Merge pull request #477 from nathanleclaire/there_is_no_volume_only_tar

Build without bind mounts for cloud editions
This commit is contained in:
Justin Cormack 2016-09-15 16:06:49 +01:00 committed by GitHub
commit 530125e0f9
11 changed files with 156 additions and 56 deletions

View File

@ -44,25 +44,97 @@ mobylinux-bios.iso: Dockerfile.bios initrd.img kernel/x86_64/vmlinuz64 isolinux.
tar cf - $^ | docker build -t moby-bios:build -f Dockerfile.bios -
docker run --net=none --log-driver=none --rm moby-bios:build cat /tmp/mobylinux-bios.iso > $@
ami: initrd.img
docker-compose build ami
docker-compose run --rm -T ami clean
docker-compose run --rm -T ami bake
common: initrd.img
$(MAKE) -C kernel
$(MAKE) -C packages
ami: common
tar cf - \
cloud initrd.img kernel/x86_64/vmlinuz64 \
| \
docker build -t moby-ami:build -f cloud/Dockerfile.ami -
# The EBS device seems not to show up without mounting in /dev, even
# with --privileged enabled.
docker run \
--rm \
--privileged \
-v /dev:/dev \
-e AWS_SECRET_ACCESS_KEY \
-e AWS_ACCESS_KEY_ID \
moby-ami:build clean
docker run \
--rm \
--privileged \
-v /dev:/dev \
-e AWS_SECRET_ACCESS_KEY \
-e AWS_ACCESS_KEY_ID \
moby-ami:build bake >./cloud/aws/ami_id.out
ami-clean-mount:
docker-compose run --rm -T ami clean-mount
docker run \
--rm \
--privileged \
-v /dev:/dev \
-e AWS_SECRET_ACCESS_KEY \
-e AWS_ACCESS_KEY_ID \
moby-ami:build clean-mount
azure: initrd.img
docker-compose build azure
docker-compose run --rm -T azure makeraw
docker build -t raw2vhd -f cloud/Dockerfile.raw2vhd cloud
docker run -v $(shell pwd):/mnt raw2vhd /mnt/mobylinux.img /mnt/mobylinux.vhd
docker-compose run --rm -T azure uploadvhd
# TODO(nathanleclaire): Migrate this to docker/editions repo.
uploadvhd: azure
docker run \
-i \
-e AZURE_STG_ACCOUNT_KEY \
-e CONTAINER_NAME \
--log-driver none \
--rm \
-v vhdartifact:/tmp \
moby-azure:build \
uploadvhd >./cloud/azure/vhd_blob_url.out
azure: common vhdartifact
tar cf - \
cloud initrd.img kernel/x86_64/vmlinuz64 \
| \
docker build -t moby-azure:build -f cloud/Dockerfile.azure -
tar cf - \
cloud \
| \
docker build -t moby-azure:raw2vhd -f cloud/Dockerfile.raw2vhd -
# -v /dev:/dev needed in addition to --privileged due to creation of
# loopback device (mount namespace?)
docker run \
--rm \
--privileged \
--log-driver none \
-v vhdartifact:/tmp \
-v /dev:/dev \
moby-azure:build \
makeraw
docker run \
--rm \
--log-driver none \
-v vhdartifact:/tmp \
moby-azure:raw2vhd
docker run \
--rm \
-i \
--log-driver none \
-v vhdartifact:/tmp \
moby-azure:build \
tarout \
| tar -xvf -
vhdartifact:
# NB: Multiple 'docker volume create' with same name does not return
# non-zero even though maybe it should. The '|| true' is included as
# future insurance.
docker volume create --name vhdartifact || true
clean:
rm -f initrd.img mobylinux.vhd mobylinux.img mobylinux-bios.iso mobylinux-efi.iso mobylinux.efi
docker images -q alpine_ami:latest | xargs docker rmi -f || true
docker images -q alpine_azure:latest | xargs docker rmi -f || true
docker images -q moby-azure:build | xargs docker rmi -f || true
docker images -q moby-azure:raw2vhd | xargs docker rmi -f || true
docker volume rm vhdartifact || true
$(MAKE) -C packages clean
$(MAKE) -C kernel clean

View File

@ -8,10 +8,15 @@ RUN apk add --update \
jq \
syslinux
RUN pip install -U awscli
RUN mkdir -p /scripts
RUN mkdir /build
RUN mkdir /scripts
WORKDIR /scripts
COPY ./build-common.sh .
COPY ./aws/common.sh .
COPY ./aws/bake-ami.sh .
COPY ./kernel/x86_64/vmlinuz64 /build
COPY ./initrd.img /build
COPY ./cloud/aws/syslinux.cfg /build/syslinux.cfg
COPY ./cloud/build-common.sh .
COPY ./cloud/aws/common.sh .
COPY ./cloud/aws/aws.sh .
COPY ./cloud/aws/bake-ami.sh .
ENTRYPOINT ["./bake-ami.sh"]
ENTRYPOINT ["./aws.sh"]

View File

@ -6,13 +6,20 @@ RUN apk add --update \
syslinux \
multipath-tools \
git \
tar \
util-linux
RUN go get -u github.com/Microsoft/azure-vhd-utils-for-go
RUN mkdir /build
RUN mkdir /scripts
WORKDIR /scripts
COPY ./build-common.sh .
COPY ./azure/bake-azure.sh .
COPY ./kernel/x86_64/vmlinuz64 /build
COPY ./initrd.img /build
COPY ./cloud/azure/syslinux.cfg /build/syslinux.cfg
COPY ./cloud/build-common.sh .
COPY ./cloud/azure/bake-azure.sh .
COPY ./cloud/azure/azure.sh .
ENTRYPOINT ["./bake-azure.sh"]
VOLUME ["/tmp"]
ENTRYPOINT ["./azure.sh"]

View File

@ -12,5 +12,7 @@ RUN apt-get update && \
# If version changes in distributed packages, this build is busted. Sanity check.
RUN qemu-img --version
RUN qemu-img --version | awk '{ if ($3 != "2.1.2,") exit 1; }'
COPY ./cloud/azure/raw2vhd.sh /raw2vhd.sh
ENTRYPOINT ["qemu-img", "convert", "-f", "raw", "-O", "vpc", "-o", "subformat=fixed"]
VOLUME ["/tmp"]
ENTRYPOINT ["/raw2vhd.sh"]

7
alpine/cloud/aws/aws.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
./bake-ami.sh "$@" 1>&2
if [ "$1" = "bake" ]
then
cat /build/ami_id.out
fi

View File

@ -10,9 +10,11 @@ set -e
PROVIDER="aws"
. "./build-common.sh"
. "${MOBY_SRC_ROOT}/cloud/aws/common.sh"
. "./common.sh"
# TODO(nathanleclaire): This could be calculated dynamically to avoid conflicts.
export AWS_DEFAULT_REGION=$(current_instance_region)
# TODO(nathanleclaire): This device could be calculated dynamically to avoid conflicts.
EBS_DEVICE=/dev/xvdb
bake_image()
@ -73,7 +75,7 @@ bake_image()
# Boom, now you (should) have a Moby AMI.
arrowecho "Created AMI: ${IMAGE_ID}"
echo "${IMAGE_ID}" >"${MOBY_SRC_ROOT}/cloud/aws/ami_id.out"
echo "${IMAGE_ID}" >"${MOBY_SRC_ROOT}/ami_id.out"
}
clean_volume_mount()
@ -124,6 +126,12 @@ clean_tagged_resources()
fi
}
if [ -z "${AWS_ACCESS_KEY_ID}" ] || [ -z "${AWS_SECRET_ACCESS_KEY}" ]
then
errecho "Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to authenticate with AWS."
exit 1
fi
case "$1" in
bake)
bake_image
@ -138,5 +146,5 @@ case "$1" in
clean_volume_mount "${TAG_KEY}"
;;
*)
echo "Command $1 not found. Usage: ./bake-ami.sh [bake|clean|clean-mount]"
errecho "Command $1 not found. Usage: ./bake-ami.sh [bake|clean|clean-mount]"
esac

12
alpine/cloud/azure/azure.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/sh
if [ "$1" = "tarout" ]
then
tar --directory /tmp -cf - -S mobylinux.vhd
else
./bake-azure.sh "$@" 1>&2
if [ "$1" = "uploadvhd" ]
then
cat vhd_blob_url.out
fi
fi

View File

@ -28,7 +28,7 @@ PROVIDER="azure"
case "$1" in
makeraw)
RAW_IMAGE="${MOBY_SRC_ROOT}/mobylinux.img"
RAW_IMAGE="/tmp/mobylinux.img"
if [ -f "${RAW_IMAGE}" ]
then
@ -57,7 +57,8 @@ case "$1" in
kpartx -d "${LOOPBACK_DEVICE}"
losetup -d "${LOOPBACK_DEVICE}"
arrowecho "Finished making raw image file"
arrowecho "Cleanup done, outputting created image. This might take a while..."
arrowecho "Finished outputting raw image file to ${RAW_IMAGE}"
;;
uploadvhd)
@ -69,11 +70,13 @@ case "$1" in
AZURE_STG_ACCOUNT_NAME=${AZURE_STG_ACCOUNT_NAME:-"dockereditions"}
CONTAINER_NAME=${CONTAINER_NAME:-"mobylinux"}
BLOBNAME=${BLOBNAME:-$(md5sum "${MOBY_SRC_ROOT}/mobylinux.vhd" | awk '{ print $1; }')-mobylinux.vhd}
BLOBNAME=${BLOBNAME:-$(md5sum "/tmp/mobylinux.vhd" | awk '{ print $1; }')-mobylinux.vhd}
BLOB_URL="https://${AZURE_STG_ACCOUNT_NAME}.blob.core.windows.net/${CONTAINER_NAME}/${BLOBNAME}"
arrowecho "Uploading VHD to ${BLOBURL}..."
azure-vhd-utils-for-go upload \
--localvhdpath "${MOBY_SRC_ROOT}/mobylinux.vhd" \
--localvhdpath "/tmp/mobylinux.vhd" \
--stgaccountname "${AZURE_STG_ACCOUNT_NAME}" \
--stgaccountkey "${AZURE_STG_ACCOUNT_KEY}" \
--containername "${CONTAINER_NAME}" \

7
alpine/cloud/azure/raw2vhd.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
set -e
>&2 echo "Converting raw image file to VHD..."
qemu-img convert -f raw -O vpc -o subformat=fixed /tmp/mobylinux.img /tmp/mobylinux.vhd 1>&2
>&2 echo "Done converting to VHD."

View File

@ -2,7 +2,7 @@
set -e
MOBY_SRC_ROOT=${MOBY_SRC_ROOT:-/mnt}
MOBY_SRC_ROOT=${MOBY_SRC_ROOT:-/build}
arrowecho()
{
@ -96,9 +96,9 @@ configure_syslinux_on_device_partition()
arrowecho "Copying image and kernel binary to partition"
# Get files needed to boot in place.
cp ${MOBY_SRC_ROOT}/cloud/${PROVIDER}/syslinux.cfg ${ROOT_PARTITION_MOUNT}
cp ${MOBY_SRC_ROOT}/syslinux.cfg ${ROOT_PARTITION_MOUNT}
cat ${ROOT_PARTITION_MOUNT}/syslinux.cfg
cp ${MOBY_SRC_ROOT}/kernel/x86_64/vmlinuz64 ${ROOT_PARTITION_MOUNT}
cp ${MOBY_SRC_ROOT}/vmlinuz64 ${ROOT_PARTITION_MOUNT}
cp ${MOBY_SRC_ROOT}/initrd.img ${ROOT_PARTITION_MOUNT}
# From http://www.syslinux.org/wiki/index.php?title=EXTLINUX:

View File

@ -1,23 +0,0 @@
version: '2'
services:
azure:
privileged: true
build:
context: cloud
dockerfile: Dockerfile.azure
network_mode: bridge
environment:
AZURE_STG_ACCOUNT_KEY:
volumes:
- .:/mnt
- /dev:/dev
ami:
privileged: true
build:
context: cloud
dockerfile: Dockerfile.ami
network_mode: bridge
volumes:
- .:/mnt
- $HOME/.aws:/root/.aws:ro
- /dev:/dev