Add Azure provider support

Signed-off-by: Nathan LeClaire <nathan.leclaire@gmail.com>

alpine/.gitignore

@@ -1,5 +1,7 @@
 *.img
 *.img.gz
+/mobylinux.img
+/mobylinux.vhd
 /mobylinux-bios.iso
 /mobylinux-efi.iso
 /mobylinux-boot.vhdx

alpine/Dockerfile

@@ -77,7 +77,8 @@ COPY packages/test/mobytest /usr/bin/
 COPY packages/sysctl/etc /etc/
 COPY packages/iptables/iptables /usr/local/sbin/
 COPY packages/containerd/etc /etc/
-COPY packages/aws/etc /etc
+COPY packages/aws/etc /etc/
+COPY packages/azure/etc /etc/
 
 RUN \
   rc-update add swap boot && \
@@ -116,6 +117,7 @@ RUN \
   rc-update add test default && \
   rc-update add containerd default && \
   rc-update add aws default && \
+  rc-update add azure default && \
   true
 
 COPY init /

alpine/Makefile

@@ -57,8 +57,15 @@ ami: common
 	docker-compose run --rm -T ami clean
 	docker-compose run --rm -T ami bake
 
+azure: common
+	docker-compose build azure
+	docker-compose run --rm -T azure makeraw
+	docker build -t raw2vhd -f cloud/Dockerfile.raw2vhd cloud
+	docker run -v $(shell pwd):/mnt raw2vhd /mnt/mobylinux.img /mnt/mobylinux.vhd
+	docker-compose run --rm -T azure uploadvhd
+
 clean:
-	rm -f initrd.img initrd.img.gz initrd-arm.img Dockerfile.armhf
+	rm -f initrd.img initrd.img.gz initrd-arm.img Dockerfile.armhf mobylinux.vhd mobylinux.img
 	rm -f mobylinux-bios.iso mobylinux-efi.iso mobylinux.efi
 	$(MAKE) -C packages clean
 	$(MAKE) -C kernel clean

alpine/cloud/Dockerfile.azure

@@ -0,0 +1,19 @@
+FROM golang:alpine
+
+RUN apk add --update \
+    bash \
+    curl \
+    e2fsprogs \
+    syslinux \
+    multipath-tools \
+    git \
+    util-linux
+
+RUN go get -u github.com/Microsoft/azure-vhd-utils-for-go
+
+RUN mkdir /scripts
+WORKDIR /scripts
+COPY ./build-common.sh .
+COPY ./azure/bake-azure.sh .
+
+ENTRYPOINT ["./bake-azure.sh"]

alpine/cloud/Dockerfile.raw2vhd

@@ -0,0 +1,16 @@
+FROM debian:jessie
+
+# Why bother with this whole song and dance?  qemu-img versions >=2.2.1 have a
+# "bug" which causes the generated VHD files to be improperly formatted for
+# running on Azure: https://bugs.launchpad.net/qemu/+bug/1490611
+# 
+# Since Alpine has only qemu-img >= 2.4.1 in its apk index, we cannot use
+# Alpine.
+RUN apt-get update && \
+    apt-get install -y qemu-utils
+
+# If version changes in distributed packages, this build is busted.  Sanity check.
+RUN qemu-img --version
+RUN qemu-img --version | awk '{ if ($3 != "2.1.2,") exit 1; }'
+
+ENTRYPOINT ["qemu-img", "convert", "-f", "raw", "-O", "vpc", "-o", "subformat=fixed"]

alpine/cloud/azure/Dockerfile.waagent

@@ -0,0 +1,28 @@
+FROM debian:jessie
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        rsyslog \
+        openssl \
+        ca-certificates \
+        ssh \
+        git \
+        parted \
+        sudo \
+        net-tools \
+        ifupdown \
+        python \
+        python-pyasn1 \
+        python-setuptools \
+        python-rpm
+
+RUN git clone https://github.com/Azure/WALinuxAgent /WALinuxAgent
+WORKDIR /WALinuxAgent
+RUN python setup.py install
+COPY ./waagent.conf /etc/waagent.conf
+RUN cp /WALinuxAgent/bin/* /usr/sbin/
+
+RUN chmod +x /usr/sbin/waagent && \
+    ln -sf /dev/stdout /var/log/waagent.log
+
+ENTRYPOINT ["/usr/sbin/waagent"]

alpine/cloud/azure/bake-azure.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Script to automate the creation of a VHD for Moby in Azure, and upload it to
+# an Azure storage account (needed in order to launch it on Azure, or upload it
+# to the Azure Marketplace).
+#
+# Usage: ./bake-azure.sh (intended to be invoked in a Docker container with
+# specific properties, see the 'alpine' dir / Makefile)
+#
+# Parameters (override as environment variables):
+#
+# AZURE_STG_ACCOUNT_NAME: Name of the storage account to upload the VHD to.
+#
+# AZURE_STG_ACCOUNT_KEY: Key needed to access the storage account to upload the
+# VHD.  This can be accessed in the storage account in the web portal.
+#
+# CONTAINER_NAME: Name of the container in the storage account to place the
+# created VHD in.  "Container" here is NOT a Docker/Linux container, it is
+# similar to "bucket" in AWS parlance.
+#
+# BLOBNAME: Name of the created VHD "blob".  e.g., "foobar-mobylinux.vhd"
+
+set -e
+
+declare -xr PROVIDER="azure"
+
+source "build-common.sh"
+
+case "$1" in
+	makeraw)
+		RAW_IMAGE="${MOBY_SRC_ROOT}/mobylinux.img"
+
+		if [ -f "${RAW_IMAGE}" ]
+		then
+			rm "${RAW_IMAGE}"
+		fi
+
+		arrowecho "Writing empty image file"
+		dd if=/dev/zero of="${RAW_IMAGE}" count=0 bs=1 seek=30G
+
+		arrowecho "Formatting image file for boot"
+		format_on_device "${RAW_IMAGE}"
+
+		arrowecho "Setting up loopback device"
+		LOOPBACK_DEVICE="$(losetup -f --show ${RAW_IMAGE})"
+
+		arrowecho "Loopback device is ${LOOPBACK_DEVICE}"
+
+		arrowecho "Mapping partition"
+		MAPPED_PARTITION="/dev/mapper/$(kpartx -av ${LOOPBACK_DEVICE} | cut -d' ' -f3)"
+		arrowecho "Partition mapped at ${MAPPED_PARTITION}"
+
+		arrowecho "Installing syslinux and dropping artifacts on partition..."
+		configure_syslinux_on_device_partition "${LOOPBACK_DEVICE}" "${MAPPED_PARTITION}"
+
+		arrowecho "Cleaning up..."
+		kpartx -d "${LOOPBACK_DEVICE}"
+		losetup -d "${LOOPBACK_DEVICE}"
+
+		arrowecho "Finished making raw image file"
+		;;
+
+	uploadvhd)
+		if [ -z "${AZURE_STG_ACCOUNT_KEY}" ]
+		then
+			errecho "Need to set AZURE_STG_ACCOUNT_KEY for the 'dockereditions' storage account."
+			exit 1
+		fi
+
+		AZURE_STG_ACCOUNT_NAME=${AZURE_STG_ACCOUNT_NAME:-"dockereditions"}
+		CONTAINER_NAME=${CONTAINER_NAME:-"mobylinux"}
+		BLOBNAME=${BLOBNAME:-$(md5sum "${MOBY_SRC_ROOT}/mobylinux.vhd" | awk '{ print $1; }')-mobylinux.vhd}
+
+		azure-vhd-utils-for-go upload \
+			--localvhdpath "${MOBY_SRC_ROOT}/mobylinux.vhd" \
+			--stgaccountname "${AZURE_STG_ACCOUNT_NAME}" \
+			--stgaccountkey "${AZURE_STG_ACCOUNT_KEY}" \
+			--containername "${CONTAINER_NAME}" \
+			--blobname "${BLOBNAME}" \
+			--overwrite
+
+		arrowecho "VHD uploaded."
+		arrowecho "https://${AZURE_STG_ACCOUNT_NAME}.blob.core.windows.net/${CONTAINER_NAME}/${BLOBNAME}"
+		;;
+
+	*)
+		errecho "Invalid usage.  Syntax: ./bake-azure.sh [makeraw|uploadvhd]"
+		exit 1
+esac

alpine/cloud/azure/syslinux.cfg

@@ -0,0 +1,5 @@
+DEFAULT linux
+LABEL linux
+    KERNEL /vmlinuz64
+    INITRD /initrd.img
+    APPEND root=/dev/sda1 rootdelay=300 console=tty1 console=ttyS0 earlyprintk=ttyS0 mobyplatform=azure

alpine/cloud/azure/waagent.conf

@@ -0,0 +1,75 @@
+#
+# Windows Azure Linux Agent Configuration
+#
+
+# Specified program is invoked with the argument "Ready" when we report ready status
+# to the endpoint server.
+Role.StateConsumer=None
+
+# Specified program is invoked with XML file argument specifying role
+#  configuration.
+Role.ConfigurationConsumer=None
+
+# Specified program is invoked with XML file argument specifying role topology.
+Role.TopologyConsumer=None
+
+# Enable instance creation
+Provisioning.Enabled=y
+
+# Password authentication for root account will be unavailable.
+Provisioning.DeleteRootPassword=y
+
+# Generate fresh host key pair.
+Provisioning.RegenerateSshHostKeyPair=n
+
+# Supported values are "rsa", "dsa" and "ecdsa".
+Provisioning.SshHostKeyPairType=rsa
+
+# Monitor host name changes and publish changes via DHCP requests.
+Provisioning.MonitorHostName=n
+
+# Decode CustomData from Base64.
+Provisioning.DecodeCustomData=y
+
+# Execute CustomData after provisioning.
+Provisioning.ExecuteCustomData=y
+
+# Format if unformatted. If 'n', resource disk will not be mounted.
+ResourceDisk.Format=y
+
+# File system on the resource disk
+# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here.
+ResourceDisk.Filesystem=ext4
+
+# Mount point for the resource disk
+ResourceDisk.MountPoint=/mnt/resource
+
+# Create and use swapfile on resource disk.
+ResourceDisk.EnableSwap=n
+
+# Size of the swapfile.
+ResourceDisk.SwapSizeMB=0
+
+# Respond to load balancer probes if requested by Windows Azure.
+LBProbeResponder=y
+
+# File to write log to.
+# '/var/log/waagent.log' if not set
+Logs.File=/dev/stdout
+
+# Enable logging to serial console (y|n)
+# When stdout is not enough...
+# 'y' if not set
+Logs.Console=y
+
+# Enable verbose logging (y|n)
+Logs.Verbose=y
+
+# Preferred network interface to communicate with Azure platform
+Network.Interface=eth0
+
+# Root device timeout in seconds.
+OS.RootDeviceScsiTimeout=300
+
+# If "None", the system default version is used.
+OS.OpensslPath=None

alpine/docker-compose.yml

@@ -27,6 +27,17 @@ services:
       context: .
       dockerfile: Dockerfile.armhf
     network_mode: bridge
+  azure:
+    privileged: true
+    build:
+      context: cloud
+      dockerfile: Dockerfile.azure
+    network_mode: bridge
+    environment:
+      AZURE_STG_ACCOUNT_KEY: 
+    volumes:
+      - .:/mnt
+      - /dev:/dev
   ami:
     privileged: true
     build:

alpine/packages/azure/etc/init.d/azure

@@ -0,0 +1,116 @@
+#!/sbin/openrc-run
+
+description="Bootstrap procedure if running on Docker Azure edition"
+
+depend()
+{
+	need docker
+	need networking
+}
+
+start()
+{
+	[ "$(mobyplatform)" != "azure" ] && exit 0
+	ebegin "Running Azure-specific initialization"
+
+	einfo "Setting hostname"
+
+	# TODO: This is probably quite fragile (splitting the returned JSON by
+	# quotes instead of properly parsing).  Would bundling 'jq' in Moby be
+	# too much overhead?
+	hostname $(wget -qO- http://169.254.169.254/metadata/v1/instanceinfo | awk -F '"' '{ print $4; }')
+
+	for i in $(seq 1 20)
+	do
+		einfo "Pulling Windows Azure Linux Agent container"
+
+		docker pull nathanleclaire/walinuxagent >/dev/null
+
+		einfo "Running Windows Azure Linux Agent container"
+
+		docker run -d \
+			--name waagent \
+			--privileged \
+			--restart unless-stopped \
+			--net host \
+			--ipc host \
+			--pid host \
+			-v /home:/home \
+			-v /etc:/etc \
+			-v /lib/modules:/lib/modules \
+			-v /lib/firmware:/lib/firmware \
+			-v /var/log:/var/log \
+			-v /var/lib/waagent:/var/lib/waagent \
+			nathanleclaire/walinuxagent
+
+		if [ $? -eq 0 ]
+		then
+			break
+		fi
+
+		# Wait for... network to come up?  DNS servers to be reachable?
+		# Not certain, but Azure continually fails to achieve this pull so
+		# far because it can't dial the DNS lookup properly.
+		# 
+		# TODO: Debug.
+		sleep 5
+	done
+
+	# Wait for custom data to arrive
+	while [ ! -f /var/lib/waagent/CustomData ]
+	do
+		sleep 5
+	done
+
+	source /var/lib/waagent/CustomData
+
+	if [ "${ROLE}" == "MANAGER" ]
+	then
+		docker swarm init \
+			--auto-accept manager \
+			--auto-accept worker
+
+		# TODO: Make this cleaner.
+		# User gets added by waagent.
+		# Need to unlock it to login via SSH.
+		passwd -u docker
+
+		checkpath --directory --mode 0700 /home/docker/.ssh
+
+		einfo "Pulling taco shell"
+
+		docker pull nathanleclaire/taco-shell >/dev/null
+
+		einfo "Running taco shell"
+
+		# Container for inbound shell access.
+		#
+		#  TODO: Move this into another service?
+		docker run -d \
+			--name taco-shell \
+			--net host \
+			--restart always \
+			-p 22:22 \
+			-v /usr/bin/docker:/usr/docker/bin/docker:ro \
+			-v /var/run/docker.sock:/var/run/docker.sock \
+			-v /home/docker/.ssh:/home/docker/.ssh:ro \
+			-v /etc/passwd:/etc/passwd:ro \
+			-v /etc/shadow:/etc/shadow:ro \
+			-v /var/log:/var/log:ro \
+			-v /etc/ssh/ssh_host_rsa_key:/etc/ssh/ssh_host_rsa_key:ro \
+			-v /etc/ssh/ssh_host_rsa_key.pub:/etc/ssh/ssh_host_rsa_key.pub:ro \
+			nathanleclaire/taco-shell
+	else
+		docker swarm join "${MANAGER_IP}:2377"
+	fi
+
+	eend 0
+}
+
+stop()
+{
+	[ "$(mobyplatform)" != "azure" ] && exit 0
+	docker rm -f waagent || true
+	docker rm -f taco-shell || true
+	passwd -l docker
+}