Update README.md (#3067)

v1.5.8: bump AMIs

.dockerignore

@@ -15,3 +15,4 @@ tests/integration/.tox
 #.dapper
 vendor/*/*/*/.git
 tmp
+docs/_site

.drone.yml

@@ -1,8 +1,22 @@
 ---
-pipeline:
-  build:
-    image: rancher/dapper:1.10.3
-    volumes:
-    - /var/run/docker.sock:/var/run/docker.sock
-    commands:
-    - dapper ci
+kind: pipeline
+name: default
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: build
+  pull: default
+  image: rancher/dapper:1.10.3
+  commands:
+  - KERNEL_CHECK=0 dapper ci
+  volumes:
+  - name: socket
+    path: /var/run/docker.sock
+
+volumes:
+- name: socket
+  host:
+    path: /var/run/docker.sock

.gitignore

@@ -6,6 +6,7 @@
 /dist
 /gopath
 /images/*/build
+/scripts/images/vmware/assets
 .dockerfile
 *.swp
 /tests/integration/MANIFEST
@@ -16,4 +17,6 @@
 __pycache__
 /.dapper
 /.trash-cache
+/trash.lock
 .idea
+.trash-conf

Dockerfile.dapper

@@ -1,5 +1,5 @@
-FROM ubuntu:16.04
-# FROM arm64=aarch64/ubuntu:16.04 arm=armhf/ubuntu:16.04
+FROM ubuntu:bionic
+# FROM arm64=arm64v8/ubuntu:bionic
 
 # get the apt-cacher proxy set
 ARG APTPROXY=
@@ -35,11 +35,12 @@ RUN echo "Acquire::http { Proxy \"$APTPROXY\"; };" >> /etc/apt/apt.conf.d/01prox
         vim \
         wget \
         xorriso \
-	telnet
+        xz-utils \
+        telnet
 
 ########## Dapper Configuration #####################
 
-ENV DAPPER_ENV VERSION DEV_BUILD RUNTEST DEBUG APTPROXY ENGINE_REGISTRY_MIRROR INTEGRATION_TESTS
+ENV DAPPER_ENV VERSION DEV_BUILD RUNTEST DEBUG APTPROXY ENGINE_REGISTRY_MIRROR KERNEL_CHECK APPEND_SYSTEM_IMAGES APPEND_USER_IMAGES
 ENV DAPPER_DOCKER_SOCKET true
 ENV DAPPER_SOURCE /go/src/github.com/rancher/os
 ENV DAPPER_OUTPUT ./bin ./dist ./build/initrd ./build/kernel
@@ -57,72 +58,80 @@ ARG OS_REPO=rancher
 ARG HOSTNAME_DEFAULT=rancher
 ARG DISTRIB_ID=RancherOS
 
-ARG DOCKER_VERSION=1.11.2
-ARG DOCKER_PATCH_VERSION=v${DOCKER_VERSION}-ros1
-ARG DOCKER_BUILD_VERSION=1.10.3
-ARG DOCKER_BUILD_PATCH_VERSION=v${DOCKER_BUILD_VERSION}-ros1
 ARG SELINUX_POLICY_URL=https://github.com/rancher/refpolicy/releases/download/v0.0.3/policy.29
 
-ARG KERNEL_VERSION_amd64=4.9.78-rancher
-ARG KERNEL_URL_amd64=https://github.com/rancher/os-kernel/releases/download/v${KERNEL_VERSION_amd64}/linux-${KERNEL_VERSION_amd64}-x86.tar.gz
-#ARG KERNEL_URL_arm64=https://github.com/imikushin/os-kernel/releases/download/Estuary-4.4.0-arm64.8/linux-4.4.0-rancher-arm64.tar.gz
+ARG KERNEL_VERSION=4.14.176-rancher
+ARG KERNEL_URL_amd64=https://github.com/rancher/os-kernel/releases/download/v${KERNEL_VERSION}/linux-${KERNEL_VERSION}-x86.tar.gz
+ARG KERNEL_URL_arm64=https://github.com/rancher/os-kernel/releases/download/v${KERNEL_VERSION}/linux-${KERNEL_VERSION}-arm64.tar.gz
 
-ARG DOCKER_URL_amd64=https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz
-ARG DOCKER_URL_arm=https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm.tgz
-ARG DOCKER_URL_arm64=https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm64.tgz
-
-ARG BUILD_DOCKER_URL_amd64=https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_BUILD_VERSION}
-ARG BUILD_DOCKER_URL_arm=https://github.com/rancher/docker/releases/download/${DOCKER_BUILD_PATCH_VERSION}/docker-${DOCKER_BUILD_VERSION}_arm
-ARG BUILD_DOCKER_URL_arm64=https://github.com/rancher/docker/releases/download/${DOCKER_BUILD_PATCH_VERSION}/docker-${DOCKER_BUILD_VERSION}_arm64
+ARG BUILD_DOCKER_URL_amd64=https://get.docker.com/builds/Linux/x86_64/docker-1.10.3
+ARG BUILD_DOCKER_URL_arm64=https://github.com/rancher/docker/releases/download/v1.10.3-ros1/docker-1.10.3_arm64
 
 ARG OS_RELEASES_YML=https://releases.rancher.com/os
 
 ARG OS_SERVICES_REPO=https://raw.githubusercontent.com/${OS_REPO}/os-services
 ARG IMAGE_NAME=${OS_REPO}/os
-ARG DFS_IMAGE=${OS_REPO}/docker:v${DOCKER_VERSION}-2
 
-ARG OS_BASE_URL_amd64=https://github.com/rancher/os-base/releases/download/v2017.02.5-1/os-base_amd64.tar.xz
-ARG OS_BASE_URL_arm64=https://github.com/rancher/os-base/releases/download/v2017.02.5-1/os-base_arm64.tar.xz
-ARG OS_BASE_URL_arm=https://github.com/rancher/os-base/releases/download/v2017.02.5-1/os-base_arm.tar.xz
+ARG OS_CONSOLE=default
+ARG OS_AUTOFORMAT=false
+
+ARG OS_BASE_URL_amd64=https://github.com/rancher/os-base/releases/download/v2018.02.11-1/os-base_amd64.tar.xz
+ARG OS_BASE_URL_arm64=https://github.com/rancher/os-base/releases/download/v2018.02.11-1/os-base_arm64.tar.xz
+
+ARG OS_INITRD_BASE_URL_amd64=https://github.com/rancher/os-initrd-base/releases/download/v2018.02.11-1/os-initrd-base-amd64.tar.gz
+ARG OS_INITRD_BASE_URL_arm64=https://github.com/rancher/os-initrd-base/releases/download/v2018.02.11-1/os-initrd-base-arm64.tar.gz
+
+ARG SYSTEM_DOCKER_VERSION=17.06-ros6
+ARG SYSTEM_DOCKER_URL_amd64=https://github.com/rancher/os-system-docker/releases/download/${SYSTEM_DOCKER_VERSION}/docker-amd64-${SYSTEM_DOCKER_VERSION}.tgz
+ARG SYSTEM_DOCKER_URL_arm64=https://github.com/rancher/os-system-docker/releases/download/${SYSTEM_DOCKER_VERSION}/docker-arm64-${SYSTEM_DOCKER_VERSION}.tgz
+
+ARG USER_DOCKER_VERSION=19.03.8
+ARG USER_DOCKER_ENGINE_VERSION=docker-${USER_DOCKER_VERSION}
+
+ARG AZURE_SERVICE=false
+ARG PROXMOXVE_SERVICE=false
 ######################################################
 
 # Set up environment and export all ARGS as ENV
 ENV ARCH=${ARCH} \
-    HOST_ARCH=${HOST_ARCH}
+    HOST_ARCH=${HOST_ARCH} \
+    XZ_DEFAULTS="-T0"
 
 ENV BUILD_DOCKER_URL=BUILD_DOCKER_URL_${ARCH} \
     BUILD_DOCKER_URL_amd64=${BUILD_DOCKER_URL_amd64} \
-    BUILD_DOCKER_URL_arm=${BUILD_DOCKER_URL_arm} \
     BUILD_DOCKER_URL_arm64=${BUILD_DOCKER_URL_arm64} \
     DAPPER_HOST_ARCH=${DAPPER_HOST_ARCH} \
-    DFS_IMAGE=${DFS_IMAGE} \
     DISTRIB_ID=${DISTRIB_ID} \
-    DOCKER_PATCH_VERSION=${DOCKER_PATCH_VERSION} \
-    DOCKER_URL=DOCKER_URL_${ARCH} \
-    DOCKER_URL_amd64=${DOCKER_URL_amd64} \
-    DOCKER_URL_arm=${DOCKER_URL_arm} \
-    DOCKER_URL_arm64=${DOCKER_URL_arm64} \
-    DOCKER_VERSION=${DOCKER_VERSION} \
     DOWNLOADS=/usr/src/downloads \
     GOPATH=/go \
-    GO_VERSION=1.7.1 \
+    GO_VERSION=1.8.5 \
     GOARCH=$ARCH \
     HOSTNAME_DEFAULT=${HOSTNAME_DEFAULT} \
     IMAGE_NAME=${IMAGE_NAME} \
-    KERNEL_VERSION=${KERNEL_VERSION_amd64} \
+    KERNEL_VERSION=${KERNEL_VERSION} \
     KERNEL_URL=KERNEL_URL_${ARCH} \
     KERNEL_URL_amd64=${KERNEL_URL_amd64} \
     KERNEL_URL_arm64=${KERNEL_URL_arm64} \
-    OS_BASE_SHA1=OS_BASE_SHA1_${ARCH} \
     OS_BASE_URL=OS_BASE_URL_${ARCH} \
     OS_BASE_URL_amd64=${OS_BASE_URL_amd64} \
-    OS_BASE_URL_arm=${OS_BASE_URL_arm} \
     OS_BASE_URL_arm64=${OS_BASE_URL_arm64} \
+    OS_INITRD_BASE_URL=OS_INITRD_BASE_URL_${ARCH} \
+    OS_INITRD_BASE_URL_amd64=${OS_INITRD_BASE_URL_amd64} \
+    OS_INITRD_BASE_URL_arm64=${OS_INITRD_BASE_URL_arm64} \
     OS_RELEASES_YML=${OS_RELEASES_YML} \
     OS_REPO=${OS_REPO} \
     OS_SERVICES_REPO=${OS_SERVICES_REPO} \
+    OS_CONSOLE=${OS_CONSOLE} \
+    OS_AUTOFORMAT=${OS_AUTOFORMAT} \
     REPO_VERSION=master \
-    SELINUX_POLICY_URL=${SELINUX_POLICY_URL}
+    SELINUX_POLICY_URL=${SELINUX_POLICY_URL} \
+    SYSTEM_DOCKER_URL=SYSTEM_DOCKER_URL_${ARCH} \
+    SYSTEM_DOCKER_URL_amd64=${SYSTEM_DOCKER_URL_amd64} \
+    SYSTEM_DOCKER_URL_arm64=${SYSTEM_DOCKER_URL_arm64} \
+    USER_DOCKER_VERSION=${USER_DOCKER_VERSION} \
+    USER_DOCKER_ENGINE_VERSION=${USER_DOCKER_ENGINE_VERSION} \
+    AZURE_SERVICE=${AZURE_SERVICE} \
+    PROXMOXVE_SERVICE=${PROXMOXVE_SERVICE}
 ENV PATH=${GOPATH}/bin:/usr/local/go/bin:$PATH
 
 RUN mkdir -p ${DOWNLOADS}
@@ -138,26 +147,16 @@ RUN echo "... Downloading  ${!KERNEL_URL}"; \
 RUN curl -pfL ${SELINUX_POLICY_URL} > ${DOWNLOADS}/$(basename ${SELINUX_POLICY_URL})
 
 # Install Go
-COPY assets/go-dnsclient.patch ${DAPPER_SOURCE}
-RUN ln -sf go-6 /usr/bin/go && \
-    curl -sfL https://storage.googleapis.com/golang/go${GO_VERSION}.src.tar.gz | tar -xzf - -C /usr/local && \
-    patch /usr/local/go/src/net/dnsclient_unix.go ${DAPPER_SOURCE}/go-dnsclient.patch && \
-    cd /usr/local/go/src && \
-    GOROOT_BOOTSTRAP=/usr GOARCH=${HOST_ARCH} GOHOSTARCH=${HOST_ARCH} ./make.bash && \
-    rm /usr/bin/go
+RUN wget -O - https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GOARCH}.tar.gz | tar -xzf - -C /usr/local && \
+    go get github.com/rancher/trash
+
+RUN mkdir -p ${GOPATH}/src/golang.org/x && cd ${GOPATH}/src/golang.org/x/ && git clone https://github.com/golang/tools && \
+    cd tools && git checkout 6adeb8aab2ded9eb693b831d5fd090c10a6ebdfa -b temp && go get golang.org/x/lint/golint
 
 # Install Host Docker
 RUN curl -fL ${!BUILD_DOCKER_URL} > /usr/bin/docker && \
     chmod +x /usr/bin/docker
 
-# Install Trash
-RUN go get github.com/rancher/trash
-
-# Install golint
-RUN go get github.com/golang/lint/golint
-
-RUN go get gopkg.in/check.v1
-
 # Install dapper
 RUN curl -sL https://releases.rancher.com/dapper/latest/dapper-`uname -s`-`uname -m | sed 's/arm.*/arm/'` > /usr/bin/dapper && \
     chmod +x /usr/bin/dapper

Makefile

@@ -1,4 +1,4 @@
-TARGETS := $(shell ls scripts | grep -vE 'clean|run|help|docs|release|build-moby|run-moby')
+TARGETS := $(shell ls scripts | grep -vE 'clean|run|help|release*|build-moby|run-moby')
 
 .dapper:
 	@echo Downloading dapper
@@ -25,9 +25,6 @@ run: build/initrd/.id .dapper
 	./.dapper -m bind build-target
 	./scripts/run
 
-docs:
-	./scripts/docs
-
 build-moby:
 	./scripts/build-moby
 
@@ -40,38 +37,46 @@ shell-bind: .dapper
 clean:
 	@./scripts/clean
 
-release: .dapper release-build qcows
+release: .dapper release-build
 
 release-build:
 	mkdir -p dist
-	./.dapper release 2>&1 | tee dist/release.log
+	./.dapper release
 
-itest:
+rpi64: .dapper
+	./scripts/release-rpi64
+
+vmware: .dapper
 	mkdir -p dist
-	./.dapper integration-test 2>&1 | tee dist/itest.log
-	grep FAIL dist/itest.log || true
+	APPEND_SYSTEM_IMAGES="rancher/os-openvmtools:10.3.10-2" \
+	./.dapper release-vmware
 
-qcows:
-	cp dist/artifacts/rancheros.iso scripts/images/openstack/
-	cd scripts/images/openstack && \
-		APPEND="console=tty1 console=ttyS0,115200n8 printk.devkmsg=on rancher.autologin=ttyS0" \
-		NAME=openstack ../../../.dapper
-	cd scripts/images/openstack && \
-		APPEND="console=tty1 rancher.debug=true printk.devkmsg=on notsc clocksource=kvm-clock rancher.network.interfaces.eth0.ipv4ll rancher.cloud_init.datasources=[digitalocean] rancher.autologin=tty1 rancher.autologin=ttyS0" \
-		NAME=digitalocean ../../../.dapper
-	cp ./scripts/images/openstack/dist/*.img dist/artifacts/
+hyperv: .dapper
+	mkdir -p dist
+	APPEND_SYSTEM_IMAGES="rancher/os-hypervvmtools:v4.14.138-rancher-1" \
+	./.dapper release-hyperv
 
-rpi:
-	# scripts/images/raspberry-pi-hypriot/dist/rancheros-raspberry-pi.zip
-	cp dist/artifacts/rootfs_arm.tar.gz scripts/images/raspberry-pi-hypriot/
-	cd scripts/images/raspberry-pi-hypriot/ \
-		&& ../../../.dapper
+azurebase: .dapper
+	mkdir -p dist
+	AZURE_SERVICE="true" \
+	APPEND_SYSTEM_IMAGES="rancher/os-hypervvmtools:v4.14.138-rancher-1 rancher/os-waagent:v2.2.34-1" \
+	./.dapper release-azurebase
 
-rpi64:
-	# scripts/images/raspberry-pi-hypriot64/dist/rancheros-raspberry-pi.zip
-	cp dist/artifacts/rootfs_arm64.tar.gz scripts/images/raspberry-pi-hypriot64/
-	cd scripts/images/raspberry-pi-hypriot64/ \
-		&& ../../../.dapper
+4glte: .dapper
+	mkdir -p dist
+	APPEND_SYSTEM_IMAGES="rancher/os-modemmanager:v1.6.4-1" \
+	./.dapper release-4glte
+
+proxmoxve: .dapper
+	mkdir -p dist
+	PROXMOXVE_SERVICE="true" \
+	APPEND_SYSTEM_IMAGES="rancher/os-qemuguestagent:v2.8.1-2" \
+	./.dapper release-proxmoxve
+
+pingan: .dapper
+	mkdir -p dist
+	APPEND_SYSTEM_IMAGES="cnrancher/os-pingan-amc:v0.0.6-1" \
+	./.dapper release-pingan
 
 help:
 	@./scripts/help

README.md

@@ -1,5 +1,16 @@
+# Development and Maintenance Status
+RancherOS 1.x is no longer being actively maintained. There are two significant reasons behind this product decision:
+1. **Docker** - The current industry requirements for a container runtime are very much evolving. Container runtimes like containerd and CRIO are now being actively considered as the default choice. RancherOS 1.x, which was specifically designed around using Docker engine only, unfortunately does not lend itself, in its current design, to this new evolving requirement.
+2. **ISV Support** - RancherOS was specifically designed as a minimalistic OS to support purpose-built containerized applications. It was not designed to be used as a general purpose OS (such as CentOS or Ubuntu). As such, most ISVs have not certified their software to run on RancherOS, nor does RancherOS even contain the necessary components for many of these applications to run.
+
+We're working on a replacement. Stay tuned!
+
 # RancherOS
 
+[![Build Status](https://drone-pr.rancher.io/api/badges/rancher/os/status.svg?branch=master)](https://drone-pr.rancher.io/rancher/os)
+[![Docker Pulls](https://img.shields.io/docker/pulls/rancher/os.svg)](https://store.docker.com/community/images/rancher/os)
+[![Go Report Card](https://goreportcard.com/badge/github.com/rancher/os)](https://goreportcard.com/badge/github.com/rancher/os)
+
 The smallest, easiest way to run Docker in production at scale.  Everything in RancherOS is a container managed by Docker.  This includes system services such as udev and rsyslog.  RancherOS includes only the bare minimum amount of software needed to run Docker.  This keeps the binary download of RancherOS very small.  Everything else can be pulled in dynamically through Docker.
 
 ## How this works
@@ -10,58 +21,72 @@ a container that runs the user Docker.  The user Docker is then the instance tha
 used to create containers.  We created this separation because it seemed logical and also
 it would really be bad if somebody did `docker rm -f $(docker ps -qa)` and deleted the entire OS.
 
-![How it works](docs/rancheros.png "How it works")
+![How it works](./rancheros.png "How it works")
 
-## Latest Release
+## Release
 
-**v1.0.3 - Docker 17.03.1-ce - Linux 4.9.34**
+- **v1.5.8 - Docker 19.03.15 - Linux 4.14.138**
 
 ### ISO
 
-- https://releases.rancher.com/os/latest/rancheros.iso
-- https://releases.rancher.com/os/v1.0.3/rancheros.iso
+- https://releases.rancher.com/os/v1.5.8/rancheros.iso
+- https://releases.rancher.com/os/v1.5.8/hyperv/rancheros.iso
+- https://releases.rancher.com/os/v1.5.8/4glte/rancheros.iso
+- https://releases.rancher.com/os/v1.5.8/vmware/rancheros.iso
+
+#### Special docker-machine Links
+
+- https://releases.rancher.com/os/v1.5.8/vmware/rancheros-autoformat.iso
+- https://releases.rancher.com/os/v1.5.8/proxmoxve/rancheros-autoformat.iso
 
 ### Additional Downloads
 
-#### Latest Links
+#### AMD64 Links
 
-* https://releases.rancher.com/os/latest/initrd
-* https://releases.rancher.com/os/latest/initrd-v1.0.3
-* https://releases.rancher.com/os/latest/iso-checksums.txt
-* https://releases.rancher.com/os/latest/rancheros-openstack.img
-* https://releases.rancher.com/os/latest/rancheros.ipxe
-* https://releases.rancher.com/os/latest/rancheros.iso
-* https://releases.rancher.com/os/latest/rancheros-v1.0.3.tar.gz
-* https://releases.rancher.com/os/latest/rootfs.tar.gz
-* https://releases.rancher.com/os/latest/vmlinuz
-* https://releases.rancher.com/os/latest/vmlinuz-4.9.34-rancher
+* https://releases.rancher.com/os/v1.5.8/initrd
+* https://releases.rancher.com/os/v1.5.8/vmlinuz
+* https://releases.rancher.com/os/v1.5.8/rancheros.ipxe
+* https://releases.rancher.com/os/v1.5.8/rootfs.tar.gz
 
-#### v1.0.3 Links
+#### ARM64 Links
 
-* https://releases.rancher.com/os/v1.0.3/initrd
-* https://releases.rancher.com/os/v1.0.3/initrd-v1.0.3
-* https://releases.rancher.com/os/v1.0.3/iso-checksums.txt
-* https://releases.rancher.com/os/v1.0.3/rancheros-openstack.img
-* https://releases.rancher.com/os/v1.0.3/rancheros.ipxe
-* https://releases.rancher.com/os/v1.0.3/rancheros.iso
-* https://releases.rancher.com/os/v1.0.3/rancheros-v1.0.3.tar.gz
-* https://releases.rancher.com/os/v1.0.3/rootfs.tar.gz
-* https://releases.rancher.com/os/v1.0.3/vmlinuz
-* https://releases.rancher.com/os/v1.0.3/vmlinuz-4.9.34-rancher
+* https://releases.rancher.com/os/v1.5.8/arm64/initrd
+* https://releases.rancher.com/os/v1.5.8/arm64/vmlinuz
+* https://releases.rancher.com/os/v1.5.8/arm64/rootfs_arm64.tar.gz
+* https://releases.rancher.com/os/v1.5.8/arm64/rancheros-raspberry-pi64.zip
 
-#### ARM Links
+#### Cloud Links
 
-* https://releases.rancher.com/os/latest/rootfs_arm.tar.gz
-* https://releases.rancher.com/os/latest/rootfs_arm64.tar.gz
-* https://releases.rancher.com/os/latest/rancheros-raspberry-pi.zip
-* https://releases.rancher.com/os/latest/rancheros-raspberry-pi64.zip
+* https://releases.rancher.com/os/v1.5.8/rancheros-openstack.img
+* https://releases.rancher.com/os/v1.5.8/rancheros-digitalocean.img
+* https://releases.rancher.com/os/v1.5.8/rancheros-cloudstack.img
+* https://releases.rancher.com/os/v1.5.8/rancheros-aliyun.vhd
+* https://releases.rancher.com/os/v1.5.8/rancheros-gce.tar.gz
 
-* https://releases.rancher.com/os/v1.0.3/rootfs_arm.tar.gz
-* https://releases.rancher.com/os/v1.0.3/rootfs_arm64.tar.gz
-* https://releases.rancher.com/os/v1.0.3/rancheros-raspberry-pi.zip
-* https://releases.rancher.com/os/v1.0.3/rancheros-raspberry-pi64.zip
+#### VMware Links
 
-**Note**: you can use `http` instead of `https` in the above URLs, e.g. for iPXE.
+* https://releases.rancher.com/os/v1.5.8/vmware/initrd
+* https://releases.rancher.com/os/v1.5.8/vmware/rancheros.vmdk
+* https://releases.rancher.com/os/v1.5.8/vmware/rootfs.tar.gz
+
+#### Hyper-V Links
+
+* https://releases.rancher.com/os/v1.5.8/hyperv/initrd
+* https://releases.rancher.com/os/v1.5.8/hyperv/rootfs.tar.gz
+
+#### Proxmox VE Links
+
+* https://releases.rancher.com/os/v1.5.8/proxmoxve/initrd
+* https://releases.rancher.com/os/v1.5.8/proxmoxve/rootfs.tar.gz
+
+#### 4G-LTE Links
+
+* https://releases.rancher.com/os/v1.5.8/4glte/initrd
+* https://releases.rancher.com/os/v1.5.8/4glte/rootfs.tar.gz
+
+**Note**:
+1. you can use `http` instead of `https` in the above URLs, e.g. for iPXE.
+2. you can use `latest` instead of `v1.5.8` in the above URLs if you want to get the latest version.
 
 ### Amazon
 
@@ -69,36 +94,34 @@ SSH keys are added to the **`rancher`** user, so you must log in using the **ran
 
 **HVM**
 
-Region | Type | AMI |
+Region | Type | AMI
 -------|------|------
-ap-south-1 | HVM | [ami-3576085a](https://ap-south-1.console.aws.amazon.com/ec2/home?region=ap-south-1#launchInstanceWizard:ami=ami-3576085a)
-eu-west-2 | HVM | [ami-4806102c](https://eu-west-2.console.aws.amazon.com/ec2/home?region=eu-west-2#launchInstanceWizard:ami=ami-4806102c)
-eu-west-1 | HVM | [ami-64b2a802](https://eu-west-1.console.aws.amazon.com/ec2/home?region=eu-west-1#launchInstanceWizard:ami=ami-64b2a802)
-ap-northeast-2 | HVM | [ami-9d03dcf3](https://ap-northeast-2.console.aws.amazon.com/ec2/home?region=ap-northeast-2#launchInstanceWizard:ami=ami-9d03dcf3)
-ap-northeast-1 | HVM | [ami-8bb1a7ec](https://ap-northeast-1.console.aws.amazon.com/ec2/home?region=ap-northeast-1#launchInstanceWizard:ami=ami-8bb1a7ec)
-sa-east-1 | HVM | [ami-ae1b71c2](https://sa-east-1.console.aws.amazon.com/ec2/home?region=sa-east-1#launchInstanceWizard:ami=ami-ae1b71c2)
-ca-central-1 | HVM | [ami-4fa7182b](https://ca-central-1.console.aws.amazon.com/ec2/home?region=ca-central-1#launchInstanceWizard:ami=ami-4fa7182b)
-ap-southeast-1 | HVM | [ami-4f921c2c](https://ap-southeast-1.console.aws.amazon.com/ec2/home?region=ap-southeast-1#launchInstanceWizard:ami=ami-4f921c2c)
-ap-southeast-2 | HVM | [ami-d64c5fb5](https://ap-southeast-2.console.aws.amazon.com/ec2/home?region=ap-southeast-2#launchInstanceWizard:ami=ami-d64c5fb5)
-eu-central-1 | HVM | [ami-8c52f4e3](https://eu-central-1.console.aws.amazon.com/ec2/home?region=eu-central-1#launchInstanceWizard:ami=ami-8c52f4e3)
-us-east-1 | HVM | [ami-067c4a10](https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#launchInstanceWizard:ami=ami-067c4a10)
-us-east-2 | HVM | [ami-b74b6ad2](https://us-east-2.console.aws.amazon.com/ec2/home?region=us-east-2#launchInstanceWizard:ami=ami-b74b6ad2)
-us-west-1 | HVM | [ami-04351964](https://us-west-1.console.aws.amazon.com/ec2/home?region=us-west-1#launchInstanceWizard:ami=ami-04351964)
-us-west-2 | HVM | [ami-bed0c7c7](https://us-west-2.console.aws.amazon.com/ec2/home?region=us-west-2#launchInstanceWizard:ami=ami-bed0c7c7)
+eu-north-1 | HVM | [ami-08b189555c5d2d8c3](https://eu-north-1.console.aws.amazon.com/ec2/home?region=eu-north-1#launchInstanceWizard:ami=ami-08b189555c5d2d8c3)
+ap-south-1 | HVM | [ami-0086964cb1ffc4fdb](https://ap-south-1.console.aws.amazon.com/ec2/home?region=ap-south-1#launchInstanceWizard:ami=ami-0086964cb1ffc4fdb)
+eu-west-3 | HVM | [ami-088930cafc1ad9f20](https://eu-west-3.console.aws.amazon.com/ec2/home?region=eu-west-3#launchInstanceWizard:ami=ami-088930cafc1ad9f20)
+eu-west-2 | HVM | [ami-0fdf07cfd187af004](https://eu-west-2.console.aws.amazon.com/ec2/home?region=eu-west-2#launchInstanceWizard:ami=ami-0fdf07cfd187af004)
+eu-west-1 | HVM | [ami-0cea454c576ececbd](https://eu-west-1.console.aws.amazon.com/ec2/home?region=eu-west-1#launchInstanceWizard:ami=ami-0cea454c576ececbd)
+ap-northeast-2 | HVM | [ami-0fdb6555f88256d12](https://ap-northeast-2.console.aws.amazon.com/ec2/home?region=ap-northeast-2#launchInstanceWizard:ami=ami-0fdb6555f88256d12)
+ap-northeast-1 | HVM | [ami-052c75c3e8757bcd9](https://ap-northeast-1.console.aws.amazon.com/ec2/home?region=ap-northeast-1#launchInstanceWizard:ami=ami-052c75c3e8757bcd9)
+sa-east-1 | HVM | [ami-04e51c9d1edad1bfd](https://sa-east-1.console.aws.amazon.com/ec2/home?region=sa-east-1#launchInstanceWizard:ami=ami-04e51c9d1edad1bfd)
+ca-central-1 | HVM | [ami-006a1ff3bf21003b3](https://ca-central-1.console.aws.amazon.com/ec2/home?region=ca-central-1#launchInstanceWizard:ami=ami-006a1ff3bf21003b3)
+ap-southeast-1 | HVM | [ami-03b14c67c74644c2b](https://ap-southeast-1.console.aws.amazon.com/ec2/home?region=ap-southeast-1#launchInstanceWizard:ami=ami-03b14c67c74644c2b)
+ap-southeast-2 | HVM | [ami-07059c8f12411bfcb](https://ap-southeast-2.console.aws.amazon.com/ec2/home?region=ap-southeast-2#launchInstanceWizard:ami=ami-07059c8f12411bfcb)
+eu-central-1 | HVM | [ami-0fc1a9332c246bc56](https://eu-central-1.console.aws.amazon.com/ec2/home?region=eu-central-1#launchInstanceWizard:ami=ami-0fc1a9332c246bc56)
+us-east-1 | HVM | [ami-02fe87f853d560d52](https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#launchInstanceWizard:ami=ami-02fe87f853d560d52)
+us-east-2 | HVM | [ami-004259f4c48585992](https://us-east-2.console.aws.amazon.com/ec2/home?region=us-east-2#launchInstanceWizard:ami=ami-004259f4c48585992)
+us-west-1 | HVM | [ami-0b382b76fadc95544](https://us-west-1.console.aws.amazon.com/ec2/home?region=us-west-1#launchInstanceWizard:ami=ami-0b382b76fadc95544)
+us-west-2 | HVM | [ami-0cdefa6a0646eb511](https://us-west-2.console.aws.amazon.com/ec2/home?region=us-west-2#launchInstanceWizard:ami=ami-0cdefa6a0646eb511)
 
-Additionally, images are available with support for Amazon EC2 Container Service (ECS) [here](https://docs.rancher.com/os/amazon-ecs/#amazon-ecs-enabled-amis).
+Additionally, images are available with support for Amazon EC2 Container Service (ECS) [here](https://rancher.com/docs/os/v1.x/en/installation/amazon-ecs/#amazon-ecs-enabled-amis).
 
-### Google Compute Engine
+### Azure
 
-We are providing a disk image that users can download and import for use in Google Compute Engine. The image can be obtained from the release artifacts for RancherOS.
-
-[Download Image](https://github.com/rancher/os/releases/download/v1.0.0/rancheros-v1.0.0.tar.gz)
-
-Please follow the directions at our [docs to launch in GCE](http://docs.rancher.com/os/running-rancheros/cloud/gce/).
+You can get RancherOS in the [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/rancher.rancheros), currently only the `rancher` user can be logged in through SSH keys.
 
 ## Documentation for RancherOS
 
-Please refer to our [RancherOS Documentation](http://docs.rancher.com/os/) website to read all about RancherOS. It has detailed information on how RancherOS works, getting-started and other details.
+Please refer to our [RancherOS Documentation](https://rancher.com/docs/os/v1.x/en/) website to read all about RancherOS. It has detailed information on how RancherOS works, getting-started and other details.
 
 ## Support, Discussion, and Community
 If you need any help with RancherOS or Rancher, please join us at either our [Rancher forums](http://forums.rancher.com) or [#rancher IRC channel](http://webchat.freenode.net/?channels=rancher) where most of our team hangs out at.
@@ -110,8 +133,9 @@ Please submit any **RancherOS** bugs, issues, and feature requests to [rancher/o
 
 Please submit any **Rancher** bugs, issues, and feature requests to [rancher/rancher](//github.com/rancher/rancher/issues).
 
-#License
-Copyright (c) 2014-2017 [Rancher Labs, Inc.](http://rancher.com)
+## License
+
+Copyright (c) 2014-2020 [Rancher Labs, Inc.](http://rancher.com)
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

assets/docker/cni/bridge.d/bridge.conf

@@ -1,12 +0,0 @@
-{
-    "name": "bridge",
-    "type": "bridge",
-    "bridge": "docker-sys",
-    "isDefaultGateway": true,
-    "ipMasq": true,
-    "hairpinMode": true,
-    "ipam": {
-        "type": "host-local",
-        "subnet": "172.18.42.1/16"
-    }
-}

assets/docker/cni/default.d

@@ -1 +0,0 @@
-bridge.d/

assets/docker/hooks/poststop.d/network.json

@@ -1,7 +0,0 @@
-{
-  "path": "/usr/bin/ros",
-  "args": [
-    "cni-glue",
-    "poststop"
-  ]
-}

assets/docker/hooks/prestart.d/network.json

@@ -1,6 +0,0 @@
-{
-  "path": "/usr/bin/ros",
-  "args": [
-    "cni-glue"
-  ]
-}

assets/go-dnsclient.patch

@@ -1,18 +0,0 @@
-296a297,300
-> 	conf.update(name)
-> }
-> 
-> func (conf *resolverConfig) update(name string) {
-300a305,316
-> }
-> 
-> func UpdateDnsConf() {
-> 	resolvConf.initOnce.Do(resolvConf.init)
-> 
-> 	// Ensure only one update at a time checks resolv.conf.
-> 	if !resolvConf.tryAcquireSema() {
-> 		return
-> 	}
-> 	defer resolvConf.releaseSema()
-> 
-> 	resolvConf.update("/etc/resolv.conf")

cmd/cloudinitexecute/authorize_ssh_keys.go

@@ -7,8 +7,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 )
 
 const (

cmd/cloudinitexecute/cloudinitexecute.go

@@ -10,10 +10,12 @@ import (
 	"strings"
 
 	rancherConfig "github.com/rancher/os/config"
+	"github.com/rancher/os/config/cloudinit/config"
 	"github.com/rancher/os/config/cloudinit/system"
-	"github.com/rancher/os/docker"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+
 	"golang.org/x/net/context"
 )
 
@@ -108,7 +110,10 @@ func ApplyConsole(cfg *rancherConfig.CloudConfig) {
 		}
 	}
 
-	util.RunCommandSequence(cfg.Runcmd)
+	err := util.RunCommandSequence(cfg.Runcmd)
+	if err != nil {
+		log.Error(err)
+	}
 }
 
 func WriteFiles(cfg *rancherConfig.CloudConfig, container string) {
@@ -121,6 +126,14 @@ func WriteFiles(cfg *rancherConfig.CloudConfig, container string) {
 			continue
 		}
 
+		content, err := config.DecodeContent(file.File.Content, file.File.Encoding)
+		if err != nil {
+			continue
+		}
+
+		file.File.Content = string(content)
+		file.File.Encoding = ""
+
 		f := system.File{
 			File: file.File,
 		}
@@ -168,7 +181,17 @@ func applyPreConsole(cfg *rancherConfig.CloudConfig) {
 }
 
 func resizeDevice(cfg *rancherConfig.CloudConfig) error {
-	cmd := exec.Command("growpart", cfg.Rancher.ResizeDevice, "1")
+	partition := "1"
+	targetPartition := fmt.Sprintf("%s%s", cfg.Rancher.ResizeDevice, partition)
+
+	if strings.Contains(cfg.Rancher.ResizeDevice, "mmcblk") {
+		partition = "2"
+		targetPartition = fmt.Sprintf("%sp%s", cfg.Rancher.ResizeDevice, partition)
+	} else if strings.Contains(cfg.Rancher.ResizeDevice, "nvme") {
+		targetPartition = fmt.Sprintf("%sp%s", cfg.Rancher.ResizeDevice, partition)
+	}
+
+	cmd := exec.Command("growpart", cfg.Rancher.ResizeDevice, partition)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	cmd.Run()
@@ -181,7 +204,7 @@ func resizeDevice(cfg *rancherConfig.CloudConfig) error {
 		return err
 	}
 
-	cmd = exec.Command("resize2fs", fmt.Sprintf("%s1", cfg.Rancher.ResizeDevice))
+	cmd = exec.Command("resize2fs", targetPartition)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()

cmd/cloudinitsave/cloudinitsave.go

@@ -24,8 +24,6 @@ import (
 	"sync"
 	"time"
 
-	yaml "github.com/cloudfoundry-incubator/candiedyaml"
-
 	"github.com/rancher/os/cmd/control"
 	"github.com/rancher/os/cmd/network"
 	rancherConfig "github.com/rancher/os/config"
@@ -33,17 +31,25 @@ import (
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/datasource/configdrive"
 	"github.com/rancher/os/config/cloudinit/datasource/file"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/aliyun"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/azure"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/cloudstack"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/digitalocean"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/ec2"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/exoscale"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/gce"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/packet"
 	"github.com/rancher/os/config/cloudinit/datasource/proccmdline"
+	"github.com/rancher/os/config/cloudinit/datasource/proxmox"
+	"github.com/rancher/os/config/cloudinit/datasource/tftp"
 	"github.com/rancher/os/config/cloudinit/datasource/url"
 	"github.com/rancher/os/config/cloudinit/datasource/vmware"
 	"github.com/rancher/os/config/cloudinit/pkg"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/netconf"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/netconf"
+	"github.com/rancher/os/pkg/util"
+
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 )
 
 const (
@@ -63,6 +69,11 @@ func Main() {
 	if err := saveCloudConfig(); err != nil {
 		log.Errorf("Failed to save cloud-config: %v", err)
 	}
+
+	// exit wpa_supplicant
+	netconf.StopWpaSupplicant()
+	// exit dhcpcd
+	netconf.StopDhcpcd()
 }
 
 func saveCloudConfig() error {
@@ -72,7 +83,7 @@ func saveCloudConfig() error {
 	log.Debugf("init: SaveCloudConfig(pre ApplyNetworkConfig): %#v", cfg.Rancher.Network)
 	network.ApplyNetworkConfig(cfg)
 
-	log.Infof("datasources that will be consided: %#v", cfg.Rancher.CloudInit.Datasources)
+	log.Infof("datasources that will be considered: %#v", cfg.Rancher.CloudInit.Datasources)
 	dss := getDatasources(cfg.Rancher.CloudInit.Datasources)
 	if len(dss) == 0 {
 		log.Errorf("currentDatasource - none found")
@@ -90,23 +101,6 @@ func saveCloudConfig() error {
 	return nil
 }
 
-func RequiresNetwork(datasource string) bool {
-	// TODO: move into the datasources (and metadatasources)
-	// and then we can enable that platforms defaults..
-	parts := strings.SplitN(datasource, ":", 2)
-	requiresNetwork, ok := map[string]bool{
-		"ec2":          true,
-		"file":         false,
-		"url":          true,
-		"cmdline":      true,
-		"configdrive":  false,
-		"digitalocean": true,
-		"gce":          true,
-		"packet":       true,
-	}[parts[0]]
-	return ok && requiresNetwork
-}
-
 func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
 	os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600)
 
@@ -180,6 +174,11 @@ func fetchAndSave(ds datasource.Datasource) error {
 		log.Errorf("Failed fetching user-data from datasource: %v", err)
 		return err
 	}
+	userDataBytes, err = decompressIfGzip(userDataBytes)
+	if err != nil {
+		log.Errorf("Failed decompressing user-data from datasource: %v", err)
+		return err
+	}
 	log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
 	metadata, err = ds.FetchMetadata()
 	if err != nil {
@@ -231,13 +230,26 @@ func getDatasources(datasources []string) []datasource.Datasource {
 
 		switch parts[0] {
 		case "*":
-			dss = append(dss, getDatasources([]string{"configdrive", "vmware", "ec2", "digitalocean", "packet", "gce"})...)
+			dss = append(dss, getDatasources([]string{"configdrive", "vmware", "ec2", "digitalocean", "packet", "gce", "cloudstack", "exoscale", "proxmox"})...)
+		case "proxmox":
+			if root == "" {
+				root = "/media/pve-config"
+			}
+			dss = append(dss, proxmox.NewDataSource(root))
+		case "exoscale":
+			dss = append(dss, exoscale.NewDatasource(root))
+		case "cloudstack":
+			for _, source := range cloudstack.NewDatasource(root) {
+				dss = append(dss, source)
+			}
 		case "ec2":
 			dss = append(dss, ec2.NewDatasource(root))
 		case "file":
 			if root != "" {
 				dss = append(dss, file.NewDatasource(root))
 			}
+		case "tftp":
+			dss = append(dss, tftp.NewDatasource(root))
 		case "url":
 			if root != "" {
 				dss = append(dss, url.NewDatasource(root))
@@ -264,6 +276,10 @@ func getDatasources(datasources []string) []datasource.Datasource {
 			if v != nil {
 				dss = append(dss, v)
 			}
+		case "aliyun":
+			dss = append(dss, aliyun.NewDatasource(root))
+		case "azure":
+			dss = append(dss, azure.NewDatasource(root))
 		}
 	}
 
@@ -271,12 +287,18 @@ func getDatasources(datasources []string) []datasource.Datasource {
 }
 
 func enableDoLinkLocal() {
-	err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
+	cfg := rancherConfig.LoadConfig()
+	dhcpTimeout := cfg.Rancher.Defaults.Network.DHCPTimeout
+	if cfg.Rancher.Network.DHCPTimeout > 0 {
+		dhcpTimeout = cfg.Rancher.Network.DHCPTimeout
+	}
+	_, err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
 		Interfaces: map[string]netconf.InterfaceConfig{
 			"eth0": {
 				IPV4LL: true,
 			},
 		},
+		DHCPTimeout: dhcpTimeout,
 	}, false, false)
 	if err != nil {
 		log.Errorf("Failed to apply link local on eth0: %v", err)
@@ -359,3 +381,13 @@ func composeToCloudConfig(bytes []byte) ([]byte, error) {
 		},
 	})
 }
+
+const gzipMagicBytes = "\x1f\x8b"
+
+func decompressIfGzip(userdataBytes []byte) ([]byte, error) {
+	if !bytes.HasPrefix(userdataBytes, []byte(gzipMagicBytes)) {
+		return userdataBytes, nil
+	}
+
+	return config.DecompressGzip(userdataBytes)
+}

cmd/control/autologin.go

@@ -8,9 +8,10 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/codegangsta/cli"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/pkg/log"
+
+	"github.com/codegangsta/cli"
 )
 
 func AutologinMain() {
@@ -73,7 +74,9 @@ func autologinAction(c *cli.Context) error {
 		// until I make time to read their source, lets just give us a way to get work done
 		loginBin = "bash"
 		args = append(args, "--login")
-		os.Setenv("PROMPT_COMMAND", `echo "[`+fmt.Sprintf("Recovery console %s@%s:${PWD}", user, cfg.Hostname)+`]"`)
+		if mode == "recovery" {
+			os.Setenv("PROMPT_COMMAND", `echo "[`+fmt.Sprintf("Recovery console %s@%s:${PWD}", user, cfg.Hostname)+`]"`)
+		}
 	} else {
 		loginBin = "login"
 		args = append(args, "-f", user)
@@ -91,7 +94,6 @@ func autologinAction(c *cli.Context) error {
 	//return syscall.Exec(loginBinPath, args, os.Environ())
 	cmd = exec.Command(loginBinPath, args...)
 	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, "SVEN", "MORE")
 
 	cmd.Stderr = os.Stderr
 	cmd.Stdout = os.Stdout

cmd/control/bootstrap.go

@@ -1,20 +1,22 @@
 package control
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/codegangsta/cli"
-
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 )
 
-func bootstrapAction(c *cli.Context) error {
+func BootstrapMain() {
+	log.InitLogger()
+
 	log.Debugf("bootstrapAction")
 	if err := UdevSettle(); err != nil {
 		log.Errorf("Failed to run udev settle: %v", err)
@@ -23,6 +25,13 @@ func bootstrapAction(c *cli.Context) error {
 	log.Debugf("bootstrapAction: loadingConfig")
 	cfg := config.LoadConfig()
 
+	log.Debugf("bootstrapAction: Rngd(%v)", cfg.Rancher.State.Rngd)
+	if cfg.Rancher.State.Rngd {
+		if err := runRngd(); err != nil {
+			log.Errorf("Failed to run rngd: %v", err)
+		}
+	}
+
 	log.Debugf("bootstrapAction: MdadmScan(%v)", cfg.Rancher.State.MdadmScan)
 	if cfg.Rancher.State.MdadmScan {
 		if err := mdadmScan(); err != nil {
@@ -30,6 +39,20 @@ func bootstrapAction(c *cli.Context) error {
 		}
 	}
 
+	log.Debugf("bootstrapAction: cryptsetup(%v)", cfg.Rancher.State.Cryptsetup)
+	if cfg.Rancher.State.Cryptsetup {
+		if err := cryptsetup(); err != nil {
+			log.Errorf("Failed to run cryptsetup: %v", err)
+		}
+	}
+
+	log.Debugf("bootstrapAction: LvmScan(%v)", cfg.Rancher.State.LvmScan)
+	if cfg.Rancher.State.LvmScan {
+		if err := vgchange(); err != nil {
+			log.Errorf("Failed to run vgchange: %v", err)
+		}
+	}
+
 	stateScript := cfg.Rancher.State.Script
 	log.Debugf("bootstrapAction: stateScript(%v)", stateScript)
 	if stateScript != "" {
@@ -39,7 +62,10 @@ func bootstrapAction(c *cli.Context) error {
 	}
 
 	log.Debugf("bootstrapAction: RunCommandSequence(%v)", cfg.Bootcmd)
-	util.RunCommandSequence(cfg.Bootcmd)
+	err := util.RunCommandSequence(cfg.Bootcmd)
+	if err != nil {
+		log.Error(err)
+	}
 
 	if cfg.Rancher.State.Dev != "" && cfg.Rancher.State.Wait {
 		waitForRoot(cfg)
@@ -56,8 +82,6 @@ func bootstrapAction(c *cli.Context) error {
 	if err := UdevSettle(); err != nil {
 		log.Errorf("Failed to run udev settle: %v", err)
 	}
-
-	return nil
 }
 
 func mdadmScan() error {
@@ -67,6 +91,56 @@ func mdadmScan() error {
 	return cmd.Run()
 }
 
+func vgchange() error {
+	cmd := exec.Command("vgchange", "--activate", "ay")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func cryptsetup() error {
+	devices, err := util.BlkidType("crypto_LUKS")
+	if err != nil {
+		return err
+	}
+
+	for _, cryptdevice := range devices {
+		fdRead, err := os.Open("/dev/console")
+		if err != nil {
+			return err
+		}
+		defer fdRead.Close()
+
+		fdWrite, err := os.OpenFile("/dev/console", os.O_WRONLY|os.O_APPEND, 0)
+		if err != nil {
+			return err
+		}
+		defer fdWrite.Close()
+
+		cmd := exec.Command("cryptsetup", "luksOpen", cryptdevice, fmt.Sprintf("luks-%s", filepath.Base(cryptdevice)))
+		cmd.Stdout = fdWrite
+		cmd.Stderr = fdWrite
+		cmd.Stdin = fdRead
+
+		if err := cmd.Run(); err != nil {
+			log.Errorf("Failed to run cryptsetup for %s: %v", cryptdevice, err)
+		}
+	}
+
+	return nil
+}
+
+func runRngd() error {
+	// use /dev/urandom as random number input for rngd
+	// this is a really bad idea
+	// since I am simple filling the kernel entropy pool with entropy coming from the kernel itself!
+	// but this does not need to consider the user's hw rngd drivers.
+	cmd := exec.Command("rngd", "-r", "/dev/urandom", "-q")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
 func runStateScript(script string) error {
 	f, err := ioutil.TempFile("", "")
 	if err != nil {

cmd/control/cli.go

@@ -4,14 +4,20 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/codegangsta/cli"
 	"github.com/rancher/os/cmd/control/service"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/pkg/log"
+
+	"github.com/codegangsta/cli"
 )
 
 func Main() {
 	log.InitLogger()
+	cli.VersionPrinter = func(c *cli.Context) {
+		cfg := config.LoadConfig()
+		runningName := cfg.Rancher.Upgrade.Image + ":" + config.Version
+		fmt.Fprintf(c.App.Writer, "version %s from os image %s\n", c.App.Version, runningName)
+	}
 	app := cli.NewApp()
 
 	app.Name = os.Args[0]
@@ -27,13 +33,6 @@ func Main() {
 	}
 
 	app.Commands = []cli.Command{
-		{
-			Name:            "bootstrap",
-			Hidden:          true,
-			HideHelp:        true,
-			SkipFlagParsing: true,
-			Action:          bootstrapAction,
-		},
 		{
 			Name:        "config",
 			ShortName:   "c",

cmd/control/config.go

@@ -1,6 +1,8 @@
 package control
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -10,12 +12,13 @@ import (
 	"strings"
 	"text/template"
 
-	yaml "github.com/cloudfoundry-incubator/candiedyaml"
-	"github.com/rancher/os/log"
-
-	"github.com/codegangsta/cli"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
+	"github.com/codegangsta/cli"
+	"github.com/pkg/errors"
 )
 
 func configSubcommands() []cli.Command {
@@ -153,6 +156,22 @@ func env2map(env []string) map[string]string {
 }
 
 func editSyslinux(c *cli.Context) error {
+	// check whether is Raspberry Pi or not
+	bytes, err := ioutil.ReadFile("/proc/device-tree/model")
+	if err == nil && strings.Contains(strings.ToLower(string(bytes)), "raspberry") {
+		buf := bufio.NewWriter(os.Stdout)
+		fmt.Fprintln(buf, "raspberry pi can not use this command")
+		buf.Flush()
+		return errors.New("raspberry pi can not use this command")
+	}
+
+	if isExist := checkGlobalCfg(); !isExist {
+		buf := bufio.NewWriter(os.Stdout)
+		fmt.Fprintln(buf, "global.cfg can not be found")
+		buf.Flush()
+		return errors.New("global.cfg can not be found")
+	}
+
 	cmd := exec.Command("system-docker", "run", "--rm", "-it",
 		"-v", "/:/host",
 		"-w", "/host",
@@ -280,5 +299,12 @@ func inputBytes(c *cli.Context) ([]byte, error) {
 		}
 		defer input.Close()
 	}
-	return ioutil.ReadAll(input)
+	content, err := ioutil.ReadAll(input)
+	if err != nil {
+		return nil, err
+	}
+	if bytes.Contains(content, []byte{13, 10}) {
+		return nil, errors.New("file format shouldn't contain CRLF characters")
+	}
+	return content, nil
 }

cmd/control/config_test.go

@@ -2,11 +2,11 @@ package control
 
 import (
 	"bytes"
+	"os"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"os"
 )
 
 func TestGenTpl(t *testing.T) {

cmd/control/console.go

@@ -5,19 +5,19 @@ import (
 	"sort"
 	"strings"
 
-	"golang.org/x/net/context"
+	"github.com/rancher/os/cmd/control/service"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/compose"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+	"github.com/rancher/os/pkg/util/network"
 
 	"github.com/codegangsta/cli"
 	"github.com/docker/docker/reference"
 	composeConfig "github.com/docker/libcompose/config"
 	"github.com/docker/libcompose/project/options"
-	"github.com/rancher/os/cmd/control/service"
-	"github.com/rancher/os/compose"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/docker"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
-	"github.com/rancher/os/util/network"
+	"golang.org/x/net/context"
 )
 
 func consoleSubcommands() []cli.Command {
@@ -43,8 +43,14 @@ func consoleSubcommands() []cli.Command {
 			Action: consoleEnable,
 		},
 		{
-			Name:   "list",
-			Usage:  "list available consoles",
+			Name:  "list",
+			Usage: "list available consoles",
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "update, u",
+					Usage: "update console cache",
+				},
+			},
 			Action: consoleList,
 		},
 	}
@@ -127,7 +133,7 @@ func consoleEnable(c *cli.Context) error {
 
 func consoleList(c *cli.Context) error {
 	cfg := config.LoadConfig()
-	consoles := availableConsoles(cfg)
+	consoles := availableConsoles(cfg, c.Bool("update"))
 	CurrentConsole := CurrentConsole()
 
 	for _, console := range consoles {
@@ -144,13 +150,20 @@ func consoleList(c *cli.Context) error {
 }
 
 func validateConsole(console string, cfg *config.CloudConfig) {
-	consoles := availableConsoles(cfg)
+	consoles := availableConsoles(cfg, false)
 	if !service.IsLocalOrURL(console) && !util.Contains(consoles, console) {
 		log.Fatalf("%s is not a valid console", console)
 	}
 }
 
-func availableConsoles(cfg *config.CloudConfig) []string {
+func availableConsoles(cfg *config.CloudConfig, update bool) []string {
+	if update {
+		err := network.UpdateCaches(cfg.Rancher.Repositories.ToArray(), "consoles")
+		if err != nil {
+			log.Debugf("Failed to update console caches: %v", err)
+		}
+
+	}
 	consoles, err := network.GetConsoles(cfg.Rancher.Repositories.ToArray())
 	if err != nil {
 		log.Fatal(err)

cmd/control/console_init.go

@@ -7,15 +7,21 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"regexp"
+	"strconv"
 	"strings"
 	"syscall"
+	"text/template"
 
-	"github.com/codegangsta/cli"
 	"github.com/rancher/os/cmd/cloudinitexecute"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/config/cmdline"
+	"github.com/rancher/os/pkg/compose"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+
+	"github.com/codegangsta/cli"
+	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -24,18 +30,15 @@ const (
 	gettyCmd    = "/sbin/agetty"
 	rancherHome = "/home/rancher"
 	startScript = "/opt/rancher/bin/start.sh"
+	runLockDir  = "/run/lock"
+	sshdFile    = "/etc/ssh/sshd_config"
+	sshdTplFile = "/etc/ssh/sshd_config.tpl"
 )
 
 type symlink struct {
 	oldname, newname string
 }
 
-func ConsoleInitMain() {
-	if err := consoleInitFunc(); err != nil {
-		log.Fatal(err)
-	}
-}
-
 func consoleInitAction(c *cli.Context) error {
 	return consoleInitFunc()
 }
@@ -63,8 +66,32 @@ func consoleInitFunc() error {
 	createHomeDir(rancherHome, 1100, 1100)
 	createHomeDir(dockerHome, 1101, 1101)
 
-	password := config.GetCmdline("rancher.password")
-	if password != "" {
+	// who & w command need this file
+	if _, err := os.Stat("/run/utmp"); os.IsNotExist(err) {
+		f, err := os.OpenFile("/run/utmp", os.O_RDWR|os.O_CREATE, 0644)
+		if err != nil {
+			log.Error(err)
+		}
+		defer f.Close()
+	}
+
+	// some software need this dir, like open-iscsi
+	if _, err := os.Stat(runLockDir); os.IsNotExist(err) {
+		if err = os.Mkdir(runLockDir, 0755); err != nil {
+			log.Error(err)
+		}
+	}
+
+	ignorePassword := false
+	for _, d := range cfg.Rancher.Disable {
+		if d == "password" {
+			ignorePassword = true
+			break
+		}
+	}
+
+	password := cmdline.GetCmdline("rancher.password")
+	if !ignorePassword && password != "" {
 		cmd := exec.Command("chpasswd")
 		cmd.Stdin = strings.NewReader(fmt.Sprint("rancher:", password))
 		if err := cmd.Run(); err != nil {
@@ -85,33 +112,105 @@ func consoleInitFunc() error {
 		log.Error(err)
 	}
 
-	if err := modifySshdConfig(); err != nil {
+	if err := modifySshdConfig(cfg); err != nil {
 		log.Error(err)
 	}
 
-	for _, link := range []symlink{
-		{"/var/lib/rancher/engine/docker", "/usr/bin/docker"},
-		{"/var/lib/rancher/engine/docker-init", "/usr/bin/docker-init"},
-		{"/var/lib/rancher/engine/docker-containerd", "/usr/bin/docker-containerd"},
-		{"/var/lib/rancher/engine/docker-containerd-ctr", "/usr/bin/docker-containerd-ctr"},
-		{"/var/lib/rancher/engine/docker-containerd-shim", "/usr/bin/docker-containerd-shim"},
-		{"/var/lib/rancher/engine/dockerd", "/usr/bin/dockerd"},
-		{"/var/lib/rancher/engine/docker-proxy", "/usr/bin/docker-proxy"},
-		{"/var/lib/rancher/engine/docker-runc", "/usr/bin/docker-runc"},
-		{"/usr/share/ros/os-release", "/usr/lib/os-release"},
-		{"/usr/share/ros/os-release", "/etc/os-release"},
-	} {
+	p, err := compose.GetProject(cfg, false, true)
+	if err != nil {
+		log.Error(err)
+	}
+
+	// check the multi engine service & generate the multi engine script
+	for _, key := range p.ServiceConfigs.Keys() {
+		serviceConfig, ok := p.ServiceConfigs.Get(key)
+		if !ok {
+			log.Errorf("Failed to get service config from the project")
+			continue
+		}
+		if _, ok := serviceConfig.Labels[config.UserDockerLabel]; ok {
+			err = util.GenerateDindEngineScript(serviceConfig.Labels[config.UserDockerLabel])
+			if err != nil {
+				log.Errorf("Failed to generate engine script: %v", err)
+				continue
+			}
+		}
+	}
+
+	baseSymlink := symLinkEngineBinary()
+
+	if _, err := os.Stat(dockerCompletionFile); err == nil {
+		baseSymlink = append(baseSymlink, symlink{
+			dockerCompletionFile, dockerCompletionLinkFile,
+		})
+	}
+
+	if cfg.Rancher.Console == "default" {
+		// add iptables symlinks for default console
+		baseSymlink = append(baseSymlink, []symlink{
+			{"/usr/sbin/iptables", "/usr/sbin/iptables-save"},
+			{"/usr/sbin/iptables", "/usr/sbin/iptables-restore"},
+			{"/usr/sbin/iptables", "/usr/sbin/ip6tables"},
+			{"/usr/sbin/iptables", "/usr/sbin/ip6tables-save"},
+			{"/usr/sbin/iptables", "/usr/sbin/ip6tables-restore"},
+			{"/usr/sbin/iptables", "/usr/bin/iptables-xml"},
+		}...)
+	}
+
+	for _, link := range baseSymlink {
 		syscall.Unlink(link.newname)
 		if err := os.Symlink(link.oldname, link.newname); err != nil {
 			log.Error(err)
 		}
 	}
 
+	// mount systemd cgroups
+	if err := os.MkdirAll("/sys/fs/cgroup/systemd", 0555); err != nil {
+		log.Error(err)
+	}
+	if err := unix.Mount("cgroup", "/sys/fs/cgroup/systemd", "cgroup", 0, "none,name=systemd"); err != nil {
+		log.Error(err)
+	}
+
 	// font backslashes need to be escaped for when issue is output! (but not the others..)
 	if err := ioutil.WriteFile("/etc/issue", []byte(config.Banner), 0644); err != nil {
 		log.Error(err)
 	}
 
+	// write out a profile.d file for the proxy settings.
+	// maybe write these on the host and bindmount into everywhere?
+	proxyLines := []string{}
+	for _, k := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} {
+		if v, ok := cfg.Rancher.Environment[k]; ok {
+			proxyLines = append(proxyLines, fmt.Sprintf("export %s=%q", k, v))
+		}
+	}
+
+	if len(proxyLines) > 0 {
+		proxyString := strings.Join(proxyLines, "\n")
+		proxyString = fmt.Sprintf("#!/bin/sh\n%s\n", proxyString)
+		if err := ioutil.WriteFile("/etc/profile.d/proxy.sh", []byte(proxyString), 0755); err != nil {
+			log.Error(err)
+		}
+	}
+
+	// write out a profile.d file for the PATH settings.
+	pathLines := []string{}
+	for _, k := range []string{"PATH", "path"} {
+		if v, ok := cfg.Rancher.Environment[k]; ok {
+			for _, p := range strings.Split(v, ",") {
+				pathLines = append(pathLines, fmt.Sprintf("export PATH=$PATH:%s", strings.TrimSpace(p)))
+			}
+		}
+	}
+	if len(pathLines) > 0 {
+		pathString := strings.Join(pathLines, "\n")
+		pathString = fmt.Sprintf("#!/bin/sh\n%s\n", pathString)
+		if err := ioutil.WriteFile("/etc/profile.d/path.sh", []byte(pathString), 0755); err != nil {
+			log.Error(err)
+		}
+	}
+
 	cmd = exec.Command("bash", "-c", `echo $(/sbin/ifconfig | grep -B1 "inet addr" |awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' |awk -F: '{ print $1 ": " $3}') >> /etc/issue`)
 	if err := cmd.Run(); err != nil {
 		log.Error(err)
@@ -152,11 +251,24 @@ func generateRespawnConf(cmdline, user string, sshd, recovery bool) string {
 		autologinBin = "/usr/bin/recovery"
 	}
 
+	config := config.LoadConfig()
+
+	allowAutoLogin := true
+	for _, d := range config.Rancher.Disable {
+		if d == "autologin" {
+			allowAutoLogin = false
+			break
+		}
+	}
+
 	for i := 1; i < 7; i++ {
 		tty := fmt.Sprintf("tty%d", i)
+		if !istty(tty) {
+			continue
+		}
 
 		respawnConf.WriteString(gettyCmd)
-		if strings.Contains(cmdline, fmt.Sprintf("rancher.autologin=%s", tty)) {
+		if allowAutoLogin && strings.Contains(cmdline, fmt.Sprintf("rancher.autologin=%s", tty)) {
 			respawnConf.WriteString(fmt.Sprintf(" -n -l %s -o %s:tty%d", autologinBin, user, i))
 		}
 		respawnConf.WriteString(fmt.Sprintf(" --noclear %s linux\n", tty))
@@ -167,8 +279,12 @@ func generateRespawnConf(cmdline, user string, sshd, recovery bool) string {
 			continue
 		}
 
+		if !istty(tty) {
+			continue
+		}
+
 		respawnConf.WriteString(gettyCmd)
-		if strings.Contains(cmdline, fmt.Sprintf("rancher.autologin=%s", tty)) {
+		if allowAutoLogin && strings.Contains(cmdline, fmt.Sprintf("rancher.autologin=%s", tty)) {
 			respawnConf.WriteString(fmt.Sprintf(" -n -l %s -o %s:%s", autologinBin, user, tty))
 		}
 		respawnConf.WriteString(fmt.Sprintf(" %s\n", tty))
@@ -207,29 +323,34 @@ func writeRespawn(user string, sshd, recovery bool) error {
 	return ioutil.WriteFile("/etc/respawn.conf", []byte(respawn), 0644)
 }
 
-func modifySshdConfig() error {
-	sshdConfig, err := ioutil.ReadFile("/etc/ssh/sshd_config")
-	if err != nil {
-		return err
-	}
-	sshdConfigString := string(sshdConfig)
-
-	for _, item := range []string{
-		"UseDNS no",
-		"PermitRootLogin no",
-		"ServerKeyBits 2048",
-		"AllowGroups docker",
-	} {
-		match, err := regexp.Match("^"+item, sshdConfig)
+func modifySshdConfig(cfg *config.CloudConfig) error {
+	_, err := os.Stat(sshdTplFile)
+	if err == nil {
+		os.Remove(sshdFile)
+		sshdTpl, err := template.ParseFiles(sshdTplFile)
 		if err != nil {
 			return err
 		}
-		if !match {
-			sshdConfigString += fmt.Sprintf("%s\n", item)
+		f, err := os.OpenFile(sshdFile, os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			return err
 		}
+		defer f.Close()
+
+		config := map[string]string{}
+		if cfg.Rancher.SSH.Port > 0 && cfg.Rancher.SSH.Port < 65355 {
+			config["Port"] = strconv.Itoa(cfg.Rancher.SSH.Port)
+		}
+		if cfg.Rancher.SSH.ListenAddress != "" {
+			config["ListenAddress"] = cfg.Rancher.SSH.ListenAddress
+		}
+
+		return sshdTpl.Execute(f, config)
+	} else if os.IsNotExist(err) {
+		return nil
 	}
 
-	return ioutil.WriteFile("/etc/ssh/sshd_config", []byte(sshdConfigString), 0644)
+	return err
 }
 
 func setupSSH(cfg *config.CloudConfig) error {
@@ -276,3 +397,10 @@ func setupSSH(cfg *config.CloudConfig) error {
 
 	return os.MkdirAll("/var/run/sshd", 0644)
 }
+
+func istty(name string) bool {
+	if f, err := os.Open(fmt.Sprintf("/dev/%s", name)); err == nil {
+		return terminal.IsTerminal(int(f.Fd()))
+	}
+	return false
+}

cmd/control/dev.go

@@ -3,8 +3,9 @@ package control
 import (
 	"fmt"
 
+	"github.com/rancher/os/pkg/util"
+
 	"github.com/codegangsta/cli"
-	"github.com/rancher/os/util"
 )
 
 func devAction(c *cli.Context) error {

cmd/control/docker_init.go

@@ -9,15 +9,19 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+
 	"github.com/codegangsta/cli"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
 )
 
 const (
-	dockerConf = "/var/lib/rancher/conf/docker"
-	dockerDone = "/run/docker-done"
-	dockerLog  = "/var/log/docker.log"
+	dockerConf               = "/var/lib/rancher/conf/docker"
+	dockerDone               = "/run/docker-done"
+	dockerLog                = "/var/log/docker.log"
+	dockerCompletionLinkFile = "/usr/share/bash-completion/completions/docker"
+	dockerCompletionFile     = "/var/lib/rancher/engine/completion"
 )
 
 func dockerInitAction(c *cli.Context) error {
@@ -29,6 +33,12 @@ func dockerInitAction(c *cli.Context) error {
 		time.Sleep(200 * time.Millisecond)
 	}
 
+	if _, err := os.Stat(dockerCompletionFile); err != nil {
+		if _, err := os.Readlink(dockerCompletionLinkFile); err == nil {
+			syscall.Unlink(dockerCompletionLinkFile)
+		}
+	}
+
 	dockerBin := ""
 	dockerPaths := []string{
 		"/usr/bin",
@@ -70,11 +80,27 @@ func dockerInitAction(c *cli.Context) error {
 	}
 
 	for _, mount := range strings.Split(string(mountInfo), "\n") {
-		if strings.Contains(mount, "/var/lib/docker /var/lib/docker") && strings.Contains(mount, "rootfs") {
-			os.Setenv("DOCKER_RAMDISK", "1")
+		if strings.Contains(mount, "/var/lib/user-docker /var/lib/docker") && strings.Contains(mount, "rootfs") {
+			os.Setenv("DOCKER_RAMDISK", "true")
 		}
 	}
 
+	cfg := config.LoadConfig()
+
+	for _, link := range symLinkEngineBinary() {
+		syscall.Unlink(link.newname)
+		if _, err := os.Stat(link.oldname); err == nil {
+			if err := os.Symlink(link.oldname, link.newname); err != nil {
+				log.Error(err)
+			}
+		}
+	}
+
+	err = checkZfsBackingFS(cfg.Rancher.Docker.StorageDriver, cfg.Rancher.Docker.Graph)
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	args := []string{
 		"bash",
 		"-c",

cmd/control/engine.go

@@ -2,28 +2,41 @@ package control
 
 import (
 	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path"
 	"sort"
+	"strconv"
 	"strings"
 
-	"golang.org/x/net/context"
+	"github.com/rancher/os/cmd/control/service"
+	"github.com/rancher/os/cmd/control/service/app"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/compose"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+	"github.com/rancher/os/pkg/util/network"
+	"github.com/rancher/os/pkg/util/versions"
 
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 	"github.com/codegangsta/cli"
 	"github.com/docker/docker/reference"
+	"github.com/docker/engine-api/types"
+	"github.com/docker/engine-api/types/filters"
+	composeConfig "github.com/docker/libcompose/config"
 	"github.com/docker/libcompose/project/options"
-	"github.com/rancher/os/cmd/control/service"
-	"github.com/rancher/os/compose"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/docker"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
-	"github.com/rancher/os/util/network"
+	composeYaml "github.com/docker/libcompose/yaml"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
 )
 
 func engineSubcommands() []cli.Command {
 	return []cli.Command{
 		{
 			Name:   "switch",
-			Usage:  "switch Docker engine without a reboot",
+			Usage:  "switch user Docker engine without a reboot",
 			Action: engineSwitch,
 			Flags: []cli.Flag{
 				cli.BoolFlag{
@@ -36,14 +49,74 @@ func engineSubcommands() []cli.Command {
 				},
 			},
 		},
+		{
+			Name:        "create",
+			Usage:       "create Dind engine without a reboot",
+			Description: "must switch user docker to 17.12.1 or earlier if using Dind",
+			ArgsUsage:   "<name>",
+			Before:      preFlightValidate,
+			Action:      engineCreate,
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "version, v",
+					Value: config.DefaultDind,
+					Usage: fmt.Sprintf("set the version for the engine, %s are available", config.SupportedDinds),
+				},
+				cli.StringFlag{
+					Name:  "network",
+					Usage: "set the network for the engine",
+				},
+				cli.StringFlag{
+					Name:  "fixed-ip",
+					Usage: "set the fixed ip for the engine",
+				},
+				cli.StringFlag{
+					Name:  "ssh-port",
+					Usage: "set the ssh port for the engine",
+				},
+				cli.StringFlag{
+					Name:  "authorized-keys",
+					Usage: "set the ssh authorized_keys absolute path for the engine",
+				},
+			},
+		},
+		{
+			Name:      "rm",
+			Usage:     "remove Dind engine without a reboot",
+			ArgsUsage: "<name>",
+			Before: func(c *cli.Context) error {
+				if len(c.Args()) != 1 {
+					return errors.New("Must specify exactly one Docker engine to remove")
+				}
+				return nil
+			},
+			Action: dindEngineRemove,
+			Flags: []cli.Flag{
+				cli.IntFlag{
+					Name:  "timeout,t",
+					Usage: "specify a shutdown timeout in seconds",
+					Value: 10,
+				},
+				cli.BoolFlag{
+					Name:  "force, f",
+					Usage: "do not prompt for input",
+				},
+			},
+		},
 		{
 			Name:   "enable",
-			Usage:  "set Docker engine to be switched on next reboot",
+			Usage:  "set user Docker engine to be switched on next reboot",
 			Action: engineEnable,
 		},
 		{
-			Name:   "list",
-			Usage:  "list available Docker engines",
+			Name:  "list",
+			Usage: "list available Docker engines (include the Dind engines)",
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "update, u",
+					Usage: "update engine cache",
+				},
+			},
 			Action: engineList,
 		},
 	}
@@ -82,6 +155,108 @@ func engineSwitch(c *cli.Context) error {
 	return nil
 }
 
+func engineCreate(c *cli.Context) error {
+	name := c.Args()[0]
+	version := c.String("version")
+	sshPort, _ := strconv.Atoi(c.String("ssh-port"))
+	if sshPort <= 0 {
+		sshPort = randomSSHPort()
+	}
+	authorizedKeys := c.String("authorized-keys")
+	network := c.String("network")
+	fixedIP := c.String("fixed-ip")
+
+	// generate & create engine compose
+	err := generateEngineCompose(name, version, sshPort, authorizedKeys, network, fixedIP)
+	if err != nil {
+		return err
+	}
+
+	// stage engine service
+	cfg := config.LoadConfig()
+	var enabledServices []string
+	if val, ok := cfg.Rancher.ServicesInclude[name]; !ok || !val {
+		cfg.Rancher.ServicesInclude[name] = true
+		enabledServices = append(enabledServices, name)
+	}
+
+	if len(enabledServices) > 0 {
+		if err := compose.StageServices(cfg, enabledServices...); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := config.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	// generate engine script
+	err = util.GenerateDindEngineScript(name)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return nil
+}
+
+func dindEngineRemove(c *cli.Context) error {
+	if !c.Bool("force") {
+		if !yes("Continue") {
+			return nil
+		}
+	}
+
+	// app.ProjectDelete needs to use this flag
+	// Allow deletion of the Dind engine
+	c.Set("force", "true")
+	// Remove volumes associated with the Dind engine container
+	c.Set("v", "true")
+
+	name := c.Args()[0]
+	cfg := config.LoadConfig()
+	p, err := compose.GetProject(cfg, true, false)
+	if err != nil {
+		log.Fatalf("Get project failed: %v", err)
+	}
+
+	// 1. service stop
+	err = app.ProjectStop(p, c)
+	if err != nil {
+		log.Fatalf("Stop project service failed: %v", err)
+	}
+
+	// 2. service delete
+	err = app.ProjectDelete(p, c)
+	if err != nil {
+		log.Fatalf("Delete project service failed: %v", err)
+	}
+
+	// 3. service delete
+	if _, ok := cfg.Rancher.ServicesInclude[name]; !ok {
+		log.Fatalf("Failed to found enabled service %s", name)
+	}
+
+	delete(cfg.Rancher.ServicesInclude, name)
+
+	if err = config.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
+		log.Fatal(err)
+	}
+
+	// 4. remove service from file
+	err = RemoveEngineFromCompose(name)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// 5. remove dind engine script
+	err = util.RemoveDindEngineScript(name)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func engineEnable(c *cli.Context) error {
 	if len(c.Args()) != 1 {
 		log.Fatal("Must specify exactly one Docker engine to enable")
@@ -104,7 +279,7 @@ func engineEnable(c *cli.Context) error {
 
 func engineList(c *cli.Context) error {
 	cfg := config.LoadConfig()
-	engines := availableEngines(cfg)
+	engines := availableEngines(cfg, c.Bool("update"))
 	currentEngine := CurrentEngine()
 
 	for _, engine := range engines {
@@ -117,17 +292,50 @@ func engineList(c *cli.Context) error {
 		}
 	}
 
+	// check the dind container
+	client, err := docker.NewSystemClient()
+	if err != nil {
+		log.Warnf("Failed to detect dind: %v", err)
+		return nil
+	}
+
+	filter := filters.NewArgs()
+	filter.Add("label", config.UserDockerLabel)
+	opts := types.ContainerListOptions{
+		All:    true,
+		Filter: filter,
+	}
+	containers, err := client.ContainerList(context.Background(), opts)
+	if err != nil {
+		log.Warnf("Failed to detect dind: %v", err)
+		return nil
+	}
+	for _, c := range containers {
+		if c.State == "running" {
+			fmt.Printf("enabled  %s\n", c.Labels[config.UserDockerLabel])
+		} else {
+			fmt.Printf("disabled %s\n", c.Labels[config.UserDockerLabel])
+		}
+	}
+
 	return nil
 }
 
 func validateEngine(engine string, cfg *config.CloudConfig) {
-	engines := availableEngines(cfg)
+	engines := availableEngines(cfg, false)
 	if !service.IsLocalOrURL(engine) && !util.Contains(engines, engine) {
 		log.Fatalf("%s is not a valid engine", engine)
 	}
 }
 
-func availableEngines(cfg *config.CloudConfig) []string {
+func availableEngines(cfg *config.CloudConfig, update bool) []string {
+	if update {
+		err := network.UpdateCaches(cfg.Rancher.Repositories.ToArray(), "engines")
+		if err != nil {
+			log.Debugf("Failed to update engine caches: %v", err)
+		}
+
+	}
 	engines, err := network.GetEngines(cfg.Rancher.Repositories.ToArray())
 	if err != nil {
 		log.Fatal(err)
@@ -157,8 +365,14 @@ func CurrentEngine() (engine string) {
 	}
 	if t, ok := image.(reference.NamedTagged); ok {
 		tag := t.Tag()
-		if !strings.HasPrefix(tag, "1.") {
-			// TODO: this assumes we only do Docker ce :/
+
+		// compatible with some patch image tags, such as 17.12.1-1,17.06.2-1,...
+		tag = strings.SplitN(tag, "-", 2)[0]
+
+		if !strings.HasPrefix(tag, "1.") && versions.LessThan(tag, "18.09.0") {
+			// >= 18.09.0, docker-<version>
+			// < 18.09.0 and >= 16.03, docker-<version>-ce
+			// < 17.03, docker-<version>
 			tag = tag + "-ce"
 		}
 		return "docker-" + tag
@@ -166,3 +380,210 @@ func CurrentEngine() (engine string) {
 
 	return
 }
+
+func preFlightValidate(c *cli.Context) error {
+	if len(c.Args()) != 1 {
+		return errors.New("Must specify one engine name")
+	}
+	name := c.Args()[0]
+	if name == "" {
+		return errors.New("Must specify one engine name")
+	}
+
+	version := c.String("version")
+	if version == "" {
+		return errors.New("Must specify one engine version")
+	}
+
+	authorizedKeys := c.String("authorized-keys")
+	if authorizedKeys != "" {
+		if _, err := os.Stat(authorizedKeys); os.IsNotExist(err) {
+			return errors.New("The authorized-keys should be an exist file, recommended to put in the /opt or /var/lib/rancher directory")
+		}
+	}
+
+	network := c.String("network")
+	if network == "" {
+		return errors.New("Must specify network")
+	}
+
+	userDefineNetwork, err := CheckUserDefineNetwork(network)
+	if err != nil {
+		return err
+	}
+
+	fixedIP := c.String("fixed-ip")
+	if fixedIP == "" {
+		return errors.New("Must specify fix ip")
+	}
+
+	err = CheckUserDefineIPv4Address(fixedIP, *userDefineNetwork)
+	if err != nil {
+		return err
+	}
+
+	isVersionMatch := false
+	for _, v := range config.SupportedDinds {
+		if v == version {
+			isVersionMatch = true
+			break
+		}
+	}
+
+	if !isVersionMatch {
+		return errors.Errorf("Engine version not supported only %v are supported", config.SupportedDinds)
+	}
+
+	if c.String("ssh-port") != "" {
+		port, err := strconv.Atoi(c.String("ssh-port"))
+		if err != nil {
+			return errors.Wrap(err, "Failed to convert ssh port to Int")
+		}
+		if port > 0 {
+			addr, err := net.ResolveTCPAddr("tcp", "localhost:"+strconv.Itoa(port))
+			if err != nil {
+				return errors.Errorf("Failed to resolve tcp addr: %v", err)
+			}
+			l, err := net.ListenTCP("tcp", addr)
+			if err != nil {
+				return errors.Errorf("Failed to listen tcp: %v", err)
+			}
+			defer l.Close()
+		}
+	}
+
+	return nil
+}
+
+func randomSSHPort() int {
+	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+	if err != nil {
+		log.Errorf("Failed to resolve tcp addr: %v", err)
+		return 0
+	}
+
+	l, err := net.ListenTCP("tcp", addr)
+	if err != nil {
+		return 0
+	}
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port
+}
+
+func generateEngineCompose(name, version string, sshPort int, authorizedKeys, network, fixedIP string) error {
+	if err := os.MkdirAll(path.Dir(config.MultiDockerConfFile), 0700); err != nil && !os.IsExist(err) {
+		log.Errorf("Failed to create directory for file %s: %v", config.MultiDockerConfFile, err)
+		return err
+	}
+
+	composeConfigs := map[string]composeConfig.ServiceConfigV1{}
+
+	if _, err := os.Stat(config.MultiDockerConfFile); err == nil {
+		// read from engine compose
+		bytes, err := ioutil.ReadFile(config.MultiDockerConfFile)
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(bytes, &composeConfigs)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := os.MkdirAll(config.MultiDockerDataDir+"/"+name, 0700); err != nil && !os.IsExist(err) {
+		log.Errorf("Failed to create directory for file %s: %v", config.MultiDockerDataDir+"/"+name, err)
+		return err
+	}
+
+	volumes := []string{
+		"/lib/modules:/lib/modules",
+		config.MultiDockerDataDir + "/" + name + ":" + config.MultiDockerDataDir + "/" + name,
+	}
+	if authorizedKeys != "" {
+		volumes = append(volumes, authorizedKeys+":/root/.ssh/authorized_keys")
+	}
+
+	composeConfigs[name] = composeConfig.ServiceConfigV1{
+		Image:       "${REGISTRY_DOMAIN}/" + version,
+		Restart:     "always",
+		Privileged:  true,
+		Net:         network,
+		Ports:       []string{strconv.Itoa(sshPort) + ":22"},
+		Volumes:     volumes,
+		VolumesFrom: []string{},
+		Command: composeYaml.Command{
+			"--storage-driver=overlay2",
+			"--data-root=" + config.MultiDockerDataDir + "/" + name,
+			"--host=unix://" + config.MultiDockerDataDir + "/" + name + "/docker-" + name + ".sock",
+		},
+		Labels: composeYaml.SliceorMap{
+			"io.rancher.os.scope":     "system",
+			"io.rancher.os.after":     "console",
+			config.UserDockerLabel:    name,
+			config.UserDockerNetLabel: network,
+			config.UserDockerFIPLabel: fixedIP,
+		},
+	}
+
+	bytes, err := yaml.Marshal(composeConfigs)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(config.MultiDockerConfFile, bytes, 0640)
+}
+
+func RemoveEngineFromCompose(name string) error {
+	composeConfigs := map[string]composeConfig.ServiceConfigV1{}
+
+	if _, err := os.Stat(config.MultiDockerConfFile); err == nil {
+		// read from engine compose
+		bytes, err := ioutil.ReadFile(config.MultiDockerConfFile)
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(bytes, &composeConfigs)
+		if err != nil {
+			return err
+		}
+	}
+
+	delete(composeConfigs, name)
+
+	bytes, err := yaml.Marshal(composeConfigs)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(config.MultiDockerConfFile, bytes, 0640)
+}
+
+func CheckUserDefineNetwork(name string) (*types.NetworkResource, error) {
+	systemClient, err := docker.NewSystemClient()
+	if err != nil {
+		return nil, err
+	}
+
+	networks, err := systemClient.NetworkList(context.Background(), types.NetworkListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, network := range networks {
+		if network.Name == name {
+			return &network, nil
+		}
+	}
+
+	return nil, errors.Errorf("Failed to found the user define network: %s", name)
+}
+
+func CheckUserDefineIPv4Address(ipv4 string, network types.NetworkResource) error {
+	for _, config := range network.IPAM.Config {
+		_, ipnet, _ := net.ParseCIDR(config.Subnet)
+		if ipnet.Contains(net.ParseIP(ipv4)) {
+			return nil
+		}
+	}
+	return errors.Errorf("IP %s is not in the specified cidr", ipv4)
+}

cmd/control/entrypoint.go

@@ -5,14 +5,14 @@ import (
 	"os/exec"
 	"syscall"
 
-	"github.com/codegangsta/cli"
-	"github.com/rancher/os/log"
-	"golang.org/x/net/context"
-
 	"github.com/rancher/os/cmd/cloudinitexecute"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/docker"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+
+	"github.com/codegangsta/cli"
+	"golang.org/x/net/context"
 )
 
 const (
@@ -85,7 +85,6 @@ func setupCommandSymlinks() {
 		{config.RosBin, "/usr/bin/cloud-init-save"},
 		{config.RosBin, "/usr/bin/dockerlaunch"},
 		{config.RosBin, "/usr/bin/respawn"},
-		{config.RosBin, "/usr/bin/system-docker"},
 		{config.RosBin, "/usr/sbin/netconf"},
 		{config.RosBin, "/usr/sbin/wait-for-docker"},
 		{config.RosBin, "/usr/sbin/poweroff"},

cmd/control/env.go

@@ -6,9 +6,10 @@ import (
 	"os/exec"
 	"syscall"
 
-	"github.com/codegangsta/cli"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/util"
+
+	"github.com/codegangsta/cli"
 )
 
 func envAction(c *cli.Context) error {

cmd/control/install.go

@@ -11,18 +11,17 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 
-	"github.com/rancher/os/log"
-
-	"github.com/codegangsta/cli"
-	"github.com/rancher/catalog-service/utils/version"
 	"github.com/rancher/os/cmd/control/install"
 	"github.com/rancher/os/cmd/power"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/dfs" // TODO: move CopyFile into util or something.
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/dfs" // TODO: move CopyFile into util or something.
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+
+	"github.com/codegangsta/cli"
+	"github.com/pkg/errors"
 )
 
 var installCommand = cli.Command{
@@ -86,6 +85,10 @@ var installCommand = cli.Command{
 			Name:  "kexec, k",
 			Usage: "reboot using kexec",
 		},
+		cli.BoolFlag{
+			Name:  "save, s",
+			Usage: "save services and images for next booting",
+		},
 		cli.BoolFlag{
 			Name:  "debug",
 			Usage: "Run installer with debug output",
@@ -94,6 +97,15 @@ var installCommand = cli.Command{
 }
 
 func installAction(c *cli.Context) error {
+	log.InitLogger()
+	debug := c.Bool("debug")
+	if debug {
+		log.Info("Log level is debug")
+		originalLevel := log.GetLevel()
+		defer log.SetLevel(originalLevel)
+		log.SetLevel(log.DebugLevel)
+	}
+
 	if runtime.GOARCH != "amd64" {
 		log.Fatalf("ros install / upgrade only supported on 'amd64', not '%s'", runtime.GOARCH)
 	}
@@ -102,13 +114,6 @@ func installAction(c *cli.Context) error {
 		log.Fatalf("invalid arguments %v", c.Args())
 	}
 
-	debug := c.Bool("debug")
-	if debug {
-		originalLevel := log.GetLevel()
-		defer log.SetLevel(originalLevel)
-		log.SetLevel(log.DebugLevel)
-	}
-
 	kappend := strings.TrimSpace(c.String("append"))
 	force := c.Bool("force")
 	kexec := c.Bool("kexec")
@@ -118,7 +123,11 @@ func installAction(c *cli.Context) error {
 	image := c.String("image")
 	cfg := config.LoadConfig()
 	if image == "" {
-		image = cfg.Rancher.Upgrade.Image + ":" + config.Version + config.Suffix
+		image = fmt.Sprintf("%s:%s%s",
+			cfg.Rancher.Upgrade.Image,
+			config.Version,
+			config.Suffix)
+		image = formatImage(image, cfg)
 	}
 
 	installType := c.String("install-type")
@@ -158,13 +167,25 @@ func installAction(c *cli.Context) error {
 	} else {
 		os.MkdirAll("/opt", 0755)
 		uc := "/opt/user_config.yml"
-		if err := util.FileCopy(cloudConfig, uc); err != nil {
-			log.WithFields(log.Fields{"cloudConfig": cloudConfig, "error": err}).Fatal("Failed to copy cloud-config")
+		if strings.HasPrefix(cloudConfig, "http://") || strings.HasPrefix(cloudConfig, "https://") {
+			if err := util.HTTPDownloadToFile(cloudConfig, uc); err != nil {
+				log.WithFields(log.Fields{"cloudConfig": cloudConfig, "error": err}).Fatal("Failed to http get cloud-config")
+			}
+		} else {
+			if err := util.FileCopy(cloudConfig, uc); err != nil {
+				log.WithFields(log.Fields{"cloudConfig": cloudConfig, "error": err}).Fatal("Failed to copy cloud-config")
+			}
 		}
 		cloudConfig = uc
 	}
 
-	if err := runInstall(image, installType, cloudConfig, device, partition, statedir, kappend, force, kexec, isoinstallerloaded, debug); err != nil {
+	savedImages := []string{}
+	if c.Bool("save") && cloudConfig != "" && installType != "upgrade" {
+		savedImages = install.GetCacheImageList(cloudConfig, cfg)
+		log.Debugf("Will cache these images: %s", savedImages)
+	}
+
+	if err := runInstall(image, installType, cloudConfig, device, partition, statedir, kappend, force, kexec, isoinstallerloaded, debug, savedImages); err != nil {
 		log.WithFields(log.Fields{"err": err}).Fatal("Failed to run install")
 		return err
 	}
@@ -177,7 +198,7 @@ func installAction(c *cli.Context) error {
 	return nil
 }
 
-func runInstall(image, installType, cloudConfig, device, partition, statedir, kappend string, force, kexec, isoinstallerloaded, debug bool) error {
+func runInstall(image, installType, cloudConfig, device, partition, statedir, kappend string, force, kexec, isoinstallerloaded, debug bool, savedImages []string) error {
 	fmt.Printf("Installing from %s\n", image)
 
 	if !force {
@@ -186,46 +207,6 @@ func runInstall(image, installType, cloudConfig, device, partition, statedir, ka
 			os.Exit(1)
 		}
 	}
-	diskType := "msdos"
-
-	if installType == "gptsyslinux" {
-		diskType = "gpt"
-	}
-
-	// Versions before 0.8.0-rc3 use the old calling convention (from the lay-down-os shell script)
-	imageVersion := strings.TrimPrefix(image, "rancher/os:")
-	if version.GreaterThan("v0.8.0-rc3", imageVersion) {
-		log.Infof("user specified to install pre v0.8.0: %s", image)
-		imageVersion = strings.Replace(imageVersion, "-", ".", -1)
-		vArray := strings.Split(imageVersion, ".")
-		if len(vArray) >= 2 {
-			v, _ := strconv.ParseFloat(vArray[0]+"."+vArray[1], 32)
-			if v < 0.8 || imageVersion == "0.8.0-rc1" {
-				log.Infof("starting installer container for %s", image)
-				if installType == "generic" ||
-					installType == "syslinux" ||
-					installType == "gptsyslinux" {
-					cmd := exec.Command("system-docker", "run", "--net=host", "--privileged", "--volumes-from=all-volumes",
-						"--entrypoint=/scripts/set-disk-partitions", image, device, diskType)
-					cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
-					if err := cmd.Run(); err != nil {
-						return err
-					}
-				}
-				cmd := exec.Command("system-docker", "run", "--net=host", "--privileged", "--volumes-from=user-volumes",
-					"--volumes-from=command-volumes", image, "-d", device, "-t", installType, "-c", cloudConfig,
-					"-a", kappend)
-				cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
-				return cmd.Run()
-			}
-		}
-	}
-
-	if _, err := os.Stat("/usr/bin/system-docker"); os.IsNotExist(err) {
-		if err := os.Symlink("/usr/bin/ros", "/usr/bin/system-docker"); err != nil {
-			log.Errorf("ln error %s", err)
-		}
-	}
 
 	useIso := false
 	// --isoinstallerloaded is used if the ros has created the installer container from and image that was on the booted iso
@@ -233,15 +214,21 @@ func runInstall(image, installType, cloudConfig, device, partition, statedir, ka
 		log.Infof("start !isoinstallerloaded")
 
 		if _, err := os.Stat("/dist/initrd-" + config.Version); os.IsNotExist(err) {
-			if err = mountBootIso(); err != nil {
-				log.Debugf("mountBootIso error %s", err)
+			deviceName, deviceType, err := getBootIso()
+			if err != nil {
+				log.Errorf("Failed to get boot iso: %v", err)
+				fmt.Println("There is no boot iso drive, terminate the task")
+				return err
+			}
+			if err = mountBootIso(deviceName, deviceType); err != nil {
+				log.Debugf("Failed to mountBootIso: %v", err)
 			} else {
 				log.Infof("trying to load /bootiso/rancheros/installer.tar.gz")
 				if _, err := os.Stat("/bootiso/rancheros/"); err == nil {
 					cmd := exec.Command("system-docker", "load", "-i", "/bootiso/rancheros/installer.tar.gz")
 					cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 					if err := cmd.Run(); err != nil {
-						log.Infof("failed to load images from /bootiso/rancheros: %s", err)
+						log.Infof("failed to load images from /bootiso/rancheros: %v", err)
 					} else {
 						log.Infof("Loaded images from /bootiso/rancheros/installer.tar.gz")
 
@@ -296,23 +283,22 @@ func runInstall(image, installType, cloudConfig, device, partition, statedir, ka
 			if statedir != "" {
 				installerCmd = append(installerCmd, "--statedir", statedir)
 			}
+			if len(savedImages) > 0 {
+				installerCmd = append(installerCmd, "--save")
+			}
 
 			// TODO: mount at /mnt for shared mount?
 			if useIso {
 				util.Unmount("/bootiso")
 			}
-
 			cmd := exec.Command("system-docker", installerCmd...)
 			log.Debugf("Run(%v)", cmd)
 			cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+
 			return cmd.Run()
 		}
 	}
 
-	// TODO: needs to pass the log level on to the container
-	log.InitLogger()
-	log.SetLevel(log.InfoLevel)
-
 	log.Debugf("running installation")
 
 	if partition == "" {
@@ -333,7 +319,7 @@ func runInstall(image, installType, cloudConfig, device, partition, statedir, ka
 			device = "/host" + device
 			//# TODO: Change this to a number so that users can specify.
 			//# Will need to make it so that our builds and packer APIs remain consistent.
-			partition = device + "1" //${partition:=${device}1}
+			partition = install.GetDefaultPartition(device)
 		}
 	}
 
@@ -344,7 +330,13 @@ func runInstall(image, installType, cloudConfig, device, partition, statedir, ka
 	if isoinstallerloaded {
 		log.Debugf("running isoinstallerloaded...")
 		// TODO: detect if its not mounted and then optionally mount?
-		if err := mountBootIso(); err != nil {
+		deviceName, deviceType, err := getBootIso()
+		if err != nil {
+			log.Errorf("Failed to get boot iso: %v", err)
+			fmt.Println("There is no boot iso drive, terminate the task")
+			return err
+		}
+		if err := mountBootIso(deviceName, deviceType); err != nil {
 			log.Errorf("error mountBootIso %s", err)
 			//return err
 		}
@@ -356,21 +348,49 @@ func runInstall(image, installType, cloudConfig, device, partition, statedir, ka
 		return err
 	}
 
+	if len(savedImages) > 0 {
+		return install.RunCacheScript(partition, savedImages)
+	}
+
 	return nil
 }
 
-func mountBootIso() error {
+func getDeviceByLabel(label string) (string, string) {
+	d, t, err := util.Blkid(label)
+	if err != nil {
+		log.Warnf("Failed to run blkid for %s", label)
+		return "", ""
+	}
+	return d, t
+}
+
+func getBootIso() (string, string, error) {
 	deviceName := "/dev/sr0"
 	deviceType := "iso9660"
-	if d, t := util.Blkid("RancherOS"); d != "" {
-		deviceName = d
-		deviceType = t
+
+	// Our ISO LABEL is RancherOS
+	// But some tools(like rufus) will change LABEL to RANCHEROS
+	for _, label := range []string{"RancherOS", "RANCHEROS"} {
+		d, t := getDeviceByLabel(label)
+		if d != "" {
+			deviceName = d
+			deviceType = t
+			continue
+		}
 	}
 
+	// Check the sr deive if exist
+	if _, err := os.Stat(deviceName); os.IsNotExist(err) {
+		return "", "", err
+	}
+
+	return deviceName, deviceType, nil
+}
+
+func mountBootIso(deviceName, deviceType string) error {
 	mountsFile, err := os.Open("/proc/mounts")
 	if err != nil {
-		log.Errorf("failed to read /proc/mounts %s", err)
-		return err
+		return errors.Wrap(err, "Failed to read /proc/mounts")
 	}
 	defer mountsFile.Close()
 
@@ -382,14 +402,15 @@ func mountBootIso() error {
 	cmd := exec.Command("mount", "-t", deviceType, deviceName, "/bootiso")
 	log.Debugf("mount (%#v)", cmd)
 
-	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	var outBuf, errBuf bytes.Buffer
+	cmd.Stdout = &outBuf
+	cmd.Stderr = &errBuf
 	err = cmd.Run()
 	if err != nil {
-		log.Errorf("tried and failed to mount %s: %s", deviceName, err)
-	} else {
-		log.Debugf("Mounted %s", deviceName)
+		return errors.Wrapf(err, "Tried and failed to mount %s: stderr output: %s", deviceName, errBuf.String())
 	}
-	return err
+	log.Debugf("Mounted %s, output: %s", deviceName, outBuf.String())
+	return nil
 }
 
 func layDownOS(image, installType, cloudConfig, device, partition, statedir, kappend string, kexec bool) error {
@@ -405,7 +426,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 	//cloudConfig := SCRIPTS_DIR + "/conf/empty.yml" //${cloudConfig:-"${SCRIPTS_DIR}/conf/empty.yml"}
 	CONSOLE := "tty0"
 	baseName := "/mnt/new_img"
-	kernelArgs := "printk.devkmsg=on rancher.state.dev=LABEL=RANCHER_STATE rancher.state.wait" // console="+CONSOLE
+	kernelArgs := "printk.devkmsg=on rancher.state.dev=LABEL=RANCHER_STATE rancher.state.wait panic=10" // console="+CONSOLE
 	if statedir != "" {
 		kernelArgs = kernelArgs + " rancher.state.directory=" + statedir
 	}
@@ -426,7 +447,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 	case "generic":
 		log.Debugf("formatAndMount")
 		var err error
-		device, partition, err = formatAndMount(baseName, device, partition)
+		device, _, err = formatAndMount(baseName, device, partition)
 		if err != nil {
 			log.Errorf("formatAndMount %s", err)
 			return err
@@ -443,7 +464,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 		}
 	case "arm":
 		var err error
-		device, partition, err = formatAndMount(baseName, device, partition)
+		_, _, err = formatAndMount(baseName, device, partition)
 		if err != nil {
 			return err
 		}
@@ -453,7 +474,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 	case "amazon-ebs-hvm":
 		CONSOLE = "ttyS0"
 		var err error
-		device, partition, err = formatAndMount(baseName, device, partition)
+		device, _, err = formatAndMount(baseName, device, partition)
 		if err != nil {
 			return err
 		}
@@ -465,7 +486,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 	case "googlecompute":
 		CONSOLE = "ttyS0"
 		var err error
-		device, partition, err = formatAndMount(baseName, device, partition)
+		device, _, err = formatAndMount(baseName, device, partition)
 		if err != nil {
 			return err
 		}
@@ -473,7 +494,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 		seedData(baseName, cloudConfig, FILES)
 	case "noformat":
 		var err error
-		device, partition, err = install.MountDevice(baseName, device, partition, false)
+		device, _, err = install.MountDevice(baseName, device, partition, false)
 		if err != nil {
 			return err
 		}
@@ -481,9 +502,14 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 		if err := os.MkdirAll(filepath.Join(baseName, statedir), 0755); err != nil {
 			return err
 		}
+		err = seedData(baseName, cloudConfig, FILES)
+		if err != nil {
+			log.Errorf("seedData %s", err)
+			return err
+		}
 	case "raid":
 		var err error
-		device, partition, err = install.MountDevice(baseName, device, partition, false)
+		device, _, err = install.MountDevice(baseName, device, partition, false)
 		if err != nil {
 			return err
 		}
@@ -491,7 +517,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 	case "bootstrap":
 		CONSOLE = "ttyS0"
 		var err error
-		device, partition, err = install.MountDevice(baseName, device, partition, true)
+		_, _, err = install.MountDevice(baseName, device, partition, true)
 		if err != nil {
 			return err
 		}
@@ -501,7 +527,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 		fallthrough
 	case "upgrade":
 		var err error
-		device, partition, err = install.MountDevice(baseName, device, partition, false)
+		device, _, err = install.MountDevice(baseName, device, partition, false)
 		if err != nil {
 			return err
 		}
@@ -514,20 +540,20 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 	kernelArgs = kernelArgs + " console=" + CONSOLE
 
 	if kappend == "" {
-		preservedAppend, _ := ioutil.ReadFile(filepath.Join(baseName, install.BootDir+"append"))
+		preservedAppend, _ := ioutil.ReadFile(filepath.Join(baseName, config.BootDir, "append"))
 		kappend = string(preservedAppend)
 	} else {
-		ioutil.WriteFile(filepath.Join(baseName, install.BootDir+"append"), []byte(kappend), 0644)
+		ioutil.WriteFile(filepath.Join(baseName, config.BootDir, "append"), []byte(kappend), 0644)
 	}
 
 	if installType == "amazon-ebs-pv" {
 		menu := install.BootVars{
 			BaseName: baseName,
-			BootDir:  install.BootDir,
+			BootDir:  config.BootDir,
 			Timeout:  0,
 			Fallback: 0, // need to be conditional on there being a 'rollback'?
 			Entries: []install.MenuEntry{
-				install.MenuEntry{"RancherOS-current", install.BootDir, VERSION, kernelArgs, kappend},
+				install.MenuEntry{"RancherOS-current", config.BootDir, VERSION, kernelArgs, kappend},
 			},
 		}
 		install.PvGrubConfig(menu)
@@ -541,7 +567,7 @@ func layDownOS(image, installType, cloudConfig, device, partition, statedir, kap
 	log.Debugf("installRancher done")
 
 	if kexec {
-		power.Kexec(false, filepath.Join(baseName, install.BootDir), kernelArgs+" "+kappend)
+		power.Kexec(false, filepath.Join(baseName, config.BootDir), kernelArgs+" "+kappend)
 	}
 
 	return nil
@@ -555,12 +581,42 @@ func seedData(baseName, cloudData string, files []string) error {
 		return err
 	}
 
-	if err = os.MkdirAll(filepath.Join(baseName, "/var/lib/rancher/conf/cloud-config.d"), 0700); err != nil {
+	stateSeedDir := "state_seed"
+	cloudConfigBase := "/var/lib/rancher/conf/cloud-config.d"
+	cloudConfigDir := ""
+
+	// If there is a separate boot partition, cloud-config should be written to RANCHER_STATE partition.
+	bootPartition, _, err := util.Blkid("RANCHER_BOOT")
+	if err != nil {
+		log.Errorf("Failed to run blkid: %s", err)
+	}
+	if bootPartition != "" {
+		stateSeedFullPath := filepath.Join(baseName, stateSeedDir)
+		if err = os.MkdirAll(stateSeedFullPath, 0700); err != nil {
+			return err
+		}
+
+		defer util.Unmount(stateSeedFullPath)
+
+		statePartition := install.GetStatePartition()
+		cmd := exec.Command("mount", statePartition, stateSeedFullPath)
+		//cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+		log.Debugf("seedData: mount %s to %s", statePartition, stateSeedFullPath)
+		if err = cmd.Run(); err != nil {
+			return err
+		}
+
+		cloudConfigDir = filepath.Join(baseName, stateSeedDir, cloudConfigBase)
+	} else {
+		cloudConfigDir = filepath.Join(baseName, cloudConfigBase)
+	}
+
+	if err = os.MkdirAll(cloudConfigDir, 0700); err != nil {
 		return err
 	}
 
 	if !strings.HasSuffix(cloudData, "empty.yml") {
-		if err = dfs.CopyFile(cloudData, baseName+"/var/lib/rancher/conf/cloud-config.d/", filepath.Base(cloudData)); err != nil {
+		if err = dfs.CopyFile(cloudData, cloudConfigDir, filepath.Base(cloudData)); err != nil {
 			return err
 		}
 	}
@@ -657,28 +713,28 @@ func setDiskpartitions(device, diskType string) error {
 		}
 	}
 	//do it!
-	log.Debugf("running dd")
+	log.Debugf("running dd device: %s", device)
 	cmd := exec.Command("dd", "if=/dev/zero", "of="+device, "bs=512", "count=2048")
 	//cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 	if err := cmd.Run(); err != nil {
 		log.Errorf("dd error %s", err)
 		return err
 	}
-	log.Debugf("running partprobe")
+	log.Debugf("running partprobe: %s", device)
 	cmd = exec.Command("partprobe", device)
 	//cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 	if err := cmd.Run(); err != nil {
-		log.Errorf("partprobe error %s", err)
+		log.Errorf("Failed to partprobe device %s: %v", device, err)
 		return err
 	}
 
-	log.Debugf("making single RANCHER_STATE partition")
+	log.Debugf("making single RANCHER_STATE partition, device: %s", device)
 	cmd = exec.Command("parted", "-s", "-a", "optimal", device,
 		"mklabel "+diskType, "--",
 		"mkpart primary ext4 1 -1")
 	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 	if err := cmd.Run(); err != nil {
-		log.Errorf("parted: %s", err)
+		log.Errorf("Failed to parted device %s: %v", device, err)
 		return err
 	}
 	return setBootable(device, diskType)
@@ -755,7 +811,7 @@ func setBootable(device, diskType string) error {
 func upgradeBootloader(device, baseName, diskType string) error {
 	log.Debugf("start upgradeBootloader")
 
-	grubDir := filepath.Join(baseName, install.BootDir+"grub")
+	grubDir := filepath.Join(baseName, config.BootDir, "grub")
 	if _, err := os.Stat(grubDir); os.IsNotExist(err) {
 		log.Debugf("%s does not exist - no need to upgrade bootloader", grubDir)
 		// we've already upgraded
@@ -763,12 +819,12 @@ func upgradeBootloader(device, baseName, diskType string) error {
 		return nil
 	}
 	// deal with systems which were previously upgraded, then rolled back, and are now being re-upgraded
-	grubBackup := filepath.Join(baseName, install.BootDir+"grub_backup")
+	grubBackup := filepath.Join(baseName, config.BootDir, "grub_backup")
 	if err := os.RemoveAll(grubBackup); err != nil {
 		log.Errorf("RemoveAll (%s): %s", grubBackup, err)
 		return err
 	}
-	backupSyslinuxDir := filepath.Join(baseName, install.BootDir+"syslinux_backup")
+	backupSyslinuxDir := filepath.Join(baseName, config.BootDir, "syslinux_backup")
 	if _, err := os.Stat(backupSyslinuxDir); !os.IsNotExist(err) {
 		backupSyslinuxLdlinuxSys := filepath.Join(backupSyslinuxDir, "ldlinux.sys")
 		if _, err := os.Stat(backupSyslinuxLdlinuxSys); !os.IsNotExist(err) {
@@ -791,7 +847,7 @@ func upgradeBootloader(device, baseName, diskType string) error {
 		return err
 	}
 
-	syslinuxDir := filepath.Join(baseName, install.BootDir+"syslinux")
+	syslinuxDir := filepath.Join(baseName, config.BootDir, "syslinux")
 	// it seems that v0.5.0 didn't have a syslinux dir, while 0.7 does
 	if _, err := os.Stat(syslinuxDir); !os.IsNotExist(err) {
 		if err := os.Rename(syslinuxDir, backupSyslinuxDir); err != nil {
@@ -812,15 +868,15 @@ func upgradeBootloader(device, baseName, diskType string) error {
 
 				cfg = strings.Replace(cfg, "current", "previous", -1)
 				// TODO consider removing the APPEND line - as the global.cfg should have the same result
-				ioutil.WriteFile(filepath.Join(baseName, install.BootDir, "linux-current.cfg"), []byte(cfg), 0644)
+				ioutil.WriteFile(filepath.Join(baseName, config.BootDir, "linux-current.cfg"), []byte(cfg), 0644)
 
 				lines := strings.Split(cfg, "\n")
 				for _, line := range lines {
 					line = strings.TrimSpace(line)
 					if strings.HasPrefix(line, "APPEND") {
-						log.Errorf("write new (%s) %s", filepath.Join(baseName, install.BootDir, "global.cfg"), err)
+						log.Errorf("write new (%s) %s", filepath.Join(baseName, config.BootDir, "global.cfg"), err)
 						// TODO: need to append any extra's the user specified
-						ioutil.WriteFile(filepath.Join(baseName, install.BootDir, "global.cfg"), []byte(cfg), 0644)
+						ioutil.WriteFile(filepath.Join(baseName, config.BootDir, "global.cfg"), []byte(cfg), 0644)
 						break
 					}
 				}
@@ -881,7 +937,7 @@ func installSyslinux(device, baseName, diskType string) error {
 		}
 	}
 
-	sysLinuxDir := filepath.Join(baseName, install.BootDir, "syslinux")
+	sysLinuxDir := filepath.Join(baseName, config.BootDir, "syslinux")
 	if err := os.MkdirAll(sysLinuxDir, 0755); err != nil {
 		log.Errorf("MkdirAll(%s)): %s", sysLinuxDir, err)
 		//return err
@@ -939,12 +995,12 @@ func installRancher(baseName, VERSION, DIST, kappend string) (string, error) {
 	log.Debugf("installRancher")
 
 	// detect if there already is a linux-current.cfg, if so, move it to linux-previous.cfg,
-	currentCfg := filepath.Join(baseName, install.BootDir, "linux-current.cfg")
+	currentCfg := filepath.Join(baseName, config.BootDir, "linux-current.cfg")
 	if _, err := os.Stat(currentCfg); !os.IsNotExist(err) {
 		existingCfg := filepath.Join(DIST, "linux-current.cfg")
 		// only remove previous if there is a change to the current
 		if different(currentCfg, existingCfg) {
-			previousCfg := filepath.Join(baseName, install.BootDir, "linux-previous.cfg")
+			previousCfg := filepath.Join(baseName, config.BootDir, "linux-previous.cfg")
 			if _, err := os.Stat(previousCfg); !os.IsNotExist(err) {
 				if err := os.Remove(previousCfg); err != nil {
 					return currentCfg, err
@@ -966,7 +1022,7 @@ func installRancher(baseName, VERSION, DIST, kappend string) (string, error) {
 		if file.Name() == "global.cfg" {
 			overwrite = false
 		}
-		if err := dfs.CopyFileOverwrite(filepath.Join(DIST, file.Name()), filepath.Join(baseName, install.BootDir), file.Name(), overwrite); err != nil {
+		if err := dfs.CopyFileOverwrite(filepath.Join(DIST, file.Name()), filepath.Join(baseName, config.BootDir), file.Name(), overwrite); err != nil {
 			log.Errorf("copy %s: %s", file.Name(), err)
 			//return err
 		}
@@ -974,7 +1030,7 @@ func installRancher(baseName, VERSION, DIST, kappend string) (string, error) {
 
 	// the general INCLUDE syslinuxcfg
 	isolinuxFile := filepath.Join(DIST, "isolinux", "isolinux.cfg")
-	syslinuxDir := filepath.Join(baseName, install.BootDir, "syslinux")
+	syslinuxDir := filepath.Join(baseName, config.BootDir, "syslinux")
 	if err := dfs.CopyFileOverwrite(isolinuxFile, syslinuxDir, "syslinux.cfg", true); err != nil {
 		log.Errorf("copy global syslinux.cfgS%s: %s", "syslinux.cfg", err)
 		//return err
@@ -984,7 +1040,7 @@ func installRancher(baseName, VERSION, DIST, kappend string) (string, error) {
 	}
 
 	// The global.cfg INCLUDE - useful for over-riding the APPEND line
-	globalFile := filepath.Join(filepath.Join(baseName, install.BootDir), "global.cfg")
+	globalFile := filepath.Join(baseName, config.BootDir, "global.cfg")
 	if _, err := os.Stat(globalFile); !os.IsNotExist(err) {
 		err := ioutil.WriteFile(globalFile, []byte("APPEND "+kappend), 0644)
 		if err != nil {

cmd/control/install/grub.go

@@ -6,7 +6,7 @@ import (
 	"os/exec"
 	"path/filepath"
 
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/pkg/log"
 )
 
 func RunGrub(baseName, device string) error {

cmd/control/install/install.go

@@ -7,12 +7,10 @@ import (
 	"strings"
 
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 )
 
-const BootDir = "boot/"
-
 type MenuEntry struct {
 	Name, BootDir, Version, KernelArgs, Append string
 }
@@ -46,19 +44,15 @@ func MountDevice(baseName, device, partition string, raw bool) (string, string,
 		//rootfs := partition
 		// Don't use ResolveDevice - it can fail, whereas `blkid -L LABEL` works more often
 
-		cfg := config.LoadConfig()
-		if d, _ := util.Blkid("RANCHER_BOOT"); d != "" {
+		d, _, err := util.Blkid("RANCHER_BOOT")
+		if err != nil {
+			log.Errorf("Failed to run blkid: %s", err)
+		}
+		if d != "" {
 			partition = d
-			baseName = filepath.Join(baseName, BootDir)
+			baseName = filepath.Join(baseName, config.BootDir)
 		} else {
-			if dev := util.ResolveDevice(cfg.Rancher.State.Dev); dev != "" {
-				// try the rancher.state.dev setting
-				partition = dev
-			} else {
-				if d, _ := util.Blkid("RANCHER_STATE"); d != "" {
-					partition = d
-				}
-			}
+			partition = GetStatePartition()
 		}
 		cmd := exec.Command("lsblk", "-no", "pkname", partition)
 		log.Debugf("Run(%v)", cmd)
@@ -74,3 +68,24 @@ func MountDevice(baseName, device, partition string, raw bool) (string, string,
 	log.Debugf("mountdevice return2 -> d: %s, p: %s", device, partition)
 	return device, partition, cmd.Run()
 }
+
+func GetStatePartition() string {
+	cfg := config.LoadConfig()
+
+	if dev := util.ResolveDevice(cfg.Rancher.State.Dev); dev != "" {
+		// try the rancher.state.dev setting
+		return dev
+	}
+	d, _, err := util.Blkid("RANCHER_STATE")
+	if err != nil {
+		log.Errorf("Failed to run blkid: %s", err)
+	}
+	return d
+}
+
+func GetDefaultPartition(device string) string {
+	if strings.Contains(device, "nvme") {
+		return device + "p1"
+	}
+	return device + "1"
+}

cmd/control/install/service.go

@@ -0,0 +1,129 @@
+package install
+
+import (
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+	"github.com/rancher/os/pkg/util/network"
+
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
+)
+
+type ImageConfig struct {
+	Image string `yaml:"image,omitempty"`
+}
+
+func GetCacheImageList(cloudconfig string, oldcfg *config.CloudConfig) []string {
+	savedImages := make([]string, 0)
+	bytes, err := readConfigFile(cloudconfig)
+	if err != nil {
+		log.WithFields(log.Fields{"err": err}).Fatal("Failed to read cloud-config")
+		return savedImages
+	}
+	r := make(map[interface{}]interface{})
+	if err := yaml.Unmarshal(bytes, &r); err != nil {
+		log.WithFields(log.Fields{"err": err}).Fatal("Failed to unmarshal cloud-config")
+		return savedImages
+	}
+	newcfg := &config.CloudConfig{}
+	if err := util.Convert(r, newcfg); err != nil {
+		log.WithFields(log.Fields{"err": err}).Fatal("Failed to convert cloud-config")
+		return savedImages
+	}
+
+	// services_include
+	for key, value := range newcfg.Rancher.ServicesInclude {
+		if value {
+			serviceImage := getServiceImage(key, "", oldcfg, newcfg)
+			if serviceImage != "" {
+				savedImages = append(savedImages, serviceImage)
+			}
+		}
+	}
+
+	// console
+	newConsole := newcfg.Rancher.Console
+	if newConsole != "" && newConsole != "default" {
+		consoleImage := getServiceImage(newConsole, "console", oldcfg, newcfg)
+		if consoleImage != "" {
+			savedImages = append(savedImages, consoleImage)
+		}
+	}
+
+	// docker engine
+	newEngine := newcfg.Rancher.Docker.Engine
+	if newEngine != "" && newEngine != oldcfg.Rancher.Docker.Engine {
+		engineImage := getServiceImage(newEngine, "docker", oldcfg, newcfg)
+		if engineImage != "" {
+			savedImages = append(savedImages, engineImage)
+		}
+
+	}
+
+	return savedImages
+}
+
+func getServiceImage(service, svctype string, oldcfg, newcfg *config.CloudConfig) string {
+	var (
+		serviceImage string
+		bytes        []byte
+		err          error
+	)
+	if len(newcfg.Rancher.Repositories.ToArray()) > 0 {
+		bytes, err = network.LoadServiceResource(service, true, newcfg)
+	} else {
+		bytes, err = network.LoadServiceResource(service, true, oldcfg)
+	}
+	if err != nil {
+		log.WithFields(log.Fields{"err": err}).Fatal("Failed to load service resource")
+		return serviceImage
+	}
+	imageConfig := map[interface{}]ImageConfig{}
+	if err = yaml.Unmarshal(bytes, &imageConfig); err != nil {
+		log.WithFields(log.Fields{"err": err}).Fatal("Failed to unmarshal service")
+		return serviceImage
+	}
+	switch svctype {
+	case "console":
+		serviceImage = formatImage(imageConfig["console"].Image, oldcfg, newcfg)
+	case "docker":
+		serviceImage = formatImage(imageConfig["docker"].Image, oldcfg, newcfg)
+	default:
+		serviceImage = formatImage(imageConfig[service].Image, oldcfg, newcfg)
+	}
+
+	return serviceImage
+}
+
+func RunCacheScript(partition string, images []string) error {
+	return util.RunScript("/scripts/cache-services.sh", partition, strings.Join(images, " "))
+}
+
+func readConfigFile(file string) ([]byte, error) {
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = nil
+			content = []byte{}
+		} else {
+			return nil, err
+		}
+	}
+	return content, err
+}
+
+func formatImage(image string, oldcfg, newcfg *config.CloudConfig) string {
+	registryDomain := newcfg.Rancher.Environment["REGISTRY_DOMAIN"]
+	if registryDomain == "" {
+		registryDomain = oldcfg.Rancher.Environment["REGISTRY_DOMAIN"]
+	}
+	image = strings.Replace(image, "${REGISTRY_DOMAIN}", registryDomain, -1)
+
+	image = strings.Replace(image, "${SUFFIX}", config.Suffix, -1)
+
+	return image
+}

cmd/control/install/syslinux.go

@@ -9,7 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/pkg/log"
 )
 
 func syslinuxConfig(menu BootVars) error {

cmd/control/os.go

@@ -3,25 +3,25 @@ package control
 import (
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"net/url"
 	"os"
 	"runtime"
 	"strings"
 
-	"golang.org/x/net/context"
+	"github.com/rancher/os/cmd/power"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/compose"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+	"github.com/rancher/os/pkg/util/network"
 
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
-	"github.com/rancher/os/log"
-
 	"github.com/codegangsta/cli"
 	dockerClient "github.com/docker/engine-api/client"
 	composeConfig "github.com/docker/libcompose/config"
 	"github.com/docker/libcompose/project/options"
-	"github.com/rancher/os/cmd/power"
-	"github.com/rancher/os/compose"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/docker"
+	"golang.org/x/net/context"
 )
 
 type Images struct {
@@ -71,8 +71,14 @@ func osSubcommands() []cli.Command {
 			},
 		},
 		{
-			Name:   "list",
-			Usage:  "list the current available versions",
+			Name:  "list",
+			Usage: "list the current available versions",
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "update, u",
+					Usage: "update engine cache",
+				},
+			},
 			Action: osMetaDataGet,
 		},
 		{
@@ -83,8 +89,7 @@ func osSubcommands() []cli.Command {
 	}
 }
 
-// TODO: this and the getLatestImage should probably move to utils/network and be suitably cached.
-func getImages() (*Images, error) {
+func getImages(update bool) (*Images, error) {
 	upgradeURL, err := getUpgradeURL()
 	if err != nil {
 		return nil, err
@@ -105,25 +110,41 @@ func getImages() (*Images, error) {
 
 		q := u.Query()
 		q.Set("current", config.Version)
+		if hypervisor := util.GetHypervisor(); hypervisor == "" {
+			q.Set("hypervisor", hypervisor)
+		}
 		u.RawQuery = q.Encode()
 		upgradeURL = u.String()
 
-		resp, err := http.Get(upgradeURL)
-		if err != nil {
-			return nil, err
+		if update {
+			_, err := network.UpdateCache(upgradeURL)
+			if err != nil {
+				log.Errorf("Failed to update os caches: %v", err)
+			}
 		}
-		defer resp.Body.Close()
-		body, err = ioutil.ReadAll(resp.Body)
+
+		body, err = network.LoadFromNetwork(upgradeURL)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	return parseBody(body)
+	images, err := parseBody(body)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := config.LoadConfig()
+	images.Current = formatImage(images.Current, cfg)
+	for i := len(images.Available) - 1; i >= 0; i-- {
+		images.Available[i] = formatImage(images.Available[i], cfg)
+	}
+
+	return images, nil
 }
 
 func osMetaDataGet(c *cli.Context) error {
-	images, err := getImages()
+	images, err := getImages(c.Bool("update"))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -135,6 +156,7 @@ func osMetaDataGet(c *cli.Context) error {
 
 	cfg := config.LoadConfig()
 	runningName := cfg.Rancher.Upgrade.Image + ":" + config.Version
+	runningName = formatImage(runningName, cfg)
 
 	foundRunning := false
 	for i := len(images.Available) - 1; i >= 0; i-- {
@@ -163,7 +185,7 @@ func osMetaDataGet(c *cli.Context) error {
 }
 
 func getLatestImage() (string, error) {
-	images, err := getImages()
+	images, err := getImages(false)
 	if err != nil {
 		return "", err
 	}
@@ -176,6 +198,10 @@ func osUpgrade(c *cli.Context) error {
 		log.Fatalf("ros install / upgrade only supported on 'amd64', not '%s'", runtime.GOARCH)
 	}
 
+	if isExist := checkGlobalCfg(); !isExist {
+		log.Fatalf("ros upgrade cannot be supported")
+	}
+
 	image := c.String("image")
 
 	if image == "" {
@@ -212,7 +238,7 @@ func osVersion(c *cli.Context) error {
 	return nil
 }
 
-func startUpgradeContainer(image string, stage, force, reboot, kexec, debug bool, upgradeConsole bool, kernelArgs string) error {
+func startUpgradeContainer(image string, stage, force, reboot, kexec, upgradeConsole, debug bool, kernelArgs string) error {
 	command := []string{
 		"-t", "rancher-upgrade",
 		"-r", config.Version,

cmd/control/preload.go

@@ -11,11 +11,13 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/codegangsta/cli"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
 
+	"github.com/codegangsta/cli"
 	dockerClient "github.com/docker/engine-api/client"
-	"github.com/rancher/os/docker"
-	"github.com/rancher/os/log"
+	"github.com/docker/engine-api/types"
 )
 
 const (
@@ -23,7 +25,11 @@ const (
 )
 
 func preloadImagesAction(c *cli.Context) error {
-	return PreloadImages(docker.NewDefaultClient, userImagesPreloadDirectory)
+	err := PreloadImages(docker.NewDefaultClient, userImagesPreloadDirectory)
+	if err != nil {
+		log.Errorf("Failed to preload user images: %v", err)
+	}
+	return err
 }
 
 func shouldLoad(file string) bool {
@@ -48,6 +54,20 @@ func PreloadImages(clientFactory func() (dockerClient.APIClient, error), imagesD
 		return err
 	}
 
+	// try to load predefined user images
+	if imagesDir == userImagesPreloadDirectory {
+		oldUserImgName := path.Join(config.ImagesPath, config.UserImages)
+		userImgfile, err := os.Stat(oldUserImgName)
+		if err == nil {
+			newUserImgName := path.Join(userImagesPreloadDirectory, userImgfile.Name())
+			if _, err = os.Stat(newUserImgName); os.IsNotExist(err) {
+				if err := os.Symlink(oldUserImgName, newUserImgName); err != nil {
+					log.Error(err)
+				}
+			}
+		}
+	}
+
 	files, err := ioutil.ReadDir(imagesDir)
 	if err != nil {
 		return err
@@ -56,6 +76,7 @@ func PreloadImages(clientFactory func() (dockerClient.APIClient, error), imagesD
 	for _, file := range files {
 		filename := path.Join(imagesDir, file.Name())
 		if !shouldLoad(filename) {
+			log.Infof("Skipping to preload the file: %s", filename)
 			continue
 		}
 
@@ -63,6 +84,7 @@ func PreloadImages(clientFactory func() (dockerClient.APIClient, error), imagesD
 		if err != nil {
 			return err
 		}
+		defer image.Close()
 		var imageReader io.Reader
 		imageReader = image
 		match, err := regexp.MatchString(".t?gz$", file.Name())
@@ -84,22 +106,26 @@ func PreloadImages(clientFactory func() (dockerClient.APIClient, error), imagesD
 			clientInitialized = true
 		}
 
-		log.Infof("Loading image %s", filename)
-		if _, err = client.ImageLoad(context.Background(), imageReader, false); err != nil {
+		var imageLoadResponse types.ImageLoadResponse
+		if imageLoadResponse, err = client.ImageLoad(context.Background(), imageReader, false); err != nil {
 			return err
 		}
-
-		if err = image.Close(); err != nil {
-			return err
+		cfg := config.LoadConfig()
+		if cfg.Rancher.PreloadWait {
+			if _, err := ioutil.ReadAll(imageLoadResponse.Body); err != nil {
+				return err
+			}
 		}
 
+		log.Infof("Finished to load image %s", filename)
+
+		log.Infof("Creating done stamp file for image %s", filename)
 		doneStamp, err := os.Create(fmt.Sprintf("%s.done", filename))
 		if err != nil {
 			return err
 		}
-		if err = doneStamp.Close(); err != nil {
-			return err
-		}
+		defer doneStamp.Close()
+		log.Infof("Finished to created the done stamp file for image %s", filename)
 	}
 
 	return nil

cmd/control/recovery_init.go

@@ -5,7 +5,8 @@ import (
 	"os/exec"
 	"syscall"
 
-	log "github.com/Sirupsen/logrus"
+	"github.com/rancher/os/pkg/log"
+
 	"github.com/codegangsta/cli"
 )
 

cmd/control/selinux.go

@@ -4,12 +4,14 @@ import (
 	"fmt"
 	"syscall"
 
-	"github.com/codegangsta/cli"
 	"github.com/rancher/os/config"
+
+	"github.com/codegangsta/cli"
 )
 
 func selinuxCommand() cli.Command {
 	app := cli.Command{}
+	app.Hidden = true
 	app.Name = "selinux"
 	app.Action = func(c *cli.Context) error {
 		argv := []string{"system-docker", "run", "-it", "--privileged", "--rm",

cmd/control/service/app/app.go

@@ -7,12 +7,12 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/rancher/os/log"
-	"golang.org/x/net/context"
+	"github.com/rancher/os/pkg/log"
 
 	"github.com/codegangsta/cli"
 	"github.com/docker/libcompose/project"
 	"github.com/docker/libcompose/project/options"
+	"golang.org/x/net/context"
 )
 
 func ProjectPs(p project.APIProject, c *cli.Context) error {
@@ -85,6 +85,7 @@ func ProjectUp(p project.APIProject, c *cli.Context) error {
 	if err != nil {
 		return cli.NewExitError(err.Error(), 1)
 	}
+
 	if c.Bool("foreground") {
 		signalChan := make(chan os.Signal, 1)
 		cleanupDone := make(chan bool)

cmd/control/service/command/command.go

@@ -3,9 +3,10 @@ package command
 import (
 	"errors"
 
+	"github.com/rancher/os/cmd/control/service/app"
+
 	"github.com/codegangsta/cli"
 	composeApp "github.com/docker/libcompose/cli/app"
-	"github.com/rancher/os/cmd/control/service/app"
 )
 
 func verifyOneOrMoreServices(c *cli.Context) error {
@@ -145,7 +146,7 @@ func LogsCommand(factory composeApp.ProjectFactory) cli.Command {
 				Value: 100,
 			},
 			cli.BoolFlag{
-				Name:  "follow",
+				Name:  "follow, f",
 				Usage: "Follow log output.",
 			},
 		},

cmd/control/service/service.go

@@ -4,15 +4,16 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/rancher/os/cmd/control/service/command"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/compose"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+	"github.com/rancher/os/pkg/util/network"
+
 	"github.com/codegangsta/cli"
 	dockerApp "github.com/docker/libcompose/cli/docker/app"
 	"github.com/docker/libcompose/project"
-	"github.com/rancher/os/cmd/control/service/command"
-	"github.com/rancher/os/compose"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
-	"github.com/rancher/os/util/network"
 )
 
 type projectFactory struct {
@@ -70,8 +71,18 @@ func serviceSubCommands() []cli.Command {
 			Action: disable,
 		},
 		{
-			Name:   "list",
-			Usage:  "list services and state",
+			Name:  "list",
+			Usage: "list services and state",
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "all, a",
+					Usage: "list all services and state",
+				},
+				cli.BoolFlag{
+					Name:  "update, u",
+					Usage: "update service cache",
+				},
+			},
 			Action: list,
 		},
 		{
@@ -173,7 +184,13 @@ func list(c *cli.Context) error {
 		clone[service] = enabled
 	}
 
-	services := availableService(cfg)
+	services := availableService(cfg, c.Bool("update"))
+
+	if c.Bool("all") {
+		for service := range cfg.Rancher.Services {
+			fmt.Printf("enabled  %s\n", service)
+		}
+	}
 
 	for _, service := range services {
 		if enabled, ok := clone[service]; ok {
@@ -209,7 +226,7 @@ func IsLocalOrURL(service string) bool {
 
 // ValidService checks to see if the service definition exists
 func ValidService(service string, cfg *config.CloudConfig) bool {
-	services := availableService(cfg)
+	services := availableService(cfg, false)
 	if !IsLocalOrURL(service) && !util.Contains(services, service) {
 		return false
 	}
@@ -222,7 +239,14 @@ func validateService(service string, cfg *config.CloudConfig) {
 	}
 }
 
-func availableService(cfg *config.CloudConfig) []string {
+func availableService(cfg *config.CloudConfig, update bool) []string {
+	if update {
+		err := network.UpdateCaches(cfg.Rancher.Repositories.ToArray(), "services")
+		if err != nil {
+			log.Debugf("Failed to update service caches: %v", err)
+		}
+
+	}
 	services, err := network.GetServices(cfg.Rancher.Repositories.ToArray())
 	if err != nil {
 		log.Fatalf("Failed to get services: %v", err)

cmd/control/switch_console.go

@@ -3,11 +3,12 @@ package control
 import (
 	"errors"
 
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/compose"
+	"github.com/rancher/os/pkg/log"
+
 	"github.com/codegangsta/cli"
 	"github.com/docker/libcompose/project/options"
-	"github.com/rancher/os/compose"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
 	"golang.org/x/net/context"
 )
 
@@ -24,6 +25,14 @@ func switchConsoleAction(c *cli.Context) error {
 		return err
 	}
 
+	// stop docker and console to avoid zombie process
+	if err = project.Stop(context.Background(), 10, "docker"); err != nil {
+		log.Errorf("Failed to stop Docker: %v", err)
+	}
+	if err = project.Stop(context.Background(), 10, "console"); err != nil {
+		log.Errorf("Failed to stop console: %v", err)
+	}
+
 	if newConsole != "default" {
 		if err = compose.LoadSpecialService(project, cfg, "console", newConsole); err != nil {
 			return err
@@ -40,8 +49,8 @@ func switchConsoleAction(c *cli.Context) error {
 		return err
 	}
 
-	if err = project.Restart(context.Background(), 10, "docker"); err != nil {
-		log.Errorf("Failed to restart Docker: %v", err)
+	if err = project.Start(context.Background(), "docker"); err != nil {
+		log.Errorf("Failed to start Docker: %v", err)
 	}
 
 	return nil

cmd/control/tlsconf.go

@@ -5,12 +5,12 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 
 	"github.com/codegangsta/cli"
 	machineUtil "github.com/docker/machine/utils"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/util"
 )
 
 const (

cmd/control/udevsettle.go

@@ -1,21 +1,64 @@
 package control
 
 import (
+	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/log"
 
 	"github.com/codegangsta/cli"
-	"github.com/rancher/os/log"
 )
 
 func udevSettleAction(c *cli.Context) {
+	if err := extraRules(); err != nil {
+		log.Error(err)
+	}
+
 	if err := UdevSettle(); err != nil {
 		log.Fatal(err)
 	}
 }
 
+func extraRules() error {
+	cfg := config.LoadConfig()
+	if len(cfg.Rancher.Network.ModemNetworks) > 0 {
+		rules, err := ioutil.ReadDir(config.UdevRulesExtrasDir)
+		if err != nil {
+			return err
+		}
+		for _, r := range rules {
+			if r.IsDir() || filepath.Ext(r.Name()) != ".rules" {
+				continue
+			}
+			err := os.Symlink(filepath.Join(config.UdevRulesExtrasDir, r.Name()), filepath.Join(config.UdevRulesDir, r.Name()))
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		rules, err := ioutil.ReadDir(config.UdevRulesDir)
+		if err != nil {
+			return err
+		}
+		for _, r := range rules {
+			if r.IsDir() || (filepath.Ext(r.Name()) != ".rules") || (r.Mode()&os.ModeSymlink != 0) {
+				continue
+			}
+			err := os.Remove(filepath.Join(config.UdevRulesDir, r.Name()))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func UdevSettle() error {
 	cmd := exec.Command("udevd", "--daemon")
+	defer exec.Command("killall", "udevd").Run()
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {

cmd/control/user_docker.go

@@ -5,21 +5,20 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"syscall"
 	"time"
 
-	"golang.org/x/net/context"
-
-	"path/filepath"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/compose"
+	rosDocker "github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 
 	"github.com/codegangsta/cli"
 	composeClient "github.com/docker/libcompose/docker/client"
 	"github.com/docker/libcompose/project"
-	"github.com/rancher/os/compose"
-	"github.com/rancher/os/config"
-	rosDocker "github.com/rancher/os/docker"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
+	"golang.org/x/net/context"
 )
 
 const (
@@ -27,6 +26,7 @@ const (
 	dockerPidFile         = "/var/run/docker.pid"
 	sourceDirectory       = "/engine"
 	destDirectory         = "/var/lib/rancher/engine"
+	dockerCompletionFName = "completion"
 )
 
 var (
@@ -95,8 +95,14 @@ func copyBinaries(source, dest string) error {
 		if err = out.Close(); err != nil {
 			return err
 		}
-		if err := os.Chmod(destFile, 0751); err != nil {
-			return err
+		if file.Name() == dockerCompletionFName {
+			if err := os.Chmod(destFile, 0644); err != nil {
+				return err
+			}
+		} else {
+			if err := os.Chmod(destFile, 0751); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -175,14 +181,14 @@ func startDocker(cfg *config.CloudConfig) error {
 		return err
 	}
 
-	cmd := []string{"docker-runc", "exec", "--", info.ID, "env"}
+	cmd := []string{"system-docker-runc", "exec", "--", info.ID, "env"}
 	log.Info(dockerCfg.AppendEnv())
 	cmd = append(cmd, dockerCfg.AppendEnv()...)
 	cmd = append(cmd, dockerCommand...)
 	cmd = append(cmd, args...)
 	log.Infof("Running %v", cmd)
 
-	return syscall.Exec("/usr/bin/ros", cmd, os.Environ())
+	return syscall.Exec("/usr/bin/system-docker-runc", cmd, os.Environ())
 }
 
 func waitForPid(service string, project *project.Project) (int, error) {

cmd/control/util.go

@@ -3,10 +3,15 @@ package control
 import (
 	"bufio"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strings"
+	"time"
 
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/log"
+
+	"github.com/pkg/errors"
 )
 
 func yes(question string) bool {
@@ -19,3 +24,63 @@ func yes(question string) bool {
 
 	return strings.ToLower(line[0:1]) == "y"
 }
+
+func formatImage(image string, cfg *config.CloudConfig) string {
+	domainRegistry := cfg.Rancher.Environment["REGISTRY_DOMAIN"]
+	if domainRegistry != "docker.io" && domainRegistry != "" {
+		return fmt.Sprintf("%s/%s", domainRegistry, image)
+	}
+	return image
+}
+
+func symLinkEngineBinary() []symlink {
+	baseSymlink := []symlink{
+		{"/usr/share/ros/os-release", "/usr/lib/os-release"},
+		{"/usr/share/ros/os-release", "/etc/os-release"},
+
+		{"/var/lib/rancher/engine/docker", "/usr/bin/docker"},
+		{"/var/lib/rancher/engine/dockerd", "/usr/bin/dockerd"},
+		{"/var/lib/rancher/engine/docker-init", "/usr/bin/docker-init"},
+		{"/var/lib/rancher/engine/docker-proxy", "/usr/bin/docker-proxy"},
+
+		// >= 18.09.0
+		{"/var/lib/rancher/engine/containerd", "/usr/bin/containerd"},
+		{"/var/lib/rancher/engine/ctr", "/usr/bin/ctr"},
+		{"/var/lib/rancher/engine/containerd-shim", "/usr/bin/containerd-shim"},
+		{"/var/lib/rancher/engine/runc", "/usr/bin/runc"},
+
+		// < 18.09.0
+		{"/var/lib/rancher/engine/docker-containerd", "/usr/bin/docker-containerd"},
+		{"/var/lib/rancher/engine/docker-containerd-ctr", "/usr/bin/docker-containerd-ctr"},
+		{"/var/lib/rancher/engine/docker-containerd-shim", "/usr/bin/docker-containerd-shim"},
+		{"/var/lib/rancher/engine/docker-runc", "/usr/bin/docker-runc"},
+	}
+	return baseSymlink
+}
+
+func checkZfsBackingFS(driver, dir string) error {
+	if driver != "zfs" {
+		return nil
+	}
+	for i := 0; i < 4; i++ {
+		mountInfo, err := ioutil.ReadFile("/proc/self/mountinfo")
+		if err != nil {
+			continue
+		}
+		for _, mount := range strings.Split(string(mountInfo), "\n") {
+			if strings.Contains(mount, dir) && strings.Contains(mount, driver) {
+				return nil
+			}
+		}
+		time.Sleep(1 * time.Second)
+	}
+	return errors.Errorf("BackingFS: %s not match storage-driver: %s", dir, driver)
+}
+
+func checkGlobalCfg() bool {
+	_, err := os.Stat("/proc/1/root/boot/global.cfg")
+	if err == nil || os.IsExist(err) {
+		return true
+	}
+	return false
+}

cmd/init/init.go

@@ -0,0 +1,90 @@
+// +build linux
+
+package init
+
+import (
+	"fmt"
+
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/dfs"
+	"github.com/rancher/os/pkg/init/b2d"
+	"github.com/rancher/os/pkg/init/cloudinit"
+	"github.com/rancher/os/pkg/init/configfiles"
+	"github.com/rancher/os/pkg/init/debug"
+	"github.com/rancher/os/pkg/init/docker"
+	"github.com/rancher/os/pkg/init/env"
+	"github.com/rancher/os/pkg/init/fsmount"
+	"github.com/rancher/os/pkg/init/hypervisor"
+	"github.com/rancher/os/pkg/init/modules"
+	"github.com/rancher/os/pkg/init/one"
+	"github.com/rancher/os/pkg/init/prepare"
+	"github.com/rancher/os/pkg/init/recovery"
+	"github.com/rancher/os/pkg/init/selinux"
+	"github.com/rancher/os/pkg/init/sharedroot"
+	"github.com/rancher/os/pkg/init/switchroot"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/sysinit"
+)
+
+func MainInit() {
+	log.InitLogger()
+	// TODO: this breaks and does nothing if the cfg is invalid (or is it due to threading?)
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Printf("Starting Recovery console: %v\n", r)
+			recovery.Recovery(nil)
+		}
+	}()
+
+	if err := RunInit(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func RunInit() error {
+	initFuncs := config.CfgFuncs{
+		{"set env", env.Init},
+		{"preparefs", prepare.FS},
+		{"save init cmdline", prepare.SaveCmdline},
+		{"mount OEM", fsmount.MountOem},
+		{"debug save cfg", debug.PrintAndLoadConfig},
+		{"load modules", modules.LoadModules},
+		{"recovery console", recovery.LoadRecoveryConsole},
+		{"b2d env", b2d.B2D},
+		{"mount STATE and bootstrap", fsmount.MountStateAndBootstrap},
+		{"cloud-init", cloudinit.CloudInit},
+		{"read cfg and log files", configfiles.ReadConfigFiles},
+		{"switchroot", switchroot.SwitchRoot},
+		{"mount OEM2", fsmount.MountOem},
+		{"mount BOOT", fsmount.MountBoot},
+		{"write cfg and log files", configfiles.WriteConfigFiles},
+		{"b2d Env", b2d.Env},
+		{"hypervisor tools", hypervisor.Tools},
+		{"preparefs2", prepare.FS},
+		{"load modules2", modules.LoadModules},
+		{"set proxy env", env.Proxy},
+		{"init SELinux", selinux.Initialize},
+		{"setupSharedRoot", sharedroot.Setup},
+		{"sysinit", sysinit.RunSysInit},
+	}
+
+	cfg, err := config.ChainCfgFuncs(nil, initFuncs)
+	if err != nil {
+		recovery.Recovery(err)
+	}
+
+	launchConfig, args := docker.GetLaunchConfig(cfg, &cfg.Rancher.SystemDocker)
+	launchConfig.Fork = !cfg.Rancher.SystemDocker.Exec
+	//launchConfig.NoLog = true
+
+	log.Info("Launching System Docker")
+	_, err = dfs.LaunchDocker(launchConfig, config.SystemDockerBin, args...)
+	if err != nil {
+		log.Errorf("Error Launching System Docker: %s", err)
+		recovery.Recovery(err)
+		return err
+	}
+	// Code never gets here - rancher.system_docker.exec=true
+
+	return one.PidOne()
+}

cmd/network/network.go

@@ -1,17 +1,31 @@
 package network
 
 import (
-	"golang.org/x/net/context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"syscall"
+	"text/template"
 
-	"github.com/rancher/os/docker"
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/hostname"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/netconf"
 
 	"github.com/docker/libnetwork/resolvconf"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/hostname"
-	"github.com/rancher/os/netconf"
+	"golang.org/x/net/context"
 )
 
+var funcMap = template.FuncMap{
+	"addFunc": func(a, b int) string {
+		return strconv.Itoa(a + b)
+	},
+}
+
 func Main() {
 	log.InitLogger()
 
@@ -28,36 +42,113 @@ func Main() {
 		log.Error(err)
 	}
 
-	select {}
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGTERM)
+	<-signalChan
+	log.Info("Received SIGTERM, shutting down")
+	netconf.StopWpaSupplicant()
+	netconf.StopDhcpcd()
 }
 
 func ApplyNetworkConfig(cfg *config.CloudConfig) {
 	log.Infof("Apply Network Config")
-	nameservers := cfg.Rancher.Network.DNS.Nameservers
-	search := cfg.Rancher.Network.DNS.Search
-	userSetDNS := len(nameservers) > 0 || len(search) > 0
-	if !userSetDNS {
-		nameservers = cfg.Rancher.Defaults.Network.DNS.Nameservers
-		search = cfg.Rancher.Defaults.Network.DNS.Search
-	}
-
-	// TODO: don't write to the file if nameservers is still empty
-	log.Infof("Writing resolv.conf (%v) %v", nameservers, search)
-	if _, err := resolvconf.Build("/etc/resolv.conf", nameservers, search, nil); err != nil {
-		log.Error(err)
-	}
+	userSetDNS := len(cfg.Rancher.Network.DNS.Nameservers) > 0 || len(cfg.Rancher.Network.DNS.Search) > 0
 
 	if err := hostname.SetHostnameFromCloudConfig(cfg); err != nil {
-		log.Error(err)
+		log.Errorf("Failed to set hostname from cloud config: %v", err)
 	}
 
 	userSetHostname := cfg.Hostname != ""
-	if err := netconf.ApplyNetworkConfigs(&cfg.Rancher.Network, userSetHostname, userSetDNS); err != nil {
-		log.Error(err)
+	if cfg.Rancher.Network.DHCPTimeout <= 0 {
+		cfg.Rancher.Network.DHCPTimeout = cfg.Rancher.Defaults.Network.DHCPTimeout
 	}
 
+	// In order to handle the STATIC mode in Wi-Fi network, we have to update the dhcpcd.conf file.
+	// https://wiki.archlinux.org/index.php/dhcpcd#Static_profile
+	if len(cfg.Rancher.Network.WifiNetworks) > 0 {
+		generateDhcpcdFiles(cfg)
+		generateWpaFiles(cfg)
+	}
+
+	dhcpSetDNS, err := netconf.ApplyNetworkConfigs(&cfg.Rancher.Network, userSetHostname, userSetDNS)
+	if err != nil {
+		log.Errorf("Failed to apply network configs(by netconf): %v", err)
+	}
+
+	if dhcpSetDNS {
+		log.Infof("DNS set by DHCP")
+	}
+
+	if !userSetDNS && !dhcpSetDNS {
+		// only write 8.8.8.8,8.8.4.4 as a last resort
+		log.Infof("Writing default resolv.conf - no user setting, and no DHCP setting")
+		if _, err := resolvconf.Build("/etc/resolv.conf",
+			cfg.Rancher.Defaults.Network.DNS.Nameservers,
+			cfg.Rancher.Defaults.Network.DNS.Search,
+			nil); err != nil {
+			log.Errorf("Failed to write resolv.conf (!userSetDNS and !dhcpSetDNS): %v", err)
+		}
+	}
+	if userSetDNS {
+		if _, err := resolvconf.Build("/etc/resolv.conf", cfg.Rancher.Network.DNS.Nameservers, cfg.Rancher.Network.DNS.Search, nil); err != nil {
+			log.Errorf("Failed to write resolv.conf (userSetDNS): %v", err)
+		} else {
+			log.Infof("writing to /etc/resolv.conf: nameservers: %v, search: %v", cfg.Rancher.Network.DNS.Nameservers, cfg.Rancher.Network.DNS.Search)
+		}
+	}
+
+	resolve, err := ioutil.ReadFile("/etc/resolv.conf")
+	log.Debugf("Resolve.conf == [%s], %v", resolve, err)
+
 	log.Infof("Apply Network Config SyncHostname")
 	if err := hostname.SyncHostname(); err != nil {
-		log.Error(err)
+		log.Errorf("Failed to sync hostname: %v", err)
+	}
+}
+
+func generateDhcpcdFiles(cfg *config.CloudConfig) {
+	networks := cfg.Rancher.Network.WifiNetworks
+	interfaces := cfg.Rancher.Network.Interfaces
+	configs := make(map[string]netconf.WifiNetworkConfig)
+	for k, v := range interfaces {
+		if c, ok := networks[v.WifiNetwork]; ok && c.Address != "" {
+			configs[k] = c
+		}
+	}
+	f, err := os.Create(config.DHCPCDConfigFile)
+	defer f.Close()
+	if err != nil {
+		log.Errorf("Failed to open file: %s err: %v", config.DHCPCDConfigFile, err)
+	}
+	templateFiles := []string{config.DHCPCDTemplateFile}
+	templateName := filepath.Base(templateFiles[0])
+	p := template.Must(template.New(templateName).ParseFiles(templateFiles...))
+	if err = p.Execute(f, configs); err != nil {
+		log.Errorf("Failed to wrote wpa configuration to %s: %v", config.DHCPCDConfigFile, err)
+	}
+}
+
+func generateWpaFiles(cfg *config.CloudConfig) {
+	networks := cfg.Rancher.Network.WifiNetworks
+	interfaces := cfg.Rancher.Network.Interfaces
+	for k, v := range interfaces {
+		if v.WifiNetwork != "" {
+			configs := make(map[string]netconf.WifiNetworkConfig)
+			filename := fmt.Sprintf(config.WPAConfigFile, k)
+			f, err := os.Create(filename)
+			if err != nil {
+				log.Errorf("Failed to open file: %s err: %v", filename, err)
+			}
+			if c, ok := networks[v.WifiNetwork]; ok {
+				configs[v.WifiNetwork] = c
+			}
+			templateFiles := []string{config.WPATemplateFile}
+			templateName := filepath.Base(templateFiles[0])
+			p := template.Must(template.New(templateName).Funcs(funcMap).ParseFiles(templateFiles...))
+			if err = p.Execute(f, configs); err != nil {
+				log.Errorf("Failed to wrote wpa configuration to %s: %v", filename, err)
+			}
+			f.Close()
+		}
 	}
 }

cmd/power/power.go

@@ -10,22 +10,23 @@ import (
 	"syscall"
 	"time"
 
-	"golang.org/x/net/context"
+	"github.com/rancher/os/cmd/control/install"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 
 	"github.com/docker/engine-api/types"
 	"github.com/docker/engine-api/types/container"
 	"github.com/docker/engine-api/types/filters"
-	"github.com/rancher/os/cmd/control/install"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
-
-	"github.com/rancher/os/docker"
-	"github.com/rancher/os/util"
+	"golang.org/x/net/context"
 )
 
 // You can't shutdown the system from a process in console because we want to stop the console container.
 // If you do that you kill yourself.  So we spawn a separate container to do power operations
 // This can up because on shutdown we want ssh to gracefully die, terminating ssh connections and not just hanging tcp session
+//
+// Be careful of container name. only [a-zA-Z0-9][a-zA-Z0-9_.-] are allowed
 func runDocker(name string) error {
 	if os.ExpandEnv("${IN_DOCKER}") == "true" {
 		return nil
@@ -36,15 +37,18 @@ func runDocker(name string) error {
 		return err
 	}
 
-	cmd := []string{name}
+	cmd := os.Args
+	log.Debugf("runDocker cmd: %s", cmd)
 
 	if name == "" {
 		name = filepath.Base(os.Args[0])
-		cmd = os.Args
 	}
 
-	existing, err := client.ContainerInspect(context.Background(), name)
+	containerName := strings.TrimPrefix(strings.Join(strings.Split(name, "/"), "-"), "-")
+
+	existing, err := client.ContainerInspect(context.Background(), containerName)
 	if err == nil && existing.ID != "" {
+		// remove the old version of reboot
 		err := client.ContainerRemove(context.Background(), types.ContainerRemoveOptions{
 			ContainerID: existing.ID,
 		})
@@ -73,12 +77,13 @@ func runDocker(name string) error {
 			},
 		},
 		&container.HostConfig{
-			PidMode: "host",
+			PidMode:     "host",
+			NetworkMode: "none",
 			VolumesFrom: []string{
 				currentContainer.ID,
 			},
 			Privileged: true,
-		}, nil, name)
+		}, nil, containerName)
 	if err != nil {
 		return err
 	}
@@ -126,8 +131,23 @@ func reboot(name string, force bool, code uint) {
 		log.Fatalf("%s: Need to be root", os.Args[0])
 	}
 
-	// Add shutdown timeout
 	cfg := config.LoadConfig()
+
+	// Validate config
+	if !force {
+		_, validationErrors, err := config.LoadConfigWithError()
+		if err != nil {
+			log.Fatal(err)
+		}
+		if validationErrors != nil && !validationErrors.Valid() {
+			for _, validationError := range validationErrors.Errors() {
+				log.Error(validationError)
+			}
+			return
+		}
+	}
+
+	// Add shutdown timeout
 	timeoutValue := cfg.Rancher.ShutdownTimeout
 	if timeoutValue == 0 {
 		timeoutValue = 60
@@ -174,7 +194,7 @@ func reboot(name string, force bool, code uint) {
 			return
 		}
 		defer util.Unmount(baseName)
-		Kexec(previouskexecFlag, filepath.Join(baseName, install.BootDir), kexecAppendFlag)
+		Kexec(previouskexecFlag, filepath.Join(baseName, config.BootDir), kexecAppendFlag)
 		return
 	}
 

cmd/power/shutdown.go

@@ -7,10 +7,11 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"github.com/codegangsta/cli"
 	"github.com/rancher/os/cmd/control/install"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/pkg/log"
+
+	"github.com/codegangsta/cli"
 )
 
 var (
@@ -27,7 +28,7 @@ func Shutdown() {
 	log.InitLogger()
 	app := cli.NewApp()
 
-	app.Name = os.Args[0]
+	app.Name = filepath.Base(os.Args[0])
 	app.Usage = fmt.Sprintf("%s RancherOS\nbuilt: %s", app.Name, config.BuildDate)
 	app.Version = config.Version
 	app.Author = "Rancher Labs, Inc."
@@ -94,13 +95,22 @@ func Shutdown() {
 	if app.Name == "poweroff" {
 		app.Flags = append(app.Flags, cli.BoolTFlag{
 			Name:        "P, poweroff",
-			Usage:       "halt the machine",
+			Usage:       "poweroff the machine",
 			Destination: &poweroffFlag,
 		})
 	} else {
+		//  shutdown -h
+		//        Equivalent to --poweroff
+		if app.Name == "shutdown" {
+			app.Flags = append(app.Flags, cli.BoolFlag{
+				Name:        "h",
+				Usage:       "poweroff the machine",
+				Destination: &poweroffFlag,
+			})
+		}
 		app.Flags = append(app.Flags, cli.BoolFlag{
 			Name:        "P, poweroff",
-			Usage:       "halt the machine",
+			Usage:       "poweroff the machine",
 			Destination: &poweroffFlag,
 		})
 	}
@@ -181,6 +191,7 @@ func Kexec(previous bool, bootDir, append string) error {
 // Reboot is used by installation / upgrade
 // TODO: add kexec option
 func Reboot() {
+	os.Args = []string{"reboot"}
 	reboot("reboot", false, syscall.LINUX_REBOOT_CMD_RESTART)
 }
 
@@ -197,8 +208,13 @@ func shutdown(c *cli.Context) error {
 	}
 
 	timeArg := c.Args().Get(0)
-	if c.App.Name == "shutdown" && timeArg != "" {
-		if timeArg != "now" {
+	// We may be called via an absolute path, so check that now and make sure we
+	// don't pass the wrong app name down. Aside from the logic in the immediate
+	// context here, the container name is derived from how we were called and
+	// cannot contain slashes.
+	appName := filepath.Base(c.App.Name)
+	if appName == "shutdown" && timeArg != "" {
+		if timeArg != "now" && timeArg != "+0" {
 			err := fmt.Errorf("Sorry, can't parse '%s' as time value (only 'now' supported)", timeArg)
 			log.Error(err)
 			return err
@@ -206,7 +222,7 @@ func shutdown(c *cli.Context) error {
 		// TODO: if there are more params, LOG them
 	}
 
-	reboot(c.App.Name, forceFlag, powerCmd)
+	reboot(appName, forceFlag, powerCmd)
 
 	return nil
 }

cmd/respawn/respawn.go

@@ -13,9 +13,10 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/codegangsta/cli"
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/pkg/log"
+
+	"github.com/codegangsta/cli"
 )
 
 var (
@@ -136,19 +137,14 @@ func execute(line string, doneChannel chan string) {
 			Setsid: true,
 		}
 
-		err := cmd.Start()
-		if err != nil {
-			log.Errorf("%s : %v", line, err)
-		}
-
-		if err == nil {
+		if err := cmd.Start(); err == nil {
 			addProcess(cmd.Process)
-			err = cmd.Wait()
+			if err = cmd.Wait(); err != nil {
+				log.Errorf("Wait cmd to exit: %s, err: %v", line, err)
+			}
 			removeProcess(cmd.Process)
-		}
-
-		if err != nil {
-			log.Errorf("%s : %v", line, err)
+		} else {
+			log.Errorf("Start cmd: %s, err: %v", line, err)
 		}
 
 		if !running {

cmd/sysinit/sysinit.go

@@ -1,13 +1,21 @@
 package sysinit
 
 import (
-	initPkg "github.com/rancher/os/init"
-	"github.com/rancher/os/log"
+	"io/ioutil"
+	"os"
+
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/sysinit"
 )
 
 func Main() {
 	log.InitLogger()
-	if err := initPkg.SysInit(); err != nil {
+
+	resolve, err := ioutil.ReadFile("/etc/resolv.conf")
+	log.Infof("Resolv.conf == [%s], %v", resolve, err)
+	log.Infof("Exec %v", os.Args)
+
+	if err := sysinit.SysInit(); err != nil {
 		log.Fatal(err)
 	}
 }

cmd/systemdocker/system-docker.go

@@ -1,23 +0,0 @@
-package systemdocker
-
-import (
-	"os"
-
-	"github.com/docker/docker/docker"
-	"github.com/rancher/os/config"
-	"github.com/rancher/os/log"
-)
-
-func Main() {
-	log.SetLevel(log.DebugLevel)
-
-	if os.Geteuid() != 0 {
-		log.Fatalf("%s: Need to be root", os.Args[0])
-	}
-
-	if os.Getenv("DOCKER_HOST") == "" {
-		os.Setenv("DOCKER_HOST", config.SystemDockerHost)
-	}
-
-	docker.RancherOSMain()
-}

cmd/wait/wait.go

@@ -4,8 +4,8 @@ import (
 	"os"
 
 	"github.com/rancher/os/config"
-	"github.com/rancher/os/docker"
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/pkg/docker"
+	"github.com/rancher/os/pkg/log"
 )
 
 func Main() {

config/cloudinit/Documentation/config-drive.md

@@ -36,5 +36,6 @@ image and use a plain directory containing the same contents:
 qemu-system-x86_64 \
     -fsdev local,id=conf,security_model=none,readonly,path=/tmp/new-drive \
     -device virtio-9p-pci,fsdev=conf,mount_tag=config-2 \
+    -device virtio-rng-pci \
     [usual qemu options here...]
 ```

config/cloudinit/config/decode.go

@@ -18,7 +18,12 @@ func DecodeBase64Content(content string) ([]byte, error) {
 }
 
 func DecodeGzipContent(content string) ([]byte, error) {
-	gzr, err := gzip.NewReader(bytes.NewReader([]byte(content)))
+	byteContent := []byte(content)
+	return DecompressGzip(byteContent)
+}
+
+func DecompressGzip(content []byte) ([]byte, error) {
+	gzr, err := gzip.NewReader(bytes.NewReader(content))
 
 	if err != nil {
 		return nil, fmt.Errorf("Unable to decode gzip: %q", err)

config/cloudinit/config/validate/validate.go

@@ -149,7 +149,7 @@ func coerceNodes(w, s Node) Node {
 	return n
 }
 
-// normalizeNodeNames replaces all occurences of '-' with '_' within key names
+// normalizeNodeNames replaces all occurrences of '-' with '_' within key names
 // and makes a note of each replacement in the report.
 func normalizeNodeNames(node Node, report *Report) Node {
 	if strings.Contains(node.name, "-") {

config/cloudinit/cover

@@ -22,6 +22,6 @@ COVERPKG=${COVERPKG//./}
 # generate arg for "go test"
 export COVER="-coverprofile ${COVEROUT}/${COVERPKG}.out"
 
-source ./test
+source test
 
 go tool cover -html=${COVEROUT}/${COVERPKG}.out

config/cloudinit/datasource/configdrive/configdrive.go

@@ -22,11 +22,11 @@ import (
 	"path"
 	"syscall"
 
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 
 	"github.com/docker/docker/pkg/mount"
-	"github.com/rancher/os/config/cloudinit/datasource"
-	"github.com/rancher/os/util"
 )
 
 const (
@@ -70,7 +70,7 @@ func (cd *ConfigDrive) Finish() error {
 
 func (cd *ConfigDrive) String() string {
 	if cd.lastError != nil {
-		return fmt.Sprintf("%s: %s (lastError: %s)", cd.Type(), cd.root, cd.lastError)
+		return fmt.Sprintf("%s: %s (lastError: %v)", cd.Type(), cd.root, cd.lastError)
 	}
 	return fmt.Sprintf("%s: %s", cd.Type(), cd.root)
 }

config/cloudinit/datasource/datasource.go

@@ -17,7 +17,7 @@ package datasource
 import (
 	"net"
 
-	"github.com/rancher/os/netconf"
+	"github.com/rancher/os/pkg/netconf"
 )
 
 type Datasource interface {
@@ -28,7 +28,7 @@ type Datasource interface {
 	FetchUserdata() ([]byte, error)
 	Type() string
 	String() string
-	// Finish gives the datasource the oportunity to clean up, unmount or release any open / cache resources
+	// Finish gives the datasource the opportunity to clean up, unmount or release any open / cache resources
 	Finish() error
 }
 
@@ -38,6 +38,7 @@ type Metadata struct {
 	Hostname      string
 	SSHPublicKeys map[string]string
 	NetworkConfig netconf.NetworkConfig
+	RootDisk      string
 
 	PublicIPv4  net.IP
 	PublicIPv6  net.IP

config/cloudinit/datasource/file/file.go

@@ -41,7 +41,7 @@ func (f *LocalFile) Finish() error {
 }
 
 func (f *LocalFile) String() string {
-	return fmt.Sprintf("%s: %s (lastError: %s)", f.Type(), f.path, f.lastError)
+	return fmt.Sprintf("%s: %s (lastError: %v)", f.Type(), f.path, f.lastError)
 }
 
 func (f *LocalFile) AvailabilityChanges() bool {

config/cloudinit/datasource/metadata/aliyun/metadata.go

@@ -0,0 +1,84 @@
+package aliyun
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/pkg/netconf"
+)
+
+const (
+	DefaultAddress = "http://100.100.100.200/"
+	apiVersion     = "2016-01-01/"
+	userdataPath   = apiVersion + "user-data/"
+	metadataPath   = apiVersion + "meta-data/"
+)
+
+type MetadataService struct {
+	metadata.Service
+}
+
+func NewDatasource(root string) *MetadataService {
+	if root == "" {
+		root = DefaultAddress
+	}
+	return &MetadataService{metadata.NewDatasource(root, apiVersion, userdataPath, metadataPath, nil)}
+}
+
+func (ms MetadataService) AvailabilityChanges() bool {
+	// TODO: if it can't find the network, maybe we can start it?
+	return false
+}
+
+func (ms MetadataService) FetchMetadata() (metadata datasource.Metadata, err error) {
+	// see https://www.alibabacloud.com/help/faq-detail/49122.htm
+	metadata.NetworkConfig = netconf.NetworkConfig{}
+
+	enablePublicKey := false
+
+	rootContents, err := ms.FetchAttributes("")
+	if err != nil {
+		return metadata, err
+	}
+	for _, c := range rootContents {
+		if c == "public-keys/" {
+			enablePublicKey = true
+			break
+		}
+	}
+	if !enablePublicKey {
+		return metadata, fmt.Errorf("The public-keys should be enable in %s", ms.Type())
+	}
+
+	keynames, err := ms.FetchAttributes("public-keys/")
+	if err != nil {
+		return metadata, err
+	}
+
+	metadata.SSHPublicKeys = map[string]string{}
+	for _, k := range keynames {
+		k = strings.TrimRight(k, "/")
+		sshkey, err := ms.FetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", k))
+		if err != nil {
+			return metadata, err
+		}
+		metadata.SSHPublicKeys[k] = sshkey
+		log.Printf("Found SSH key for %q\n", k)
+	}
+
+	if hostname, err := ms.FetchAttribute("hostname"); err == nil {
+		metadata.Hostname = hostname
+		log.Printf("Found hostname  %s\n", hostname)
+	} else {
+		return metadata, err
+	}
+
+	return metadata, nil
+}
+
+func (ms MetadataService) Type() string {
+	return "aliyun-metadata-service"
+}

config/cloudinit/datasource/metadata/aliyun/metadata_test.go

@@ -0,0 +1,78 @@
+package aliyun
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
+	"github.com/rancher/os/config/cloudinit/pkg"
+)
+
+func TestType(t *testing.T) {
+	want := "aliyun-metadata-service"
+	if kind := (MetadataService{}).Type(); kind != want {
+		t.Fatalf("bad type: want %q, got %q", want, kind)
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root         string
+		metadataPath string
+		resources    map[string]string
+		expect       datasource.Metadata
+		clientErr    error
+		expectErr    error
+	}{
+		{
+			root:         "/",
+			metadataPath: "2016-01-01/meta-data/",
+			resources: map[string]string{
+				"/2016-01-01/meta-data/": "hostname\n",
+			},
+			expectErr: fmt.Errorf("The public-keys should be enable in aliyun-metadata-service"),
+		},
+		{
+			root:         "/",
+			metadataPath: "2016-01-01/meta-data/",
+			resources: map[string]string{
+				"/2016-01-01/meta-data/":                           "hostname\npublic-keys/\n",
+				"/2016-01-01/meta-data/hostname":                   "host",
+				"/2016-01-01/meta-data/public-keys/":               "xx/",
+				"/2016-01-01/meta-data/public-keys/xx/":            "openssh-key",
+				"/2016-01-01/meta-data/public-keys/xx/openssh-key": "key",
+			},
+			expect: datasource.Metadata{
+				Hostname:      "host",
+				SSHPublicKeys: map[string]string{"xx": "key"},
+			},
+		},
+		{
+			clientErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")},
+			expectErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")},
+		},
+	} {
+		service := &MetadataService{metadata.Service{
+			Root:         tt.root,
+			Client:       &test.HTTPClient{Resources: tt.resources, Err: tt.clientErr},
+			MetadataPath: tt.metadataPath,
+		}}
+		metadata, err := service.FetchMetadata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): \nwant %q, \ngot %q\n", tt.resources, tt.expectErr, err)
+		}
+		if !reflect.DeepEqual(tt.expect, metadata) {
+			t.Fatalf("bad fetch (%q): \nwant %#v, \ngot %#v\n", tt.resources, tt.expect, metadata)
+		}
+	}
+}
+
+func Error(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

config/cloudinit/datasource/metadata/azure/metadata.go

@@ -0,0 +1,155 @@
+package azure
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"strconv"
+
+	"github.com/rancher/os/config/cloudinit/config"
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+)
+
+const (
+	metadataHeader   = "true"
+	metadataVersion  = "2019-02-01"
+	metadataEndpoint = "http://169.254.169.254/metadata/"
+)
+
+type MetadataService struct {
+	metadata.Service
+}
+
+func NewDatasource(root string) *MetadataService {
+	if root == "" {
+		root = metadataEndpoint
+	}
+	return &MetadataService{metadata.NewDatasource(root, "instance?api-version="+metadataVersion+"&format=json", "", "", assembleHeader())}
+}
+
+func (ms MetadataService) ConfigRoot() string {
+	return ms.Root + "instance"
+}
+
+func (ms MetadataService) AvailabilityChanges() bool {
+	// TODO: if it can't find the network, maybe we can start it?
+	return false
+}
+
+func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
+	d, err := ms.FetchData(ms.MetadataURL())
+	if err != nil {
+		return datasource.Metadata{}, err
+	}
+	type Plan struct {
+		Name      string `json:"name,omitempty"`
+		Product   string `json:"product,omitempty"`
+		Publisher string `json:"publisher,omitempty"`
+	}
+	type PublicKey struct {
+		KeyData string `json:"keyData,omitempty"`
+		Path    string `json:"path,omitempty"`
+	}
+	type Compute struct {
+		AZEnvironment        string      `json:"azEnvironment,omitempty"`
+		CustomData           string      `json:"customData,omitempty"`
+		Location             string      `json:"location,omitempty"`
+		Name                 string      `json:"name,omitempty"`
+		Offer                string      `json:"offer,omitempty"`
+		OSType               string      `json:"osType,omitempty"`
+		PlacementGroupID     string      `json:"placementGroupId,omitempty"`
+		Plan                 Plan        `json:"plan,omitempty"`
+		PlatformFaultDomain  string      `json:"platformFaultDomain,omitempty"`
+		PlatformUpdateDomain string      `json:"platformUpdateDomain,omitempty"`
+		Provider             string      `json:"provider,omitempty"`
+		PublicKeys           []PublicKey `json:"publicKeys,omitempty"`
+		Publisher            string      `json:"publisher,omitempty"`
+		ResourceGroupName    string      `json:"resourceGroupName,omitempty"`
+		SKU                  string      `json:"sku,omitempty"`
+		SubscriptionID       string      `json:"subscriptionId,omitempty"`
+		Tags                 string      `json:"tags,omitempty"`
+		Version              string      `json:"version,omitempty"`
+		VMID                 string      `json:"vmId,omitempty"`
+		VMScaleSetName       string      `json:"vmScaleSetName,omitempty"`
+		VMSize               string      `json:"vmSize,omitempty"`
+		Zone                 string      `json:"zone,omitempty"`
+	}
+	type IPAddress struct {
+		PrivateIPAddress string `json:"privateIpAddress,omitempty"`
+		PublicIPAddress  string `json:"publicIpAddress,omitempty"`
+	}
+	type Subnet struct {
+		Address string `json:"address,omitempty"`
+		Prefix  string `json:"prefix,omitempty"`
+	}
+	type IPV4 struct {
+		IPAddress []IPAddress `json:"ipAddress,omitempty"`
+		Subnet    []Subnet    `json:"subnet,omitempty"`
+	}
+	type IPV6 struct {
+		IPAddress []IPAddress `json:"ipAddress,omitempty"`
+	}
+	type Interface struct {
+		IPV4       IPV4   `json:"ipv4,omitempty"`
+		IPV6       IPV6   `json:"ipv6,omitempty"`
+		MacAddress string `json:"macAddress,omitempty"`
+	}
+	type Network struct {
+		Interface []Interface `json:"interface,omitempty"`
+	}
+	type Instance struct {
+		Compute Compute `json:"compute,omitempty"`
+		Network Network `json:"network,omitempty"`
+	}
+	instance := &Instance{}
+	if err := json.Unmarshal(d, instance); err != nil {
+		return datasource.Metadata{}, err
+	}
+	m := datasource.Metadata{
+		Hostname:      instance.Compute.Name,
+		SSHPublicKeys: make(map[string]string, 0),
+	}
+	if len(instance.Network.Interface) > 0 {
+		if len(instance.Network.Interface[0].IPV4.IPAddress) > 0 {
+			m.PublicIPv4 = net.ParseIP(instance.Network.Interface[0].IPV4.IPAddress[0].PublicIPAddress)
+			m.PrivateIPv4 = net.ParseIP(instance.Network.Interface[0].IPV4.IPAddress[0].PrivateIPAddress)
+		}
+		if len(instance.Network.Interface[0].IPV6.IPAddress) > 0 {
+			m.PublicIPv6 = net.ParseIP(instance.Network.Interface[0].IPV6.IPAddress[0].PublicIPAddress)
+			m.PrivateIPv6 = net.ParseIP(instance.Network.Interface[0].IPV6.IPAddress[0].PrivateIPAddress)
+		}
+	}
+	for i, k := range instance.Compute.PublicKeys {
+		m.SSHPublicKeys[strconv.Itoa(i)] = k.KeyData
+	}
+	return m, nil
+}
+
+func (ms MetadataService) FetchUserdata() ([]byte, error) {
+	d, err := ms.FetchData(ms.UserdataURL())
+	if err != nil {
+		return []byte{}, err
+	}
+	return config.DecodeBase64Content(string(d))
+}
+
+func (ms MetadataService) Type() string {
+	return "azure-metadata-service"
+}
+
+func (ms MetadataService) MetadataURL() string {
+	// metadata: http://169.254.169.254/metadata/instance?api-version=2019-02-01&format=json
+	return ms.Root + "instance?api-version=" + metadataVersion + "&format=json"
+}
+
+func (ms MetadataService) UserdataURL() string {
+	// userdata: http://169.254.169.254/metadata/instance/compute/customData?api-version=2019-02-01&format=text
+	return ms.Root + "instance/compute/customData?api-version=" + metadataVersion + "&format=text"
+}
+
+func assembleHeader() http.Header {
+	h := http.Header{}
+	h.Add("Metadata", metadataHeader)
+	return h
+}

config/cloudinit/datasource/metadata/azure/metadata_test.go

@@ -0,0 +1,166 @@
+package azure
+
+import (
+	"bytes"
+	"net"
+	"reflect"
+	"testing"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
+)
+
+func TestType(t *testing.T) {
+	want := "azure-metadata-service"
+	if kind := (MetadataService{}).Type(); kind != want {
+		t.Fatalf("bad type: want %q, got %q", want, kind)
+	}
+}
+
+func TestMetadataURL(t *testing.T) {
+	want := "http://169.254.169.254/metadata/instance?api-version=2019-02-01&format=json"
+	ms := NewDatasource("")
+	if url := ms.MetadataURL(); url != want {
+		t.Fatalf("bad url: want %q, got %q", want, url)
+	}
+}
+
+func TestUserdataURL(t *testing.T) {
+	want := "http://169.254.169.254/metadata/instance/compute/customData?api-version=2019-02-01&format=text"
+	ms := NewDatasource("")
+	if url := ms.UserdataURL(); url != want {
+		t.Fatalf("bad url: want %q, got %q", want, url)
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root         string
+		metadataPath string
+		resources    map[string]string
+		expect       datasource.Metadata
+		clientErr    error
+		expectErr    error
+	}{
+		{
+			root: "/metadata/",
+			resources: map[string]string{
+				"/metadata/instance?api-version=2019-02-01&format=json": `{
+	"compute": {
+		"azEnvironment": "AZUREPUBLICCLOUD",
+		"location": "westus",
+		"name": "rancheros",
+		"offer": "",
+		"osType": "Linux",
+		"placementGroupId": "",
+		"plan": {
+			"name": "",
+			"product": "",
+			"publisher": ""
+		},
+		"platformFaultDomain": "0",
+		"platformUpdateDomain": "0",
+		"provider": "Microsoft.Compute",
+		"publicKeys": [{
+			"keyData":"publickey1",
+			"path": "/home/rancher/.ssh/authorized_keys"
+		}],
+		"publisher": "",
+		"resourceGroupName": "rancheros",
+		"sku": "Enterprise",
+		"subscriptionId": "xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx",
+		"tags": "",
+		"version": "",
+		"vmId": "453945c8-3923-4366-b2d3-ea4c80e9b70e",
+		"vmScaleSetName": "",
+		"vmSize": "Standard_A1",
+		"zone": ""
+	},
+	"network": {
+		"interface": [{
+			"ipv4": {
+				"ipAddress": [{
+					"privateIpAddress": "192.168.1.2",
+					"publicIpAddress": "5.6.7.8"
+				}],
+				"subnet": [{
+					"address": "192.168.1.0",
+					"prefix": "24"
+				}]
+			},
+			"ipv6": {
+				"ipAddress": []
+			},
+			"macAddress": "002248020E1E"
+		}]
+	}
+}
+`,
+			},
+			expect: datasource.Metadata{
+				PrivateIPv4: net.ParseIP("192.168.1.2"),
+				PublicIPv4:  net.ParseIP("5.6.7.8"),
+				SSHPublicKeys: map[string]string{
+					"0": "publickey1",
+				},
+				Hostname: "rancheros",
+			},
+		},
+	} {
+		service := &MetadataService{
+			Service: metadata.Service{
+				Root:   tt.root,
+				Client: &test.HTTPClient{Resources: tt.resources, Err: tt.clientErr},
+			},
+		}
+		metadata, err := service.FetchMetadata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): \nwant %#v,\n got %#v", tt.resources, tt.expectErr, err)
+		}
+		if !reflect.DeepEqual(tt.expect, metadata) {
+			t.Fatalf("bad fetch (%q): \nwant %#v,\n got %#v", tt.resources, tt.expect, metadata)
+		}
+	}
+}
+
+func TestFetchUserdata(t *testing.T) {
+	for _, tt := range []struct {
+		root         string
+		userdataPath string
+		resources    map[string]string
+		userdata     []byte
+		clientErr    error
+		expectErr    error
+	}{
+		{
+			root: "/metadata/",
+			resources: map[string]string{
+				"/metadata/instance/compute/customData?api-version=2019-02-01&format=text": "I2Nsb3VkLWNvbmZpZwpob3N0bmFtZTogcmFuY2hlcjE=",
+			},
+			userdata: []byte(`#cloud-config
+hostname: rancher1`),
+		},
+	} {
+		service := &MetadataService{
+			Service: metadata.Service{
+				Root:   tt.root,
+				Client: &test.HTTPClient{Resources: tt.resources, Err: tt.clientErr},
+			},
+		}
+		data, err := service.FetchUserdata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): want %q, got %q", tt.resources, tt.expectErr, err)
+		}
+		if !bytes.Equal(data, tt.userdata) {
+			t.Fatalf("bad userdata (%q): want %q, got %q", tt.resources, tt.userdata, data)
+		}
+	}
+}
+
+func Error(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

config/cloudinit/datasource/metadata/cloudstack/metadata.go

@@ -0,0 +1,112 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudstack
+
+import (
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/config/cloudinit/pkg"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/netconf"
+)
+
+const (
+	apiVersion   = "latest/"
+	userdataPath = apiVersion + "user-data"
+	metadataPath = apiVersion + "meta-data/"
+
+	serverIdentifier = "dhcp_server_identifier"
+)
+
+type MetadataService struct {
+	metadata.Service
+}
+
+func NewDatasource(root string) []*MetadataService {
+	roots := make([]string, 0, 5)
+
+	if root == "" {
+		if links, err := netconf.GetValidLinkList(); err == nil {
+			log.Infof("Checking to see if a cloudstack server-identifier is available")
+			for _, link := range links {
+				linkName := link.Attrs().Name
+				log.Infof("searching for cloudstack server %s on %s", serverIdentifier, linkName)
+				lease := netconf.GetDhcpLease(linkName)
+				if server, ok := lease[serverIdentifier]; ok {
+					log.Infof("found cloudstack server '%s'", server)
+					server = "http://" + server + "/"
+					roots = append(roots, server)
+				}
+			}
+		} else {
+			log.Errorf("error getting LinkList: %s", err)
+		}
+	} else {
+		roots = append(roots, root)
+	}
+
+	sources := make([]*MetadataService, 0, len(roots))
+	for _, server := range roots {
+		datasource := metadata.NewDatasourceWithCheckPath(server, apiVersion, metadataPath, userdataPath, metadataPath, nil)
+		sources = append(sources, &MetadataService{datasource})
+	}
+	return sources
+}
+
+func (ms MetadataService) AvailabilityChanges() bool {
+	// TODO: if it can't find the network, maybe we can start it?
+	return false
+}
+
+func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
+	metadata := datasource.Metadata{}
+
+	if sshKeys, err := ms.FetchAttributes("public-keys"); err == nil {
+		metadata.SSHPublicKeys = map[string]string{}
+		for i, sshkey := range sshKeys {
+			log.Printf("Found SSH key %d", i)
+			metadata.SSHPublicKeys[strconv.Itoa(i)] = sshkey
+		}
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+
+	if hostname, err := ms.FetchAttribute("local-hostname"); err == nil {
+		metadata.Hostname = strings.Split(hostname, " ")[0]
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+
+	if localAddr, err := ms.FetchAttribute("local-ipv4"); err == nil {
+		metadata.PrivateIPv4 = net.ParseIP(localAddr)
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+	if publicAddr, err := ms.FetchAttribute("public-ipv4"); err == nil {
+		metadata.PublicIPv4 = net.ParseIP(publicAddr)
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+
+	return metadata, nil
+}
+
+func (ms MetadataService) Type() string {
+	return "cloudstack-metadata-service"
+}

config/cloudinit/datasource/metadata/cloudstack/metadata_test.go

@@ -0,0 +1,102 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudstack
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"testing"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
+	"github.com/rancher/os/config/cloudinit/pkg"
+)
+
+func TestType(t *testing.T) {
+	want := "cloudstack-metadata-service"
+	if kind := (MetadataService{}).Type(); kind != want {
+		t.Fatalf("bad type: want %q, got %q", want, kind)
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root         string
+		metadataPath string
+		resources    map[string]string
+		expect       datasource.Metadata
+		clientErr    error
+		expectErr    error
+	}{
+		{
+			root:         "/",
+			metadataPath: "latest/meta-data/",
+			resources: map[string]string{
+				"/latest/meta-data/local-hostname": "host",
+				"/latest/meta-data/local-ipv4":     "1.2.3.4",
+				"/latest/meta-data/public-ipv4":    "5.6.7.8",
+				"/latest/meta-data/public-keys":    "key\n",
+			},
+			expect: datasource.Metadata{
+				Hostname:      "host",
+				PrivateIPv4:   net.ParseIP("1.2.3.4"),
+				PublicIPv4:    net.ParseIP("5.6.7.8"),
+				SSHPublicKeys: map[string]string{"0": "key"},
+			},
+		},
+		{
+			root:         "/",
+			metadataPath: "latest/meta-data/",
+			resources: map[string]string{
+				"/latest/meta-data/local-hostname": "host domain another_domain",
+				"/latest/meta-data/local-ipv4":     "21.2.3.4",
+				"/latest/meta-data/public-ipv4":    "25.6.7.8",
+				"/latest/meta-data/public-keys":    "key\n",
+			},
+			expect: datasource.Metadata{
+				Hostname:      "host",
+				PrivateIPv4:   net.ParseIP("21.2.3.4"),
+				PublicIPv4:    net.ParseIP("25.6.7.8"),
+				SSHPublicKeys: map[string]string{"0": "key"},
+			},
+		},
+		{
+			clientErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")},
+			expectErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")},
+		},
+	} {
+		service := &MetadataService{metadata.Service{
+			Root:         tt.root,
+			Client:       &test.HTTPClient{Resources: tt.resources, Err: tt.clientErr},
+			MetadataPath: tt.metadataPath,
+		}}
+		metadata, err := service.FetchMetadata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): \nwant %q, \ngot %q\n", tt.resources, tt.expectErr, err)
+		}
+		if !reflect.DeepEqual(tt.expect, metadata) {
+			t.Fatalf("bad fetch (%q): \nwant %#v, \ngot %#v\n", tt.resources, tt.expect, metadata)
+		}
+	}
+}
+
+func Error(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

config/cloudinit/datasource/metadata/digitalocean/metadata.go

@@ -17,14 +17,12 @@ package digitalocean
 import (
 	"encoding/json"
 	"fmt"
-	"strconv"
-
-	"github.com/rancher/os/netconf"
-
 	"net"
+	"strconv"
 
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/pkg/netconf"
 )
 
 const (

config/cloudinit/datasource/metadata/digitalocean/metadata_test.go

@@ -20,12 +20,11 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/rancher/os/netconf"
-
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
 	"github.com/rancher/os/config/cloudinit/pkg"
+	"github.com/rancher/os/pkg/netconf"
 )
 
 func TestType(t *testing.T) {

config/cloudinit/datasource/metadata/ec2/metadata.go

@@ -15,25 +15,29 @@
 package ec2
 
 import (
-	"bufio"
-	"bytes"
 	"fmt"
 	"log"
 	"net"
 	"strings"
 
-	"github.com/rancher/os/netconf"
-
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata"
 	"github.com/rancher/os/config/cloudinit/pkg"
+	"github.com/rancher/os/pkg/netconf"
 )
 
 const (
 	DefaultAddress = "http://169.254.169.254/"
 	apiVersion     = "latest/"
-	userdataPath   = apiVersion + "user-data/"
+	userdataPath   = apiVersion + "user-data"
 	metadataPath   = apiVersion + "meta-data/"
+
+	defaultXVRootDisk   = "/dev/xvda"
+	defaultNVMeRootDisk = "/dev/nvme0n1"
+)
+
+var (
+	nvmeInstanceTypes = []string{"c5", "c5d", "i3.metal", "m5", "m5d", "r5", "r5d", "t3", "z1d"}
 )
 
 type MetadataService struct {
@@ -57,7 +61,7 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 	metadata := datasource.Metadata{}
 	metadata.NetworkConfig = netconf.NetworkConfig{}
 
-	if keynames, err := ms.fetchAttributes("public-keys"); err == nil {
+	if keynames, err := ms.FetchAttributes("public-keys"); err == nil {
 		keyIDs := make(map[string]string)
 		for _, keyname := range keynames {
 			tokens := strings.SplitN(keyname, "=", 2)
@@ -69,7 +73,7 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 
 		metadata.SSHPublicKeys = map[string]string{}
 		for name, id := range keyIDs {
-			sshkey, err := ms.fetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", id))
+			sshkey, err := ms.FetchAttribute(fmt.Sprintf("public-keys/%s/openssh-key", id))
 			if err != nil {
 				return metadata, err
 			}
@@ -80,44 +84,44 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 		return metadata, err
 	}
 
-	if hostname, err := ms.fetchAttribute("hostname"); err == nil {
+	if hostname, err := ms.FetchAttribute("hostname"); err == nil {
 		metadata.Hostname = strings.Split(hostname, " ")[0]
 	} else if _, ok := err.(pkg.ErrNotFound); !ok {
 		return metadata, err
 	}
 
 	// TODO: these are only on the first interface - it looks like you can have as many as you need...
-	if localAddr, err := ms.fetchAttribute("local-ipv4"); err == nil {
+	if localAddr, err := ms.FetchAttribute("local-ipv4"); err == nil {
 		metadata.PrivateIPv4 = net.ParseIP(localAddr)
 	} else if _, ok := err.(pkg.ErrNotFound); !ok {
 		return metadata, err
 	}
-	if publicAddr, err := ms.fetchAttribute("public-ipv4"); err == nil {
+	if publicAddr, err := ms.FetchAttribute("public-ipv4"); err == nil {
 		metadata.PublicIPv4 = net.ParseIP(publicAddr)
 	} else if _, ok := err.(pkg.ErrNotFound); !ok {
 		return metadata, err
 	}
 
 	metadata.NetworkConfig.Interfaces = make(map[string]netconf.InterfaceConfig)
-	if macs, err := ms.fetchAttributes("network/interfaces/macs"); err != nil {
+	if macs, err := ms.FetchAttributes("network/interfaces/macs"); err != nil {
 		for _, mac := range macs {
-			if deviceNumber, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/device-number", mac)); err != nil {
+			if deviceNumber, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/device-number", mac)); err != nil {
 				network := netconf.InterfaceConfig{
 					DHCP: true,
 				}
 				/* Looks like we must use DHCP for aws
 				// private ipv4
-				if subnetCidrBlock, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv4-cidr-block", mac)); err != nil {
+				if subnetCidrBlock, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv4-cidr-block", mac)); err != nil {
 					cidr := strings.Split(subnetCidrBlock, "/")
-					if localAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac)); err != nil {
+					if localAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac)); err != nil {
 						for _, addr := range localAddr {
 							network.Addresses = append(network.Addresses, addr+"/"+cidr[1])
 						}
 					}
 				}
 				// ipv6
-				if localAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac)); err != nil {
-					if subnetCidrBlock, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-block", mac)); err != nil {
+				if localAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac)); err != nil {
+					if subnetCidrBlock, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-block", mac)); err != nil {
 						for i, addr := range localAddr {
 							cidr := strings.Split(subnetCidrBlock[i], "/")
 							network.Addresses = append(network.Addresses, addr+"/"+cidr[1])
@@ -126,8 +130,8 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 				}
 				*/
 				// disabled - it looks to me like you don't actually put the public IP on the eth device
-				/*				if publicAddr, err := ms.fetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/public-ipv4s", mac)); err != nil {
-									if vpcCidrBlock, err := ms.fetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv4-cidr-block", mac)); err != nil {
+				/*				if publicAddr, err := ms.FetchAttributes(fmt.Sprintf("network/interfaces/macs/%s/public-ipv4s", mac)); err != nil {
+									if vpcCidrBlock, err := ms.FetchAttribute(fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv4-cidr-block", mac)); err != nil {
 										cidr := strings.Split(vpcCidrBlock, "/")
 										network.Addresses = append(network.Addresses, publicAddr+"/"+cidr[1])
 									}
@@ -139,31 +143,23 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 		}
 	}
 
+	// With C5 and M5 instances, EBS volumes are exposed as NVMe block devices.
+	// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html
+	metadata.RootDisk = defaultXVRootDisk
+	if instanceType, err := ms.FetchAttribute("instance-type"); err == nil {
+		for _, nvmeType := range nvmeInstanceTypes {
+			if strings.HasPrefix(instanceType, nvmeType) {
+				metadata.RootDisk = defaultNVMeRootDisk
+				break
+			}
+		}
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+
 	return metadata, nil
 }
 
 func (ms MetadataService) Type() string {
 	return "ec2-metadata-service"
 }
-
-func (ms MetadataService) fetchAttributes(key string) ([]string, error) {
-	url := ms.MetadataURL() + key
-	resp, err := ms.FetchData(url)
-	if err != nil {
-		return nil, err
-	}
-	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
-	data := make([]string, 0)
-	for scanner.Scan() {
-		data = append(data, scanner.Text())
-	}
-	return data, scanner.Err()
-}
-
-func (ms MetadataService) fetchAttribute(key string) (string, error) {
-	attrs, err := ms.fetchAttributes(key)
-	if err == nil && len(attrs) > 0 {
-		return attrs[0], nil
-	}
-	return "", err
-}

config/cloudinit/datasource/metadata/ec2/metadata_test.go

@@ -24,7 +24,7 @@ import (
 	"github.com/rancher/os/config/cloudinit/datasource/metadata"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
 	"github.com/rancher/os/config/cloudinit/pkg"
-	"github.com/rancher/os/netconf"
+	"github.com/rancher/os/pkg/netconf"
 )
 
 func TestType(t *testing.T) {
@@ -34,114 +34,6 @@ func TestType(t *testing.T) {
 	}
 }
 
-func TestFetchAttributes(t *testing.T) {
-	for _, s := range []struct {
-		resources map[string]string
-		err       error
-		tests     []struct {
-			path string
-			val  []string
-		}
-	}{
-		{
-			resources: map[string]string{
-				"/":      "a\nb\nc/",
-				"/c/":    "d\ne/",
-				"/c/e/":  "f",
-				"/a":     "1",
-				"/b":     "2",
-				"/c/d":   "3",
-				"/c/e/f": "4",
-			},
-			tests: []struct {
-				path string
-				val  []string
-			}{
-				{"/", []string{"a", "b", "c/"}},
-				{"/b", []string{"2"}},
-				{"/c/d", []string{"3"}},
-				{"/c/e/", []string{"f"}},
-			},
-		},
-		{
-			err: fmt.Errorf("test error"),
-			tests: []struct {
-				path string
-				val  []string
-			}{
-				{"", nil},
-			},
-		},
-	} {
-		service := MetadataService{metadata.Service{
-			Client: &test.HTTPClient{Resources: s.resources, Err: s.err},
-		}}
-		for _, tt := range s.tests {
-			attrs, err := service.fetchAttributes(tt.path)
-			if err != s.err {
-				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
-			}
-			if !reflect.DeepEqual(attrs, tt.val) {
-				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs)
-			}
-		}
-	}
-}
-
-func TestFetchAttribute(t *testing.T) {
-	for _, s := range []struct {
-		resources map[string]string
-		err       error
-		tests     []struct {
-			path string
-			val  string
-		}
-	}{
-		{
-			resources: map[string]string{
-				"/":      "a\nb\nc/",
-				"/c/":    "d\ne/",
-				"/c/e/":  "f",
-				"/a":     "1",
-				"/b":     "2",
-				"/c/d":   "3",
-				"/c/e/f": "4",
-			},
-			tests: []struct {
-				path string
-				val  string
-			}{
-				{"/a", "1"},
-				{"/b", "2"},
-				{"/c/d", "3"},
-				{"/c/e/f", "4"},
-			},
-		},
-		{
-			err: fmt.Errorf("test error"),
-			tests: []struct {
-				path string
-				val  string
-			}{
-				{"", ""},
-			},
-		},
-	} {
-		service := MetadataService{metadata.Service{
-			Client: &test.HTTPClient{Resources: s.resources, Err: s.err},
-		}}
-		for _, tt := range s.tests {
-			attr, err := service.fetchAttribute(tt.path)
-			if err != s.err {
-				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
-			}
-			if attr != tt.val {
-				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr)
-			}
-		}
-	}
-}
-
 func TestFetchMetadata(t *testing.T) {
 	for _, tt := range []struct {
 		root         string
@@ -175,6 +67,7 @@ func TestFetchMetadata(t *testing.T) {
 				PrivateIPv4:   net.ParseIP("1.2.3.4"),
 				PublicIPv4:    net.ParseIP("5.6.7.8"),
 				SSHPublicKeys: map[string]string{"test1": "key"},
+				RootDisk:      "/dev/xvda",
 				NetworkConfig: netconf.NetworkConfig{
 					Interfaces: map[string]netconf.InterfaceConfig{
 					/*			"eth0": netconf.InterfaceConfig{
@@ -197,12 +90,14 @@ func TestFetchMetadata(t *testing.T) {
 				"/2009-04-04/meta-data/public-keys":               "0=test1\n",
 				"/2009-04-04/meta-data/public-keys/0":             "openssh-key",
 				"/2009-04-04/meta-data/public-keys/0/openssh-key": "key",
+				"/2009-04-04/meta-data/instance-type":             "m5.large",
 			},
 			expect: datasource.Metadata{
 				Hostname:      "host",
 				PrivateIPv4:   net.ParseIP("21.2.3.4"),
 				PublicIPv4:    net.ParseIP("25.6.7.8"),
 				SSHPublicKeys: map[string]string{"test1": "key"},
+				RootDisk:      "/dev/nvme0n1",
 				NetworkConfig: netconf.NetworkConfig{
 					Interfaces: map[string]netconf.InterfaceConfig{
 					/*						"eth0": netconf.InterfaceConfig{

config/cloudinit/datasource/metadata/exoscale/metadata.go

@@ -0,0 +1,92 @@
+package exoscale
+
+import (
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/config/cloudinit/pkg"
+	"github.com/rancher/os/pkg/log"
+)
+
+const (
+	defaultAddress = "http://169.254.169.254/"
+	apiVersion     = "1.0/"
+	userdataPath   = apiVersion + "user-data"
+	metadataPath   = apiVersion + "meta-data/"
+)
+
+type MetadataService struct {
+	metadata.Service
+}
+
+func NewDatasource(root string) *MetadataService {
+	if root == "" {
+		root = defaultAddress
+	}
+
+	return &MetadataService{
+		metadata.NewDatasourceWithCheckPath(
+			root,
+			apiVersion,
+			metadataPath,
+			userdataPath,
+			metadataPath,
+			nil,
+		),
+	}
+}
+
+func (ms MetadataService) IsAvailable() bool {
+	checkURL := ms.Root + ms.IsAvailableCheckPath
+	var err error
+	_, err = ms.Client.GetRetry(checkURL)
+	if err != nil {
+		log.Errorf("%s: %s (lastError: %v)", "IsAvailable", checkURL, err)
+	}
+	return (err == nil)
+}
+
+func (ms MetadataService) AvailabilityChanges() bool {
+	// TODO: if it can't find the network, maybe we can start it?
+	return false
+}
+
+func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
+	metadata := datasource.Metadata{}
+
+	if sshKeys, err := ms.FetchAttributes("public-keys"); err == nil {
+		metadata.SSHPublicKeys = map[string]string{}
+		for i, sshkey := range sshKeys {
+			log.Printf("Found SSH key %d", i)
+			metadata.SSHPublicKeys[strconv.Itoa(i)] = sshkey
+		}
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+
+	if hostname, err := ms.FetchAttribute("local-hostname"); err == nil {
+		metadata.Hostname = strings.Split(hostname, " ")[0]
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+
+	if localAddr, err := ms.FetchAttribute("local-ipv4"); err == nil {
+		metadata.PrivateIPv4 = net.ParseIP(localAddr)
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+	if publicAddr, err := ms.FetchAttribute("public-ipv4"); err == nil {
+		metadata.PublicIPv4 = net.ParseIP(publicAddr)
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return metadata, err
+	}
+
+	return metadata, nil
+}
+
+func (ms MetadataService) Type() string {
+	return "exoscale-metadata-service"
+}

config/cloudinit/datasource/metadata/exoscale/metadata_test.go

@@ -0,0 +1,88 @@
+package exoscale
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"testing"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata"
+	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
+	"github.com/rancher/os/config/cloudinit/pkg"
+)
+
+func TestType(t *testing.T) {
+	want := "exoscale-metadata-service"
+	if kind := (MetadataService{}).Type(); kind != want {
+		t.Fatalf("bad type: want %q, got %q", want, kind)
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root         string
+		metadataPath string
+		resources    map[string]string
+		expect       datasource.Metadata
+		clientErr    error
+		expectErr    error
+	}{
+		{
+			root:         "/",
+			metadataPath: "1.0/meta-data/",
+			resources: map[string]string{
+				"/1.0/meta-data/local-hostname": "host",
+				"/1.0/meta-data/local-ipv4":     "1.2.3.4",
+				"/1.0/meta-data/public-ipv4":    "5.6.7.8",
+				"/1.0/meta-data/public-keys":    "key\n",
+			},
+			expect: datasource.Metadata{
+				Hostname:      "host",
+				PrivateIPv4:   net.ParseIP("1.2.3.4"),
+				PublicIPv4:    net.ParseIP("5.6.7.8"),
+				SSHPublicKeys: map[string]string{"0": "key"},
+			},
+		},
+		{
+			root:         "/",
+			metadataPath: "1.0/meta-data/",
+			resources: map[string]string{
+				"/1.0/meta-data/local-hostname": "host domain another_domain",
+				"/1.0/meta-data/local-ipv4":     "21.2.3.4",
+				"/1.0/meta-data/public-ipv4":    "25.6.7.8",
+				"/1.0/meta-data/public-keys":    "key\n",
+			},
+			expect: datasource.Metadata{
+				Hostname:      "host",
+				PrivateIPv4:   net.ParseIP("21.2.3.4"),
+				PublicIPv4:    net.ParseIP("25.6.7.8"),
+				SSHPublicKeys: map[string]string{"0": "key"},
+			},
+		},
+		{
+			clientErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")},
+			expectErr: pkg.ErrTimeout{Err: fmt.Errorf("test error")},
+		},
+	} {
+		service := &MetadataService{metadata.Service{
+			Root:         tt.root,
+			Client:       &test.HTTPClient{Resources: tt.resources, Err: tt.clientErr},
+			MetadataPath: tt.metadataPath,
+		}}
+		metadata, err := service.FetchMetadata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): \nwant %q, \ngot %q\n", tt.resources, tt.expectErr, err)
+		}
+		if !reflect.DeepEqual(tt.expect, metadata) {
+			t.Fatalf("bad fetch (%q): \nwant %#v, \ngot %#v\n", tt.resources, tt.expect, metadata)
+		}
+	}
+}
+
+func Error(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

config/cloudinit/datasource/metadata/gce/metadata.go

@@ -21,8 +21,6 @@ import (
 	"strconv"
 	"strings"
 
-	//"github.com/rancher/os/netconf"
-
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata"
 )
@@ -59,14 +57,19 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 		return datasource.Metadata{}, err
 	}
 
-	projectSSHKeys, err := ms.fetchString("project/attributes/sshKeys")
+	projectSSHKeys, err := ms.fetchString("project/attributes/ssh-keys")
 	if err != nil {
 		return datasource.Metadata{}, err
 	}
-	instanceSSHKeys, err := ms.fetchString("instance/attributes/sshKeys")
+	instanceSSHKeys, err := ms.fetchString("instance/attributes/ssh-keys")
 	if err != nil {
 		return datasource.Metadata{}, err
 	}
+	blockProjectSSHKeys, err := ms.fetchString("instance/attributes/block-project-ssh-keys")
+	if err != nil {
+		return datasource.Metadata{}, err
+	}
+
 	md := datasource.Metadata{
 		PublicIPv4:    public,
 		PrivateIPv4:   local,
@@ -91,8 +94,10 @@ func (ms MetadataService) FetchMetadata() (datasource.Metadata, error) {
 		md.NetworkConfig.Interfaces["eth0"] = network
 	}
 	*/
-
-	keyStrings := strings.Split(projectSSHKeys+"\n"+instanceSSHKeys, "\n")
+	keyStrings := strings.Split(instanceSSHKeys, "\n")
+	if blockProjectSSHKeys != "true" {
+		keyStrings = append(keyStrings, strings.Split(projectSSHKeys, "\n")...)
+	}
 
 	i := 0
 	for _, keyString := range keyStrings {

config/cloudinit/datasource/metadata/gce/metadata_test.go

@@ -20,8 +20,6 @@ import (
 	"reflect"
 	"testing"
 
-	//"github.com/rancher/os/netconf"
-
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"

config/cloudinit/datasource/metadata/metadata.go

@@ -15,34 +15,45 @@
 package metadata
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"net/http"
 	"strings"
 
 	"github.com/rancher/os/config/cloudinit/pkg"
-	"github.com/rancher/os/log"
+	"github.com/rancher/os/pkg/log"
 )
 
 type Service struct {
-	Root         string
-	Client       pkg.Getter
-	APIVersion   string
-	UserdataPath string
-	MetadataPath string
-	lastError    error
+	Root                 string
+	Client               pkg.Getter
+	APIVersion           string
+	IsAvailableCheckPath string
+	UserdataPath         string
+	MetadataPath         string
+	lastError            error
 }
 
+// NewDatasource creates as HTTP based cloud-data service with the corresponding paths for the user-data and meta-data.
+// To check the available in IsAvailable, the apiVersion is used as path.
 func NewDatasource(root, apiVersion, userdataPath, metadataPath string, header http.Header) Service {
+	return NewDatasourceWithCheckPath(root, apiVersion, apiVersion, userdataPath, metadataPath, header)
+}
+
+// NewDatasourceWithCheckPath creates as HTTP based cloud-data service with the corresponding paths for the user-data and meta-data.
+func NewDatasourceWithCheckPath(root, apiVersion, isAvailableCheckPath, userdataPath, metadataPath string, header http.Header) Service {
 	if !strings.HasSuffix(root, "/") {
 		root += "/"
 	}
-	return Service{root, pkg.NewHTTPClientHeader(header), apiVersion, userdataPath, metadataPath, nil}
+	return Service{root, pkg.NewHTTPClientHeader(header), apiVersion, isAvailableCheckPath, userdataPath, metadataPath, nil}
 }
 
 func (ms Service) IsAvailable() bool {
-	_, ms.lastError = ms.Client.Get(ms.Root + ms.APIVersion)
+	checkURL := ms.Root + ms.IsAvailableCheckPath
+	_, ms.lastError = ms.Client.Get(checkURL)
 	if ms.lastError != nil {
-		log.Errorf("%s: %s (lastError: %s)", "IsAvailable", ms.Root+":"+ms.UserdataPath, ms.lastError)
+		log.Errorf("%s: %s (lastError: %v)", "IsAvailable", checkURL, ms.lastError)
 	}
 	return (ms.lastError == nil)
 }
@@ -52,7 +63,7 @@ func (ms *Service) Finish() error {
 }
 
 func (ms *Service) String() string {
-	return fmt.Sprintf("%s: %s (lastError: %s)", "metadata", ms.Root+ms.UserdataPath, ms.lastError)
+	return fmt.Sprintf("%s: %s (lastError: %v)", "metadata", ms.UserdataURL(), ms.lastError)
 }
 
 func (ms Service) AvailabilityChanges() bool {
@@ -84,3 +95,25 @@ func (ms Service) MetadataURL() string {
 func (ms Service) UserdataURL() string {
 	return (ms.Root + ms.UserdataPath)
 }
+
+func (ms Service) FetchAttributes(key string) ([]string, error) {
+	url := ms.MetadataURL() + key
+	resp, err := ms.FetchData(url)
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
+	data := make([]string, 0)
+	for scanner.Scan() {
+		data = append(data, scanner.Text())
+	}
+	return data, scanner.Err()
+}
+
+func (ms Service) FetchAttribute(key string) (string, error) {
+	attrs, err := ms.FetchAttributes(key)
+	if err == nil && len(attrs) > 0 {
+		return attrs[0], nil
+	}
+	return "", err
+}

config/cloudinit/datasource/metadata/metadata_test.go

@@ -17,6 +17,7 @@ package metadata
 import (
 	"bytes"
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/rancher/os/config/cloudinit/datasource/metadata/test"
@@ -32,14 +33,14 @@ func TestAvailabilityChanges(t *testing.T) {
 
 func TestIsAvailable(t *testing.T) {
 	for _, tt := range []struct {
-		root       string
-		apiVersion string
-		resources  map[string]string
-		expect     bool
+		root      string
+		checkPath string
+		resources map[string]string
+		expect    bool
 	}{
 		{
-			root:       "/",
-			apiVersion: "2009-04-04",
+			root:      "/",
+			checkPath: "2009-04-04",
 			resources: map[string]string{
 				"/2009-04-04": "",
 			},
@@ -52,9 +53,9 @@ func TestIsAvailable(t *testing.T) {
 		},
 	} {
 		service := &Service{
-			Root:       tt.root,
-			Client:     &test.HTTPClient{Resources: tt.resources, Err: nil},
-			APIVersion: tt.apiVersion,
+			Root:                 tt.root,
+			Client:               &test.HTTPClient{Resources: tt.resources, Err: nil},
+			IsAvailableCheckPath: tt.checkPath,
 		}
 		if a := service.IsAvailable(); a != tt.expect {
 			t.Fatalf("bad isAvailable (%q): want %t, got %t", tt.resources, tt.expect, a)
@@ -177,6 +178,114 @@ func TestNewDatasource(t *testing.T) {
 	}
 }
 
+func TestFetchAttributes(t *testing.T) {
+	for _, s := range []struct {
+		resources map[string]string
+		err       error
+		tests     []struct {
+			path string
+			val  []string
+		}
+	}{
+		{
+			resources: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"/", []string{"a", "b", "c/"}},
+				{"/b", []string{"2"}},
+				{"/c/d", []string{"3"}},
+				{"/c/e/", []string{"f"}},
+			},
+		},
+		{
+			err: fmt.Errorf("test error"),
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"", nil},
+			},
+		},
+	} {
+		service := &Service{
+			Client: &test.HTTPClient{Resources: s.resources, Err: s.err},
+		}
+		for _, tt := range s.tests {
+			attrs, err := service.FetchAttributes(tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
+			}
+			if !reflect.DeepEqual(attrs, tt.val) {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs)
+			}
+		}
+	}
+}
+
+func TestFetchAttribute(t *testing.T) {
+	for _, s := range []struct {
+		resources map[string]string
+		err       error
+		tests     []struct {
+			path string
+			val  string
+		}
+	}{
+		{
+			resources: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"/a", "1"},
+				{"/b", "2"},
+				{"/c/d", "3"},
+				{"/c/e/f", "4"},
+			},
+		},
+		{
+			err: fmt.Errorf("test error"),
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"", ""},
+			},
+		},
+	} {
+		service := &Service{
+			Client: &test.HTTPClient{Resources: s.resources, Err: s.err},
+		}
+		for _, tt := range s.tests {
+			attr, err := service.FetchAttribute(tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
+			}
+			if attr != tt.val {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr)
+			}
+		}
+	}
+}
+
 func Error(err error) string {
 	if err != nil {
 		return err.Error()

config/cloudinit/datasource/metadata/packet/metadata.go

@@ -15,16 +15,14 @@
 package packet
 
 import (
-	"bytes"
 	"fmt"
-	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/datasource/metadata"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/netconf"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/netconf"
 
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 	packetMetadata "github.com/packethost/packngo/metadata"
@@ -50,8 +48,7 @@ func NewDatasource(root string) *MetadataService {
 }
 
 func (ms *MetadataService) FetchMetadata() (metadata datasource.Metadata, err error) {
-	c := packetMetadata.NewClient(http.DefaultClient)
-	m, err := c.Metadata.Get()
+	m, err := packetMetadata.GetMetadata()
 	if err != nil {
 		log.Errorf("Failed to get Packet metadata: %v", err)
 		return
@@ -72,24 +69,24 @@ func (ms *MetadataService) FetchMetadata() (metadata datasource.Metadata, err er
 		Interfaces: map[string]netconf.InterfaceConfig{},
 	}
 	for _, iface := range m.Network.Interfaces {
-		netCfg.Interfaces["mac="+iface.Mac] = netconf.InterfaceConfig{
+		netCfg.Interfaces["mac="+iface.MAC] = netconf.InterfaceConfig{
 			Bond: "bond0",
 		}
 	}
 	for _, addr := range m.Network.Addresses {
-		bondCfg.Addresses = append(bondCfg.Addresses, fmt.Sprintf("%s/%d", addr.Address, addr.Cidr))
-		if addr.Gateway != "" {
-			if addr.AddressFamily == 4 {
+		bondCfg.Addresses = append(bondCfg.Addresses, fmt.Sprintf("%s/%d", addr.Address, addr.NetworkBits))
+		if addr.Gateway != nil && len(addr.Gateway) > 0 {
+			if addr.Family == packetMetadata.IPv4 {
 				if addr.Public {
-					bondCfg.Gateway = addr.Gateway
+					bondCfg.Gateway = addr.Gateway.String()
 				}
 			} else {
-				bondCfg.GatewayIpv6 = addr.Gateway
+				bondCfg.GatewayIpv6 = addr.Gateway.String()
 			}
 		}
 
-		if addr.AddressFamily == 4 && strings.HasPrefix(addr.Gateway, "10.") {
-			bondCfg.PostUp = append(bondCfg.PostUp, "ip route add 10.0.0.0/8 via "+addr.Gateway)
+		if addr.Family == packetMetadata.IPv4 && strings.HasPrefix(addr.Gateway.String(), "10.") {
+			bondCfg.PostUp = append(bondCfg.PostUp, "ip route add 10.0.0.0/8 via "+addr.Gateway.String())
 		}
 	}
 
@@ -124,7 +121,7 @@ func (ms *MetadataService) FetchMetadata() (metadata datasource.Metadata, err er
 	*/
 	metadata.Hostname = m.Hostname
 	metadata.SSHPublicKeys = map[string]string{}
-	for i, key := range m.SshKeys {
+	for i, key := range m.SSHKeys {
 		metadata.SSHPublicKeys[strconv.Itoa(i)] = key
 	}
 
@@ -132,9 +129,9 @@ func (ms *MetadataService) FetchMetadata() (metadata datasource.Metadata, err er
 
 	// This is not really the right place - perhaps we should add a call-home function in each datasource to be called after the network is applied
 	//(see the original in cmd/cloudsave/packet)
-	if _, err = http.Post(m.PhoneHomeURL, "application/json", bytes.NewReader([]byte{})); err != nil {
-		log.Errorf("Failed to post to Packet phone home URL: %v", err)
-	}
+	//if _, err = http.Post(m.PhoneHomeURL, "application/json", bytes.NewReader([]byte{})); err != nil {
+	//log.Errorf("Failed to post to Packet phone home URL: %v", err)
+	//}
 
 	return
 }

config/cloudinit/datasource/proccmdline/proc_cmdline.go

@@ -20,10 +20,9 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"github.com/rancher/os/log"
-
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/pkg"
+	"github.com/rancher/os/pkg/log"
 )
 
 const (
@@ -57,7 +56,7 @@ func (c *ProcCmdline) Finish() error {
 }
 
 func (c *ProcCmdline) String() string {
-	return fmt.Sprintf("%s: %s (lastError: %s)", c.Type(), c.Location, c.lastError)
+	return fmt.Sprintf("%s: %s (lastError: %v)", c.Type(), c.Location, c.lastError)
 }
 
 func (c *ProcCmdline) AvailabilityChanges() bool {

config/cloudinit/datasource/proxmox/proxmox.go

@@ -0,0 +1,114 @@
+package proxmox
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"syscall"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+
+	"github.com/docker/docker/pkg/mount"
+)
+
+const (
+	configDev           = "/dev/sr0"
+	configDevMountPoint = "/media/pve-config"
+)
+
+type Proxmox struct {
+	root                string
+	readFile            func(filename string) ([]byte, error)
+	lastError           error
+	availabilityChanges bool
+}
+
+func NewDataSource(root string) *Proxmox {
+	return &Proxmox{root, ioutil.ReadFile, nil, true}
+}
+
+func (pve *Proxmox) IsAvailable() bool {
+	if pve.root == configDevMountPoint {
+		pve.lastError = MountConfigDrive()
+		if pve.lastError != nil {
+			log.Error(pve.lastError)
+			pve.availabilityChanges = false
+			return false
+		}
+		defer pve.Finish()
+	}
+
+	_, pve.lastError = os.Stat(pve.root)
+	return !os.IsNotExist(pve.lastError)
+}
+
+func (pve *Proxmox) Finish() error {
+	return UnmountConfigDrive()
+}
+
+func (pve *Proxmox) String() string {
+	if pve.lastError != nil {
+		return fmt.Sprintf("%s: %s (lastError: %v)", pve.Type(), pve.root, pve.lastError)
+	}
+	return fmt.Sprintf("%s: %s", pve.Type(), pve.root)
+}
+
+func (pve *Proxmox) AvailabilityChanges() bool {
+	return pve.availabilityChanges
+}
+
+func (pve *Proxmox) ConfigRoot() string {
+	return pve.root
+}
+
+func (pve *Proxmox) FetchMetadata() (metadata datasource.Metadata, err error) {
+	return datasource.Metadata{}, nil
+}
+
+func (pve *Proxmox) FetchUserdata() ([]byte, error) {
+	return pve.tryReadFile(path.Join(pve.root, "user-data"))
+}
+
+func (pve *Proxmox) Type() string {
+	return "proxmox"
+}
+
+func (pve *Proxmox) tryReadFile(filename string) ([]byte, error) {
+	if pve.root == configDevMountPoint {
+		pve.lastError = MountConfigDrive()
+		if pve.lastError != nil {
+			log.Error(pve.lastError)
+			return nil, pve.lastError
+		}
+		defer pve.Finish()
+	}
+	log.Debugf("Attempting to read from %q\n", filename)
+	data, err := pve.readFile(filename)
+	if os.IsNotExist(err) {
+		err = nil
+	}
+	if err != nil {
+		log.Errorf("ERROR read cloud-config file(%s) - err: %q", filename, err)
+	}
+	return data, err
+}
+
+func MountConfigDrive() error {
+	if err := os.MkdirAll(configDevMountPoint, 700); err != nil {
+		return err
+	}
+
+	fsType, err := util.GetFsType(configDev)
+	if err != nil {
+		return err
+	}
+
+	return mount.Mount(configDev, configDevMountPoint, fsType, "ro")
+}
+
+func UnmountConfigDrive() error {
+	return syscall.Unmount(configDevMountPoint, 0)
+}

config/cloudinit/datasource/proxmox/proxmox_test.go

@@ -0,0 +1,77 @@
+package proxmox
+
+import (
+	"testing"
+
+	"github.com/rancher/os/config/cloudinit/datasource/test"
+)
+
+func TestFetchUserdata(t *testing.T) {
+	for _, tt := range []struct {
+		root     string
+		files    test.MockFilesystem
+		userdata string
+	}{
+		{
+			root:     "/",
+			files:    test.NewMockFilesystem(),
+			userdata: "",
+		},
+		{
+			root:     "/media/config",
+			files:    test.NewMockFilesystem(test.File{Path: "/media/config/user-data", Contents: "userdata"}),
+			userdata: "userdata",
+		},
+	} {
+		pve := Proxmox{tt.root, tt.files.ReadFile, nil, true}
+		userdata, err := pve.FetchUserdata()
+		if err != nil {
+			t.Fatalf("bad error for %+v: want %v, got %q", tt, nil, err)
+		}
+		if string(userdata) != tt.userdata {
+			t.Fatalf("bad userdata for %+v: want %q, got %q", tt, tt.userdata, userdata)
+		}
+	}
+}
+
+func TestConfigRoot(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		configRoot string
+	}{
+		{
+			root:       "/",
+			configRoot: "/",
+		},
+		{
+			root:       "/media/pve-config",
+			configRoot: "/media/pve-config",
+		},
+	} {
+		pve := Proxmox{tt.root, nil, nil, true}
+		if configRoot := pve.ConfigRoot(); configRoot != tt.configRoot {
+			t.Fatalf("bad config root for %q: want %q, got %q", tt, tt.configRoot, configRoot)
+		}
+	}
+}
+
+func TestNewDataSource(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		expectRoot string
+	}{
+		{
+			root:       "",
+			expectRoot: "",
+		},
+		{
+			root:       "/media/pve-config",
+			expectRoot: "/media/pve-config",
+		},
+	} {
+		service := NewDataSource(tt.root)
+		if service.root != tt.expectRoot {
+			t.Fatalf("bad root (%q): want %q, got %q", tt.root, tt.expectRoot, service.root)
+		}
+	}
+}

config/cloudinit/datasource/tftp/tftp.go

@@ -0,0 +1,83 @@
+package tftp
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/rancher/os/config/cloudinit/datasource"
+
+	"github.com/pin/tftp"
+)
+
+type Client interface {
+	Receive(filename string, mode string) (io.WriterTo, error)
+}
+
+type RemoteFile struct {
+	host      string
+	path      string
+	client    Client
+	stream    io.WriterTo
+	lastError error
+}
+
+func NewDatasource(hostAndPath string) *RemoteFile {
+	parts := strings.SplitN(hostAndPath, "/", 2)
+
+	if len(parts) < 2 {
+		return &RemoteFile{hostAndPath, "", nil, nil, nil}
+	}
+
+	host := parts[0]
+	if match, _ := regexp.MatchString(":[0-9]{2,5}$", host); !match {
+		// No port, using default port 69
+		host += ":69"
+	}
+
+	path := parts[1]
+	if client, lastError := tftp.NewClient(host); lastError == nil {
+		return &RemoteFile{host, path, client, nil, nil}
+	}
+
+	return &RemoteFile{host, path, nil, nil, nil}
+}
+
+func (f *RemoteFile) IsAvailable() bool {
+	f.stream, f.lastError = f.client.Receive(f.path, "octet")
+	return f.lastError == nil
+}
+
+func (f *RemoteFile) Finish() error {
+	return nil
+}
+
+func (f *RemoteFile) String() string {
+	return fmt.Sprintf("%s, host:%s, path:%s (lastError: %v)", f.Type(), f.host, f.path, f.lastError)
+}
+
+func (f *RemoteFile) AvailabilityChanges() bool {
+	return false
+}
+
+func (f *RemoteFile) ConfigRoot() string {
+	return ""
+}
+
+func (f *RemoteFile) FetchMetadata() (datasource.Metadata, error) {
+	return datasource.Metadata{}, nil
+}
+
+func (f *RemoteFile) FetchUserdata() ([]byte, error) {
+	var b bytes.Buffer
+
+	_, err := f.stream.WriteTo(&b)
+
+	return b.Bytes(), err
+}
+
+func (f *RemoteFile) Type() string {
+	return "tftp"
+}

config/cloudinit/datasource/tftp/tftp_test.go

@@ -0,0 +1,92 @@
+package tftp
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"testing"
+)
+
+type mockClient struct {
+}
+
+type mockReceiver struct {
+}
+
+func (r mockReceiver) WriteTo(w io.Writer) (n int64, err error) {
+	b := []byte("cloud-config file")
+	w.Write(b)
+	return int64(len(b)), nil
+}
+
+func (c mockClient) Receive(filename string, mode string) (io.WriterTo, error) {
+	if filename == "does-not-exist" {
+		return &mockReceiver{}, fmt.Errorf("does not exist")
+	}
+	return &mockReceiver{}, nil
+}
+
+var _ Client = (*mockClient)(nil)
+
+func TestNewDatasource(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		expectHost string
+		expectPath string
+	}{
+		{
+			root:       "127.0.0.1/test/file.yaml",
+			expectHost: "127.0.0.1:69",
+			expectPath: "test/file.yaml",
+		},
+		{
+			root:       "127.0.0.1/test/file.yaml",
+			expectHost: "127.0.0.1:69",
+			expectPath: "test/file.yaml",
+		},
+	} {
+		ds := NewDatasource(tt.root)
+		if ds.host != tt.expectHost || ds.path != tt.expectPath {
+			t.Fatalf("bad host or path (%q): want host=%s, got %s, path=%s, got %s", tt.root, tt.expectHost, ds.host, tt.expectPath, ds.path)
+		}
+	}
+}
+
+func TestIsAvailable(t *testing.T) {
+	for _, tt := range []struct {
+		remoteFile *RemoteFile
+		expect     bool
+	}{
+		{
+			remoteFile: &RemoteFile{"1.2.3.4", "test", &mockClient{}, nil, nil},
+			expect:     true,
+		},
+		{
+			remoteFile: &RemoteFile{"1.2.3.4", "does-not-exist", &mockClient{}, nil, nil},
+			expect:     false,
+		},
+	} {
+		if tt.remoteFile.IsAvailable() != tt.expect {
+			t.Fatalf("expected remote file %s to be %v", tt.remoteFile.path, tt.expect)
+		}
+	}
+}
+
+func TestFetchUserdata(t *testing.T) {
+	rf := &RemoteFile{"1.2.3.4", "test", &mockClient{}, &mockReceiver{}, nil}
+	b, _ := rf.FetchUserdata()
+
+	expect := []byte("cloud-config file")
+
+	if len(b) != len(expect) || !reflect.DeepEqual(b, expect) {
+		t.Fatalf("expected length of buffer to be %d was %d. Expected %s, got %s", len(expect), len(b), string(expect), string(b))
+	}
+}
+
+func TestType(t *testing.T) {
+	rf := &RemoteFile{"1.2.3.4", "test", &mockClient{}, nil, nil}
+
+	if rf.Type() != "tftp" {
+		t.Fatalf("expected remote file Type() to return %s got %s", "tftp", rf.Type())
+	}
+}

config/cloudinit/datasource/url/url.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/rancher/os/config/cloudinit/datasource"
 	"github.com/rancher/os/config/cloudinit/pkg"
+	"github.com/rancher/os/pkg/util/network"
 )
 
 type RemoteFile struct {
@@ -31,8 +32,9 @@ func NewDatasource(url string) *RemoteFile {
 }
 
 func (f *RemoteFile) IsAvailable() bool {
+	network.SetProxyEnvironmentVariables()
 	client := pkg.NewHTTPClient()
-	_, f.lastError = client.Get(f.url)
+	_, f.lastError = client.GetRetry(f.url)
 	return (f.lastError == nil)
 }
 
@@ -41,7 +43,7 @@ func (f *RemoteFile) Finish() error {
 }
 
 func (f *RemoteFile) String() string {
-	return fmt.Sprintf("%s: %s (lastError: %s)", f.Type(), f.url, f.lastError)
+	return fmt.Sprintf("%s: %s (lastError: %v)", f.Type(), f.url, f.lastError)
 }
 
 func (f *RemoteFile) AvailabilityChanges() bool {

config/cloudinit/datasource/vmware/vmware.go

@@ -21,8 +21,8 @@ import (
 
 	"github.com/rancher/os/config/cloudinit/config"
 	"github.com/rancher/os/config/cloudinit/datasource"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/netconf"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/netconf"
 )
 
 type readConfigFunction func(key string) (string, error)
@@ -40,7 +40,7 @@ func (v VMWare) Finish() error {
 }
 
 func (v VMWare) String() string {
-	return fmt.Sprintf("%s: %s (lastError: %s)", v.Type(), v.ovfFileName, v.lastError)
+	return fmt.Sprintf("%s: %s (lastError: %v)", v.Type(), v.ovfFileName, v.lastError)
 }
 
 func (v VMWare) AvailabilityChanges() bool {
@@ -77,6 +77,13 @@ func (v VMWare) FetchMetadata() (metadata datasource.Metadata, err error) {
 		}
 		metadata.NetworkConfig.DNS.Nameservers = append(metadata.NetworkConfig.DNS.Nameservers, val)
 	}
+	dnsServers, _ := v.read("dns.servers")
+	for _, val := range strings.Split(dnsServers, ",") {
+		if val == "" {
+			break
+		}
+		metadata.NetworkConfig.DNS.Nameservers = append(metadata.NetworkConfig.DNS.Nameservers, val)
+	}
 
 	for i := 0; ; i++ {
 		//if domain := saveConfig("dns.domain.%d", i); domain == "" {
@@ -86,6 +93,13 @@ func (v VMWare) FetchMetadata() (metadata datasource.Metadata, err error) {
 		}
 		metadata.NetworkConfig.DNS.Search = append(metadata.NetworkConfig.DNS.Search, val)
 	}
+	dnsDomains, _ := v.read("dns.domains")
+	for _, val := range strings.Split(dnsDomains, ",") {
+		if val == "" {
+			break
+		}
+		metadata.NetworkConfig.DNS.Search = append(metadata.NetworkConfig.DNS.Search, val)
+	}
 
 	metadata.NetworkConfig.Interfaces = make(map[string]netconf.InterfaceConfig)
 	found := true
@@ -120,6 +134,11 @@ func (v VMWare) FetchMetadata() (metadata datasource.Metadata, err error) {
 			if address == "" {
 				break
 			}
+			netmask, _ := v.read("interface.%d.ip.%d.netmask", i, a)
+			if netmask != "" {
+				ones, _ := net.IPMask(net.ParseIP(netmask).To4()).Size()
+				address = fmt.Sprintf("%s/%d", address, ones)
+			}
 			netDevice.Addresses = append(netDevice.Addresses, address)
 			found = true
 			netDevice.DHCP = false

config/cloudinit/datasource/vmware/vmware_amd64.go

@@ -18,10 +18,9 @@ import (
 	"io/ioutil"
 	"os"
 
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
-
 	"github.com/rancher/os/config/cloudinit/pkg"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 
 	"github.com/sigma/vmw-guestinfo/rpcvmx"
 	"github.com/sigma/vmw-guestinfo/vmcheck"
@@ -33,7 +32,10 @@ type ovfWrapper struct {
 }
 
 func (ovf ovfWrapper) readConfig(key string) (string, error) {
-	return ovf.env.Properties["guestinfo."+key], nil
+	if val := ovf.env.Properties["guestinfo."+key]; val != "" {
+		return val, nil
+	}
+	return readConfig(key)
 }
 
 func NewDatasource(fileName string) *VMWare {
@@ -78,9 +80,14 @@ func (v VMWare) IsAvailable() bool {
 	}
 	if v.ovfFileName != "" {
 		_, v.lastError = os.Stat(v.ovfFileName)
-		return !os.IsNotExist(v.lastError)
+		if !os.IsNotExist(v.lastError) {
+			// when GuestInfo is empty, the DataSource should not be available.
+			return v.checkGuestInfo()
+		}
+		return false
 	}
-	return vmcheck.IsVirtualWorld()
+	// when GuestInfo is empty, the DataSource should not be available.
+	return vmcheck.IsVirtualWorld() && v.checkGuestInfo()
 }
 
 func readConfig(key string) (string, error) {
@@ -107,3 +114,23 @@ func urlDownload(url string) ([]byte, error) {
 	client := pkg.NewHTTPClient()
 	return client.GetRetry(url)
 }
+
+func (v VMWare) checkGuestInfo() bool {
+	userData, err := v.FetchUserdata()
+	if err == nil && string(userData) != "" {
+		return true
+	}
+	metadata, err := v.FetchMetadata()
+	if err == nil {
+		if metadata.Hostname != "" {
+			return true
+		}
+		if len(metadata.NetworkConfig.DNS.Nameservers) > 0 || len(metadata.NetworkConfig.DNS.Search) > 0 {
+			return true
+		}
+		if len(metadata.NetworkConfig.Interfaces) > 0 {
+			return true
+		}
+	}
+	return false
+}

config/cloudinit/datasource/vmware/vmware_test.go

@@ -25,7 +25,7 @@ import (
 	"testing"
 
 	"github.com/rancher/os/config/cloudinit/datasource"
-	"github.com/rancher/os/netconf"
+	"github.com/rancher/os/pkg/netconf"
 )
 
 type MockHypervisor map[string]string
@@ -361,8 +361,8 @@ func TestOvfTransport(t *testing.T) {
 		}
 		v.urlDownload = fakeDownloader
 
-		metadata, err := v.FetchMetadata()
-		userdata, err := v.FetchUserdata()
+		metadata, _ := v.FetchMetadata()
+		userdata, _ := v.FetchUserdata()
 
 		if !reflect.DeepEqual(tt.metadata, metadata) {
 			t.Errorf("bad metadata (#%d): want %#v, got %#v", i, tt.metadata, metadata)

config/cloudinit/datasource/waagent/waagent.go

@@ -1,128 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package waagent
-
-import (
-	"encoding/xml"
-	"fmt"
-	"io/ioutil"
-	"net"
-	"os"
-	"path"
-
-	"github.com/rancher/os/log"
-
-	"github.com/rancher/os/config/cloudinit/datasource"
-)
-
-type Waagent struct {
-	root      string
-	readFile  func(filename string) ([]byte, error)
-	lastError error
-}
-
-func NewDatasource(root string) *Waagent {
-	return &Waagent{root, ioutil.ReadFile, nil}
-}
-
-func (a *Waagent) IsAvailable() bool {
-	_, a.lastError = os.Stat(path.Join(a.root, "provisioned"))
-	return !os.IsNotExist(a.lastError)
-}
-
-func (a *Waagent) Finish() error {
-	return nil
-}
-
-func (a *Waagent) String() string {
-	return fmt.Sprintf("%s: %s (lastError: %s)", a.Type(), a.root, a.lastError)
-}
-
-func (a *Waagent) AvailabilityChanges() bool {
-	return true
-}
-
-func (a *Waagent) ConfigRoot() string {
-	return a.root
-}
-
-func (a *Waagent) FetchMetadata() (metadata datasource.Metadata, err error) {
-	var metadataBytes []byte
-	if metadataBytes, err = a.tryReadFile(path.Join(a.root, "SharedConfig.xml")); err != nil {
-		return
-	}
-	if len(metadataBytes) == 0 {
-		return
-	}
-
-	type Instance struct {
-		ID             string `xml:"id,attr"`
-		Address        string `xml:"address,attr"`
-		InputEndpoints struct {
-			Endpoints []struct {
-				LoadBalancedPublicAddress string `xml:"loadBalancedPublicAddress,attr"`
-			} `xml:"Endpoint"`
-		}
-	}
-
-	type SharedConfig struct {
-		Incarnation struct {
-			Instance string `xml:"instance,attr"`
-		}
-		Instances struct {
-			Instances []Instance `xml:"Instance"`
-		}
-	}
-
-	var m SharedConfig
-	if err = xml.Unmarshal(metadataBytes, &m); err != nil {
-		return
-	}
-
-	var instance Instance
-	for _, i := range m.Instances.Instances {
-		if i.ID == m.Incarnation.Instance {
-			instance = i
-			break
-		}
-	}
-
-	metadata.PrivateIPv4 = net.ParseIP(instance.Address)
-	for _, e := range instance.InputEndpoints.Endpoints {
-		host, _, err := net.SplitHostPort(e.LoadBalancedPublicAddress)
-		if err == nil {
-			metadata.PublicIPv4 = net.ParseIP(host)
-			break
-		}
-	}
-	return
-}
-
-func (a *Waagent) FetchUserdata() ([]byte, error) {
-	return a.tryReadFile(path.Join(a.root, "CustomData"))
-}
-
-func (a *Waagent) Type() string {
-	return "Waagent"
-}
-
-func (a *Waagent) tryReadFile(filename string) ([]byte, error) {
-	log.Printf("Attempting to read from %q\n", filename)
-	data, err := a.readFile(filename)
-	if os.IsNotExist(err) {
-		err = nil
-	}
-	return data, err
-}

config/cloudinit/datasource/waagent/waagent_test.go

@@ -1,166 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package waagent
-
-import (
-	"net"
-	"reflect"
-	"testing"
-
-	"github.com/rancher/os/config/cloudinit/datasource"
-	"github.com/rancher/os/config/cloudinit/datasource/test"
-)
-
-func TestFetchMetadata(t *testing.T) {
-	for _, tt := range []struct {
-		root     string
-		files    test.MockFilesystem
-		metadata datasource.Metadata
-	}{
-		{
-			root:  "/",
-			files: test.NewMockFilesystem(),
-		},
-		{
-			root:  "/",
-			files: test.NewMockFilesystem(test.File{Path: "/SharedConfig.xml", Contents: ""}),
-		},
-		{
-			root:  "/var/lib/Waagent",
-			files: test.NewMockFilesystem(test.File{Path: "/var/lib/Waagent/SharedConfig.xml", Contents: ""}),
-		},
-		{
-			root: "/var/lib/Waagent",
-			files: test.NewMockFilesystem(test.File{Path: "/var/lib/Waagent/SharedConfig.xml", Contents: `<?xml version="1.0" encoding="utf-8"?>
-<SharedConfig version="1.0.0.0" goalStateIncarnation="1">
-  <Deployment name="c8f9e4c9c18948e1bebf57c5685da756" guid="{1d10394f-c741-4a1a-a6bb-278f213c5a5e}" incarnation="0" isNonCancellableTopologyChangeEnabled="false">
-    <Service name="core-test-1" guid="{00000000-0000-0000-0000-000000000000}" />
-    <ServiceInstance name="c8f9e4c9c18948e1bebf57c5685da756.0" guid="{1e202e9a-8ffe-4915-b6ef-4118c9628fda}" />
-  </Deployment>
-  <Incarnation number="1" instance="core-test-1" guid="{8767eb4b-b445-4783-b1f5-6c0beaf41ea0}" />
-  <Role guid="{53ecc81e-257f-fbc9-a53a-8cf1a0a122b4}" name="core-test-1" settleTimeSeconds="0" />
-  <LoadBalancerSettings timeoutSeconds="0" waitLoadBalancerProbeCount="8">
-    <Probes>
-      <Probe name="D41D8CD98F00B204E9800998ECF8427E" />
-      <Probe name="C9DEC1518E1158748FA4B6081A8266DD" />
-    </Probes>
-  </LoadBalancerSettings>
-  <OutputEndpoints>
-    <Endpoint name="core-test-1:openInternalEndpoint" type="SFS">
-      <Target instance="core-test-1" endpoint="openInternalEndpoint" />
-    </Endpoint>
-  </OutputEndpoints>
-  <Instances>
-    <Instance id="core-test-1" address="100.73.202.64">
-      <FaultDomains randomId="0" updateId="0" updateCount="0" />
-      <InputEndpoints>
-        <Endpoint name="openInternalEndpoint" address="100.73.202.64" protocol="any" isPublic="false" enableDirectServerReturn="false" isDirectAddress="false" disableStealthMode="false">
-          <LocalPorts>
-            <LocalPortSelfManaged />
-          </LocalPorts>
-        </Endpoint>
-        <Endpoint name="ssh" address="100.73.202.64:22" protocol="tcp" hostName="core-test-1ContractContract" isPublic="true" loadBalancedPublicAddress="191.239.39.77:22" enableDirectServerReturn="false" isDirectAddress="false" disableStealthMode="false">
-          <LocalPorts>
-            <LocalPortRange from="22" to="22" />
-          </LocalPorts>
-        </Endpoint>
-      </InputEndpoints>
-    </Instance>
-  </Instances>
-</SharedConfig>`}),
-			metadata: datasource.Metadata{
-				PrivateIPv4: net.ParseIP("100.73.202.64"),
-				PublicIPv4:  net.ParseIP("191.239.39.77"),
-			},
-		},
-	} {
-		a := Waagent{tt.root, tt.files.ReadFile, nil}
-		metadata, err := a.FetchMetadata()
-		if err != nil {
-			t.Fatalf("bad error for %+v: want %v, got %q", tt, nil, err)
-		}
-		if !reflect.DeepEqual(tt.metadata, metadata) {
-			t.Fatalf("bad metadata for %+v: want %#v, got %#v", tt, tt.metadata, metadata)
-		}
-	}
-}
-
-func TestFetchUserdata(t *testing.T) {
-	for _, tt := range []struct {
-		root  string
-		files test.MockFilesystem
-	}{
-		{
-			"/",
-			test.NewMockFilesystem(),
-		},
-		{
-			"/",
-			test.NewMockFilesystem(test.File{Path: "/CustomData", Contents: ""}),
-		},
-		{
-			"/var/lib/Waagent/",
-			test.NewMockFilesystem(test.File{Path: "/var/lib/Waagent/CustomData", Contents: ""}),
-		},
-	} {
-		a := Waagent{tt.root, tt.files.ReadFile, nil}
-		_, err := a.FetchUserdata()
-		if err != nil {
-			t.Fatalf("bad error for %+v: want %v, got %q", tt, nil, err)
-		}
-	}
-}
-
-func TestConfigRoot(t *testing.T) {
-	for _, tt := range []struct {
-		root       string
-		configRoot string
-	}{
-		{
-			"/",
-			"/",
-		},
-		{
-			"/var/lib/Waagent",
-			"/var/lib/Waagent",
-		},
-	} {
-		a := Waagent{tt.root, nil, nil}
-		if configRoot := a.ConfigRoot(); configRoot != tt.configRoot {
-			t.Fatalf("bad config root for %q: want %q, got %q", tt, tt.configRoot, configRoot)
-		}
-	}
-}
-
-func TestNewDatasource(t *testing.T) {
-	for _, tt := range []struct {
-		root       string
-		expectRoot string
-	}{
-		{
-			root:       "",
-			expectRoot: "",
-		},
-		{
-			root:       "/var/lib/Waagent",
-			expectRoot: "/var/lib/Waagent",
-		},
-	} {
-		service := NewDatasource(tt.root)
-		if service.root != tt.expectRoot {
-			t.Fatalf("bad root (%q): want %q, got %q", tt.root, tt.expectRoot, service.root)
-		}
-	}
-}

config/cloudinit/network/interface_test.go

@@ -363,7 +363,7 @@ func TestFilename(t *testing.T) {
 		{logicalInterface{name: "iface", hwaddr: net.HardwareAddr([]byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab}), configDepth: 1}, "01-iface"},
 	} {
 		if tt.i.Filename() != tt.f {
-			t.Fatalf("bad filename (%q): got %q, want %q", tt.i, tt.i.Filename(), tt.f)
+			t.Fatalf("bad filename: got %q, want %q", tt.i.Filename(), tt.f)
 		}
 	}
 }

config/cloudinit/network/packet.go

@@ -17,7 +17,7 @@ package network
 import (
 	"net"
 
-	"github.com/rancher/os/netconf"
+	"github.com/rancher/os/pkg/netconf"
 )
 
 func ProcessPacketNetconf(netdata netconf.NetworkConfig) ([]InterfaceGenerator, error) {

config/cloudinit/network/vmware.go

@@ -57,7 +57,7 @@ func ProcessVMwareNetconf(config map[string]string) ([]InterfaceGenerator, error
 		var dhcp bool
 		iface := &physicalInterface{}
 
-		log.Printf("Proccessing interface %d", i)
+		log.Printf("Processing interface %d", i)
 
 		log.Println("Processing DHCP")
 		if dhcp, err = processDHCPConfig(config, fmt.Sprintf("interface.%d.", i)); err != nil {

config/cloudinit/pkg/http_client.go

@@ -18,11 +18,12 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	neturl "net/url"
 	"strings"
 	"time"
+
+	"github.com/rancher/os/pkg/log"
 )
 
 const (
@@ -120,14 +121,14 @@ func (h *HTTPClient) GetRetry(rawurl string) ([]byte, error) {
 
 	duration := h.InitialBackoff
 	for retry := 1; retry <= h.MaxRetries; retry++ {
-		log.Printf("Fetching data from %s. Attempt #%d", dataURL, retry)
+		log.Debugf("Fetching data from %s. Attempt #%d", dataURL, retry)
 
 		data, err := h.Get(dataURL)
 		switch err.(type) {
 		case ErrNetwork:
-			log.Printf(err.Error())
+			log.Debugf(err.Error())
 		case ErrServer:
-			log.Printf(err.Error())
+			log.Debugf(err.Error())
 		case ErrNotFound:
 			return data, err
 		default:
@@ -135,7 +136,7 @@ func (h *HTTPClient) GetRetry(rawurl string) ([]byte, error) {
 		}
 
 		duration = ExpBackoff(duration, h.MaxBackoff)
-		log.Printf("Sleeping for %v...", duration)
+		log.Debugf("Sleeping for %v...", duration)
 		time.Sleep(duration)
 	}
 

config/cloudinit/test

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-source ./build
+source build
 
 SRC=$(find . -name '*.go' \
 	-not -path "./vendor/*")

config/cmdline/cmdline.go

@@ -0,0 +1,196 @@
+package cmdline
+
+import (
+	"io/ioutil"
+	"strings"
+	"unicode"
+
+	"github.com/rancher/os/pkg/util"
+
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
+)
+
+func Read(parseAll bool) (m map[interface{}]interface{}, err error) {
+	cmdLine, err := ioutil.ReadFile("/proc/cmdline")
+	if err != nil {
+		return nil, err
+	}
+
+	if len(cmdLine) == 0 {
+		return nil, nil
+	}
+
+	cmdLineObj := Parse(strings.TrimSpace(util.UnescapeKernelParams(string(cmdLine))), parseAll)
+
+	return cmdLineObj, nil
+}
+
+func GetCmdline(key string) interface{} {
+	parseAll := true
+	if strings.HasPrefix(key, "cc.") || strings.HasPrefix(key, "rancher.") {
+		// the normal case
+		parseAll = false
+	}
+	cmdline, _ := Read(parseAll)
+	v, _ := GetOrSetVal(key, cmdline, nil)
+	return v
+}
+
+func GetOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) {
+	parts := strings.Split(args, ".")
+
+	tData := data
+	if value != nil {
+		tData = util.MapCopy(data)
+	}
+	t := tData
+	for i, part := range parts {
+		val, ok := t[part]
+		last := i+1 == len(parts)
+
+		// Reached end, set the value
+		if last && value != nil {
+			if s, ok := value.(string); ok {
+				value = UnmarshalOrReturnString(s)
+			}
+
+			t[part] = value
+			return value, tData
+		}
+
+		// Missing intermediate key, create key
+		if !last && value != nil && !ok {
+			newData := map[interface{}]interface{}{}
+			t[part] = newData
+			t = newData
+			continue
+		}
+
+		if !ok {
+			break
+		}
+
+		if last {
+			return val, tData
+		}
+
+		newData, ok := val.(map[interface{}]interface{})
+		if !ok {
+			break
+		}
+
+		t = newData
+	}
+
+	return "", tData
+}
+
+// Replace newlines, colons, and question marks with random strings
+// This is done to avoid YAML treating these as special characters
+var (
+	newlineMagicString      = "9XsJcx6dR5EERYCC"
+	colonMagicString        = "V0Rc21pIVknMm2rr"
+	questionMarkMagicString = "FoPL6JLMAaJqKMJT"
+)
+
+func reverseReplacement(result interface{}) interface{} {
+	switch val := result.(type) {
+	case map[interface{}]interface{}:
+		for k, v := range val {
+			val[k] = reverseReplacement(v)
+		}
+		return val
+	case []interface{}:
+		for i, item := range val {
+			val[i] = reverseReplacement(item)
+		}
+		return val
+	case string:
+		val = strings.Replace(val, newlineMagicString, "\n", -1)
+		val = strings.Replace(val, colonMagicString, ":", -1)
+		val = strings.Replace(val, questionMarkMagicString, "?", -1)
+		return val
+	}
+
+	return result
+}
+
+func UnmarshalOrReturnString(value string) (result interface{}) {
+	value = strings.Replace(value, "\n", newlineMagicString, -1)
+	value = strings.Replace(value, ":", colonMagicString, -1)
+	value = strings.Replace(value, "?", questionMarkMagicString, -1)
+	if err := yaml.Unmarshal([]byte(value), &result); err != nil {
+		result = value
+	}
+	result = reverseReplacement(result)
+	return
+}
+
+//splitCmdLine splits on spaces except when a space is within a quoted or bracketed string.
+func splitCmdLine(cmdLine string) []string {
+	lastRune := rune(0)
+	f := func(c rune) bool {
+		switch {
+		case c == lastRune:
+			lastRune = rune(0)
+			return false
+		case lastRune != rune(0):
+			return false
+		case unicode.In(c, unicode.Quotation_Mark):
+			lastRune = c
+			return false
+		case c == '[':
+			lastRune = ']'
+			return false
+		default:
+			return c == ' '
+		}
+	}
+	return strings.FieldsFunc(cmdLine, f)
+}
+
+func Parse(cmdLine string, parseAll bool) map[interface{}]interface{} {
+	result := map[interface{}]interface{}{}
+
+outer:
+	for _, part := range splitCmdLine(cmdLine) {
+		if strings.HasPrefix(part, "cc.") {
+			part = part[3:]
+		} else if !strings.HasPrefix(part, "rancher.") {
+			if !parseAll {
+				continue
+			}
+		}
+
+		var value string
+		kv := strings.SplitN(part, "=", 2)
+
+		if len(kv) == 1 {
+			value = "true"
+		} else {
+			value = kv[1]
+		}
+
+		current := result
+		keys := strings.Split(kv[0], ".")
+		for i, key := range keys {
+			if i == len(keys)-1 {
+				current[key] = UnmarshalOrReturnString(value)
+			} else {
+				if obj, ok := current[key]; ok {
+					if newCurrent, ok := obj.(map[interface{}]interface{}); ok {
+						current = newCurrent
+					} else {
+						continue outer
+					}
+				} else {
+					newCurrent := make(map[interface{}]interface{})
+					current[key] = newCurrent
+					current = newCurrent
+				}
+			}
+		}
+	}
+
+	return result
+}

config/config.go

@@ -5,7 +5,8 @@ import (
 	"strings"
 
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/config/cmdline"
+	"github.com/rancher/os/pkg/util"
 )
 
 const Banner = `
@@ -34,6 +35,7 @@ func Merge(bytes []byte) error {
 
 func Export(private, full bool) (string, error) {
 	rawCfg := loadRawConfig("", full)
+	rawCfg = filterAdditional(rawCfg)
 	if !private {
 		rawCfg = filterPrivateKeys(rawCfg)
 	}
@@ -41,6 +43,21 @@ func Export(private, full bool) (string, error) {
 	bytes, err := yaml.Marshal(rawCfg)
 	return string(bytes), err
 }
+func filterPrivateKeys(data map[interface{}]interface{}) map[interface{}]interface{} {
+	for _, privateKey := range PrivateKeys {
+		_, data = filterKey(data, strings.Split(privateKey, "."))
+	}
+
+	return data
+}
+
+func filterAdditional(data map[interface{}]interface{}) map[interface{}]interface{} {
+	for _, additional := range Additional {
+		_, data = filterKey(data, strings.Split(additional, "."))
+	}
+
+	return data
+}
 
 func Get(key string) (interface{}, error) {
 	cfg := LoadConfig()
@@ -50,23 +67,17 @@ func Get(key string) (interface{}, error) {
 		return nil, err
 	}
 
-	v, _ := getOrSetVal(key, data, nil)
+	v, _ := cmdline.GetOrSetVal(key, data, nil)
 	return v, nil
 }
 
-func GetCmdline(key string) interface{} {
-	cmdline := readCmdline()
-	v, _ := getOrSetVal(key, cmdline, nil)
-	return v
-}
-
 func Set(key string, value interface{}) error {
 	existing, err := readConfigs(nil, false, true, CloudConfigFile)
 	if err != nil {
 		return err
 	}
 
-	_, modified := getOrSetVal(key, existing, value)
+	_, modified := cmdline.GetOrSetVal(key, existing, value)
 
 	c := &CloudConfig{}
 	if err = util.Convert(modified, c); err != nil {

config/config_test.go

@@ -3,9 +3,10 @@ package config
 import (
 	"testing"
 
-	yaml "github.com/cloudfoundry-incubator/candiedyaml"
+	"github.com/rancher/os/config/cmdline"
+	"github.com/rancher/os/pkg/util"
 
-	"github.com/rancher/os/util"
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 	"github.com/stretchr/testify/require"
 )
 
@@ -57,50 +58,50 @@ func TestFilterKey(t *testing.T) {
 func TestUnmarshalOrReturnString(t *testing.T) {
 	assert := require.New(t)
 
-	assert.Equal("ab", unmarshalOrReturnString("ab"))
-	assert.Equal("a\nb", unmarshalOrReturnString("a\nb"))
-	assert.Equal("a\n", unmarshalOrReturnString("a\n"))
-	assert.Equal("\nb", unmarshalOrReturnString("\nb"))
-	assert.Equal("a,b", unmarshalOrReturnString("a,b"))
-	assert.Equal("a,", unmarshalOrReturnString("a,"))
-	assert.Equal(",b", unmarshalOrReturnString(",b"))
+	assert.Equal("ab", cmdline.UnmarshalOrReturnString("ab"))
+	assert.Equal("a\nb", cmdline.UnmarshalOrReturnString("a\nb"))
+	assert.Equal("a\n", cmdline.UnmarshalOrReturnString("a\n"))
+	assert.Equal("\nb", cmdline.UnmarshalOrReturnString("\nb"))
+	assert.Equal("a,b", cmdline.UnmarshalOrReturnString("a,b"))
+	assert.Equal("a,", cmdline.UnmarshalOrReturnString("a,"))
+	assert.Equal(",b", cmdline.UnmarshalOrReturnString(",b"))
 
-	assert.Equal(int64(10), unmarshalOrReturnString("10"))
-	assert.Equal(true, unmarshalOrReturnString("true"))
-	assert.Equal(false, unmarshalOrReturnString("false"))
+	assert.Equal(int64(10), cmdline.UnmarshalOrReturnString("10"))
+	assert.Equal(true, cmdline.UnmarshalOrReturnString("true"))
+	assert.Equal(false, cmdline.UnmarshalOrReturnString("false"))
 
-	assert.Equal([]interface{}{"a"}, unmarshalOrReturnString("[a]"))
-	assert.Equal([]interface{}{"a"}, unmarshalOrReturnString("[\"a\"]"))
+	assert.Equal([]interface{}{"a"}, cmdline.UnmarshalOrReturnString("[a]"))
+	assert.Equal([]interface{}{"a"}, cmdline.UnmarshalOrReturnString("[\"a\"]"))
 
-	assert.Equal([]interface{}{"a,"}, unmarshalOrReturnString("[\"a,\"]"))
-	assert.Equal([]interface{}{" a, "}, unmarshalOrReturnString("[\" a, \"]"))
-	assert.Equal([]interface{}{",a"}, unmarshalOrReturnString("[\",a\"]"))
-	assert.Equal([]interface{}{" ,a "}, unmarshalOrReturnString("[\" ,a \"]"))
+	assert.Equal([]interface{}{"a,"}, cmdline.UnmarshalOrReturnString("[\"a,\"]"))
+	assert.Equal([]interface{}{" a, "}, cmdline.UnmarshalOrReturnString("[\" a, \"]"))
+	assert.Equal([]interface{}{",a"}, cmdline.UnmarshalOrReturnString("[\",a\"]"))
+	assert.Equal([]interface{}{" ,a "}, cmdline.UnmarshalOrReturnString("[\" ,a \"]"))
 
-	assert.Equal([]interface{}{"a\n"}, unmarshalOrReturnString("[\"a\n\"]"))
-	assert.Equal([]interface{}{" a\n "}, unmarshalOrReturnString("[\" a\n \"]"))
-	assert.Equal([]interface{}{"\na"}, unmarshalOrReturnString("[\"\na\"]"))
-	assert.Equal([]interface{}{" \na "}, unmarshalOrReturnString("[\" \na \"]"))
+	assert.Equal([]interface{}{"a\n"}, cmdline.UnmarshalOrReturnString("[\"a\n\"]"))
+	assert.Equal([]interface{}{" a\n "}, cmdline.UnmarshalOrReturnString("[\" a\n \"]"))
+	assert.Equal([]interface{}{"\na"}, cmdline.UnmarshalOrReturnString("[\"\na\"]"))
+	assert.Equal([]interface{}{" \na "}, cmdline.UnmarshalOrReturnString("[\" \na \"]"))
 
-	assert.Equal([]interface{}{"a", "b"}, unmarshalOrReturnString("[a,b]"))
-	assert.Equal([]interface{}{"a", "b"}, unmarshalOrReturnString("[\"a\",\"b\"]"))
+	assert.Equal([]interface{}{"a", "b"}, cmdline.UnmarshalOrReturnString("[a,b]"))
+	assert.Equal([]interface{}{"a", "b"}, cmdline.UnmarshalOrReturnString("[\"a\",\"b\"]"))
 
-	assert.Equal([]interface{}{"a,", "b"}, unmarshalOrReturnString("[\"a,\",b]"))
-	assert.Equal([]interface{}{"a", ",b"}, unmarshalOrReturnString("[a,\",b\"]"))
-	assert.Equal([]interface{}{" a, ", " ,b "}, unmarshalOrReturnString("[\" a, \",\" ,b \"]"))
+	assert.Equal([]interface{}{"a,", "b"}, cmdline.UnmarshalOrReturnString("[\"a,\",b]"))
+	assert.Equal([]interface{}{"a", ",b"}, cmdline.UnmarshalOrReturnString("[a,\",b\"]"))
+	assert.Equal([]interface{}{" a, ", " ,b "}, cmdline.UnmarshalOrReturnString("[\" a, \",\" ,b \"]"))
 
-	assert.Equal([]interface{}{"a\n", "b"}, unmarshalOrReturnString("[\"a\n\",b]"))
-	assert.Equal([]interface{}{"a", "\nb"}, unmarshalOrReturnString("[a,\"\nb\"]"))
-	assert.Equal([]interface{}{" a\n ", " \nb "}, unmarshalOrReturnString("[\" a\n \",\" \nb \"]"))
+	assert.Equal([]interface{}{"a\n", "b"}, cmdline.UnmarshalOrReturnString("[\"a\n\",b]"))
+	assert.Equal([]interface{}{"a", "\nb"}, cmdline.UnmarshalOrReturnString("[a,\"\nb\"]"))
+	assert.Equal([]interface{}{" a\n ", " \nb "}, cmdline.UnmarshalOrReturnString("[\" a\n \",\" \nb \"]"))
 
-	assert.Equal([]interface{}{"a", int64(10)}, unmarshalOrReturnString("[a,10]"))
-	assert.Equal([]interface{}{int64(10), "a"}, unmarshalOrReturnString("[10,a]"))
+	assert.Equal([]interface{}{"a", int64(10)}, cmdline.UnmarshalOrReturnString("[a,10]"))
+	assert.Equal([]interface{}{int64(10), "a"}, cmdline.UnmarshalOrReturnString("[10,a]"))
 
-	assert.Equal([]interface{}{"a", true}, unmarshalOrReturnString("[a,true]"))
-	assert.Equal([]interface{}{false, "a"}, unmarshalOrReturnString("[false,a]"))
+	assert.Equal([]interface{}{"a", true}, cmdline.UnmarshalOrReturnString("[a,true]"))
+	assert.Equal([]interface{}{false, "a"}, cmdline.UnmarshalOrReturnString("[false,a]"))
 }
 
-func TestParseCmdline(t *testing.T) {
+func TestCmdlineParse(t *testing.T) {
 	assert := require.New(t)
 
 	assert.Equal(map[interface{}]interface{}{
@@ -108,55 +109,79 @@ func TestParseCmdline(t *testing.T) {
 			"key1": "value1",
 			"key2": "value2",
 		},
-	}, parseCmdline("a b rancher.key1=value1 c rancher.key2=value2"))
+	}, cmdline.Parse("a b rancher.key1=value1 c rancher.key2=value2", false), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"key": "a,b",
 		},
-	}, parseCmdline("rancher.key=a,b"))
+	}, cmdline.Parse("rancher.key=a,b", false), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"key": "a\nb",
 		},
-	}, parseCmdline("rancher.key=a\nb"))
+	}, cmdline.Parse("rancher.key=a\nb", false), false)
+
+	assert.Equal(map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
+			"key": "a b",
+		},
+	}, cmdline.Parse("rancher.key='a b'", false), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"key": "a:b",
 		},
-	}, parseCmdline("rancher.key=a:b"))
+	}, cmdline.Parse("rancher.key=a:b", false), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"key": int64(5),
 		},
-	}, parseCmdline("rancher.key=5"))
+	}, cmdline.Parse("rancher.key=5", false), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"rescue": true,
 		},
-	}, parseCmdline("rancher.rescue"))
+	}, cmdline.Parse("rancher.rescue", false), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"keyArray": []interface{}{int64(1), int64(2)},
 		},
-	}, parseCmdline("rancher.keyArray=[1,2]"))
+	}, cmdline.Parse("rancher.keyArray=[1,2]", false), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"strArray": []interface{}{"url:http://192.168.1.100/cloud-config?a=b"},
 		},
-	}, parseCmdline("rancher.strArray=[\"url:http://192.168.1.100/cloud-config?a=b\"]"))
+	}, cmdline.Parse("rancher.strArray=[\"url:http://192.168.1.100/cloud-config?a=b\"]", false), false)
 
 	assert.Equal(map[interface{}]interface{}{
 		"rancher": map[interface{}]interface{}{
 			"strArray": []interface{}{"url:http://192.168.1.100/cloud-config?a=b"},
 		},
-	}, parseCmdline("rancher.strArray=[url:http://192.168.1.100/cloud-config?a=b]"))
+	}, cmdline.Parse("rancher.strArray=[url:http://192.168.1.100/cloud-config?a=b]", false), false)
+
+	assert.Equal(map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
+			"strArray": []interface{}{"part1 part2", "part3"},
+		},
+	}, cmdline.Parse("rancher.strArray=['part1 part2',part3]", false), false)
+
+	assert.Equal(map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
+			"strArray": []interface{}{"part1 part2", "part3"},
+		},
+	}, cmdline.Parse("rancher.strArray=[\"part1 part2\",part3]", false), false)
+
+	assert.Equal(map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
+			"strArray": []interface{}{"part1 part2", "part3"},
+		},
+	}, cmdline.Parse("rancher.strArray=[ \"part1 part2\", part3 ]", false), false)
 }
 
 func TestGet(t *testing.T) {
@@ -181,7 +206,7 @@ func TestGet(t *testing.T) {
 	}
 
 	for k, v := range tests {
-		val, _ := getOrSetVal(k, data, nil)
+		val, _ := cmdline.GetOrSetVal(k, data, nil)
 		assert.Equal(v, val)
 	}
 }
@@ -225,8 +250,8 @@ func TestSet(t *testing.T) {
 	}
 
 	for k, v := range tests {
-		_, tData := getOrSetVal(k, data, v)
-		val, _ := getOrSetVal(k, tData, nil)
+		_, tData := cmdline.GetOrSetVal(k, data, v)
+		val, _ := cmdline.GetOrSetVal(k, tData, nil)
 		data = tData
 		assert.Equal(v, val)
 	}

config/data_funcs.go

@@ -1,12 +1,8 @@
 package config
 
 import (
-	yaml "github.com/cloudfoundry-incubator/candiedyaml"
-	"github.com/rancher/os/log"
-
-	"strings"
-
-	"github.com/rancher/os/util"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
 )
 
 type CfgFunc func(*CloudConfig) (*CloudConfig, error)
@@ -14,6 +10,7 @@ type CfgFuncData struct {
 	Name string
 	Func CfgFunc
 }
+
 type CfgFuncs []CfgFuncData
 
 func ChainCfgFuncs(cfg *CloudConfig, cfgFuncs CfgFuncs) (*CloudConfig, error) {
@@ -29,7 +26,7 @@ func ChainCfgFuncs(cfg *CloudConfig, cfgFuncs CfgFuncs) (*CloudConfig, error) {
 		}
 		var err error
 		if cfg, err = cfgFunc(cfg); err != nil {
-			log.Errorf("Failed [%d/%d] %s: %s", i, len, name, err)
+			log.Errorf("Failed [%d/%d] %s: %v", i, len, name, err)
 			return cfg, err
 		}
 		log.Debugf("[%d/%d] Done %s", i, len, name)
@@ -71,145 +68,3 @@ func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest m
 
 	return
 }
-
-func filterPrivateKeys(data map[interface{}]interface{}) map[interface{}]interface{} {
-	for _, privateKey := range PrivateKeys {
-		_, data = filterKey(data, strings.Split(privateKey, "."))
-	}
-
-	return data
-}
-
-func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) {
-	parts := strings.Split(args, ".")
-
-	tData := data
-	if value != nil {
-		tData = util.MapCopy(data)
-	}
-	t := tData
-	for i, part := range parts {
-		val, ok := t[part]
-		last := i+1 == len(parts)
-
-		// Reached end, set the value
-		if last && value != nil {
-			if s, ok := value.(string); ok {
-				value = unmarshalOrReturnString(s)
-			}
-
-			t[part] = value
-			return value, tData
-		}
-
-		// Missing intermediate key, create key
-		if !last && value != nil && !ok {
-			newData := map[interface{}]interface{}{}
-			t[part] = newData
-			t = newData
-			continue
-		}
-
-		if !ok {
-			break
-		}
-
-		if last {
-			return val, tData
-		}
-
-		newData, ok := val.(map[interface{}]interface{})
-		if !ok {
-			break
-		}
-
-		t = newData
-	}
-
-	return "", tData
-}
-
-// Replace newlines, colons, and question marks with random strings
-// This is done to avoid YAML treating these as special characters
-var (
-	newlineMagicString      = "9XsJcx6dR5EERYCC"
-	colonMagicString        = "V0Rc21pIVknMm2rr"
-	questionMarkMagicString = "FoPL6JLMAaJqKMJT"
-)
-
-func reverseReplacement(result interface{}) interface{} {
-	switch val := result.(type) {
-	case map[interface{}]interface{}:
-		for k, v := range val {
-			val[k] = reverseReplacement(v)
-		}
-		return val
-	case []interface{}:
-		for i, item := range val {
-			val[i] = reverseReplacement(item)
-		}
-		return val
-	case string:
-		val = strings.Replace(val, newlineMagicString, "\n", -1)
-		val = strings.Replace(val, colonMagicString, ":", -1)
-		val = strings.Replace(val, questionMarkMagicString, "?", -1)
-		return val
-	}
-
-	return result
-}
-
-func unmarshalOrReturnString(value string) (result interface{}) {
-	value = strings.Replace(value, "\n", newlineMagicString, -1)
-	value = strings.Replace(value, ":", colonMagicString, -1)
-	value = strings.Replace(value, "?", questionMarkMagicString, -1)
-	if err := yaml.Unmarshal([]byte(value), &result); err != nil {
-		result = value
-	}
-	result = reverseReplacement(result)
-	return
-}
-
-func parseCmdline(cmdLine string) map[interface{}]interface{} {
-	result := make(map[interface{}]interface{})
-
-outer:
-	for _, part := range strings.Split(cmdLine, " ") {
-		if strings.HasPrefix(part, "cc.") {
-			part = part[3:]
-		} else if !strings.HasPrefix(part, "rancher.") {
-			continue
-		}
-
-		var value string
-		kv := strings.SplitN(part, "=", 2)
-
-		if len(kv) == 1 {
-			value = "true"
-		} else {
-			value = kv[1]
-		}
-
-		current := result
-		keys := strings.Split(kv[0], ".")
-		for i, key := range keys {
-			if i == len(keys)-1 {
-				current[key] = unmarshalOrReturnString(value)
-			} else {
-				if obj, ok := current[key]; ok {
-					if newCurrent, ok := obj.(map[interface{}]interface{}); ok {
-						current = newCurrent
-					} else {
-						continue outer
-					}
-				} else {
-					newCurrent := make(map[interface{}]interface{})
-					current[key] = newCurrent
-					current = newCurrent
-				}
-			}
-		}
-	}
-
-	return result
-}

config/disk.go

@@ -9,13 +9,16 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/rancher/os/config/cloudinit/datasource"
+	"github.com/rancher/os/config/cloudinit/initialize"
+	"github.com/rancher/os/config/cmdline"
+	"github.com/rancher/os/pkg/log"
+	"github.com/rancher/os/pkg/util"
+
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 	"github.com/docker/engine-api/types"
 	composeConfig "github.com/docker/libcompose/config"
-	"github.com/rancher/os/config/cloudinit/datasource"
-	"github.com/rancher/os/config/cloudinit/initialize"
-	"github.com/rancher/os/log"
-	"github.com/rancher/os/util"
+	"github.com/xeipuuv/gojsonschema"
 )
 
 func ReadConfig(bytes []byte, substituteMetadataVars bool, files ...string) (*CloudConfig, error) {
@@ -46,14 +49,48 @@ func loadRawDiskConfig(dirPrefix string, full bool) map[interface{}]interface{}
 	return util.Merge(rawCfg, additionalCfgs)
 }
 
+func loadRawDiskConfigWithError(dirPrefix string, full bool) (map[interface{}]interface{}, error) {
+	var rawCfg map[interface{}]interface{}
+	rawCfg, err := readConfigs(nil, true, true, OsConfigFile, OemConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	files := CloudConfigDirFiles(dirPrefix)
+	files = append(files, path.Join(dirPrefix, CloudConfigFile))
+	additionalCfgs, err := readConfigs(nil, true, true, files...)
+	if err != nil {
+		return nil, err
+	}
+	return util.Merge(rawCfg, additionalCfgs), nil
+}
+
 func loadRawConfig(dirPrefix string, full bool) map[interface{}]interface{} {
 	rawCfg := loadRawDiskConfig(dirPrefix, full)
-	rawCfg = util.Merge(rawCfg, readCmdline())
+	procCmdline, err := cmdline.Read(false)
+	if err != nil {
+		log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
+	}
+	rawCfg = util.Merge(rawCfg, procCmdline)
 	rawCfg = util.Merge(rawCfg, readElidedCmdline(rawCfg))
 	rawCfg = applyDebugFlags(rawCfg)
 	return mergeMetadata(rawCfg, readMetadata())
 }
 
+func loadRawConfigWithError(dirPrefix string, full bool) (map[interface{}]interface{}, error) {
+	rawCfg, err := loadRawDiskConfigWithError(dirPrefix, full)
+	if err != nil {
+		return nil, err
+	}
+	procCmdline, err := cmdline.Read(false)
+	if err != nil {
+		log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
+	}
+	rawCfg = util.Merge(rawCfg, procCmdline)
+	rawCfg = util.Merge(rawCfg, readElidedCmdline(rawCfg))
+	rawCfg = applyDebugFlags(rawCfg)
+	return mergeMetadata(rawCfg, readMetadata()), nil
+}
+
 func LoadConfig() *CloudConfig {
 	cfg := LoadConfigWithPrefix("")
 
@@ -66,6 +103,21 @@ func LoadConfig() *CloudConfig {
 	return cfg
 }
 
+func LoadConfigWithError() (*CloudConfig, *gojsonschema.Result, error) {
+	rawCfg, err := loadRawConfigWithError("", true)
+	if err != nil {
+		return &CloudConfig{}, nil, err
+	}
+	cfg := &CloudConfig{}
+	if err := util.Convert(rawCfg, cfg); err != nil {
+		validationErrors, err := ValidateRawCfg(rawCfg)
+		return &CloudConfig{}, validationErrors, err
+	}
+	cfg = amendNils(cfg)
+	cfg = amendContainerNames(cfg)
+	return cfg, nil, nil
+}
+
 func LoadConfigWithPrefix(dirPrefix string) *CloudConfig {
 	rawCfg := loadRawConfig(dirPrefix, true)
 
@@ -107,7 +159,7 @@ func Insert(m interface{}, args ...interface{}) interface{} {
 }
 
 func SaveInitCmdline(cmdLineArgs string) {
-	elidedCfg := parseCmdline(cmdLineArgs)
+	elidedCfg := cmdline.Parse(cmdLineArgs, false)
 
 	env := Insert(make(map[interface{}]interface{}), interface{}("EXTRA_CMDLINE"), interface{}(cmdLineArgs))
 	rancher := Insert(make(map[interface{}]interface{}), interface{}("environment"), env)
@@ -155,10 +207,10 @@ func applyDebugFlags(rawCfg map[interface{}]interface{}) map[interface{}]interfa
 	}
 
 	log.SetLevel(log.DebugLevel)
-	_, rawCfg = getOrSetVal("rancher.docker.debug", rawCfg, true)
-	_, rawCfg = getOrSetVal("rancher.system_docker.debug", rawCfg, true)
-	_, rawCfg = getOrSetVal("rancher.bootstrap_docker.debug", rawCfg, true)
-	_, rawCfg = getOrSetVal("rancher.log", rawCfg, true)
+	_, rawCfg = cmdline.GetOrSetVal("rancher.docker.debug", rawCfg, true)
+	_, rawCfg = cmdline.GetOrSetVal("rancher.system_docker.debug", rawCfg, true)
+	_, rawCfg = cmdline.GetOrSetVal("rancher.bootstrap_docker.debug", rawCfg, true)
+	_, rawCfg = cmdline.GetOrSetVal("rancher.log", rawCfg, true)
 
 	return rawCfg
 }
@@ -200,6 +252,11 @@ func mergeMetadata(rawCfg map[interface{}]interface{}, md datasource.Metadata) m
 
 	out["ssh_authorized_keys"] = finalKeys
 
+	rancherOut, _ := out["rancher"].(map[interface{}]interface{})
+	if _, ok := rancherOut["resize_device"]; md.RootDisk != "" && !ok {
+		rancherOut["resize_device"] = md.RootDisk
+	}
+
 	return out
 }
 
@@ -216,7 +273,7 @@ func readElidedCmdline(rawCfg map[interface{}]interface{}) map[interface{}]inter
 	for k, v := range rawCfg {
 		if key, _ := k.(string); key == "EXTRA_CMDLINE" {
 			if val, ok := v.(string); ok {
-				cmdLineObj := parseCmdline(strings.TrimSpace(util.UnescapeKernelParams(string(val))))
+				cmdLineObj := cmdline.Parse(strings.TrimSpace(util.UnescapeKernelParams(string(val))), false)
 
 				return cmdLineObj
 			}
@@ -225,22 +282,6 @@ func readElidedCmdline(rawCfg map[interface{}]interface{}) map[interface{}]inter
 	return nil
 }
 
-func readCmdline() map[interface{}]interface{} {
-	cmdLine, err := ioutil.ReadFile("/proc/cmdline")
-	if err != nil {
-		log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
-		return nil
-	}
-
-	if len(cmdLine) == 0 {
-		return nil
-	}
-
-	cmdLineObj := parseCmdline(strings.TrimSpace(util.UnescapeKernelParams(string(cmdLine))))
-
-	return cmdLineObj
-}
-
 func amendNils(c *CloudConfig) *CloudConfig {
 	t := *c
 	if t.Rancher.Environment == nil {

config/docker_config.go

@@ -8,7 +8,7 @@ import (
 )
 
 func (d *DockerConfig) FullArgs() []string {
-	args := []string{"daemon"}
+	args := []string{}
 	args = append(args, generateEngineOptsSlice(d.EngineOpts)...)
 	args = append(args, d.ExtraArgs...)
 	if d.TLS {

config/schema.go

@@ -1,204 +1,218 @@
 package config
 
 var schema = `{
-  "type": "object",
-  "additionalProperties": false,
+	"type": "object",
+	"additionalProperties": false,
 
-  "properties": {
-    "ssh_authorized_keys": {"$ref": "#/definitions/list_of_strings"},
-    "write_files": {
-      "type": "array",
-      "items": {"$ref": "#/definitions/file_config"}
-    },
-    "hostname": {"type": "string"},
-    "mounts": {"type": "array"},
-    "rancher": {"$ref": "#/definitions/rancher_config"},
-    "runcmd": {"type": "array"},
-    "bootcmd": {"type": "array"}
-  },
+	"properties": {
+		"ssh_authorized_keys": {"$ref": "#/definitions/list_of_strings"},
+		"write_files": {
+			"type": "array",
+			"items": {"$ref": "#/definitions/file_config"}
+		},
+		"hostname": {"type": "string"},
+		"mounts": {"type": "array"},
+		"rancher": {"$ref": "#/definitions/rancher_config"},
+		"runcmd": {"type": "array"},
+		"bootcmd": {"type": "array"}
+	},
 
-  "definitions": {
-    "rancher_config": {
-      "id": "#/definitions/rancher_config",
-      "type": "object",
-      "additionalProperties": false,
+	"definitions": {
+		"rancher_config": {
+			"id": "#/definitions/rancher_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "console": {"type": "string"},
-        "environment": {"type": "object"},
-        "cloud_init_services": {"type": "object"},
-        "services": {"type": "object"},
-        "bootstrap": {"type": "object"},
-        "autoformat": {"type": "object"},
-        "bootstrap_docker": {"$ref": "#/definitions/docker_config"},
-        "cloud_init": {"$ref": "#/definitions/cloud_init_config"},
-        "debug": {"type": "boolean"},
-        "rm_usr": {"type": "boolean"},
-        "no_sharedroot": {"type": "boolean"},
-        "log": {"type": "boolean"},
-        "force_console_rebuild": {"type": "boolean"},
-        "recovery": {"type": "boolean"},
-        "disable": {"$ref": "#/definitions/list_of_strings"},
-        "services_include": {"type": "object"},
-        "modules": {"$ref": "#/definitions/list_of_strings"},
-        "network": {"$ref": "#/definitions/network_config"},
-        "default_network": {"type": "object"},
-        "repositories": {"type": "object"},
-        "ssh": {"$ref": "#/definitions/ssh_config"},
-        "state": {"$ref": "#/definitions/state_config"},
-        "system_docker": {"$ref": "#/definitions/docker_config"},
-        "upgrade": {"$ref": "#/definitions/upgrade_config"},
-        "docker": {"$ref": "#/definitions/docker_config"},
-        "registry_auths": {"type": "object"},
-        "defaults": {"$ref": "#/definitions/defaults_config"},
-        "resize_device": {"type": "string"},
-        "sysctl": {"type": "object"},
-        "restart_services": {"type": "array"},
-	"hypervisor_service": {"type": "boolean"},
-        "shutdown_timeout": {"type": "integer"}
-      }
-    },
+			"properties": {
+				"console": {"type": "string"},
+				"environment": {"type": "object"},
+				"cloud_init_services": {"type": "object"},
+				"services": {"type": "object"},
+				"bootstrap": {"type": "object"},
+				"autoformat": {"type": "object"},
+				"bootstrap_docker": {"$ref": "#/definitions/docker_config"},
+				"cloud_init": {"$ref": "#/definitions/cloud_init_config"},
+				"debug": {"type": "boolean"},
+				"rm_usr": {"type": "boolean"},
+				"no_sharedroot": {"type": "boolean"},
+				"log": {"type": "boolean"},
+				"force_console_rebuild": {"type": "boolean"},
+				"recovery": {"type": "boolean"},
+				"disable": {"$ref": "#/definitions/list_of_strings"},
+				"services_include": {"type": "object"},
+				"modules": {"$ref": "#/definitions/list_of_strings"},
+				"network": {"$ref": "#/definitions/network_config"},
+				"repositories": {"type": "object"},
+				"ssh": {"$ref": "#/definitions/ssh_config"},
+				"state": {"$ref": "#/definitions/state_config"},
+				"system_docker": {"$ref": "#/definitions/docker_config"},
+				"upgrade": {"$ref": "#/definitions/upgrade_config"},
+				"docker": {"$ref": "#/definitions/docker_config"},
+				"registry_auths": {"type": "object"},
+				"defaults": {"$ref": "#/definitions/defaults_config"},
+				"resize_device": {"type": "string"},
+				"sysctl": {"type": "object"},
+				"restart_services": {"type": "array"},
+				"hypervisor_service": {"type": "boolean"},
+				"shutdown_timeout": {"type": "integer"},
+				"http_load_retries": {"type": "integer"},
+				"preload_wait": {"type": "boolean"}
+			}
+		},
 
-    "file_config": {
-      "id": "#/definitions/file_config",
-      "type": "object",
-      "additionalProperties": false,
+		"file_config": {
+			"id": "#/definitions/file_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "encoding": {"type": "string"},
-        "container": {"type": "string"},
-        "content": {"type": "string"},
-        "owner": {"type": "string"},
-        "path": {"type": "string"},
-        "permissions": {"type": "string"}
-      }
-    },
+			"properties": {
+				"encoding": {"type": "string"},
+				"container": {"type": "string"},
+				"content": {"type": "string"},
+				"owner": {"type": "string"},
+				"path": {"type": "string"},
+				"permissions": {"type": "string"}
+			}
+		},
 
-    "network_config": {
-      "id": "#/definitions/network_config",
-      "type": "object",
-      "additionalProperties": false,
+		"network_config": {
+			"id": "#/definitions/network_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "pre_cmds": {"$ref": "#/definitions/list_of_strings"},
-        "dns": {"type": "object"},
-        "interfaces": {"type": "object"},
-        "post_cmds": {"$ref": "#/definitions/list_of_strings"},
-        "http_proxy": {"type": "string"},
-        "https_proxy": {"type": "string"},
-        "no_proxy": {"type": "string"}
-      }
-    },
+			"properties": {
+				"pre_cmds": {"$ref": "#/definitions/list_of_strings"},
+				"dhcp_timeout": {"type": "integer"},
+				"dns": {"type": "object"},
+				"interfaces": {"type": "object"},
+				"post_cmds": {"$ref": "#/definitions/list_of_strings"},
+				"http_proxy": {"type": "string"},
+				"https_proxy": {"type": "string"},
+				"no_proxy": {"type": "string"},
+				"wifi_networks": {"type": "object"},
+				"modem_networks": {"type": "object"}
+			}
+		},
 
-    "upgrade_config": {
-      "id": "#/definitions/upgrade_config",
-      "type": "object",
-      "additionalProperties": false,
+		"upgrade_config": {
+			"id": "#/definitions/upgrade_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "url": {"type": "string"},
-        "image": {"type": "string"},
-        "rollback": {"type": "string"}
-      }
-    },
+			"properties": {
+				"url": {"type": "string"},
+				"image": {"type": "string"},
+				"rollback": {"type": "string"},
+				"policy": {"type": "string"}
+			}
+		},
 
-    "docker_config": {
-      "id": "#/definitions/docker_config",
-      "type": "object",
-      "additionalProperties": false,
+		"docker_config": {
+			"id": "#/definitions/docker_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "engine": {"type": "string"},
-        "tls": {"type": "boolean"},
-        "tls_args": {"$ref": "#/definitions/list_of_strings"},
-        "args": {"$ref": "#/definitions/list_of_strings"},
-        "extra_args": {"$ref": "#/definitions/list_of_strings"},
-        "server_cert": {"type": "string"},
-        "server_key": {"type": "string"},
-        "ca_cert": {"type": "string"},
-        "ca_key": {"type": "string"},
-        "environment": {"$ref": "#/definitions/list_of_strings"},
-        "storage_context": {"type": "string"},
-        "exec": {"type": ["boolean", "null"]},
-        "bridge": {"type": "string"},
-        "config_file": {"type": "string"},
-        "containerd": {"type": "string"},
-        "debug": {"type": ["boolean", "null"]},
-        "exec_root": {"type": "string"},
-        "group": {"type": "string"},
-        "graph": {"type": "string"},
-        "host": {"type": "array"},
-        "live_restore": {"type": ["boolean", "null"]},
-        "log_driver": {"type": "string"},
-        "log_opts": {"type": "object"},
-        "pid_file": {"type": "string"},
-        "registry_mirror": {"type": "string"},
-        "restart": {"type": ["boolean", "null"]},
-        "selinux_enabled": {"type": ["boolean", "null"]},
-        "storage_driver": {"type": "string"},
-        "userland_proxy": {"type": ["boolean", "null"]},
-        "insecure_registry": {"$ref": "#/definitions/list_of_strings"}
-      }
-    },
+			"properties": {
+				"engine": {"type": "string"},
+				"tls": {"type": "boolean"},
+				"tls_args": {"$ref": "#/definitions/list_of_strings"},
+				"args": {"$ref": "#/definitions/list_of_strings"},
+				"extra_args": {"$ref": "#/definitions/list_of_strings"},
+				"server_cert": {"type": "string"},
+				"server_key": {"type": "string"},
+				"ca_cert": {"type": "string"},
+				"ca_key": {"type": "string"},
+				"environment": {"$ref": "#/definitions/list_of_strings"},
+				"storage_context": {"type": "string"},
+				"exec": {"type": ["boolean", "null"]},
+				"bridge": {"type": "string"},
+				"bip": {"type": "string"},
+				"config_file": {"type": "string"},
+				"containerd": {"type": "string"},
+				"debug": {"type": ["boolean", "null"]},
+				"exec_root": {"type": "string"},
+				"group": {"type": "string"},
+				"graph": {"type": "string"},
+				"host": {"type": "array"},
+				"live_restore": {"type": ["boolean", "null"]},
+				"log_driver": {"type": "string"},
+				"log_opts": {"type": "object"},
+				"pid_file": {"type": "string"},
+				"registry_mirror": {"type": "string"},
+				"restart": {"type": ["boolean", "null"]},
+				"selinux_enabled": {"type": ["boolean", "null"]},
+				"storage_driver": {"type": "string"},
+				"userland_proxy": {"type": ["boolean", "null"]},
+				"insecure_registry": {"$ref": "#/definitions/list_of_strings"}
+			}
+		},
 
-    "ssh_config": {
-      "id": "#/definitions/ssh_config",
-      "type": "object",
-      "additionalProperties": false,
+		"ssh_config": {
+			"id": "#/definitions/ssh_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "keys": {"type": "object"},
-        "daemon": {"type": "boolean"}
-      }
-    },
+			"properties": {
+				"keys": {"type": "object"},
+				"daemon": {"type": "boolean"},
+				"port": {"type": "integer"},
+				"listen_address": {"type": "string"}
+			}
+		},
 
-    "state_config": {
-      "id": "#/definitions/state_config",
-      "type": "object",
-      "additionalProperties": false,
+		"state_config": {
+			"id": "#/definitions/state_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "directory": {"type": "string"},
-        "fstype": {"type": "string"},
-        "dev": {"type": "string"},
-        "wait": {"type": "boolean"},
-        "required": {"type": "boolean"},
-        "autoformat": {"$ref": "#/definitions/list_of_strings"},
-        "mdadm_scan": {"type": "boolean"},
-        "script": {"type": "string"},
-        "oem_fstype": {"type": "string"},
-        "oem_dev": {"type": "string"}
-      }
-    },
+			"properties": {
+				"directory": {"type": "string"},
+				"fstype": {"type": "string"},
+				"dev": {"type": "string"},
+				"wait": {"type": "boolean"},
+				"required": {"type": "boolean"},
+				"autoformat": {"$ref": "#/definitions/list_of_strings"},
+				"mdadm_scan": {"type": "boolean"},
+				"cryptsetup": {"type": "boolean"},
+				"lvm_scan": {"type": "boolean"},
+				"rngd": {"type": "boolean"},
+				"script": {"type": "string"},
+				"oem_fstype": {"type": "string"},
+				"oem_dev": {"type": "string"},
+				"boot_fstype": {"type": "string"},
+				"boot_dev": {"type": "string"}
+			}
+		},
 
-    "cloud_init_config": {
-      "id": "#/definitions/cloud_init_config",
-      "type": "object",
-      "additionalProperties": false,
+		"cloud_init_config": {
+			"id": "#/definitions/cloud_init_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "datasources": {"$ref": "#/definitions/list_of_strings"}
-      }
-    },
+			"properties": {
+				"datasources": {"$ref": "#/definitions/list_of_strings"}
+			}
+		},
 
-    "defaults_config": {
-      "id": "#/definitions/defaults_config",
-      "type": "object",
-      "additionalProperties": false,
+		"defaults_config": {
+			"id": "#/definitions/defaults_config",
+			"type": "object",
+			"additionalProperties": false,
 
-      "properties": {
-        "hostname": {"type": "string"},
-        "docker": {"type": "object"},
-        "network": {"$ref": "#/definitions/network_config"}
-      }
-    },
+			"properties": {
+				"hostname": {"type": "string"},
+				"docker": {"type": "object"},
+				"network": {"$ref": "#/definitions/network_config"},
+				"system_docker_logs": {"type": "string"}
+			}
+		},
 
-    "list_of_strings": {
-      "type": "array",
-      "items": {"type": "string"},
-      "uniqueItems": true
-    }
-  }
+		"list_of_strings": {
+			"type": "array",
+			"items": {"type": "string"},
+			"uniqueItems": true
+		}
+	}
 }
 
 `