From 13b34a66686c2d0c965e3964d49f33ec96703bfd Mon Sep 17 00:00:00 2001 From: Josh Curl Date: Wed, 9 Nov 2016 11:08:30 -0800 Subject: [PATCH] Early cloud-init --- cmd/cloudinitsave/cloudinitsave.go | 137 +++++++++++------- cmd/network/network.go | 44 +----- compose/project.go | 2 +- config/config.go | 2 +- config/disk.go | 23 ++- config/types.go | 1 + docker/service.go | 4 +- docs/_includes/os-sidebar.html | 10 +- .../built-in-system-services/index.md | 67 +-------- docs/os/boot-process/cloud-init/index.md | 28 ++++ .../image-preloading}/index.md | 6 +- images/02-cloudinit/Dockerfile | 3 - images/02-cloudinit/cloud-init.sh | 15 -- init/bootstrap.go | 20 +++ init/init.go | 99 +++++++++++-- os-config.tpl.yml | 78 ++++------ tests/cloud_init_test.go | 20 +++ tests/network_from_url_test.go | 2 +- 18 files changed, 312 insertions(+), 249 deletions(-) rename docs/os/{system-services => boot-process}/built-in-system-services/index.md (53%) create mode 100644 docs/os/boot-process/cloud-init/index.md rename docs/os/{configuration/prepacking-docker-images => boot-process/image-preloading}/index.md (91%) delete mode 100644 images/02-cloudinit/Dockerfile delete mode 100755 images/02-cloudinit/cloud-init.sh create mode 100644 tests/cloud_init_test.go diff --git a/cmd/cloudinitsave/cloudinitsave.go b/cmd/cloudinitsave/cloudinitsave.go index 5ced8b93..1348d040 100644 --- a/cmd/cloudinitsave/cloudinitsave.go +++ b/cmd/cloudinitsave/cloudinitsave.go @@ -17,10 +17,10 @@ package cloudinitsave import ( "errors" - "flag" "os" "strings" "sync" + "syscall" "time" yaml "github.com/cloudfoundry-incubator/candiedyaml" @@ -36,7 +36,9 @@ import ( "github.com/coreos/coreos-cloudinit/datasource/proc_cmdline" "github.com/coreos/coreos-cloudinit/datasource/url" "github.com/coreos/coreos-cloudinit/pkg" + "github.com/docker/docker/pkg/mount" "github.com/rancher/os/cmd/cloudinitsave/gce" + "github.com/rancher/os/cmd/network" rancherConfig "github.com/rancher/os/config" "github.com/rancher/os/netconf" "github.com/rancher/os/util" @@ -46,73 +48,42 @@ const ( datasourceInterval = 100 * time.Millisecond datasourceMaxInterval = 30 * time.Second datasourceTimeout = 5 * time.Minute + configDevName = "config-2" + configDev = "LABEL=" + configDevName + configDevMountPoint = "/media/config-2" ) -var ( - network bool - flags *flag.FlagSet -) - -func init() { - flags = flag.NewFlagSet(os.Args[0], flag.ContinueOnError) - flags.BoolVar(&network, "network", true, "use network based datasources") -} - func Main() { - flags.Parse(os.Args[1:]) + log.Info("Running cloud-init-save") - log.Infof("Running cloud-init-save: network=%v", network) + cfg := rancherConfig.LoadConfig() + network.ApplyNetworkConfig(cfg) - if err := saveCloudConfig(); err != nil { + if err := SaveCloudConfig(true); err != nil { log.Errorf("Failed to save cloud-config: %v", err) } } -func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error { - os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600) - - if len(scriptBytes) > 0 { - log.Infof("Writing to %s", rancherConfig.CloudConfigScriptFile) - if err := util.WriteFileAtomic(rancherConfig.CloudConfigScriptFile, scriptBytes, 500); err != nil { - log.Errorf("Error while writing file %s: %v", rancherConfig.CloudConfigScriptFile, err) - return err - } - } - - if len(cloudConfigBytes) > 0 { - if err := util.WriteFileAtomic(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil { - return err - } - log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes)) - } - - metaDataBytes, err := yaml.Marshal(metadata) - if err != nil { +func MountConfigDrive() error { + if err := os.MkdirAll(configDevMountPoint, 644); err != nil { return err } - if err = util.WriteFileAtomic(rancherConfig.MetaDataFile, metaDataBytes, 400); err != nil { - return err - } - log.Infof("Written to %s:\n%s", rancherConfig.MetaDataFile, string(metaDataBytes)) + configDev := util.ResolveDevice(configDev) - return nil -} - -func currentDatasource() (datasource.Datasource, error) { - cfg := rancherConfig.LoadConfig() - - dss := getDatasources(cfg) - if len(dss) == 0 { - return nil, nil + if configDev == "" { + return mount.Mount(configDevName, configDevMountPoint, "9p", "trans=virtio,version=9p2000.L") } - ds := selectDatasource(dss) - return ds, nil + return mount.Mount(configDev, configDevMountPoint, "iso9660,vfat", "") } -func saveCloudConfig() error { - userDataBytes, metadata, err := fetchUserData() +func UnmountConfigDrive() error { + return syscall.Unmount(configDevMountPoint, 0) +} + +func SaveCloudConfig(network bool) error { + userDataBytes, metadata, err := fetchUserData(network) if err != nil { return err } @@ -146,9 +117,67 @@ func saveCloudConfig() error { return saveFiles(userDataBytes, scriptBytes, metadata) } -func fetchUserData() ([]byte, datasource.Metadata, error) { +func RequiresNetwork(datasource string) bool { + parts := strings.SplitN(datasource, ":", 2) + requiresNetwork, ok := map[string]bool{ + "ec2": true, + "file": false, + "url": true, + "cmdline": true, + "configdrive": false, + "digitalocean": true, + "gce": true, + "packet": true, + }[parts[0]] + return ok && requiresNetwork +} + +func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error { + os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600) + + if len(scriptBytes) > 0 { + log.Infof("Writing to %s", rancherConfig.CloudConfigScriptFile) + if err := util.WriteFileAtomic(rancherConfig.CloudConfigScriptFile, scriptBytes, 500); err != nil { + log.Errorf("Error while writing file %s: %v", rancherConfig.CloudConfigScriptFile, err) + return err + } + } + + if len(cloudConfigBytes) > 0 { + if err := util.WriteFileAtomic(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil { + return err + } + log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes)) + } + + metaDataBytes, err := yaml.Marshal(metadata) + if err != nil { + return err + } + + if err = util.WriteFileAtomic(rancherConfig.MetaDataFile, metaDataBytes, 400); err != nil { + return err + } + log.Infof("Written to %s:\n%s", rancherConfig.MetaDataFile, string(metaDataBytes)) + + return nil +} + +func currentDatasource(network bool) (datasource.Datasource, error) { + cfg := rancherConfig.LoadConfig() + + dss := getDatasources(cfg, network) + if len(dss) == 0 { + return nil, nil + } + + ds := selectDatasource(dss) + return ds, nil +} + +func fetchUserData(network bool) ([]byte, datasource.Metadata, error) { var metadata datasource.Metadata - ds, err := currentDatasource() + ds, err := currentDatasource(network) if err != nil || ds == nil { log.Errorf("Failed to select datasource: %v", err) return nil, metadata, err @@ -170,7 +199,7 @@ func fetchUserData() ([]byte, datasource.Metadata, error) { // getDatasources creates a slice of possible Datasources for cloudinit based // on the different source command-line flags. -func getDatasources(cfg *rancherConfig.CloudConfig) []datasource.Datasource { +func getDatasources(cfg *rancherConfig.CloudConfig, network bool) []datasource.Datasource { dss := make([]datasource.Datasource, 0, 5) for _, ds := range cfg.Rancher.CloudInit.Datasources { diff --git a/cmd/network/network.go b/cmd/network/network.go index 1e26b395..63f5fd75 100644 --- a/cmd/network/network.go +++ b/cmd/network/network.go @@ -1,54 +1,24 @@ package network import ( - "flag" - "os" - - "golang.org/x/net/context" - log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/resolvconf" "github.com/rancher/os/config" - "github.com/rancher/os/docker" "github.com/rancher/os/hostname" "github.com/rancher/os/netconf" ) -var ( - stopNetworkPre bool - flags *flag.FlagSet -) - -func init() { - flags = flag.NewFlagSet(os.Args[0], flag.ContinueOnError) - flags.BoolVar(&stopNetworkPre, "stop-network-pre", false, "") -} - func Main() { - flags.Parse(os.Args[1:]) - - log.Infof("Running network: stop-network-pre=%v", stopNetworkPre) - - if stopNetworkPre { - client, err := docker.NewSystemClient() - if err != nil { - log.Error(err) - } - - err = client.ContainerStop(context.Background(), "network-pre", 10) - if err != nil { - log.Error(err) - } - - _, err = client.ContainerWait(context.Background(), "network-pre") - if err != nil { - log.Error(err) - } - } + log.Infof("Running network") cfg := config.LoadConfig() + ApplyNetworkConfig(cfg) + select {} +} + +func ApplyNetworkConfig(cfg *config.CloudConfig) { nameservers := cfg.Rancher.Network.Dns.Nameservers search := cfg.Rancher.Network.Dns.Search userSetDns := len(nameservers) > 0 || len(search) > 0 @@ -77,6 +47,4 @@ func Main() { if err := hostname.SyncHostname(); err != nil { log.Error(err) } - - select {} } diff --git a/compose/project.go b/compose/project.go index 37b4905e..2c1294f6 100644 --- a/compose/project.go +++ b/compose/project.go @@ -201,7 +201,7 @@ func newCoreServiceProject(cfg *config.CloudConfig, useNetwork, loadConsole bool go func() { for event := range projectEvents { - if event.EventType == events.ContainerStarted && event.ServiceName == "ntp" { + if event.EventType == events.ContainerStarted && event.ServiceName == "network" { useNetwork = true } } diff --git a/config/config.go b/config/config.go index 108a24a4..8d24c3c5 100644 --- a/config/config.go +++ b/config/config.go @@ -18,7 +18,7 @@ func Merge(bytes []byte) error { } func Export(private, full bool) (string, error) { - rawCfg := loadRawDiskConfig(full) + rawCfg := loadRawDiskConfig("", full) if !private { rawCfg = filterPrivateKeys(rawCfg) } diff --git a/config/disk.go b/config/disk.go index 92691c4a..a19e334c 100644 --- a/config/disk.go +++ b/config/disk.go @@ -31,27 +31,32 @@ func ReadConfig(bytes []byte, substituteMetadataVars bool, files ...string) (*Cl return c, nil } -func loadRawDiskConfig(full bool) map[interface{}]interface{} { +func loadRawDiskConfig(dirPrefix string, full bool) map[interface{}]interface{} { var rawCfg map[interface{}]interface{} if full { rawCfg, _ = readConfigs(nil, true, false, OsConfigFile, OemConfigFile) } - files := append(CloudConfigDirFiles(), CloudConfigFile) + files := CloudConfigDirFiles(dirPrefix) + files = append(files, path.Join(dirPrefix, CloudConfigFile)) additionalCfgs, _ := readConfigs(nil, true, false, files...) return util.Merge(rawCfg, additionalCfgs) } -func loadRawConfig() map[interface{}]interface{} { - rawCfg := loadRawDiskConfig(true) +func loadRawConfig(dirPrefix string) map[interface{}]interface{} { + rawCfg := loadRawDiskConfig(dirPrefix, true) rawCfg = util.Merge(rawCfg, readCmdline()) rawCfg = applyDebugFlags(rawCfg) return mergeMetadata(rawCfg, readMetadata()) } func LoadConfig() *CloudConfig { - rawCfg := loadRawConfig() + return LoadConfigWithPrefix("") +} + +func LoadConfigWithPrefix(dirPrefix string) *CloudConfig { + rawCfg := loadRawConfig(dirPrefix) cfg := &CloudConfig{} if err := util.Convert(rawCfg, cfg); err != nil { @@ -63,8 +68,10 @@ func LoadConfig() *CloudConfig { return cfg } -func CloudConfigDirFiles() []string { - files, err := ioutil.ReadDir(CloudConfigDir) +func CloudConfigDirFiles(dirPrefix string) []string { + cloudConfigDir := path.Join(dirPrefix, CloudConfigDir) + + files, err := ioutil.ReadDir(cloudConfigDir) if err != nil { if os.IsNotExist(err) { // do nothing @@ -78,7 +85,7 @@ func CloudConfigDirFiles() []string { var finalFiles []string for _, file := range files { if !file.IsDir() && !strings.HasPrefix(file.Name(), ".") { - finalFiles = append(finalFiles, path.Join(CloudConfigDir, file.Name())) + finalFiles = append(finalFiles, path.Join(cloudConfigDir, file.Name())) } } diff --git a/config/types.go b/config/types.go index 33700668..90e9e0b2 100644 --- a/config/types.go +++ b/config/types.go @@ -101,6 +101,7 @@ type RancherConfig struct { Environment map[string]string `yaml:"environment,omitempty"` Services map[string]*composeConfig.ServiceConfigV1 `yaml:"services,omitempty"` BootstrapContainers map[string]*composeConfig.ServiceConfigV1 `yaml:"bootstrap,omitempty"` + CloudInitServices map[string]*composeConfig.ServiceConfigV1 `yaml:"cloud_init_services,omitempty"` BootstrapDocker DockerConfig `yaml:"bootstrap_docker,omitempty"` CloudInit CloudInit `yaml:"cloud_init,omitempty"` Debug bool `yaml:"debug,omitempty"` diff --git a/docker/service.go b/docker/service.go index 561d96b0..fdc9045a 100644 --- a/docker/service.go +++ b/docker/service.go @@ -41,9 +41,7 @@ func (s *Service) DependentServices() []project.ServiceRelationship { } if s.requiresUserDocker() { - // Linking to cloud-init is a hack really. The problem is we need to link to something - // that will trigger a reload - rels = appendLink(rels, "cloud-init", false, s.project) + rels = appendLink(rels, "docker", false, s.project) } else if s.missingImage() { rels = appendLink(rels, "network", false, s.project) } diff --git a/docs/_includes/os-sidebar.html b/docs/_includes/os-sidebar.html index 52db9d0f..12387ed2 100644 --- a/docs/_includes/os-sidebar.html +++ b/docs/_includes/os-sidebar.html @@ -81,7 +81,6 @@
  • Setting up Docker TLS
  • Private Registries
  • Switching Docker Versions
    • -
  • Pre-packing Docker Images
  • Users
  • Resizing a Device Partition
  • sysctl Settings
    • @@ -96,10 +95,17 @@ +
  • + + +
  • diff --git a/docs/os/system-services/built-in-system-services/index.md b/docs/os/boot-process/built-in-system-services/index.md similarity index 53% rename from docs/os/system-services/built-in-system-services/index.md rename to docs/os/boot-process/built-in-system-services/index.md index 7a5410e4..f8ef473b 100644 --- a/docs/os/system-services/built-in-system-services/index.md +++ b/docs/os/boot-process/built-in-system-services/index.md @@ -1,54 +1,19 @@ --- title: Built-in System Services in RancherOS layout: os-default - +redirect_from: + - os/system-services/built-in-system-services/ --- ## Built-in System Services To launch RancherOS, we have built-in system services. They are defined in the [Docker Compose](https://docs.docker.com/compose/compose-file/) format, and can be found in the default system config file, `/usr/share/ros/os-config.yml`. You can [add your own system services]({{site.baseurl}}/os/system-services/) or override services in the cloud-config. -In start up order, here are the groups of services: +### preload-user-images -1. Device and power management: -- udev-cold -- udev -- acpid +Read more about [image preloading]({{site.baseurl}}/os/boot-process/image-preloading/). -2. syslog - -3. System configuration and networking: -- preload-system-images -- cloud-init-pre -- network-pre -- ntp -- cloud-init -- network - -4. User interaction: -- console -- docker - -5. Post configuration: -- preload-user-images - -### preload-system-images & preload-user-images - -Read more about [pre-packing Docker images]({{site.baseurl}}/os/configuration/prepacking-docker-images/). - -### cloud-init-pre - -User-data (i.e. [cloud-config]({{site.baseurl}}/os/configuration/#cloud-config)) and metadata from cloud provider, VM runtime, or a management service, is loaded in this service. - -The user-data is written to: - -* `/var/lib/rancher/conf/cloud-config.d/boot.yml` - If the user-data is a cloud-config, i.e. begins with `#cloud-config` and is YAML format. -* `/var/lib/rancher/conf/cloud-config-script` - If the user-data is a script, i.e begins with `#!`. -* `/var/lib/rancher/conf/metadata` - If it is serialized cloud provider metadata. - -It is configured by the `rancher.cloud_init.datasources` list in [cloud-config]({{site.baseurl}}/os/configuration/#cloud-config). It is pre-configured in cloud-provider specific images (e.g. AWS, GCE). - -### network-pre +### network During this service, networking is set up, e.g. hostname, interfaces, and DNS. @@ -58,16 +23,6 @@ It is configured by `hostname` and `rancher.network`[settings]({{site.baseurl}}/ Runs `ntpd` in a System Docker container. -### cloud-init - -It does the same thing as cloud-init-pre, but in this step, it can also use the network to fetch user-data and metadata (e.g. in cloud providers). - - -### network - -Completes setting up networking with configuration obtained by cloud-init. - - ### console This service provides the RancherOS user interface by running `sshd` and `getty`. It completes the RancherOS configuration on start up: @@ -90,20 +45,8 @@ This service provides the RancherOS user interface by running `sshd` and `getty` 6. Runs `/etc/rc.local` if it exists and is executable. Any errors are ignored. - ### docker This system service runs the user docker daemon. Normally it runs inside the console system container by running `docker-init` script which, in turn, looks for docker binaries in `/opt/bin`, `/usr/local/bin` and `/usr/bin`, adds the first found directory with docker binaries to PATH and runs `dockerlaunch docker daemon` appending the passed arguments. Docker daemon args are read from `rancher.docker.args` cloud-config property (followed by `rancher.docker.extra_args`). - -### RancherOS Configuration Load Order - -[Cloud-config]({{site.baseurl}}/os/configuration/#cloud-config/) is read by system services when they need to get configuration. Each additional file overwrites and extends the previous configuration file. - -1. `/usr/share/ros/os-config.yml` - This is the system default configuration, which should **not** be modified by users. -2. `/usr/share/ros/oem/oem-config.yml` - This will typically exist by OEM, which should **not** be modified by users. -3. Files in `/var/lib/rancher/conf/cloud-config.d/` ordered by filename. If a file is passed in through user-data, it is written by cloud-init and saved as `/var/lib/rancher/conf/cloud-config.d/boot.yml`. -4. `/var/lib/rancher/conf/cloud-config.yml` - If you set anything with `ros config set`, the changes are saved in this file. -5. Kernel parameters with names starting with `rancher`. -6. `/var/lib/rancher/conf/metadata` - Metadata added by cloud-init. diff --git a/docs/os/boot-process/cloud-init/index.md b/docs/os/boot-process/cloud-init/index.md new file mode 100644 index 00000000..7a14e2bd --- /dev/null +++ b/docs/os/boot-process/cloud-init/index.md @@ -0,0 +1,28 @@ +--- +title: Cloud-init +layout: os-default + +--- + +## Cloud-init + +Userdata and metadata can be fetched from a cloud provider, VM runtime, or management service during the RancherOS boot process. Since v0.8.0, this process occurs while RancherOS is still running from memory and before System Docker starts. It is configured by the `rancher.cloud_init.datasources` configuration parameter. For cloud-provider specific images, such as AWS and GCE, the datasource is pre-configured. + +### Userdata + +Userdata is a file given by users when launching RancherOS hosts. It is stored in different locations depending on its format. If the userdata is a [cloud-config]({{site.baseurl}}/os/configuration/#cloud-config) file, indicated by beginning with `#cloud-config` and being in YAML format, it is stored in `/var/lib/rancher/conf/cloud-config.d/boot.yml`. If the userdata is a script, indicated by beginning with `#!`, it is stored in `/var/lib/rancher/conf/cloud-config-script`. + +### Metadata + +Although the specifics vary based on provider, a metadata file will typically contain information about the RancherOS host and contain additional configuration. Its primary purpose within RancherOS is to provide an alternate source for SSH keys and hostname configuration. For example, AWS launches hosts with a set of authorized keys and RancherOS obtains these via metadata. Metadata is stored in `/var/lib/rancher/conf/metadata`. + +## Configuration Load Order + +[Cloud-config]({{site.baseurl}}/os/configuration/#cloud-config/) is read by system services when they need to get configuration. Each additional file overwrites and extends the previous configuration file. + +1. `/usr/share/ros/os-config.yml` - This is the system default configuration, which should **not** be modified by users. +2. `/usr/share/ros/oem/oem-config.yml` - This will typically exist by OEM, which should **not** be modified by users. +3. Files in `/var/lib/rancher/conf/cloud-config.d/` ordered by filename. If a file is passed in through user-data, it is written by cloud-init and saved as `/var/lib/rancher/conf/cloud-config.d/boot.yml`. +4. `/var/lib/rancher/conf/cloud-config.yml` - If you set anything with `ros config set`, the changes are saved in this file. +5. Kernel parameters with names starting with `rancher`. +6. `/var/lib/rancher/conf/metadata` - Metadata added by cloud-init. diff --git a/docs/os/configuration/prepacking-docker-images/index.md b/docs/os/boot-process/image-preloading/index.md similarity index 91% rename from docs/os/configuration/prepacking-docker-images/index.md rename to docs/os/boot-process/image-preloading/index.md index 534da2ee..fc2ab41e 100644 --- a/docs/os/configuration/prepacking-docker-images/index.md +++ b/docs/os/boot-process/image-preloading/index.md @@ -1,10 +1,12 @@ --- -title: Pre-packing Docker Images +title: Image Preloading layout: os-default +redirect_from: + - os/configuration/prepacking-docker-images/ --- -## Pre-packing Docker Images +## Image Preloading --- On boot, RancherOS scans `/var/lib/rancher/preload/docker` and `/var/lib/rancher/preload/system-docker` directories and tries to load container image archives it finds there, with `docker load` and `system-docker load`. diff --git a/images/02-cloudinit/Dockerfile b/images/02-cloudinit/Dockerfile deleted file mode 100644 index 88aa0d6c..00000000 --- a/images/02-cloudinit/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM rancher/os-base -COPY cloud-init.sh / -CMD ["/cloud-init.sh"] diff --git a/images/02-cloudinit/cloud-init.sh b/images/02-cloudinit/cloud-init.sh deleted file mode 100755 index 3c6c0c0a..00000000 --- a/images/02-cloudinit/cloud-init.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -x -e - -MOUNT_POINT=/media/config-2 -CONFIG_DEV=$(ros dev "LABEL=config-2") - -mkdir -p ${MOUNT_POINT} - -if [ -e "${CONFIG_DEV}" ]; then - mount -t iso9660,vfat ${CONFIG_DEV} ${MOUNT_POINT} -else - mount -t 9p -o trans=virtio,version=9p2000.L config-2 ${MOUNT_POINT} 2>/dev/null || true -fi - -cloud-init-save -network=${CLOUD_INIT_NETWORK:-true} diff --git a/init/bootstrap.go b/init/bootstrap.go index 026197bc..a69383ea 100644 --- a/init/bootstrap.go +++ b/init/bootstrap.go @@ -19,6 +19,12 @@ func bootstrapServices(cfg *config.CloudConfig) (*config.CloudConfig, error) { return cfg, err } +func runCloudInitServiceSet(cfg *config.CloudConfig) (*config.CloudConfig, error) { + log.Info("Running cloud-init services") + _, err := compose.RunServiceSet("cloud-init", cfg, cfg.Rancher.CloudInitServices) + return cfg, err +} + func startDocker(cfg *config.CloudConfig) (chan interface{}, error) { launchConfig, args := getLaunchConfig(cfg, &cfg.Rancher.BootstrapDocker) launchConfig.Fork = true @@ -62,3 +68,17 @@ func bootstrap(cfg *config.CloudConfig) error { bootstrapServices) return err } + +func runCloudInitServices(cfg *config.CloudConfig) error { + c, err := startDocker(cfg) + if err != nil { + return err + } + + defer stopDocker(c) + + _, err = config.ChainCfgFuncs(cfg, + loadImages, + runCloudInitServiceSet) + return err +} diff --git a/init/init.go b/init/init.go index befd6924..e7a875c7 100644 --- a/init/init.go +++ b/init/init.go @@ -5,6 +5,7 @@ package init import ( "bufio" "fmt" + "io/ioutil" "os" "os/exec" "strings" @@ -12,6 +13,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/mount" + "github.com/rancher/os/cmd/cloudinitsave" "github.com/rancher/os/config" "github.com/rancher/os/dfs" "github.com/rancher/os/util" @@ -143,23 +145,18 @@ func tryMountState(cfg *config.CloudConfig) error { return mountState(cfg) } -func tryMountAndBootstrap(cfg *config.CloudConfig) (*config.CloudConfig, error) { +func tryMountAndBootstrap(cfg *config.CloudConfig) (*config.CloudConfig, bool, error) { if !isInitrd() || cfg.Rancher.State.Dev == "" { - return cfg, nil + return cfg, false, nil } if err := tryMountState(cfg); !cfg.Rancher.State.Required && err != nil { - return cfg, nil + return cfg, false, nil } else if err != nil { - return cfg, err + return cfg, false, err } - log.Debugf("Switching to new root at %s %s", STATE, cfg.Rancher.State.Directory) - if err := switchRoot(STATE, cfg.Rancher.State.Directory, cfg.Rancher.RmUsr); err != nil { - return cfg, err - } - - return mountOem(cfg) + return cfg, true, nil } func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*dfs.Config, []string) { @@ -218,6 +215,9 @@ func RunInit() error { } boot2DockerEnvironment := false + var shouldSwitchRoot bool + var cloudConfigBootFile []byte + var metadataFile []byte initFuncs := []config.CfgFunc{ func(c *config.CloudConfig) (*config.CloudConfig, error) { return c, dfs.PrepareFs(&mountConfig) @@ -265,7 +265,84 @@ func RunInit() error { return cfg, nil }, - tryMountAndBootstrap, + func(cfg *config.CloudConfig) (*config.CloudConfig, error) { + var err error + cfg, shouldSwitchRoot, err = tryMountAndBootstrap(cfg) + if err != nil { + return nil, err + } + return cfg, nil + }, + func(cfg *config.CloudConfig) (*config.CloudConfig, error) { + if err := os.MkdirAll(config.CloudConfigDir, os.ModeDir|0755); err != nil { + log.Error(err) + } + + cfg.Rancher.CloudInit.Datasources = config.LoadConfigWithPrefix(STATE).Rancher.CloudInit.Datasources + if err := config.Set("rancher.cloud_init.datasources", cfg.Rancher.CloudInit.Datasources); err != nil { + log.Error(err) + } + + network := false + for _, datasource := range cfg.Rancher.CloudInit.Datasources { + if cloudinitsave.RequiresNetwork(datasource) { + network = true + break + } + } + + if network { + if err := runCloudInitServices(cfg); err != nil { + log.Error(err) + } + } else { + if err := cloudinitsave.MountConfigDrive(); err != nil { + log.Error(err) + } + if err := cloudinitsave.SaveCloudConfig(false); err != nil { + log.Error(err) + } + if err := cloudinitsave.UnmountConfigDrive(); err != nil { + log.Error(err) + } + } + return cfg, nil + }, + func(cfg *config.CloudConfig) (*config.CloudConfig, error) { + var err error + cloudConfigBootFile, err = ioutil.ReadFile(config.CloudConfigBootFile) + if err != nil { + log.Error(err) + } + metadataFile, err = ioutil.ReadFile(config.MetaDataFile) + if err != nil { + log.Error(err) + } + return cfg, nil + }, + func(cfg *config.CloudConfig) (*config.CloudConfig, error) { + if !shouldSwitchRoot { + return cfg, nil + } + log.Debugf("Switching to new root at %s %s", STATE, cfg.Rancher.State.Directory) + if err := switchRoot(STATE, cfg.Rancher.State.Directory, cfg.Rancher.RmUsr); err != nil { + return cfg, err + } + return cfg, nil + }, + mountOem, + func(cfg *config.CloudConfig) (*config.CloudConfig, error) { + if err := os.MkdirAll(config.CloudConfigDir, os.ModeDir|0755); err != nil { + log.Error(err) + } + if err := util.WriteFileAtomic(config.CloudConfigBootFile, cloudConfigBootFile, 400); err != nil { + log.Error(err) + } + if err := util.WriteFileAtomic(config.MetaDataFile, metadataFile, 400); err != nil { + log.Error(err) + } + return cfg, nil + }, func(cfg *config.CloudConfig) (*config.CloudConfig, error) { if boot2DockerEnvironment { if err := config.Set("rancher.state.dev", cfg.Rancher.State.Dev); err != nil { diff --git a/os-config.tpl.yml b/os-config.tpl.yml index 0d2d652f..128a0593 100644 --- a/os-config.tpl.yml +++ b/os-config.tpl.yml @@ -30,6 +30,30 @@ rancher: - /lib/firmware:/lib/firmware - /usr/bin/ros:/usr/bin/ros:ro - /usr/share/ros:/usr/share/ros:ro + cloud_init_services: + cloud-init: + image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}} + command: cloud-init-save + labels: + io.rancher.os.detach: "false" + io.rancher.os.scope: system + io.rancher.os.after: ntp + log_driver: json-file + net: host + uts: host + pid: host + ipc: host + privileged: true + volumes: + - /dev:/host/dev + - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher + - /lib/modules:/lib/modules + - /lib/firmware:/lib/firmware + - /usr/bin/ros:/usr/bin/ros:ro + - /usr/bin/ros:/usr/bin/cloud-init-save + - /usr/share/ros:/usr/share/ros:ro + - /var/lib/rancher:/var/lib/rancher + - /var/lib/rancher/conf:/var/lib/rancher/conf bootstrap_docker: bridge: none storage_driver: overlay @@ -80,43 +104,13 @@ rancher: - command-volumes - user-volumes - system-volumes - cloud-init: - image: {{.OS_REPO}}/os-cloudinit:{{.VERSION}}{{.SUFFIX}} - labels: - io.rancher.os.detach: "false" - io.rancher.os.reloadconfig: "true" - io.rancher.os.scope: system - io.rancher.os.after: ntp - net: host - uts: host - privileged: true - volumes_from: - - command-volumes - - system-volumes cloud-init-execute: image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}} command: cloud-init-execute -pre-console labels: io.rancher.os.detach: "false" io.rancher.os.scope: system - io.rancher.os.after: cloud-init - net: host - uts: host - privileged: true - volumes_from: - - system-volumes - volumes: - - /usr/bin/ros:/usr/bin/ros - - /usr/bin/ros:/usr/bin/cloud-init-execute - cloud-init-pre: - image: {{.OS_REPO}}/os-cloudinit:{{.VERSION}}{{.SUFFIX}} - environment: - - CLOUD_INIT_NETWORK=false - labels: - io.rancher.os.detach: "false" - io.rancher.os.reloadconfig: "true" - io.rancher.os.scope: system - io.rancher.os.after: udev + io.rancher.os.after: ntp net: host uts: host privileged: true @@ -151,7 +145,7 @@ rancher: command: ros console-init labels: io.rancher.os.scope: system - io.rancher.os.after: network + io.rancher.os.after: cloud-init-execute io.docker.compose.rebuild: always io.rancher.os.console: default net: host @@ -178,25 +172,13 @@ rancher: read_only: true volumes: - /var/lib/docker:/var/lib/docker - network-pre: + network: image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}} command: netconf labels: io.rancher.os.scope: system - io.rancher.os.after: cloud-init-pre - net: host - uts: host - pid: host - privileged: true - volumes_from: - - command-volumes - - system-volumes - network: - image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}} - command: netconf --stop-network-pre - labels: - io.rancher.os.scope: system - io.rancher.os.after: cloud-init-execute + io.rancher.os.after: udev + io.rancher.os.reloadconfig: "true" net: host uts: host pid: host @@ -209,7 +191,7 @@ rancher: command: ntpd --nofork -g labels: io.rancher.os.scope: system - io.rancher.os.after: network-pre + io.rancher.os.after: network net: host uts: host privileged: true diff --git a/tests/cloud_init_test.go b/tests/cloud_init_test.go new file mode 100644 index 00000000..a04d0d05 --- /dev/null +++ b/tests/cloud_init_test.go @@ -0,0 +1,20 @@ +package integration + +import . "gopkg.in/check.v1" + +func (s *QemuSuite) TestReadDatasourcesFromDisk(c *C) { + s.RunQemu(c) + + s.CheckCall(c, ` +sudo tee /var/lib/rancher/conf/cloud-config.d/datasources.yml << EOF +rancher: + cloud_init: + datasources: + - url:https://gist.githubusercontent.com/joshwget/e1c49f8b1ddeeba01bc9d0a3be01ed60/raw/9168b380fde182d53acea487d49b680648a0ca5b/gistfile1.txt +EOF +`) + + s.Reboot(c) + + s.CheckCall(c, "sudo ros config get rancher.log | grep true") +} diff --git a/tests/network_from_url_test.go b/tests/network_from_url_test.go index 8c411c50..1effaa39 100644 --- a/tests/network_from_url_test.go +++ b/tests/network_from_url_test.go @@ -4,7 +4,7 @@ import . "gopkg.in/check.v1" func (s *QemuSuite) TestNetworkFromUrl(c *C) { netArgs := []string{"-net", "nic,vlan=0,model=virtio"} - args := []string{"--cloud-config", "./tests/assets/test_10/cloud-config.yml"} + args := []string{"--append", "rancher.cloud_init.datasources=[url:https://gist.githubusercontent.com/joshwget/0bdc616cd26162ad87c535644c8b1ef6/raw/8cce947c08cf006e932b71d92ddbb96bae8e3325/gistfile1.txt]"} for i := 0; i < 7; i++ { args = append(args, netArgs...) }