Replace wait-for-network services with retries

Dockerfile.dapper

@@ -101,9 +101,11 @@ RUN if [ -n "${!VBOX_MODULES_URL}" ]; then \
     ;fi
 
 # Install Go
+COPY assets/go-dnsclient.patch ${DAPPER_SOURCE}
 ENV GO_VERSION 1.6.2
 RUN ln -sf go-6 /usr/bin/go && \
     curl -sfL https://storage.googleapis.com/golang/go${GO_VERSION}.src.tar.gz | tar -xzf - -C /usr/local && \
+    patch /usr/local/go/src/net/dnsclient_unix.go ${DAPPER_SOURCE}/go-dnsclient.patch && \
     cd /usr/local/go/src && \
     GOROOT_BOOTSTRAP=/usr GOARCH=${HOST_ARCH} GOHOSTARCH=${HOST_ARCH} ./make.bash && \
     rm /usr/bin/go

assets/go-dnsclient.patch

@@ -0,0 +1,24 @@
+265,270d264
+< 	// Ensure only one update at a time checks resolv.conf.
+< 	if !conf.tryAcquireSema() {
+< 		return
+< 	}
+< 	defer conf.releaseSema()
+< 
+276a271,280
+> 	conf.update(name)
+> }
+> 
+> func (conf *resolverConfig) update(name string) {
+> 	// Ensure only one update at a time checks resolv.conf.
+> 	if !conf.tryAcquireSema() {
+> 		return
+> 	}
+> 	defer conf.releaseSema()
+> 
+293a298,302
+> }
+> 
+> func UpdateDnsConf() {
+> 	resolvConf.initOnce.Do(resolvConf.init)
+> 	resolvConf.update("/etc/resolv.conf")

cmd/network/network.go

@@ -3,44 +3,50 @@ package network
 import (
 	"flag"
 	"os"
-	"os/exec"
+
+	"golang.org/x/net/context"
 
 	log "github.com/Sirupsen/logrus"
 
 	"github.com/docker/libnetwork/resolvconf"
 	"github.com/rancher/netconf"
 	"github.com/rancher/os/config"
+	"github.com/rancher/os/docker"
 	"github.com/rancher/os/hostname"
 )
 
-const (
-	NETWORK_DONE     = "/var/run/network.done"
-	WAIT_FOR_NETWORK = "wait-for-network"
-)
-
 var (
-	daemon bool
+	stopNetworkPre bool
-	flags  *flag.FlagSet
+	flags          *flag.FlagSet
 )
 
 func init() {
 	flags = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
-	flags.BoolVar(&daemon, "daemon", false, "run dhcpd as daemon")
+	flags.BoolVar(&stopNetworkPre, "stop-network-pre", false, "")
-}
-
-func sendTerm(proc string) {
-	cmd := exec.Command("killall", "-TERM", proc)
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stdout
-	cmd.Run()
 }
 
 func Main() {
 	flags.Parse(os.Args[1:])
 
-	log.Infof("Running network: daemon=%v", daemon)
+	log.Infof("Running network: stop-network-pre=%v", stopNetworkPre)
+
+	if stopNetworkPre {
+		client, err := docker.NewSystemClient()
+		if err != nil {
+			log.Error(err)
+		}
+
+		err = client.ContainerStop(context.Background(), "network-pre", 10)
+		if err != nil {
+			log.Error(err)
+		}
+
+		_, err = client.ContainerWait(context.Background(), "network-pre")
+		if err != nil {
+			log.Error(err)
+		}
+	}
 
-	os.Remove(NETWORK_DONE) // ignore error
 	cfg, err := config.LoadConfig()
 	if err != nil {
 		log.Fatal(err)
@@ -75,14 +81,5 @@ func Main() {
 		log.Error(err)
 	}
 
-	if f, err := os.Create(NETWORK_DONE); err != nil {
+	select {}
-		log.Error(err)
-	} else {
-		f.Close()
-	}
-	sendTerm(WAIT_FOR_NETWORK)
-
-	if daemon {
-		select {}
-	}
 }

cmd/waitfornetwork/waitfornetwork.go

@@ -1,23 +0,0 @@
-package waitfornetwork
-
-import (
-	"github.com/rancher/os/cmd/network"
-	"os"
-	"os/signal"
-	"syscall"
-)
-
-func handleTerm() {
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, syscall.SIGTERM)
-	<-c
-	os.Exit(0)
-}
-
-func Main() {
-	go handleTerm()
-	if _, err := os.Stat(network.NETWORK_DONE); err == nil {
-		os.Exit(0)
-	}
-	select {}
-}

images/02-network/Dockerfile

@@ -1,3 +0,0 @@
-FROM rancher/os-base
-COPY network.sh  /
-CMD ["/network.sh"]

images/02-network/network.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-set -x -e
-
-netconf -daemon=${DAEMON:-false}

main.go

@@ -16,7 +16,6 @@ import (
 	"github.com/rancher/os/cmd/systemdocker"
 	"github.com/rancher/os/cmd/userdocker"
 	"github.com/rancher/os/cmd/wait"
-	"github.com/rancher/os/cmd/waitfornetwork"
 	"github.com/rancher/os/config"
 	osInit "github.com/rancher/os/init"
 )
@@ -53,7 +52,6 @@ func main() {
 	registerCmd("/usr/bin/ros", control.Main)
 	registerCmd("/usr/bin/cloud-init", cloudinit.Main)
 	registerCmd("/usr/sbin/netconf", network.Main)
-	registerCmd("/usr/sbin/wait-for-network", waitfornetwork.Main)
 	registerCmd("/usr/sbin/wait-for-docker", wait.Main)
 
 	if !reexec.Init() {

os-config.tpl.yml

@@ -149,13 +149,12 @@ rancher:
       - /usr/bin/ros:/usr/bin/ros:ro
       - /usr/bin/ros:/usr/bin/cloud-init:ro
       - /usr/bin/ros:/usr/sbin/netconf:ro
-      - /usr/bin/ros:/usr/sbin/wait-for-network:ro
       - /usr/bin/ros:/usr/sbin/wait-for-docker:ro
     console:
       image: {{.OS_REPO}}/os-console:{{.VERSION}}{{.SUFFIX}}
       labels:
         io.rancher.os.scope: system
-        io.rancher.os.after: wait-for-network
+        io.rancher.os.after: network
         io.docker.compose.rebuild: always
       net: host
       uts: host
@@ -181,9 +180,9 @@ rancher:
       - /var/lib/system-docker:/var/lib/system-docker
       - /var/lib/rkt:/var/lib/rkt
     network-pre:
-      image: {{.OS_REPO}}/os-network:{{.VERSION}}{{.SUFFIX}}
+      image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}}
+      command: netconf
       labels:
-        io.rancher.os.detach: "false"
         io.rancher.os.scope: system
         io.rancher.os.after: cloud-init-pre
       net: host
@@ -194,12 +193,11 @@ rancher:
       - command-volumes
       - system-volumes
     network:
-      image: {{.OS_REPO}}/os-network:{{.VERSION}}{{.SUFFIX}}
+      image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}}
+      command: netconf --stop-network-pre
       labels:
         io.rancher.os.scope: system
         io.rancher.os.after: cloud-init
-      environment:
-      - DAEMON=true
       net: host
       uts: host
       pid: host
@@ -207,35 +205,11 @@ rancher:
       volumes_from:
       - command-volumes
       - system-volumes
-    wait-for-network-pre:
-      image: {{.OS_REPO}}/os-network:{{.VERSION}}{{.SUFFIX}}
-      command: wait-for-network
-      labels:
-        io.rancher.os.detach: "false"
-        io.rancher.os.scope: system
-        io.rancher.os.after: network-pre
-      pid: host
-      privileged: true
-      volumes_from:
-      - command-volumes
-      - system-volumes
-    wait-for-network:
-      image: {{.OS_REPO}}/os-network:{{.VERSION}}{{.SUFFIX}}
-      command: wait-for-network
-      labels:
-        io.rancher.os.detach: "false"
-        io.rancher.os.scope: system
-        io.rancher.os.after: network
-      pid: host
-      privileged: true
-      volumes_from:
-      - command-volumes
-      - system-volumes
     ntp:
       image: {{.OS_REPO}}/os-ntp:{{.VERSION}}{{.SUFFIX}}
       labels:
         io.rancher.os.scope: system
-        io.rancher.os.after: wait-for-network-pre
+        io.rancher.os.after: network-pre
       net: host
       uts: host
       privileged: true

tests/integration/assets/test_10/cloud-config.yml

@@ -2,4 +2,4 @@
 rancher:
   cloud_init:
     datasources:
-    - url:https://s3-us-west-2.amazonaws.com/rancher-os-test/cloud-config.yml
+    - url:https://gist.githubusercontent.com/joshwget/0bdc616cd26162ad87c535644c8b1ef6/raw/8cce947c08cf006e932b71d92ddbb96bae8e3325/gistfile1.txt

tests/integration/assets/test_18/cloud-config.yml

@@ -0,0 +1,11 @@
+#cloud-config
+rancher:
+  services:
+    missing_image:
+      image: tianon/true
+      labels:
+        io.rancher.os.scope: system
+  services_include:
+    debian-console: true
+ssh_authorized_keys:
+  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC85w9stZyiLQp/DkVO6fqwiShYcj1ClKdtCqgHtf+PLpJkFReSFu8y21y+ev09gsSMRRrjF7yt0pUHV6zncQhVeqsZtgc5WbELY2DOYUGmRn/CCvPbXovoBrQjSorqlBmpuPwsStYLr92Xn+VVsMNSUIegHY22DphGbDKG85vrKB8HxUxGIDxFBds/uE8FhSy+xsoyT/jUZDK6pgq2HnGl6D81ViIlKecpOpWlW3B+fea99ADNyZNVvDzbHE5pcI3VRw8u59WmpWOUgT6qacNVACl8GqpBvQk8sw7O/X9DSZHCKafeD9G5k+GYbAUz92fKWrx/lOXfUXPS3+c8dRIF

tests/integration/rostest/test_10_network_from_url.py

@@ -28,9 +28,6 @@ set -x -e
 ip link show dev br0
 ip link show dev br0.100 | grep br0.100@br0
 ip link show dev eth1.100 | grep 'master br0'
-ip link show dev eth6 | grep 'master bond0'
-ip link show dev eth7 | grep 'master bond0'
-[ "$(</sys/class/net/bond0/bonding/mode)" = "active-backup 1" ]
 
 SCRIPT
 sudo bash test-merge

tests/integration/rostest/test_18_network_on_boot.py

@@ -0,0 +1,26 @@
+import pytest
+import rostest.util as u
+from rostest.util import SSH
+
+pytestmark = pytest.mark.skipif(u.arch != 'amd64', reason='amd64 network setup impossible to replicate for arm64')
+
+cloud_config_path = './tests/integration/assets/test_18/cloud-config.yml'
+
+net_args_arch = {'amd64': ['-net', 'nic,vlan=1,model=virtio'],
+                 'arm64': ['-device', 'virtio-net-device']}
+net_args_arch['arm'] = net_args_arch['arm64']
+net_args = net_args_arch[u.arch]
+
+
+@pytest.fixture(scope="module")
+def qemu(request):
+    q = u.run_qemu(request,
+                   run_args=['--cloud-config', cloud_config_path] +
+                   net_args + net_args + net_args)
+    u.flush_out(q.stdout)
+    return q
+
+
+def test_network_resources_loaded(qemu):
+    SSH(qemu).check_call("apt-get --version")
+    SSH(qemu).check_call("sudo system-docker images | grep tianon/true")

util/network/network.go

@@ -4,8 +4,10 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"strings"
+	"time"
 
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 
@@ -44,41 +46,32 @@ func GetServices(urls []string) ([]string, error) {
 	return result, nil
 }
 
-func retryHttp(f func() (*http.Response, error), times int) (resp *http.Response, err error) {
+func loadFromNetwork(location string) ([]byte, error) {
-	for i := 0; i < times; i++ {
+	var err error
-		if resp, err = f(); err == nil {
+	for i := 0; i < 300; i++ {
-			return
+		net.UpdateDnsConf()
+
+		var resp *http.Response
+		resp, err = http.Get(location)
+		if err == nil {
+			defer resp.Body.Close()
+			if resp.StatusCode != http.StatusOK {
+				return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode)
+			}
+
+			bytes, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return nil, err
+			}
+
+			cacheAdd(location, bytes)
+			return bytes, nil
 		}
-		log.Warnf("Error making HTTP request: %s. Retrying", err)
-	}
-	return
-}
 
-func loadFromNetwork(location string, network bool) ([]byte, error) {
+		time.Sleep(100 * time.Millisecond)
-	bytes := cacheLookup(location)
-	if bytes != nil {
-		return bytes, nil
 	}
 
-	resp, err := retryHttp(func() (*http.Response, error) {
+	return nil, err
-		return http.Get(location)
-	}, 8)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode)
-	}
-	defer resp.Body.Close()
-
-	bytes, err = ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	cacheAdd(location, bytes)
-
-	return bytes, nil
 }
 
 func LoadResource(location string, network bool) ([]byte, error) {
@@ -86,7 +79,7 @@ func LoadResource(location string, network bool) ([]byte, error) {
 		if !network {
 			return nil, ErrNoNetwork
 		}
-		return loadFromNetwork(location, network)
+		return loadFromNetwork(location)
 	} else if strings.HasPrefix(location, "/") {
 		return ioutil.ReadFile(location)
 	}
@@ -111,12 +104,12 @@ func LoadServiceResource(name string, useNetwork bool, cfg *config.CloudConfig)
 	urls := cfg.Rancher.Repositories.ToArray()
 	for _, url := range urls {
 		serviceUrl := serviceUrl(url, name)
-		bytes, err := LoadResource(serviceUrl, useNetwork)
+		bytes, err = LoadResource(serviceUrl, useNetwork)
 		if err == nil {
 			log.Debugf("Loaded %s from %s", name, serviceUrl)
 			return bytes, nil
 		}
 	}
 
-	return nil, ErrNotFound
+	return nil, err
 }