Merge pull request #2062 from SvenDowideit/resolv.conf_race

Stop overwriting a good resolv.conf file with the default one

Dockerfile.dapper

@@ -105,7 +105,7 @@ ENV BUILD_DOCKER_URL=BUILD_DOCKER_URL_${ARCH} \
     DOCKER_VERSION=${DOCKER_VERSION} \
     DOWNLOADS=/usr/src/downloads \
     GOPATH=/go \
-    GO_VERSION=1.7.1 \
+    GO_VERSION=1.7.6 \
     GOARCH=$ARCH \
     HOSTNAME_DEFAULT=${HOSTNAME_DEFAULT} \
     IMAGE_NAME=${IMAGE_NAME} \
@@ -141,7 +141,6 @@ RUN curl -pfL ${SELINUX_POLICY_URL} > ${DOWNLOADS}/$(basename ${SELINUX_POLICY_U
 COPY assets/go-dnsclient.patch ${DAPPER_SOURCE}
 RUN ln -sf go-6 /usr/bin/go && \
     curl -sfL https://storage.googleapis.com/golang/go${GO_VERSION}.src.tar.gz | tar -xzf - -C /usr/local && \
-    patch /usr/local/go/src/net/dnsclient_unix.go ${DAPPER_SOURCE}/go-dnsclient.patch && \
     cd /usr/local/go/src && \
     GOROOT_BOOTSTRAP=/usr GOARCH=${HOST_ARCH} GOHOSTARCH=${HOST_ARCH} ./make.bash && \
     rm /usr/bin/go

cmd/cloudinitsave/cloudinitsave.go

@@ -271,7 +271,7 @@ func getDatasources(datasources []string) []datasource.Datasource {
 }
 
 func enableDoLinkLocal() {
-	err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
+	_, err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
 		Interfaces: map[string]netconf.InterfaceConfig{
 			"eth0": {
 				IPV4LL: true,

cmd/network/network.go

@@ -10,6 +10,7 @@ import (
 	"github.com/rancher/os/config"
 	"github.com/rancher/os/hostname"
 	"github.com/rancher/os/netconf"
+	"io/ioutil"
 )
 
 func Main() {
@@ -33,29 +34,40 @@ func Main() {
 
 func ApplyNetworkConfig(cfg *config.CloudConfig) {
 	log.Infof("Apply Network Config")
-	nameservers := cfg.Rancher.Network.DNS.Nameservers
+	userSetDNS := len(cfg.Rancher.Network.DNS.Nameservers) > 0 || len(cfg.Rancher.Network.DNS.Search) > 0
-	search := cfg.Rancher.Network.DNS.Search
+	if userSetDNS {
-	userSetDNS := len(nameservers) > 0 || len(search) > 0
+		if _, err := resolvconf.Build("/etc/resolv.conf", cfg.Rancher.Network.DNS.Nameservers, cfg.Rancher.Network.DNS.Search, nil); err != nil {
-	if !userSetDNS {
-		nameservers = cfg.Rancher.Defaults.Network.DNS.Nameservers
-		search = cfg.Rancher.Defaults.Network.DNS.Search
-	}
-
-	// TODO: don't write to the file if nameservers is still empty
-	log.Infof("Writing resolv.conf (%v) %v", nameservers, search)
-	if _, err := resolvconf.Build("/etc/resolv.conf", nameservers, search, nil); err != nil {
 			log.Error(err)
 		}
+	}
 
 	if err := hostname.SetHostnameFromCloudConfig(cfg); err != nil {
 		log.Error(err)
 	}
 
 	userSetHostname := cfg.Hostname != ""
-	if err := netconf.ApplyNetworkConfigs(&cfg.Rancher.Network, userSetHostname, userSetDNS); err != nil {
+	dhcpSetDNS, err := netconf.ApplyNetworkConfigs(&cfg.Rancher.Network, userSetHostname, userSetDNS)
+	if err != nil {
 		log.Error(err)
 	}
 
+	if dhcpSetDNS {
+		log.Infof("DNS set by DHCP")
+	}
+
+	if !userSetDNS && !dhcpSetDNS {
+		// only write 8.8.8.8,8.8.4.4 as a last resort
+		log.Infof("Writing default resolv.conf - no user setting, and no DHCP setting")
+		if _, err := resolvconf.Build("/etc/resolv.conf",
+			cfg.Rancher.Defaults.Network.DNS.Nameservers,
+			cfg.Rancher.Defaults.Network.DNS.Search,
+			nil); err != nil {
+			log.Error(err)
+		}
+	}
+	resolve, err := ioutil.ReadFile("/etc/resolv.conf")
+	log.Debugf("Resolve.conf == [%s], %s", resolve, err)
+
 	log.Infof("Apply Network Config SyncHostname")
 	if err := hostname.SyncHostname(); err != nil {
 		log.Error(err)

cmd/sysinit/sysinit.go

@@ -3,10 +3,17 @@ package sysinit
 import (
 	initPkg "github.com/rancher/os/init"
 	"github.com/rancher/os/log"
+	"io/ioutil"
+	"os"
 )
 
 func Main() {
 	log.InitLogger()
+
+	resolve, err := ioutil.ReadFile("/etc/resolv.conf")
+	log.Infof("2Resolv.conf == [%s], %s", resolve, err)
+	log.Infof("Exec %v", os.Args)
+
 	if err := initPkg.SysInit(); err != nil {
 		log.Fatal(err)
 	}

config/types.go

@@ -46,6 +46,7 @@ const (
 	CloudConfigScriptFile  = "/var/lib/rancher/conf/cloud-config-script"
 	MetaDataFile           = "/var/lib/rancher/conf/metadata"
 	CloudConfigFile        = "/var/lib/rancher/conf/cloud-config.yml"
+	EtcResolvConfFile      = "/etc/resolv.conf"
 )
 
 var (

dfs/scratch.go

@@ -357,15 +357,20 @@ ff02::2    ip6-allrouters
 127.0.1.1       `+hostname)
 
 	if len(cfg.DNSConfig.Nameservers) != 0 {
-		log.Infof("Writing resolv.conf (%v) %v", cfg.DNSConfig.Nameservers, cfg.DNSConfig.Search)
+		resolve, err := ioutil.ReadFile("/etc/resolv.conf")
-		if _, err := resolvconf.Build("/etc/resolv.conf", cfg.DNSConfig.Nameservers, cfg.DNSConfig.Search, nil); err != nil {
+		log.Debugf("Resolve.conf == [%s], err", resolve, err)
+
+		if err != nil {
+			log.Infof("scratch Writing empty resolv.conf (%v) %v", []string{}, []string{})
+			if _, err := resolvconf.Build("/etc/resolv.conf", []string{}, []string{}, nil); err != nil {
 				return err
 			}
 		}
+	}
 
 	if cfg.BridgeName != "" && cfg.BridgeName != "none" {
 		log.Debugf("Creating bridge %s (%s)", cfg.BridgeName, cfg.BridgeAddress)
-		if err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
+		if _, err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
 			Interfaces: map[string]netconf.InterfaceConfig{
 				cfg.BridgeName: {
 					Address: cfg.BridgeAddress,
@@ -374,6 +379,7 @@ ff02::2    ip6-allrouters
 				},
 			},
 		}, false, false); err != nil {
+			log.Errorf("Error creating bridge: %s", err)
 			return err
 		}
 	}

init/init.go

@@ -344,6 +344,7 @@ func RunInit() error {
 			if err := runCloudInitServices(cfg); err != nil {
 				log.Error(err)
 			}
+
 			// It'd be nice to push to rsyslog before this, but we don't have network
 			log.AddRSyslogHook()
 
@@ -355,6 +356,7 @@ func RunInit() error {
 				config.CloudConfigBootFile,
 				config.CloudConfigNetworkFile,
 				config.MetaDataFile,
+				config.EtcResolvConfFile,
 			}
 			// And all the files in /var/log/boot/
 			// TODO: I wonder if we can put this code into the log module, and have things write to the buffer until we FsReady()
@@ -418,6 +420,9 @@ func RunInit() error {
 			return cfg, nil
 		}},
 		config.CfgFuncData{"b2d Env", func(cfg *config.CloudConfig) (*config.CloudConfig, error) {
+
+			log.Debugf("memory Resolve.conf == [%s]", configFiles["/etc/resolv.conf"])
+
 			if boot2DockerEnvironment {
 				if err := config.Set("rancher.state.dev", cfg.Rancher.State.Dev); err != nil {
 					log.Errorf("Failed to update rancher.state.dev: %v", err)

log/log.go

@@ -14,6 +14,7 @@ import (
 	"github.com/rancher/os/config/cmdline"
 )
 
+var logFile *os.File
 var userHook *ShowuserlogHook
 var defaultLogLevel logrus.Level
 var debugThisLogger = false
@@ -154,7 +155,8 @@ func InitDeferedLogger() {
 		// write to dmesg until we can write to file. (maybe we can do this if rancher.debug=true?)
 		f, err := os.OpenFile("/dev/kmsg", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
 		if err == nil {
-			logrus.SetOutput(f)
+			logFile = f
+			logrus.SetOutput(logFile)
 		}
 
 		pwd, err := os.Getwd()
@@ -223,7 +225,8 @@ func FsReady() {
 		if debugThisLogger {
 			logrus.Infof("Setting log output for %s to: %s", os.Args[0], filename)
 		}
-		logrus.SetOutput(f)
+		logFile = f
+		logrus.SetOutput(logFile)
 	}
 }
 

netconf/netconf_linux.go

@@ -157,7 +157,7 @@ func populateDefault(netCfg *NetworkConfig) {
 	}
 }
 
-func ApplyNetworkConfigs(netCfg *NetworkConfig, userSetHostname, userSetDNS bool) error {
+func ApplyNetworkConfigs(netCfg *NetworkConfig, userSetHostname, userSetDNS bool) (bool, error) {
 	populateDefault(netCfg)
 
 	log.Debugf("Config: %#v", netCfg)
@@ -169,7 +169,8 @@ func ApplyNetworkConfigs(netCfg *NetworkConfig, userSetHostname, userSetDNS bool
 
 	links, err := netlink.LinkList()
 	if err != nil {
-		return err
+		log.Errorf("error getting LinkList: %s", err)
+		return false, err
 	}
 
 	wg := sync.WaitGroup{}
@@ -180,7 +181,20 @@ func ApplyNetworkConfigs(netCfg *NetworkConfig, userSetHostname, userSetDNS bool
 	}
 	wg.Wait()
 
-	return err
+	// make sure there was a DHCP set dns - or tell ros to write 8.8.8.8,8.8.8.4
+	log.Infof("Checking to see if DNS was set by DHCP")
+	dnsSet := false
+	for _, link := range links {
+		linkName := link.Attrs().Name
+		log.Infof("dns testing %s", linkName)
+		lease := getDhcpLease(linkName)
+		if _, ok := lease["domain_name_servers"]; ok {
+			log.Infof("dns was dhcp set for %s", linkName)
+			dnsSet = true
+		}
+	}
+
+	return dnsSet, err
 }
 
 func applyOuter(link netlink.Link, netCfg *NetworkConfig, wg *sync.WaitGroup, userSetHostname, userSetDNS bool) {
@@ -222,14 +236,36 @@ func applyOuter(link netlink.Link, netCfg *NetworkConfig, wg *sync.WaitGroup, us
 	}(linkName, match)
 }
 
-func hasDhcp(iface string) bool {
+func getDhcpLease(iface string) (lease map[string]string) {
+	lease = make(map[string]string)
+
+	out := getDhcpLeaseString(iface)
+	log.Debugf("getDhcpLease %s: %s", iface, out)
+
+	lines := strings.Split(string(out), "\n")
+	for _, line := range lines {
+		l := strings.SplitN(line, "=", 2)
+		log.Debugf("line: %v", l)
+		if len(l) > 1 {
+			lease[l[0]] = l[1]
+		}
+	}
+	return lease
+}
+
+func getDhcpLeaseString(iface string) []byte {
 	cmd := exec.Command("dhcpcd", "-U", iface)
 	//cmd.Stderr = os.Stderr
 	out, err := cmd.Output()
 	if err != nil {
 		log.Error(err)
 	}
-	log.Debugf("dhcpcd -u %s: %s", iface, out)
+	return out
+}
+
+func hasDhcp(iface string) bool {
+	out := getDhcpLeaseString(iface)
+	log.Debugf("dhcpcd -U %s: %s", iface, out)
 	return len(out) > 0
 }
 

os-config.tpl.yml

@@ -50,6 +50,7 @@ rancher:
       ipc: host
       privileged: true
       volumes:
+      - /etc/resolv.conf:/etc/resolv.conf
       - /dev:/host/dev
       - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher
       - /lib/modules:/lib/modules

scripts/run

@@ -31,6 +31,7 @@ while [ "$#" -gt 0 ]; do
             ;;
         --pxe)
             BOOT_PXE=1
+            FORMAT=0
             ;;
         --do)
             BOOT_HD=1

util/network/network.go

@@ -88,15 +88,11 @@ func loadFromNetwork(location string) ([]byte, error) {
 	SetProxyEnvironmentVariables(cfg)
 
 	var err error
-	// Sven thinks that the dhcpcd --wait we added makes this less necessary
-	//for i := 0; i < 300; i++ {
-	updateDNSCache()
 
 	var resp *http.Response
 	log.Debugf("LoadFromNetwork(%s)", location)
 	resp, err = http.Get(location)
-	log.Debugf("LoadFromNetwork(%s) returned %v", location, resp)
+	log.Debugf("LoadFromNetwork(%s) returned %v, %v", location, resp, err)
-	log.Debugf("LoadFromNetwork(%s) error %v", location, err)
 	if err == nil {
 		defer resp.Body.Close()
 		if resp.StatusCode != http.StatusOK {
@@ -112,9 +108,6 @@ func loadFromNetwork(location string) ([]byte, error) {
 		return bytes, nil
 	}
 
-	//	time.Sleep(100 * time.Millisecond)
-	//}
-
 	return nil, err
 }
 

util/network/update_dns_cache.go

@@ -1,9 +0,0 @@
-// +build dnspatch
-
-package network
-
-import "net"
-
-func updateDNSCache() {
-	net.UpdateDnsConf()
-}

util/network/update_dns_cache_patched.go

@@ -1,5 +0,0 @@
-// +build !dnspatch
-
-package network
-
-func updateDNSCache() {}