Ensure /etc/hosts has a header always - Fix conformance test

We have 2 scenarios where we copy /etc/hosts
- with host network (we just copy the /etc/hosts from node)
- without host network (create a fresh /etc/hosts from pod info)

We are having trouble figuring out whether a /etc/hosts in a
pod/container has been "fixed-up" or not. And whether we used
host network or a fresh /etc/hosts in the various ways we start
up the tests which are:

- VM/box against a remote cluster
- As a container inside the k8s cluster
- DIND scenario in CI where test runs inside a managed container

Please see previous mis-guided attempt to fix this problem at
ba20e63446 In this commit we revert
the code from there as well.

So we should make sure:
- we always add a header if we touched the file
- we add slightly different headers so we can figure out if we used the
  host network or not.

Update the test case to inject /etc/hosts from node to another path
(/etc/hosts-original) as well and use that to compare.

pkg/kubelet/kubelet_pods.go

@@ -68,6 +68,11 @@ import (
 	"k8s.io/kubernetes/third_party/forked/golang/expansion"
 )
 
+const (
+	managedHostsHeader                = "# Kubernetes-managed hosts file.\n"
+	managedHostsHeaderWithHostNetwork = "# Kubernetes-managed hosts file (host network).\n"
+)
+
 // Get a list of pods that have data directories.
 func (kl *Kubelet) listPodsFromDisk() ([]types.UID, error) {
 	podInfos, err := ioutil.ReadDir(kl.getPodsDir())
@@ -356,15 +361,18 @@ func nodeHostsFileContent(hostsFilePath string, hostAliases []v1.HostAlias) ([]b
 	if err != nil {
 		return nil, err
 	}
-	hostsFileContent = append(hostsFileContent, hostsEntriesFromHostAliases(hostAliases)...)
+	var buffer bytes.Buffer
-	return hostsFileContent, nil
+	buffer.WriteString(managedHostsHeaderWithHostNetwork)
+	buffer.Write(hostsFileContent)
+	buffer.Write(hostsEntriesFromHostAliases(hostAliases))
+	return buffer.Bytes(), nil
 }
 
 // managedHostsFileContent generates the content of the managed etc hosts based on Pod IP and other
 // information.
 func managedHostsFileContent(hostIP, hostName, hostDomainName string, hostAliases []v1.HostAlias) []byte {
 	var buffer bytes.Buffer
-	buffer.WriteString("# Kubernetes-managed hosts file.\n")
+	buffer.WriteString(managedHostsHeader)
 	buffer.WriteString("127.0.0.1\tlocalhost\n")                      // ipv4 localhost
 	buffer.WriteString("::1\tlocalhost ip6-localhost ip6-loopback\n") // ipv6 localhost
 	buffer.WriteString("fe00::0\tip6-localnet\n")
@@ -376,9 +384,8 @@ func managedHostsFileContent(hostIP, hostName, hostDomainName string, hostAliase
 	} else {
 		buffer.WriteString(fmt.Sprintf("%s\t%s\n", hostIP, hostName))
 	}
-	hostsFileContent := buffer.Bytes()
+	buffer.Write(hostsEntriesFromHostAliases(hostAliases))
-	hostsFileContent = append(hostsFileContent, hostsEntriesFromHostAliases(hostAliases)...)
+	return buffer.Bytes()
-	return hostsFileContent
 }
 
 func hostsEntriesFromHostAliases(hostAliases []v1.HostAlias) []byte {

pkg/kubelet/kubelet_pods_test.go

@@ -563,7 +563,8 @@ fe00::1	ip6-allnodes
 fe00::2	ip6-allrouters
 123.45.67.89	some.domain
 `,
-			`# hosts file for testing.
+			`# Kubernetes-managed hosts file (host network).
+# hosts file for testing.
 127.0.0.1	localhost
 ::1	localhost ip6-localhost ip6-loopback
 fe00::0	ip6-localnet
@@ -585,7 +586,8 @@ fe00::1	ip6-allnodes
 fe00::2	ip6-allrouters
 12.34.56.78	another.domain
 `,
-			`# another hosts file for testing.
+			`# Kubernetes-managed hosts file (host network).
+# another hosts file for testing.
 127.0.0.1	localhost
 ::1	localhost ip6-localhost ip6-loopback
 fe00::0	ip6-localnet
@@ -609,7 +611,8 @@ fe00::1	ip6-allnodes
 fe00::2	ip6-allrouters
 123.45.67.89	some.domain
 `,
-			`# hosts file for testing.
+			`# Kubernetes-managed hosts file (host network).
+# hosts file for testing.
 127.0.0.1	localhost
 ::1	localhost ip6-localhost ip6-loopback
 fe00::0	ip6-localnet
@@ -639,7 +642,8 @@ fe00::1	ip6-allnodes
 fe00::2	ip6-allrouters
 12.34.56.78	another.domain
 `,
-			`# another hosts file for testing.
+			`# Kubernetes-managed hosts file (host network).
+# another hosts file for testing.
 127.0.0.1	localhost
 ::1	localhost ip6-localhost ip6-loopback
 fe00::0	ip6-localnet

test/e2e/common/kubelet_etc_hosts.go

@@ -17,8 +17,6 @@ limitations under the License.
 package common
 
 import (
-	"io/ioutil"
-	"os"
 	"strings"
 	"time"
 
@@ -34,6 +32,8 @@ const (
 	etcHostsPodName            = "test-pod"
 	etcHostsHostNetworkPodName = "test-host-network-pod"
 	etcHostsPartialContent     = "# Kubernetes-managed hosts file."
+	etcHostsPath               = "/etc/hosts"
+	etcHostsOriginalPath       = "/etc/hosts-original"
 )
 
 var etcHostsImageName = imageutils.GetE2EImage(imageutils.Netexec)
@@ -42,7 +42,6 @@ type KubeletManagedHostConfig struct {
 	hostNetworkPod *v1.Pod
 	pod            *v1.Pod
 	f              *framework.Framework
-	tmpEtcHostFile *os.File
 }
 
 var _ = framework.KubeDescribe("KubeletManagedEtcHosts", func() {
@@ -62,8 +61,6 @@ var _ = framework.KubeDescribe("KubeletManagedEtcHosts", func() {
 
 		By("Running the test")
 		config.verifyEtcHosts()
-
-		config.cleanup()
 	})
 })
 
@@ -81,26 +78,6 @@ func (config *KubeletManagedHostConfig) verifyEtcHosts() {
 }
 
 func (config *KubeletManagedHostConfig) setup() {
-	etcHostContents := `127.0.0.1	localhost
-::1	localhost ip6-localhost ip6-loopback
-fe00::0	ip6-localnet
-ff00::0	ip6-mcastprefix
-ff02::1	ip6-allnodes
-ff02::2	ip6-allrouters`
-
-	// Write the data to a temp file.
-	var err error
-	config.tmpEtcHostFile, err = ioutil.TempFile("", "etc-hosts")
-	if err != nil {
-		framework.Failf("failed to create temp file for /etc/hosts: %v", err)
-	}
-	if _, err := config.tmpEtcHostFile.Write([]byte(etcHostContents)); err != nil {
-		framework.Failf("Failed to write temp file for /etc/hosts data: %v", err)
-	}
-	if err := config.tmpEtcHostFile.Close(); err != nil {
-		framework.Failf("Failed to close temp file: %v", err)
-	}
-
 	By("Creating hostNetwork=false pod")
 	config.createPodWithoutHostNetwork()
 
@@ -108,12 +85,6 @@ ff02::2	ip6-allrouters`
 	config.createPodWithHostNetwork()
 }
 
-func (config *KubeletManagedHostConfig) cleanup() {
-	if config.tmpEtcHostFile != nil {
-		os.Remove(config.tmpEtcHostFile.Name())
-	}
-}
-
 func (config *KubeletManagedHostConfig) createPodWithoutHostNetwork() {
 	podSpec := config.createPodSpec(etcHostsPodName)
 	config.pod = config.f.PodClient().CreateSync(podSpec)
@@ -137,16 +108,24 @@ func assertManagedStatus(
 	etcHostsContent := ""
 
 	for startTime := time.Now(); time.Since(startTime) < retryTimeout; {
-		etcHostsContent = config.getEtcHostsContent(podName, name)
+		etcHostsContent = config.getFileContents(podName, name, etcHostsPath)
-		isManaged := strings.Contains(etcHostsContent, etcHostsPartialContent)
+		etcHostsOriginalContent := config.getFileContents(podName, name, etcHostsOriginalPath)
 
+		// Make sure there is some content in both files
+		if len(etcHostsContent) > 0 && len(etcHostsOriginalContent) > 0 {
+			// if the files match, kubernetes did not touch the file at all
+			// if the file has the header, kubernetes is not using host network
+			// and is constructing the file based on Pod IP
+			isManaged := strings.HasPrefix(etcHostsContent, etcHostsPartialContent) &&
+				etcHostsContent != etcHostsOriginalContent
 			if expectedIsManaged == isManaged {
 				return
 			}
+		}
 
 		glog.Warningf(
-			"For pod: %s, name: %s, expected %t, actual %t (/etc/hosts was %q), retryCount: %d",
+			"For pod: %s, name: %s, expected %t, (/etc/hosts was %q), (/etc/hosts-original was %q), retryCount: %d",
-			podName, name, expectedIsManaged, isManaged, etcHostsContent, retryCount)
+			podName, name, expectedIsManaged, etcHostsContent, etcHostsOriginalContent, retryCount)
 
 		retryCount++
 		time.Sleep(100 * time.Millisecond)
@@ -163,8 +142,8 @@ func assertManagedStatus(
 	}
 }
 
-func (config *KubeletManagedHostConfig) getEtcHostsContent(podName, containerName string) string {
+func (config *KubeletManagedHostConfig) getFileContents(podName, containerName, path string) string {
-	return config.f.ExecCommandInContainer(podName, containerName, "cat", "/etc/hosts")
+	return config.f.ExecCommandInContainer(podName, containerName, "cat", path)
 }
 
 func (config *KubeletManagedHostConfig) createPodSpec(podName string) *v1.Pod {
@@ -184,6 +163,12 @@ func (config *KubeletManagedHostConfig) createPodSpec(podName string) *v1.Pod {
 						"sleep",
 						"900",
 					},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "host-etc-hosts",
+							MountPath: etcHostsOriginalPath,
+						},
+					},
 				},
 				{
 					Name:            "busybox-2",
@@ -193,6 +178,12 @@ func (config *KubeletManagedHostConfig) createPodSpec(podName string) *v1.Pod {
 						"sleep",
 						"900",
 					},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "host-etc-hosts",
+							MountPath: etcHostsOriginalPath,
+						},
+					},
 				},
 				{
 					Name:            "busybox-3",
@@ -205,7 +196,11 @@ func (config *KubeletManagedHostConfig) createPodSpec(podName string) *v1.Pod {
 					VolumeMounts: []v1.VolumeMount{
 						{
 							Name:      "host-etc-hosts",
-							MountPath: "/etc/hosts",
+							MountPath: etcHostsPath,
+						},
+						{
+							Name:      "host-etc-hosts",
+							MountPath: etcHostsOriginalPath,
 						},
 					},
 				},
@@ -215,7 +210,7 @@ func (config *KubeletManagedHostConfig) createPodSpec(podName string) *v1.Pod {
 					Name: "host-etc-hosts",
 					VolumeSource: v1.VolumeSource{
 						HostPath: &v1.HostPathVolumeSource{
-							Path: config.tmpEtcHostFile.Name(),
+							Path: etcHostsPath,
 							Type: hostPathType,
 						},
 					},
@@ -227,6 +222,8 @@ func (config *KubeletManagedHostConfig) createPodSpec(podName string) *v1.Pod {
 }
 
 func (config *KubeletManagedHostConfig) createPodSpecWithHostNetwork(podName string) *v1.Pod {
+	hostPathType := new(v1.HostPathType)
+	*hostPathType = v1.HostPathType(string(v1.HostPathFileOrCreate))
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: podName,
@@ -243,6 +240,12 @@ func (config *KubeletManagedHostConfig) createPodSpecWithHostNetwork(podName str
 						"sleep",
 						"900",
 					},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "host-etc-hosts",
+							MountPath: etcHostsOriginalPath,
+						},
+					},
 				},
 				{
 					Name:            "busybox-2",
@@ -252,6 +255,23 @@ func (config *KubeletManagedHostConfig) createPodSpecWithHostNetwork(podName str
 						"sleep",
 						"900",
 					},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "host-etc-hosts",
+							MountPath: etcHostsOriginalPath,
+						},
+					},
+				},
+			},
+			Volumes: []v1.Volume{
+				{
+					Name: "host-etc-hosts",
+					VolumeSource: v1.VolumeSource{
+						HostPath: &v1.HostPathVolumeSource{
+							Path: etcHostsPath,
+							Type: hostPathType,
+						},
+					},
 				},
 			},
 		},