Merge pull request #30728 from Random-Liu/fix-node-e2e-vm-leak

Automatic merge from submit-queue

Node E2E: Stop throwing fatal error in buildGo to fix vm leak.

Fixes https://github.com/kubernetes/kubernetes/issues/30719.

Avoid throwing out fatal error during `buildGo`.  As is pointed out in https://github.com/kubernetes/kubernetes/issues/30719#issuecomment-240283268, we delete vm instances in `defer`. However, `defer` will not be called if there is a fatal error (process exits immediately) 
http://stackoverflow.com/questions/17888610/are-deferred-functions-called-when-calling-log-fatalln.

This PR:
* changes all fatal error during create/delete instance to normal error so as to make sure we have a chance to delete the instance.
* adds log for instance deletion so as to better surface similar error in the future.

@dchen1107 @yujuhong 
/cc @kubernetes/sig-node

test/e2e_node/e2e_build.go

@@ -37,11 +37,11 @@ var buildTargets = []string{
 	"vendor/github.com/onsi/ginkgo/ginkgo",
 }
 
-func buildGo() {
+func buildGo() error {
 	glog.Infof("Building k8s binaries...")
 	k8sRoot, err := getK8sRootDir()
 	if err != nil {
-		glog.Fatalf("Failed to locate kubernetes root directory %v.", err)
+		return fmt.Errorf("failed to locate kubernetes root directory %v.", err)
 	}
 	targets := strings.Join(buildTargets, " ")
 	cmd := exec.Command("make", "-C", k8sRoot, fmt.Sprintf("WHAT=%s", targets))
@@ -49,8 +49,9 @@ func buildGo() {
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()
 	if err != nil {
-		glog.Fatalf("Failed to build go packages %v\n", err)
+		return fmt.Errorf("failed to build go packages %v\n", err)
 	}
+	return nil
 }
 
 func getK8sBin(bin string) (string, error) {

test/e2e_node/e2e_node_suite_test.go

@@ -101,7 +101,7 @@ func TestE2eNode(t *testing.T) {
 // Setup the kubelet on the node
 var _ = SynchronizedBeforeSuite(func() []byte {
 	if *buildServices {
-		buildGo()
+		Expect(buildGo()).To(Succeed())
 	}
 	if framework.TestContext.NodeName == "" {
 		hostname, err := os.Hostname()

test/e2e_node/e2e_remote.go

@@ -82,12 +82,14 @@ func GetHostnameOrIp(hostname string) string {
 // the binaries k8s required for node e2e tests
 func CreateTestArchive() (string, error) {
 	// Build the executables
-	buildGo()
+	if err := buildGo(); err != nil {
+		return "", fmt.Errorf("failed to build the depedencies: %v", err)
+	}
 
 	// Make sure we can find the newly built binaries
 	buildOutputDir, err := getK8sBuildOutputDir()
 	if err != nil {
-		glog.Fatalf("Failed to locate kubernetes build output directory %v", err)
+		return "", fmt.Errorf("failed to locate kubernetes build output directory %v", err)
 	}
 
 	ginkgoTest := filepath.Join(buildOutputDir, "e2e_node.test")

test/e2e_node/runner/run_e2e.go

@@ -502,9 +502,11 @@ func getComputeClient() (*compute.Service, error) {
 }
 
 func deleteInstance(image string) {
-	_, err := computeService.Instances.Delete(*project, *zone, imageToInstanceName(image)).Do()
+	instanceName := imageToInstanceName(image)
+	glog.Infof("Deleting instance %q", instanceName)
+	_, err := computeService.Instances.Delete(*project, *zone, instanceName).Do()
 	if err != nil {
-		glog.Infof("Error deleting instance %s", imageToInstanceName(image))
+		glog.Errorf("Error deleting instance %q: %v", instanceName, err)
 	}
 }