Merge pull request #24808 from screeley44/gluster_errors

Automatic merge from submit-queue

read gluster log to surface glusterfs plugin errors properly in describe events

glusterfs.go does not properly expose errors as all mount errors go to a log file, I propose we read the log file to expose the errors without asking the users to 'go look at this log'

This PR does the following:
1.  adds a gluster option for log-level=ERROR to remove all noise from log file
2.  change log file name and path based on PV + Pod name - so specific per PV and Pod
3.  create a utility to read the last two lines of the log file when failure occurs

old behavior:
```
  13s	13s	1	{kubelet 127.0.0.1}		Warning	FailedMount	Unable to mount volumes for pod "bb-gluster-pod2_default(34b18c6b-070d-11e6-8e95-52540092b5fb)": glusterfs: mount failed: Mount failed: exit status 1
Mounting arguments: 192.168.234.147:myVol2 /var/lib/kubelet/pods/34b18c6b-070d-11e6-8e95-52540092b5fb/volumes/kubernetes.io~glusterfs/pv-gluster glusterfs [log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pv-gluster/glusterfs.log]
Output: Mount failed. Please check the log file for more details.
```

improved behavior: (updated after suggestions from community)
```
  34m		34m		1	{kubelet 127.0.0.1}			Warning		FailedMount	Unable to mount volumes for pod "bb-multi-pod1_default(e7d7f790-0d4b-11e6-a275-52540092b5fb)": glusterfs: mount failed: Mount failed: exit status 1
Mounting arguments: 192.168.123.222:myVol2 /var/lib/kubelet/pods/e7d7f790-0d4b-11e6-a275-52540092b5fb/volumes/kubernetes.io~glusterfs/pv-gluster2 glusterfs [log-level=ERROR log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pv-gluster2/bb-multi-pod1-glusterfs.log]
Output: Mount failed. Please check the log file for more details.

 the following error information was pulled from the log to help resolve this issue: 
[2016-04-28 14:21:29.109697] E [socket.c:2332:socket_connect_finish] 0-glusterfs: connection to 192.168.123.222:24007 failed (Connection timed out)
[2016-04-28 14:21:29.109767] E [glusterfsd-mgmt.c:1819:mgmt_rpc_notify] 0-glusterfsd-mgmt: failed to connect with remote-host: 192.168.123.222 (Transport endpoint is not connected)

```

also this PR is alternate approach to :  #24624

pkg/volume/glusterfs/glusterfs.go

@@ -240,7 +240,12 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error {
 	if err := os.MkdirAll(p, 0750); err != nil {
 		return fmt.Errorf("glusterfs: mkdir failed: %v", err)
 	}
-	log := path.Join(p, "glusterfs.log")
+
+	// adding log-level ERROR to remove noise
+	// and more specific log path so each pod has
+	// it's own log based on PV + Pod
+	log := path.Join(p, b.pod.Name+"-glusterfs.log")
+	options = append(options, "log-level=ERROR")
 	options = append(options, "log-file="+log)
 
 	addr := make(map[string]struct{})
@@ -255,8 +260,18 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error {
 	for hostIP := range addr {
 		errs = b.mounter.Mount(hostIP+":"+b.path, dir, "glusterfs", options)
 		if errs == nil {
+			glog.Infof("glusterfs: successfully mounted %s", dir)
 			return nil
 		}
 	}
+
+	// Failed mount scenario.
+	// Since gluster does not return eror text
+	// it all goes in a log file, we will read the log file
+	logerror := readGlusterLog(log, b.pod.Name)
+	if logerror != nil {
+		// return fmt.Errorf("glusterfs: mount failed: %v", logerror)
+		return fmt.Errorf("glusterfs: mount failed: %v the following error information was pulled from the glusterfs log to help diagnose this issue: %v", errs, logerror)
+	}
 	return fmt.Errorf("glusterfs: mount failed: %v", errs)
 }

pkg/volume/glusterfs/glusterfs_util.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package glusterfs
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+
+	"github.com/golang/glog"
+)
+
+// readGlusterLog will take the last 2 lines of the log file
+// on failure of gluster SetUp and return those so kubelet can
+// properly expose them
+// return nil on any failure
+func readGlusterLog(path string, podName string) error {
+
+	var line1 string
+	var line2 string
+	linecount := 0
+
+	glog.Infof("glusterfs: failure, now attempting to read the gluster log for pod %s", podName)
+
+	// Check and make sure path exists
+	if len(path) == 0 {
+		return fmt.Errorf("glusterfs: log file does not exist for pod: %s", podName)
+	}
+
+	// open the log file
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("glusterfs: could not open log file for pod: %s", podName)
+	}
+	defer file.Close()
+
+	// read in and scan the file using scanner
+	// from stdlib
+	fscan := bufio.NewScanner(file)
+
+	// rather than guessing on bytes or using Seek
+	// going to scan entire file and take the last two lines
+	// generally the file should be small since it is pod specific
+	for fscan.Scan() {
+		if linecount > 0 {
+			line1 = line2
+		}
+		line2 = "\n" + fscan.Text()
+
+		linecount++
+	}
+
+	if linecount > 0 {
+		return fmt.Errorf("%v", line1+line2+"\n")
+	}
+	return nil
+}