Merge pull request #978 from nathanleclaire/collect_system_logs

Collect system container logs for diagnostics

alpine/packages/diagnostics/capture.go

@@ -15,7 +15,8 @@ import (
 )
 
 const (
-	defaultCommandTimeout = 5 * time.Second
+	defaultCaptureTimeout = 5 * time.Second
+	defaultCommandTimeout = defaultCaptureTimeout
 
 	// Might eventually have some pretty long (~30s) traces in here, so 35
 	// seconds seems reasonable.

alpine/packages/diagnostics/http.go

@@ -19,7 +19,7 @@ import (
 )
 
 var (
-	errDockerPingNotOK = errors.New("Docker /_ping did not return OK")
+	errDockerRespNotOK = errors.New("Docker API call did not return OK")
 )
 
 const (
@@ -39,15 +39,58 @@ func init() {
 	for _, c := range commonCmdCaptures {
 		cloudCaptures = append(cloudCaptures, c)
 	}
+
+	cloudCaptures = append(cloudCaptures, SystemContainerCapturer{})
 }
 
 // HTTPDiagnosticListener sets a health check and optional diagnostic endpoint
 // for cloud editions.
 type HTTPDiagnosticListener struct{}
 
+func dockerHTTPGet(ctx context.Context, url string) (*http.Response, error) {
+	client := &http.Client{
+		Transport: &UnixSocketRoundTripper{},
+	}
+
+	return dockerHTTPGetWithClient(ctx, url, client)
+}
+
+func dockerHTTPGetWithClient(ctx context.Context, url string, client *http.Client) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req = req.WithContext(ctx)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return resp, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return resp, errDockerRespNotOK
+	}
+
+	return resp, err
+
+}
+
 // UnixSocketRoundTripper provides a way to make HTTP request to Docker socket
 // directly.
-type UnixSocketRoundTripper struct{}
+type UnixSocketRoundTripper struct {
+	Stream bool
+	conn   *httputil.ClientConn
+}
+
+// Close will close the connection if the caller needs to clean up after
+// themselves in a streaming request.
+func (u UnixSocketRoundTripper) Close() error {
+	if u.conn != nil {
+		return u.conn.Close()
+	}
+	return nil
+}
 
 // RoundTrip dials the Docker UNIX socket to make a HTTP request.
 func (u UnixSocketRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
@@ -55,43 +98,29 @@ func (u UnixSocketRoundTripper) RoundTrip(req *http.Request) (*http.Response, er
 	if err != nil {
 		return nil, err
 	}
-	conn := httputil.NewClientConn(dial, nil)
+	u.conn = httputil.NewClientConn(dial, nil)
-	defer conn.Close()
+
-	return conn.Do(req)
+	// If the client makes a streaming request (e.g., /container/x/logs)
+	// it's their responsibility to close the connection, because it needs
+	// to remain open to stream the response body.
+	if !u.Stream {
+		defer u.conn.Close()
+	}
+
+	return u.conn.Do(req)
 }
 
 // Listen starts the HTTPDiagnosticListener and sets up handlers for its endpoints
 func (h HTTPDiagnosticListener) Listen() {
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		client := &http.Client{
-			Transport: &UnixSocketRoundTripper{},
-		}
-
-		req, err := http.NewRequest(http.MethodGet, "/_ping", nil)
-		if err != nil {
-			http.Error(w, "Error creating HTTP request to talk to Docker", http.StatusInternalServerError)
-			return
-		}
-
 		ctx, cancel := context.WithTimeout(context.Background(), healthcheckTimeout)
 		defer cancel()
 
-		req = req.WithContext(ctx)
-
 		errCh := make(chan error)
 
 		go func() {
-			resp, err := client.Do(req)
+			_, err := dockerHTTPGet(ctx, "/_ping")
-			if err != nil {
 			errCh <- err
-				return
-			}
-			defer resp.Body.Close()
-			if resp.StatusCode != http.StatusOK {
-				errCh <- errDockerPingNotOK
-				return
-			}
-			errCh <- nil
 		}()
 
 		select {

alpine/packages/diagnostics/system_log_capture.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+)
+
+const (
+	systemLogDir         = "editionslogs"
+	systemContainerLabel = "com.docker.editions.system"
+)
+
+// SystemContainerCapturer gets the logs from containers which are run
+// specifically for Docker Editions.
+type SystemContainerCapturer struct{}
+
+// Capture writes output from a CommandCapturer to a tar archive
+func (s SystemContainerCapturer) Capture(parentCtx context.Context, w *tar.Writer) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultCaptureTimeout)
+	defer cancel()
+
+	errCh := make(chan error)
+
+	go func() {
+		resp, err := dockerHTTPGet(ctx, "/containers/json?all=1&label="+systemContainerLabel)
+		if err != nil {
+			errCh <- err
+			return
+		}
+
+		defer resp.Body.Close()
+
+		names := []struct {
+			ID    string `json:"id"`
+			Names []string
+		}{}
+
+		if err := json.NewDecoder(resp.Body).Decode(&names); err != nil {
+			errCh <- err
+			return
+		}
+
+		for _, c := range names {
+			transport := &UnixSocketRoundTripper{
+				Stream: true,
+			}
+
+			client := &http.Client{
+				Transport: transport,
+			}
+
+			resp, err := dockerHTTPGetWithClient(ctx, "/containers/"+c.ID+"/logs?stderr=1&stdout=1&timestamps=1&tail=all", client)
+			if err != nil {
+				log.Println("ERROR (get request):", err)
+				continue
+			}
+
+			defer resp.Body.Close()
+
+			// logs makes streaming request where the original http
+			// conn is left open so we must clean up after
+			// ourselves when we're done reading
+			defer transport.Close()
+
+			logLines, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				log.Println("ERROR (reading response):", err)
+				continue
+			}
+
+			// Docker API returns a Names array where the names are
+			// like this: ["/foobar", "/quux"]
+			//
+			// Use the first one from it.
+			//
+			// Additionally, the slash from it helps delimit a path
+			// separator in the tar archive.
+			//
+			// TODO(nathanleclaire): This seems fragile, but I'm
+			// not sure what approach would be much better.
+			tarWrite(w, bytes.NewBuffer(logLines), systemLogDir+c.Names[0])
+		}
+
+		errCh <- nil
+	}()
+
+	select {
+	case <-ctx.Done():
+		log.Println("System container log capture context error", ctx.Err())
+	case err := <-errCh:
+		if err != nil {
+			log.Println("System container log capture error", err)
+		}
+		log.Println("System container log capture finished successfully")
+	}
+}