From bbea2863c26430b4bef3fc347bbf19997d4df95c Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Thu, 4 Jun 2015 13:02:09 -0700 Subject: [PATCH 1/2] Add docker dependencies: timeutils, jsonmessage --- Godeps/Godeps.json | 10 + .../docker/pkg/jsonmessage/jsonmessage.go | 172 ++++++++++++++++++ .../pkg/jsonmessage/jsonmessage_test.go | 38 ++++ .../docker/docker/pkg/timeutils/json.go | 26 +++ .../docker/docker/pkg/timeutils/utils.go | 29 +++ .../docker/docker/pkg/timeutils/utils_test.go | 36 ++++ 6 files changed, 311 insertions(+) create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/jsonmessage/jsonmessage.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/jsonmessage/jsonmessage_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/json.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/utils.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/utils_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index cfc31c06cb7..03eb7e0d061 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -149,6 +149,11 @@ "Comment": "v2.2.0-23-g5ca8014", "Rev": "5ca80149b9d3f8b863af0e2bb6742e608603bd99" }, + { + "ImportPath": "github.com/docker/docker/pkg/jsonmessage", + "Comment": "v1.7.rc", + "Rev": "0cd6c05d8112e9246b734107d54e2855e3d5fec5" + }, { "ImportPath": "github.com/docker/docker/pkg/mount", "Comment": "v1.4.1-1714-ged66853", @@ -164,6 +169,11 @@ "Comment": "v1.4.1-1714-ged66853", "Rev": "ed6685369740035b0af9675bf9add52d0af7657b" }, + { + "ImportPath": "github.com/docker/docker/pkg/timeutils", + "Comment": "v1.7.rc", + "Rev": "0cd6c05d8112e9246b734107d54e2855e3d5fec5" + }, { "ImportPath": "github.com/docker/docker/pkg/units", "Comment": "v1.4.1-1714-ged66853", diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/jsonmessage/jsonmessage.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/jsonmessage/jsonmessage.go new file mode 100644 index 00000000000..7db1626e48c --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/jsonmessage/jsonmessage.go @@ -0,0 +1,172 @@ +package jsonmessage + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "time" + + "github.com/docker/docker/pkg/term" + "github.com/docker/docker/pkg/timeutils" + "github.com/docker/docker/pkg/units" +) + +type JSONError struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +func (e *JSONError) Error() string { + return e.Message +} + +type JSONProgress struct { + terminalFd uintptr + Current int `json:"current,omitempty"` + Total int `json:"total,omitempty"` + Start int64 `json:"start,omitempty"` +} + +func (p *JSONProgress) String() string { + var ( + width = 200 + pbBox string + numbersBox string + timeLeftBox string + ) + + ws, err := term.GetWinsize(p.terminalFd) + if err == nil { + width = int(ws.Width) + } + + if p.Current <= 0 && p.Total <= 0 { + return "" + } + current := units.HumanSize(float64(p.Current)) + if p.Total <= 0 { + return fmt.Sprintf("%8v", current) + } + total := units.HumanSize(float64(p.Total)) + percentage := int(float64(p.Current)/float64(p.Total)*100) / 2 + if percentage > 50 { + percentage = 50 + } + if width > 110 { + // this number can't be negetive gh#7136 + numSpaces := 0 + if 50-percentage > 0 { + numSpaces = 50 - percentage + } + pbBox = fmt.Sprintf("[%s>%s] ", strings.Repeat("=", percentage), strings.Repeat(" ", numSpaces)) + } + numbersBox = fmt.Sprintf("%8v/%v", current, total) + + if p.Current > 0 && p.Start > 0 && percentage < 50 { + fromStart := time.Now().UTC().Sub(time.Unix(int64(p.Start), 0)) + perEntry := fromStart / time.Duration(p.Current) + left := time.Duration(p.Total-p.Current) * perEntry + left = (left / time.Second) * time.Second + + if width > 50 { + timeLeftBox = " " + left.String() + } + } + return pbBox + numbersBox + timeLeftBox +} + +type JSONMessage struct { + Stream string `json:"stream,omitempty"` + Status string `json:"status,omitempty"` + Progress *JSONProgress `json:"progressDetail,omitempty"` + ProgressMessage string `json:"progress,omitempty"` //deprecated + ID string `json:"id,omitempty"` + From string `json:"from,omitempty"` + Time int64 `json:"time,omitempty"` + Error *JSONError `json:"errorDetail,omitempty"` + ErrorMessage string `json:"error,omitempty"` //deprecated +} + +func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error { + if jm.Error != nil { + if jm.Error.Code == 401 { + return fmt.Errorf("Authentication is required.") + } + return jm.Error + } + var endl string + if isTerminal && jm.Stream == "" && jm.Progress != nil { + // [2K = erase entire current line + fmt.Fprintf(out, "%c[2K\r", 27) + endl = "\r" + } else if jm.Progress != nil && jm.Progress.String() != "" { //disable progressbar in non-terminal + return nil + } + if jm.Time != 0 { + fmt.Fprintf(out, "%s ", time.Unix(jm.Time, 0).Format(timeutils.RFC3339NanoFixed)) + } + if jm.ID != "" { + fmt.Fprintf(out, "%s: ", jm.ID) + } + if jm.From != "" { + fmt.Fprintf(out, "(from %s) ", jm.From) + } + if jm.Progress != nil && isTerminal { + fmt.Fprintf(out, "%s %s%s", jm.Status, jm.Progress.String(), endl) + } else if jm.ProgressMessage != "" { //deprecated + fmt.Fprintf(out, "%s %s%s", jm.Status, jm.ProgressMessage, endl) + } else if jm.Stream != "" { + fmt.Fprintf(out, "%s%s", jm.Stream, endl) + } else { + fmt.Fprintf(out, "%s%s\n", jm.Status, endl) + } + return nil +} + +func DisplayJSONMessagesStream(in io.Reader, out io.Writer, terminalFd uintptr, isTerminal bool) error { + var ( + dec = json.NewDecoder(in) + ids = make(map[string]int) + diff = 0 + ) + for { + var jm JSONMessage + if err := dec.Decode(&jm); err != nil { + if err == io.EOF { + break + } + return err + } + + if jm.Progress != nil { + jm.Progress.terminalFd = terminalFd + } + if jm.ID != "" && (jm.Progress != nil || jm.ProgressMessage != "") { + line, ok := ids[jm.ID] + if !ok { + line = len(ids) + ids[jm.ID] = line + if isTerminal { + fmt.Fprintf(out, "\n") + } + diff = 0 + } else { + diff = len(ids) - line + } + if jm.ID != "" && isTerminal { + // [{diff}A = move cursor up diff rows + fmt.Fprintf(out, "%c[%dA", 27, diff) + } + } + err := jm.Display(out, isTerminal) + if jm.ID != "" && isTerminal { + // [{diff}B = move cursor down diff rows + fmt.Fprintf(out, "%c[%dB", 27, diff) + } + if err != nil { + return err + } + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/jsonmessage/jsonmessage_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/jsonmessage/jsonmessage_test.go new file mode 100644 index 00000000000..4c3f5666b3f --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/jsonmessage/jsonmessage_test.go @@ -0,0 +1,38 @@ +package jsonmessage + +import ( + "testing" +) + +func TestError(t *testing.T) { + je := JSONError{404, "Not found"} + if je.Error() != "Not found" { + t.Fatalf("Expected 'Not found' got '%s'", je.Error()) + } +} + +func TestProgress(t *testing.T) { + jp := JSONProgress{} + if jp.String() != "" { + t.Fatalf("Expected empty string, got '%s'", jp.String()) + } + + expected := " 1 B" + jp2 := JSONProgress{Current: 1} + if jp2.String() != expected { + t.Fatalf("Expected %q, got %q", expected, jp2.String()) + } + + expected = "[=========================> ] 50 B/100 B" + jp3 := JSONProgress{Current: 50, Total: 100} + if jp3.String() != expected { + t.Fatalf("Expected %q, got %q", expected, jp3.String()) + } + + // this number can't be negetive gh#7136 + expected = "[==================================================>] 50 B/40 B" + jp4 := JSONProgress{Current: 50, Total: 40} + if jp4.String() != expected { + t.Fatalf("Expected %q, got %q", expected, jp4.String()) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/json.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/json.go new file mode 100644 index 00000000000..8043d69d18f --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/json.go @@ -0,0 +1,26 @@ +package timeutils + +import ( + "errors" + "time" +) + +const ( + // RFC3339NanoFixed is our own version of RFC339Nano because we want one + // that pads the nano seconds part with zeros to ensure + // the timestamps are aligned in the logs. + RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + // JSONFormat is the format used by FastMarshalJSON + JSONFormat = `"` + time.RFC3339Nano + `"` +) + +// FastMarshalJSON avoids one of the extra allocations that +// time.MarshalJSON is making. +func FastMarshalJSON(t time.Time) (string, error) { + if y := t.Year(); y < 0 || y >= 10000 { + // RFC 3339 is clear that years are 4 digits exactly. + // See golang.org/issue/4556#c15 for more discussion. + return "", errors.New("time.MarshalJSON: year outside of range [0,9999]") + } + return t.Format(JSONFormat), nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/utils.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/utils.go new file mode 100644 index 00000000000..6af16a1d7fc --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/utils.go @@ -0,0 +1,29 @@ +package timeutils + +import ( + "strconv" + "strings" + "time" +) + +// GetTimestamp tries to parse given string as RFC3339 time +// or Unix timestamp (with seconds precision), if successful +//returns a Unix timestamp as string otherwise returns value back. +func GetTimestamp(value string) string { + var format string + if strings.Contains(value, ".") { + format = time.RFC3339Nano + } else { + format = time.RFC3339 + } + + loc := time.FixedZone(time.Now().Zone()) + if len(value) < len(format) { + format = format[:len(value)] + } + t, err := time.ParseInLocation(format, value, loc) + if err != nil { + return value + } + return strconv.FormatInt(t.Unix(), 10) +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/utils_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/utils_test.go new file mode 100644 index 00000000000..1d724fb2ac9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/timeutils/utils_test.go @@ -0,0 +1,36 @@ +package timeutils + +import ( + "testing" +) + +func TestGetTimestamp(t *testing.T) { + cases := []struct{ in, expected string }{ + {"0", "-62167305600"}, // 0 gets parsed year 0 + + // Partial RFC3339 strings get parsed with second precision + {"2006-01-02T15:04:05.999999999+07:00", "1136189045"}, + {"2006-01-02T15:04:05.999999999Z", "1136214245"}, + {"2006-01-02T15:04:05.999999999", "1136214245"}, + {"2006-01-02T15:04:05", "1136214245"}, + {"2006-01-02T15:04", "1136214240"}, + {"2006-01-02T15", "1136214000"}, + {"2006-01-02T", "1136160000"}, + {"2006-01-02", "1136160000"}, + {"2006", "1136073600"}, + {"2015-05-13T20:39:09Z", "1431549549"}, + + // unix timestamps returned as is + {"1136073600", "1136073600"}, + + // String fallback + {"invalid", "invalid"}, + } + + for _, c := range cases { + o := GetTimestamp(c.in) + if o != c.expected { + t.Fatalf("wrong value for '%s'. expected:'%s' got:'%s'", c.in, c.expected, o) + } + } +} From 636b40ffa1992815c5063fe8b3f8b9fce67aaf7f Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Thu, 4 Jun 2015 17:37:07 -0700 Subject: [PATCH 2/2] Filtered out unfriendly error from docker when registry is not reachable(code: 502, 503, 504) --- pkg/kubelet/dockertools/docker.go | 22 ++++++++++++++-- pkg/kubelet/dockertools/docker_test.go | 36 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go index 9954370591e..d0945931aaf 100644 --- a/pkg/kubelet/dockertools/docker.go +++ b/pkg/kubelet/dockertools/docker.go @@ -19,6 +19,7 @@ package dockertools import ( "fmt" "math/rand" + "net/http" "os" "strconv" "strings" @@ -31,6 +32,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/types" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" utilerrors "github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors" + "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/parsers" docker "github.com/fsouza/go-dockerclient" "github.com/golang/glog" @@ -113,6 +115,22 @@ func parseImageName(image string) (string, string) { return parsers.ParseRepositoryTag(image) } +func filterHTTPError(err error, image string) error { + // docker/docker/pull/11314 prints detailed error info for docker pull. + // When it hits 502, it returns a verbose html output including an inline svg, + // which makes the output of kubectl get pods much harder to parse. + // Here converts such verbose output to a concise one. + jerr, ok := err.(*jsonmessage.JSONError) + if ok && (jerr.Code == http.StatusBadGateway || + jerr.Code == http.StatusServiceUnavailable || + jerr.Code == http.StatusGatewayTimeout) { + glog.V(2).Infof("Pulling image %q failed: %v", image, err) + return fmt.Errorf("image pull failed for %s because the registry is temporarily unavailbe.", image) + } else { + return err + } +} + func (p dockerPuller) Pull(image string, secrets []api.Secret) error { repoToPull, tag := parseImageName(image) @@ -149,7 +167,7 @@ func (p dockerPuller) Pull(image string, secrets []api.Secret) error { return fmt.Errorf("image pull failed for %s, this may be because there are no credentials on this request. details: (%v)", image, err) } - return err + return filterHTTPError(err, image) } var pullErrs []error @@ -160,7 +178,7 @@ func (p dockerPuller) Pull(image string, secrets []api.Secret) error { return nil } - pullErrs = append(pullErrs, err) + pullErrs = append(pullErrs, filterHTTPError(err, image)) } return utilerrors.NewAggregate(pullErrs) diff --git a/pkg/kubelet/dockertools/docker_test.go b/pkg/kubelet/dockertools/docker_test.go index 5b7b2e863dc..97c260ff942 100644 --- a/pkg/kubelet/dockertools/docker_test.go +++ b/pkg/kubelet/dockertools/docker_test.go @@ -22,6 +22,7 @@ import ( "hash/adler32" "reflect" "sort" + "strings" "testing" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" @@ -31,6 +32,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/network" "github.com/GoogleCloudPlatform/kubernetes/pkg/types" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/docker/docker/pkg/jsonmessage" docker "github.com/fsouza/go-dockerclient" ) @@ -238,6 +240,40 @@ func TestPullWithNoSecrets(t *testing.T) { } } +func TestPullWithJSONError(t *testing.T) { + tests := map[string]struct { + imageName string + err error + expectedError string + }{ + "Json error": { + "ubuntu", + &jsonmessage.JSONError{Code: 50, Message: "Json error"}, + "Json error", + }, + "Bad gateway": { + "ubuntu", + &jsonmessage.JSONError{Code: 502, Message: "\n\n \n \n \n

Oops, there was an error!

\n

We have been contacted of this error, feel free to check out status.docker.com\n to see if there is a bigger issue.

\n\n \n"}, + "because the registry is temporarily unavailbe", + }, + } + for i, test := range tests { + fakeKeyring := &credentialprovider.FakeKeyring{} + fakeClient := &FakeDockerClient{ + Errors: map[string]error{"pull": test.err}, + } + puller := &dockerPuller{ + client: fakeClient, + keyring: fakeKeyring, + } + err := puller.Pull(test.imageName, []api.Secret{}) + if err == nil || !strings.Contains(err.Error(), test.expectedError) { + t.Errorf("%d: expect error %s, got : %s", i, test.expectedError, err) + continue + } + } +} + func TestPullWithSecrets(t *testing.T) { // auth value is equivalent to: "username":"passed-user","password":"passed-password" dockerCfg := map[string]map[string]string{"index.docker.io/v1/": {"email": "passed-email", "auth": "cGFzc2VkLXVzZXI6cGFzc2VkLXBhc3N3b3Jk"}}