From fa339d7390a9bb227cff1fea6a48af7c8727b21b Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 5 Jun 2020 23:22:26 -0400 Subject: [PATCH] apimachinery: add util/net helpers for decoding/encoding Warning headers --- .../k8s.io/apimachinery/pkg/util/net/http.go | 234 +++++++++++ .../apimachinery/pkg/util/net/http_test.go | 390 ++++++++++++++++++ 2 files changed, 624 insertions(+) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/http.go b/staging/src/k8s.io/apimachinery/pkg/util/net/http.go index 20c9a245bb2..8ac4b1f8d16 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/http.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/http.go @@ -21,15 +21,20 @@ import ( "bytes" "context" "crypto/tls" + "errors" "fmt" "io" + "mime" "net" "net/http" "net/url" "os" "path" + "regexp" "strconv" "strings" + "unicode" + "unicode/utf8" "golang.org/x/net/http2" "k8s.io/klog/v2" @@ -482,3 +487,232 @@ func CloneHeader(in http.Header) http.Header { } return out } + +// WarningHeader contains a single RFC2616 14.46 warnings header +type WarningHeader struct { + // Codeindicates the type of warning. 299 is a miscellaneous persistent warning + Code int + // Agent contains the name or pseudonym of the server adding the Warning header. + // A single "-" is recommended when agent is unknown. + Agent string + // Warning text + Text string +} + +// ParseWarningHeaders extract RFC2616 14.46 warnings headers from the specified set of header values. +// Multiple comma-separated warnings per header are supported. +// If errors are encountered on a header, the remainder of that header are skipped and subsequent headers are parsed. +// Returns successfully parsed warnings and any errors encountered. +func ParseWarningHeaders(headers []string) ([]WarningHeader, []error) { + var ( + results []WarningHeader + errs []error + ) + for _, header := range headers { + for len(header) > 0 { + result, remainder, err := ParseWarningHeader(header) + if err != nil { + errs = append(errs, err) + break + } + results = append(results, result) + header = remainder + } + } + return results, errs +} + +var ( + codeMatcher = regexp.MustCompile(`^[0-9]{3}$`) + wordDecoder = &mime.WordDecoder{} +) + +// ParseWarningHeader extracts one RFC2616 14.46 warning from the specified header, +// returning an error if the header does not contain a correctly formatted warning. +// Any remaining content in the header is returned. +func ParseWarningHeader(header string) (result WarningHeader, remainder string, err error) { + // https://tools.ietf.org/html/rfc2616#section-14.46 + // updated by + // https://tools.ietf.org/html/rfc7234#section-5.5 + // https://tools.ietf.org/html/rfc7234#appendix-A + // Some requirements regarding production and processing of the Warning + // header fields have been relaxed, as it is not widely implemented. + // Furthermore, the Warning header field no longer uses RFC 2047 + // encoding, nor does it allow multiple languages, as these aspects were + // not implemented. + // + // Format is one of: + // warn-code warn-agent "warn-text" + // warn-code warn-agent "warn-text" "warn-date" + // + // warn-code is a three digit number + // warn-agent is unquoted and contains no spaces + // warn-text is quoted with backslash escaping (RFC2047-encoded according to RFC2616, not encoded according to RFC7234) + // warn-date is optional, quoted, and in HTTP-date format (no embedded or escaped quotes) + // + // additional warnings can optionally be included in the same header by comma-separating them: + // warn-code warn-agent "warn-text" "warn-date"[, warn-code warn-agent "warn-text" "warn-date", ...] + + // tolerate leading whitespace + header = strings.TrimSpace(header) + + parts := strings.SplitN(header, " ", 3) + if len(parts) != 3 { + return WarningHeader{}, "", errors.New("invalid warning header: fewer than 3 segments") + } + code, agent, textDateRemainder := parts[0], parts[1], parts[2] + + // verify code format + if !codeMatcher.Match([]byte(code)) { + return WarningHeader{}, "", errors.New("invalid warning header: code segment is not 3 digits between 100-299") + } + codeInt, _ := strconv.ParseInt(code, 10, 64) + + // verify agent presence + if len(agent) == 0 { + return WarningHeader{}, "", errors.New("invalid warning header: empty agent segment") + } + if !utf8.ValidString(agent) || hasAnyRunes(agent, unicode.IsControl) { + return WarningHeader{}, "", errors.New("invalid warning header: invalid agent") + } + + // verify textDateRemainder presence + if len(textDateRemainder) == 0 { + return WarningHeader{}, "", errors.New("invalid warning header: empty text segment") + } + + // extract text + text, dateAndRemainder, err := parseQuotedString(textDateRemainder) + if err != nil { + return WarningHeader{}, "", fmt.Errorf("invalid warning header: %v", err) + } + // tolerate RFC2047-encoded text from warnings produced according to RFC2616 + if decodedText, err := wordDecoder.DecodeHeader(text); err == nil { + text = decodedText + } + if !utf8.ValidString(text) || hasAnyRunes(text, unicode.IsControl) { + return WarningHeader{}, "", errors.New("invalid warning header: invalid text") + } + result = WarningHeader{Code: int(codeInt), Agent: agent, Text: text} + + if len(dateAndRemainder) > 0 { + if dateAndRemainder[0] == '"' { + // consume date + foundEndQuote := false + for i := 1; i < len(dateAndRemainder); i++ { + if dateAndRemainder[i] == '"' { + foundEndQuote = true + remainder = strings.TrimSpace(dateAndRemainder[i+1:]) + break + } + } + if !foundEndQuote { + return WarningHeader{}, "", errors.New("invalid warning header: unterminated date segment") + } + } else { + remainder = dateAndRemainder + } + } + if len(remainder) > 0 { + if remainder[0] == ',' { + // consume comma if present + remainder = strings.TrimSpace(remainder[1:]) + } else { + return WarningHeader{}, "", errors.New("invalid warning header: unexpected token after warn-date") + } + } + + return result, remainder, nil +} + +func parseQuotedString(quotedString string) (string, string, error) { + if len(quotedString) == 0 { + return "", "", errors.New("invalid quoted string: 0-length") + } + + if quotedString[0] != '"' { + return "", "", errors.New("invalid quoted string: missing initial quote") + } + + quotedString = quotedString[1:] + var remainder string + escaping := false + closedQuote := false + result := &bytes.Buffer{} +loop: + for i := 0; i < len(quotedString); i++ { + b := quotedString[i] + switch b { + case '"': + if escaping { + result.WriteByte(b) + escaping = false + } else { + closedQuote = true + remainder = strings.TrimSpace(quotedString[i+1:]) + break loop + } + case '\\': + if escaping { + result.WriteByte(b) + escaping = false + } else { + escaping = true + } + default: + result.WriteByte(b) + escaping = false + } + } + + if !closedQuote { + return "", "", errors.New("invalid quoted string: missing closing quote") + } + return result.String(), remainder, nil +} + +func NewWarningHeader(code int, agent, text string) (string, error) { + if code < 0 || code > 999 { + return "", errors.New("code must be between 0 and 999") + } + if len(agent) == 0 { + agent = "-" + } else if !utf8.ValidString(agent) || strings.ContainsAny(agent, `\"`) || hasAnyRunes(agent, unicode.IsSpace, unicode.IsControl) { + return "", errors.New("agent must be valid UTF-8 and must not contain spaces, quotes, backslashes, or control characters") + } + if !utf8.ValidString(text) || hasAnyRunes(text, unicode.IsControl) { + return "", errors.New("text must be valid UTF-8 and must not contain control characters") + } + return fmt.Sprintf("%03d %s %s", code, agent, makeQuotedString(text)), nil +} + +func hasAnyRunes(s string, runeCheckers ...func(rune) bool) bool { + for _, r := range s { + for _, checker := range runeCheckers { + if checker(r) { + return true + } + } + } + return false +} + +func makeQuotedString(s string) string { + result := &bytes.Buffer{} + // opening quote + result.WriteRune('"') + for _, c := range s { + switch c { + case '"', '\\': + // escape " and \ + result.WriteRune('\\') + result.WriteRune(c) + default: + // write everything else as-is + result.WriteRune(c) + } + } + // closing quote + result.WriteRune('"') + return result.String() +} diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/http_test.go b/staging/src/k8s.io/apimachinery/pkg/util/net/http_test.go index da483c3563b..cd4f7f743ea 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/http_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/http_test.go @@ -618,3 +618,393 @@ func TestSourceIPs(t *testing.T) { }) } } + +func TestParseWarningHeader(t *testing.T) { + tests := []struct { + name string + + header string + + wantResult WarningHeader + wantRemainder string + wantErr string + }{ + // invalid cases + { + name: "empty", + header: ``, + wantErr: "fewer than 3 segments", + }, + { + name: "bad code", + header: `A B`, + wantErr: "fewer than 3 segments", + }, + { + name: "short code", + header: `1 - "text"`, + wantErr: "not 3 digits", + }, + { + name: "bad code", + header: `A - "text"`, + wantErr: "not 3 digits", + }, + { + name: "invalid date quoting", + header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT `, + wantErr: "unterminated date segment", + }, + { + name: "invalid post-date", + header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT" other`, + wantErr: "unexpected token after warn-date", + }, + { + name: "agent control character", + header: " 299 agent\u0000name \"text\"", + wantErr: "invalid agent", + }, + { + name: "agent non-utf8 character", + header: " 299 agent\xc5name \"text\"", + wantErr: "invalid agent", + }, + { + name: "text control character", + header: " 299 - \"text\u0000\"content", + wantErr: "invalid text", + }, + { + name: "text non-utf8 character", + header: " 299 - \"text\xc5\"content", + wantErr: "invalid text", + }, + + // valid cases + { + name: "ok", + header: `299 - "text"`, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text`}, + }, + { + name: "ok", + header: `299 - "text\"\\\a\b\c"`, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`}, + }, + // big code + { + name: "big code", + header: `321 - "text"`, + wantResult: WarningHeader{Code: 321, Agent: "-", Text: "text"}, + }, + // RFC 2047 decoding + { + name: "ok, rfc 2047, iso-8859-1, q", + header: `299 - "=?iso-8859-1?q?this=20is=20some=20text?="`, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `this is some text`}, + }, + { + name: "ok, rfc 2047, utf-8, b", + header: `299 - "=?UTF-8?B?VGhpcyBpcyBhIGhvcnNleTog8J+Qjg==?= And =?UTF-8?B?VGhpcyBpcyBhIGhvcnNleTog8J+Qjg==?="`, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `This is a horsey: 🐎 And This is a horsey: 🐎`}, + }, + { + name: "ok, rfc 2047, utf-8, q", + header: `299 - "=?UTF-8?Q?This is a \"horsey\": =F0=9F=90=8E?="`, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `This is a "horsey": 🐎`}, + }, + { + name: "ok, rfc 2047, unknown charset", + header: `299 - "=?UTF-9?Q?This is a horsey: =F0=9F=90=8E?="`, + wantResult: WarningHeader{Code: 299, Agent: "-", Text: `=?UTF-9?Q?This is a horsey: =F0=9F=90=8E?=`}, + }, + { + name: "ok with spaces", + header: ` 299 - "text\"\\\a\b\c" `, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`}, + }, + { + name: "ok with date", + header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT" `, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`}, + }, + { + name: "ok with date and comma", + header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT" , `, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`}, + }, + { + name: "ok with comma", + header: ` 299 - "text\"\\\a\b\c" , `, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`}, + }, + { + name: "ok with date and comma and remainder", + header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT" , remainder `, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`}, + wantRemainder: "remainder", + }, + { + name: "ok with comma and remainder", + header: ` 299 - "text\"\\\a\b\c" ,remainder text,second remainder`, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`}, + wantRemainder: "remainder text,second remainder", + }, + { + name: "ok with utf-8 content directly in warn-text", + header: ` 299 - "Test of Iñtërnâtiônàlizætiøn,💝🐹🌇⛔" `, + wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `Test of Iñtërnâtiônàlizætiøn,💝🐹🌇⛔`}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotResult, gotRemainder, err := ParseWarningHeader(tt.header) + switch { + case err == nil && len(tt.wantErr) > 0: + t.Errorf("ParseWarningHeader() no error, expected error %q", tt.wantErr) + return + case err != nil && len(tt.wantErr) == 0: + t.Errorf("ParseWarningHeader() error %q, expected no error", err) + return + case err != nil && len(tt.wantErr) > 0 && !strings.Contains(err.Error(), tt.wantErr): + t.Errorf("ParseWarningHeader() error %q, expected error %q", err, tt.wantErr) + return + } + if err != nil { + return + } + if !reflect.DeepEqual(gotResult, tt.wantResult) { + t.Errorf("ParseWarningHeader() gotResult = %#v, want %#v", gotResult, tt.wantResult) + } + if gotRemainder != tt.wantRemainder { + t.Errorf("ParseWarningHeader() gotRemainder = %v, want %v", gotRemainder, tt.wantRemainder) + } + }) + } +} + +func TestNewWarningHeader(t *testing.T) { + tests := []struct { + name string + + code int + agent string + text string + + want string + wantErr string + }{ + // invalid cases + { + name: "code too low", + code: -1, + agent: `-`, + text: `example warning`, + wantErr: "between 0 and 999", + }, + { + name: "code too high", + code: 1000, + agent: `-`, + text: `example warning`, + wantErr: "between 0 and 999", + }, + { + name: "agent with space", + code: 299, + agent: `test agent`, + text: `example warning`, + wantErr: `agent must be valid`, + }, + { + name: "agent with newline", + code: 299, + agent: "test\nagent", + text: `example warning`, + wantErr: `agent must be valid`, + }, + { + name: "agent with backslash", + code: 299, + agent: `test\agent`, + text: `example warning`, + wantErr: `agent must be valid`, + }, + { + name: "agent with quote", + code: 299, + agent: `test"agent"`, + text: `example warning`, + wantErr: `agent must be valid`, + }, + { + name: "agent with control character", + code: 299, + agent: "test\u0000agent", + text: `example warning`, + wantErr: `agent must be valid`, + }, + { + name: "agent with non-UTF8", + code: 299, + agent: "test\xc5agent", + text: `example warning`, + wantErr: `agent must be valid`, + }, + { + name: "text with newline", + code: 299, + agent: `-`, + text: "Test of new\nline", + wantErr: "text must be valid", + }, + { + name: "text with control character", + code: 299, + agent: `-`, + text: "Test of control\u0000character", + wantErr: "text must be valid", + }, + { + name: "text with non-UTF8", + code: 299, + agent: `-`, + text: "Test of control\xc5character", + wantErr: "text must be valid", + }, + + { + name: "valid empty text", + code: 299, + agent: `-`, + text: ``, + want: `299 - ""`, + }, + { + name: "valid empty agent", + code: 299, + agent: ``, + text: `example warning`, + want: `299 - "example warning"`, + }, + { + name: "valid low code", + code: 1, + agent: `-`, + text: `example warning`, + want: `001 - "example warning"`, + }, + { + name: "valid high code", + code: 999, + agent: `-`, + text: `example warning`, + want: `999 - "example warning"`, + }, + { + name: "valid utf-8", + code: 299, + agent: `-`, + text: `Test of "Iñtërnâtiônàlizætiøn,💝🐹🌇⛔"`, + want: `299 - "Test of \"Iñtërnâtiônàlizætiøn,💝🐹🌇⛔\""`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewWarningHeader(tt.code, tt.agent, tt.text) + + switch { + case err == nil && len(tt.wantErr) > 0: + t.Fatalf("ParseWarningHeader() no error, expected error %q", tt.wantErr) + case err != nil && len(tt.wantErr) == 0: + t.Fatalf("ParseWarningHeader() error %q, expected no error", err) + case err != nil && len(tt.wantErr) > 0 && !strings.Contains(err.Error(), tt.wantErr): + t.Fatalf("ParseWarningHeader() error %q, expected error %q", err, tt.wantErr) + } + if err != nil { + return + } + + if got != tt.want { + t.Fatalf("NewWarningHeader() = %v, want %v", got, tt.want) + } + + roundTrip, remaining, err := ParseWarningHeader(got) + if err != nil { + t.Fatalf("error roundtripping: %v", err) + } + if len(remaining) > 0 { + t.Fatalf("unexpected remainder roundtripping: %s", remaining) + } + agent := tt.agent + if len(agent) == 0 { + agent = "-" + } + expect := WarningHeader{Code: tt.code, Agent: agent, Text: tt.text} + if roundTrip != expect { + t.Fatalf("after round trip, want:\n%#v\ngot\n%#v", expect, roundTrip) + } + }) + } +} + +func TestParseWarningHeaders(t *testing.T) { + tests := []struct { + name string + + headers []string + + want []WarningHeader + wantErrs []string + }{ + { + name: "empty", + headers: []string{}, + want: nil, + wantErrs: []string{}, + }, + { + name: "multi-header with error", + headers: []string{ + `299 - "warning 1.1",299 - "warning 1.2"`, + `299 - "warning 2", 299 - "warning unquoted`, + ` 299 - "warning 3.1" , 299 - "warning 3.2" `, + }, + want: []WarningHeader{ + {Code: 299, Agent: "-", Text: "warning 1.1"}, + {Code: 299, Agent: "-", Text: "warning 1.2"}, + {Code: 299, Agent: "-", Text: "warning 2"}, + {Code: 299, Agent: "-", Text: "warning 3.1"}, + {Code: 299, Agent: "-", Text: "warning 3.2"}, + }, + wantErrs: []string{"invalid warning header: invalid quoted string: missing closing quote"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErrs := ParseWarningHeaders(tt.headers) + + switch { + case len(gotErrs) != len(tt.wantErrs): + t.Fatalf("ParseWarningHeader() got %v, expected %v", gotErrs, tt.wantErrs) + case len(gotErrs) == len(tt.wantErrs) && len(gotErrs) > 0: + gotErrStrings := []string{} + for _, err := range gotErrs { + gotErrStrings = append(gotErrStrings, err.Error()) + } + if !reflect.DeepEqual(gotErrStrings, tt.wantErrs) { + t.Fatalf("ParseWarningHeader() got %v, expected %v", gotErrs, tt.wantErrs) + } + } + if len(gotErrs) > 0 { + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseWarningHeaders() got %#v, want %#v", got, tt.want) + } + }) + } +}