apimachinery: add util/net helpers for decoding/encoding Warning headers

This commit is contained in:
Jordan Liggitt 2020-06-05 23:22:26 -04:00
parent 3918393e04
commit fa339d7390
2 changed files with 624 additions and 0 deletions

View File

@ -21,15 +21,20 @@ import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"mime"
"net"
"net/http"
"net/url"
"os"
"path"
"regexp"
"strconv"
"strings"
"unicode"
"unicode/utf8"
"golang.org/x/net/http2"
"k8s.io/klog/v2"
@ -482,3 +487,232 @@ func CloneHeader(in http.Header) http.Header {
}
return out
}
// WarningHeader contains a single RFC2616 14.46 warnings header
type WarningHeader struct {
// Codeindicates the type of warning. 299 is a miscellaneous persistent warning
Code int
// Agent contains the name or pseudonym of the server adding the Warning header.
// A single "-" is recommended when agent is unknown.
Agent string
// Warning text
Text string
}
// ParseWarningHeaders extract RFC2616 14.46 warnings headers from the specified set of header values.
// Multiple comma-separated warnings per header are supported.
// If errors are encountered on a header, the remainder of that header are skipped and subsequent headers are parsed.
// Returns successfully parsed warnings and any errors encountered.
func ParseWarningHeaders(headers []string) ([]WarningHeader, []error) {
var (
results []WarningHeader
errs []error
)
for _, header := range headers {
for len(header) > 0 {
result, remainder, err := ParseWarningHeader(header)
if err != nil {
errs = append(errs, err)
break
}
results = append(results, result)
header = remainder
}
}
return results, errs
}
var (
codeMatcher = regexp.MustCompile(`^[0-9]{3}$`)
wordDecoder = &mime.WordDecoder{}
)
// ParseWarningHeader extracts one RFC2616 14.46 warning from the specified header,
// returning an error if the header does not contain a correctly formatted warning.
// Any remaining content in the header is returned.
func ParseWarningHeader(header string) (result WarningHeader, remainder string, err error) {
// https://tools.ietf.org/html/rfc2616#section-14.46
// updated by
// https://tools.ietf.org/html/rfc7234#section-5.5
// https://tools.ietf.org/html/rfc7234#appendix-A
// Some requirements regarding production and processing of the Warning
// header fields have been relaxed, as it is not widely implemented.
// Furthermore, the Warning header field no longer uses RFC 2047
// encoding, nor does it allow multiple languages, as these aspects were
// not implemented.
//
// Format is one of:
// warn-code warn-agent "warn-text"
// warn-code warn-agent "warn-text" "warn-date"
//
// warn-code is a three digit number
// warn-agent is unquoted and contains no spaces
// warn-text is quoted with backslash escaping (RFC2047-encoded according to RFC2616, not encoded according to RFC7234)
// warn-date is optional, quoted, and in HTTP-date format (no embedded or escaped quotes)
//
// additional warnings can optionally be included in the same header by comma-separating them:
// warn-code warn-agent "warn-text" "warn-date"[, warn-code warn-agent "warn-text" "warn-date", ...]
// tolerate leading whitespace
header = strings.TrimSpace(header)
parts := strings.SplitN(header, " ", 3)
if len(parts) != 3 {
return WarningHeader{}, "", errors.New("invalid warning header: fewer than 3 segments")
}
code, agent, textDateRemainder := parts[0], parts[1], parts[2]
// verify code format
if !codeMatcher.Match([]byte(code)) {
return WarningHeader{}, "", errors.New("invalid warning header: code segment is not 3 digits between 100-299")
}
codeInt, _ := strconv.ParseInt(code, 10, 64)
// verify agent presence
if len(agent) == 0 {
return WarningHeader{}, "", errors.New("invalid warning header: empty agent segment")
}
if !utf8.ValidString(agent) || hasAnyRunes(agent, unicode.IsControl) {
return WarningHeader{}, "", errors.New("invalid warning header: invalid agent")
}
// verify textDateRemainder presence
if len(textDateRemainder) == 0 {
return WarningHeader{}, "", errors.New("invalid warning header: empty text segment")
}
// extract text
text, dateAndRemainder, err := parseQuotedString(textDateRemainder)
if err != nil {
return WarningHeader{}, "", fmt.Errorf("invalid warning header: %v", err)
}
// tolerate RFC2047-encoded text from warnings produced according to RFC2616
if decodedText, err := wordDecoder.DecodeHeader(text); err == nil {
text = decodedText
}
if !utf8.ValidString(text) || hasAnyRunes(text, unicode.IsControl) {
return WarningHeader{}, "", errors.New("invalid warning header: invalid text")
}
result = WarningHeader{Code: int(codeInt), Agent: agent, Text: text}
if len(dateAndRemainder) > 0 {
if dateAndRemainder[0] == '"' {
// consume date
foundEndQuote := false
for i := 1; i < len(dateAndRemainder); i++ {
if dateAndRemainder[i] == '"' {
foundEndQuote = true
remainder = strings.TrimSpace(dateAndRemainder[i+1:])
break
}
}
if !foundEndQuote {
return WarningHeader{}, "", errors.New("invalid warning header: unterminated date segment")
}
} else {
remainder = dateAndRemainder
}
}
if len(remainder) > 0 {
if remainder[0] == ',' {
// consume comma if present
remainder = strings.TrimSpace(remainder[1:])
} else {
return WarningHeader{}, "", errors.New("invalid warning header: unexpected token after warn-date")
}
}
return result, remainder, nil
}
func parseQuotedString(quotedString string) (string, string, error) {
if len(quotedString) == 0 {
return "", "", errors.New("invalid quoted string: 0-length")
}
if quotedString[0] != '"' {
return "", "", errors.New("invalid quoted string: missing initial quote")
}
quotedString = quotedString[1:]
var remainder string
escaping := false
closedQuote := false
result := &bytes.Buffer{}
loop:
for i := 0; i < len(quotedString); i++ {
b := quotedString[i]
switch b {
case '"':
if escaping {
result.WriteByte(b)
escaping = false
} else {
closedQuote = true
remainder = strings.TrimSpace(quotedString[i+1:])
break loop
}
case '\\':
if escaping {
result.WriteByte(b)
escaping = false
} else {
escaping = true
}
default:
result.WriteByte(b)
escaping = false
}
}
if !closedQuote {
return "", "", errors.New("invalid quoted string: missing closing quote")
}
return result.String(), remainder, nil
}
func NewWarningHeader(code int, agent, text string) (string, error) {
if code < 0 || code > 999 {
return "", errors.New("code must be between 0 and 999")
}
if len(agent) == 0 {
agent = "-"
} else if !utf8.ValidString(agent) || strings.ContainsAny(agent, `\"`) || hasAnyRunes(agent, unicode.IsSpace, unicode.IsControl) {
return "", errors.New("agent must be valid UTF-8 and must not contain spaces, quotes, backslashes, or control characters")
}
if !utf8.ValidString(text) || hasAnyRunes(text, unicode.IsControl) {
return "", errors.New("text must be valid UTF-8 and must not contain control characters")
}
return fmt.Sprintf("%03d %s %s", code, agent, makeQuotedString(text)), nil
}
func hasAnyRunes(s string, runeCheckers ...func(rune) bool) bool {
for _, r := range s {
for _, checker := range runeCheckers {
if checker(r) {
return true
}
}
}
return false
}
func makeQuotedString(s string) string {
result := &bytes.Buffer{}
// opening quote
result.WriteRune('"')
for _, c := range s {
switch c {
case '"', '\\':
// escape " and \
result.WriteRune('\\')
result.WriteRune(c)
default:
// write everything else as-is
result.WriteRune(c)
}
}
// closing quote
result.WriteRune('"')
return result.String()
}

View File

@ -618,3 +618,393 @@ func TestSourceIPs(t *testing.T) {
})
}
}
func TestParseWarningHeader(t *testing.T) {
tests := []struct {
name string
header string
wantResult WarningHeader
wantRemainder string
wantErr string
}{
// invalid cases
{
name: "empty",
header: ``,
wantErr: "fewer than 3 segments",
},
{
name: "bad code",
header: `A B`,
wantErr: "fewer than 3 segments",
},
{
name: "short code",
header: `1 - "text"`,
wantErr: "not 3 digits",
},
{
name: "bad code",
header: `A - "text"`,
wantErr: "not 3 digits",
},
{
name: "invalid date quoting",
header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT `,
wantErr: "unterminated date segment",
},
{
name: "invalid post-date",
header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT" other`,
wantErr: "unexpected token after warn-date",
},
{
name: "agent control character",
header: " 299 agent\u0000name \"text\"",
wantErr: "invalid agent",
},
{
name: "agent non-utf8 character",
header: " 299 agent\xc5name \"text\"",
wantErr: "invalid agent",
},
{
name: "text control character",
header: " 299 - \"text\u0000\"content",
wantErr: "invalid text",
},
{
name: "text non-utf8 character",
header: " 299 - \"text\xc5\"content",
wantErr: "invalid text",
},
// valid cases
{
name: "ok",
header: `299 - "text"`,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text`},
},
{
name: "ok",
header: `299 - "text\"\\\a\b\c"`,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`},
},
// big code
{
name: "big code",
header: `321 - "text"`,
wantResult: WarningHeader{Code: 321, Agent: "-", Text: "text"},
},
// RFC 2047 decoding
{
name: "ok, rfc 2047, iso-8859-1, q",
header: `299 - "=?iso-8859-1?q?this=20is=20some=20text?="`,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `this is some text`},
},
{
name: "ok, rfc 2047, utf-8, b",
header: `299 - "=?UTF-8?B?VGhpcyBpcyBhIGhvcnNleTog8J+Qjg==?= And =?UTF-8?B?VGhpcyBpcyBhIGhvcnNleTog8J+Qjg==?="`,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `This is a horsey: 🐎 And This is a horsey: 🐎`},
},
{
name: "ok, rfc 2047, utf-8, q",
header: `299 - "=?UTF-8?Q?This is a \"horsey\": =F0=9F=90=8E?="`,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `This is a "horsey": 🐎`},
},
{
name: "ok, rfc 2047, unknown charset",
header: `299 - "=?UTF-9?Q?This is a horsey: =F0=9F=90=8E?="`,
wantResult: WarningHeader{Code: 299, Agent: "-", Text: `=?UTF-9?Q?This is a horsey: =F0=9F=90=8E?=`},
},
{
name: "ok with spaces",
header: ` 299 - "text\"\\\a\b\c" `,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`},
},
{
name: "ok with date",
header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT" `,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`},
},
{
name: "ok with date and comma",
header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT" , `,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`},
},
{
name: "ok with comma",
header: ` 299 - "text\"\\\a\b\c" , `,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`},
},
{
name: "ok with date and comma and remainder",
header: ` 299 - "text\"\\\a\b\c" "Tue, 15 Nov 1994 08:12:31 GMT" , remainder `,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`},
wantRemainder: "remainder",
},
{
name: "ok with comma and remainder",
header: ` 299 - "text\"\\\a\b\c" ,remainder text,second remainder`,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `text"\abc`},
wantRemainder: "remainder text,second remainder",
},
{
name: "ok with utf-8 content directly in warn-text",
header: ` 299 - "Test of Iñtërnâtiônàlizætiøn,💝🐹🌇⛔" `,
wantResult: WarningHeader{Code: 299, Agent: `-`, Text: `Test of Iñtërnâtiônàlizætiøn,💝🐹🌇⛔`},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotResult, gotRemainder, err := ParseWarningHeader(tt.header)
switch {
case err == nil && len(tt.wantErr) > 0:
t.Errorf("ParseWarningHeader() no error, expected error %q", tt.wantErr)
return
case err != nil && len(tt.wantErr) == 0:
t.Errorf("ParseWarningHeader() error %q, expected no error", err)
return
case err != nil && len(tt.wantErr) > 0 && !strings.Contains(err.Error(), tt.wantErr):
t.Errorf("ParseWarningHeader() error %q, expected error %q", err, tt.wantErr)
return
}
if err != nil {
return
}
if !reflect.DeepEqual(gotResult, tt.wantResult) {
t.Errorf("ParseWarningHeader() gotResult = %#v, want %#v", gotResult, tt.wantResult)
}
if gotRemainder != tt.wantRemainder {
t.Errorf("ParseWarningHeader() gotRemainder = %v, want %v", gotRemainder, tt.wantRemainder)
}
})
}
}
func TestNewWarningHeader(t *testing.T) {
tests := []struct {
name string
code int
agent string
text string
want string
wantErr string
}{
// invalid cases
{
name: "code too low",
code: -1,
agent: `-`,
text: `example warning`,
wantErr: "between 0 and 999",
},
{
name: "code too high",
code: 1000,
agent: `-`,
text: `example warning`,
wantErr: "between 0 and 999",
},
{
name: "agent with space",
code: 299,
agent: `test agent`,
text: `example warning`,
wantErr: `agent must be valid`,
},
{
name: "agent with newline",
code: 299,
agent: "test\nagent",
text: `example warning`,
wantErr: `agent must be valid`,
},
{
name: "agent with backslash",
code: 299,
agent: `test\agent`,
text: `example warning`,
wantErr: `agent must be valid`,
},
{
name: "agent with quote",
code: 299,
agent: `test"agent"`,
text: `example warning`,
wantErr: `agent must be valid`,
},
{
name: "agent with control character",
code: 299,
agent: "test\u0000agent",
text: `example warning`,
wantErr: `agent must be valid`,
},
{
name: "agent with non-UTF8",
code: 299,
agent: "test\xc5agent",
text: `example warning`,
wantErr: `agent must be valid`,
},
{
name: "text with newline",
code: 299,
agent: `-`,
text: "Test of new\nline",
wantErr: "text must be valid",
},
{
name: "text with control character",
code: 299,
agent: `-`,
text: "Test of control\u0000character",
wantErr: "text must be valid",
},
{
name: "text with non-UTF8",
code: 299,
agent: `-`,
text: "Test of control\xc5character",
wantErr: "text must be valid",
},
{
name: "valid empty text",
code: 299,
agent: `-`,
text: ``,
want: `299 - ""`,
},
{
name: "valid empty agent",
code: 299,
agent: ``,
text: `example warning`,
want: `299 - "example warning"`,
},
{
name: "valid low code",
code: 1,
agent: `-`,
text: `example warning`,
want: `001 - "example warning"`,
},
{
name: "valid high code",
code: 999,
agent: `-`,
text: `example warning`,
want: `999 - "example warning"`,
},
{
name: "valid utf-8",
code: 299,
agent: `-`,
text: `Test of "Iñtërnâtiônàlizætiøn,💝🐹🌇⛔"`,
want: `299 - "Test of \"Iñtërnâtiônàlizætiøn,💝🐹🌇⛔\""`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NewWarningHeader(tt.code, tt.agent, tt.text)
switch {
case err == nil && len(tt.wantErr) > 0:
t.Fatalf("ParseWarningHeader() no error, expected error %q", tt.wantErr)
case err != nil && len(tt.wantErr) == 0:
t.Fatalf("ParseWarningHeader() error %q, expected no error", err)
case err != nil && len(tt.wantErr) > 0 && !strings.Contains(err.Error(), tt.wantErr):
t.Fatalf("ParseWarningHeader() error %q, expected error %q", err, tt.wantErr)
}
if err != nil {
return
}
if got != tt.want {
t.Fatalf("NewWarningHeader() = %v, want %v", got, tt.want)
}
roundTrip, remaining, err := ParseWarningHeader(got)
if err != nil {
t.Fatalf("error roundtripping: %v", err)
}
if len(remaining) > 0 {
t.Fatalf("unexpected remainder roundtripping: %s", remaining)
}
agent := tt.agent
if len(agent) == 0 {
agent = "-"
}
expect := WarningHeader{Code: tt.code, Agent: agent, Text: tt.text}
if roundTrip != expect {
t.Fatalf("after round trip, want:\n%#v\ngot\n%#v", expect, roundTrip)
}
})
}
}
func TestParseWarningHeaders(t *testing.T) {
tests := []struct {
name string
headers []string
want []WarningHeader
wantErrs []string
}{
{
name: "empty",
headers: []string{},
want: nil,
wantErrs: []string{},
},
{
name: "multi-header with error",
headers: []string{
`299 - "warning 1.1",299 - "warning 1.2"`,
`299 - "warning 2", 299 - "warning unquoted`,
` 299 - "warning 3.1" , 299 - "warning 3.2" `,
},
want: []WarningHeader{
{Code: 299, Agent: "-", Text: "warning 1.1"},
{Code: 299, Agent: "-", Text: "warning 1.2"},
{Code: 299, Agent: "-", Text: "warning 2"},
{Code: 299, Agent: "-", Text: "warning 3.1"},
{Code: 299, Agent: "-", Text: "warning 3.2"},
},
wantErrs: []string{"invalid warning header: invalid quoted string: missing closing quote"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, gotErrs := ParseWarningHeaders(tt.headers)
switch {
case len(gotErrs) != len(tt.wantErrs):
t.Fatalf("ParseWarningHeader() got %v, expected %v", gotErrs, tt.wantErrs)
case len(gotErrs) == len(tt.wantErrs) && len(gotErrs) > 0:
gotErrStrings := []string{}
for _, err := range gotErrs {
gotErrStrings = append(gotErrStrings, err.Error())
}
if !reflect.DeepEqual(gotErrStrings, tt.wantErrs) {
t.Fatalf("ParseWarningHeader() got %v, expected %v", gotErrs, tt.wantErrs)
}
}
if len(gotErrs) > 0 {
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("ParseWarningHeaders() got %#v, want %#v", got, tt.want)
}
})
}
}