Adding a limit on the maximum bytes accepted to be decoded in a resource

write request.
This commit is contained in:
Chao Xu 2019-02-06 16:58:24 -08:00
parent 22b74dc67b
commit b971b12d3c
14 changed files with 109 additions and 9 deletions

View File

@ -130,6 +130,7 @@ func TestAddFlags(t *testing.T) {
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(10 * 1024 * 1024),
MaxRequestBodyBytes: int64(10 * 1024 * 1024),
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{

View File

@ -365,6 +365,17 @@ func NewTooManyRequestsError(message string) *StatusError {
}}
}
// NewRequestEntityTooLargeError returns an error indicating that the request
// entity was too large.
func NewRequestEntityTooLargeError(message string) *StatusError {
return &StatusError{metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusRequestEntityTooLarge,
Reason: metav1.StatusReasonRequestEntityTooLarge,
Message: fmt.Sprintf("Request entity too large: %s", message),
}}
}
// NewGenericServerResponse returns a new error for server responses that are not in a recognizable form.
func NewGenericServerResponse(code int, verb string, qualifiedResource schema.GroupResource, name, serverMessage string, retryAfterSeconds int, isUnexpectedResponse bool) *StatusError {
reason := metav1.StatusReasonUnknown
@ -551,6 +562,19 @@ func IsTooManyRequests(err error) bool {
return false
}
// IsRequestEntityTooLargeError determines if err is an error which indicates
// the request entity is too large.
func IsRequestEntityTooLargeError(err error) bool {
if ReasonForError(err) == metav1.StatusReasonRequestEntityTooLarge {
return true
}
switch t := err.(type) {
case APIStatus:
return t.Status().Code == http.StatusRequestEntityTooLarge
}
return false
}
// IsUnexpectedServerError returns true if the server response was not in the expected API format,
// and may be the result of another HTTP actor.
func IsUnexpectedServerError(err error) bool {

View File

@ -746,6 +746,10 @@ const (
// Status code 406
StatusReasonNotAcceptable StatusReason = "NotAcceptable"
// StatusReasonRequestEntityTooLarge means that the request entity is too large.
// Status code 413
StatusReasonRequestEntityTooLarge StatusReason = "RequestEntityTooLarge"
// StatusReasonUnsupportedMediaType means that the content type sent by the client is not acceptable
// to the server - for instance, attempting to send protobuf for a resource that supports only json and yaml.
// API calls that return UnsupportedMediaType can never succeed.

View File

@ -87,6 +87,10 @@ type APIGroupVersion struct {
// OpenAPIModels exposes the OpenAPI models to each individual handler.
OpenAPIModels openapiproto.Models
// The limit on the request body size that would be accepted and decoded in a write request.
// 0 means no limit.
MaxRequestBodyBytes int64
}
// InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container.

View File

@ -85,7 +85,7 @@ func createHandler(r rest.NamedCreater, scope RequestScope, admit admission.Inte
decoder := scope.Serializer.DecoderToVersion(s.Serializer, scope.HubGroupVersion)
body, err := readBody(req)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return

View File

@ -72,7 +72,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope RequestSco
options := &metav1.DeleteOptions{}
if allowsOptions {
body, err := readBody(req)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return
@ -226,7 +226,7 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope RequestSco
options := &metav1.DeleteOptions{}
if checkBody {
body, err := readBody(req)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return

View File

@ -96,7 +96,7 @@ func PatchResource(r rest.Patcher, scope RequestScope, admit admission.Interface
return
}
patchBytes, err := readBody(req)
patchBytes, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return

View File

@ -20,6 +20,7 @@ import (
"context"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
@ -70,6 +71,8 @@ type RequestScope struct {
// HubGroupVersion indicates what version objects read from etcd or incoming requests should be converted to for in-memory handling.
HubGroupVersion schema.GroupVersion
MaxRequestBodyBytes int64
}
func (scope *RequestScope) err(err error, w http.ResponseWriter, req *http.Request) {
@ -333,9 +336,23 @@ func summarizeData(data []byte, maxLength int) string {
}
}
func readBody(req *http.Request) ([]byte, error) {
func limitedReadBody(req *http.Request, limit int64) ([]byte, error) {
defer req.Body.Close()
return ioutil.ReadAll(req.Body)
if limit <= 0 {
return ioutil.ReadAll(req.Body)
}
lr := &io.LimitedReader{
R: req.Body,
N: limit + 1,
}
data, err := ioutil.ReadAll(lr)
if err != nil {
return nil, err
}
if lr.N <= 0 {
return nil, errors.NewRequestEntityTooLargeError(fmt.Sprintf("limit is %d", limit))
}
return data, nil
}
func parseTimeout(str string) time.Duration {

View File

@ -70,7 +70,7 @@ func UpdateResource(r rest.Updater, scope RequestScope, admit admission.Interfac
return
}
body, err := readBody(req)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return

View File

@ -513,6 +513,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
HubGroupVersion: schema.GroupVersion{Group: fqKindToRegister.Group, Version: runtime.APIVersionInternal},
MetaGroupVersion: metav1.SchemeGroupVersion,
MaxRequestBodyBytes: a.group.MaxRequestBodyBytes,
}
if a.group.MetaGroupVersion != nil {
reqScope.MetaGroupVersion = *a.group.MetaGroupVersion

View File

@ -159,6 +159,9 @@ type Config struct {
// patch may cause.
// This affects all places that applies json patch in the binary.
JSONPatchMaxCopyBytes int64
// The limit on the request body size that would be accepted and decoded in a write request.
// 0 means no limit.
MaxRequestBodyBytes int64
// MaxRequestsInFlight is the maximum number of parallel non-long-running requests. Every further
// request has to wait. Applies only to non-mutating requests.
MaxRequestsInFlight int
@ -268,7 +271,13 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
// on the size increase the "copy" operations in a json patch
// can cause. See
// https://github.com/etcd-io/etcd/blob/release-3.3/etcdserver/server.go#L90.
JSONPatchMaxCopyBytes: int64(10 * 1024 * 1024),
JSONPatchMaxCopyBytes: int64(10 * 1024 * 1024),
// 10MB is the recommended maximum client request size in bytes
// the etcd server should accept. Thus, we set it as the
// maximum bytes accepted to be decoded in a resource write
// request. See
// https://github.com/etcd-io/etcd/blob/release-3.3/etcdserver/server.go#L90.
MaxRequestBodyBytes: int64(10 * 1024 * 1024),
EnableAPIResponseCompression: utilfeature.DefaultFeatureGate.Enabled(features.APIResponseCompression),
// Default to treating watch as a long-running operation
@ -461,6 +470,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),
enableAPIResponseCompression: c.EnableAPIResponseCompression,
maxRequestBodyBytes: c.MaxRequestBodyBytes,
}
for {

View File

@ -156,6 +156,10 @@ type GenericAPIServer struct {
// HandlerChainWaitGroup allows you to wait for all chain handlers finish after the server shutdown.
HandlerChainWaitGroup *utilwaitgroup.SafeWaitGroup
// The limit on the request body size that would be accepted and decoded in a write request.
// 0 means no limit.
maxRequestBodyBytes int64
}
// DelegationTarget is an interface which allows for composition of API servers with top level handling that works
@ -336,6 +340,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
}
apiGroupVersion.OpenAPIModels = openAPIModels
apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes
if err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer); err != nil {
return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err)

View File

@ -45,7 +45,12 @@ type ServerRunOptions struct {
// We intentionally did not add a flag for this option. Users of the
// apiserver library can wire it to a flag.
JSONPatchMaxCopyBytes int64
TargetRAMMB int
// The limit on the request body size that would be accepted and
// decoded in a write request. 0 means no limit.
// We intentionally did not add a flag for this option. Users of the
// apiserver library can wire it to a flag.
MaxRequestBodyBytes int64
TargetRAMMB int
}
func NewServerRunOptions() *ServerRunOptions {
@ -56,6 +61,7 @@ func NewServerRunOptions() *ServerRunOptions {
RequestTimeout: defaults.RequestTimeout,
MinRequestTimeout: defaults.MinRequestTimeout,
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
}
}
@ -68,6 +74,7 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
c.RequestTimeout = s.RequestTimeout
c.MinRequestTimeout = s.MinRequestTimeout
c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
c.PublicAddress = s.AdvertiseAddress
return nil
@ -116,6 +123,10 @@ func (s *ServerRunOptions) Validate() []error {
errors = append(errors, fmt.Errorf("--json-patch-max-copy-bytes can not be negative value"))
}
if s.MaxRequestBodyBytes < 0 {
errors = append(errors, fmt.Errorf("--max-resource-write-bytes can not be negative value"))
}
return errors
}

View File

@ -41,6 +41,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
TargetRAMMB: -65536,
},
expectErr: "--target-ram-mb can not be negative value",
@ -55,6 +56,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
TargetRAMMB: 65536,
},
expectErr: "--max-requests-inflight can not be negative value",
@ -69,6 +71,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
TargetRAMMB: 65536,
},
expectErr: "--max-mutating-requests-inflight can not be negative value",
@ -83,6 +86,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
RequestTimeout: -time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
TargetRAMMB: 65536,
},
expectErr: "--request-timeout can not be negative value",
@ -97,6 +101,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: -1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
TargetRAMMB: 65536,
},
expectErr: "--min-request-timeout can not be negative value",
@ -111,10 +116,26 @@ func TestServerRunOptionsValidate(t *testing.T) {
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: -10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
TargetRAMMB: 65536,
},
expectErr: "--json-patch-max-copy-bytes can not be negative value",
},
{
name: "Test when MaxRequestBodyBytes is negative value",
testOptions: &ServerRunOptions{
AdvertiseAddress: net.ParseIP("192.168.10.10"),
CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"},
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: -10 * 1024 * 1024,
TargetRAMMB: 65536,
},
expectErr: "--max-resource-write-bytes can not be negative value",
},
{
name: "Test when ServerRunOptions is valid",
testOptions: &ServerRunOptions{
@ -125,6 +146,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
TargetRAMMB: 65536,
},
},