diff --git a/go.mod b/go.mod index 89f0d604a70..64ac5d4d164 100644 --- a/go.mod +++ b/go.mod @@ -97,7 +97,7 @@ require ( github.com/urfave/negroni v1.0.0 // indirect github.com/vishvananda/netlink v1.1.0 github.com/vmware/govmomi v0.20.3 - go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f + go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 @@ -414,7 +414,7 @@ replace ( github.com/xlab/handysort => github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 github.com/xordataexchange/crypt => github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 go.etcd.io/bbolt => go.etcd.io/bbolt v1.3.5 - go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f // 54ba9589114f is the SHA for git tag v3.4.9 + go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 // 18dfb9cca345 is the SHA for git tag v3.4.10 go.mongodb.org/mongo-driver => go.mongodb.org/mongo-driver v1.1.2 go.opencensus.io => go.opencensus.io v0.22.2 go.uber.org/atomic => go.uber.org/atomic v1.4.0 diff --git a/go.sum b/go.sum index 74982254b0f..f49229589a7 100644 --- a/go.sum +++ b/go.sum @@ -467,8 +467,8 @@ github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSf github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f h1:pBCD+Z7cy5WPTq+R6MmJJvDRpn88cp7bmTypBsn91g4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 h1:2gOG36vt1BhUqpzxwZLZJxUim2dHB05vw+RAn4Q6YOU= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.mod b/staging/src/k8s.io/apiextensions-apiserver/go.mod index cee4e0bf952..d511bd68cb8 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.mod +++ b/staging/src/k8s.io/apiextensions-apiserver/go.mod @@ -18,7 +18,7 @@ require ( github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 - go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f + go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 google.golang.org/grpc v1.27.0 gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.0.0 diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.sum b/staging/src/k8s.io/apiextensions-apiserver/go.sum index 5d0616ef920..7895e50f7e9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.sum +++ b/staging/src/k8s.io/apiextensions-apiserver/go.sum @@ -386,8 +386,8 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f h1:pBCD+Z7cy5WPTq+R6MmJJvDRpn88cp7bmTypBsn91g4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 h1:2gOG36vt1BhUqpzxwZLZJxUim2dHB05vw+RAn4Q6YOU= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA= diff --git a/staging/src/k8s.io/apiserver/go.mod b/staging/src/k8s.io/apiserver/go.mod index 23864b8d532..b3480c1ebd5 100644 --- a/staging/src/k8s.io/apiserver/go.mod +++ b/staging/src/k8s.io/apiserver/go.mod @@ -31,7 +31,7 @@ require ( github.com/stretchr/testify v1.4.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect go.etcd.io/bbolt v1.3.5 // indirect - go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f + go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 go.uber.org/zap v1.10.0 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e diff --git a/staging/src/k8s.io/apiserver/go.sum b/staging/src/k8s.io/apiserver/go.sum index 093ac08c202..9ed3f5b489a 100644 --- a/staging/src/k8s.io/apiserver/go.sum +++ b/staging/src/k8s.io/apiserver/go.sum @@ -299,8 +299,8 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f h1:pBCD+Z7cy5WPTq+R6MmJJvDRpn88cp7bmTypBsn91g4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 h1:2gOG36vt1BhUqpzxwZLZJxUim2dHB05vw+RAn4Q6YOU= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/staging/src/k8s.io/kube-aggregator/go.sum b/staging/src/k8s.io/kube-aggregator/go.sum index f46302ad135..5e248fa84dc 100644 --- a/staging/src/k8s.io/kube-aggregator/go.sum +++ b/staging/src/k8s.io/kube-aggregator/go.sum @@ -332,8 +332,8 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f h1:pBCD+Z7cy5WPTq+R6MmJJvDRpn88cp7bmTypBsn91g4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 h1:2gOG36vt1BhUqpzxwZLZJxUim2dHB05vw+RAn4Q6YOU= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/staging/src/k8s.io/legacy-cloud-providers/go.sum b/staging/src/k8s.io/legacy-cloud-providers/go.sum index 48652911034..84db1efc33f 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/go.sum +++ b/staging/src/k8s.io/legacy-cloud-providers/go.sum @@ -304,7 +304,7 @@ github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59b github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= diff --git a/staging/src/k8s.io/sample-apiserver/go.sum b/staging/src/k8s.io/sample-apiserver/go.sum index f3b6a599f7f..1ae10bb331c 100644 --- a/staging/src/k8s.io/sample-apiserver/go.sum +++ b/staging/src/k8s.io/sample-apiserver/go.sum @@ -329,8 +329,8 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f h1:pBCD+Z7cy5WPTq+R6MmJJvDRpn88cp7bmTypBsn91g4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 h1:2gOG36vt1BhUqpzxwZLZJxUim2dHB05vw+RAn4Q6YOU= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/vendor/go.etcd.io/etcd/auth/simple_token.go b/vendor/go.etcd.io/etcd/auth/simple_token.go index 934978c9857..a9dc5b715f6 100644 --- a/vendor/go.etcd.io/etcd/auth/simple_token.go +++ b/vendor/go.etcd.io/etcd/auth/simple_token.go @@ -37,7 +37,7 @@ const ( // var for testing purposes var ( - simpleTokenTTL = 5 * time.Minute + simpleTokenTTLDefault = 300 * time.Second simpleTokenTTLResolution = 1 * time.Second ) @@ -47,6 +47,7 @@ type simpleTokenTTLKeeper struct { stopc chan struct{} deleteTokenFunc func(string) mu *sync.Mutex + simpleTokenTTL time.Duration } func (tm *simpleTokenTTLKeeper) stop() { @@ -58,12 +59,12 @@ func (tm *simpleTokenTTLKeeper) stop() { } func (tm *simpleTokenTTLKeeper) addSimpleToken(token string) { - tm.tokens[token] = time.Now().Add(simpleTokenTTL) + tm.tokens[token] = time.Now().Add(tm.simpleTokenTTL) } func (tm *simpleTokenTTLKeeper) resetSimpleToken(token string) { if _, ok := tm.tokens[token]; ok { - tm.tokens[token] = time.Now().Add(simpleTokenTTL) + tm.tokens[token] = time.Now().Add(tm.simpleTokenTTL) } } @@ -101,6 +102,7 @@ type tokenSimple struct { simpleTokenKeeper *simpleTokenTTLKeeper simpleTokensMu sync.Mutex simpleTokens map[string]string // token -> username + simpleTokenTTL time.Duration } func (t *tokenSimple) genTokenPrefix() (string, error) { @@ -157,6 +159,10 @@ func (t *tokenSimple) invalidateUser(username string) { } func (t *tokenSimple) enable() { + if t.simpleTokenTTL <= 0 { + t.simpleTokenTTL = simpleTokenTTLDefault + } + delf := func(tk string) { if username, ok := t.simpleTokens[tk]; ok { if t.lg != nil { @@ -177,6 +183,7 @@ func (t *tokenSimple) enable() { stopc: make(chan struct{}), deleteTokenFunc: delf, mu: &t.simpleTokensMu, + simpleTokenTTL: t.simpleTokenTTL, } go t.simpleTokenKeeper.run() } @@ -234,10 +241,14 @@ func (t *tokenSimple) isValidSimpleToken(ctx context.Context, token string) bool return false } -func newTokenProviderSimple(lg *zap.Logger, indexWaiter func(uint64) <-chan struct{}) *tokenSimple { +func newTokenProviderSimple(lg *zap.Logger, indexWaiter func(uint64) <-chan struct{}, TokenTTL time.Duration) *tokenSimple { + if lg == nil { + lg = zap.NewNop() + } return &tokenSimple{ - lg: lg, - simpleTokens: make(map[string]string), - indexWaiter: indexWaiter, + lg: lg, + simpleTokens: make(map[string]string), + indexWaiter: indexWaiter, + simpleTokenTTL: TokenTTL, } } diff --git a/vendor/go.etcd.io/etcd/auth/store.go b/vendor/go.etcd.io/etcd/auth/store.go index bf3e474bf80..ba1d95ff2ca 100644 --- a/vendor/go.etcd.io/etcd/auth/store.go +++ b/vendor/go.etcd.io/etcd/auth/store.go @@ -23,6 +23,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "go.etcd.io/etcd/auth/authpb" "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" @@ -59,6 +60,7 @@ var ( ErrRoleNotFound = errors.New("auth: role not found") ErrRoleEmpty = errors.New("auth: role name is empty") ErrAuthFailed = errors.New("auth: authentication failed, invalid user ID or password") + ErrNoPasswordUser = errors.New("auth: authentication failed, password was given for no password user") ErrPermissionDenied = errors.New("auth: permission denied") ErrRoleNotGranted = errors.New("auth: role is not granted to the user") ErrPermissionNotGranted = errors.New("auth: permission is not granted to the role") @@ -360,7 +362,7 @@ func (as *authStore) CheckPassword(username, password string) (uint64, error) { } if user.Options != nil && user.Options.NoPassword { - return 0, ErrAuthFailed + return 0, ErrNoPasswordUser } return getRevision(tx), nil @@ -994,7 +996,7 @@ func (as *authStore) IsAdminPermitted(authInfo *AuthInfo) error { if !as.IsAuthEnabled() { return nil } - if authInfo == nil { + if authInfo == nil || authInfo.Username == "" { return ErrUserEmpty } @@ -1351,7 +1353,8 @@ func decomposeOpts(lg *zap.Logger, optstr string) (string, map[string]string, er func NewTokenProvider( lg *zap.Logger, tokenOpts string, - indexWaiter func(uint64) <-chan struct{}) (TokenProvider, error) { + indexWaiter func(uint64) <-chan struct{}, + TokenTTL time.Duration) (TokenProvider, error) { tokenType, typeSpecificOpts, err := decomposeOpts(lg, tokenOpts) if err != nil { return nil, ErrInvalidAuthOpts @@ -1364,7 +1367,7 @@ func NewTokenProvider( } else { plog.Warningf("simple token is not cryptographically signed") } - return newTokenProviderSimple(lg, indexWaiter), nil + return newTokenProviderSimple(lg, indexWaiter, TokenTTL), nil case tokenTypeJWT: return newTokenProviderJWT(lg, typeSpecificOpts) diff --git a/vendor/go.etcd.io/etcd/clientv3/watch.go b/vendor/go.etcd.io/etcd/clientv3/watch.go index 87d222d1d68..4ae3a0b33fb 100644 --- a/vendor/go.etcd.io/etcd/clientv3/watch.go +++ b/vendor/go.etcd.io/etcd/clientv3/watch.go @@ -25,6 +25,7 @@ import ( pb "go.etcd.io/etcd/etcdserver/etcdserverpb" mvccpb "go.etcd.io/etcd/mvcc/mvccpb" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -616,7 +617,10 @@ func (w *watchGrpcStream) run() { }, } req := &pb.WatchRequest{RequestUnion: cr} - wc.Send(req) + lg.Info("sending watch cancel request for failed dispatch", zap.Int64("watch-id", pbresp.WatchId)) + if err := wc.Send(req); err != nil { + lg.Warning("failed to send watch cancel request", zap.Int64("watch-id", pbresp.WatchId), zap.Error(err)) + } } // watch client failed on Recv; spawn another if possible @@ -637,6 +641,21 @@ func (w *watchGrpcStream) run() { return case ws := <-w.closingc: + if ws.id != -1 { + // client is closing an established watch; close it on the server proactively instead of waiting + // to close when the next message arrives + cancelSet[ws.id] = struct{}{} + cr := &pb.WatchRequest_CancelRequest{ + CancelRequest: &pb.WatchCancelRequest{ + WatchId: ws.id, + }, + } + req := &pb.WatchRequest{RequestUnion: cr} + lg.Info("sending watch cancel request for closed watcher", zap.Int64("watch-id", ws.id)) + if err := wc.Send(req); err != nil { + lg.Warning("failed to send watch cancel request", zap.Int64("watch-id", ws.id), zap.Error(err)) + } + } w.closeSubstream(ws) delete(closing, ws) // no more watchers on this stream, shutdown diff --git a/vendor/go.etcd.io/etcd/embed/config.go b/vendor/go.etcd.io/etcd/embed/config.go index 2f64d927f2a..a21a41b0209 100644 --- a/vendor/go.etcd.io/etcd/embed/config.go +++ b/vendor/go.etcd.io/etcd/embed/config.go @@ -273,6 +273,9 @@ type Config struct { AuthToken string `json:"auth-token"` BcryptCost uint `json:"bcrypt-cost"` + //The AuthTokenTTL in seconds of the simple token + AuthTokenTTL uint `json:"auth-token-ttl"` + ExperimentalInitialCorruptCheck bool `json:"experimental-initial-corrupt-check"` ExperimentalCorruptCheckTime time.Duration `json:"experimental-corrupt-check-time"` ExperimentalEnableV2V3 string `json:"experimental-enable-v2v3"` @@ -335,6 +338,10 @@ type Config struct { // Only valid if "logger" option is "capnslog". // WARN: DO NOT USE THIS! LogPkgLevels string `json:"log-package-levels"` + + // UnsafeNoFsync disables all uses of fsync. + // Setting this is unsafe and will cause data loss. + UnsafeNoFsync bool `json:"unsafe-no-fsync"` } // configYAML holds the config suitable for yaml parsing @@ -406,8 +413,9 @@ func NewConfig() *Config { CORS: map[string]struct{}{"*": {}}, HostWhitelist: map[string]struct{}{"*": {}}, - AuthToken: "simple", - BcryptCost: uint(bcrypt.DefaultCost), + AuthToken: "simple", + BcryptCost: uint(bcrypt.DefaultCost), + AuthTokenTTL: 300, PreVote: false, // TODO: enable by default in v3.5 diff --git a/vendor/go.etcd.io/etcd/embed/etcd.go b/vendor/go.etcd.io/etcd/embed/etcd.go index ac7dbc987fb..30e56f668a1 100644 --- a/vendor/go.etcd.io/etcd/embed/etcd.go +++ b/vendor/go.etcd.io/etcd/embed/etcd.go @@ -192,6 +192,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ClientCertAuthEnabled: cfg.ClientTLSInfo.ClientCertAuth, AuthToken: cfg.AuthToken, BcryptCost: cfg.BcryptCost, + TokenTTL: cfg.AuthTokenTTL, CORS: cfg.CORS, HostWhitelist: cfg.HostWhitelist, InitialCorruptCheck: cfg.ExperimentalInitialCorruptCheck, @@ -204,6 +205,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { Debug: cfg.Debug, ForceNewCluster: cfg.ForceNewCluster, EnableGRPCGateway: cfg.EnableGRPCGateway, + UnsafeNoFsync: cfg.UnsafeNoFsync, EnableLeaseCheckpoint: cfg.ExperimentalEnableLeaseCheckpoint, CompactionBatchLimit: cfg.ExperimentalCompactionBatchLimit, } @@ -811,7 +813,7 @@ func (e *Etcd) GetLogger() *zap.Logger { func parseCompactionRetention(mode, retention string) (ret time.Duration, err error) { h, err := strconv.Atoi(retention) - if err == nil { + if err == nil && h >= 0 { switch mode { case CompactorModeRevision: ret = time.Duration(int64(h)) diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/v2discovery/discovery.go b/vendor/go.etcd.io/etcd/etcdserver/api/v2discovery/discovery.go index cf770b37859..16ccfde8e27 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/v2discovery/discovery.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/v2discovery/discovery.go @@ -218,7 +218,7 @@ func (d *discovery) createSelf(contents string) error { return err } -func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) { +func (d *discovery) checkCluster() ([]*client.Node, uint64, uint64, error) { configKey := path.Join("/", d.cluster, "_config") ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout) // find cluster size @@ -247,7 +247,7 @@ func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) { } return nil, 0, 0, err } - size, err := strconv.Atoi(resp.Node.Value) + size, err := strconv.ParseUint(resp.Node.Value, 10, 0) if err != nil { return nil, 0, 0, ErrBadSizeKey } @@ -288,7 +288,7 @@ func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) { if path.Base(nodes[i].Key) == path.Base(d.selfKey()) { break } - if i >= size-1 { + if uint64(i) >= size-1 { return nodes[:size], size, resp.Index, ErrFullCluster } } @@ -316,7 +316,7 @@ func (d *discovery) logAndBackoffForRetry(step string) { d.clock.Sleep(retryTimeInSecond) } -func (d *discovery) checkClusterRetry() ([]*client.Node, int, uint64, error) { +func (d *discovery) checkClusterRetry() ([]*client.Node, uint64, uint64, error) { if d.retries < nRetries { d.logAndBackoffForRetry("cluster status check") return d.checkCluster() @@ -336,8 +336,8 @@ func (d *discovery) waitNodesRetry() ([]*client.Node, error) { return nil, ErrTooManyRetries } -func (d *discovery) waitNodes(nodes []*client.Node, size int, index uint64) ([]*client.Node, error) { - if len(nodes) > size { +func (d *discovery) waitNodes(nodes []*client.Node, size uint64, index uint64) ([]*client.Node, error) { + if uint64(len(nodes)) > size { nodes = nodes[:size] } // watch from the next index @@ -369,16 +369,16 @@ func (d *discovery) waitNodes(nodes []*client.Node, size int, index uint64) ([]* } // wait for others - for len(all) < size { + for uint64(len(all)) < size { if d.lg != nil { d.lg.Info( "found peers from discovery server; waiting for more", zap.String("discovery-url", d.url.String()), zap.Int("found-peers", len(all)), - zap.Int("needed-peers", size-len(all)), + zap.Int("needed-peers", int(size-uint64(len(all)))), ) } else { - plog.Noticef("found %d peer(s), waiting for %d more", len(all), size-len(all)) + plog.Noticef("found %d peer(s), waiting for %d more", len(all), size-uint64(len(all))) } resp, err := w.Next(context.Background()) if err != nil { @@ -415,7 +415,7 @@ func (d *discovery) selfKey() string { return path.Join("/", d.cluster, d.id.String()) } -func nodesToCluster(ns []*client.Node, size int) (string, error) { +func nodesToCluster(ns []*client.Node, size uint64) (string, error) { s := make([]string, len(ns)) for i, n := range ns { s[i] = n.Value @@ -425,7 +425,7 @@ func nodesToCluster(ns []*client.Node, size int) (string, error) { if err != nil { return us, ErrInvalidURL } - if m.Len() != size { + if uint64(m.Len()) != size { return us, ErrDuplicateName } return us, nil diff --git a/vendor/go.etcd.io/etcd/etcdserver/backend.go b/vendor/go.etcd.io/etcd/etcdserver/backend.go index 3eace1a33c6..aa443cac86f 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/backend.go +++ b/vendor/go.etcd.io/etcd/etcdserver/backend.go @@ -31,6 +31,7 @@ import ( func newBackend(cfg ServerConfig) backend.Backend { bcfg := backend.DefaultBackendConfig() bcfg.Path = cfg.backendPath() + bcfg.UnsafeNoFsync = cfg.UnsafeNoFsync if cfg.BackendBatchLimit != 0 { bcfg.BatchLimit = cfg.BackendBatchLimit if cfg.Logger != nil { diff --git a/vendor/go.etcd.io/etcd/etcdserver/config.go b/vendor/go.etcd.io/etcd/etcdserver/config.go index 88cd721c325..b8cdc037a7f 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/config.go +++ b/vendor/go.etcd.io/etcd/etcdserver/config.go @@ -126,6 +126,7 @@ type ServerConfig struct { AuthToken string BcryptCost uint + TokenTTL uint // InitialCorruptCheck is true to check data corruption on boot // before serving any peer/client traffic. @@ -157,6 +158,10 @@ type ServerConfig struct { LeaseCheckpointInterval time.Duration EnableGRPCGateway bool + + // UnsafeNoFsync disables all uses of fsync. + // Setting this is unsafe and will cause data loss. + UnsafeNoFsync bool `json:"unsafe-no-fsync"` } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/raft_internal_stringer.go b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/raft_internal_stringer.go index 3d3536a326d..31e121ee0a6 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/raft_internal_stringer.go +++ b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/raft_internal_stringer.go @@ -137,7 +137,7 @@ type loggableValueCompare struct { Result Compare_CompareResult `protobuf:"varint,1,opt,name=result,proto3,enum=etcdserverpb.Compare_CompareResult"` Target Compare_CompareTarget `protobuf:"varint,2,opt,name=target,proto3,enum=etcdserverpb.Compare_CompareTarget"` Key []byte `protobuf:"bytes,3,opt,name=key,proto3"` - ValueSize int `protobuf:"bytes,7,opt,name=value_size,proto3"` + ValueSize int64 `protobuf:"varint,7,opt,name=value_size,proto3"` RangeEnd []byte `protobuf:"bytes,64,opt,name=range_end,proto3"` } @@ -146,7 +146,7 @@ func newLoggableValueCompare(c *Compare, cv *Compare_Value) *loggableValueCompar c.Result, c.Target, c.Key, - len(cv.Value), + int64(len(cv.Value)), c.RangeEnd, } } @@ -160,7 +160,7 @@ func (*loggableValueCompare) ProtoMessage() {} // To preserve proto encoding of the key bytes, a faked out proto type is used here. type loggablePutRequest struct { Key []byte `protobuf:"bytes,1,opt,name=key,proto3"` - ValueSize int `protobuf:"varint,2,opt,name=value_size,proto3"` + ValueSize int64 `protobuf:"varint,2,opt,name=value_size,proto3"` Lease int64 `protobuf:"varint,3,opt,name=lease,proto3"` PrevKv bool `protobuf:"varint,4,opt,name=prev_kv,proto3"` IgnoreValue bool `protobuf:"varint,5,opt,name=ignore_value,proto3"` @@ -170,7 +170,7 @@ type loggablePutRequest struct { func NewLoggablePutRequest(request *PutRequest) *loggablePutRequest { return &loggablePutRequest{ request.Key, - len(request.Value), + int64(len(request.Value)), request.Lease, request.PrevKv, request.IgnoreValue, diff --git a/vendor/go.etcd.io/etcd/etcdserver/metrics.go b/vendor/go.etcd.io/etcd/etcdserver/metrics.go index e0c0cde8553..57e21ff3769 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/metrics.go +++ b/vendor/go.etcd.io/etcd/etcdserver/metrics.go @@ -184,7 +184,13 @@ func init() { } func monitorFileDescriptor(lg *zap.Logger, done <-chan struct{}) { - ticker := time.NewTicker(5 * time.Second) + + // This ticker will check File Descriptor Requirements ,and count all fds in used. + // And recorded some logs when in used >= limit/5*4. Just recorded message. + // If fds was more than 10K,It's low performance due to FDUsage() works. + // So need to increase it. + // See https://github.com/etcd-io/etcd/issues/11969 for more detail. + ticker := time.NewTicker(10 * time.Minute) defer ticker.Stop() for { used, err := runtime.FDUsage() diff --git a/vendor/go.etcd.io/etcd/etcdserver/raft.go b/vendor/go.etcd.io/etcd/etcdserver/raft.go index e7cf0729929..df08e3de3fe 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/raft.go +++ b/vendor/go.etcd.io/etcd/etcdserver/raft.go @@ -465,6 +465,9 @@ func startNode(cfg ServerConfig, cl *membership.RaftCluster, ids []types.ID) (id plog.Panicf("create wal error: %v", err) } } + if cfg.UnsafeNoFsync { + w.SetUnsafeNoFsync() + } peers := make([]raft.Peer, len(ids)) for i, id := range ids { var ctx []byte @@ -527,7 +530,7 @@ func restartNode(cfg ServerConfig, snapshot *raftpb.Snapshot) (types.ID, *member if snapshot != nil { walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term } - w, id, cid, st, ents := readWAL(cfg.Logger, cfg.WALDir(), walsnap) + w, id, cid, st, ents := readWAL(cfg.Logger, cfg.WALDir(), walsnap, cfg.UnsafeNoFsync) if cfg.Logger != nil { cfg.Logger.Info( @@ -582,7 +585,7 @@ func restartAsStandaloneNode(cfg ServerConfig, snapshot *raftpb.Snapshot) (types if snapshot != nil { walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term } - w, id, cid, st, ents := readWAL(cfg.Logger, cfg.WALDir(), walsnap) + w, id, cid, st, ents := readWAL(cfg.Logger, cfg.WALDir(), walsnap, cfg.UnsafeNoFsync) // discard the previously uncommitted entries for i, ent := range ents { diff --git a/vendor/go.etcd.io/etcd/etcdserver/server.go b/vendor/go.etcd.io/etcd/etcdserver/server.go index fb98c05b7c6..a341625dccb 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/server.go +++ b/vendor/go.etcd.io/etcd/etcdserver/server.go @@ -553,6 +553,7 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) { func(index uint64) <-chan struct{} { return srv.applyWait.Wait(index) }, + time.Duration(cfg.TokenTTL)*time.Second, ) if err != nil { if cfg.Logger != nil { diff --git a/vendor/go.etcd.io/etcd/etcdserver/storage.go b/vendor/go.etcd.io/etcd/etcdserver/storage.go index 9a000b8da66..d87e51af0cb 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/storage.go +++ b/vendor/go.etcd.io/etcd/etcdserver/storage.go @@ -82,7 +82,7 @@ func (st *storage) Release(snap raftpb.Snapshot) error { // readWAL reads the WAL at the given snap and returns the wal, its latest HardState and cluster ID, and all entries that appear // after the position of the given snap in the WAL. // The snap must have been previously saved to the WAL, or this call will panic. -func readWAL(lg *zap.Logger, waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) { +func readWAL(lg *zap.Logger, waldir string, snap walpb.Snapshot, unsafeNoFsync bool) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) { var ( err error wmetadata []byte @@ -97,6 +97,9 @@ func readWAL(lg *zap.Logger, waldir string, snap walpb.Snapshot) (w *wal.WAL, id plog.Fatalf("open wal error: %v", err) } } + if unsafeNoFsync { + w.SetUnsafeNoFsync() + } if wmetadata, st, ents, err = w.ReadAll(); err != nil { w.Close() // we can only repair ErrUnexpectedEOF and we never repair twice. diff --git a/vendor/go.etcd.io/etcd/etcdserver/v3_server.go b/vendor/go.etcd.io/etcd/etcdserver/v3_server.go index 70b7177d39e..1fa8e4e6fc7 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/v3_server.go +++ b/vendor/go.etcd.io/etcd/etcdserver/v3_server.go @@ -428,9 +428,10 @@ func (s *EtcdServer) Authenticate(ctx context.Context, r *pb.AuthenticateRequest return nil, err } + // internalReq doesn't need to have Password because the above s.AuthStore().CheckPassword() already did it. + // In addition, it will let a WAL entry not record password as a plain text. internalReq := &pb.InternalAuthenticateRequest{ Name: r.Name, - Password: r.Password, SimpleToken: st, } diff --git a/vendor/go.etcd.io/etcd/mvcc/backend/backend.go b/vendor/go.etcd.io/etcd/mvcc/backend/backend.go index 26f196fbff3..523d358d5ae 100644 --- a/vendor/go.etcd.io/etcd/mvcc/backend/backend.go +++ b/vendor/go.etcd.io/etcd/mvcc/backend/backend.go @@ -123,6 +123,8 @@ type BackendConfig struct { MmapSize uint64 // Logger logs backend-side operations. Logger *zap.Logger + // UnsafeNoFsync disables all uses of fsync. + UnsafeNoFsync bool `json:"unsafe-no-fsync"` } func DefaultBackendConfig() BackendConfig { @@ -150,6 +152,8 @@ func newBackend(bcfg BackendConfig) *backend { } bopts.InitialMmapSize = bcfg.mmapSize() bopts.FreelistType = bcfg.BackendFreelistType + bopts.NoSync = bcfg.UnsafeNoFsync + bopts.NoGrowSync = bcfg.UnsafeNoFsync db, err := bolt.Open(bcfg.Path, 0600, bopts) if err != nil { diff --git a/vendor/go.etcd.io/etcd/mvcc/watchable_store.go b/vendor/go.etcd.io/etcd/mvcc/watchable_store.go index 72c6b8be4ba..55d0e9dee91 100644 --- a/vendor/go.etcd.io/etcd/mvcc/watchable_store.go +++ b/vendor/go.etcd.io/etcd/mvcc/watchable_store.go @@ -30,9 +30,8 @@ import ( var ( // chanBufLen is the length of the buffered chan // for sending out watched events. - // TODO: find a good buf value. 1024 is just a random one that - // seems to be reasonable. - chanBufLen = 1024 + // See https://github.com/etcd-io/etcd/issues/11906 for more detail. + chanBufLen = 128 // maxWatchersPerSync is the number of watchers to sync in a single batch maxWatchersPerSync = 512 diff --git a/vendor/go.etcd.io/etcd/pkg/fileutil/dir_unix.go b/vendor/go.etcd.io/etcd/pkg/fileutil/dir_unix.go index 58a77dfc1a9..4ce15dc6bcf 100644 --- a/vendor/go.etcd.io/etcd/pkg/fileutil/dir_unix.go +++ b/vendor/go.etcd.io/etcd/pkg/fileutil/dir_unix.go @@ -18,5 +18,10 @@ package fileutil import "os" +const ( + // PrivateDirMode grants owner to make/remove files inside the directory. + PrivateDirMode = 0700 +) + // OpenDir opens a directory for syncing. func OpenDir(path string) (*os.File, error) { return os.Open(path) } diff --git a/vendor/go.etcd.io/etcd/pkg/fileutil/dir_windows.go b/vendor/go.etcd.io/etcd/pkg/fileutil/dir_windows.go index c123395c004..a10a90583c7 100644 --- a/vendor/go.etcd.io/etcd/pkg/fileutil/dir_windows.go +++ b/vendor/go.etcd.io/etcd/pkg/fileutil/dir_windows.go @@ -21,6 +21,11 @@ import ( "syscall" ) +const ( + // PrivateDirMode grants owner to make/remove files inside the directory. + PrivateDirMode = 0777 +) + // OpenDir opens a directory in windows with write access for syncing. func OpenDir(path string) (*os.File, error) { fd, err := openDir(path) diff --git a/vendor/go.etcd.io/etcd/pkg/fileutil/fileutil.go b/vendor/go.etcd.io/etcd/pkg/fileutil/fileutil.go index 5d9fb530395..5ff6369242f 100644 --- a/vendor/go.etcd.io/etcd/pkg/fileutil/fileutil.go +++ b/vendor/go.etcd.io/etcd/pkg/fileutil/fileutil.go @@ -27,8 +27,6 @@ import ( const ( // PrivateFileMode grants owner to read/write a file. PrivateFileMode = 0600 - // PrivateDirMode grants owner to make/remove files inside the directory. - PrivateDirMode = 0700 ) var plog = capnslog.NewPackageLogger("go.etcd.io/etcd", "pkg/fileutil") @@ -46,14 +44,22 @@ func IsDirWriteable(dir string) error { // TouchDirAll is similar to os.MkdirAll. It creates directories with 0700 permission if any directory // does not exists. TouchDirAll also ensures the given directory is writable. func TouchDirAll(dir string) error { - // If path is already a directory, MkdirAll does nothing - // and returns nil. - err := os.MkdirAll(dir, PrivateDirMode) - if err != nil { - // if mkdirAll("a/text") and "text" is not - // a directory, this will return syscall.ENOTDIR - return err + // If path is already a directory, MkdirAll does nothing and returns nil, so, + // first check if dir exist with an expected permission mode. + if Exist(dir) { + err := CheckDirPermission(dir, PrivateDirMode) + if err != nil { + return err + } + } else { + err := os.MkdirAll(dir, PrivateDirMode) + if err != nil { + // if mkdirAll("a/text") and "text" is not + // a directory, this will return syscall.ENOTDIR + return err + } } + return IsDirWriteable(dir) } @@ -102,3 +108,22 @@ func ZeroToEnd(f *os.File) error { _, err = f.Seek(off, io.SeekStart) return err } + +// CheckDirPermission checks permission on an existing dir. +// Returns error if dir is empty or exist with a different permission than specified. +func CheckDirPermission(dir string, perm os.FileMode) error { + if !Exist(dir) { + return fmt.Errorf("directory %q empty, cannot check permission.", dir) + } + //check the existing permission on the directory + dirInfo, err := os.Stat(dir) + if err != nil { + return err + } + dirMode := dirInfo.Mode().Perm() + if dirMode != perm { + err = fmt.Errorf("directory %q,%q exist without desired file permission %q.", dir, dirInfo.Mode(), os.FileMode(PrivateDirMode)) + return err + } + return nil +} diff --git a/vendor/go.etcd.io/etcd/pkg/transport/BUILD b/vendor/go.etcd.io/etcd/pkg/transport/BUILD index 09bc2a808de..55f9b3c4deb 100644 --- a/vendor/go.etcd.io/etcd/pkg/transport/BUILD +++ b/vendor/go.etcd.io/etcd/pkg/transport/BUILD @@ -20,6 +20,7 @@ go_library( importpath = "go.etcd.io/etcd/pkg/transport", visibility = ["//visibility:public"], deps = [ + "//vendor/go.etcd.io/etcd/pkg/fileutil:go_default_library", "//vendor/go.etcd.io/etcd/pkg/tlsutil:go_default_library", "//vendor/go.uber.org/zap:go_default_library", ], diff --git a/vendor/go.etcd.io/etcd/pkg/transport/listener.go b/vendor/go.etcd.io/etcd/pkg/transport/listener.go index 80e35bda59a..7260e4d079c 100644 --- a/vendor/go.etcd.io/etcd/pkg/transport/listener.go +++ b/vendor/go.etcd.io/etcd/pkg/transport/listener.go @@ -31,6 +31,7 @@ import ( "strings" "time" + "go.etcd.io/etcd/pkg/fileutil" "go.etcd.io/etcd/pkg/tlsutil" "go.uber.org/zap" @@ -114,10 +115,17 @@ func (info TLSInfo) Empty() bool { } func SelfCert(lg *zap.Logger, dirpath string, hosts []string, additionalUsages ...x509.ExtKeyUsage) (info TLSInfo, err error) { - if err = os.MkdirAll(dirpath, 0700); err != nil { + info.Logger = lg + err = fileutil.TouchDirAll(dirpath) + if err != nil { + if info.Logger != nil { + info.Logger.Warn( + "cannot create cert directory", + zap.Error(err), + ) + } return } - info.Logger = lg certPath := filepath.Join(dirpath, "cert.pem") keyPath := filepath.Join(dirpath, "key.pem") diff --git a/vendor/go.etcd.io/etcd/version/version.go b/vendor/go.etcd.io/etcd/version/version.go index ce2acaef1d6..b1a002f6114 100644 --- a/vendor/go.etcd.io/etcd/version/version.go +++ b/vendor/go.etcd.io/etcd/version/version.go @@ -26,7 +26,7 @@ import ( var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.4.9" + Version = "3.4.10" APIVersion = "unknown" // Git SHA Value will be set during build diff --git a/vendor/go.etcd.io/etcd/wal/decoder.go b/vendor/go.etcd.io/etcd/wal/decoder.go index f2f01fd881c..d007fe1c748 100644 --- a/vendor/go.etcd.io/etcd/wal/decoder.go +++ b/vendor/go.etcd.io/etcd/wal/decoder.go @@ -59,6 +59,11 @@ func (d *decoder) decode(rec *walpb.Record) error { return d.decodeRecord(rec) } +// raft max message size is set to 1 MB in etcd server +// assume projects set reasonable message size limit, +// thus entry size should never exceed 10 MB +const maxWALEntrySizeLimit = int64(10 * 1024 * 1024) + func (d *decoder) decodeRecord(rec *walpb.Record) error { if len(d.brs) == 0 { return io.EOF @@ -79,6 +84,9 @@ func (d *decoder) decodeRecord(rec *walpb.Record) error { } recBytes, padBytes := decodeFrameSize(l) + if recBytes >= maxWALEntrySizeLimit-padBytes { + return ErrMaxWALEntrySizeLimitExceeded + } data := make([]byte, recBytes+padBytes) if _, err = io.ReadFull(d.brs[0], data); err != nil { diff --git a/vendor/go.etcd.io/etcd/wal/wal.go b/vendor/go.etcd.io/etcd/wal/wal.go index 0451f94d058..1aced50bdd1 100644 --- a/vendor/go.etcd.io/etcd/wal/wal.go +++ b/vendor/go.etcd.io/etcd/wal/wal.go @@ -56,12 +56,15 @@ var ( plog = capnslog.NewPackageLogger("go.etcd.io/etcd", "wal") - ErrMetadataConflict = errors.New("wal: conflicting metadata found") - ErrFileNotFound = errors.New("wal: file not found") - ErrCRCMismatch = errors.New("wal: crc mismatch") - ErrSnapshotMismatch = errors.New("wal: snapshot mismatch") - ErrSnapshotNotFound = errors.New("wal: snapshot not found") - crcTable = crc32.MakeTable(crc32.Castagnoli) + ErrMetadataConflict = errors.New("wal: conflicting metadata found") + ErrFileNotFound = errors.New("wal: file not found") + ErrCRCMismatch = errors.New("wal: crc mismatch") + ErrSnapshotMismatch = errors.New("wal: snapshot mismatch") + ErrSnapshotNotFound = errors.New("wal: snapshot not found") + ErrSliceOutOfRange = errors.New("wal: slice bounds out of range") + ErrMaxWALEntrySizeLimitExceeded = errors.New("wal: max entry size limit exceeded") + ErrDecoderNotFound = errors.New("wal: decoder not found") + crcTable = crc32.MakeTable(crc32.Castagnoli) ) // WAL is a logical representation of the stable storage. @@ -84,6 +87,8 @@ type WAL struct { decoder *decoder // decoder to decode records readClose func() error // closer for decode reader + unsafeNoSync bool // if set, do not fsync + mu sync.Mutex enti uint64 // index of the last entry saved to the wal encoder *encoder // encoder to encode records @@ -93,7 +98,8 @@ type WAL struct { } // Create creates a WAL ready for appending records. The given metadata is -// recorded at the head of each WAL file, and can be retrieved with ReadAll. +// recorded at the head of each WAL file, and can be retrieved with ReadAll +// after the file is Open. func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) { if Exist(dirpath) { return nil, os.ErrExist @@ -233,6 +239,10 @@ func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) { return w, nil } +func (w *WAL) SetUnsafeNoFsync() { + w.unsafeNoSync = true +} + func (w *WAL) cleanupWAL(lg *zap.Logger) { var err error if err = w.Close(); err != nil { @@ -428,6 +438,10 @@ func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb. defer w.mu.Unlock() rec := &walpb.Record{} + + if w.decoder == nil { + return nil, state, nil, ErrDecoderNotFound + } decoder := w.decoder var match bool @@ -435,8 +449,15 @@ func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb. switch rec.Type { case entryType: e := mustUnmarshalEntry(rec.Data) + // 0 <= e.Index-w.start.Index - 1 < len(ents) if e.Index > w.start.Index { - ents = append(ents[:e.Index-w.start.Index-1], e) + // prevent "panic: runtime error: slice bounds out of range [:13038096702221461992] with capacity 0" + up := e.Index - w.start.Index - 1 + if up > uint64(len(ents)) { + // return error before append call causes runtime panic + return nil, state, nil, ErrSliceOutOfRange + } + ents = append(ents[:up], e) } w.enti = e.Index @@ -768,6 +789,9 @@ func (w *WAL) cut() error { } func (w *WAL) sync() error { + if w.unsafeNoSync { + return nil + } if w.encoder != nil { if err := w.encoder.flush(); err != nil { return err diff --git a/vendor/modules.txt b/vendor/modules.txt index 3ea1b97a710..a67d8b97fd6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -725,7 +725,7 @@ github.com/vmware/govmomi/vim25/xml github.com/xiang90/probing # go.etcd.io/bbolt v1.3.5 => go.etcd.io/bbolt v1.3.5 go.etcd.io/bbolt -# go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f => go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f +# go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 => go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 go.etcd.io/etcd/auth go.etcd.io/etcd/auth/authpb go.etcd.io/etcd/client