Pin go.etcd.io/etcd to 3.4.4 and rebuild vendor

This commit is contained in:
jingyih 2020-03-16 10:58:56 -07:00
parent 0b6bed9577
commit 1f7ccbcfc3
26 changed files with 279 additions and 89 deletions

4
go.mod
View File

@ -120,7 +120,7 @@ require (
github.com/vishvananda/netlink v1.0.0
github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936 // indirect
github.com/vmware/govmomi v0.20.3
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
golang.org/x/lint v0.0.0-20190409202823-959b441ac422
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9
@ -485,7 +485,7 @@ replace (
github.com/xlab/handysort => github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1
github.com/xordataexchange/crypt => github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77
go.etcd.io/bbolt => go.etcd.io/bbolt v1.3.3
go.etcd.io/etcd => go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 // 3cf2f69b5738 is the SHA for git tag v3.4.3
go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd // c65a9e2dd1fd is the SHA for git tag v3.4.4
go.mongodb.org/mongo-driver => go.mongodb.org/mongo-driver v1.1.2
go.opencensus.io => go.opencensus.io v0.21.0
go.uber.org/atomic => go.uber.org/atomic v1.3.2

4
go.sum
View File

@ -512,8 +512,8 @@ github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSf
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd h1:JwlSZ9TSTcRHDGbIr5QG3MEmKyMK99BbNLMJtwnJ0rY=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA=
go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=

View File

@ -18,7 +18,7 @@ require (
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.4.0
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd
google.golang.org/grpc v1.26.0
gopkg.in/yaml.v2 v2.2.8
k8s.io/api v0.0.0

View File

@ -309,8 +309,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd h1:JwlSZ9TSTcRHDGbIr5QG3MEmKyMK99BbNLMJtwnJ0rY=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA=

View File

@ -33,7 +33,7 @@ require (
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.4.0
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd
go.uber.org/zap v1.10.0
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9

View File

@ -236,8 +236,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd h1:JwlSZ9TSTcRHDGbIr5QG3MEmKyMK99BbNLMJtwnJ0rY=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

View File

@ -255,8 +255,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd h1:JwlSZ9TSTcRHDGbIr5QG3MEmKyMK99BbNLMJtwnJ0rY=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

View File

@ -236,7 +236,7 @@ github.com/vmware/govmomi v0.20.3 h1:gpw/0Ku+6RgF3jsi7fnCLmlcikBHfKBCUcu1qgc16OU
github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

View File

@ -252,8 +252,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd h1:JwlSZ9TSTcRHDGbIr5QG3MEmKyMK99BbNLMJtwnJ0rY=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

6
vendor/go.etcd.io/etcd/auth/jwt.go generated vendored
View File

@ -105,7 +105,7 @@ func (t *tokenJWT) assign(ctx context.Context, username string, revision uint64)
token, err := tk.SignedString(t.key)
if err != nil {
if t.lg != nil {
t.lg.Warn(
t.lg.Debug(
"failed to sign a JWT token",
zap.String("user-name", username),
zap.Uint64("revision", revision),
@ -118,7 +118,7 @@ func (t *tokenJWT) assign(ctx context.Context, username string, revision uint64)
}
if t.lg != nil {
t.lg.Info(
t.lg.Debug(
"created/assigned a new JWT token",
zap.String("user-name", username),
zap.Uint64("revision", revision),
@ -136,7 +136,7 @@ func newTokenProviderJWT(lg *zap.Logger, optMap map[string]string) (*tokenJWT, e
err = opts.ParseWithDefaults(optMap)
if err != nil {
if lg != nil {
lg.Warn("problem loading JWT options", zap.Error(err))
lg.Error("problem loading JWT options", zap.Error(err))
} else {
plog.Errorf("problem loading JWT options: %s", err)
}

View File

@ -306,7 +306,7 @@ func (as *authStore) Authenticate(ctx context.Context, username, password string
return nil, ErrAuthFailed
}
if user.Options.NoPassword {
if user.Options != nil && user.Options.NoPassword {
return nil, ErrAuthFailed
}
@ -344,7 +344,7 @@ func (as *authStore) CheckPassword(username, password string) (uint64, error) {
return 0, ErrAuthFailed
}
if user.Options.NoPassword {
if user.Options != nil && user.Options.NoPassword {
return 0, ErrAuthFailed
}
@ -388,7 +388,8 @@ func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse,
var hashed []byte
var err error
if r.Options != nil && !r.Options.NoPassword {
noPassword := r.Options != nil && r.Options.NoPassword
if !noPassword {
hashed, err = bcrypt.GenerateFromPassword([]byte(r.Password), as.bcryptCost)
if err != nil {
if as.lg != nil {

View File

@ -113,10 +113,9 @@ func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOp
return nil, status.Errorf(codes.Unimplemented, "clientv3/retry_interceptor: cannot retry on ClientStreams, set Disable()")
}
newStreamer, err := streamer(ctx, desc, cc, method, grpcOpts...)
logger.Warn("retry stream intercept", zap.Error(err))
if err != nil {
// TODO(mwitkow): Maybe dial and transport errors should be retriable?
return nil, err
logger.Error("streamer failed to create ClientStream", zap.Error(err))
return nil, err // TODO(mwitkow): Maybe dial and transport errors should be retriable?
}
retryingStreamer := &serverStreamingRetryingStream{
client: c,
@ -185,6 +184,7 @@ func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error {
if !attemptRetry {
return lastErr // success or hard failure
}
// We start off from attempt 1, because zeroth was already made on normal SendMsg().
for attempt := uint(1); attempt < s.callOpts.max; attempt++ {
if err := waitRetryBackoff(s.ctx, attempt, s.callOpts); err != nil {
@ -192,12 +192,13 @@ func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error {
}
newStream, err := s.reestablishStreamAndResendBuffer(s.ctx)
if err != nil {
// TODO(mwitkow): Maybe dial and transport errors should be retriable?
return err
s.client.lg.Error("failed reestablishStreamAndResendBuffer", zap.Error(err))
return err // TODO(mwitkow): Maybe dial and transport errors should be retriable?
}
s.setStream(newStream)
s.client.lg.Warn("retrying RecvMsg", zap.Error(lastErr))
attemptRetry, lastErr = s.receiveMsgAndIndicateRetry(m)
//fmt.Printf("Received message and indicate: %v %v\n", attemptRetry, lastErr)
if !attemptRetry {
return lastErr
}

View File

@ -36,7 +36,6 @@ go_library(
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/go.etcd.io/bbolt:go_default_library",
"//vendor/go.etcd.io/etcd/auth:go_default_library",
"//vendor/go.etcd.io/etcd/clientv3:go_default_library",
"//vendor/go.etcd.io/etcd/etcdserver/api:go_default_library",
"//vendor/go.etcd.io/etcd/etcdserver/api/membership:go_default_library",
"//vendor/go.etcd.io/etcd/etcdserver/api/rafthttp:go_default_library",

View File

@ -37,11 +37,17 @@ const (
)
// NewPeerHandler generates an http.Handler to handle etcd peer requests.
func NewPeerHandler(lg *zap.Logger, s etcdserver.ServerPeer) http.Handler {
return newPeerHandler(lg, s, s.RaftHandler(), s.LeaseHandler())
func NewPeerHandler(lg *zap.Logger, s etcdserver.ServerPeerV2) http.Handler {
return newPeerHandler(lg, s, s.RaftHandler(), s.LeaseHandler(), s.HashKVHandler())
}
func newPeerHandler(lg *zap.Logger, s etcdserver.Server, raftHandler http.Handler, leaseHandler http.Handler) http.Handler {
func newPeerHandler(
lg *zap.Logger,
s etcdserver.Server,
raftHandler http.Handler,
leaseHandler http.Handler,
hashKVHandler http.Handler,
) http.Handler {
peerMembersHandler := newPeerMembersHandler(lg, s.Cluster())
peerMemberPromoteHandler := newPeerMemberPromoteHandler(lg, s)
@ -55,6 +61,9 @@ func newPeerHandler(lg *zap.Logger, s etcdserver.Server, raftHandler http.Handle
mux.Handle(leasehttp.LeasePrefix, leaseHandler)
mux.Handle(leasehttp.LeaseInternalPrefix, leaseHandler)
}
if hashKVHandler != nil {
mux.Handle(etcdserver.PeerHashKVPath, hashKVHandler)
}
mux.HandleFunc(versionPath, versionHandler(s.Cluster(), serveVersion))
return mux
}

View File

@ -657,8 +657,8 @@ func (c *RaftCluster) IsReadyToRemoveVotingMember(id uint64) bool {
}
func (c *RaftCluster) IsReadyToPromoteMember(id uint64) bool {
nmembers := 1
nstarted := 0
nmembers := 1 // We count the learner to be promoted for the future quorum
nstarted := 1 // and we also count it as started.
for _, member := range c.VotingMembers() {
if member.IsStarted() {

View File

@ -226,6 +226,9 @@ func (s *Snapshotter) snapNames() ([]string, error) {
if err != nil {
return nil, err
}
if err = s.cleanupSnapdir(names); err != nil {
return nil, err
}
snaps := checkSuffix(s.lg, names)
if len(snaps) == 0 {
return nil, ErrNoSnapshot
@ -253,3 +256,21 @@ func checkSuffix(lg *zap.Logger, names []string) []string {
}
return snaps
}
// cleanupSnapdir removes any files that should not be in the snapshot directory:
// - db.tmp prefixed files that can be orphaned by defragmentation
func (s *Snapshotter) cleanupSnapdir(filenames []string) error {
for _, filename := range filenames {
if strings.HasPrefix(filename, "db.tmp") {
if s.lg != nil {
s.lg.Info("found orphaned defragmentation file; deleting", zap.String("path", filename))
} else {
plog.Infof("found orphaned defragmentation file; deleting: %s", filename)
}
if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) {
return fmt.Errorf("failed to remove orphaned defragmentation file %s: %v", filename, rmErr)
}
}
}
return nil
}

View File

@ -15,11 +15,15 @@
package etcdserver
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"time"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
"go.etcd.io/etcd/mvcc"
@ -230,10 +234,12 @@ func (s *EtcdServer) checkHashKV() error {
mismatch(uint64(s.ID()))
}
checkedCount := 0
for _, p := range peers {
if p.resp == nil {
continue
}
checkedCount++
id := p.resp.Header.MemberId
// leader expects follower's latest revision less than or equal to leader's
@ -298,62 +304,56 @@ func (s *EtcdServer) checkHashKV() error {
mismatch(id)
}
}
if lg != nil {
lg.Info("finished peer corruption check", zap.Int("number-of-peers-checked", checkedCount))
} else {
plog.Infof("finished peer corruption check")
}
return nil
}
type peerHashKVResp struct {
type peerInfo struct {
id types.ID
eps []string
}
resp *clientv3.HashKVResponse
type peerHashKVResp struct {
peerInfo
resp *pb.HashKVResponse
err error
}
func (s *EtcdServer) getPeerHashKVs(rev int64) (resps []*peerHashKVResp) {
func (s *EtcdServer) getPeerHashKVs(rev int64) []*peerHashKVResp {
// TODO: handle the case when "s.cluster.Members" have not
// been populated (e.g. no snapshot to load from disk)
mbs := s.cluster.Members()
pss := make([]peerHashKVResp, len(mbs))
for _, m := range mbs {
members := s.cluster.Members()
peers := make([]peerInfo, 0, len(members))
for _, m := range members {
if m.ID == s.ID() {
continue
}
pss = append(pss, peerHashKVResp{id: m.ID, eps: m.PeerURLs})
peers = append(peers, peerInfo{id: m.ID, eps: m.PeerURLs})
}
lg := s.getLogger()
for _, p := range pss {
var resps []*peerHashKVResp
for _, p := range peers {
if len(p.eps) == 0 {
continue
}
cli, cerr := clientv3.New(clientv3.Config{
DialTimeout: s.Cfg.ReqTimeout(),
Endpoints: p.eps,
})
if cerr != nil {
if lg != nil {
lg.Warn(
"failed to create client to peer URL",
zap.String("local-member-id", s.ID().String()),
zap.String("remote-peer-id", p.id.String()),
zap.Strings("remote-peer-endpoints", p.eps),
zap.Error(cerr),
)
} else {
plog.Warningf("%s failed to create client to peer %q for hash checking (%q)", s.ID(), p.eps, cerr.Error())
}
continue
}
respsLen := len(resps)
for _, c := range cli.Endpoints() {
var lastErr error
for _, ep := range p.eps {
ctx, cancel := context.WithTimeout(context.Background(), s.Cfg.ReqTimeout())
var resp *clientv3.HashKVResponse
resp, cerr = cli.HashKV(ctx, c, rev)
var resp *pb.HashKVResponse
resp, lastErr = s.getPeerHashKVHTTP(ctx, ep, rev)
cancel()
if cerr == nil {
resps = append(resps, &peerHashKVResp{id: p.id, eps: p.eps, resp: resp, err: nil})
if lastErr == nil {
resps = append(resps, &peerHashKVResp{peerInfo: p, resp: resp, err: nil})
break
}
if lg != nil {
@ -361,17 +361,17 @@ func (s *EtcdServer) getPeerHashKVs(rev int64) (resps []*peerHashKVResp) {
"failed hash kv request",
zap.String("local-member-id", s.ID().String()),
zap.Int64("requested-revision", rev),
zap.String("remote-peer-endpoint", c),
zap.Error(cerr),
zap.String("remote-peer-endpoint", ep),
zap.Error(lastErr),
)
} else {
plog.Warningf("%s hash-kv error %q on peer %q with revision %d", s.ID(), cerr.Error(), c, rev)
plog.Warningf("%s hash-kv error %q on peer %q with revision %d", s.ID(), lastErr.Error(), ep, rev)
}
}
cli.Close()
// failed to get hashKV from all endpoints of this peer
if respsLen == len(resps) {
resps = append(resps, &peerHashKVResp{id: p.id, eps: p.eps, resp: nil, err: cerr})
resps = append(resps, &peerHashKVResp{peerInfo: p, resp: nil, err: lastErr})
}
}
return resps
@ -410,3 +410,112 @@ func (a *applierV3Corrupt) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantR
func (a *applierV3Corrupt) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) {
return nil, ErrCorrupt
}
type ServerPeerV2 interface {
ServerPeer
HashKVHandler() http.Handler
}
const PeerHashKVPath = "/members/hashkv"
type hashKVHandler struct {
lg *zap.Logger
server *EtcdServer
}
func (s *EtcdServer) HashKVHandler() http.Handler {
return &hashKVHandler{lg: s.getLogger(), server: s}
}
func (h *hashKVHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.Header().Set("Allow", http.MethodGet)
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
return
}
if r.URL.Path != PeerHashKVPath {
http.Error(w, "bad path", http.StatusBadRequest)
return
}
defer r.Body.Close()
b, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, "error reading body", http.StatusBadRequest)
return
}
req := &pb.HashKVRequest{}
if err = json.Unmarshal(b, req); err != nil {
h.lg.Warn("failed to unmarshal request", zap.Error(err))
http.Error(w, "error unmarshalling request", http.StatusBadRequest)
return
}
hash, rev, compactRev, err := h.server.KV().HashByRev(req.Revision)
if err != nil {
h.lg.Warn(
"failed to get hashKV",
zap.Int64("requested-revision", req.Revision),
zap.Error(err),
)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
resp := &pb.HashKVResponse{Header: &pb.ResponseHeader{Revision: rev}, Hash: hash, CompactRevision: compactRev}
respBytes, err := json.Marshal(resp)
if err != nil {
h.lg.Warn("failed to marshal hashKV response", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("X-Etcd-Cluster-ID", h.server.Cluster().ID().String())
w.Header().Set("Content-Type", "application/json")
w.Write(respBytes)
}
// getPeerHashKVHTTP fetch hash of kv store at the given rev via http call to the given url
func (s *EtcdServer) getPeerHashKVHTTP(ctx context.Context, url string, rev int64) (*pb.HashKVResponse, error) {
cc := &http.Client{Transport: s.peerRt}
hashReq := &pb.HashKVRequest{Revision: rev}
hashReqBytes, err := json.Marshal(hashReq)
if err != nil {
return nil, err
}
requestUrl := url + PeerHashKVPath
req, err := http.NewRequest(http.MethodGet, requestUrl, bytes.NewReader(hashReqBytes))
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
req.Header.Set("Content-Type", "application/json")
req.Cancel = ctx.Done()
resp, err := cc.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode == http.StatusBadRequest {
if strings.Contains(string(b), mvcc.ErrCompacted.Error()) {
return nil, rpctypes.ErrCompacted
}
if strings.Contains(string(b), mvcc.ErrFutureRev.Error()) {
return nil, rpctypes.ErrFutureRev
}
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unknown error: %s", string(b))
}
hashResp := &pb.HashKVResponse{}
if err := json.Unmarshal(b, hashResp); err != nil {
return nil, err
}
return hashResp, nil
}

View File

@ -807,12 +807,13 @@ func (s *EtcdServer) start() {
func (s *EtcdServer) purgeFile() {
var dberrc, serrc, werrc <-chan error
var dbdonec, sdonec, wdonec <-chan struct{}
if s.Cfg.MaxSnapFiles > 0 {
dberrc = fileutil.PurgeFile(s.getLogger(), s.Cfg.SnapDir(), "snap.db", s.Cfg.MaxSnapFiles, purgeFileInterval, s.done)
serrc = fileutil.PurgeFile(s.getLogger(), s.Cfg.SnapDir(), "snap", s.Cfg.MaxSnapFiles, purgeFileInterval, s.done)
dbdonec, dberrc = fileutil.PurgeFileWithDoneNotify(s.getLogger(), s.Cfg.SnapDir(), "snap.db", s.Cfg.MaxSnapFiles, purgeFileInterval, s.stopping)
sdonec, serrc = fileutil.PurgeFileWithDoneNotify(s.getLogger(), s.Cfg.SnapDir(), "snap", s.Cfg.MaxSnapFiles, purgeFileInterval, s.stopping)
}
if s.Cfg.MaxWALFiles > 0 {
werrc = fileutil.PurgeFile(s.getLogger(), s.Cfg.WALDir(), "wal", s.Cfg.MaxWALFiles, purgeFileInterval, s.done)
wdonec, werrc = fileutil.PurgeFileWithDoneNotify(s.getLogger(), s.Cfg.WALDir(), "wal", s.Cfg.MaxWALFiles, purgeFileInterval, s.stopping)
}
lg := s.getLogger()
@ -836,6 +837,15 @@ func (s *EtcdServer) purgeFile() {
plog.Fatalf("failed to purge wal file %v", e)
}
case <-s.stopping:
if dbdonec != nil {
<-dbdonec
}
if sdonec != nil {
<-sdonec
}
if wdonec != nil {
<-wdonec
}
return
}
}

View File

@ -577,12 +577,7 @@ func (s *EtcdServer) raftRequestOnce(ctx context.Context, r pb.InternalRaftReque
}
func (s *EtcdServer) raftRequest(ctx context.Context, r pb.InternalRaftRequest) (proto.Message, error) {
for {
resp, err := s.raftRequestOnce(ctx, r)
if err != auth.ErrAuthOldRevision {
return resp, err
}
}
return s.raftRequestOnce(ctx, r)
}
// doSerialize handles the auth logic, with permissions checked by "chk", for a serialized request "get". Returns a non-nil error on authentication failure.

View File

@ -369,13 +369,27 @@ func (b *backend) defrag() error {
b.batchTx.tx = nil
tmpdb, err := bolt.Open(b.db.Path()+".tmp", 0600, boltOpenOptions)
// Create a temporary file to ensure we start with a clean slate.
// Snapshotter.cleanupSnapdir cleans up any of these that are found during startup.
dir := filepath.Dir(b.db.Path())
temp, err := ioutil.TempFile(dir, "db.tmp.*")
if err != nil {
return err
}
options := bolt.Options{}
if boltOpenOptions != nil {
options = *boltOpenOptions
}
options.OpenFile = func(path string, i int, mode os.FileMode) (file *os.File, err error) {
return temp, nil
}
tdbp := temp.Name()
tmpdb, err := bolt.Open(tdbp, 0600, &options)
if err != nil {
return err
}
dbp := b.db.Path()
tdbp := tmpdb.Path()
size1, sizeInUse1 := b.Size(), b.SizeInUse()
if b.lg != nil {
b.lg.Info(
@ -387,11 +401,17 @@ func (b *backend) defrag() error {
zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse1))),
)
}
// gofail: var defragBeforeCopy struct{}
err = defragdb(b.db, tmpdb, defragLimit)
if err != nil {
tmpdb.Close()
os.RemoveAll(tmpdb.Path())
if rmErr := os.RemoveAll(tmpdb.Path()); rmErr != nil {
if b.lg != nil {
b.lg.Error("failed to remove db.tmp after defragmentation completed", zap.Error(rmErr))
} else {
plog.Fatalf("failed to remove db.tmp after defragmentation completed: %v", rmErr)
}
}
return err
}
@ -411,6 +431,7 @@ func (b *backend) defrag() error {
plog.Fatalf("cannot close database (%s)", err)
}
}
// gofail: var defragBeforeRename struct{}
err = os.Rename(tdbp, dbp)
if err != nil {
if b.lg != nil {

View File

@ -43,6 +43,7 @@ func (s *store) scheduleCompaction(compactMainRev int64, keep map[revision]struc
rev = bytesToRev(key)
if _, ok := keep[rev]; !ok {
tx.UnsafeDelete(keyBucketName, key)
keyCompactions++
}
}

View File

@ -296,6 +296,14 @@ var (
// overridden by mvcc initialization
reportCompactRevMu sync.RWMutex
reportCompactRev = func() float64 { return 0 }
totalPutSizeGauge = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "etcd_debugging",
Subsystem: "mvcc",
Name: "total_put_size_in_bytes",
Help: "The total size of put kv pairs seen by this member.",
})
)
func init() {
@ -325,6 +333,7 @@ func init() {
prometheus.MustRegister(hashRevSec)
prometheus.MustRegister(currentRev)
prometheus.MustRegister(compactRev)
prometheus.MustRegister(totalPutSizeGauge)
}
// ReportEventReceived reports that an event is received.

View File

@ -21,14 +21,15 @@ type metricsTxnWrite struct {
ranges uint
puts uint
deletes uint
putSize int64
}
func newMetricsTxnRead(tr TxnRead) TxnRead {
return &metricsTxnWrite{&txnReadWrite{tr}, 0, 0, 0}
return &metricsTxnWrite{&txnReadWrite{tr}, 0, 0, 0, 0}
}
func newMetricsTxnWrite(tw TxnWrite) TxnWrite {
return &metricsTxnWrite{tw, 0, 0, 0}
return &metricsTxnWrite{tw, 0, 0, 0, 0}
}
func (tw *metricsTxnWrite) Range(key, end []byte, ro RangeOptions) (*RangeResult, error) {
@ -43,6 +44,8 @@ func (tw *metricsTxnWrite) DeleteRange(key, end []byte) (n, rev int64) {
func (tw *metricsTxnWrite) Put(key, value []byte, lease lease.LeaseID) (rev int64) {
tw.puts++
size := int64(len(key) + len(value))
tw.putSize += size
return tw.TxnWrite.Put(key, value, lease)
}
@ -60,6 +63,7 @@ func (tw *metricsTxnWrite) End() {
puts := float64(tw.puts)
putCounter.Add(puts)
putCounterDebug.Add(puts) // TODO: remove in 3.5 release
totalPutSizeGauge.Add(float64(tw.putSize))
deletes := float64(tw.deletes)
deleteCounter.Add(deletes)

View File

@ -25,13 +25,23 @@ import (
)
func PurgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) <-chan error {
return purgeFile(lg, dirname, suffix, max, interval, stop, nil)
return purgeFile(lg, dirname, suffix, max, interval, stop, nil, nil)
}
func PurgeFileWithDoneNotify(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) (<-chan struct{}, <-chan error) {
doneC := make(chan struct{})
errC := purgeFile(lg, dirname, suffix, max, interval, stop, nil, doneC)
return doneC, errC
}
// purgeFile is the internal implementation for PurgeFile which can post purged files to purgec if non-nil.
func purgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string) <-chan error {
// if donec is non-nil, the function closes it to notify its exit.
func purgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string, donec chan<- struct{}) <-chan error {
errC := make(chan error, 1)
go func() {
if donec != nil {
defer close(donec)
}
for {
fnames, err := ReadDir(dirname)
if err != nil {

View File

@ -26,7 +26,7 @@ import (
var (
// MinClusterVersion is the min cluster version this etcd binary is compatible with.
MinClusterVersion = "3.0.0"
Version = "3.4.3"
Version = "3.4.4"
APIVersion = "unknown"
// Git SHA Value will be set during build

2
vendor/modules.txt vendored
View File

@ -733,7 +733,7 @@ github.com/vmware/govmomi/vim25/xml
github.com/xiang90/probing
# go.etcd.io/bbolt v1.3.3 => go.etcd.io/bbolt v1.3.3
go.etcd.io/bbolt
# go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 => go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
# go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200224211402-c65a9e2dd1fd
go.etcd.io/etcd/auth
go.etcd.io/etcd/auth/authpb
go.etcd.io/etcd/client