From 273d20791ee612540eae306d86aebcb0f12a0159 Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Tue, 24 Mar 2015 15:54:50 -0700 Subject: [PATCH 1/2] Upgrade go-etcd to v0.4.6 (Take 3) --- Godeps/Godeps.json | 14 +- .../etcd/etcdserver/etcdhttp/httptypes/doc.go | 19 +++ .../etcdserver/etcdhttp/httptypes/errors.go | 49 ++++++ .../etcdhttp/httptypes/errors_test.go | 47 ++++++ .../etcdserver/etcdhttp/httptypes/member.go | 67 ++++++++ .../etcdhttp/httptypes/member_test.go | 135 ++++++++++++++++ .../github.com/coreos/go-etcd/etcd/client.go | 59 ++++--- .../coreos/go-etcd/etcd/client_test.go | 6 +- .../github.com/coreos/go-etcd/etcd/cluster.go | 33 +--- .../github.com/coreos/go-etcd/etcd/error.go | 3 +- .../src/github.com/coreos/go-etcd/etcd/get.go | 5 + .../github.com/coreos/go-etcd/etcd/options.go | 10 +- .../coreos/go-etcd/etcd/requests.go | 150 ++++++++++-------- .../coreos/go-etcd/etcd/set_curl_chan_test.go | 6 +- .../coreos/go-etcd/etcd/set_update_create.go | 2 +- 15 files changed, 472 insertions(+), 133 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/doc.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 78cf0ed67e1..bf54f7b731e 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -62,10 +62,20 @@ "Comment": "v0.1-62-g8d75e11", "Rev": "8d75e11374a1928608c906fe745b538483e7aeb2" }, + { + "ImportPath": "github.com/coreos/etcd/etcdserver/etcdhttp/httptypes", + "Comment": "v2.0.4-288-g866a9d4", + "Rev": "866a9d4e41401657ea44bf539b2c5561d6fdcd67" + }, + { + "ImportPath": "github.com/coreos/etcd/pkg/types", + "Comment": "v2.0.4-288-g866a9d4", + "Rev": "866a9d4e41401657ea44bf539b2c5561d6fdcd67" + }, { "ImportPath": "github.com/coreos/go-etcd/etcd", - "Comment": "v0.2.0-rc1-120-g23142f6", - "Rev": "23142f6773a676cc2cae8dd0cb90b2ea761c853f" + "Comment": "v0.4.6-8-g60e12ca", + "Rev": "60e12cac3db8ffce00b576b4af0e7b0a968f1003" }, { "ImportPath": "github.com/coreos/go-systemd/dbus", diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/doc.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/doc.go new file mode 100644 index 00000000000..fa0158020e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/doc.go @@ -0,0 +1,19 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package httptypes defines how etcd's HTTP API entities are serialized to and deserialized from JSON. +*/ + +package httptypes diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors.go new file mode 100644 index 00000000000..7e0d275ebbb --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors.go @@ -0,0 +1,49 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httptypes + +import ( + "encoding/json" + "log" + "net/http" +) + +type HTTPError struct { + Message string `json:"message"` + // HTTP return code + Code int `json:"-"` +} + +func (e HTTPError) Error() string { + return e.Message +} + +// TODO(xiangli): handle http write errors +func (e HTTPError) WriteTo(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(e.Code) + b, err := json.Marshal(e) + if err != nil { + log.Panicf("marshal HTTPError should never fail: %v", err) + } + w.Write(b) +} + +func NewHTTPError(code int, m string) *HTTPError { + return &HTTPError{ + Message: m, + Code: code, + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors_test.go new file mode 100644 index 00000000000..f5cec6d4579 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/errors_test.go @@ -0,0 +1,47 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httptypes + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestHTTPErrorWriteTo(t *testing.T) { + err := NewHTTPError(http.StatusBadRequest, "what a bad request you made!") + rr := httptest.NewRecorder() + err.WriteTo(rr) + + wcode := http.StatusBadRequest + wheader := http.Header(map[string][]string{ + "Content-Type": []string{"application/json"}, + }) + wbody := `{"message":"what a bad request you made!"}` + + if wcode != rr.Code { + t.Errorf("HTTP status code %d, want %d", rr.Code, wcode) + } + + if !reflect.DeepEqual(wheader, rr.HeaderMap) { + t.Errorf("HTTP headers %v, want %v", rr.HeaderMap, wheader) + } + + gbody := rr.Body.String() + if wbody != gbody { + t.Errorf("HTTP body %q, want %q", gbody, wbody) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member.go new file mode 100644 index 00000000000..30ecbb53939 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member.go @@ -0,0 +1,67 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httptypes + +import ( + "encoding/json" + + "github.com/coreos/etcd/pkg/types" +) + +type Member struct { + ID string `json:"id"` + Name string `json:"name"` + PeerURLs []string `json:"peerURLs"` + ClientURLs []string `json:"clientURLs"` +} + +type MemberCreateRequest struct { + PeerURLs types.URLs +} + +type MemberUpdateRequest struct { + MemberCreateRequest +} + +func (m *MemberCreateRequest) UnmarshalJSON(data []byte) error { + s := struct { + PeerURLs []string `json:"peerURLs"` + }{} + + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + urls, err := types.NewURLs(s.PeerURLs) + if err != nil { + return err + } + + m.PeerURLs = urls + return nil +} + +type MemberCollection []Member + +func (c *MemberCollection) MarshalJSON() ([]byte, error) { + d := struct { + Members []Member `json:"members"` + }{ + Members: []Member(*c), + } + + return json.Marshal(d) +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member_test.go new file mode 100644 index 00000000000..e0b29d88311 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/etcdserver/etcdhttp/httptypes/member_test.go @@ -0,0 +1,135 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httptypes + +import ( + "encoding/json" + "net/url" + "reflect" + "testing" + + "github.com/coreos/etcd/pkg/types" +) + +func TestMemberUnmarshal(t *testing.T) { + tests := []struct { + body []byte + wantMember Member + wantError bool + }{ + // no URLs, just check ID & Name + { + body: []byte(`{"id": "c", "name": "dungarees"}`), + wantMember: Member{ID: "c", Name: "dungarees", PeerURLs: nil, ClientURLs: nil}, + }, + + // both client and peer URLs + { + body: []byte(`{"peerURLs": ["http://127.0.0.1:4001"], "clientURLs": ["http://127.0.0.1:4001"]}`), + wantMember: Member{ + PeerURLs: []string{ + "http://127.0.0.1:4001", + }, + ClientURLs: []string{ + "http://127.0.0.1:4001", + }, + }, + }, + + // multiple peer URLs + { + body: []byte(`{"peerURLs": ["http://127.0.0.1:4001", "https://example.com"]}`), + wantMember: Member{ + PeerURLs: []string{ + "http://127.0.0.1:4001", + "https://example.com", + }, + ClientURLs: nil, + }, + }, + + // multiple client URLs + { + body: []byte(`{"clientURLs": ["http://127.0.0.1:4001", "https://example.com"]}`), + wantMember: Member{ + PeerURLs: nil, + ClientURLs: []string{ + "http://127.0.0.1:4001", + "https://example.com", + }, + }, + }, + + // invalid JSON + { + body: []byte(`{"peerU`), + wantError: true, + }, + } + + for i, tt := range tests { + got := Member{} + err := json.Unmarshal(tt.body, &got) + if tt.wantError != (err != nil) { + t.Errorf("#%d: want error %t, got %v", i, tt.wantError, err) + continue + } + + if !reflect.DeepEqual(tt.wantMember, got) { + t.Errorf("#%d: incorrect output: want=%#v, got=%#v", i, tt.wantMember, got) + } + } +} + +func TestMemberCreateRequestUnmarshal(t *testing.T) { + body := []byte(`{"peerURLs": ["http://127.0.0.1:8081", "https://127.0.0.1:8080"]}`) + want := MemberCreateRequest{ + PeerURLs: types.URLs([]url.URL{ + url.URL{Scheme: "http", Host: "127.0.0.1:8081"}, + url.URL{Scheme: "https", Host: "127.0.0.1:8080"}, + }), + } + + var req MemberCreateRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Fatalf("Unmarshal returned unexpected err=%v", err) + } + + if !reflect.DeepEqual(want, req) { + t.Fatalf("Failed to unmarshal MemberCreateRequest: want=%#v, got=%#v", want, req) + } +} + +func TestMemberCreateRequestUnmarshalFail(t *testing.T) { + tests := [][]byte{ + // invalid JSON + []byte(``), + []byte(`{`), + + // spot-check validation done in types.NewURLs + []byte(`{"peerURLs": "foo"}`), + []byte(`{"peerURLs": ["."]}`), + []byte(`{"peerURLs": []}`), + []byte(`{"peerURLs": ["http://127.0.0.1:4001/foo"]}`), + []byte(`{"peerURLs": ["http://127.0.0.1"]}`), + } + + for i, tt := range tests { + var req MemberCreateRequest + if err := json.Unmarshal(tt, &req); err == nil { + t.Errorf("#%d: expected err, got nil", i) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go index f6ae5486173..8ecb50ee53f 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go @@ -7,12 +7,16 @@ import ( "errors" "io" "io/ioutil" + "math/rand" "net" "net/http" "net/url" "os" "path" + "strings" "time" + + "github.com/coreos/etcd/etcdserver/etcdhttp/httptypes" ) // See SetConsistency for how to use these constants. @@ -28,6 +32,10 @@ const ( defaultBufferSize = 10 ) +func init() { + rand.Seed(int64(time.Now().Nanosecond())) +} + type Config struct { CertFile string `json:"certFile"` KeyFile string `json:"keyFile"` @@ -64,8 +72,7 @@ func NewClient(machines []string) *Client { config := Config{ // default timeout is one second DialTimeout: time.Second, - // default consistency level is STRONG - Consistency: STRONG_CONSISTENCY, + Consistency: WEAK_CONSISTENCY, } client := &Client{ @@ -89,8 +96,7 @@ func NewTLSClient(machines []string, cert, key, caCert string) (*Client, error) config := Config{ // default timeout is one second DialTimeout: time.Second, - // default consistency level is STRONG - Consistency: STRONG_CONSISTENCY, + Consistency: WEAK_CONSISTENCY, CertFile: cert, KeyFile: key, CaCertFile: make([]string, 0), @@ -292,30 +298,37 @@ func (c *Client) SyncCluster() bool { // internalSyncCluster syncs cluster information using the given machine list. func (c *Client) internalSyncCluster(machines []string) bool { for _, machine := range machines { - httpPath := c.createHttpPath(machine, path.Join(version, "machines")) + httpPath := c.createHttpPath(machine, path.Join(version, "members")) resp, err := c.httpClient.Get(httpPath) if err != nil { // try another machine in the cluster continue - } else { - b, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - // try another machine in the cluster - continue - } - - // update Machines List - c.cluster.updateFromStr(string(b)) - - // update leader - // the first one in the machine list is the leader - c.cluster.switchLeader(0) - - logger.Debug("sync.machines ", c.cluster.Machines) - c.saveConfig() - return true } + + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + // try another machine in the cluster + continue + } + + var mCollection httptypes.MemberCollection + if err := json.Unmarshal(b, &mCollection); err != nil { + // try another machine + continue + } + + urls := make([]string, 0) + for _, m := range mCollection { + urls = append(urls, m.ClientURLs...) + } + + // update Machines List + c.cluster.updateFromStr(strings.Join(urls, ",")) + + logger.Debug("sync.machines ", c.cluster.Machines) + c.saveConfig() + return true } return false } diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go index c245e479844..66d79d73320 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go @@ -10,7 +10,7 @@ import ( ) // To pass this test, we need to create a cluster of 3 machines -// The server should be listening on 127.0.0.1:4001, 4002, 4003 +// The server should be listening on localhost:4001, 4002, 4003 func TestSync(t *testing.T) { fmt.Println("Make sure there are three nodes at 0.0.0.0:4001-4003") @@ -36,8 +36,8 @@ func TestSync(t *testing.T) { if err != nil { t.Fatal(err) } - if host != "127.0.0.1" { - t.Fatal("Host must be 127.0.0.1") + if host != "localhost" { + t.Fatal("Host must be localhost") } } diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go index aaa20546e32..787cf753ba2 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go @@ -1,13 +1,14 @@ package etcd import ( - "net/url" + "math/rand" "strings" ) type Cluster struct { Leader string `json:"leader"` Machines []string `json:"machines"` + picked int } func NewCluster(machines []string) *Cluster { @@ -18,34 +19,16 @@ func NewCluster(machines []string) *Cluster { // default leader and machines return &Cluster{ - Leader: machines[0], + Leader: "", Machines: machines, + picked: rand.Intn(len(machines)), } } -// switchLeader switch the current leader to machines[num] -func (cl *Cluster) switchLeader(num int) { - logger.Debugf("switch.leader[from %v to %v]", - cl.Leader, cl.Machines[num]) - - cl.Leader = cl.Machines[num] -} +func (cl *Cluster) failure() { cl.picked = rand.Intn(len(cl.Machines)) } +func (cl *Cluster) pick() string { return cl.Machines[cl.picked] } func (cl *Cluster) updateFromStr(machines string) { - cl.Machines = strings.Split(machines, ", ") -} - -func (cl *Cluster) updateLeader(leader string) { - logger.Debugf("update.leader[%s,%s]", cl.Leader, leader) - cl.Leader = leader -} - -func (cl *Cluster) updateLeaderFromURL(u *url.URL) { - var leader string - if u.Scheme == "" { - leader = "http://" + u.Host - } else { - leader = u.Scheme + "://" + u.Host - } - cl.updateLeader(leader) + cl.Machines = strings.Split(machines, ",") + cl.picked = rand.Intn(len(cl.Machines)) } diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go index 7e692872472..66dca54b5c4 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go @@ -6,7 +6,8 @@ import ( ) const ( - ErrCodeEtcdNotReachable = 501 + ErrCodeEtcdNotReachable = 501 + ErrCodeUnhandledHTTPStatus = 502 ) var ( diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go index 976bf07fd74..09fe641c253 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go @@ -18,9 +18,14 @@ func (c *Client) Get(key string, sort, recursive bool) (*Response, error) { } func (c *Client) RawGet(key string, sort, recursive bool) (*RawResponse, error) { + var q bool + if c.config.Consistency == STRONG_CONSISTENCY { + q = true + } ops := Options{ "recursive": recursive, "sorted": sort, + "quorum": q, } return c.get(key, ops) diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go index 701c9b35b97..d21c96f0805 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go @@ -17,11 +17,11 @@ type validOptions map[string]reflect.Kind // values are meant to be used as constants. var ( VALID_GET_OPTIONS = validOptions{ - "recursive": reflect.Bool, - "consistent": reflect.Bool, - "sorted": reflect.Bool, - "wait": reflect.Bool, - "waitIndex": reflect.Uint64, + "recursive": reflect.Bool, + "quorum": reflect.Bool, + "sorted": reflect.Bool, + "wait": reflect.Bool, + "waitIndex": reflect.Uint64, } VALID_PUT_OPTIONS = validOptions{ diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go index 5d8b45a2d39..70d9db2defb 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go @@ -3,8 +3,9 @@ package etcd import ( "errors" "fmt" + "io" "io/ioutil" - "math/rand" + "net" "net/http" "net/url" "path" @@ -38,15 +39,9 @@ func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan // getCancelable issues a cancelable GET request func (c *Client) getCancelable(key string, options Options, cancel <-chan bool) (*RawResponse, error) { - logger.Debugf("get %s [%s]", key, c.cluster.Leader) + logger.Debugf("get %s [%s]", key, c.cluster.pick()) p := keyToPath(key) - // If consistency level is set to STRONG, append - // the `consistent` query string. - if c.config.Consistency == STRONG_CONSISTENCY { - options["consistent"] = true - } - str, err := options.toParameters(VALID_GET_OPTIONS) if err != nil { return nil, err @@ -72,7 +67,7 @@ func (c *Client) get(key string, options Options) (*RawResponse, error) { func (c *Client) put(key string, value string, ttl uint64, options Options) (*RawResponse, error) { - logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader) + logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.pick()) p := keyToPath(key) str, err := options.toParameters(VALID_PUT_OPTIONS) @@ -93,7 +88,7 @@ func (c *Client) put(key string, value string, ttl uint64, // post issues a POST request func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) { - logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader) + logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.pick()) p := keyToPath(key) req := NewRawRequest("POST", p, buildValues(value, ttl), nil) @@ -108,7 +103,7 @@ func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error // delete issues a DELETE request func (c *Client) delete(key string, options Options) (*RawResponse, error) { - logger.Debugf("delete %s [%s]", key, c.cluster.Leader) + logger.Debugf("delete %s [%s]", key, c.cluster.pick()) p := keyToPath(key) str, err := options.toParameters(VALID_DELETE_OPTIONS) @@ -129,7 +124,6 @@ func (c *Client) delete(key string, options Options) (*RawResponse, error) { // SendRequest sends a HTTP request and returns a Response as defined by etcd func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { - var req *http.Request var resp *http.Response var httpPath string @@ -179,6 +173,7 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { // we connect to a leader sleep := 25 * time.Millisecond maxSleep := time.Second + for attempt := 0; ; attempt++ { if attempt > 0 { select { @@ -192,16 +187,9 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { } } - logger.Debug("Connecting to etcd: attempt", attempt+1, "for", rr.RelativePath) + logger.Debug("Connecting to etcd: attempt ", attempt+1, " for ", rr.RelativePath) - if rr.Method == "GET" && c.config.Consistency == WEAK_CONSISTENCY { - // If it's a GET and consistency level is set to WEAK, - // then use a random machine. - httpPath = c.getHttpPath(true, rr.RelativePath) - } else { - // Else use the leader. - httpPath = c.getHttpPath(false, rr.RelativePath) - } + httpPath = c.getHttpPath(rr.RelativePath) // Return a cURL command if curlChan is set if c.cURLch != nil { @@ -214,21 +202,29 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { logger.Debug("send.request.to ", httpPath, " | method ", rr.Method) - reqLock.Lock() - if rr.Values == nil { - if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil { - return nil, err - } - } else { - body := strings.NewReader(rr.Values.Encode()) - if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil { - return nil, err - } + req, err := func() (*http.Request, error) { + reqLock.Lock() + defer reqLock.Unlock() - req.Header.Set("Content-Type", - "application/x-www-form-urlencoded; param=value") + if rr.Values == nil { + if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil { + return nil, err + } + } else { + body := strings.NewReader(rr.Values.Encode()) + if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil { + return nil, err + } + + req.Header.Set("Content-Type", + "application/x-www-form-urlencoded; param=value") + } + return req, nil + }() + + if err != nil { + return nil, err } - reqLock.Unlock() resp, err = c.httpClient.Do(req) defer func() { @@ -248,24 +244,24 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { // network error, change a machine! if err != nil { - logger.Debug("network error:", err.Error()) + logger.Debug("network error: ", err.Error()) lastResp := http.Response{} if checkErr := checkRetry(c.cluster, numReqs, lastResp, err); checkErr != nil { return nil, checkErr } - c.cluster.switchLeader(attempt % len(c.cluster.Machines)) + c.cluster.failure() continue } // if there is no error, it should receive response - logger.Debug("recv.response.from", httpPath) + logger.Debug("recv.response.from ", httpPath) if validHttpStatusCode[resp.StatusCode] { // try to read byte code and break the loop respBody, err = ioutil.ReadAll(resp.Body) if err == nil { - logger.Debug("recv.success.", httpPath) + logger.Debug("recv.success ", httpPath) break } // ReadAll error may be caused due to cancel request @@ -274,22 +270,15 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { return nil, ErrRequestCancelled default: } - } - // if resp is TemporaryRedirect, set the new leader and retry - if resp.StatusCode == http.StatusTemporaryRedirect { - u, err := resp.Location() - - if err != nil { - logger.Warning(err) - } else { - // Update cluster leader based on redirect location - // because it should point to the leader address - c.cluster.updateLeaderFromURL(u) - logger.Debug("recv.response.relocate", u.String()) + if err == io.ErrUnexpectedEOF { + // underlying connection was closed prematurely, probably by timeout + // TODO: empty body or unexpectedEOF can cause http.Transport to get hosed; + // this allows the client to detect that and take evasive action. Need + // to revisit once code.google.com/p/go/issues/detail?id=8648 gets fixed. + respBody = []byte{} + break } - resp.Body.Close() - continue } if checkErr := checkRetry(c.cluster, numReqs, *resp, @@ -314,34 +303,53 @@ func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response, err error) error { - if numReqs >= 2*len(cluster.Machines) { - return newError(ErrCodeEtcdNotReachable, - "Tried to connect to each peer twice and failed", 0) + if isEmptyResponse(lastResp) { + if !isConnectionError(err) { + return err + } + } else if !shouldRetry(lastResp) { + body := []byte("nil") + if lastResp.Body != nil { + if b, err := ioutil.ReadAll(lastResp.Body); err == nil { + body = b + } + } + errStr := fmt.Sprintf("unhandled http status [%s] with body [%s]", http.StatusText(lastResp.StatusCode), body) + return newError(ErrCodeUnhandledHTTPStatus, errStr, 0) } - code := lastResp.StatusCode - if code == http.StatusInternalServerError { + if numReqs > 2*len(cluster.Machines) { + errStr := fmt.Sprintf("failed to propose on members %v twice [last error: %v]", cluster.Machines, err) + return newError(ErrCodeEtcdNotReachable, errStr, 0) + } + if shouldRetry(lastResp) { + // sleep some time and expect leader election finish time.Sleep(time.Millisecond * 200) - } - logger.Warning("bad response status code", code) + logger.Warning("bad response status code", lastResp.StatusCode) return nil } -func (c *Client) getHttpPath(random bool, s ...string) string { - var machine string - if random { - machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))] - } else { - machine = c.cluster.Leader - } +func isEmptyResponse(r http.Response) bool { return r.StatusCode == 0 } - fullPath := machine + "/" + version +func isConnectionError(err error) bool { + _, ok := err.(*net.OpError) + return ok +} + +// shouldRetry returns whether the reponse deserves retry. +func shouldRetry(r http.Response) bool { + // TODO: only retry when the cluster is in leader election + // We cannot do it exactly because etcd doesn't support it well. + return r.StatusCode == http.StatusInternalServerError +} + +func (c *Client) getHttpPath(s ...string) string { + fullPath := c.cluster.pick() + "/" + version for _, seg := range s { fullPath = fullPath + "/" + seg } - return fullPath } @@ -360,11 +368,13 @@ func buildValues(value string, ttl uint64) url.Values { return v } -// convert key string to http path exclude version +// convert key string to http path exclude version, including URL escaping // for example: key[foo] -> path[keys/foo] +// key[/%z] -> path[keys/%25z] // key[/] -> path[keys/] func keyToPath(key string) string { - p := path.Join("keys", key) + // URL-escape our key, except for slashes + p := strings.Replace(url.QueryEscape(path.Join("keys", key)), "%2F", "/", -1) // corner case: if key is "/" or "//" ect // path join will clear the tailing "/" diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go index 756e317815a..87c86b83082 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go @@ -19,7 +19,7 @@ func TestSetCurlChan(t *testing.T) { } expected := fmt.Sprintf("curl -X PUT %s/v2/keys/foo -d value=bar -d ttl=5", - c.cluster.Leader) + c.cluster.pick()) actual := c.RecvCURL() if expected != actual { t.Fatalf(`Command "%s" is not equal to expected value "%s"`, @@ -32,8 +32,8 @@ func TestSetCurlChan(t *testing.T) { t.Fatal(err) } - expected = fmt.Sprintf("curl -X GET %s/v2/keys/foo?consistent=true&recursive=false&sorted=false", - c.cluster.Leader) + expected = fmt.Sprintf("curl -X GET %s/v2/keys/foo?quorum=true&recursive=false&sorted=false", + c.cluster.pick()) actual = c.RecvCURL() if expected != actual { t.Fatalf(`Command "%s" is not equal to expected value "%s"`, diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go index cb0d5674775..e2840cf3567 100644 --- a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go @@ -13,7 +13,7 @@ func (c *Client) Set(key string, value string, ttl uint64) (*Response, error) { return raw.Unmarshal() } -// Set sets the given key to a directory. +// SetDir sets the given key to a directory. // It will create a new directory or replace the old key value pair by a directory. // It will not replace a existing directory. func (c *Client) SetDir(key string, ttl uint64) (*Response, error) { From 8238298fbbf27eca9741cbbe6f41c4c76e191232 Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Tue, 24 Mar 2015 16:20:18 -0700 Subject: [PATCH 2/2] Include coreos/etcd/pkg/types pakcage --- .../github.com/coreos/etcd/pkg/types/id.go | 41 ++++ .../coreos/etcd/pkg/types/id_test.go | 95 +++++++++ .../github.com/coreos/etcd/pkg/types/set.go | 178 +++++++++++++++++ .../coreos/etcd/pkg/types/set_test.go | 186 ++++++++++++++++++ .../github.com/coreos/etcd/pkg/types/slice.go | 22 +++ .../coreos/etcd/pkg/types/slice_test.go | 30 +++ .../github.com/coreos/etcd/pkg/types/urls.go | 74 +++++++ .../coreos/etcd/pkg/types/urls_test.go | 169 ++++++++++++++++ 8 files changed, 795 insertions(+) create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice_test.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls.go create mode 100644 Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls_test.go diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id.go new file mode 100644 index 00000000000..88cb9e63494 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id.go @@ -0,0 +1,41 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "strconv" +) + +// ID represents a generic identifier which is canonically +// stored as a uint64 but is typically represented as a +// base-16 string for input/output +type ID uint64 + +func (i ID) String() string { + return strconv.FormatUint(uint64(i), 16) +} + +// IDFromString attempts to create an ID from a base-16 string. +func IDFromString(s string) (ID, error) { + i, err := strconv.ParseUint(s, 16, 64) + return ID(i), err +} + +// IDSlice implements the sort interface +type IDSlice []ID + +func (p IDSlice) Len() int { return len(p) } +func (p IDSlice) Less(i, j int) bool { return uint64(p[i]) < uint64(p[j]) } +func (p IDSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id_test.go new file mode 100644 index 00000000000..97d168f58e2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/id_test.go @@ -0,0 +1,95 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "sort" + "testing" +) + +func TestIDString(t *testing.T) { + tests := []struct { + input ID + want string + }{ + { + input: 12, + want: "c", + }, + { + input: 4918257920282737594, + want: "444129853c343bba", + }, + } + + for i, tt := range tests { + got := tt.input.String() + if tt.want != got { + t.Errorf("#%d: ID.String failure: want=%v, got=%v", i, tt.want, got) + } + } +} + +func TestIDFromString(t *testing.T) { + tests := []struct { + input string + want ID + }{ + { + input: "17", + want: 23, + }, + { + input: "612840dae127353", + want: 437557308098245459, + }, + } + + for i, tt := range tests { + got, err := IDFromString(tt.input) + if err != nil { + t.Errorf("#%d: IDFromString failure: err=%v", i, err) + continue + } + if tt.want != got { + t.Errorf("#%d: IDFromString failure: want=%v, got=%v", i, tt.want, got) + } + } +} + +func TestIDFromStringFail(t *testing.T) { + tests := []string{ + "", + "XXX", + "612840dae127353612840dae127353", + } + + for i, tt := range tests { + _, err := IDFromString(tt) + if err == nil { + t.Fatalf("#%d: IDFromString expected error, but err=nil", i) + } + } +} + +func TestIDSlice(t *testing.T) { + g := []ID{10, 500, 5, 1, 100, 25} + w := []ID{1, 5, 10, 25, 100, 500} + sort.Sort(IDSlice(g)) + if !reflect.DeepEqual(g, w) { + t.Errorf("slice after sort = %#v, want %#v", g, w) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set.go new file mode 100644 index 00000000000..32287522b11 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set.go @@ -0,0 +1,178 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "sort" + "sync" +) + +type Set interface { + Add(string) + Remove(string) + Contains(string) bool + Equals(Set) bool + Length() int + Values() []string + Copy() Set + Sub(Set) Set +} + +func NewUnsafeSet(values ...string) *unsafeSet { + set := &unsafeSet{make(map[string]struct{})} + for _, v := range values { + set.Add(v) + } + return set +} + +func NewThreadsafeSet(values ...string) *tsafeSet { + us := NewUnsafeSet(values...) + return &tsafeSet{us, sync.RWMutex{}} +} + +type unsafeSet struct { + d map[string]struct{} +} + +// Add adds a new value to the set (no-op if the value is already present) +func (us *unsafeSet) Add(value string) { + us.d[value] = struct{}{} +} + +// Remove removes the given value from the set +func (us *unsafeSet) Remove(value string) { + delete(us.d, value) +} + +// Contains returns whether the set contains the given value +func (us *unsafeSet) Contains(value string) (exists bool) { + _, exists = us.d[value] + return +} + +// ContainsAll returns whether the set contains all given values +func (us *unsafeSet) ContainsAll(values []string) bool { + for _, s := range values { + if !us.Contains(s) { + return false + } + } + return true +} + +// Equals returns whether the contents of two sets are identical +func (us *unsafeSet) Equals(other Set) bool { + v1 := sort.StringSlice(us.Values()) + v2 := sort.StringSlice(other.Values()) + v1.Sort() + v2.Sort() + return reflect.DeepEqual(v1, v2) +} + +// Length returns the number of elements in the set +func (us *unsafeSet) Length() int { + return len(us.d) +} + +// Values returns the values of the Set in an unspecified order. +func (us *unsafeSet) Values() (values []string) { + values = make([]string, 0) + for val, _ := range us.d { + values = append(values, val) + } + return +} + +// Copy creates a new Set containing the values of the first +func (us *unsafeSet) Copy() Set { + cp := NewUnsafeSet() + for val, _ := range us.d { + cp.Add(val) + } + + return cp +} + +// Sub removes all elements in other from the set +func (us *unsafeSet) Sub(other Set) Set { + oValues := other.Values() + result := us.Copy().(*unsafeSet) + + for _, val := range oValues { + if _, ok := result.d[val]; !ok { + continue + } + delete(result.d, val) + } + + return result +} + +type tsafeSet struct { + us *unsafeSet + m sync.RWMutex +} + +func (ts *tsafeSet) Add(value string) { + ts.m.Lock() + defer ts.m.Unlock() + ts.us.Add(value) +} + +func (ts *tsafeSet) Remove(value string) { + ts.m.Lock() + defer ts.m.Unlock() + ts.us.Remove(value) +} + +func (ts *tsafeSet) Contains(value string) (exists bool) { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Contains(value) +} + +func (ts *tsafeSet) Equals(other Set) bool { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Equals(other) +} + +func (ts *tsafeSet) Length() int { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Length() +} + +func (ts *tsafeSet) Values() (values []string) { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Values() +} + +func (ts *tsafeSet) Copy() Set { + ts.m.RLock() + defer ts.m.RUnlock() + usResult := ts.us.Copy().(*unsafeSet) + return &tsafeSet{usResult, sync.RWMutex{}} +} + +func (ts *tsafeSet) Sub(other Set) Set { + ts.m.RLock() + defer ts.m.RUnlock() + usResult := ts.us.Sub(other).(*unsafeSet) + return &tsafeSet{usResult, sync.RWMutex{}} +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set_test.go new file mode 100644 index 00000000000..ff1ecc68d3c --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/set_test.go @@ -0,0 +1,186 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "sort" + "testing" +) + +func TestUnsafeSet(t *testing.T) { + driveSetTests(t, NewUnsafeSet()) +} + +func TestThreadsafeSet(t *testing.T) { + driveSetTests(t, NewThreadsafeSet()) +} + +// Check that two slices contents are equal; order is irrelevant +func equal(a, b []string) bool { + as := sort.StringSlice(a) + bs := sort.StringSlice(b) + as.Sort() + bs.Sort() + return reflect.DeepEqual(as, bs) +} + +func driveSetTests(t *testing.T, s Set) { + // Verify operations on an empty set + eValues := []string{} + values := s.Values() + if !reflect.DeepEqual(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + if l := s.Length(); l != 0 { + t.Fatalf("Expected length=0, got %d", l) + } + for _, v := range []string{"foo", "bar", "baz"} { + if s.Contains(v) { + t.Fatalf("Expect s.Contains(%q) to be fale, got true", v) + } + } + + // Add three items, ensure they show up + s.Add("foo") + s.Add("bar") + s.Add("baz") + + eValues = []string{"foo", "bar", "baz"} + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + + for _, v := range eValues { + if !s.Contains(v) { + t.Fatalf("Expect s.Contains(%q) to be true, got false", v) + } + } + + if l := s.Length(); l != 3 { + t.Fatalf("Expected length=3, got %d", l) + } + + // Add the same item a second time, ensuring it is not duplicated + s.Add("foo") + + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + if l := s.Length(); l != 3 { + t.Fatalf("Expected length=3, got %d", l) + } + + // Remove all items, ensure they are gone + s.Remove("foo") + s.Remove("bar") + s.Remove("baz") + + eValues = []string{} + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + + if l := s.Length(); l != 0 { + t.Fatalf("Expected length=0, got %d", l) + } + + // Create new copies of the set, and ensure they are unlinked to the + // original Set by making modifications + s.Add("foo") + s.Add("bar") + cp1 := s.Copy() + cp2 := s.Copy() + s.Remove("foo") + cp3 := s.Copy() + cp1.Add("baz") + + for i, tt := range []struct { + want []string + got []string + }{ + {[]string{"bar"}, s.Values()}, + {[]string{"foo", "bar", "baz"}, cp1.Values()}, + {[]string{"foo", "bar"}, cp2.Values()}, + {[]string{"bar"}, cp3.Values()}, + } { + if !equal(tt.want, tt.got) { + t.Fatalf("case %d: expect values=%v got %v", i, tt.want, tt.got) + } + } + + for i, tt := range []struct { + want bool + got bool + }{ + {true, s.Equals(cp3)}, + {true, cp3.Equals(s)}, + {false, s.Equals(cp2)}, + {false, s.Equals(cp1)}, + {false, cp1.Equals(s)}, + {false, cp2.Equals(s)}, + {false, cp2.Equals(cp1)}, + } { + if tt.got != tt.want { + t.Fatalf("case %d: want %t, got %t", i, tt.want, tt.got) + + } + } + + // Subtract values from a Set, ensuring a new Set is created and + // the original Sets are unmodified + sub1 := cp1.Sub(s) + sub2 := cp2.Sub(cp1) + + for i, tt := range []struct { + want []string + got []string + }{ + {[]string{"foo", "bar", "baz"}, cp1.Values()}, + {[]string{"foo", "bar"}, cp2.Values()}, + {[]string{"bar"}, s.Values()}, + {[]string{"foo", "baz"}, sub1.Values()}, + {[]string{}, sub2.Values()}, + } { + if !equal(tt.want, tt.got) { + t.Fatalf("case %d: expect values=%v got %v", i, tt.want, tt.got) + } + } +} + +func TestUnsafeSetContainsAll(t *testing.T) { + vals := []string{"foo", "bar", "baz"} + s := NewUnsafeSet(vals...) + + tests := []struct { + strs []string + wcontain bool + }{ + {[]string{}, true}, + {vals[:1], true}, + {vals[:2], true}, + {vals, true}, + {[]string{"cuz"}, false}, + {[]string{vals[0], "cuz"}, false}, + } + for i, tt := range tests { + if g := s.ContainsAll(tt.strs); g != tt.wcontain { + t.Errorf("#%d: ok = %v, want %v", i, g, tt.wcontain) + } + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice.go new file mode 100644 index 00000000000..0327950f706 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice.go @@ -0,0 +1,22 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +// Uint64Slice implements sort interface +type Uint64Slice []uint64 + +func (p Uint64Slice) Len() int { return len(p) } +func (p Uint64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice_test.go new file mode 100644 index 00000000000..95e37e04d20 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/slice_test.go @@ -0,0 +1,30 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "sort" + "testing" +) + +func TestUint64Slice(t *testing.T) { + g := Uint64Slice{10, 500, 5, 1, 100, 25} + w := Uint64Slice{1, 5, 10, 25, 100, 500} + sort.Sort(g) + if !reflect.DeepEqual(g, w) { + t.Errorf("slice after sort = %#v, want %#v", g, w) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls.go new file mode 100644 index 00000000000..ce2483ffaaa --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls.go @@ -0,0 +1,74 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "errors" + "fmt" + "net" + "net/url" + "sort" + "strings" +) + +type URLs []url.URL + +func NewURLs(strs []string) (URLs, error) { + all := make([]url.URL, len(strs)) + if len(all) == 0 { + return nil, errors.New("no valid URLs given") + } + for i, in := range strs { + in = strings.TrimSpace(in) + u, err := url.Parse(in) + if err != nil { + return nil, err + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("URL scheme must be http or https: %s", in) + } + if _, _, err := net.SplitHostPort(u.Host); err != nil { + return nil, fmt.Errorf(`URL address does not have the form "host:port": %s`, in) + } + if u.Path != "" { + return nil, fmt.Errorf("URL must not contain a path: %s", in) + } + all[i] = *u + } + us := URLs(all) + us.Sort() + + return us, nil +} + +func (us URLs) String() string { + return strings.Join(us.StringSlice(), ",") +} + +func (us *URLs) Sort() { + sort.Sort(us) +} +func (us URLs) Len() int { return len(us) } +func (us URLs) Less(i, j int) bool { return us[i].String() < us[j].String() } +func (us URLs) Swap(i, j int) { us[i], us[j] = us[j], us[i] } + +func (us URLs) StringSlice() []string { + out := make([]string, len(us)) + for i := range us { + out[i] = us[i].String() + } + + return out +} diff --git a/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls_test.go b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls_test.go new file mode 100644 index 00000000000..41caa5d68a5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/etcd/pkg/types/urls_test.go @@ -0,0 +1,169 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "reflect" + "testing" + + "github.com/coreos/etcd/pkg/testutil" +) + +func TestNewURLs(t *testing.T) { + tests := []struct { + strs []string + wurls URLs + }{ + { + []string{"http://127.0.0.1:4001"}, + testutil.MustNewURLs(t, []string{"http://127.0.0.1:4001"}), + }, + // it can trim space + { + []string{" http://127.0.0.1:4001 "}, + testutil.MustNewURLs(t, []string{"http://127.0.0.1:4001"}), + }, + // it does sort + { + []string{ + "http://127.0.0.2:4001", + "http://127.0.0.1:4001", + }, + testutil.MustNewURLs(t, []string{ + "http://127.0.0.1:4001", + "http://127.0.0.2:4001", + }), + }, + } + for i, tt := range tests { + urls, _ := NewURLs(tt.strs) + if !reflect.DeepEqual(urls, tt.wurls) { + t.Errorf("#%d: urls = %+v, want %+v", i, urls, tt.wurls) + } + } +} + +func TestURLsString(t *testing.T) { + tests := []struct { + us URLs + wstr string + }{ + { + URLs{}, + "", + }, + { + testutil.MustNewURLs(t, []string{"http://127.0.0.1:4001"}), + "http://127.0.0.1:4001", + }, + { + testutil.MustNewURLs(t, []string{ + "http://127.0.0.1:4001", + "http://127.0.0.2:4001", + }), + "http://127.0.0.1:4001,http://127.0.0.2:4001", + }, + { + testutil.MustNewURLs(t, []string{ + "http://127.0.0.2:4001", + "http://127.0.0.1:4001", + }), + "http://127.0.0.2:4001,http://127.0.0.1:4001", + }, + } + for i, tt := range tests { + g := tt.us.String() + if g != tt.wstr { + t.Errorf("#%d: string = %s, want %s", i, g, tt.wstr) + } + } +} + +func TestURLsSort(t *testing.T) { + g := testutil.MustNewURLs(t, []string{ + "http://127.0.0.4:4001", + "http://127.0.0.2:4001", + "http://127.0.0.1:4001", + "http://127.0.0.3:4001", + }) + w := testutil.MustNewURLs(t, []string{ + "http://127.0.0.1:4001", + "http://127.0.0.2:4001", + "http://127.0.0.3:4001", + "http://127.0.0.4:4001", + }) + gurls := URLs(g) + gurls.Sort() + if !reflect.DeepEqual(g, w) { + t.Errorf("URLs after sort = %#v, want %#v", g, w) + } +} + +func TestURLsStringSlice(t *testing.T) { + tests := []struct { + us URLs + wstr []string + }{ + { + URLs{}, + []string{}, + }, + { + testutil.MustNewURLs(t, []string{"http://127.0.0.1:4001"}), + []string{"http://127.0.0.1:4001"}, + }, + { + testutil.MustNewURLs(t, []string{ + "http://127.0.0.1:4001", + "http://127.0.0.2:4001", + }), + []string{"http://127.0.0.1:4001", "http://127.0.0.2:4001"}, + }, + { + testutil.MustNewURLs(t, []string{ + "http://127.0.0.2:4001", + "http://127.0.0.1:4001", + }), + []string{"http://127.0.0.2:4001", "http://127.0.0.1:4001"}, + }, + } + for i, tt := range tests { + g := tt.us.StringSlice() + if !reflect.DeepEqual(g, tt.wstr) { + t.Errorf("#%d: string slice = %+v, want %+v", i, g, tt.wstr) + } + } +} + +func TestNewURLsFail(t *testing.T) { + tests := [][]string{ + // no urls given + {}, + // missing protocol scheme + {"://127.0.0.1:4001"}, + // unsupported scheme + {"mailto://127.0.0.1:4001"}, + // not conform to host:port + {"http://127.0.0.1"}, + // contain a path + {"http://127.0.0.1:4001/path"}, + } + for i, tt := range tests { + _, err := NewURLs(tt) + if err == nil { + t.Errorf("#%d: err = nil, but error", i) + } + } +}