From f390dc7918e6d67f32b4509199f6d902b661877e Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Thu, 16 Apr 2015 02:29:41 +0000 Subject: [PATCH] Update the prometheus library and its dependencies to the most recent versions. The process metrics on the master have been missing since sometime between 0.13 and 0.14, and updating the libraries fixes them. My hypothesis is that our switch away from using cgo to build statically linked binaries caused the breakage by causing prometheus's process metric collector, which had cgo-specific build logic, to be disabled. Prometheus has since changed that logic. --- Godeps/Godeps.json | 18 +- .../{ext => pbutil}/all_test.go | 55 +---- .../{ext => pbutil}/decode.go | 2 +- .../{ext => pbutil}/doc.go | 4 +- .../{ext => pbutil}/encode.go | 2 +- .../{ext => pbutil}/fixtures_test.go | 2 +- .../prometheus/client_golang/model/metric.go | 36 +-- .../client_golang/model/metric_test.go | 63 ++++- .../client_golang/model/signature.go | 89 +++++-- .../client_golang/model/signature_test.go | 218 ++++++++++++++---- .../client_golang/model/timestamp.go | 1 + .../client_golang/prometheus/examples_test.go | 18 ++ .../prometheus/process_collector.go | 42 +++- .../prometheus/process_collector_procfs.go | 63 ----- .../prometheus/process_collector_rest.go | 24 -- .../client_golang/prometheus/push.go | 65 ++++++ .../client_golang/prometheus/registry.go | 139 ++++++----- .../client_golang/prometheus/registry_test.go | 130 +++++++---- .../client_golang/prometheus/summary.go | 15 +- .../client_golang/prometheus/summary_test.go | 11 + .../client_golang/text/bench_test.go | 8 +- .../client_golang/text/parse_test.go | 2 +- .../prometheus/client_golang/text/proto.go | 4 +- .../github.com/prometheus/procfs/.travis.yml | 5 + .../github.com/prometheus/procfs/README.md | 6 +- .../github.com/prometheus/procfs/proc_stat.go | 26 ++- 26 files changed, 674 insertions(+), 374 deletions(-) rename Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/{ext => pbutil}/all_test.go (87%) rename Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/{ext => pbutil}/decode.go (99%) rename Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/{ext => pbutil}/doc.go (86%) rename Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/{ext => pbutil}/encode.go (99%) rename Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/{ext => pbutil}/fixtures_test.go (99%) delete mode 100644 Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_procfs.go delete mode 100644 Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_rest.go create mode 100644 Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/push.go create mode 100644 Godeps/_workspace/src/github.com/prometheus/procfs/.travis.yml diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ee6ed2d9010..94d2a678789 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -317,8 +317,8 @@ "Rev": "05017fcccf23c823bfdea560dcc958a136e54fb7" }, { - "ImportPath": "github.com/matttproud/golang_protobuf_extensions/ext", - "Rev": "ba7d65ac66e9da93a714ca18f6d1bc7a0c09100c" + "ImportPath": "github.com/matttproud/golang_protobuf_extensions/pbutil", + "Rev": "fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a" }, { "ImportPath": "github.com/miekg/dns", @@ -348,18 +348,18 @@ }, { "ImportPath": "github.com/prometheus/client_golang/model", - "Comment": "0.2.0-5-gde5f7a2", - "Rev": "de5f7a2db9d883392ce3ad667087280fe1ff9cea" + "Comment": "0.4.0-1-g692492e", + "Rev": "692492e54b553a81013254cc1fba4b6dd76fad30" }, { "ImportPath": "github.com/prometheus/client_golang/prometheus", - "Comment": "0.2.0-5-gde5f7a2", - "Rev": "de5f7a2db9d883392ce3ad667087280fe1ff9cea" + "Comment": "0.4.0-1-g692492e", + "Rev": "692492e54b553a81013254cc1fba4b6dd76fad30" }, { "ImportPath": "github.com/prometheus/client_golang/text", - "Comment": "0.2.0-5-gde5f7a2", - "Rev": "de5f7a2db9d883392ce3ad667087280fe1ff9cea" + "Comment": "0.4.0-1-g692492e", + "Rev": "692492e54b553a81013254cc1fba4b6dd76fad30" }, { "ImportPath": "github.com/prometheus/client_model/go", @@ -368,7 +368,7 @@ }, { "ImportPath": "github.com/prometheus/procfs", - "Rev": "6c34ef819e19b4e16f410100ace4aa006f0e3bf8" + "Rev": "490cc6eb5fa45bf8a8b7b73c8bc82a8160e8531d" }, { "ImportPath": "github.com/rackspace/gophercloud", diff --git a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/all_test.go b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/all_test.go similarity index 87% rename from Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/all_test.go rename to Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/all_test.go index 7270b67a38f..094156e66c5 100644 --- a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/all_test.go +++ b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/all_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ext +package pbutil import ( "bytes" @@ -21,6 +21,8 @@ import ( "testing" "testing/quick" + "github.com/matttproud/golang_protobuf_extensions/pbtest" + . "github.com/golang/protobuf/proto" . "github.com/golang/protobuf/proto/testdata" ) @@ -138,10 +140,10 @@ I expect it may. Let's hope you enjoy testing as much as we do.`), func TestEndToEndValid(t *testing.T) { for _, test := range [][]Message{ - []Message{&Empty{}}, - []Message{&GoEnum{Foo: FOO_FOO1.Enum()}, &Empty{}, &GoEnum{Foo: FOO_FOO1.Enum()}}, - []Message{&GoEnum{Foo: FOO_FOO1.Enum()}}, - []Message{&Strings{ + {&Empty{}}, + {&GoEnum{Foo: FOO_FOO1.Enum()}, &Empty{}, &GoEnum{Foo: FOO_FOO1.Enum()}}, + {&GoEnum{Foo: FOO_FOO1.Enum()}}, + {&Strings{ StringField: String(`This is my gigantic, unhappy string. It exceeds the encoding size of a single byte varint. We are using it to fuzz test the correctness of the header decoding mechanisms, which may prove problematic. @@ -176,45 +178,6 @@ I expect it may. Let's hope you enjoy testing as much as we do.`), } } -// visitMessage empties the private state fields of the quick.Value()-generated -// Protocol Buffer messages, for they cause an inordinate amount of problems. -// This is because we are using an automated fuzz generator on a type with -// private fields. -func visitMessage(m Message) { - t := reflect.TypeOf(m) - if t.Kind() != reflect.Ptr { - return - } - derefed := t.Elem() - if derefed.Kind() != reflect.Struct { - return - } - v := reflect.ValueOf(m) - elem := v.Elem() - for i := 0; i < elem.NumField(); i++ { - field := elem.FieldByIndex([]int{i}) - fieldType := field.Type() - if fieldType.Implements(reflect.TypeOf((*Message)(nil)).Elem()) { - visitMessage(field.Interface().(Message)) - } - if field.Kind() == reflect.Slice { - for i := 0; i < field.Len(); i++ { - elem := field.Index(i) - elemType := elem.Type() - if elemType.Implements(reflect.TypeOf((*Message)(nil)).Elem()) { - visitMessage(elem.Interface().(Message)) - } - } - } - } - if field := elem.FieldByName("XXX_unrecognized"); field.IsValid() { - field.Set(reflect.ValueOf([]byte{})) - } - if field := elem.FieldByName("XXX_extensions"); field.IsValid() { - field.Set(reflect.ValueOf(nil)) - } -} - // rndMessage generates a random valid Protocol Buffer message. func rndMessage(r *rand.Rand) Message { var t reflect.Type @@ -307,7 +270,9 @@ func rndMessage(r *rand.Rand) Message { if !ok { panic("attempt to generate illegal item; consult item 11") } - visitMessage(v.Interface().(Message)) + if err := pbtest.SanitizeGenerated(v.Interface().(Message)); err != nil { + panic(err) + } return v.Interface().(Message) } diff --git a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/decode.go b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/decode.go similarity index 99% rename from Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/decode.go rename to Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/decode.go index 28b520e4bbb..66d9b5458f8 100644 --- a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/decode.go +++ b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/decode.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ext +package pbutil import ( "encoding/binary" diff --git a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/doc.go b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/doc.go similarity index 86% rename from Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/doc.go rename to Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/doc.go index 652907bd2e0..c318385cbed 100644 --- a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/doc.go +++ b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/doc.go @@ -12,5 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ext enables record length-delimited Protocol Buffer streaming. -package ext +// Package pbutil provides record length-delimited Protocol Buffer streaming. +package pbutil diff --git a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/encode.go b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/encode.go similarity index 99% rename from Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/encode.go rename to Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/encode.go index 473b31dccaf..4b76ea9a1d8 100644 --- a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/encode.go +++ b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/encode.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ext +package pbutil import ( "encoding/binary" diff --git a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/fixtures_test.go b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/fixtures_test.go similarity index 99% rename from Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/fixtures_test.go rename to Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/fixtures_test.go index 07e75c54da5..d6d9b25594f 100644 --- a/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/ext/fixtures_test.go +++ b/Godeps/_workspace/src/github.com/matttproud/golang_protobuf_extensions/pbutil/fixtures_test.go @@ -27,7 +27,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -package ext +package pbutil import ( . "github.com/golang/protobuf/proto" diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go index 74f394b92e0..32f9d7fbca7 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go @@ -14,10 +14,8 @@ package model import ( - "encoding/binary" "encoding/json" "fmt" - "hash/fnv" "sort" "strings" ) @@ -66,37 +64,7 @@ func (m Metric) String() string { // Fingerprint returns a Metric's Fingerprint. func (m Metric) Fingerprint() Fingerprint { - labelNames := make([]string, 0, len(m)) - maxLength := 0 - - for labelName, labelValue := range m { - labelNames = append(labelNames, string(labelName)) - if len(labelName) > maxLength { - maxLength = len(labelName) - } - if len(labelValue) > maxLength { - maxLength = len(labelValue) - } - } - - sort.Strings(labelNames) - - summer := fnv.New64a() - buf := make([]byte, maxLength) - - for _, labelName := range labelNames { - labelValue := m[LabelName(labelName)] - - copy(buf, labelName) - summer.Write(buf[:len(labelName)]) - - summer.Write(separator) - - copy(buf, labelValue) - summer.Write(buf[:len(labelValue)]) - } - - return Fingerprint(binary.LittleEndian.Uint64(summer.Sum(nil))) + return metricToFingerprint(m) } // Clone returns a copy of the Metric. @@ -133,7 +101,7 @@ type COWMetric struct { // Set sets a label name in the wrapped Metric to a given value and copies the // Metric initially, if it is not already a copy. -func (m COWMetric) Set(ln LabelName, lv LabelValue) { +func (m *COWMetric) Set(ln LabelName, lv LabelValue) { m.doCOW() m.Metric[ln] = lv } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go index 7c31bdfb45e..d51b1842f6a 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go @@ -22,7 +22,7 @@ func testMetric(t testing.TB) { }{ { input: Metric{}, - fingerprint: 2676020557754725067, + fingerprint: 14695981039346656037, }, { input: Metric{ @@ -30,31 +30,27 @@ func testMetric(t testing.TB) { "occupation": "robot", "manufacturer": "westinghouse", }, - fingerprint: 13260944541294022935, + fingerprint: 11310079640881077873, }, { input: Metric{ "x": "y", }, - fingerprint: 1470933794305433534, + fingerprint: 13948396922932177635, }, - // The following two demonstrate a bug in fingerprinting. They - // should not have the same fingerprint with a sane - // fingerprinting function. See - // https://github.com/prometheus/client_golang/issues/74 . { input: Metric{ "a": "bb", "b": "c", }, - fingerprint: 3734646176939799877, + fingerprint: 3198632812309449502, }, { input: Metric{ "a": "b", "bb": "c", }, - fingerprint: 3734646176939799877, + fingerprint: 5774953389407657638, }, } @@ -74,3 +70,52 @@ func BenchmarkMetric(b *testing.B) { testMetric(b) } } + +func TestCOWMetric(t *testing.T) { + testMetric := Metric{ + "to_delete": "test1", + "to_change": "test2", + } + + scenarios := []struct { + fn func(*COWMetric) + out Metric + }{ + { + fn: func(cm *COWMetric) { + cm.Delete("to_delete") + }, + out: Metric{ + "to_change": "test2", + }, + }, + { + fn: func(cm *COWMetric) { + cm.Set("to_change", "changed") + }, + out: Metric{ + "to_delete": "test1", + "to_change": "changed", + }, + }, + } + + for i, s := range scenarios { + orig := testMetric.Clone() + cm := &COWMetric{ + Metric: orig, + } + + s.fn(cm) + + // Test that the original metric was not modified. + if !orig.Equal(testMetric) { + t.Fatalf("%d. original metric changed; expected %v, got %v", i, testMetric, orig) + } + + // Test that the new metric has the right changes. + if !cm.Metric.Equal(s.out) { + t.Fatalf("%d. copied metric doesn't contain expected changes; expected %v, got %v", i, s.out, cm.Metric) + } + } +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature.go index 4b392fb187f..cc77b192dd1 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature.go @@ -17,6 +17,7 @@ import ( "bytes" "hash" "hash/fnv" + "sync" ) // SeparatorByte is a byte that cannot occur in valid UTF-8 sequences and is @@ -28,7 +29,7 @@ var ( // cache the signature of an empty label set. emptyLabelSignature = fnv.New64a().Sum64() - hashAndBufPool = make(chan *hashAndBuf, 1024) + hashAndBufPool sync.Pool ) type hashAndBuf struct { @@ -37,19 +38,15 @@ type hashAndBuf struct { } func getHashAndBuf() *hashAndBuf { - select { - case hb := <-hashAndBufPool: - return hb - default: + hb := hashAndBufPool.Get() + if hb == nil { return &hashAndBuf{h: fnv.New64a()} } + return hb.(*hashAndBuf) } func putHashAndBuf(hb *hashAndBuf) { - select { - case hashAndBufPool <- hb: - default: - } + hashAndBufPool.Put(hb) } // LabelsToSignature returns a unique signature (i.e., fingerprint) for a given @@ -63,10 +60,10 @@ func LabelsToSignature(labels map[string]string) uint64 { hb := getHashAndBuf() defer putHashAndBuf(hb) - for k, v := range labels { - hb.b.WriteString(k) + for labelName, labelValue := range labels { + hb.b.WriteString(labelName) hb.b.WriteByte(SeparatorByte) - hb.b.WriteString(v) + hb.b.WriteString(labelValue) hb.h.Write(hb.b.Bytes()) result ^= hb.h.Sum64() hb.h.Reset() @@ -75,10 +72,34 @@ func LabelsToSignature(labels map[string]string) uint64 { return result } -// LabelValuesToSignature returns a unique signature (i.e., fingerprint) for the -// values of a given label set. -func LabelValuesToSignature(labels map[string]string) uint64 { - if len(labels) == 0 { +// metricToFingerprint works exactly as LabelsToSignature but takes a Metric as +// parameter (rather than a label map) and returns a Fingerprint. +func metricToFingerprint(m Metric) Fingerprint { + if len(m) == 0 { + return Fingerprint(emptyLabelSignature) + } + + var result uint64 + hb := getHashAndBuf() + defer putHashAndBuf(hb) + + for labelName, labelValue := range m { + hb.b.WriteString(string(labelName)) + hb.b.WriteByte(SeparatorByte) + hb.b.WriteString(string(labelValue)) + hb.h.Write(hb.b.Bytes()) + result ^= hb.h.Sum64() + hb.h.Reset() + hb.b.Reset() + } + return Fingerprint(result) +} + +// SignatureForLabels works like LabelsToSignature but takes a Metric as +// parameter (rather than a label map) and only includes the labels with the +// specified LabelNames into the signature calculation. +func SignatureForLabels(m Metric, labels LabelNames) uint64 { + if len(m) == 0 || len(labels) == 0 { return emptyLabelSignature } @@ -86,8 +107,10 @@ func LabelValuesToSignature(labels map[string]string) uint64 { hb := getHashAndBuf() defer putHashAndBuf(hb) - for _, v := range labels { - hb.b.WriteString(v) + for _, label := range labels { + hb.b.WriteString(string(label)) + hb.b.WriteByte(SeparatorByte) + hb.b.WriteString(string(m[label])) hb.h.Write(hb.b.Bytes()) result ^= hb.h.Sum64() hb.h.Reset() @@ -95,3 +118,33 @@ func LabelValuesToSignature(labels map[string]string) uint64 { } return result } + +// SignatureWithoutLabels works like LabelsToSignature but takes a Metric as +// parameter (rather than a label map) and excludes the labels with any of the +// specified LabelNames from the signature calculation. +func SignatureWithoutLabels(m Metric, labels map[LabelName]struct{}) uint64 { + if len(m) == 0 { + return emptyLabelSignature + } + + var result uint64 + hb := getHashAndBuf() + defer putHashAndBuf(hb) + + for labelName, labelValue := range m { + if _, exclude := labels[labelName]; exclude { + continue + } + hb.b.WriteString(string(labelName)) + hb.b.WriteByte(SeparatorByte) + hb.b.WriteString(string(labelValue)) + hb.h.Write(hb.b.Bytes()) + result ^= hb.h.Sum64() + hb.h.Reset() + hb.b.Reset() + } + if result == 0 { + return emptyLabelSignature + } + return result +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature_test.go index ad20fa3ff87..7b3327d4406 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature_test.go @@ -15,10 +15,11 @@ package model import ( "runtime" + "sync" "testing" ) -func testLabelsToSignature(t testing.TB) { +func TestLabelsToSignature(t *testing.T) { var scenarios = []struct { in map[string]string out uint64 @@ -42,57 +43,112 @@ func testLabelsToSignature(t testing.TB) { } } -func TestLabelToSignature(t *testing.T) { - testLabelsToSignature(t) -} - -func TestEmptyLabelSignature(t *testing.T) { - input := []map[string]string{nil, {}} - - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - - alloc := ms.Alloc - - for _, labels := range input { - LabelsToSignature(labels) +func TestMetricToFingerprint(t *testing.T) { + var scenarios = []struct { + in Metric + out Fingerprint + }{ + { + in: Metric{}, + out: 14695981039346656037, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, + out: 12952432476264840823, + }, } - runtime.ReadMemStats(&ms) + for i, scenario := range scenarios { + actual := metricToFingerprint(scenario.in) - if got := ms.Alloc; alloc != got { - t.Fatal("expected LabelsToSignature with empty labels not to perform allocations") - } -} - -func BenchmarkLabelToSignature(b *testing.B) { - for i := 0; i < b.N; i++ { - testLabelsToSignature(b) - } -} - -func benchmarkLabelValuesToSignature(b *testing.B, l map[string]string, e uint64) { - for i := 0; i < b.N; i++ { - if a := LabelValuesToSignature(l); a != e { - b.Fatalf("expected signature of %d for %s, got %d", e, l, a) + if actual != scenario.out { + t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) } } } -func BenchmarkLabelValuesToSignatureScalar(b *testing.B) { - benchmarkLabelValuesToSignature(b, nil, 14695981039346656037) +func TestSignatureForLabels(t *testing.T) { + var scenarios = []struct { + in Metric + labels LabelNames + out uint64 + }{ + { + in: Metric{}, + labels: nil, + out: 14695981039346656037, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, + labels: LabelNames{"fear", "name"}, + out: 12952432476264840823, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough", "foo": "bar"}, + labels: LabelNames{"fear", "name"}, + out: 12952432476264840823, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, + labels: LabelNames{}, + out: 14695981039346656037, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, + labels: nil, + out: 14695981039346656037, + }, + } + + for i, scenario := range scenarios { + actual := SignatureForLabels(scenario.in, scenario.labels) + + if actual != scenario.out { + t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) + } + } } -func BenchmarkLabelValuesToSignatureSingle(b *testing.B) { - benchmarkLabelValuesToSignature(b, map[string]string{"first-label": "first-label-value"}, 2653746141194979650) -} +func TestSignatureWithoutLabels(t *testing.T) { + var scenarios = []struct { + in Metric + labels map[LabelName]struct{} + out uint64 + }{ + { + in: Metric{}, + labels: nil, + out: 14695981039346656037, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, + labels: map[LabelName]struct{}{"fear": struct{}{}, "name": struct{}{}}, + out: 14695981039346656037, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough", "foo": "bar"}, + labels: map[LabelName]struct{}{"foo": struct{}{}}, + out: 12952432476264840823, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, + labels: map[LabelName]struct{}{}, + out: 12952432476264840823, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, + labels: nil, + out: 12952432476264840823, + }, + } -func BenchmarkLabelValuesToSignatureDouble(b *testing.B) { - benchmarkLabelValuesToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value"}, 8893559499616767364) -} + for i, scenario := range scenarios { + actual := SignatureWithoutLabels(scenario.in, scenario.labels) -func BenchmarkLabelValuesToSignatureTriple(b *testing.B) { - benchmarkLabelValuesToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 1685970066862087833) + if actual != scenario.out { + t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) + } + } } func benchmarkLabelToSignature(b *testing.B, l map[string]string, e uint64) { @@ -118,3 +174,83 @@ func BenchmarkLabelToSignatureDouble(b *testing.B) { func BenchmarkLabelToSignatureTriple(b *testing.B) { benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676) } + +func benchmarkMetricToFingerprint(b *testing.B, m Metric, e Fingerprint) { + for i := 0; i < b.N; i++ { + if a := metricToFingerprint(m); a != e { + b.Fatalf("expected signature of %d for %s, got %d", e, m, a) + } + } +} + +func BenchmarkMetricToFingerprintScalar(b *testing.B) { + benchmarkMetricToFingerprint(b, nil, 14695981039346656037) +} + +func BenchmarkMetricToFingerprintSingle(b *testing.B) { + benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value"}, 5147259542624943964) +} + +func BenchmarkMetricToFingerprintDouble(b *testing.B) { + benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value"}, 18269973311206963528) +} + +func BenchmarkMetricToFingerprintTriple(b *testing.B) { + benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676) +} + +func TestEmptyLabelSignature(t *testing.T) { + input := []map[string]string{nil, {}} + + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + + alloc := ms.Alloc + + for _, labels := range input { + LabelsToSignature(labels) + } + + runtime.ReadMemStats(&ms) + + if got := ms.Alloc; alloc != got { + t.Fatal("expected LabelsToSignature with empty labels not to perform allocations") + } +} + +func benchmarkMetricToFingerprintConc(b *testing.B, m Metric, e Fingerprint, concLevel int) { + var start, end sync.WaitGroup + start.Add(1) + end.Add(concLevel) + + for i := 0; i < concLevel; i++ { + go func() { + start.Wait() + for j := b.N / concLevel; j >= 0; j-- { + if a := metricToFingerprint(m); a != e { + b.Fatalf("expected signature of %d for %s, got %d", e, m, a) + } + } + end.Done() + }() + } + b.ResetTimer() + start.Done() + end.Wait() +} + +func BenchmarkMetricToFingerprintTripleConc1(b *testing.B) { + benchmarkMetricToFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 1) +} + +func BenchmarkMetricToFingerprintTripleConc2(b *testing.B) { + benchmarkMetricToFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 2) +} + +func BenchmarkMetricToFingerprintTripleConc4(b *testing.B) { + benchmarkMetricToFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 4) +} + +func BenchmarkMetricToFingerprintTripleConc8(b *testing.B) { + benchmarkMetricToFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 8) +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/timestamp.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/timestamp.go index 09bd87710d2..afffdcf753e 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/timestamp.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/timestamp.go @@ -88,6 +88,7 @@ func (t Timestamp) String() string { return strconv.FormatFloat(float64(t)/float64(second), 'f', -1, 64) } +// MarshalJSON implements the json.Marshaler interface. func (t Timestamp) MarshalJSON() ([]byte, error) { return []byte(t.String()), nil } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go index 5e62967b0be..d106c42c6b7 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go @@ -18,8 +18,10 @@ import ( "fmt" "math" "net/http" + "os" "runtime" "sort" + "time" dto "github.com/prometheus/client_model/go" @@ -498,3 +500,19 @@ func ExampleHistogram() { // > // > } + +func ExamplePushCollectors() { + hostname, _ := os.Hostname() + completionTime := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "db_backup_last_completion_time", + Help: "The timestamp of the last succesful completion of a DB backup.", + }) + completionTime.Set(float64(time.Now().Unix())) + if err := prometheus.PushCollectors( + "db_backup", hostname, + "http://pushgateway:9091", + completionTime, + ); err != nil { + fmt.Println("Could not push completion time to Pushgateway:", err) + } +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector.go index 7fd1227fa6f..d8cf0eda347 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector.go @@ -13,6 +13,8 @@ package prometheus +import "github.com/prometheus/procfs" + type processCollector struct { pid int collectFn func(chan<- Metric) @@ -79,7 +81,7 @@ func NewProcessCollectorPIDFn( } // Set up process metric collection if supported by the runtime. - if processCollectSupported() { + if _, err := procfs.NewStat(); err == nil { c.collectFn = c.processCollect } @@ -100,3 +102,41 @@ func (c *processCollector) Describe(ch chan<- *Desc) { func (c *processCollector) Collect(ch chan<- Metric) { c.collectFn(ch) } + +// TODO(ts): Bring back error reporting by reverting 7faf9e7 as soon as the +// client allows users to configure the error behavior. +func (c *processCollector) processCollect(ch chan<- Metric) { + pid, err := c.pidFn() + if err != nil { + return + } + + p, err := procfs.NewProc(pid) + if err != nil { + return + } + + if stat, err := p.NewStat(); err == nil { + c.cpuTotal.Set(stat.CPUTime()) + ch <- c.cpuTotal + c.vsize.Set(float64(stat.VirtualMemory())) + ch <- c.vsize + c.rss.Set(float64(stat.ResidentMemory())) + ch <- c.rss + + if startTime, err := stat.StartTime(); err == nil { + c.startTime.Set(startTime) + ch <- c.startTime + } + } + + if fds, err := p.FileDescriptorsLen(); err == nil { + c.openFDs.Set(float64(fds)) + ch <- c.openFDs + } + + if limits, err := p.NewLimits(); err == nil { + c.maxFDs.Set(float64(limits.OpenFiles)) + ch <- c.maxFDs + } +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_procfs.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_procfs.go deleted file mode 100644 index 5a09ded1478..00000000000 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_procfs.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2015 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build linux,cgo plan9,cgo solaris,cgo - -package prometheus - -import "github.com/prometheus/procfs" - -func processCollectSupported() bool { - if _, err := procfs.NewStat(); err == nil { - return true - } - return false -} - -// TODO(ts): Bring back error reporting by reverting 7faf9e7 as soon as the -// client allows users to configure the error behavior. -func (c *processCollector) processCollect(ch chan<- Metric) { - pid, err := c.pidFn() - if err != nil { - return - } - - p, err := procfs.NewProc(pid) - if err != nil { - return - } - - if stat, err := p.NewStat(); err == nil { - c.cpuTotal.Set(stat.CPUTime()) - ch <- c.cpuTotal - c.vsize.Set(float64(stat.VirtualMemory())) - ch <- c.vsize - c.rss.Set(float64(stat.ResidentMemory())) - ch <- c.rss - - if startTime, err := stat.StartTime(); err == nil { - c.startTime.Set(startTime) - ch <- c.startTime - } - } - - if fds, err := p.FileDescriptorsLen(); err == nil { - c.openFDs.Set(float64(fds)) - ch <- c.openFDs - } - - if limits, err := p.NewLimits(); err == nil { - c.maxFDs.Set(float64(limits.OpenFiles)) - ch <- c.maxFDs - } -} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_rest.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_rest.go deleted file mode 100644 index 0b3698ab8ff..00000000000 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_rest.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2015 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !linux,!plan9,!solaris !cgo - -package prometheus - -func processCollectSupported() bool { - return false -} - -func (c *processCollector) processCollect(ch chan<- Metric) { - panic("unreachable") -} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/push.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/push.go new file mode 100644 index 00000000000..1c33848a353 --- /dev/null +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/push.go @@ -0,0 +1,65 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright (c) 2013, The Prometheus Authors +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. + +package prometheus + +// Push triggers a metric collection by the default registry and pushes all +// collected metrics to the Pushgateway specified by addr. See the Pushgateway +// documentation for detailed implications of the job and instance +// parameter. instance can be left empty. You can use just host:port or ip:port +// as url, in which case 'http://' is added automatically. You can also include +// the schema in the URL. However, do not include the '/metrics/jobs/...' part. +// +// Note that all previously pushed metrics with the same job and instance will +// be replaced with the metrics pushed by this call. (It uses HTTP method 'PUT' +// to push to the Pushgateway.) +func Push(job, instance, url string) error { + return defRegistry.Push(job, instance, url, "PUT") +} + +// PushAdd works like Push, but only previously pushed metrics with the same +// name (and the same job and instance) will be replaced. (It uses HTTP method +// 'POST' to push to the Pushgateway.) +func PushAdd(job, instance, url string) error { + return defRegistry.Push(job, instance, url, "POST") +} + +// PushCollectors works like Push, but it does not collect from the default +// registry. Instead, it collects from the provided collectors. It is a +// convenient way to push only a few metrics. +func PushCollectors(job, instance, url string, collectors ...Collector) error { + return pushCollectors(job, instance, url, "PUT", collectors...) +} + +// PushAddCollectors works like PushAdd, but it does not collect from the +// default registry. Instead, it collects from the provided collectors. It is a +// convenient way to push only a few metrics. +func PushAddCollectors(job, instance, url string, collectors ...Collector) error { + return pushCollectors(job, instance, url, "POST", collectors...) +} + +func pushCollectors(job, instance, url, method string, collectors ...Collector) error { + r := newRegistry() + for _, collector := range collectors { + if _, err := r.Register(collector); err != nil { + return err + } + } + return r.Push(job, instance, url, method) +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry.go index 550754172f7..2515311b94d 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry.go @@ -158,14 +158,19 @@ func Unregister(c Collector) bool { // SetMetricFamilyInjectionHook sets a function that is called whenever metrics // are collected. The hook function must be set before metrics collection begins // (i.e. call SetMetricFamilyInjectionHook before setting the HTTP handler.) The -// MetricFamily protobufs returned by the hook function are added to the -// delivered metrics. Each returned MetricFamily must have a unique name (also -// taking into account the MetricFamilies created in the regular way). +// MetricFamily protobufs returned by the hook function are merged with the +// metrics collected in the usual way. // // This is a way to directly inject MetricFamily protobufs managed and owned by -// the caller. The caller has full responsibility. No sanity checks are -// performed on the returned protobufs (besides the name checks described -// above). The function must be callable at any time and concurrently. +// the caller. The caller has full responsibility. As no registration of the +// injected metrics has happened, there is no descriptor to check against, and +// there are no registration-time checks. If collect-time checks are disabled +// (see function EnableCollectChecks), no sanity checks are performed on the +// returned protobufs at all. If collect-checks are enabled, type and uniqueness +// checks are performed, but no further consistency checks (which would require +// knowledge of a metric descriptor). +// +// The function must be callable at any time and concurrently. func SetMetricFamilyInjectionHook(hook func() []*dto.MetricFamily) { defRegistry.metricFamilyInjectionHook = hook } @@ -187,30 +192,10 @@ func EnableCollectChecks(b bool) { defRegistry.collectChecksEnabled = b } -// Push triggers a metric collection and pushes all collected metrics to the -// Pushgateway specified by addr. See the Pushgateway documentation for detailed -// implications of the job and instance parameter. instance can be left -// empty. The Pushgateway will then use the client's IP number instead. Use just -// host:port or ip:port ass addr. (Don't add 'http://' or any path.) -// -// Note that all previously pushed metrics with the same job and instance will -// be replaced with the metrics pushed by this call. (It uses HTTP method 'PUT' -// to push to the Pushgateway.) -func Push(job, instance, addr string) error { - return defRegistry.Push(job, instance, addr, "PUT") -} - -// PushAdd works like Push, but only previously pushed metrics with the same -// name (and the same job and instance) will be replaced. (It uses HTTP method -// 'POST' to push to the Pushgateway.) -func PushAdd(job, instance, addr string) error { - return defRegistry.Push(job, instance, addr, "POST") -} - // encoder is a function that writes a dto.MetricFamily to an io.Writer in a // certain encoding. It returns the number of bytes written and any error -// encountered. Note that ext.WriteDelimited and text.MetricFamilyToText are -// encoders. +// encountered. Note that pbutil.WriteDelimited and pbutil.MetricFamilyToText +// are encoders. type encoder func(io.Writer, *dto.MetricFamily) (int, error) type registry struct { @@ -346,10 +331,13 @@ func (r *registry) Unregister(c Collector) bool { return true } -func (r *registry) Push(job, instance, addr, method string) error { - u := fmt.Sprintf("http://%s/metrics/jobs/%s", addr, url.QueryEscape(job)) +func (r *registry) Push(job, instance, pushURL, method string) error { + if !strings.Contains(pushURL, "://") { + pushURL = "http://" + pushURL + } + pushURL = fmt.Sprintf("%s/metrics/jobs/%s", pushURL, url.QueryEscape(job)) if instance != "" { - u += "/instances/" + url.QueryEscape(instance) + pushURL += "/instances/" + url.QueryEscape(instance) } buf := r.getBuf() defer r.giveBuf(buf) @@ -359,7 +347,7 @@ func (r *registry) Push(job, instance, addr, method string) error { } return err } - req, err := http.NewRequest(method, u, buf) + req, err := http.NewRequest(method, pushURL, buf) if err != nil { return err } @@ -370,7 +358,7 @@ func (r *registry) Push(job, instance, addr, method string) error { } defer resp.Body.Close() if resp.StatusCode != 202 { - return fmt.Errorf("unexpected status code %d while pushing to %s", resp.StatusCode, u) + return fmt.Errorf("unexpected status code %d while pushing to %s", resp.StatusCode, pushURL) } return nil } @@ -479,10 +467,26 @@ func (r *registry) writePB(w io.Writer, writeEncoded encoder) (int, error) { if r.metricFamilyInjectionHook != nil { for _, mf := range r.metricFamilyInjectionHook() { - if _, exists := metricFamiliesByName[mf.GetName()]; exists { - return 0, fmt.Errorf("metric family with duplicate name injected: %s", mf) + existingMF, exists := metricFamiliesByName[mf.GetName()] + if !exists { + metricFamiliesByName[mf.GetName()] = mf + if r.collectChecksEnabled { + for _, m := range mf.Metric { + if err := r.checkConsistency(mf, m, nil, metricHashes); err != nil { + return 0, err + } + } + } + continue + } + for _, m := range mf.Metric { + if r.collectChecksEnabled { + if err := r.checkConsistency(existingMF, m, nil, metricHashes); err != nil { + return 0, err + } + } + existingMF.Metric = append(existingMF.Metric, m) } - metricFamiliesByName[mf.GetName()] = mf } } @@ -523,11 +527,42 @@ func (r *registry) checkConsistency(metricFamily *dto.MetricFamily, dtoMetric *d ) } + // Is the metric unique (i.e. no other metric with the same name and the same label values)? + h := fnv.New64a() + var buf bytes.Buffer + buf.WriteString(metricFamily.GetName()) + buf.WriteByte(model.SeparatorByte) + h.Write(buf.Bytes()) + for _, lp := range dtoMetric.Label { + buf.Reset() + buf.WriteString(lp.GetValue()) + buf.WriteByte(model.SeparatorByte) + h.Write(buf.Bytes()) + } + metricHash := h.Sum64() + if _, exists := metricHashes[metricHash]; exists { + return fmt.Errorf( + "collected metric %q was collected before with the same name and label values", + dtoMetric, + ) + } + metricHashes[metricHash] = struct{}{} + + if desc == nil { + return nil // Nothing left to check if we have no desc. + } + // Desc consistency with metric family. + if metricFamily.GetName() != desc.fqName { + return fmt.Errorf( + "collected metric %q has name %q but should have %q", + dtoMetric, metricFamily.GetName(), desc.fqName, + ) + } if metricFamily.GetHelp() != desc.help { return fmt.Errorf( "collected metric %q has help %q but should have %q", - dtoMetric, desc.help, metricFamily.GetHelp(), + dtoMetric, metricFamily.GetHelp(), desc.help, ) } @@ -557,27 +592,6 @@ func (r *registry) checkConsistency(metricFamily *dto.MetricFamily, dtoMetric *d } } - // Is the metric unique (i.e. no other metric with the same name and the same label values)? - h := fnv.New64a() - var buf bytes.Buffer - buf.WriteString(desc.fqName) - buf.WriteByte(model.SeparatorByte) - h.Write(buf.Bytes()) - for _, lp := range dtoMetric.Label { - buf.Reset() - buf.WriteString(lp.GetValue()) - buf.WriteByte(model.SeparatorByte) - h.Write(buf.Bytes()) - } - metricHash := h.Sum64() - if _, exists := metricHashes[metricHash]; exists { - return fmt.Errorf( - "collected metric %q was collected before with the same name and label values", - dtoMetric, - ) - } - metricHashes[metricHash] = struct{}{} - r.mtx.RLock() // Remaining checks need the read lock. defer r.mtx.RUnlock() @@ -712,6 +726,15 @@ func (s metricSorter) Swap(i, j int) { } func (s metricSorter) Less(i, j int) bool { + if len(s[i].Label) != len(s[j].Label) { + // This should not happen. The metrics are + // inconsistent. However, we have to deal with the fact, as + // people might use custom collectors or metric family injection + // to create inconsistent metrics. So let's simply compare the + // number of labels in this case. That will still yield + // reproducible sorting. + return len(s[i].Label) < len(s[j].Label) + } for n, lp := range s[i].Label { vi := lp.GetValue() vj := s[j].Label[n].GetValue() diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry_test.go index 95579e512b4..bbf11f69778 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry_test.go @@ -61,31 +61,29 @@ func testHandler(t testing.TB) { varintBuf := make([]byte, binary.MaxVarintLen32) - externalMetricFamily := []*dto.MetricFamily{ - { - Name: proto.String("externalname"), - Help: proto.String("externaldocstring"), - Type: dto.MetricType_COUNTER.Enum(), - Metric: []*dto.Metric{ - { - Label: []*dto.LabelPair{ - { - Name: proto.String("externallabelname"), - Value: proto.String("externalval1"), - }, - { - Name: proto.String("externalconstname"), - Value: proto.String("externalconstvalue"), - }, + externalMetricFamily := &dto.MetricFamily{ + Name: proto.String("externalname"), + Help: proto.String("externaldocstring"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("externallabelname"), + Value: proto.String("externalval1"), }, - Counter: &dto.Counter{ - Value: proto.Float64(1), + { + Name: proto.String("externalconstname"), + Value: proto.String("externalconstvalue"), }, }, + Counter: &dto.Counter{ + Value: proto.Float64(1), + }, }, }, } - marshaledExternalMetricFamily, err := proto.Marshal(externalMetricFamily[0]) + marshaledExternalMetricFamily, err := proto.Marshal(externalMetricFamily) if err != nil { t.Fatal(err) } @@ -216,16 +214,42 @@ metric: < expectedMetricFamilyAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric: label: counter: > metric: label: counter: > `) + externalMetricFamilyWithSameName := &dto.MetricFamily{ + Name: proto.String("name"), + Help: proto.String("inconsistent help string does not matter here"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("constname"), + Value: proto.String("constvalue"), + }, + { + Name: proto.String("labelname"), + Value: proto.String("different_val"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(42), + }, + }, + }, + } + + expectedMetricFamilyMergedWithExternalAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric: label: counter: > metric: label: counter: > metric: label: counter: > +`) + type output struct { headers map[string]string body []byte } var scenarios = []struct { - headers map[string]string - out output - withCounter bool - withExternalMF bool + headers map[string]string + out output + collector Collector + externalMF []*dto.MetricFamily }{ { // 0 headers: map[string]string{ @@ -281,7 +305,7 @@ metric: < }, body: expectedMetricFamilyAsText, }, - withCounter: true, + collector: metricVec, }, { // 5 headers: map[string]string{ @@ -293,7 +317,7 @@ metric: < }, body: expectedMetricFamilyAsBytes, }, - withCounter: true, + collector: metricVec, }, { // 6 headers: map[string]string{ @@ -305,7 +329,7 @@ metric: < }, body: externalMetricFamilyAsText, }, - withExternalMF: true, + externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 7 headers: map[string]string{ @@ -317,7 +341,7 @@ metric: < }, body: externalMetricFamilyAsBytes, }, - withExternalMF: true, + externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 8 headers: map[string]string{ @@ -335,8 +359,8 @@ metric: < []byte{}, ), }, - withCounter: true, - withExternalMF: true, + collector: metricVec, + externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 9 headers: map[string]string{ @@ -359,7 +383,7 @@ metric: < }, body: expectedMetricFamilyAsText, }, - withCounter: true, + collector: metricVec, }, { // 11 headers: map[string]string{ @@ -377,8 +401,8 @@ metric: < []byte{}, ), }, - withCounter: true, - withExternalMF: true, + collector: metricVec, + externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 12 headers: map[string]string{ @@ -396,8 +420,8 @@ metric: < []byte{}, ), }, - withCounter: true, - withExternalMF: true, + collector: metricVec, + externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 13 headers: map[string]string{ @@ -415,8 +439,8 @@ metric: < []byte{}, ), }, - withCounter: true, - withExternalMF: true, + collector: metricVec, + externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 14 headers: map[string]string{ @@ -434,20 +458,42 @@ metric: < []byte{}, ), }, - withCounter: true, - withExternalMF: true, + collector: metricVec, + externalMF: []*dto.MetricFamily{externalMetricFamily}, + }, + { // 15 + headers: map[string]string{ + "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text", + }, + out: output{ + headers: map[string]string{ + "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=compact-text`, + }, + body: bytes.Join( + [][]byte{ + externalMetricFamilyAsProtoCompactText, + expectedMetricFamilyMergedWithExternalAsProtoCompactText, + }, + []byte{}, + ), + }, + collector: metricVec, + externalMF: []*dto.MetricFamily{ + externalMetricFamily, + externalMetricFamilyWithSameName, + }, }, } for i, scenario := range scenarios { registry := newRegistry() registry.collectChecksEnabled = true - if scenario.withCounter { - registry.Register(metricVec) + if scenario.collector != nil { + registry.Register(scenario.collector) } - if scenario.withExternalMF { + if scenario.externalMF != nil { registry.metricFamilyInjectionHook = func() []*dto.MetricFamily { - return externalMetricFamily + return scenario.externalMF } } writer := &fakeResponseWriter{ diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go index a731f4e8d34..dd336c51994 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go @@ -16,6 +16,7 @@ package prometheus import ( "fmt" "hash/fnv" + "math" "sort" "sync" "time" @@ -277,10 +278,8 @@ func (s *summary) Write(out *dto.Metric) error { s.bufMtx.Lock() s.mtx.Lock() - - if len(s.hotBuf) != 0 { - s.swapBufs(time.Now()) - } + // Swap bufs even if hotBuf is empty to set new hotBufExpTime. + s.swapBufs(time.Now()) s.bufMtx.Unlock() s.flushColdBuf() @@ -288,9 +287,15 @@ func (s *summary) Write(out *dto.Metric) error { sum.SampleSum = proto.Float64(s.sum) for _, rank := range s.sortedObjectives { + var q float64 + if s.headStream.Count() == 0 { + q = math.NaN() + } else { + q = s.headStream.Query(rank) + } qs = append(qs, &dto.Quantile{ Quantile: proto.Float64(rank), - Value: proto.Float64(s.headStream.Query(rank)), + Value: proto.Float64(q), }) } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary_test.go index 40d05fae52f..0790cdfe727 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary_test.go @@ -289,6 +289,11 @@ func TestSummaryVecConcurrency(t *testing.T) { } func TestSummaryDecay(t *testing.T) { + if testing.Short() { + t.Skip("Skipping test in short mode.") + // More because it depends on timing than because it is particularly long... + } + sum := NewSummary(SummaryOpts{ Name: "test_summary", Help: "helpless", @@ -315,6 +320,12 @@ func TestSummaryDecay(t *testing.T) { } } tick.Stop() + // Wait for MaxAge without observations and make sure quantiles are NaN. + time.Sleep(100 * time.Millisecond) + sum.Write(m) + if got := *m.Summary.Quantile[0].Value; !math.IsNaN(got) { + t.Errorf("got %f, want NaN after expiration", got) + } } func getBounds(vars []float64, q, ε float64) (min, max float64) { diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/bench_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/bench_test.go index df887e0fc6d..a97409ed3f9 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/bench_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/bench_test.go @@ -21,7 +21,7 @@ import ( "testing" dto "github.com/prometheus/client_model/go" - "github.com/matttproud/golang_protobuf_extensions/ext" + "github.com/matttproud/golang_protobuf_extensions/pbutil" ) // Benchmarks to show how much penalty text format parsing actually inflicts. @@ -101,7 +101,7 @@ func BenchmarkParseProto(b *testing.B) { in := bytes.NewReader(data) for { family.Reset() - if _, err := ext.ReadDelimited(in, family); err != nil { + if _, err := pbutil.ReadDelimited(in, family); err != nil { if err == io.EOF { break } @@ -129,7 +129,7 @@ func BenchmarkParseProtoGzip(b *testing.B) { } for { family.Reset() - if _, err := ext.ReadDelimited(in, family); err != nil { + if _, err := pbutil.ReadDelimited(in, family); err != nil { if err == io.EOF { break } @@ -156,7 +156,7 @@ func BenchmarkParseProtoMap(b *testing.B) { in := bytes.NewReader(data) for { family := &dto.MetricFamily{} - if _, err := ext.ReadDelimited(in, family); err != nil { + if _, err := pbutil.ReadDelimited(in, family); err != nil { if err == io.EOF { break } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse_test.go index 6b7adfff557..cc3e6470f0a 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse_test.go @@ -385,7 +385,7 @@ request_duration_microseconds_count 2693 }, }, }, - }, + }, } for i, scenario := range scenarios { diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/proto.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/proto.go index 058adfc6f60..e82bbb3b408 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/proto.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/proto.go @@ -18,7 +18,7 @@ import ( "io" "github.com/golang/protobuf/proto" - "github.com/matttproud/golang_protobuf_extensions/ext" + "github.com/matttproud/golang_protobuf_extensions/pbutil" dto "github.com/prometheus/client_model/go" ) @@ -27,7 +27,7 @@ import ( // protobuf format and returns the number of bytes written and any error // encountered. func WriteProtoDelimited(w io.Writer, p *dto.MetricFamily) (int, error) { - return ext.WriteDelimited(w, p) + return pbutil.WriteDelimited(w, p) } // WriteProtoText writes the MetricFamily to the writer in text format and diff --git a/Godeps/_workspace/src/github.com/prometheus/procfs/.travis.yml b/Godeps/_workspace/src/github.com/prometheus/procfs/.travis.yml new file mode 100644 index 00000000000..b1e6743f9c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/prometheus/procfs/.travis.yml @@ -0,0 +1,5 @@ +language: go +go: + - 1.3 + - 1.4 + - tip diff --git a/Godeps/_workspace/src/github.com/prometheus/procfs/README.md b/Godeps/_workspace/src/github.com/prometheus/procfs/README.md index 0edf4966636..761d31cd4dc 100644 --- a/Godeps/_workspace/src/github.com/prometheus/procfs/README.md +++ b/Godeps/_workspace/src/github.com/prometheus/procfs/README.md @@ -4,8 +4,4 @@ This procfs package provides functions to retrieve system, kernel and process metrics from the pseudo-filesystem proc. [![GoDoc](https://godoc.org/github.com/prometheus/procfs?status.png)](https://godoc.org/github.com/prometheus/procfs) -[![Circle CI](https://circleci.com/gh/prometheus/procfs.svg?style=svg)](https://circleci.com/gh/prometheus/procfs) - -# Testing - - $ go test +[![Build Status](https://travis-ci.org/prometheus/procfs.svg?branch=master)](https://travis-ci.org/prometheus/procfs) diff --git a/Godeps/_workspace/src/github.com/prometheus/procfs/proc_stat.go b/Godeps/_workspace/src/github.com/prometheus/procfs/proc_stat.go index 1e027762eec..30a403b6c7e 100644 --- a/Godeps/_workspace/src/github.com/prometheus/procfs/proc_stat.go +++ b/Godeps/_workspace/src/github.com/prometheus/procfs/proc_stat.go @@ -7,8 +7,22 @@ import ( "os" ) -// #include -import "C" +// Originally, this USER_HZ value was dynamically retrieved via a sysconf call which +// required cgo. However, that caused a lot of problems regarding +// cross-compilation. Alternatives such as running a binary to determine the +// value, or trying to derive it in some other way were all problematic. +// After much research it was determined that USER_HZ is actually hardcoded to +// 100 on all Go-supported platforms as of the time of this writing. This is +// why we decided to hardcode it here as well. It is not impossible that there +// could be systems with exceptions, but they should be very exotic edge cases, +// and in that case, the worst outcome will be two misreported metrics. +// +// See also the following discussions: +// +// - https://github.com/prometheus/node_exporter/issues/52 +// - https://github.com/prometheus/procfs/pull/2 +// - http://stackoverflow.com/questions/17410841/how-does-user-hz-solve-the-jiffy-scaling-issue +const userHZ = 100 // ProcStat provides status information about the process, // read from /proc/[pid]/stat. @@ -152,14 +166,10 @@ func (s ProcStat) StartTime() (float64, error) { if err != nil { return 0, err } - return float64(stat.BootTime) + (float64(s.Starttime) / ticks()), nil + return float64(stat.BootTime) + (float64(s.Starttime) / userHZ), nil } // CPUTime returns the total CPU user and system time in seconds. func (s ProcStat) CPUTime() float64 { - return float64(s.UTime+s.STime) / ticks() -} - -func ticks() float64 { - return float64(C.sysconf(C._SC_CLK_TCK)) // most likely 100 + return float64(s.UTime+s.STime) / userHZ }