From 76325ca851bcba98bf5bf95b9a36a32385de46f6 Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Mon, 2 Mar 2015 01:11:37 +0000 Subject: [PATCH 1/2] Update the version of the prometheus client library we have vendored so that we can use true histograms rather than the old Summary type that calculated quantiles on the client side. --- Godeps/Godeps.json | 37 +- .../ww}/goautoneg/Makefile | 0 .../ww}/goautoneg/README.txt | 0 .../ww}/goautoneg/autoneg.go | 0 .../ww}/goautoneg/autoneg_test.go | 0 .../perks/quantile/bench_test.go | 0 .../perks/quantile/example_test.go | 21 +- .../perks/quantile/exampledata.txt | 0 .../perks/quantile/stream.go | 0 .../perks/quantile/stream_test.go | 9 +- .../client_golang/_vendor/goautoneg/MANIFEST | 1 - .../client_golang/model/labelname.go | 8 + .../prometheus/client_golang/model/metric.go | 25 +- .../client_golang/model/metric_test.go | 18 + .../client_golang/model/timestamp.go | 17 +- .../client_golang/prometheus/README.md | 6 +- .../prometheus/benchmark_test.go | 28 ++ .../client_golang/prometheus/counter.go | 2 +- .../client_golang/prometheus/desc.go | 2 +- .../prometheus/example_selfcollector_test.go | 2 +- .../client_golang/prometheus/examples_test.go | 54 ++- .../client_golang/prometheus/histogram.go | 344 ++++++++++++++++++ .../prometheus/histogram_test.go | 318 ++++++++++++++++ .../client_golang/prometheus/http.go | 2 +- .../prometheus/process_collector_procfs.go | 33 +- .../client_golang/prometheus/registry.go | 10 +- .../client_golang/prometheus/registry_test.go | 2 +- .../client_golang/prometheus/summary.go | 43 ++- .../client_golang/prometheus/summary_test.go | 8 + .../client_golang/prometheus/value.go | 2 +- .../prometheus/client_golang/text/create.go | 52 ++- .../client_golang/text/create_test.go | 94 ++++- .../prometheus/client_golang/text/parse.go | 114 +++++- .../client_golang/text/parse_test.go | 63 +++- .../prometheus/client_golang/text/proto.go | 2 +- .../prometheus/client_model/go/metrics.pb.go | 91 ++++- .../src/github.com/prometheus/procfs/doc.go | 4 +- 37 files changed, 1279 insertions(+), 133 deletions(-) rename Godeps/_workspace/src/{github.com/prometheus/client_golang/_vendor => bitbucket.org/ww}/goautoneg/Makefile (100%) rename Godeps/_workspace/src/{github.com/prometheus/client_golang/_vendor => bitbucket.org/ww}/goautoneg/README.txt (100%) rename Godeps/_workspace/src/{github.com/prometheus/client_golang/_vendor => bitbucket.org/ww}/goautoneg/autoneg.go (100%) rename Godeps/_workspace/src/{github.com/prometheus/client_golang/_vendor => bitbucket.org/ww}/goautoneg/autoneg_test.go (100%) rename Godeps/_workspace/src/github.com/{prometheus/client_golang/_vendor => beorn7}/perks/quantile/bench_test.go (100%) rename Godeps/_workspace/src/github.com/{prometheus/client_golang/_vendor => beorn7}/perks/quantile/example_test.go (87%) rename Godeps/_workspace/src/github.com/{prometheus/client_golang/_vendor => beorn7}/perks/quantile/exampledata.txt (100%) rename Godeps/_workspace/src/github.com/{prometheus/client_golang/_vendor => beorn7}/perks/quantile/stream.go (100%) rename Godeps/_workspace/src/github.com/{prometheus/client_golang/_vendor => beorn7}/perks/quantile/stream_test.go (92%) delete mode 100644 Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/MANIFEST create mode 100644 Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram.go create mode 100644 Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ae37f15984a..c041d87dc77 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -5,6 +5,11 @@ "./..." ], "Deps": [ + { + "ImportPath": "bitbucket.org/ww/goautoneg", + "Comment": "null-5", + "Rev": "75cd24fc2f2c2a2088577d12123ddee5f54e0675" + }, { "ImportPath": "code.google.com/p/gcfg", "Rev": "c2d3050044d05357eaf6c3547249ba57c5e235cb" @@ -44,6 +49,10 @@ "Comment": "v0.6.2-10-g51fe59a", "Rev": "51fe59aca108dc5680109e7b2051cbdcfa5a253c" }, + { + "ImportPath": "github.com/beorn7/perks/quantile", + "Rev": "b965b613227fddccbfffe13eae360ed3fa822f8d" + }, { "ImportPath": "github.com/coreos/go-etcd/etcd", "Comment": "v0.2.0-rc1-120-g23142f6", @@ -194,39 +203,29 @@ "Comment": "v1.0-28-g8adf9e1730c5", "Rev": "8adf9e1730c55cdc590de7d49766cb2acc88d8f2" }, - { - "ImportPath": "github.com/prometheus/client_golang/_vendor/goautoneg", - "Comment": "0.1.0-11-gc70db11", - "Rev": "c70db11f1ee77a34066aa41345dca4b105c2ed06" - }, - { - "ImportPath": "github.com/prometheus/client_golang/_vendor/perks/quantile", - "Comment": "0.1.0-11-gc70db11", - "Rev": "c70db11f1ee77a34066aa41345dca4b105c2ed06" - }, { "ImportPath": "github.com/prometheus/client_golang/model", - "Comment": "0.1.0-11-gc70db11", - "Rev": "c70db11f1ee77a34066aa41345dca4b105c2ed06" + "Comment": "0.2.0-5-gde5f7a2", + "Rev": "de5f7a2db9d883392ce3ad667087280fe1ff9cea" }, { "ImportPath": "github.com/prometheus/client_golang/prometheus", - "Comment": "0.1.0-11-gc70db11", - "Rev": "c70db11f1ee77a34066aa41345dca4b105c2ed06" + "Comment": "0.2.0-5-gde5f7a2", + "Rev": "de5f7a2db9d883392ce3ad667087280fe1ff9cea" }, { "ImportPath": "github.com/prometheus/client_golang/text", - "Comment": "0.1.0-11-gc70db11", - "Rev": "c70db11f1ee77a34066aa41345dca4b105c2ed06" + "Comment": "0.2.0-5-gde5f7a2", + "Rev": "de5f7a2db9d883392ce3ad667087280fe1ff9cea" }, { "ImportPath": "github.com/prometheus/client_model/go", - "Comment": "model-0.0.2-10-gbc9454c", - "Rev": "bc9454ca562dc050e060ea61a1c0e562a189850f" + "Comment": "model-0.0.2-12-gfa8ad6f", + "Rev": "fa8ad6fec33561be4280a8f0514318c79d7f6cb6" }, { "ImportPath": "github.com/prometheus/procfs", - "Rev": "92faa308558161acab0ada1db048e9996ecec160" + "Rev": "6c34ef819e19b4e16f410100ace4aa006f0e3bf8" }, { "ImportPath": "github.com/racker/perigee", diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/Makefile b/Godeps/_workspace/src/bitbucket.org/ww/goautoneg/Makefile similarity index 100% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/Makefile rename to Godeps/_workspace/src/bitbucket.org/ww/goautoneg/Makefile diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/README.txt b/Godeps/_workspace/src/bitbucket.org/ww/goautoneg/README.txt similarity index 100% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/README.txt rename to Godeps/_workspace/src/bitbucket.org/ww/goautoneg/README.txt diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/autoneg.go b/Godeps/_workspace/src/bitbucket.org/ww/goautoneg/autoneg.go similarity index 100% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/autoneg.go rename to Godeps/_workspace/src/bitbucket.org/ww/goautoneg/autoneg.go diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/autoneg_test.go b/Godeps/_workspace/src/bitbucket.org/ww/goautoneg/autoneg_test.go similarity index 100% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/autoneg_test.go rename to Godeps/_workspace/src/bitbucket.org/ww/goautoneg/autoneg_test.go diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/bench_test.go b/Godeps/_workspace/src/github.com/beorn7/perks/quantile/bench_test.go similarity index 100% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/bench_test.go rename to Godeps/_workspace/src/github.com/beorn7/perks/quantile/bench_test.go diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/example_test.go b/Godeps/_workspace/src/github.com/beorn7/perks/quantile/example_test.go similarity index 87% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/example_test.go rename to Godeps/_workspace/src/github.com/beorn7/perks/quantile/example_test.go index 43121e26305..ab3293aaf2a 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/example_test.go +++ b/Godeps/_workspace/src/github.com/beorn7/perks/quantile/example_test.go @@ -5,11 +5,12 @@ package quantile_test import ( "bufio" "fmt" - "github.com/bmizerany/perks/quantile" "log" "os" "strconv" "time" + + "github.com/beorn7/perks/quantile" ) func Example_simple() { @@ -17,7 +18,11 @@ func Example_simple() { go sendFloats(ch) // Compute the 50th, 90th, and 99th percentile. - q := quantile.NewTargeted(0.50, 0.90, 0.99) + q := quantile.NewTargeted(map[float64]float64{ + 0.50: 0.005, + 0.90: 0.001, + 0.99: 0.0001, + }) for v := range ch { q.Insert(v) } @@ -28,8 +33,8 @@ func Example_simple() { fmt.Println("count:", q.Count()) // Output: // perc50: 5 - // perc90: 14 - // perc99: 40 + // perc90: 16 + // perc99: 223 // count: 2388 } @@ -52,7 +57,7 @@ func Example_mergeMultipleStreams() { // even if we do not plan to query them all here. ch := make(chan quantile.Samples) getDBQuerySamples(ch) - q := quantile.NewTargeted(0.90) + q := quantile.NewTargeted(map[float64]float64{0.90: 0.001}) for samples := range ch { q.Merge(samples) } @@ -67,7 +72,11 @@ func Example_window() { go sendStreamValues(ch) tick := time.NewTicker(1 * time.Minute) - q := quantile.NewTargeted(0.90, 0.95, 0.99) + q := quantile.NewTargeted(map[float64]float64{ + 0.90: 0.001, + 0.95: 0.0005, + 0.99: 0.0001, + }) for { select { case t := <-tick.C: diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/exampledata.txt b/Godeps/_workspace/src/github.com/beorn7/perks/quantile/exampledata.txt similarity index 100% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/exampledata.txt rename to Godeps/_workspace/src/github.com/beorn7/perks/quantile/exampledata.txt diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/stream.go b/Godeps/_workspace/src/github.com/beorn7/perks/quantile/stream.go similarity index 100% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/stream.go rename to Godeps/_workspace/src/github.com/beorn7/perks/quantile/stream.go diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/stream_test.go b/Godeps/_workspace/src/github.com/beorn7/perks/quantile/stream_test.go similarity index 92% rename from Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/stream_test.go rename to Godeps/_workspace/src/github.com/beorn7/perks/quantile/stream_test.go index 707b871508f..4dba05449cb 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/perks/quantile/stream_test.go +++ b/Godeps/_workspace/src/github.com/beorn7/perks/quantile/stream_test.go @@ -113,7 +113,8 @@ func TestHighBiasedQuery(t *testing.T) { verifyHighPercsWithRelativeEpsilon(t, a, s) } -func TestTargetedMerge(t *testing.T) { +// BrokenTestTargetedMerge is broken, see Merge doc comment. +func BrokenTestTargetedMerge(t *testing.T) { rand.Seed(42) s1 := NewTargeted(Targets) s2 := NewTargeted(Targets) @@ -123,7 +124,8 @@ func TestTargetedMerge(t *testing.T) { verifyPercsWithAbsoluteEpsilon(t, a, s1) } -func TestLowBiasedMerge(t *testing.T) { +// BrokenTestLowBiasedMerge is broken, see Merge doc comment. +func BrokenTestLowBiasedMerge(t *testing.T) { rand.Seed(42) s1 := NewLowBiased(RelativeEpsilon) s2 := NewLowBiased(RelativeEpsilon) @@ -133,7 +135,8 @@ func TestLowBiasedMerge(t *testing.T) { verifyLowPercsWithRelativeEpsilon(t, a, s2) } -func TestHighBiasedMerge(t *testing.T) { +// BrokenTestHighBiasedMerge is broken, see Merge doc comment. +func BrokenTestHighBiasedMerge(t *testing.T) { rand.Seed(42) s1 := NewHighBiased(RelativeEpsilon) s2 := NewHighBiased(RelativeEpsilon) diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/MANIFEST b/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/MANIFEST deleted file mode 100644 index 71bfe39aa9d..00000000000 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/_vendor/goautoneg/MANIFEST +++ /dev/null @@ -1 +0,0 @@ -Imported at 75cd24fc2f2c from https://bitbucket.org/ww/goautoneg. diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/labelname.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/labelname.go index 047e7565559..75b2e79dae0 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/labelname.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/labelname.go @@ -33,6 +33,14 @@ const ( // JobLabel is the label name indicating the job from which a timeseries // was scraped. JobLabel LabelName = "job" + + // BucketLabel is used for the label that defines the upper bound of a + // bucket of a histogram ("le" -> "less or equal"). + BucketLabel = "le" + + // QuantileLabel is used for the label that defines the quantile in a + // summary. + QuantileLabel = "quantile" ) // A LabelName is a key for a LabelSet or Metric. It has a value associated diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go index 246a5626924..74f394b92e0 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go @@ -22,6 +22,8 @@ import ( "strings" ) +var separator = []byte{0} + // A Metric is similar to a LabelSet, but the key difference is that a Metric is // a singleton and refers to one and only one stream of samples. type Metric map[LabelName]LabelValue @@ -64,23 +66,34 @@ func (m Metric) String() string { // Fingerprint returns a Metric's Fingerprint. func (m Metric) Fingerprint() Fingerprint { - labelLength := len(m) - labelNames := make([]string, 0, labelLength) + labelNames := make([]string, 0, len(m)) + maxLength := 0 - for labelName := range m { + for labelName, labelValue := range m { labelNames = append(labelNames, string(labelName)) + if len(labelName) > maxLength { + maxLength = len(labelName) + } + if len(labelValue) > maxLength { + maxLength = len(labelValue) + } } sort.Strings(labelNames) summer := fnv.New64a() + buf := make([]byte, maxLength) for _, labelName := range labelNames { labelValue := m[LabelName(labelName)] - summer.Write([]byte(labelName)) - summer.Write([]byte{0}) - summer.Write([]byte(labelValue)) + copy(buf, labelName) + summer.Write(buf[:len(labelName)]) + + summer.Write(separator) + + copy(buf, labelValue) + summer.Write(buf[:len(labelValue)]) } return Fingerprint(binary.LittleEndian.Uint64(summer.Sum(nil))) diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go index 66a4c8b669f..7c31bdfb45e 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go @@ -38,6 +38,24 @@ func testMetric(t testing.TB) { }, fingerprint: 1470933794305433534, }, + // The following two demonstrate a bug in fingerprinting. They + // should not have the same fingerprint with a sane + // fingerprinting function. See + // https://github.com/prometheus/client_golang/issues/74 . + { + input: Metric{ + "a": "bb", + "b": "c", + }, + fingerprint: 3734646176939799877, + }, + { + input: Metric{ + "a": "b", + "bb": "c", + }, + fingerprint: 3734646176939799877, + }, } for i, scenario := range scenarios { diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/timestamp.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/timestamp.go index eb4304a8e50..09bd87710d2 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/timestamp.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/timestamp.go @@ -14,18 +14,14 @@ package model import ( + "math" "strconv" native_time "time" ) -// TODO(julius): Should this use milliseconds/nanoseconds instead? This is -// mostly hidden from the user of these types when using the -// methods below, so it will be easy to change this later -// without requiring significant user code changes. - -// Timestamp is the number of seconds since the epoch (1970-01-01 00:00 UTC) -// without leap seconds. +// Timestamp is the number of milliseconds since the epoch +// (1970-01-01 00:00 UTC) excluding leap seconds. type Timestamp int64 const ( @@ -36,6 +32,13 @@ const ( second = int64(native_time.Second / MinimumTick) // The number of nanoseconds per minimum tick. nanosPerTick = int64(MinimumTick / native_time.Nanosecond) + + // Earliest is the earliest timestamp representable. Handy for + // initializing a high watermark. + Earliest = Timestamp(math.MinInt64) + // Latest is the latest timestamp representable. Handy for initializing + // a low watermark. + Latest = Timestamp(math.MaxInt64) ) // Equal reports whether two timestamps represent the same instant. diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/README.md b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/README.md index e700e265f7c..81032bed886 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/README.md +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/README.md @@ -29,9 +29,10 @@ var ( Namespace: "my_company", Subsystem: "storage", Name: "documents_total_size_bytes", - Help: "The total size of all documents in the storage."}}) + Help: "The total size of all documents in the storage.", + }) ) - + func main() { http.Handle("/metrics", prometheus.Handler()) @@ -50,4 +51,3 @@ func init() { # Documentation [![GoDoc](https://godoc.org/github.com/prometheus/client_golang?status.png)](https://godoc.org/github.com/prometheus/client_golang) - diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/benchmark_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/benchmark_test.go index d43a857e63a..6ae7333fcc9 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/benchmark_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/benchmark_test.go @@ -129,3 +129,31 @@ func BenchmarkSummaryNoLabels(b *testing.B) { m.Observe(3.1415) } } + +func BenchmarkHistogramWithLabelValues(b *testing.B) { + m := NewHistogramVec( + HistogramOpts{ + Name: "benchmark_histogram", + Help: "A histogram to benchmark it.", + }, + []string{"one", "two", "three"}, + ) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.WithLabelValues("eins", "zwei", "drei").Observe(3.1415) + } +} + +func BenchmarkHistogramNoLabels(b *testing.B) { + m := NewHistogram(HistogramOpts{ + Name: "benchmark_histogram", + Help: "A histogram to benchmark it.", + }, + ) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Observe(3.1415) + } +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/counter.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/counter.go index d715ee0bb5f..f8d633fbd55 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/counter.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/counter.go @@ -74,7 +74,7 @@ func (c *counter) Add(v float64) { // CounterVec is a Collector that bundles a set of Counters that all share the // same Desc, but have different values for their variable labels. This is used // if you want to count the same thing partitioned by various dimensions -// (e.g. number of http requests, partitioned by response code and +// (e.g. number of HTTP requests, partitioned by response code and // method). Create instances with NewCounterVec. // // CounterVec embeds MetricVec. See there for a full list of methods with diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/desc.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/desc.go index e09f86df4b4..7e1f9e85372 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/desc.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/desc.go @@ -13,7 +13,7 @@ import ( dto "github.com/prometheus/client_model/go" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" ) var ( diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/example_selfcollector_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/example_selfcollector_test.go index d82917c5572..608deeb0279 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/example_selfcollector_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/example_selfcollector_test.go @@ -16,7 +16,7 @@ package prometheus_test import ( "runtime" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go index 2b0e7f43de5..5e62967b0be 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go @@ -23,7 +23,7 @@ import ( dto "github.com/prometheus/client_model/go" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" "github.com/prometheus/client_golang/prometheus" ) @@ -129,7 +129,7 @@ func ExampleCounterVec() { httpReqs := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "http_requests_total", - Help: "How many HTTP requests processed, partitioned by status code and http method.", + Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", ConstLabels: prometheus.Labels{"env": *binaryVersion}, }, []string{"code", "method"}, @@ -200,7 +200,7 @@ func ExampleRegister() { fmt.Println("taskCounter registered.") } // Don't forget to tell the HTTP server about the Prometheus handler. - // (In a real program, you still need to start the http server...) + // (In a real program, you still need to start the HTTP server...) http.Handle("/metrics", prometheus.Handler()) // Now you can start workers and give every one of them a pointer to @@ -240,7 +240,7 @@ func ExampleRegister() { // Prometheus will not allow you to ever export metrics with // inconsistent help strings or label names. After unregistering, the - // unregistered metrics will cease to show up in the /metrics http + // unregistered metrics will cease to show up in the /metrics HTTP // response, but the registry still remembers that those metrics had // been exported before. For this example, we will now choose a // different name. (In a real program, you would obviously not export @@ -452,3 +452,49 @@ func ExampleSummaryVec() { // > // ] } + +func ExampleHistogram() { + temps := prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "pond_temperature_celsius", + Help: "The temperature of the frog pond.", // Sorry, we can't measure how badly it smells. + Buckets: prometheus.LinearBuckets(20, 5, 5), // 5 buckets, each 5 centigrade wide. + }) + + // Simulate some observations. + for i := 0; i < 1000; i++ { + temps.Observe(30 + math.Floor(120*math.Sin(float64(i)*0.1))/10) + } + + // Just for demonstration, let's check the state of the histogram by + // (ab)using its Write method (which is usually only used by Prometheus + // internally). + metric := &dto.Metric{} + temps.Write(metric) + fmt.Println(proto.MarshalTextString(metric)) + + // Output: + // histogram: < + // sample_count: 1000 + // sample_sum: 29969.50000000001 + // bucket: < + // cumulative_count: 192 + // upper_bound: 20 + // > + // bucket: < + // cumulative_count: 366 + // upper_bound: 25 + // > + // bucket: < + // cumulative_count: 501 + // upper_bound: 30 + // > + // bucket: < + // cumulative_count: 638 + // upper_bound: 35 + // > + // bucket: < + // cumulative_count: 816 + // upper_bound: 40 + // > + // > +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram.go new file mode 100644 index 00000000000..9b36a611506 --- /dev/null +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram.go @@ -0,0 +1,344 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "fmt" + "hash/fnv" + "math" + "sort" + "sync/atomic" + + "github.com/golang/protobuf/proto" + + "github.com/prometheus/client_golang/model" + dto "github.com/prometheus/client_model/go" +) + +// A Histogram counts individual observations from an event or sample stream in +// configurable buckets. Similar to a summary, it also provides a sum of +// observations and an observation count. +// +// On the Prometheus server, quantiles can be calculated from a Histogram using +// the histogram_quantile function in the query language. +// +// Note that Histograms, in contrast to Summaries, can be aggregated with the +// Prometheus query language (see the documentation for detailed +// procedures). However, Histograms require the user to pre-define suitable +// buckets, and they are in general less accurate. The Observe method of a +// Histogram has a very low performance overhead in comparison with the Observe +// method of a Summary. +// +// To create Histogram instances, use NewHistogram. +type Histogram interface { + Metric + Collector + + // Observe adds a single observation to the histogram. + Observe(float64) +} + +var ( + // DefBuckets are the default Histogram buckets. The default buckets are + // tailored to broadly measure the response time (in seconds) of a + // network service. Most likely, however, you will be required to define + // buckets customized to your use case. + DefBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} + + errBucketLabelNotAllowed = fmt.Errorf( + "%q is not allowed as label name in histograms", model.BucketLabel, + ) +) + +// LinearBuckets creates 'count' buckets, each 'width' wide, where the lowest +// bucket has an upper bound of 'start'. The final +Inf bucket is not counted +// and not included in the returned slice. The returned slice is meant to be +// used for the Buckets field of HistogramOpts. +// +// The function panics if 'count' is zero or negative. +func LinearBuckets(start, width float64, count int) []float64 { + if count < 1 { + panic("LinearBuckets needs a positive count") + } + buckets := make([]float64, count) + for i := range buckets { + buckets[i] = start + start += width + } + return buckets +} + +// ExponentialBuckets creates 'count' buckets, where the lowest bucket has an +// upper bound of 'start' and each following bucket's upper bound is 'factor' +// times the previous bucket's upper bound. The final +Inf bucket is not counted +// and not included in the returned slice. The returned slice is meant to be +// used for the Buckets field of HistogramOpts. +// +// The function panics if 'count' is 0 or negative, if 'start' is 0 or negative, +// or if 'factor' is less than or equal 1. +func ExponentialBuckets(start, factor float64, count int) []float64 { + if count < 1 { + panic("ExponentialBuckets needs a positive count") + } + if start <= 0 { + panic("ExponentialBuckets needs a positive start value") + } + if factor <= 1 { + panic("ExponentialBuckets needs a factor greater than 1") + } + buckets := make([]float64, count) + for i := range buckets { + buckets[i] = start + start *= factor + } + return buckets +} + +// HistogramOpts bundles the options for creating a Histogram metric. It is +// mandatory to set Name and Help to a non-empty string. All other fields are +// optional and can safely be left at their zero value. +type HistogramOpts struct { + // Namespace, Subsystem, and Name are components of the fully-qualified + // name of the Histogram (created by joining these components with + // "_"). Only Name is mandatory, the others merely help structuring the + // name. Note that the fully-qualified name of the Histogram must be a + // valid Prometheus metric name. + Namespace string + Subsystem string + Name string + + // Help provides information about this Histogram. Mandatory! + // + // Metrics with the same fully-qualified name must have the same Help + // string. + Help string + + // ConstLabels are used to attach fixed labels to this + // Histogram. Histograms with the same fully-qualified name must have the + // same label names in their ConstLabels. + // + // Note that in most cases, labels have a value that varies during the + // lifetime of a process. Those labels are usually managed with a + // HistogramVec. ConstLabels serve only special purposes. One is for the + // special case where the value of a label does not change during the + // lifetime of a process, e.g. if the revision of the running binary is + // put into a label. Another, more advanced purpose is if more than one + // Collector needs to collect Histograms with the same fully-qualified + // name. In that case, those Summaries must differ in the values of + // their ConstLabels. See the Collector examples. + // + // If the value of a label never changes (not even between binaries), + // that label most likely should not be a label at all (but part of the + // metric name). + ConstLabels Labels + + // Buckets defines the buckets into which observations are counted. Each + // element in the slice is the upper inclusive bound of a bucket. The + // values must be sorted in strictly increasing order. There is no need + // to add a highest bucket with +Inf bound, it will be added + // implicitly. The default value is DefObjectives. + Buckets []float64 +} + +// NewHistogram creates a new Histogram based on the provided HistogramOpts. It +// panics if the buckets in HistogramOpts are not in strictly increasing order. +func NewHistogram(opts HistogramOpts) Histogram { + return newHistogram( + NewDesc( + BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + opts.Help, + nil, + opts.ConstLabels, + ), + opts, + ) +} + +func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogram { + if len(desc.variableLabels) != len(labelValues) { + panic(errInconsistentCardinality) + } + + for _, n := range desc.variableLabels { + if n == model.BucketLabel { + panic(errBucketLabelNotAllowed) + } + } + for _, lp := range desc.constLabelPairs { + if lp.GetName() == model.BucketLabel { + panic(errBucketLabelNotAllowed) + } + } + + if len(opts.Buckets) == 0 { + opts.Buckets = DefBuckets + } + + h := &histogram{ + desc: desc, + upperBounds: opts.Buckets, + labelPairs: makeLabelPairs(desc, labelValues), + } + for i, upperBound := range h.upperBounds { + if i < len(h.upperBounds)-1 { + if upperBound >= h.upperBounds[i+1] { + panic(fmt.Errorf( + "histogram buckets must be in increasing order: %f >= %f", + upperBound, h.upperBounds[i+1], + )) + } + } else { + if math.IsInf(upperBound, +1) { + // The +Inf bucket is implicit. Remove it here. + h.upperBounds = h.upperBounds[:i] + } + } + } + // Finally we know the final length of h.upperBounds and can make counts. + h.counts = make([]uint64, len(h.upperBounds)) + + h.Init(h) // Init self-collection. + return h +} + +type histogram struct { + SelfCollector + // Note that there is no mutex required. + + desc *Desc + + upperBounds []float64 + counts []uint64 + + labelPairs []*dto.LabelPair + + sumBits uint64 // The bits of the float64 representing the sum of all observations. + count uint64 +} + +func (h *histogram) Desc() *Desc { + return h.desc +} + +func (h *histogram) Observe(v float64) { + // TODO(beorn7): For small numbers of buckets (<30), a linear search is + // slightly faster than the binary search. If we really care, we could + // switch from one search strategy to the other depending on the number + // of buckets. + // + // Microbenchmarks (BenchmarkHistogramNoLabels): + // 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op + // 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op + // 300 buckets: 154 ns/op linear - binary 61.6 ns/op + i := sort.SearchFloat64s(h.upperBounds, v) + if i < len(h.counts) { + atomic.AddUint64(&h.counts[i], 1) + } + atomic.AddUint64(&h.count, 1) + for { + oldBits := atomic.LoadUint64(&h.sumBits) + newBits := math.Float64bits(math.Float64frombits(oldBits) + v) + if atomic.CompareAndSwapUint64(&h.sumBits, oldBits, newBits) { + break + } + } +} + +func (h *histogram) Write(out *dto.Metric) error { + his := &dto.Histogram{} + buckets := make([]*dto.Bucket, len(h.upperBounds)) + + his.SampleSum = proto.Float64(math.Float64frombits(atomic.LoadUint64(&h.sumBits))) + his.SampleCount = proto.Uint64(atomic.LoadUint64(&h.count)) + var count uint64 + for i, upperBound := range h.upperBounds { + count += atomic.LoadUint64(&h.counts[i]) + buckets[i] = &dto.Bucket{ + CumulativeCount: proto.Uint64(count), + UpperBound: proto.Float64(upperBound), + } + } + his.Bucket = buckets + out.Histogram = his + out.Label = h.labelPairs + return nil +} + +// HistogramVec is a Collector that bundles a set of Histograms that all share the +// same Desc, but have different values for their variable labels. This is used +// if you want to count the same thing partitioned by various dimensions +// (e.g. HTTP request latencies, partitioned by status code and method). Create +// instances with NewHistogramVec. +type HistogramVec struct { + MetricVec +} + +// NewHistogramVec creates a new HistogramVec based on the provided HistogramOpts and +// partitioned by the given label names. At least one label name must be +// provided. +func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec { + desc := NewDesc( + BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + opts.Help, + labelNames, + opts.ConstLabels, + ) + return &HistogramVec{ + MetricVec: MetricVec{ + children: map[uint64]Metric{}, + desc: desc, + hash: fnv.New64a(), + newMetric: func(lvs ...string) Metric { + return newHistogram(desc, opts, lvs...) + }, + }, + } +} + +// GetMetricWithLabelValues replaces the method of the same name in +// MetricVec. The difference is that this method returns a Histogram and not a +// Metric so that no type conversion is required. +func (m *HistogramVec) GetMetricWithLabelValues(lvs ...string) (Histogram, error) { + metric, err := m.MetricVec.GetMetricWithLabelValues(lvs...) + if metric != nil { + return metric.(Histogram), err + } + return nil, err +} + +// GetMetricWith replaces the method of the same name in MetricVec. The +// difference is that this method returns a Histogram and not a Metric so that no +// type conversion is required. +func (m *HistogramVec) GetMetricWith(labels Labels) (Histogram, error) { + metric, err := m.MetricVec.GetMetricWith(labels) + if metric != nil { + return metric.(Histogram), err + } + return nil, err +} + +// WithLabelValues works as GetMetricWithLabelValues, but panics where +// GetMetricWithLabelValues would have returned an error. By not returning an +// error, WithLabelValues allows shortcuts like +// myVec.WithLabelValues("404", "GET").Observe(42.21) +func (m *HistogramVec) WithLabelValues(lvs ...string) Histogram { + return m.MetricVec.WithLabelValues(lvs...).(Histogram) +} + +// With works as GetMetricWith, but panics where GetMetricWithLabels would have +// returned an error. By not returning an error, With allows shortcuts like +// myVec.With(Labels{"code": "404", "method": "GET"}).Observe(42.21) +func (m *HistogramVec) With(labels Labels) Histogram { + return m.MetricVec.With(labels).(Histogram) +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram_test.go new file mode 100644 index 00000000000..855af469501 --- /dev/null +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram_test.go @@ -0,0 +1,318 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "math" + "math/rand" + "reflect" + "sort" + "sync" + "testing" + "testing/quick" + + dto "github.com/prometheus/client_model/go" +) + +func benchmarkHistogramObserve(w int, b *testing.B) { + b.StopTimer() + + wg := new(sync.WaitGroup) + wg.Add(w) + + g := new(sync.WaitGroup) + g.Add(1) + + s := NewHistogram(HistogramOpts{}) + + for i := 0; i < w; i++ { + go func() { + g.Wait() + + for i := 0; i < b.N; i++ { + s.Observe(float64(i)) + } + + wg.Done() + }() + } + + b.StartTimer() + g.Done() + wg.Wait() +} + +func BenchmarkHistogramObserve1(b *testing.B) { + benchmarkHistogramObserve(1, b) +} + +func BenchmarkHistogramObserve2(b *testing.B) { + benchmarkHistogramObserve(2, b) +} + +func BenchmarkHistogramObserve4(b *testing.B) { + benchmarkHistogramObserve(4, b) +} + +func BenchmarkHistogramObserve8(b *testing.B) { + benchmarkHistogramObserve(8, b) +} + +func benchmarkHistogramWrite(w int, b *testing.B) { + b.StopTimer() + + wg := new(sync.WaitGroup) + wg.Add(w) + + g := new(sync.WaitGroup) + g.Add(1) + + s := NewHistogram(HistogramOpts{}) + + for i := 0; i < 1000000; i++ { + s.Observe(float64(i)) + } + + for j := 0; j < w; j++ { + outs := make([]dto.Metric, b.N) + + go func(o []dto.Metric) { + g.Wait() + + for i := 0; i < b.N; i++ { + s.Write(&o[i]) + } + + wg.Done() + }(outs) + } + + b.StartTimer() + g.Done() + wg.Wait() +} + +func BenchmarkHistogramWrite1(b *testing.B) { + benchmarkHistogramWrite(1, b) +} + +func BenchmarkHistogramWrite2(b *testing.B) { + benchmarkHistogramWrite(2, b) +} + +func BenchmarkHistogramWrite4(b *testing.B) { + benchmarkHistogramWrite(4, b) +} + +func BenchmarkHistogramWrite8(b *testing.B) { + benchmarkHistogramWrite(8, b) +} + +// Intentionally adding +Inf here to test if that case is handled correctly. +// Also, getCumulativeCounts depends on it. +var testBuckets = []float64{-2, -1, -0.5, 0, 0.5, 1, 2, math.Inf(+1)} + +func TestHistogramConcurrency(t *testing.T) { + rand.Seed(42) + + it := func(n uint32) bool { + mutations := int(n%1e4 + 1e4) + concLevel := int(n%5 + 1) + total := mutations * concLevel + + var start, end sync.WaitGroup + start.Add(1) + end.Add(concLevel) + + sum := NewHistogram(HistogramOpts{ + Name: "test_histogram", + Help: "helpless", + Buckets: testBuckets, + }) + + allVars := make([]float64, total) + var sampleSum float64 + for i := 0; i < concLevel; i++ { + vals := make([]float64, mutations) + for j := 0; j < mutations; j++ { + v := rand.NormFloat64() + vals[j] = v + allVars[i*mutations+j] = v + sampleSum += v + } + + go func(vals []float64) { + start.Wait() + for _, v := range vals { + sum.Observe(v) + } + end.Done() + }(vals) + } + sort.Float64s(allVars) + start.Done() + end.Wait() + + m := &dto.Metric{} + sum.Write(m) + if got, want := int(*m.Histogram.SampleCount), total; got != want { + t.Errorf("got sample count %d, want %d", got, want) + } + if got, want := *m.Histogram.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 { + t.Errorf("got sample sum %f, want %f", got, want) + } + + wantCounts := getCumulativeCounts(allVars) + + if got, want := len(m.Histogram.Bucket), len(testBuckets)-1; got != want { + t.Errorf("got %d buckets in protobuf, want %d", got, want) + } + for i, wantBound := range testBuckets { + if i == len(testBuckets)-1 { + break // No +Inf bucket in protobuf. + } + if gotBound := *m.Histogram.Bucket[i].UpperBound; gotBound != wantBound { + t.Errorf("got bound %f, want %f", gotBound, wantBound) + } + if gotCount, wantCount := *m.Histogram.Bucket[i].CumulativeCount, wantCounts[i]; gotCount != wantCount { + t.Errorf("got count %d, want %d", gotCount, wantCount) + } + } + return true + } + + if err := quick.Check(it, nil); err != nil { + t.Error(err) + } +} + +func TestHistogramVecConcurrency(t *testing.T) { + rand.Seed(42) + + objectives := make([]float64, 0, len(DefObjectives)) + for qu := range DefObjectives { + + objectives = append(objectives, qu) + } + sort.Float64s(objectives) + + it := func(n uint32) bool { + mutations := int(n%1e4 + 1e4) + concLevel := int(n%7 + 1) + vecLength := int(n%3 + 1) + + var start, end sync.WaitGroup + start.Add(1) + end.Add(concLevel) + + his := NewHistogramVec( + HistogramOpts{ + Name: "test_histogram", + Help: "helpless", + Buckets: []float64{-2, -1, -0.5, 0, 0.5, 1, 2, math.Inf(+1)}, + }, + []string{"label"}, + ) + + allVars := make([][]float64, vecLength) + sampleSums := make([]float64, vecLength) + for i := 0; i < concLevel; i++ { + vals := make([]float64, mutations) + picks := make([]int, mutations) + for j := 0; j < mutations; j++ { + v := rand.NormFloat64() + vals[j] = v + pick := rand.Intn(vecLength) + picks[j] = pick + allVars[pick] = append(allVars[pick], v) + sampleSums[pick] += v + } + + go func(vals []float64) { + start.Wait() + for i, v := range vals { + his.WithLabelValues(string('A' + picks[i])).Observe(v) + } + end.Done() + }(vals) + } + for _, vars := range allVars { + sort.Float64s(vars) + } + start.Done() + end.Wait() + + for i := 0; i < vecLength; i++ { + m := &dto.Metric{} + s := his.WithLabelValues(string('A' + i)) + s.Write(m) + + if got, want := len(m.Histogram.Bucket), len(testBuckets)-1; got != want { + t.Errorf("got %d buckets in protobuf, want %d", got, want) + } + if got, want := int(*m.Histogram.SampleCount), len(allVars[i]); got != want { + t.Errorf("got sample count %d, want %d", got, want) + } + if got, want := *m.Histogram.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 { + t.Errorf("got sample sum %f, want %f", got, want) + } + + wantCounts := getCumulativeCounts(allVars[i]) + + for j, wantBound := range testBuckets { + if j == len(testBuckets)-1 { + break // No +Inf bucket in protobuf. + } + if gotBound := *m.Histogram.Bucket[j].UpperBound; gotBound != wantBound { + t.Errorf("got bound %f, want %f", gotBound, wantBound) + } + if gotCount, wantCount := *m.Histogram.Bucket[j].CumulativeCount, wantCounts[j]; gotCount != wantCount { + t.Errorf("got count %d, want %d", gotCount, wantCount) + } + } + } + return true + } + + if err := quick.Check(it, nil); err != nil { + t.Error(err) + } +} + +func getCumulativeCounts(vars []float64) []uint64 { + counts := make([]uint64, len(testBuckets)) + for _, v := range vars { + for i := len(testBuckets) - 1; i >= 0; i-- { + if v > testBuckets[i] { + break + } + counts[i]++ + } + } + return counts +} + +func TestBuckets(t *testing.T) { + got := LinearBuckets(-15, 5, 6) + want := []float64{-15, -10, -5, 0, 5, 10} + if !reflect.DeepEqual(got, want) { + t.Errorf("linear buckets: got %v, want %v", got, want) + } + + got = ExponentialBuckets(100, 1.2, 3) + want = []float64{100, 120, 144} + if !reflect.DeepEqual(got, want) { + t.Errorf("linear buckets: got %v, want %v", got, want) + } +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/http.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/http.go index 818c90fb65f..dac92fd907c 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/http.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/http.go @@ -47,7 +47,7 @@ func nowSeries(t ...time.Time) nower { } // InstrumentHandler wraps the given HTTP handler for instrumentation. It -// registers four metric collectors (if not already done) and reports http +// registers four metric collectors (if not already done) and reports HTTP // metrics to the (newly or already) registered collectors: http_requests_total // (CounterVec), http_request_duration_microseconds (Summary), // http_request_size_bytes (Summary), http_response_size_bytes (Summary). Each diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_procfs.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_procfs.go index fab1f03d56a..5a09ded1478 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_procfs.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/process_collector_procfs.go @@ -24,26 +24,20 @@ func processCollectSupported() bool { return false } +// TODO(ts): Bring back error reporting by reverting 7faf9e7 as soon as the +// client allows users to configure the error behavior. func (c *processCollector) processCollect(ch chan<- Metric) { pid, err := c.pidFn() if err != nil { - c.reportCollectErrors(ch, err) return } p, err := procfs.NewProc(pid) if err != nil { - c.reportCollectErrors(ch, err) return } - if stat, err := p.NewStat(); err != nil { - // Report collect errors for metrics depending on stat. - ch <- NewInvalidMetric(c.vsize.Desc(), err) - ch <- NewInvalidMetric(c.rss.Desc(), err) - ch <- NewInvalidMetric(c.startTime.Desc(), err) - ch <- NewInvalidMetric(c.cpuTotal.Desc(), err) - } else { + if stat, err := p.NewStat(); err == nil { c.cpuTotal.Set(stat.CPUTime()) ch <- c.cpuTotal c.vsize.Set(float64(stat.VirtualMemory())) @@ -51,34 +45,19 @@ func (c *processCollector) processCollect(ch chan<- Metric) { c.rss.Set(float64(stat.ResidentMemory())) ch <- c.rss - if startTime, err := stat.StartTime(); err != nil { - ch <- NewInvalidMetric(c.startTime.Desc(), err) - } else { + if startTime, err := stat.StartTime(); err == nil { c.startTime.Set(startTime) ch <- c.startTime } } - if fds, err := p.FileDescriptorsLen(); err != nil { - ch <- NewInvalidMetric(c.openFDs.Desc(), err) - } else { + if fds, err := p.FileDescriptorsLen(); err == nil { c.openFDs.Set(float64(fds)) ch <- c.openFDs } - if limits, err := p.NewLimits(); err != nil { - ch <- NewInvalidMetric(c.maxFDs.Desc(), err) - } else { + if limits, err := p.NewLimits(); err == nil { c.maxFDs.Set(float64(limits.OpenFiles)) ch <- c.maxFDs } } - -func (c *processCollector) reportCollectErrors(ch chan<- Metric, err error) { - ch <- NewInvalidMetric(c.cpuTotal.Desc(), err) - ch <- NewInvalidMetric(c.openFDs.Desc(), err) - ch <- NewInvalidMetric(c.maxFDs.Desc(), err) - ch <- NewInvalidMetric(c.vsize.Desc(), err) - ch <- NewInvalidMetric(c.rss.Desc(), err) - ch <- NewInvalidMetric(c.startTime.Desc(), err) -} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry.go index 50d3a5816bc..550754172f7 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry.go @@ -33,11 +33,11 @@ import ( "strings" "sync" + "bitbucket.org/ww/goautoneg" + "github.com/golang/protobuf/proto" + dto "github.com/prometheus/client_model/go" - "code.google.com/p/goprotobuf/proto" - - "github.com/prometheus/client_golang/_vendor/goautoneg" "github.com/prometheus/client_golang/model" "github.com/prometheus/client_golang/text" ) @@ -171,7 +171,7 @@ func SetMetricFamilyInjectionHook(hook func() []*dto.MetricFamily) { } // PanicOnCollectError sets the behavior whether a panic is caused upon an error -// while metrics are collected and served to the http endpoint. By default, an +// while metrics are collected and served to the HTTP endpoint. By default, an // internal server error (status code 500) is served with an error message. func PanicOnCollectError(b bool) { defRegistry.panicOnCollectError = b @@ -464,6 +464,8 @@ func (r *registry) writePB(w io.Writer, writeEncoded encoder) (int, error) { metricFamily.Type = dto.MetricType_SUMMARY.Enum() case dtoMetric.Untyped != nil: metricFamily.Type = dto.MetricType_UNTYPED.Enum() + case dtoMetric.Histogram != nil: + metricFamily.Type = dto.MetricType_HISTOGRAM.Enum() default: return 0, fmt.Errorf("empty metric collected: %s", dtoMetric) } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry_test.go index b3603cb9f6a..95579e512b4 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/registry_test.go @@ -25,7 +25,7 @@ import ( "net/http" "testing" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go index 812a5dee70f..a731f4e8d34 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go @@ -20,11 +20,12 @@ import ( "sync" "time" - "code.google.com/p/goprotobuf/proto" + "github.com/beorn7/perks/quantile" + "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" - "github.com/prometheus/client_golang/_vendor/perks/quantile" + "github.com/prometheus/client_golang/model" ) // A Summary captures individual observations from an event or sample stream and @@ -35,6 +36,12 @@ import ( // Summary provides the median, the 90th and the 99th percentile of the latency // as rank estimations. // +// Note that the rank estimations cannot be aggregated in a meaningful way with +// the Prometheus query language (i.e. you cannot average or add them). If you +// need aggregatable quantiles (e.g. you want the 99th percentile latency of all +// queries served across all instances of a service), consider the Histogram +// metric type. See the Prometheus documentation for more details. +// // To create Summary instances, use NewSummary. type Summary interface { Metric @@ -44,9 +51,13 @@ type Summary interface { Observe(float64) } -// DefObjectives are the default Summary quantile values. var ( + // DefObjectives are the default Summary quantile values. DefObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001} + + errQuantileLabelNotAllowed = fmt.Errorf( + "%q is not allowed as label name in summaries", model.QuantileLabel, + ) ) // Default values for SummaryOpts. @@ -110,7 +121,10 @@ type SummaryOpts struct { // AgeBuckets is the number of buckets used to exclude observations that // are older than MaxAge from the summary. A higher number has a // resource penalty, so only increase it if the higher resolution is - // really required. The default value is DefAgeBuckets. + // really required. For very high observation rates, you might want to + // reduce the number of age buckets. With only one age bucket, you will + // effectively see a complete reset of the summary each time MaxAge has + // passed. The default value is DefAgeBuckets. AgeBuckets uint32 // BufCap defines the default sample stream buffer size. The default @@ -119,10 +133,6 @@ type SummaryOpts struct { // is the internal buffer size of the underlying package // "github.com/bmizerany/perks/quantile"). BufCap uint32 - - // Epsilon is the error epsilon for the quantile rank estimate. Must be - // positive. The default is DefEpsilon. - Epsilon float64 } // TODO: Great fuck-up with the sliding-window decay algorithm... The Merge @@ -158,6 +168,17 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary { panic(errInconsistentCardinality) } + for _, n := range desc.variableLabels { + if n == model.QuantileLabel { + panic(errQuantileLabelNotAllowed) + } + } + for _, lp := range desc.constLabelPairs { + if lp.GetName() == model.QuantileLabel { + panic(errQuantileLabelNotAllowed) + } + } + if len(opts.Objectives) == 0 { opts.Objectives = DefObjectives } @@ -358,7 +379,7 @@ func (s quantSort) Less(i, j int) bool { // SummaryVec is a Collector that bundles a set of Summaries that all share the // same Desc, but have different values for their variable labels. This is used // if you want to count the same thing partitioned by various dimensions -// (e.g. http request latencies, partitioned by status code and method). Create +// (e.g. HTTP request latencies, partitioned by status code and method). Create // instances with NewSummaryVec. type SummaryVec struct { MetricVec @@ -411,14 +432,14 @@ func (m *SummaryVec) GetMetricWith(labels Labels) (Summary, error) { // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. By not returning an // error, WithLabelValues allows shortcuts like -// myVec.WithLabelValues("404", "GET").Add(42) +// myVec.WithLabelValues("404", "GET").Observe(42.21) func (m *SummaryVec) WithLabelValues(lvs ...string) Summary { return m.MetricVec.WithLabelValues(lvs...).(Summary) } // With works as GetMetricWith, but panics where GetMetricWithLabels would have // returned an error. By not returning an error, With allows shortcuts like -// myVec.With(Labels{"code": "404", "method": "GET"}).Add(42) +// myVec.With(Labels{"code": "404", "method": "GET"}).Observe(42.21) func (m *SummaryVec) With(labels Labels) Summary { return m.MetricVec.With(labels).(Summary) } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary_test.go index 3f83796ea68..40d05fae52f 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary_test.go @@ -120,6 +120,10 @@ func BenchmarkSummaryWrite8(b *testing.B) { } func TestSummaryConcurrency(t *testing.T) { + if testing.Short() { + t.Skip("Skipping test in short mode.") + } + rand.Seed(42) it := func(n uint32) bool { @@ -195,6 +199,10 @@ func TestSummaryConcurrency(t *testing.T) { } func TestSummaryVecConcurrency(t *testing.T) { + if testing.Short() { + t.Skip("Skipping test in short mode.") + } + rand.Seed(42) objectives := make([]float64, 0, len(DefObjectives)) diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/value.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/value.go index 2db1d11fb94..107d43e3721 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/value.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/value.go @@ -22,7 +22,7 @@ import ( dto "github.com/prometheus/client_model/go" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" ) // ValueType is an enumeration of metric types that represent a simple value. diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/create.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/create.go index 6aaa2811dbd..44304599466 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/create.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/create.go @@ -24,8 +24,10 @@ import ( "bytes" "fmt" "io" + "math" "strings" + "github.com/prometheus/client_golang/model" dto "github.com/prometheus/client_model/go" ) @@ -116,7 +118,7 @@ func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (int, error) { for _, q := range metric.Summary.Quantile { n, err = writeSample( name, metric, - "quantile", fmt.Sprint(q.GetQuantile()), + model.QuantileLabel, fmt.Sprint(q.GetQuantile()), q.GetValue(), out, ) @@ -139,6 +141,54 @@ func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (int, error) { float64(metric.Summary.GetSampleCount()), out, ) + case dto.MetricType_HISTOGRAM: + if metric.Histogram == nil { + return written, fmt.Errorf( + "expected summary in metric %s", metric, + ) + } + infSeen := false + for _, q := range metric.Histogram.Bucket { + n, err = writeSample( + name+"_bucket", metric, + model.BucketLabel, fmt.Sprint(q.GetUpperBound()), + float64(q.GetCumulativeCount()), + out, + ) + written += n + if err != nil { + return written, err + } + if math.IsInf(q.GetUpperBound(), +1) { + infSeen = true + } + } + if !infSeen { + n, err = writeSample( + name+"_bucket", metric, + model.BucketLabel, "+Inf", + float64(metric.Histogram.GetSampleCount()), + out, + ) + if err != nil { + return written, err + } + written += n + } + n, err = writeSample( + name+"_sum", metric, "", "", + metric.Histogram.GetSampleSum(), + out, + ) + if err != nil { + return written, err + } + written += n + n, err = writeSample( + name+"_count", metric, "", "", + float64(metric.Histogram.GetSampleCount()), + out, + ) default: return written, fmt.Errorf( "unexpected type in metric %s", metric, diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/create_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/create_test.go index 00b67413206..fe938de80c7 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/create_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/create_test.go @@ -19,7 +19,7 @@ import ( "strings" "testing" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) @@ -219,6 +219,98 @@ summary_name{name_1="value 1",name_2="value 2",quantile="0.9"} 2 summary_name{name_1="value 1",name_2="value 2",quantile="0.99"} 3 summary_name_sum{name_1="value 1",name_2="value 2"} 2010.1971 summary_name_count{name_1="value 1",name_2="value 2"} 4711 +`, + }, + // 4: Histogram + { + in: &dto.MetricFamily{ + Name: proto.String("request_duration_microseconds"), + Help: proto.String("The response latency."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + &dto.Metric{ + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(2693), + SampleSum: proto.Float64(1756047.3), + Bucket: []*dto.Bucket{ + &dto.Bucket{ + UpperBound: proto.Float64(100), + CumulativeCount: proto.Uint64(123), + }, + &dto.Bucket{ + UpperBound: proto.Float64(120), + CumulativeCount: proto.Uint64(412), + }, + &dto.Bucket{ + UpperBound: proto.Float64(144), + CumulativeCount: proto.Uint64(592), + }, + &dto.Bucket{ + UpperBound: proto.Float64(172.8), + CumulativeCount: proto.Uint64(1524), + }, + &dto.Bucket{ + UpperBound: proto.Float64(math.Inf(+1)), + CumulativeCount: proto.Uint64(2693), + }, + }, + }, + }, + }, + }, + out: `# HELP request_duration_microseconds The response latency. +# TYPE request_duration_microseconds histogram +request_duration_microseconds_bucket{le="100"} 123 +request_duration_microseconds_bucket{le="120"} 412 +request_duration_microseconds_bucket{le="144"} 592 +request_duration_microseconds_bucket{le="172.8"} 1524 +request_duration_microseconds_bucket{le="+Inf"} 2693 +request_duration_microseconds_sum 1.7560473e+06 +request_duration_microseconds_count 2693 +`, + }, + // 5: Histogram with missing +Inf bucket. + { + in: &dto.MetricFamily{ + Name: proto.String("request_duration_microseconds"), + Help: proto.String("The response latency."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + &dto.Metric{ + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(2693), + SampleSum: proto.Float64(1756047.3), + Bucket: []*dto.Bucket{ + &dto.Bucket{ + UpperBound: proto.Float64(100), + CumulativeCount: proto.Uint64(123), + }, + &dto.Bucket{ + UpperBound: proto.Float64(120), + CumulativeCount: proto.Uint64(412), + }, + &dto.Bucket{ + UpperBound: proto.Float64(144), + CumulativeCount: proto.Uint64(592), + }, + &dto.Bucket{ + UpperBound: proto.Float64(172.8), + CumulativeCount: proto.Uint64(1524), + }, + }, + }, + }, + }, + }, + out: `# HELP request_duration_microseconds The response latency. +# TYPE request_duration_microseconds histogram +request_duration_microseconds_bucket{le="100"} 123 +request_duration_microseconds_bucket{le="120"} 412 +request_duration_microseconds_bucket{le="144"} 592 +request_duration_microseconds_bucket{le="172.8"} 1524 +request_duration_microseconds_bucket{le="+Inf"} 2693 +request_duration_microseconds_sum 1.7560473e+06 +request_duration_microseconds_count 2693 `, }, } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse.go index bbfb887b85f..e317d6850be 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse.go @@ -24,7 +24,7 @@ import ( dto "github.com/prometheus/client_model/go" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" "github.com/prometheus/client_golang/model" ) @@ -59,14 +59,19 @@ type Parser struct { currentMetric *dto.Metric currentLabelPair *dto.LabelPair - // The remaining member variables are only used for summaries. + // The remaining member variables are only used for summaries/histograms. + currentLabels map[string]string // All labels including '__name__' but excluding 'quantile'/'le' + // Summary specific. summaries map[uint64]*dto.Metric // Key is created with LabelsToSignature. - currentLabels map[string]string // All labels including '__name__' but excluding 'quantile'. currentQuantile float64 + // Histogram specific. + histograms map[uint64]*dto.Metric // Key is created with LabelsToSignature. + currentBucket float64 // These tell us if the currently processed line ends on '_count' or - // '_sum' respectively and belong to a summary, representing the sample - // count and sum of that summary. - currentIsSummaryCount, currentIsSummarySum bool + // '_sum' respectively and belong to a summary/histogram, representing the sample + // count and sum of that summary/histogram. + currentIsSummaryCount, currentIsSummarySum bool + currentIsHistogramCount, currentIsHistogramSum bool } // TextToMetricFamilies reads 'in' as the simple and flat text-based exchange @@ -111,7 +116,11 @@ func (p *Parser) reset(in io.Reader) { if p.summaries == nil || len(p.summaries) > 0 { p.summaries = map[uint64]*dto.Metric{} } + if p.histograms == nil || len(p.histograms) > 0 { + p.histograms = map[uint64]*dto.Metric{} + } p.currentQuantile = math.NaN() + p.currentBucket = math.NaN() } // startOfLine represents the state where the next byte read from p.buf is the @@ -224,13 +233,14 @@ func (p *Parser) readingMetricName() stateFn { // p.currentByte) is either the first byte of the label set (i.e. a '{'), or the // first byte of the value (otherwise). func (p *Parser) readingLabels() stateFn { - // Alas, summaries are really special... We have to reset the - // currentLabels map and the currentQuantile before starting to + // Summaries/histograms are special. We have to reset the + // currentLabels map, currentQuantile and currentBucket before starting to // read labels. - if p.currentMF.GetType() == dto.MetricType_SUMMARY { + if p.currentMF.GetType() == dto.MetricType_SUMMARY || p.currentMF.GetType() == dto.MetricType_HISTOGRAM { p.currentLabels = map[string]string{} p.currentLabels[string(model.MetricNameLabel)] = p.currentMF.GetName() p.currentQuantile = math.NaN() + p.currentBucket = math.NaN() } if p.currentByte != '{' { return p.readingValue @@ -262,10 +272,10 @@ func (p *Parser) startLabelName() stateFn { p.parseError(fmt.Sprintf("label name %q is reserved", model.MetricNameLabel)) return nil } - // Once more, special summary treatment... Don't add 'quantile' + // Special summary/histogram treatment. Don't add 'quantile' and 'le' // labels to 'real' labels. - if p.currentMF.GetType() != dto.MetricType_SUMMARY || - p.currentLabelPair.GetName() != "quantile" { + if !(p.currentMF.GetType() == dto.MetricType_SUMMARY && p.currentLabelPair.GetName() == model.QuantileLabel) && + !(p.currentMF.GetType() == dto.MetricType_HISTOGRAM && p.currentLabelPair.GetName() == model.BucketLabel) { p.currentMetric.Label = append(p.currentMetric.Label, p.currentLabelPair) } if p.skipBlankTabIfCurrentBlankTab(); p.err != nil { @@ -292,14 +302,26 @@ func (p *Parser) startLabelValue() stateFn { return nil } p.currentLabelPair.Value = proto.String(p.currentToken.String()) - // Once more, special treatment of summaries: + // Special treatment of summaries: // - Quantile labels are special, will result in dto.Quantile later. // - Other labels have to be added to currentLabels for signature calculation. if p.currentMF.GetType() == dto.MetricType_SUMMARY { - if p.currentLabelPair.GetName() == "quantile" { + if p.currentLabelPair.GetName() == model.QuantileLabel { if p.currentQuantile, p.err = strconv.ParseFloat(p.currentLabelPair.GetValue(), 64); p.err != nil { // Create a more helpful error message. - p.parseError(fmt.Sprintf("expected float as value for quantile label, got %q", p.currentLabelPair.GetValue())) + p.parseError(fmt.Sprintf("expected float as value for 'quantile' label, got %q", p.currentLabelPair.GetValue())) + return nil + } + } else { + p.currentLabels[p.currentLabelPair.GetName()] = p.currentLabelPair.GetValue() + } + } + // Similar special treatment of histograms. + if p.currentMF.GetType() == dto.MetricType_HISTOGRAM { + if p.currentLabelPair.GetName() == model.BucketLabel { + if p.currentBucket, p.err = strconv.ParseFloat(p.currentLabelPair.GetValue(), 64); p.err != nil { + // Create a more helpful error message. + p.parseError(fmt.Sprintf("expected float as value for 'le' label, got %q", p.currentLabelPair.GetValue())) return nil } } else { @@ -328,7 +350,7 @@ func (p *Parser) startLabelValue() stateFn { // p.currentByte) is the first byte of the sample value (i.e. a float). func (p *Parser) readingValue() stateFn { // When we are here, we have read all the labels, so for the - // infamous special case of a summary, we can finally find out + // special case of a summary/histogram, we can finally find out // if the metric already exists. if p.currentMF.GetType() == dto.MetricType_SUMMARY { signature := model.LabelsToSignature(p.currentLabels) @@ -338,6 +360,14 @@ func (p *Parser) readingValue() stateFn { p.summaries[signature] = p.currentMetric p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) } + } else if p.currentMF.GetType() == dto.MetricType_HISTOGRAM { + signature := model.LabelsToSignature(p.currentLabels) + if histogram := p.histograms[signature]; histogram != nil { + p.currentMetric = histogram + } else { + p.histograms[signature] = p.currentMetric + p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) + } } else { p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) } @@ -376,6 +406,25 @@ func (p *Parser) readingValue() stateFn { }, ) } + case dto.MetricType_HISTOGRAM: + // *sigh* + if p.currentMetric.Histogram == nil { + p.currentMetric.Histogram = &dto.Histogram{} + } + switch { + case p.currentIsHistogramCount: + p.currentMetric.Histogram.SampleCount = proto.Uint64(uint64(value)) + case p.currentIsHistogramSum: + p.currentMetric.Histogram.SampleSum = proto.Float64(value) + case !math.IsNaN(p.currentBucket): + p.currentMetric.Histogram.Bucket = append( + p.currentMetric.Histogram.Bucket, + &dto.Bucket{ + UpperBound: proto.Float64(p.currentBucket), + CumulativeCount: proto.Uint64(uint64(value)), + }, + ) + } default: p.err = fmt.Errorf("unexpected type for metric name %q", p.currentMF.GetName()) } @@ -598,11 +647,13 @@ func (p *Parser) readTokenAsLabelValue() { func (p *Parser) setOrCreateCurrentMF() { p.currentIsSummaryCount = false p.currentIsSummarySum = false + p.currentIsHistogramCount = false + p.currentIsHistogramSum = false name := p.currentToken.String() if p.currentMF = p.metricFamiliesByName[name]; p.currentMF != nil { return } - // Try out if this is a _sum or _count for a summary. + // Try out if this is a _sum or _count for a summary/histogram. summaryName := summaryMetricName(name) if p.currentMF = p.metricFamiliesByName[summaryName]; p.currentMF != nil { if p.currentMF.GetType() == dto.MetricType_SUMMARY { @@ -615,6 +666,18 @@ func (p *Parser) setOrCreateCurrentMF() { return } } + histogramName := histogramMetricName(name) + if p.currentMF = p.metricFamiliesByName[histogramName]; p.currentMF != nil { + if p.currentMF.GetType() == dto.MetricType_HISTOGRAM { + if isCount(name) { + p.currentIsHistogramCount = true + } + if isSum(name) { + p.currentIsHistogramSum = true + } + return + } + } p.currentMF = &dto.MetricFamily{Name: proto.String(name)} p.metricFamiliesByName[name] = p.currentMF } @@ -647,6 +710,10 @@ func isSum(name string) bool { return len(name) > 4 && name[len(name)-4:] == "_sum" } +func isBucket(name string) bool { + return len(name) > 7 && name[len(name)-7:] == "_bucket" +} + func summaryMetricName(name string) string { switch { case isCount(name): @@ -657,3 +724,16 @@ func summaryMetricName(name string) string { return name } } + +func histogramMetricName(name string) string { + switch { + case isCount(name): + return name[:len(name)-6] + case isSum(name): + return name[:len(name)-4] + case isBucket(name): + return name[:len(name)-7] + default: + return name + } +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse_test.go index af938390bd0..6b7adfff557 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/parse_test.go @@ -18,7 +18,7 @@ import ( "strings" "testing" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) @@ -335,6 +335,57 @@ my_summary{n1="val3", quantile="0.2"} 4711 }, }, }, + // 4: The histogram. + { + in: ` +# HELP request_duration_microseconds The response latency. +# TYPE request_duration_microseconds histogram +request_duration_microseconds_bucket{le="100"} 123 +request_duration_microseconds_bucket{le="120"} 412 +request_duration_microseconds_bucket{le="144"} 592 +request_duration_microseconds_bucket{le="172.8"} 1524 +request_duration_microseconds_bucket{le="+Inf"} 2693 +request_duration_microseconds_sum 1.7560473e+06 +request_duration_microseconds_count 2693 +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("request_duration_microseconds"), + Help: proto.String("The response latency."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + &dto.Metric{ + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(2693), + SampleSum: proto.Float64(1756047.3), + Bucket: []*dto.Bucket{ + &dto.Bucket{ + UpperBound: proto.Float64(100), + CumulativeCount: proto.Uint64(123), + }, + &dto.Bucket{ + UpperBound: proto.Float64(120), + CumulativeCount: proto.Uint64(412), + }, + &dto.Bucket{ + UpperBound: proto.Float64(144), + CumulativeCount: proto.Uint64(592), + }, + &dto.Bucket{ + UpperBound: proto.Float64(172.8), + CumulativeCount: proto.Uint64(1524), + }, + &dto.Bucket{ + UpperBound: proto.Float64(math.Inf(+1)), + CumulativeCount: proto.Uint64(2693), + }, + }, + }, + }, + }, + }, + }, + }, } for i, scenario := range scenarios { @@ -427,7 +478,7 @@ line"} 3.14 # TYPE metric summary metric{quantile="bla"} 3.14 `, - err: "text format parsing error in line 3: expected float as value for quantile label", + err: "text format parsing error in line 3: expected float as value for 'quantile' label", }, // 8: { @@ -500,6 +551,14 @@ metric 4.12 in: `{label="bla"} 3.14 2`, err: "text format parsing error in line 1: invalid metric name", }, + // 18: + { + in: ` +# TYPE metric histogram +metric_bucket{le="bla"} 3.14 +`, + err: "text format parsing error in line 3: expected float as value for 'le' label", + }, } for i, scenario := range scenarios { diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/proto.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/proto.go index a46c418bcb8..058adfc6f60 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/text/proto.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/text/proto.go @@ -17,7 +17,7 @@ import ( "fmt" "io" - "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" "github.com/matttproud/golang_protobuf_extensions/ext" dto "github.com/prometheus/client_model/go" diff --git a/Godeps/_workspace/src/github.com/prometheus/client_model/go/metrics.pb.go b/Godeps/_workspace/src/github.com/prometheus/client_model/go/metrics.pb.go index eb7ab8e82c9..b065f8683f0 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_model/go/metrics.pb.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_model/go/metrics.pb.go @@ -15,27 +15,28 @@ It has these top-level messages: Quantile Summary Untyped + Histogram + Bucket Metric MetricFamily */ package io_prometheus_client -import proto "code.google.com/p/goprotobuf/proto" -import json "encoding/json" +import proto "github.com/golang/protobuf/proto" import math "math" -// Reference proto, json, and math imports to suppress error if they are not otherwise used. +// Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal -var _ = &json.SyntaxError{} var _ = math.Inf type MetricType int32 const ( - MetricType_COUNTER MetricType = 0 - MetricType_GAUGE MetricType = 1 - MetricType_SUMMARY MetricType = 2 - MetricType_UNTYPED MetricType = 3 + MetricType_COUNTER MetricType = 0 + MetricType_GAUGE MetricType = 1 + MetricType_SUMMARY MetricType = 2 + MetricType_UNTYPED MetricType = 3 + MetricType_HISTOGRAM MetricType = 4 ) var MetricType_name = map[int32]string{ @@ -43,12 +44,14 @@ var MetricType_name = map[int32]string{ 1: "GAUGE", 2: "SUMMARY", 3: "UNTYPED", + 4: "HISTOGRAM", } var MetricType_value = map[string]int32{ - "COUNTER": 0, - "GAUGE": 1, - "SUMMARY": 2, - "UNTYPED": 3, + "COUNTER": 0, + "GAUGE": 1, + "SUMMARY": 2, + "UNTYPED": 3, + "HISTOGRAM": 4, } func (x MetricType) Enum() *MetricType { @@ -196,12 +199,69 @@ func (m *Untyped) GetValue() float64 { return 0 } +type Histogram struct { + SampleCount *uint64 `protobuf:"varint,1,opt,name=sample_count" json:"sample_count,omitempty"` + SampleSum *float64 `protobuf:"fixed64,2,opt,name=sample_sum" json:"sample_sum,omitempty"` + Bucket []*Bucket `protobuf:"bytes,3,rep,name=bucket" json:"bucket,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Histogram) Reset() { *m = Histogram{} } +func (m *Histogram) String() string { return proto.CompactTextString(m) } +func (*Histogram) ProtoMessage() {} + +func (m *Histogram) GetSampleCount() uint64 { + if m != nil && m.SampleCount != nil { + return *m.SampleCount + } + return 0 +} + +func (m *Histogram) GetSampleSum() float64 { + if m != nil && m.SampleSum != nil { + return *m.SampleSum + } + return 0 +} + +func (m *Histogram) GetBucket() []*Bucket { + if m != nil { + return m.Bucket + } + return nil +} + +type Bucket struct { + CumulativeCount *uint64 `protobuf:"varint,1,opt,name=cumulative_count" json:"cumulative_count,omitempty"` + UpperBound *float64 `protobuf:"fixed64,2,opt,name=upper_bound" json:"upper_bound,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Bucket) Reset() { *m = Bucket{} } +func (m *Bucket) String() string { return proto.CompactTextString(m) } +func (*Bucket) ProtoMessage() {} + +func (m *Bucket) GetCumulativeCount() uint64 { + if m != nil && m.CumulativeCount != nil { + return *m.CumulativeCount + } + return 0 +} + +func (m *Bucket) GetUpperBound() float64 { + if m != nil && m.UpperBound != nil { + return *m.UpperBound + } + return 0 +} + type Metric struct { Label []*LabelPair `protobuf:"bytes,1,rep,name=label" json:"label,omitempty"` Gauge *Gauge `protobuf:"bytes,2,opt,name=gauge" json:"gauge,omitempty"` Counter *Counter `protobuf:"bytes,3,opt,name=counter" json:"counter,omitempty"` Summary *Summary `protobuf:"bytes,4,opt,name=summary" json:"summary,omitempty"` Untyped *Untyped `protobuf:"bytes,5,opt,name=untyped" json:"untyped,omitempty"` + Histogram *Histogram `protobuf:"bytes,7,opt,name=histogram" json:"histogram,omitempty"` TimestampMs *int64 `protobuf:"varint,6,opt,name=timestamp_ms" json:"timestamp_ms,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -245,6 +305,13 @@ func (m *Metric) GetUntyped() *Untyped { return nil } +func (m *Metric) GetHistogram() *Histogram { + if m != nil { + return m.Histogram + } + return nil +} + func (m *Metric) GetTimestampMs() int64 { if m != nil && m.TimestampMs != nil { return *m.TimestampMs diff --git a/Godeps/_workspace/src/github.com/prometheus/procfs/doc.go b/Godeps/_workspace/src/github.com/prometheus/procfs/doc.go index c6400345f6c..e2acd6d40a6 100644 --- a/Godeps/_workspace/src/github.com/prometheus/procfs/doc.go +++ b/Godeps/_workspace/src/github.com/prometheus/procfs/doc.go @@ -22,7 +22,7 @@ // "fmt" // "log" // -// "github.com/prometheus/client_golang/procfs" +// "github.com/prometheus/procfs" // ) // // func main() { @@ -31,7 +31,7 @@ // log.Fatalf("could not get process: %s", err) // } // -// stat, err := p.Stat() +// stat, err := p.NewStat() // if err != nil { // log.Fatalf("could not get process stat: %s", err) // } From 0d5fe5d0787e1afb88b1f106d266a5afc65471c6 Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Mon, 2 Mar 2015 02:15:16 +0000 Subject: [PATCH 2/2] Use a prometheus Histogram rather than a Summary for tracking apiserver latency. --- pkg/apiserver/apiserver.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 2307c0ae4ad..fb076d077a3 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -52,10 +52,12 @@ var ( }, []string{"handler", "verb", "resource", "code"}, ) - requestLatencies = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ + requestLatencies = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ Name: "apiserver_request_latencies", - Help: "Response latency summary in microseconds for each request handler and verb.", + Help: "Response latency distribution in microseconds for each request handler and verb.", + // Use buckets ranging from 125 ms to 8 seconds. + Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7), }, []string{"handler", "verb"}, )