From 624e57d292e653969c0941c86729b7ad1a4274d1 Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Sun, 21 Jul 2013 17:09:48 +0200 Subject: [PATCH 1/3] Enclose artifact generation process into Makefile. Completely decouple the build process from Travis and other things we cannot control. --- .gitignore | 1 + .travis.yml | 2 +- Makefile | 55 +++++++++++++++++++++++++++++++++---- Makefile.TRAVIS | 31 --------------------- examples/delegator/Makefile | 6 ++-- examples/random/Makefile | 6 ++-- examples/simple/Makefile | 6 ++-- prometheus/Makefile | 13 +++++---- 8 files changed, 69 insertions(+), 51 deletions(-) delete mode 100644 Makefile.TRAVIS diff --git a/.gitignore b/.gitignore index 4482aca..f6fc2e8 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ _testmain.go *~ *# +.build diff --git a/.travis.yml b/.travis.yml index 2c7ccd3..afeb72d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,4 +4,4 @@ go: - 1.1 script: - - make -f Makefile.TRAVIS + - make -f Makefile diff --git a/Makefile b/Makefile index c43f4c3..3ba28b1 100644 --- a/Makefile +++ b/Makefile @@ -11,14 +11,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -MAKE_ARTIFACTS = search_index +OS = $(shell uname) +ARCH = $(shell uname -m) + +BUILD_PATH = $(PWD)/.build + +export GO_VERSION = 1.1 +export GOOS = $(subst Darwin,darwin,$(subst Linux,linux,$(OS))) +export GOARCH = $(subst x86_64,amd64,$(ARCH)) +export GOPKG = go$(GO_VERSION).$(GOOS)-$(GOARCH).tar.gz +export GOROOT = $(BUILD_PATH)/root/go +export GOPATH = $(BUILD_PATH)/root/gopath +export GOCC = $(GOROOT)/bin/go +export TMPDIR = /tmp +export GOENV = TMPDIR=$(TMPDIR) GOROOT=$(GOROOT) GOPATH=$(GOPATH) +export GO = $(GOENV) $(GOCC) +export GOFMT = $(GOROOT)/bin/gofmt + +FULL_GOPATH = $(GOPATH)/src/github.com/prometheus/client_golang +FULL_GOPATH_BASE = $(GOPATH)/src/github.com/prometheus + +MAKE_ARTIFACTS = search_index $(BUILD_PATH) all: test -build: +$(BUILD_PATH): + mkdir -vp $(BUILD_PATH) + +$(BUILD_PATH)/cache: $(BUILD_PATH) + mkdir -vp $(BUILD_PATH)/cache + +$(BUILD_PATH)/root: $(BUILD_PATH) + mkdir -vp $(BUILD_PATH)/root + +$(BUILD_PATH)/cache/$(GOPKG): $(BUILD_PATH)/cache + curl -o $@ http://go.googlecode.com/files/$(GOPKG) + +$(GOCC): $(BUILD_PATH)/root $(BUILD_PATH)/cache/$(GOPKG) + tar -C $(BUILD_PATH)/root -xzf $(BUILD_PATH)/cache/$(GOPKG) + touch $@ + +build: source_path dependencies $(MAKE) -C prometheus build $(MAKE) -C examples build +dependencies: source_path $(GOCC) + $(GO) get github.com/matttproud/gocheck + test: build $(MAKE) -C prometheus test $(MAKE) -C examples test @@ -28,18 +67,24 @@ advice: test $(MAKE) -C examples advice format: - find . -iname '*.go' -exec gofmt -w -s=true '{}' ';' + find . -iname '*.go' | grep -v './.build/' | xargs -n1 -P1 $(GOFMT) -w -s=true search_index: godoc -index -write_index -index_files='search_index' +# source_path is responsible for ensuring that the builder has not done anything +# stupid like working on Prometheus outside of ${GOPATH}. +source_path: + -[ -d "$(FULL_GOPATH)" ] || { mkdir -vp $(FULL_GOPATH_BASE) ; ln -s "$(PWD)" "$(FULL_GOPATH)" ; } + [ -d "$(FULL_GOPATH)" ] + documentation: search_index godoc -http=:6060 -index -index_files='search_index' clean: $(MAKE) -C examples clean - rm -f $(MAKE_ARTIFACTS) + rm -rf $(MAKE_ARTIFACTS) find . -iname '*~' -exec rm -f '{}' ';' find . -iname '*#' -exec rm -f '{}' ';' -.PHONY: advice build clean documentation format test +.PHONY: advice build clean documentation format source_path test diff --git a/Makefile.TRAVIS b/Makefile.TRAVIS deleted file mode 100644 index 9a6b35b..0000000 --- a/Makefile.TRAVIS +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2013 Prometheus Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PROMETHEUS_TARGET := "${GOPATH}/src/github.com/prometheus" - -all: test - -preparation: - mkdir -vp $(PROMETHEUS_TARGET) - ln -sf "$(PWD)" $(PROMETHEUS_TARGET) - -dependencies: - go get code.google.com/p/goprotobuf/proto - go get github.com/matttproud/gocheck - go get github.com/matttproud/golang_protobuf_extensions/ext - go get github.com/prometheus/client_model/go - -test: dependencies preparation - $(MAKE) test - -.PHONY: dependencies preparation test diff --git a/examples/delegator/Makefile b/examples/delegator/Makefile index c60b94c..2d8ef05 100644 --- a/examples/delegator/Makefile +++ b/examples/delegator/Makefile @@ -18,13 +18,13 @@ all: test build: delegator delegator: - go build . + $(GO) build . test: build - go test . $(GO_TEST_FLAGS) + $(GO) test . $(GO_TEST_FLAGS) advice: - go tool vet . + $(GO) tool vet . clean: rm -f $(MAKE_ARTIFACTS) diff --git a/examples/random/Makefile b/examples/random/Makefile index c54b305..0d5e48a 100644 --- a/examples/random/Makefile +++ b/examples/random/Makefile @@ -18,13 +18,13 @@ all: test build: random random: - go build . + $(GO) build . test: build - go test . $(GO_TEST_FLAGS) + $(GO) test . $(GO_TEST_FLAGS) advice: - go tool vet . + $(GO) tool vet . clean: rm -f $(MAKE_ARTIFACTS) diff --git a/examples/simple/Makefile b/examples/simple/Makefile index dba8a98..88526d3 100644 --- a/examples/simple/Makefile +++ b/examples/simple/Makefile @@ -18,13 +18,13 @@ all: test build: simple simple: - go build . + $(GO) build . test: build - go test . $(GO_TEST_FLAGS) + $(GO) test . $(GO_TEST_FLAGS) advice: - go tool vet . + $(GO) tool vet . clean: rm -f $(MAKE_ARTIFACTS) diff --git a/prometheus/Makefile b/prometheus/Makefile index 85b7b46..4c1e51d 100644 --- a/prometheus/Makefile +++ b/prometheus/Makefile @@ -13,13 +13,16 @@ all: test -build: - go build ./... +build: dependencies + $(GO) build ./... + +dependencies: $(GOCC) + $(GO) get -d test: build - go test ./... $(GO_TEST_FLAGS) + $(GO) test ./... $(GO_TEST_FLAGS) advice: - go tool vet . + $(GO) tool vet . -.PHONY: advice build test +.PHONY: advice build dependencies test From a10d055c32ec3ef50b9d1b921975f71b948515bb Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Sun, 21 Jul 2013 17:14:29 +0200 Subject: [PATCH 2/3] Include sample count and sum in Proto output. --- prometheus/histogram.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/prometheus/histogram.go b/prometheus/histogram.go index 85389a2..5a806bb 100644 --- a/prometheus/histogram.go +++ b/prometheus/histogram.go @@ -87,6 +87,8 @@ type histogram struct { type histogramVector struct { buckets []Bucket labels map[string]string + sum float64 + count uint64 } func (h *histogram) Add(labels map[string]string, value float64) { @@ -124,6 +126,9 @@ func (h *histogram) Add(labels map[string]string, value float64) { } histogram.buckets[lastIndex].Add(value) + + histogram.sum += value + histogram.count++ } func (h *histogram) String() string { @@ -313,7 +318,10 @@ func (metric *histogram) dumpChildren(f *dto.MetricFamily) { f.Type = dto.MetricType_SUMMARY.Enum() for signature, child := range metric.values { - c := &dto.Summary{} + c := &dto.Summary{ + SampleSum: proto.Float64(child.sum), + SampleCount: proto.Uint64(child.count), + } m := &dto.Metric{ Summary: c, From 93130ba5c3bd52e920b7709c12c2a9a7cfb4af70 Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Sun, 21 Jul 2013 17:34:37 +0200 Subject: [PATCH 3/3] Introduce histogram purging support. --- prometheus/histogram.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/prometheus/histogram.go b/prometheus/histogram.go index 5a806bb..3d5f9e7 100644 --- a/prometheus/histogram.go +++ b/prometheus/histogram.go @@ -13,6 +13,7 @@ import ( "math" "strconv" "sync" + "time" dto "github.com/prometheus/client_model/go" @@ -54,6 +55,7 @@ type HistogramSpecification struct { BucketBuilder BucketBuilder ReportablePercentiles []float64 Starts []float64 + PurgeInterval time.Duration } type Histogram interface { @@ -82,6 +84,9 @@ type histogram struct { values map[uint64]*histogramVector // These are the percentile values that will be reported on marshalling. reportablePercentiles []float64 + + purgeInterval time.Duration + lastPurge time.Time } type histogramVector struct { @@ -249,6 +254,8 @@ func formatFloat(value float64) string { } func (h *histogram) MarshalJSON() ([]byte, error) { + h.Purge() + h.mutex.RLock() defer h.mutex.RUnlock() @@ -274,10 +281,29 @@ func (h *histogram) MarshalJSON() ([]byte, error) { }) } +func (h *histogram) Purge() { + if h.purgeInterval == 0 { + return + } + + h.mutex.Lock() + defer h.mutex.Unlock() + if time.Since(h.lastPurge) < h.purgeInterval { + return + } + + h.resetAll() + h.lastPurge = time.Now() +} + func (h *histogram) ResetAll() { h.mutex.Lock() defer h.mutex.Unlock() + h.resetAll() +} + +func (h *histogram) resetAll() { for signature, value := range h.values { for _, bucket := range value.buckets { bucket.Reset() @@ -294,6 +320,8 @@ func NewHistogram(specification *HistogramSpecification) Histogram { bucketStarts: specification.Starts, reportablePercentiles: specification.ReportablePercentiles, values: map[uint64]*histogramVector{}, + lastPurge: time.Now(), + purgeInterval: specification.PurgeInterval, } return metric @@ -307,11 +335,14 @@ func NewDefaultHistogram() Histogram { Starts: LogarithmicSizedBucketsFor(0, 4096), BucketBuilder: AccumulatingBucketBuilder(EvictAndReplaceWith(10, AverageReducer), 50), ReportablePercentiles: []float64{0.01, 0.05, 0.5, 0.90, 0.99}, + PurgeInterval: 15 * time.Minute, }, ) } func (metric *histogram) dumpChildren(f *dto.MetricFamily) { + metric.Purge() + metric.mutex.RLock() defer metric.mutex.RUnlock()