Merge pull request #5706 from ipfs/fix/4132

fix prometheus concurrent map write bug

cmd/ipfs/daemon.go

@@ -23,11 +23,11 @@ import (
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
 	migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
 
+	mprome "gx/ipfs/QmQXBfkuwgMaPx334WuL9NmyrKnbZ5udaWnHTHEsts2x3T/go-metrics-prometheus"
 	cmds "gx/ipfs/QmSXUokcP4TJpFfqozT69AVAYRtzXVMUjzQVkYX41R9Svs/go-ipfs-cmds"
 	ma "gx/ipfs/QmT4U94DnD8FRfqr21obWY32HLM5VExccPKMjQHofeYqr9/go-multiaddr"
-	"gx/ipfs/QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH/client_golang/prometheus"
+	"gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus"
 	"gx/ipfs/Qmaabb1tJZ2CX5cp6MuuiGgns71NYoxdgQP6Xdid1dVceC/go-multiaddr-net"
-	mprome "gx/ipfs/QmbqQP7GMwJCePfEvbMSZmyPifAz1kKZifo8vwVnVHt1wL/go-metrics-prometheus"
 	"gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
 )
 

core/corehttp/metrics.go

@@ -6,13 +6,14 @@ import (
 
 	core "github.com/ipfs/go-ipfs/core"
 
-	prometheus "gx/ipfs/QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH/client_golang/prometheus"
+	prometheus "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus"
+	promhttp "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus/promhttp"
 )
 
 // This adds the scraping endpoint which Prometheus uses to fetch metrics.
 func MetricsScrapingOption(path string) ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
-		mux.Handle(path, prometheus.UninstrumentedHandler())
+		mux.Handle(path, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
 		return mux, nil
 	}
 }
@@ -20,8 +21,74 @@ func MetricsScrapingOption(path string) ServeOption {
 // This adds collection of net/http-related metrics
 func MetricsCollectionOption(handlerName string) ServeOption {
 	return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
+		// Adapted from github.com/prometheus/client_golang/prometheus/http.go
+		// Work around https://github.com/prometheus/client_golang/pull/311
+		opts := prometheus.SummaryOpts{
+			Subsystem:   "http",
+			ConstLabels: prometheus.Labels{"handler": handlerName},
+			Objectives:  map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		}
+
+		reqCnt := prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace:   opts.Namespace,
+				Subsystem:   opts.Subsystem,
+				Name:        "requests_total",
+				Help:        "Total number of HTTP requests made.",
+				ConstLabels: opts.ConstLabels,
+			},
+			[]string{"method", "code"},
+		)
+		if err := prometheus.Register(reqCnt); err != nil {
+			if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+				reqCnt = are.ExistingCollector.(*prometheus.CounterVec)
+			} else {
+				return nil, err
+			}
+		}
+
+		opts.Name = "request_duration_seconds"
+		opts.Help = "The HTTP request latencies in seconds."
+		reqDur := prometheus.NewSummaryVec(opts, nil)
+		if err := prometheus.Register(reqDur); err != nil {
+			if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+				reqDur = are.ExistingCollector.(*prometheus.SummaryVec)
+			} else {
+				return nil, err
+			}
+		}
+
+		opts.Name = "request_size_bytes"
+		opts.Help = "The HTTP request sizes in bytes."
+		reqSz := prometheus.NewSummaryVec(opts, nil)
+		if err := prometheus.Register(reqSz); err != nil {
+			if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+				reqSz = are.ExistingCollector.(*prometheus.SummaryVec)
+			} else {
+				return nil, err
+			}
+		}
+
+		opts.Name = "response_size_bytes"
+		opts.Help = "The HTTP response sizes in bytes."
+		resSz := prometheus.NewSummaryVec(opts, nil)
+		if err := prometheus.Register(resSz); err != nil {
+			if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+				resSz = are.ExistingCollector.(*prometheus.SummaryVec)
+			} else {
+				return nil, err
+			}
+		}
+
+		// Construct the mux
 		childMux := http.NewServeMux()
-		mux.HandleFunc("/", prometheus.InstrumentHandler(handlerName, childMux))
+		var promMux http.Handler = childMux
+		promMux = promhttp.InstrumentHandlerResponseSize(resSz, promMux)
+		promMux = promhttp.InstrumentHandlerRequestSize(reqSz, promMux)
+		promMux = promhttp.InstrumentHandlerDuration(reqDur, promMux)
+		promMux = promhttp.InstrumentHandlerCounter(reqCnt, promMux)
+		mux.Handle("/", promMux)
+
 		return childMux, nil
 	}
 }

package.json

@@ -42,9 +42,9 @@
       "version": "0.0.0"
     },
     {
-      "hash": "QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH",
+      "hash": "QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS",
       "name": "client_golang",
-      "version": "0.1.3"
+      "version": "0.1.4"
     },
     {
       "hash": "QmV1DPm5F46LvQMxCVPhu35zHgZEeMvyVtpxjb5TwfGiua",
@@ -160,9 +160,9 @@
     },
     {
       "author": "ipfs",
-      "hash": "QmbqQP7GMwJCePfEvbMSZmyPifAz1kKZifo8vwVnVHt1wL",
+      "hash": "QmQXBfkuwgMaPx334WuL9NmyrKnbZ5udaWnHTHEsts2x3T",
       "name": "go-metrics-prometheus",
-      "version": "0.3.9"
+      "version": "0.3.10"
     },
     {
       "author": "ipfs",