diff --git a/docs/protoapi_metrics.md b/docs/protoapi_metrics.md new file mode 100644 index 0000000..59a8f05 --- /dev/null +++ b/docs/protoapi_metrics.md @@ -0,0 +1,56 @@ +# protoapi metrics + +## Usage + +Import middleware and use in echo. + +example: +```go +package main + +import ( + "github.com/labstack/echo" + "github.com/yoozoo/protoapi/protoapigo/metrics" + "code/generated/by/protoapi/demo" +) + +type Service struct{} + +var _ demo.DemoService = &Service{} + +func main() { + e := echo.New() + + service := &Service{} + calsvr.RegisterCalcService(e, c) + + // use metrics middleware + m := metrics.MetricsFunc() + e.Use(m) + + e.Logger.Fatal(e.Start(":8080")) +} +``` + +## Metrics endpoint + +By default, echo server exports metrics under the `/metrics` path on its client port. + +The metrics can be fetched with `curl`: + +```sh +$ curl -L http://localhost:8080/metrics +``` +## More Options +Function `metrics.MetricsFunc()` can have `Option` parameters. +```go +package metrics + +func Registry(r prometheus.Registerer) Option {} + +func MetricsPath(v string) Option {} + +func Namespace(v string) Option {} + +func Subsystem(v string) Option {} +``` diff --git a/protoapigo/metrics/grafana.json b/protoapigo/metrics/grafana.json new file mode 100644 index 0000000..b05476c --- /dev/null +++ b/protoapigo/metrics/grafana.json @@ -0,0 +1,940 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": 12, + "links": [], + "refresh": false, + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(request_total{job=\"echo\",host=~\"$Host\",status=~\"$Status\",node=~\"$Node\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "total request", + "refId": "A", + "target": "alias(sumSeries(echo.host.$Host.domain.$Domain.ReqQps.mean), 'Request Rate')", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "服务请求", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(response_size_sum{job=\"echo\",host=~\"$Host\",status=~\"$Status\",node=~\"$Node\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "response size", + "refId": "A", + "target": "alias(sumSeries(echo.host.$Host.domain.$Domain.$Status.BytesOut.mean), 'Bytes Out')", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "服务流量", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(request_duration{job=\"echo\",host=~\"$Host\",status=~\"$Status\",node=~\"$Node\"})", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "max", + "refId": "A", + "target": "" + }, + { + "expr": "min(request_duration{job=\"echo\",host=~\"$Host\",status=~\"$Status\",node=~\"$Node\"})", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "min", + "refId": "B" + }, + { + "expr": "rate(request_duration_sum{job=\"echo\",host=~\"$Host\",status=~\"$Status\",node=~\"$Node\"}[1m])/rate(echo_request_duration_count{job=\"echo\",host=~\"$Host\",status=~\"$Status\",node=~\"$Node\"}[1m])", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "mean-{{status}}", + "refId": "C" + }, + { + "expr": "sum(request_duration{job=\"echo\",host=~\"$Host\",status=~\"$Status\",node=~\"$Node\"}) by (quantile)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quantile={{quantile}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "服务耗时", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{job=\"echo\",instance=~\"$Instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "target": "aliasByNode(echo.host.$Host.runtime.MemStats.Alloc.value, 5, 2)", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "内存", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbits", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_heap_alloc_bytes{job=\"echo\",instance=~\"$Instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "target": "aliasByNode(echo.host.$Host.runtime.MemStats.HeapAlloc.value, 5, 2)", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "堆", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbits", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_heap_objects{job=\"echo\",instance=~\"$Instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "target": "aliasByNode(echo.host.$Host.runtime.MemStats.HeapObjects.value, 5, 2)", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "堆对象数", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbits", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{job=\"echo\",instance=~\"$Instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}}-{{instance}}", + "refId": "A", + "target": "aliasByNode(echo.host.$Host.runtime.MemStats.PauseNs.mean, 5, 2)", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "GC Pause Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"echo\",instance=~\"$Instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "target": "aliasByNode(echo.host.$Host.runtime.NumGoroutine.value, 4, 2)", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_threads{job=\"echo\",instance=~\"$Instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "target": "aliasByNode(echo.host.$Host.runtime.NumThread.value, 4, 2)", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Host", + "options": [], + "query": "label_values(request_total, host)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Status", + "options": [], + "query": "label_values(request_total, status)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Node", + "options": [], + "query": "label_values(request_total, node)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "echo.www.localhost.com:80", + "value": "echo.www.localhost.com:80" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Instance", + "options": [], + "query": "label_values(go_memstats_alloc_bytes{job=\"echo\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now/d", + "to": "now/d" + }, + "timepicker": { + "refresh.intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time.options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "echo-prometheus", + "version": 19 + } diff --git a/protoapigo/metrics/metrics.go b/protoapigo/metrics/metrics.go new file mode 100644 index 0000000..da4532e --- /dev/null +++ b/protoapigo/metrics/metrics.go @@ -0,0 +1,94 @@ +package metrics + +import ( + "os" + "strconv" + "time" + + "github.com/labstack/echo" + "github.com/labstack/gommon/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var defaultLabelNames = []string{"node", "host", "uri", "status"} + +func MetricsFunc(options ...Option) echo.MiddlewareFunc { + opts := applyOptions(options...) + + reqCount := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: opts.Namespace, + Subsystem: opts.Subsystem, + Name: "request_total", + Help: "Total request count.", + }, + defaultLabelNames, + ) + + reqDur := prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: opts.Namespace, + Subsystem: opts.Subsystem, + Name: "request_duration", + Help: "Request duration in nanoseconds.", + }, + defaultLabelNames, + ) + + reqSize := prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: opts.Namespace, + Subsystem: opts.Subsystem, + Name: "request_size", + Help: "Request size in bytes.", + }, + defaultLabelNames, + ) + + resSize := prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: opts.Namespace, + Subsystem: opts.Subsystem, + Name: "response_size", + Help: "Response size in bytes.", + }, + defaultLabelNames, + ) + + opts.Registry.MustRegister(reqCount, reqDur, reqSize, resSize) + + hostname, err := os.Hostname() + if err != nil { + log.Warnf("os.Hostname() error:%v", err) + hostname = "-" + } + + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + req := c.Request() + res := c.Response() + + // 拦截metrics path，默认"/noauth/metrics" + if req.URL.Path == opts.MetricsPath { + promhttp.Handler().ServeHTTP(c.Response(), c.Request()) + return nil + } + + start := time.Now() + if err := next(c); err != nil { + c.Error(err) + } + + latency := time.Since(start) + status := strconv.Itoa(res.Status) + + reqCount.WithLabelValues(hostname, req.Host, req.RequestURI, status).Inc() + reqDur.WithLabelValues(hostname, req.Host, req.RequestURI, status).Observe(float64(latency.Nanoseconds())) + reqSize.WithLabelValues(hostname, req.Host, req.RequestURI, status).Observe(float64(req.ContentLength)) + resSize.WithLabelValues(hostname, req.Host, req.RequestURI, status).Observe(float64(res.Size)) + + return nil + } + } +} diff --git a/protoapigo/metrics/option.go b/protoapigo/metrics/option.go new file mode 100644 index 0000000..7ffc613 --- /dev/null +++ b/protoapigo/metrics/option.go @@ -0,0 +1,53 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var defaultMetricPath = "/metrics" + +type Option func(c *Options) + +type Options struct { + Registry prometheus.Registerer + MetricsPath string + Namespace string + Subsystem string +} + +func Registry(r prometheus.Registerer) Option { + return func(o *Options) { + o.Registry = r + } +} + +func MetricsPath(v string) Option { + return func(o *Options) { + o.MetricsPath = v + } +} + +func Namespace(v string) Option { + return func(o *Options) { + o.Namespace = v + } +} + +func Subsystem(v string) Option { + return func(o *Options) { + o.Subsystem = v + } +} + +func applyOptions(options ...Option) Options { + opts := Options{ + Registry: prometheus.DefaultRegisterer, + MetricsPath: defaultMetricPath, + } + + for _, option := range options { + option(&opts) + } + + return opts +}