mirror of
https://github.com/minio/minio.git
synced 2026-02-04 18:00:15 -05:00
feat: Add Metrics V3 API (#19068)
Metrics v3 is mainly a reorganization of metrics into smaller groups of metrics and the removal of internal aggregation of metrics received from peer nodes in a MinIO cluster. This change adds the endpoint `/minio/metrics/v3` as the top-level metrics endpoint and under this, various sub-endpoints are implemented. These are currently documented in `docs/metrics/v3.md` The handler will serve metrics at any path `/minio/metrics/v3/PATH`, as follows: when PATH is a sub-endpoint listed above => serves the group of metrics under that path; or when PATH is a (non-empty) parent directory of the sub-endpoints listed above => serves metrics from each child sub-endpoint of PATH. otherwise, returns a no resource found error All available metrics are listed in the `docs/metrics/v3.md`. More will be added subsequently.
This commit is contained in:
Aditya Manthramurthy
committed by
GitHub
GitHub
parent
2dfa9adc5d
commit
b2c5b75efa
220
cmd/metrics-v3-api.go
Normal file
220
cmd/metrics-v3-api.go
Normal file
@@ -0,0 +1,220 @@
|
||||
// Copyright (c) 2015-2024 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
const (
|
||||
apiRejectedAuthTotal MetricName = "rejected_auth_total"
|
||||
apiRejectedHeaderTotal MetricName = "rejected_header_total"
|
||||
apiRejectedTimestampTotal MetricName = "rejected_timestamp_total"
|
||||
apiRejectedInvalidTotal MetricName = "rejected_invalid_total"
|
||||
|
||||
apiRequestsWaitingTotal MetricName = "waiting_total"
|
||||
apiRequestsIncomingTotal MetricName = "incoming_total"
|
||||
|
||||
apiRequestsInFlightTotal MetricName = "inflight_total"
|
||||
apiRequestsTotal MetricName = "total"
|
||||
apiRequestsErrorsTotal MetricName = "errors_total"
|
||||
apiRequests5xxErrorsTotal MetricName = "5xx_errors_total"
|
||||
apiRequests4xxErrorsTotal MetricName = "4xx_errors_total"
|
||||
apiRequestsCanceledTotal MetricName = "canceled_total"
|
||||
|
||||
apiRequestsTTFBSecondsDistribution MetricName = "ttfb_seconds_distribution"
|
||||
|
||||
apiTrafficSentBytes MetricName = "traffic_sent_bytes"
|
||||
apiTrafficRecvBytes MetricName = "traffic_received_bytes"
|
||||
)
|
||||
|
||||
var (
|
||||
apiRejectedAuthTotalMD = NewCounterMD(apiRejectedAuthTotal,
|
||||
"Total number of requests rejected for auth failure", "type")
|
||||
apiRejectedHeaderTotalMD = NewCounterMD(apiRejectedHeaderTotal,
|
||||
"Total number of requests rejected for invalid header", "type")
|
||||
apiRejectedTimestampTotalMD = NewCounterMD(apiRejectedTimestampTotal,
|
||||
"Total number of requests rejected for invalid timestamp", "type")
|
||||
apiRejectedInvalidTotalMD = NewCounterMD(apiRejectedInvalidTotal,
|
||||
"Total number of invalid requests", "type")
|
||||
|
||||
apiRequestsWaitingTotalMD = NewGaugeMD(apiRequestsWaitingTotal,
|
||||
"Total number of requests in the waiting queue", "type")
|
||||
apiRequestsIncomingTotalMD = NewGaugeMD(apiRequestsIncomingTotal,
|
||||
"Total number of incoming requests", "type")
|
||||
|
||||
apiRequestsInFlightTotalMD = NewGaugeMD(apiRequestsInFlightTotal,
|
||||
"Total number of requests currently in flight", "name", "type")
|
||||
apiRequestsTotalMD = NewCounterMD(apiRequestsTotal,
|
||||
"Total number of requests", "name", "type")
|
||||
apiRequestsErrorsTotalMD = NewCounterMD(apiRequestsErrorsTotal,
|
||||
"Total number of requests with (4xx and 5xx) errors", "name", "type")
|
||||
apiRequests5xxErrorsTotalMD = NewCounterMD(apiRequests5xxErrorsTotal,
|
||||
"Total number of requests with 5xx errors", "name", "type")
|
||||
apiRequests4xxErrorsTotalMD = NewCounterMD(apiRequests4xxErrorsTotal,
|
||||
"Total number of requests with 4xx errors", "name", "type")
|
||||
apiRequestsCanceledTotalMD = NewCounterMD(apiRequestsCanceledTotal,
|
||||
"Total number of requests canceled by the client", "name", "type")
|
||||
|
||||
apiRequestsTTFBSecondsDistributionMD = NewCounterMD(apiRequestsTTFBSecondsDistribution,
|
||||
"Distribution of time to first byte across API calls", "name", "type", "le")
|
||||
|
||||
apiTrafficSentBytesMD = NewCounterMD(apiTrafficSentBytes,
|
||||
"Total number of bytes sent", "type")
|
||||
apiTrafficRecvBytesMD = NewCounterMD(apiTrafficRecvBytes,
|
||||
"Total number of bytes received", "type")
|
||||
)
|
||||
|
||||
// loadAPIRequestsHTTPMetrics - reads S3 HTTP metrics.
|
||||
//
|
||||
// This is a `MetricsLoaderFn`.
|
||||
//
|
||||
// This includes node level S3 HTTP metrics.
|
||||
//
|
||||
// This function currently ignores `opts`.
|
||||
func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
|
||||
// Collect node level S3 HTTP metrics.
|
||||
httpStats := globalHTTPStats.toServerHTTPStats(false)
|
||||
|
||||
// Currently we only collect S3 API related stats, so we set the "type"
|
||||
// label to "s3".
|
||||
|
||||
m.Set(apiRejectedAuthTotal, float64(httpStats.TotalS3RejectedAuth), "type", "s3")
|
||||
m.Set(apiRejectedTimestampTotal, float64(httpStats.TotalS3RejectedTime), "type", "s3")
|
||||
m.Set(apiRejectedHeaderTotal, float64(httpStats.TotalS3RejectedHeader), "type", "s3")
|
||||
m.Set(apiRejectedInvalidTotal, float64(httpStats.TotalS3RejectedInvalid), "type", "s3")
|
||||
m.Set(apiRequestsWaitingTotal, float64(httpStats.S3RequestsInQueue), "type", "s3")
|
||||
m.Set(apiRequestsIncomingTotal, float64(httpStats.S3RequestsIncoming), "type", "s3")
|
||||
|
||||
for name, value := range httpStats.CurrentS3Requests.APIStats {
|
||||
m.Set(apiRequestsInFlightTotal, float64(value), "name", name, "type", "s3")
|
||||
}
|
||||
for name, value := range httpStats.TotalS3Requests.APIStats {
|
||||
m.Set(apiRequestsTotal, float64(value), "name", name, "type", "s3")
|
||||
}
|
||||
for name, value := range httpStats.TotalS3Errors.APIStats {
|
||||
m.Set(apiRequestsErrorsTotal, float64(value), "name", name, "type", "s3")
|
||||
}
|
||||
for name, value := range httpStats.TotalS35xxErrors.APIStats {
|
||||
m.Set(apiRequests5xxErrorsTotal, float64(value), "name", name, "type", "s3")
|
||||
}
|
||||
for name, value := range httpStats.TotalS34xxErrors.APIStats {
|
||||
m.Set(apiRequests4xxErrorsTotal, float64(value), "name", name, "type", "s3")
|
||||
}
|
||||
for name, value := range httpStats.TotalS3Canceled.APIStats {
|
||||
m.Set(apiRequestsCanceledTotal, float64(value), "name", name, "type", "s3")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadAPIRequestsTTFBMetrics - loads S3 TTFB metrics.
|
||||
//
|
||||
// This is a `MetricsLoaderFn`.
|
||||
func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
|
||||
renameLabels := map[string]string{"api": "name"}
|
||||
m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, renameLabels, nil,
|
||||
"type", "s3")
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadAPIRequestsNetworkMetrics - loads S3 network metrics.
|
||||
//
|
||||
// This is a `MetricsLoaderFn`.
|
||||
func loadAPIRequestsNetworkMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
|
||||
connStats := globalConnStats.toServerConnStats()
|
||||
m.Set(apiTrafficSentBytes, float64(connStats.s3OutputBytes), "type", "s3")
|
||||
m.Set(apiTrafficRecvBytes, float64(connStats.s3InputBytes), "type", "s3")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Metric Descriptions for bucket level S3 metrics.
|
||||
var (
|
||||
apiBucketTrafficSentBytesMD = NewCounterMD(apiTrafficSentBytes,
|
||||
"Total number of bytes received for a bucket", "bucket", "type")
|
||||
apiBucketTrafficRecvBytesMD = NewCounterMD(apiTrafficRecvBytes,
|
||||
"Total number of bytes sent for a bucket", "bucket", "type")
|
||||
|
||||
apiBucketRequestsInFlightMD = NewGaugeMD(apiRequestsInFlightTotal,
|
||||
"Total number of requests currently in flight for a bucket", "bucket", "name", "type")
|
||||
apiBucketRequestsTotalMD = NewCounterMD(apiRequestsTotal,
|
||||
"Total number of requests for a bucket", "bucket", "name", "type")
|
||||
apiBucketRequestsCanceledMD = NewCounterMD(apiRequestsCanceledTotal,
|
||||
"Total number of requests canceled by the client for a bucket", "bucket", "name", "type")
|
||||
apiBucketRequests4xxErrorsMD = NewCounterMD(apiRequests4xxErrorsTotal,
|
||||
"Total number of requests with 4xx errors for a bucket", "bucket", "name", "type")
|
||||
apiBucketRequests5xxErrorsMD = NewCounterMD(apiRequests5xxErrorsTotal,
|
||||
"Total number of requests with 5xx errors for a bucket", "bucket", "name", "type")
|
||||
|
||||
apiBucketRequestsTTFBSecondsDistributionMD = NewCounterMD(apiRequestsTTFBSecondsDistribution,
|
||||
"Distribution of time to first byte across API calls for a bucket",
|
||||
"bucket", "name", "le", "type")
|
||||
)
|
||||
|
||||
// loadAPIBucketHTTPMetrics - loads bucket level S3 HTTP metrics.
|
||||
//
|
||||
// This is a `MetricsLoaderFn`.
|
||||
//
|
||||
// This includes bucket level S3 HTTP metrics and S3 network in/out metrics.
|
||||
func loadAPIBucketHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
|
||||
if len(buckets) == 0 {
|
||||
return nil
|
||||
}
|
||||
for bucket, inOut := range globalBucketConnStats.getBucketS3InOutBytes(buckets) {
|
||||
recvBytes := inOut.In
|
||||
if recvBytes > 0 {
|
||||
m.Set(apiTrafficSentBytes, float64(recvBytes), "bucket", bucket, "type", "s3")
|
||||
}
|
||||
sentBytes := inOut.Out
|
||||
if sentBytes > 0 {
|
||||
m.Set(apiTrafficRecvBytes, float64(sentBytes), "bucket", bucket, "type", "s3")
|
||||
}
|
||||
|
||||
httpStats := globalBucketHTTPStats.load(bucket)
|
||||
for k, v := range httpStats.currentS3Requests.Load(false) {
|
||||
m.Set(apiRequestsInFlightTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
||||
}
|
||||
|
||||
for k, v := range httpStats.totalS3Requests.Load(false) {
|
||||
m.Set(apiRequestsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
||||
}
|
||||
|
||||
for k, v := range httpStats.totalS3Canceled.Load(false) {
|
||||
m.Set(apiRequestsCanceledTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
||||
}
|
||||
|
||||
for k, v := range httpStats.totalS34xxErrors.Load(false) {
|
||||
m.Set(apiRequests4xxErrorsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
||||
}
|
||||
|
||||
for k, v := range httpStats.totalS35xxErrors.Load(false) {
|
||||
m.Set(apiRequests5xxErrorsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadAPIBucketTTFBMetrics - loads bucket S3 TTFB metrics.
|
||||
//
|
||||
// This is a `MetricsLoaderFn`.
|
||||
func loadAPIBucketTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
|
||||
renameLabels := map[string]string{"api": "name"}
|
||||
m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, renameLabels,
|
||||
buckets, "type", "s3")
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user