mirror of
https://github.com/minio/minio.git
synced 2026-02-04 18:00:15 -05:00
Add cluster and bucket replication metrics in metrics-v3 (#19546)
endpoint: /minio/metrics/v3/cluster/replication metrics: - average_active_workers - average_queued_bytes - average_queued_count - average_transfer_rate - current_active_workers - current_transfer_rate - last_minute_queued_bytes - last_minute_queued_count - max_active_workers - max_queued_bytes - max_queued_count - max_transfer_rate - recent_backlog_count endpoint: /minio/metrics/v3/api/bucket/replication metrics: - last_hour_failed_bytes - last_hour_failed_count - last_minute_failed_bytes - last_minute_failed_count - latency_ms - proxied_delete_tagging_requests_total - proxied_get_requests_failures - proxied_get_requests_total - proxied_get_tagging_requests_failures - proxied_get_tagging_requests_total - proxied_head_requests_failures - proxied_head_requests_total - proxied_put_tagging_requests_failures - proxied_put_tagging_requests_total - sent_bytes - sent_count - total_failed_bytes - total_failed_count - proxied_delete_tagging_requests_failures
This commit is contained in:
Shireesh Anjal
155
cmd/metrics-v3-bucket-replication.go
Normal file
155
cmd/metrics-v3-bucket-replication.go
Normal file
@@ -0,0 +1,155 @@
|
||||
// Copyright (c) 2015-2024 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
const (
|
||||
bucketReplLastHrFailedBytes = "last_hour_failed_bytes"
|
||||
bucketReplLastHrFailedCount = "last_hour_failed_count"
|
||||
bucketReplLastMinFailedBytes = "last_minute_failed_bytes"
|
||||
bucketReplLastMinFailedCount = "last_minute_failed_count"
|
||||
bucketReplLatencyMs = "latency_ms"
|
||||
bucketReplProxiedDeleteTaggingRequestsTotal = "proxied_delete_tagging_requests_total"
|
||||
bucketReplProxiedGetRequestsFailures = "proxied_get_requests_failures"
|
||||
bucketReplProxiedGetRequestsTotal = "proxied_get_requests_total"
|
||||
bucketReplProxiedGetTaggingRequestsFailures = "proxied_get_tagging_requests_failures"
|
||||
bucketReplProxiedGetTaggingRequestsTotal = "proxied_get_tagging_requests_total"
|
||||
bucketReplProxiedHeadRequestsFailures = "proxied_head_requests_failures"
|
||||
bucketReplProxiedHeadRequestsTotal = "proxied_head_requests_total"
|
||||
bucketReplProxiedPutTaggingRequestsFailures = "proxied_put_tagging_requests_failures"
|
||||
bucketReplProxiedPutTaggingRequestsTotal = "proxied_put_tagging_requests_total"
|
||||
bucketReplSentBytes = "sent_bytes"
|
||||
bucketReplSentCount = "sent_count"
|
||||
bucketReplTotalFailedBytes = "total_failed_bytes"
|
||||
bucketReplTotalFailedCount = "total_failed_count"
|
||||
bucketReplProxiedDeleteTaggingRequestsFailures = "proxied_delete_tagging_requests_failures"
|
||||
bucketL = "bucket"
|
||||
operationL = "operation"
|
||||
targetArnL = "targetArn"
|
||||
)
|
||||
|
||||
var (
|
||||
bucketReplLastHrFailedBytesMD = NewGaugeMD(bucketReplLastHrFailedBytes,
|
||||
"Total number of bytes failed at least once to replicate in the last hour on a bucket",
|
||||
bucketL)
|
||||
bucketReplLastHrFailedCountMD = NewGaugeMD(bucketReplLastHrFailedCount,
|
||||
"Total number of objects which failed replication in the last hour on a bucket",
|
||||
bucketL)
|
||||
bucketReplLastMinFailedBytesMD = NewGaugeMD(bucketReplLastMinFailedBytes,
|
||||
"Total number of bytes failed at least once to replicate in the last full minute on a bucket",
|
||||
bucketL)
|
||||
bucketReplLastMinFailedCountMD = NewGaugeMD(bucketReplLastMinFailedCount,
|
||||
"Total number of objects which failed replication in the last full minute on a bucket",
|
||||
bucketL)
|
||||
bucketReplLatencyMsMD = NewGaugeMD(bucketReplLatencyMs,
|
||||
"Replication latency on a bucket in milliseconds",
|
||||
bucketL, operationL, rangeL, targetArnL)
|
||||
bucketReplProxiedDeleteTaggingRequestsTotalMD = NewCounterMD(bucketReplProxiedDeleteTaggingRequestsTotal,
|
||||
"Number of DELETE tagging requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplProxiedGetRequestsFailuresMD = NewCounterMD(bucketReplProxiedGetRequestsFailures,
|
||||
"Number of failures in GET requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplProxiedGetRequestsTotalMD = NewCounterMD(bucketReplProxiedGetRequestsTotal,
|
||||
"Number of GET requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplProxiedGetTaggingRequestsFailuresMD = NewCounterMD(bucketReplProxiedGetTaggingRequestsFailures,
|
||||
"Number of failures in GET tagging requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplProxiedGetTaggingRequestsTotalMD = NewCounterMD(bucketReplProxiedGetTaggingRequestsTotal,
|
||||
"Number of GET tagging requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplProxiedHeadRequestsFailuresMD = NewCounterMD(bucketReplProxiedHeadRequestsFailures,
|
||||
"Number of failures in HEAD requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplProxiedHeadRequestsTotalMD = NewCounterMD(bucketReplProxiedHeadRequestsTotal,
|
||||
"Number of HEAD requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplProxiedPutTaggingRequestsFailuresMD = NewCounterMD(bucketReplProxiedPutTaggingRequestsFailures,
|
||||
"Number of failures in PUT tagging requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplProxiedPutTaggingRequestsTotalMD = NewCounterMD(bucketReplProxiedPutTaggingRequestsTotal,
|
||||
"Number of PUT tagging requests proxied to replication target",
|
||||
bucketL)
|
||||
bucketReplSentBytesMD = NewCounterMD(bucketReplSentBytes,
|
||||
"Total number of bytes replicated to the target",
|
||||
bucketL)
|
||||
bucketReplSentCountMD = NewCounterMD(bucketReplSentCount,
|
||||
"Total number of objects replicated to the target",
|
||||
bucketL)
|
||||
bucketReplTotalFailedBytesMD = NewCounterMD(bucketReplTotalFailedBytes,
|
||||
"Total number of bytes failed at least once to replicate since server start",
|
||||
bucketL)
|
||||
bucketReplTotalFailedCountMD = NewCounterMD(bucketReplTotalFailedCount,
|
||||
"Total number of objects which failed replication since server start",
|
||||
bucketL)
|
||||
bucketReplProxiedDeleteTaggingRequestsFailuresMD = NewCounterMD(bucketReplProxiedDeleteTaggingRequestsFailures,
|
||||
"Number of failures in DELETE tagging requests proxied to replication target",
|
||||
bucketL)
|
||||
)
|
||||
|
||||
// loadBucketReplicationMetrics - `BucketMetricsLoaderFn` for bucket replication metrics
|
||||
// such as latency and sent bytes.
|
||||
func loadBucketReplicationMetrics(ctx context.Context, m MetricValues, c *metricsCache, buckets []string) error {
|
||||
if globalSiteReplicationSys.isEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
dataUsageInfo, err := c.dataUsageInfo.Get()
|
||||
if err != nil {
|
||||
metricsLogIf(ctx, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
bucketReplStats := globalReplicationStats.getAllLatest(dataUsageInfo.BucketsUsage)
|
||||
for _, bucket := range buckets {
|
||||
labels := []string{bucketL, bucket}
|
||||
if s, ok := bucketReplStats[bucket]; ok {
|
||||
stats := s.ReplicationStats
|
||||
if stats.hasReplicationUsage() {
|
||||
for arn, stat := range stats.Stats {
|
||||
m.Set(bucketReplLastHrFailedBytes, float64(stat.Failed.LastHour.Bytes), labels...)
|
||||
m.Set(bucketReplLastHrFailedCount, float64(stat.Failed.LastHour.Count), labels...)
|
||||
m.Set(bucketReplLastMinFailedBytes, float64(stat.Failed.LastMinute.Bytes), labels...)
|
||||
m.Set(bucketReplLastMinFailedCount, float64(stat.Failed.LastMinute.Count), labels...)
|
||||
m.Set(bucketReplProxiedDeleteTaggingRequestsTotal, float64(s.ProxyStats.RmvTagTotal), labels...)
|
||||
m.Set(bucketReplProxiedGetRequestsFailures, float64(s.ProxyStats.GetFailedTotal), labels...)
|
||||
m.Set(bucketReplProxiedGetRequestsTotal, float64(s.ProxyStats.GetTotal), labels...)
|
||||
m.Set(bucketReplProxiedGetTaggingRequestsFailures, float64(s.ProxyStats.GetTagFailedTotal), labels...)
|
||||
m.Set(bucketReplProxiedGetTaggingRequestsTotal, float64(s.ProxyStats.GetTagTotal), labels...)
|
||||
m.Set(bucketReplProxiedHeadRequestsFailures, float64(s.ProxyStats.HeadFailedTotal), labels...)
|
||||
m.Set(bucketReplProxiedHeadRequestsTotal, float64(s.ProxyStats.HeadTotal), labels...)
|
||||
m.Set(bucketReplProxiedPutTaggingRequestsFailures, float64(s.ProxyStats.PutTagFailedTotal), labels...)
|
||||
m.Set(bucketReplProxiedPutTaggingRequestsTotal, float64(s.ProxyStats.PutTagTotal), labels...)
|
||||
m.Set(bucketReplSentCount, float64(stat.ReplicatedCount), labels...)
|
||||
m.Set(bucketReplTotalFailedBytes, float64(stat.Failed.Totals.Bytes), labels...)
|
||||
m.Set(bucketReplTotalFailedCount, float64(stat.Failed.Totals.Count), labels...)
|
||||
m.Set(bucketReplProxiedDeleteTaggingRequestsFailures, float64(s.ProxyStats.RmvTagFailedTotal), labels...)
|
||||
m.Set(bucketReplSentBytes, float64(stat.ReplicatedSize), labels...)
|
||||
|
||||
SetHistogramValues(m, bucketReplLatencyMs, stat.Latency.getUploadLatency(), bucketL, bucket, operationL, "upload", targetArnL, arn)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user