mirror of
https://github.com/minio/minio.git
synced 2026-02-09 04:10:15 -05:00
re-implement data usage crawler to be more efficient (#9075)
Implementation overview: https://gist.github.com/klauspost/1801c858d5e0df391114436fdad6987b
This commit is contained in:
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
@@ -77,6 +78,11 @@ type objectHistogramInterval struct {
|
||||
start, end int64
|
||||
}
|
||||
|
||||
const (
|
||||
// dataUsageBucketLen must be length of ObjectsHistogramIntervals
|
||||
dataUsageBucketLen = 7
|
||||
)
|
||||
|
||||
// ObjectsHistogramIntervals is the list of all intervals
|
||||
// of object sizes to be included in objects histogram.
|
||||
var ObjectsHistogramIntervals = []objectHistogramInterval{
|
||||
@@ -86,20 +92,25 @@ var ObjectsHistogramIntervals = []objectHistogramInterval{
|
||||
{"BETWEEN_10_MB_AND_64_MB", 1024 * 1024 * 10, 1024*1024*64 - 1},
|
||||
{"BETWEEN_64_MB_AND_128_MB", 1024 * 1024 * 64, 1024*1024*128 - 1},
|
||||
{"BETWEEN_128_MB_AND_512_MB", 1024 * 1024 * 128, 1024*1024*512 - 1},
|
||||
{"GREATER_THAN_512_MB", 1024 * 1024 * 512, -1},
|
||||
{"GREATER_THAN_512_MB", 1024 * 1024 * 512, math.MaxInt64},
|
||||
}
|
||||
|
||||
// DataUsageInfo represents data usage stats of the underlying Object API
|
||||
type DataUsageInfo struct {
|
||||
// The timestamp of when the data usage info is generated
|
||||
// LastUpdate is the timestamp of when the data usage info was last updated.
|
||||
// This does not indicate a full scan.
|
||||
LastUpdate time.Time `json:"lastUpdate"`
|
||||
|
||||
ObjectsCount uint64 `json:"objectsCount"`
|
||||
// Objects total size
|
||||
ObjectsTotalSize uint64 `json:"objectsTotalSize"`
|
||||
ObjectsTotalSize uint64 `json:"objectsTotalSize"`
|
||||
|
||||
// ObjectsSizesHistogram contains information on objects across all buckets.
|
||||
// See ObjectsHistogramIntervals.
|
||||
ObjectsSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
|
||||
|
||||
BucketsCount uint64 `json:"bucketsCount"`
|
||||
BucketsCount uint64 `json:"bucketsCount"`
|
||||
// BucketsSizes is "bucket name" -> size.
|
||||
BucketsSizes map[string]uint64 `json:"bucketsSizes"`
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user