Added 'close' to results channel in Walk() (#9956)

Readiness as no reasoning to be cluster scope
because that is not how the k8s networking works
for pods, all the pods to a deployment are not
sharing the network in a singleton. Instead they
are run as local scopes to themselves, with
readiness failures the pod is potentially taken
out of the network to be resolvable - this
affects the distributed setup in myriad of
different ways.

Instead readiness should behave like liveness
with local scope alone, and should be a dummy
implementation.

This PR all the startup times and overal k8s
startup time dramatically improves.

Added another handler called as `/minio/health/cluster`
to understand the cluster scope health.

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386"
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64"
 }
 
 function _build() {

buildscripts/verify-healing.sh

@@ -33,6 +33,7 @@ function start_minio_3_node() {
     declare -a ARGS
     export MINIO_ACCESS_KEY=minio
     export MINIO_SECRET_KEY=minio123
+    export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
     start_port=$(shuf -i 10000-65000 -n 1)
     for i in $(seq 1 3); do

cmd/admin-handlers-config-kv.go

@@ -173,7 +173,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	cfg := globalServerConfig
-	if globalSafeMode {
+	if newObjectLayerFn() == nil {
 		var err error
 		cfg, err = getValidConfig(objectAPI)
 		if err != nil {

cmd/admin-handlers.go

@@ -708,10 +708,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// find number of disks in the setup, ignore any errors here.
-	info, _ := objectAPI.StorageInfo(ctx, false)
-	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
-
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
 	if hip.clientToken == "" && !hip.forceStart && !hip.forceStop {
 		nh, exists := globalAllHealState.getHealSequence(healPath)
@@ -754,7 +750,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 			respCh <- hr
 		}()
 	case hip.clientToken == "":
-		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), numDisks, hip.hs, hip.forceStart)
+		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart)
 		go func() {
 			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
 			hr := healResp{respBytes, apiErr, errMsg}
@@ -1399,10 +1395,8 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	mode := ""
-	if globalSafeMode {
-		mode = "safe"
-	} else {
+	mode := "safe"
+	if newObjectLayerFn() != nil {
 		mode = "online"
 	}
 

cmd/admin-heal-ops.go

@@ -77,9 +77,6 @@ type healSequenceStatus struct {
 	FailureDetail string            `json:"Detail,omitempty"`
 	StartTime     time.Time         `json:"StartTime"`
 
-	// disk information
-	NumDisks int `json:"NumDisks"`
-
 	// settings for the heal sequence
 	HealSettings madmin.HealOpts `json:"Settings"`
 
@@ -95,8 +92,8 @@ type allHealState struct {
 	healSeqMap map[string]*healSequence
 }
 
-// initHealState - initialize healing apparatus
-func initHealState() *allHealState {
+// newHealState - initialize global heal state management
+func newHealState() *allHealState {
 	healState := &allHealState{
 		healSeqMap: make(map[string]*healSequence),
 	}
@@ -368,7 +365,7 @@ type healSequence struct {
 // NewHealSequence - creates healSettings, assumes bucket and
 // objPrefix are already validated.
 func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
-	numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
+	hs madmin.HealOpts, forceStart bool) *healSequence {
 
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
@@ -388,7 +385,6 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
-			NumDisks:     numDisks,
 		},
 		traverseAndHealDoneCh: make(chan error),
 		cancelCtx:             cancel,
@@ -677,11 +673,6 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	bucketsOnly := true // heal buckets only, not objects.
-	if err := h.healItems(bucketsOnly); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
-
 	for {
 		select {
 		case source, ok := <-h.sourceCh:
@@ -716,7 +707,7 @@ func (h *healSequence) healFromSourceCh() {
 	h.healItemsFromSourceCh()
 }
 
-func (h *healSequence) healItems(bucketsOnly bool) error {
+func (h *healSequence) healDiskMeta() error {
 	// Start with format healing
 	if err := h.healDiskFormat(); err != nil {
 		return err
@@ -728,7 +719,11 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 	}
 
 	// Start healing the bucket config prefix.
-	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
+	return h.healMinioSysMeta(bucketConfigPrefix)()
+}
+
+func (h *healSequence) healItems(bucketsOnly bool) error {
+	if err := h.healDiskMeta(); err != nil {
 		return err
 	}
 

cmd/background-heal-ops.go

@@ -100,7 +100,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 	}
 }
 
-func initHealRoutine() *healRoutine {
+func newHealRoutine() *healRoutine {
 	return &healRoutine{
 		tasks:  make(chan healTask),
 		doneCh: make(chan struct{}),
@@ -108,22 +108,22 @@ func initHealRoutine() *healRoutine {
 
 }
 
-func startBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	// Run the background healer
-	globalBackgroundHealRoutine = initHealRoutine()
+	globalBackgroundHealRoutine = newHealRoutine()
 	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
-	// Launch the background healer sequence to track
-	// background healing operations, ignore errors
-	// errors are handled into offline disks already.
-	info, _ := objAPI.StorageInfo(ctx, false)
-	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
-	nh := newBgHealSequence(numDisks)
-	globalBackgroundHealState.LaunchNewHealSequence(nh)
-}
+	nh := newBgHealSequence()
+	// Heal any disk format and metadata early, if possible.
+	if err := nh.healDiskMeta(); err != nil {
+		if newObjectLayerFn() != nil {
+			// log only in situations, when object layer
+			// has fully initialized.
+			logger.LogIf(nh.ctx, err)
+		}
+	}
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	go startBackgroundHealing(ctx, objAPI)
+	globalBackgroundHealState.LaunchNewHealSequence(nh)
 }
 
 // healDiskFormat - heals format.json, return value indicates if a
@@ -140,12 +140,20 @@ func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpt
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
 	// We will not notify peers if healing is not required.
 	if err == nil {
-		for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+		// Notify servers in background and retry if needed.
+		go func() {
+		retry:
+			for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
+				if nerr.Err != nil {
+					if nerr.Err.Error() == errServerNotInitialized.Error() {
+						time.Sleep(time.Second)
+						goto retry
+					}
+					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+					logger.LogIf(ctx, nerr.Err)
+				}
 			}
-		}
+		}()
 	}
 
 	return res, nil

cmd/bucket-handlers.go

@@ -47,7 +47,7 @@ import (
 const (
 	getBucketVersioningResponse = `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
 	objectLockConfig            = "object-lock.xml"
-	bucketTaggingConfigFile     = "tagging.xml"
+	bucketTaggingConfig         = "tagging.xml"
 )
 
 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -1123,7 +1123,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1191,7 +1191,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, nil); err != nil {
+	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -76,7 +76,44 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 
 	if globalIsGateway {
 		// This code is needed only for gateway implementations.
-		if configFile == bucketPolicyConfig {
+		switch configFile {
+		case bucketSSEConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.EncryptionConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketLifecycleConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.LifecycleConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketTaggingConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.TaggingConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketNotificationConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.NotificationConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketPolicyConfig:
 			if configData == nil {
 				return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
 			}
@@ -107,7 +144,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		meta.LifecycleConfigXML = configData
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
-	case bucketTaggingConfigFile:
+	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
 	case objectLockConfig:
 		meta.ObjectLockConfigXML = configData

cmd/bucket-metadata.go

@@ -206,7 +206,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 		bucketLifecycleConfig,
 		bucketQuotaConfigFile,
 		bucketSSEConfig,
-		bucketTaggingConfigFile,
+		bucketTaggingConfig,
 		objectLockConfig,
 	}
 
@@ -255,7 +255,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.LifecycleConfigXML = configData
 		case bucketSSEConfig:
 			b.EncryptionConfigXML = configData
-		case bucketTaggingConfigFile:
+		case bucketTaggingConfig:
 			b.TaggingConfigXML = configData
 		case objectLockConfig:
 			b.ObjectLockConfigXML = configData
@@ -303,9 +303,15 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 // deleteBucketMetadata deletes bucket metadata
 // If config does not exist no error is returned.
 func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketMetadataFile)
-	if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
-		return err
+	metadataFiles := []string{
+		dataUsageCacheName,
+		bucketMetadataFile,
+	}
+	for _, metaFile := range metadataFiles {
+		configFile := path.Join(bucketConfigPrefix, bucket, metaFile)
+		if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
+			return err
+		}
 	}
 	return nil
 }

cmd/config-current.go

@@ -325,13 +325,40 @@ func lookupConfigs(s config.Config) {
 
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to initialize etcd config")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		}
 	}
 
 	if etcdCfg.Enabled {
-		globalEtcdClient, err = etcd.New(etcdCfg)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		if globalEtcdClient == nil {
+			globalEtcdClient, err = etcd.New(etcdCfg)
+			if err != nil {
+				if globalIsGateway {
+					logger.FatalIf(err, "Unable to initialize etcd config")
+				} else {
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+				}
+			}
+		}
+
+		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil && globalDNSConfig == nil {
+			globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
+				dns.DomainNames(globalDomainNames),
+				dns.DomainIPs(globalDomainIPs),
+				dns.DomainPort(globalMinioPort),
+				dns.CoreDNSPath(etcdCfg.CoreDNSPath),
+			)
+			if err != nil {
+				if globalIsGateway {
+					logger.FatalIf(err, "Unable to initialize DNS config")
+				} else {
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+						globalDomainNames, err))
+				}
+			}
 		}
 	}
 
@@ -342,19 +369,6 @@ func lookupConfigs(s config.Config) {
 	// but not federation.
 	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
 
-	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
-		globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
-			dns.DomainNames(globalDomainNames),
-			dns.DomainIPs(globalDomainIPs),
-			dns.DomainPort(globalMinioPort),
-			dns.CoreDNSPath(etcdCfg.CoreDNSPath),
-		)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
-				globalDomainNames, err))
-		}
-	}
-
 	globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
@@ -377,7 +391,11 @@ func lookupConfigs(s config.Config) {
 
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to setup cache")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
+		}
 	}
 
 	if globalCacheConfig.Enabled {

cmd/config.go

@@ -149,7 +149,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 	if err != nil {
 		// Config not found for some reason, allow things to continue
 		// by initializing a new fresh config in safe mode.
-		if err == errConfigNotFound && globalSafeMode {
+		if err == errConfigNotFound && newObjectLayerFn() == nil {
 			return newServerConfig(), nil
 		}
 		return nil, err

cmd/config/api/help.go

@@ -33,12 +33,6 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
-		config.HelpKV{
-			Key:         apiReadyDeadline,
-			Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`,
-			Optional:    true,
-			Type:        "duration",
-		},
 		config.HelpKV{
 			Key:         apiCorsAllowOrigin,
 			Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,

cmd/config/cache/config.go

@@ -37,6 +37,7 @@ type Config struct {
 	After         int      `json:"after"`
 	WatermarkLow  int      `json:"watermark_low"`
 	WatermarkHigh int      `json:"watermark_high"`
+	Range         bool     `json:"range"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling

cmd/config/cache/help.go

@@ -68,5 +68,11 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         Range,
+			Description: `set to "on" or "off" caching of independent range requests per object, defaults to "on"`,
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -34,6 +34,7 @@ const (
 	After         = "after"
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
+	Range         = "range"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -43,6 +44,7 @@ const (
 	EnvCacheAfter         = "MINIO_CACHE_AFTER"
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
+	EnvCacheRange         = "MINIO_CACHE_RANGE"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -84,6 +86,10 @@ var (
 			Key:   WatermarkHigh,
 			Value: DefaultWaterMarkHigh,
 		},
+		config.KV{
+			Key:   Range,
+			Value: config.EnableOn,
+		},
 	}
 )
 
@@ -195,5 +201,15 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		err := errors.New("config high watermark value should be greater than low watermark value")
 		return cfg, config.ErrInvalidCacheWatermarkHigh(err)
 	}
+
+	cfg.Range = true // by default range caching is enabled.
+	if rangeStr := env.Get(EnvCacheRange, kvs.Get(Range)); rangeStr != "" {
+		rng, err := config.ParseBool(rangeStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheRange(err)
+		}
+		cfg.Range = rng
+	}
+
 	return cfg, nil
 }

cmd/config/errors.go

@@ -96,6 +96,12 @@ var (
 		"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
 	)
 
+	ErrInvalidCacheRange = newErrFn(
+		"Invalid cache range value",
+		"Please check the passed value",
+		"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
+	)
+
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",

cmd/config/etcd/dns/etcd_dns.go

@@ -47,6 +47,11 @@ func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, err
 	})
 }
 
+// Close closes the internal etcd client and cannot be used further
+func (c *CoreDNS) Close() {
+	c.etcdClient.Close()
+}
+
 // List - Retrieves list of DNS entries for the domain.
 func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
 	var srvRecords = map[string][]SrvRecord{}

cmd/disk-cache-backend.go

@@ -33,6 +33,7 @@ import (
 	"time"
 
 	"github.com/djherbis/atime"
+	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -125,16 +126,19 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 
 // represents disk cache struct
 type diskCache struct {
-	gcCounter uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	// is set to 0 if drive is offline
-	online uint32
+	online       uint32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	purgeRunning int32
 
-	dir           string // caching directory
-	quotaPct      int    // max usage in %
+	triggerGC     chan struct{}
+	dir           string         // caching directory
+	stats         CacheDiskStats // disk cache stats for prometheus
+	quotaPct      int            // max usage in %
 	pool          sync.Pool
 	after         int // minimum accesses before an object is cached.
 	lowWatermark  int
 	highWatermark int
+	enableRange   bool
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
@@ -142,16 +146,24 @@ type diskCache struct {
 }
 
 // Inits the disk cache dir if it is not initialized already.
-func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
+func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCache, error) {
+	quotaPct := config.MaxUse
+	if quotaPct == 0 {
+		quotaPct = config.Quota
+	}
+
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
 		dir:           dir,
+		triggerGC:     make(chan struct{}),
+		stats:         CacheDiskStats{Dir: dir},
 		quotaPct:      quotaPct,
-		after:         after,
-		lowWatermark:  lowWatermark,
-		highWatermark: highWatermark,
+		after:         config.After,
+		lowWatermark:  config.WatermarkLow,
+		highWatermark: config.WatermarkHigh,
+		enableRange:   config.Range,
 		online:        1,
 		pool: sync.Pool{
 			New: func() interface{} {
@@ -161,6 +173,8 @@ func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int)
 		},
 		nsMutex: newNSLock(false),
 	}
+	go cache.purgeWait(ctx)
+	cache.diskUsageHigh() // update if cache usage is already high.
 	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
 		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
 	}
@@ -181,7 +195,12 @@ func (c *diskCache) diskUsageLow() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) < gcStopPct
+	low := int(usedPercent) < gcStopPct
+	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	if low {
+		atomic.StoreInt32(&c.stats.UsageState, 0)
+	}
+	return low
 }
 
 // Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
@@ -196,7 +215,12 @@ func (c *diskCache) diskUsageHigh() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) >= gcTriggerPct
+	high := int(usedPercent) >= gcTriggerPct
+	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	if high {
+		atomic.StoreInt32(&c.stats.UsageState, 1)
+	}
+	return high
 }
 
 // Returns if size space can be allocated without exceeding
@@ -230,24 +254,36 @@ var (
 	errDoneForNow = errors.New("done for now")
 )
 
+func (c *diskCache) purgeWait(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+		case <-c.triggerGC: // wait here until someone triggers.
+			c.purge(ctx)
+		}
+	}
+}
+
 // Purge cache entries that were not accessed.
 func (c *diskCache) purge(ctx context.Context) {
-	if c.diskUsageLow() {
+	if atomic.LoadInt32(&c.purgeRunning) == 1 || c.diskUsageLow() {
 		return
 	}
+
 	toFree := c.toClear()
 	if toFree == 0 {
 		return
 	}
+
+	atomic.StoreInt32(&c.purgeRunning, 1) // do not run concurrent purge()
+	defer atomic.StoreInt32(&c.purgeRunning, 0)
+
 	// expiry for cleaning up old cache.json files that
 	// need to be cleaned up.
 	expiry := UTCNow().Add(-cacheExpiryDays)
 	// defaulting max hits count to 100
-	scorer, err := newFileScorer(toFree, time.Now().Unix(), 100)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
+	// ignore error we know what value we are passing.
+	scorer, _ := newFileScorer(toFree, time.Now().Unix(), 100)
 
 	// this function returns FileInfo for cached range files and cache data file.
 	fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
@@ -326,27 +362,20 @@ func (c *diskCache) purge(ctx context.Context) {
 		return
 	}
 
-	for _, path := range scorer.fileNames() {
-		removeAll(path)
-		slashIdx := strings.LastIndex(path, SlashSeparator)
-		pathPrefix := path[0:slashIdx]
-		fname := path[slashIdx+1:]
-		if fname == cacheDataFile {
-			removeAll(pathPrefix)
+	scorer.purgeFunc(func(qfile queuedFile) {
+		fileName := qfile.name
+		removeAll(fileName)
+		slashIdx := strings.LastIndex(fileName, SlashSeparator)
+		if slashIdx >= 0 {
+			fileNamePrefix := fileName[0:slashIdx]
+			fname := fileName[slashIdx+1:]
+			if fname == cacheDataFile {
+				removeAll(fileNamePrefix)
+			}
 		}
-	}
-}
+	})
 
-func (c *diskCache) incGCCounter() {
-	atomic.AddUint64(&c.gcCounter, 1)
-}
-
-func (c *diskCache) resetGCCounter() {
-	atomic.StoreUint64(&c.gcCounter, 0)
-}
-
-func (c *diskCache) gcCount() uint64 {
-	return atomic.LoadUint64(&c.gcCounter)
+	scorer.reset()
 }
 
 // sets cache drive status
@@ -630,7 +659,7 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
 	if c.diskUsageHigh() {
-		c.incGCCounter()
+		c.triggerGC <- struct{}{}
 		io.Copy(ioutil.Discard, data)
 		return errDiskFull
 	}

cmd/disk-cache-stats.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019, 2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,14 +16,27 @@
 
 package cmd
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+)
+
+// CacheDiskStats represents cache disk statistics
+// such as current disk usage and available.
+type CacheDiskStats struct {
+	// indicates if usage is high or low, if high value is '1', if low its '0'
+	UsageState int32
+	// indicates the current usage percentage of this cache disk
+	UsagePercent uint64
+	Dir          string
+}
 
 // CacheStats - represents bytes served from cache,
 // cache hits and cache misses.
 type CacheStats struct {
-	BytesServed uint64
-	Hits        uint64
-	Misses      uint64
+	BytesServed  uint64
+	Hits         uint64
+	Misses       uint64
+	GetDiskStats func() []CacheDiskStats
 }
 
 // Increase total bytes served from cache

cmd/disk-cache-utils.go

@@ -404,6 +404,14 @@ func (f *fileScorer) trimQueue() {
 	}
 }
 
+func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
+	e := f.queue.Front()
+	for e != nil {
+		p(e.Value.(queuedFile))
+		e = e.Next()
+	}
+}
+
 // fileNames returns all queued file names.
 func (f *fileScorer) fileNames() []string {
 	res := make([]string, 0, f.queue.Len())

cmd/disk-cache.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio/cmd/config/cache"
@@ -264,10 +265,11 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
+
 	// Since we got here, we are serving the request from backend,
 	// and also adding the object to the cache.
 	if dcache.diskUsageHigh() {
-		dcache.incGCCounter()
+		dcache.triggerGC <- struct{}{} // this is non-blocking
 	}
 
 	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
@@ -292,6 +294,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 	if rs != nil {
 		go func() {
+			// if range caching is disabled, download entire object.
+			if !dcache.enableRange {
+				rs = nil
+			}
 			// fill cache in the background for range GET requests
 			bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 			if bErr != nil {
@@ -301,7 +307,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			oi, _, _, err := dcache.statRange(ctx, bucket, object, rs)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				// use a new context to avoid locker prematurely timing out operation when the GetObjectNInfo returns.
+				dcache.Put(context.Background(), bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				return
 			}
 		}()
 		return bkReader, bkErr
@@ -522,11 +530,7 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 			return nil, false, errors.New("Atime support required for disk caching")
 		}
 
-		quota := config.MaxUse
-		if quota == 0 {
-			quota = config.Quota
-		}
-		cache, err := newDiskCache(dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
+		cache, err := newDiskCache(ctx, dir, config)
 		if err != nil {
 			return nil, false, err
 		}
@@ -666,7 +670,17 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
 		},
 	}
-
+	c.cacheStats.GetDiskStats = func() []CacheDiskStats {
+		cacheDiskStats := make([]CacheDiskStats, len(c.cache))
+		for i := range c.cache {
+			cacheDiskStats[i] = CacheDiskStats{
+				Dir: c.cache[i].stats.Dir,
+			}
+			atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&c.cache[i].stats.UsageState))
+			atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&c.cache[i].stats.UsagePercent))
+		}
+		return cacheDiskStats
+	}
 	if migrateSw {
 		go c.migrateCacheFromV1toV2(ctx)
 	}
@@ -686,19 +700,9 @@ func (c *cacheObjects) gc(ctx context.Context) {
 			if c.migrating {
 				continue
 			}
-			var wg sync.WaitGroup
 			for _, dcache := range c.cache {
-				if dcache.gcCount() == 0 {
-					continue
-				}
-				wg.Add(1)
-				go func(d *diskCache) {
-					defer wg.Done()
-					d.resetGCCounter()
-					d.purge(ctx)
-				}(dcache)
+				dcache.triggerGC <- struct{}{}
 			}
-			wg.Wait()
 		}
 	}
 }

cmd/endpoint.go

@@ -44,8 +44,6 @@ const (
 
 	// URLEndpointType - URL style endpoint type enum.
 	URLEndpointType
-
-	retryInterval = 5 // In Seconds.
 )
 
 // Endpoint - any type of endpoint.
@@ -302,7 +300,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 	resolvedList := make([]bool, len(endpoints))
 	// Mark the starting time
 	startTime := time.Now()
-	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
+	keepAliveTicker := time.NewTicker(10 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 	for {
 		// Break if the local endpoint is found already Or all the endpoints are resolved.

cmd/erasure-decode.go

@@ -145,9 +145,9 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
-			disk := p.readers[i]
-			if disk == nil {
-				// Since disk is nil, trigger another read.
+			rr := p.readers[i]
+			if rr == nil {
+				// Since reader is nil, trigger another read.
 				readTriggerCh <- true
 				return
 			}
@@ -160,7 +160,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 			// For the last shard, the shardsize might be less than previous shard sizes.
 			// Hence the following statement ensures that the buffer size is reset to the right size.
 			p.buf[bufIdx] = p.buf[bufIdx][:p.shardSize]
-			_, err := disk.ReadAt(p.buf[bufIdx], p.offset)
+			_, err := rr.ReadAt(p.buf[bufIdx], p.offset)
 			if err != nil {
 				if _, ok := err.(*errHashMismatch); ok {
 					atomic.StoreInt32(&healRequired, 1)

cmd/fs-v1.go

@@ -283,6 +283,13 @@ func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter,
 		}
 		logger.LogIf(ctx, err)
 		cache.Info.BloomFilter = nil
+
+		if cache.root() == nil {
+			if intDataUpdateTracker.debug {
+				logger.Info(color.Green("CrawlAndGetDataUsage:") + " No root added. Adding empty")
+			}
+			cache.replace(cache.Info.Name, dataUsageRoot, dataUsageEntry{})
+		}
 		if cache.Info.LastUpdate.After(bCache.Info.LastUpdate) {
 			if intDataUpdateTracker.debug {
 				logger.Info(color.Green("CrawlAndGetDataUsage:")+" Saving bucket %q cache with %d entries", b.Name, len(cache.Cache))
@@ -1479,9 +1486,5 @@ func (fs *FSObjects) IsReady(_ context.Context) bool {
 		return false
 	}
 
-	globalObjLayerMutex.RLock()
-	res := globalObjectAPI != nil && !globalSafeMode
-	globalObjLayerMutex.RUnlock()
-
-	return res
+	return newObjectLayerFn() != nil
 }

cmd/gateway-main.go

@@ -45,6 +45,61 @@ var (
 	}
 )
 
+// GatewayLocker implements custom NewNSLock implementation
+type GatewayLocker struct {
+	ObjectLayer
+	nsMutex *nsLockMap
+}
+
+// NewNSLock - implements gateway level locker
+func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
+}
+
+// Walk - implements common gateway level Walker, to walk on all objects recursively at a prefix
+func (l *GatewayLocker) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+	walk := func(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+		var marker string
+
+		// Make sure the results channel is ready to be read when we're done.
+		defer close(results)
+
+		for {
+			// set maxKeys to '0' to list maximum possible objects in single call.
+			loi, err := l.ObjectLayer.ListObjects(ctx, bucket, prefix, marker, "", 0)
+			if err != nil {
+				return err
+			}
+			marker = loi.NextMarker
+			for _, obj := range loi.Objects {
+				select {
+				case results <- obj:
+				case <-ctx.Done():
+					return nil
+				}
+			}
+			if !loi.IsTruncated {
+				break
+			}
+		}
+		return nil
+	}
+
+	if err := l.ObjectLayer.Walk(ctx, bucket, prefix, results); err != nil {
+		if _, ok := err.(NotImplemented); ok {
+			return walk(ctx, bucket, prefix, results)
+		}
+		return err
+	}
+
+	return nil
+}
+
+// NewGatewayLayerWithLocker - initialize gateway with locker.
+func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
+	return &GatewayLocker{ObjectLayer: gwLayer, nsMutex: newNSLock(false)}
+}
+
 // RegisterGatewayCommand registers a new command for gateway.
 func RegisterGatewayCommand(cmd cli.Command) error {
 	cmd.Flags = append(append(cmd.Flags, ServerFlags...), GlobalFlags...)
@@ -147,7 +202,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	gatewayHandleEnvVars()
 
 	// Set system resources to maximum.
-	logger.LogIf(GlobalContext, setMaxResources())
+	setMaxResources()
 
 	// Set when gateway is enabled
 	globalIsGateway = true

cmd/gateway-unsupported.go

@@ -30,22 +30,6 @@ import (
 	"github.com/minio/minio/pkg/madmin"
 )
 
-// GatewayLocker implements custom NeNSLock implementation
-type GatewayLocker struct {
-	ObjectLayer
-	nsMutex *nsLockMap
-}
-
-// NewNSLock - implements gateway level locker
-func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
-}
-
-// NewGatewayLayerWithLocker - initialize gateway with locker.
-func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
-	return &GatewayLocker{ObjectLayer: gwLayer, nsMutex: newNSLock(false)}
-}
-
 // GatewayUnsupported list of unsupported call stubs for gateway.
 type GatewayUnsupported struct{}
 

cmd/global-heal.go

@@ -34,7 +34,7 @@ var leaderLockTimeout = newDynamicTimeout(time.Minute, time.Minute)
 
 // NewBgHealSequence creates a background healing sequence
 // operation which crawls all objects and heal them.
-func newBgHealSequence(numDisks int) *healSequence {
+func newBgHealSequence() *healSequence {
 
 	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
 	ctx, cancelCtx := context.WithCancel(logger.SetReqInfo(GlobalContext, reqInfo))
@@ -54,7 +54,6 @@ func newBgHealSequence(numDisks int) *healSequence {
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
-			NumDisks:     numDisks,
 		},
 		cancelCtx:          cancelCtx,
 		ctx:                ctx,

cmd/handler-utils.go

@@ -401,7 +401,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioPeerVersionMismatch",
 			Description:    desc,
-			HTTPStatusCode: http.StatusBadRequest,
+			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
 		desc := fmt.Sprintf("Expected 'storage' API version '%s', instead found '%s', please upgrade the servers",
@@ -409,7 +409,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioStorageVersionMismatch",
 			Description:    desc,
-			HTTPStatusCode: http.StatusBadRequest,
+			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
 		desc := fmt.Sprintf("Expected 'lock' API version '%s', instead found '%s', please upgrade the servers",
@@ -417,7 +417,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioLockVersionMismatch",
 			Description:    desc,
-			HTTPStatusCode: http.StatusBadRequest,
+			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, adminPathPrefix):
 		var desc string
@@ -431,7 +431,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(r.Context(), w, APIError{
 			Code:           "XMinioAdminVersionMismatch",
 			Description:    desc,
-			HTTPStatusCode: http.StatusBadRequest,
+			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	default:
 		desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)

cmd/healthcheck-handler.go

@@ -21,13 +21,11 @@ import (
 	"net/http"
 )
 
-// ReadinessCheckHandler returns if the server is ready to receive requests.
-// For FS - Checks if the backend disk is available
-// For Erasure backend - Checks if all the erasure sets are writable
-func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ReadinessCheckHandler")
+// ClusterCheckHandler returns if the server is ready for requests.
+func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClusterCheckCheckHandler")
 
-	objLayer := newObjectLayerWithoutSafeModeFn()
+	objLayer := newObjectLayerFn()
 	// Service not initialized yet
 	if objLayer == nil {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
@@ -37,7 +35,7 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getReadyDeadline())
 	defer cancel()
 
-	if !objLayer.IsReady(ctx) && newObjectLayerFn() == nil {
+	if !objLayer.IsReady(ctx) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
@@ -45,6 +43,13 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
+// ReadinessCheckHandler Checks if the process is up. Always returns success.
+func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	// TODO: only implement this function to notify that this pod is
+	// busy, at a local scope in future, for now '200 OK'.
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}
+
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)

cmd/healthcheck-router.go

@@ -26,6 +26,7 @@ const (
 	healthCheckPath          = "/health"
 	healthCheckLivenessPath  = "/live"
 	healthCheckReadinessPath = "/ready"
+	healthCheckClusterPath   = "/cluster"
 	healthCheckPathPrefix    = minioReservedBucketPath + healthCheckPath
 )
 
@@ -35,6 +36,9 @@ func registerHealthCheckRouter(router *mux.Router) {
 	// Healthcheck router
 	healthRouter := router.PathPrefix(healthCheckPathPrefix).Subrouter()
 
+	// Cluster check handler to verify cluster is active
+	healthRouter.Methods(http.MethodGet).Path(healthCheckClusterPath).HandlerFunc(httpTraceAll(ClusterCheckHandler))
+
 	// Liveness handler
 	healthRouter.Methods(http.MethodGet).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler))
 	healthRouter.Methods(http.MethodHead).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler))

cmd/lock-rest-client.go

@@ -18,15 +18,14 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"errors"
 	"io"
-	"sync/atomic"
-	"time"
-
 	"net/url"
 
 	"github.com/minio/minio/cmd/http"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/dsync"
@@ -36,7 +35,6 @@ import (
 type lockRESTClient struct {
 	restClient *rest.Client
 	endpoint   Endpoint
-	connected  int32
 }
 
 func toLockError(err error) error {
@@ -62,10 +60,6 @@ func (client *lockRESTClient) String() string {
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *lockRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	if !client.IsOnline() {
-		return nil, errors.New("Lock rest server node is down")
-	}
-
 	if values == nil {
 		values = make(url.Values)
 	}
@@ -75,26 +69,16 @@ func (client *lockRESTClient) call(method string, values url.Values, body io.Rea
 		return respBody, nil
 	}
 
-	if isNetworkError(err) {
-		time.AfterFunc(time.Second, func() {
-			// After 1 seconds, take this lock client online for a retry.
-			atomic.StoreInt32(&client.connected, 1)
-		})
-
-		atomic.StoreInt32(&client.connected, 0)
-	}
-
 	return nil, toLockError(err)
 }
 
 // IsOnline - returns whether REST client failed to connect or not.
 func (client *lockRESTClient) IsOnline() bool {
-	return atomic.LoadInt32(&client.connected) == 1
+	return client.restClient.IsOnline()
 }
 
 // Close - marks the client as closed.
 func (client *lockRESTClient) Close() error {
-	atomic.StoreInt32(&client.connected, 0)
 	client.restClient.Close()
 	return nil
 }
@@ -173,9 +157,16 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return &lockRESTClient{endpoint: endpoint, restClient: restClient, connected: 0}
+		logger.Fatal(err, "Unable to create lock rest client")
+	}
+	restClient.HealthCheckFn = func() bool {
+		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
+		respBody, err := restClient.CallWithContext(ctx, lockRESTMethodHealth, nil, nil, -1)
+		xhttp.DrainBody(respBody)
+		cancel()
+		var ne *rest.NetworkError
+		return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
 	}
 
-	return &lockRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}
+	return &lockRESTClient{endpoint: endpoint, restClient: restClient}
 }

cmd/lock-rest-client_test.go

@@ -29,7 +29,7 @@ func TestLockRESTlient(t *testing.T) {
 		t.Fatalf("unexpected error %v", err)
 	}
 	lkClient := newlockRESTClient(endpoint)
-	if lkClient.connected == 0 {
+	if !lkClient.IsOnline() {
 		t.Fatalf("unexpected error. connection failed")
 	}
 

cmd/lock-rest-server-common.go

@@ -27,6 +27,7 @@ const (
 )
 
 const (
+	lockRESTMethodHealth  = "/health"
 	lockRESTMethodLock    = "/lock"
 	lockRESTMethodRLock   = "/rlock"
 	lockRESTMethodUnlock  = "/unlock"

cmd/lock-rest-server.go

@@ -78,6 +78,11 @@ func getLockArgs(r *http.Request) (args dsync.LockArgs, err error) {
 	return args, nil
 }
 
+// HealthHandler returns success if request is authenticated.
+func (l *lockRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {
+	l.IsValid(w, r)
+}
+
 // LockHandler - Acquires a lock.
 func (l *lockRESTServer) LockHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
@@ -345,6 +350,7 @@ func registerLockRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
 			}
 
 			subrouter := router.PathPrefix(path.Join(lockRESTPrefix, endpoint.Path)).Subrouter()
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(httpTraceHdrs(lockServer.HealthHandler)).Queries(queries...)
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler)).Queries(queries...)
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler)).Queries(queries...)
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler)).Queries(queries...)

cmd/merge-walk-pool.go

@@ -28,6 +28,7 @@ const (
 
 // mergeWalk - represents the go routine that does the merge walk.
 type mergeWalk struct {
+	added      time.Time
 	entryChs   []FileInfoCh
 	endWalkCh  chan struct{}   // To signal when mergeWalk go-routine should end.
 	endTimerCh chan<- struct{} // To signal when timer go-routine should end.
@@ -56,7 +57,7 @@ func NewMergeWalkPool(timeout time.Duration) *MergeWalkPool {
 // Release - selects a mergeWalk from the pool based on the input
 // listParams, removes it from the pool, and returns the MergeWalkResult
 // channel.
-// Returns nil if listParams does not have an asccociated mergeWalk.
+// Returns nil if listParams does not have an associated mergeWalk.
 func (t *MergeWalkPool) Release(params listParams) ([]FileInfoCh, chan struct{}) {
 	t.Lock()
 	defer t.Unlock()
@@ -65,6 +66,7 @@ func (t *MergeWalkPool) Release(params listParams) ([]FileInfoCh, chan struct{})
 		if len(walks) > 0 {
 			// Pop out the first valid walk entry.
 			walk := walks[0]
+			walks[0] = mergeWalk{} // clear references.
 			walks = walks[1:]
 			if len(walks) > 0 {
 				t.pool[params] = walks
@@ -91,17 +93,54 @@ func (t *MergeWalkPool) Set(params listParams, resultChs []FileInfoCh, endWalkCh
 	t.Lock()
 	defer t.Unlock()
 
+	// If we are above the limit delete at least one entry from the pool.
+	if len(t.pool) > treeWalkEntryLimit {
+		age := time.Now()
+		var oldest listParams
+		for k, v := range t.pool {
+			if len(v) == 0 {
+				delete(t.pool, k)
+				continue
+			}
+			// The first element is the oldest, so we only check that.
+			if v[0].added.Before(age) {
+				oldest = k
+			}
+		}
+		// Invalidate and delete oldest.
+		if walks, ok := t.pool[oldest]; ok {
+			walk := walks[0]
+			walks[0] = mergeWalk{} // clear references.
+			walks = walks[1:]
+			if len(walks) > 0 {
+				t.pool[params] = walks
+			} else {
+				delete(t.pool, params)
+			}
+			walk.endTimerCh <- struct{}{}
+		}
+	}
+
 	// Should be a buffered channel so that Release() never blocks.
 	endTimerCh := make(chan struct{}, 1)
 
 	walkInfo := mergeWalk{
+		added:      UTCNow(),
 		entryChs:   resultChs,
 		endWalkCh:  endWalkCh,
 		endTimerCh: endTimerCh,
 	}
 
 	// Append new walk info.
-	t.pool[params] = append(t.pool[params], walkInfo)
+	walks := t.pool[params]
+	if len(walks) < treeWalkSameEntryLimit {
+		t.pool[params] = append(walks, walkInfo)
+	} else {
+		// We are at limit, invalidate oldest, move list down and add new as last.
+		walks[0].endTimerCh <- struct{}{}
+		copy(walks, walks[1:])
+		walks[len(walks)-1] = walkInfo
+	}
 
 	// Timer go-routine which times out after t.timeOut seconds.
 	go func(endTimerCh <-chan struct{}, walkInfo mergeWalk) {

cmd/merge-walk-pool_test.go

@@ -74,7 +74,7 @@ func TestManyMergeWalksSameParam(t *testing.T) {
 		break
 	default:
 		// Create many treeWalk go-routines for the same params.
-		for i := 0; i < 10; i++ {
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
 			endWalkCh := make(chan struct{})
 			walkChs := make([]FileInfoCh, 0)
 			tw.Set(params, walkChs, endWalkCh)
@@ -82,16 +82,62 @@ func TestManyMergeWalksSameParam(t *testing.T) {
 
 		tw.Lock()
 		if walks, ok := tw.pool[params]; ok {
-			if len(walks) != 10 {
+			if len(walks) != treeWalkSameEntryLimit {
 				t.Error("There aren't as many walks as were Set")
 			}
 		}
 		tw.Unlock()
-		for i := 0; i < 10; i++ {
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
 			tw.Lock()
 			if walks, ok := tw.pool[params]; ok {
-				// Before ith Release we should have 10-i treeWalk go-routines.
-				if 10-i != len(walks) {
+				// Before ith Release we should have n-i treeWalk go-routines.
+				if treeWalkSameEntryLimit-i != len(walks) {
+					t.Error("There aren't as many walks as were Set")
+				}
+			}
+			tw.Unlock()
+			tw.Release(params)
+		}
+	}
+
+}
+
+// Test if multiple merge walkers for the same listParams are managed as expected by the pool
+// but that treeWalkSameEntryLimit is respected.
+func TestManyMergeWalksSameParamPrune(t *testing.T) {
+	// Create a treeWalkPool.
+	tw := NewMergeWalkPool(5 * time.Second)
+
+	// Create sample params.
+	params := listParams{
+		bucket: "test-bucket",
+	}
+
+	select {
+	// This timeout is an upper-bound. This is started
+	// before the first treeWalk go-routine's timeout period starts.
+	case <-time.After(5 * time.Second):
+		break
+	default:
+		// Create many treeWalk go-routines for the same params.
+		for i := 0; i < treeWalkSameEntryLimit*4; i++ {
+			endWalkCh := make(chan struct{})
+			walkChs := make([]FileInfoCh, 0)
+			tw.Set(params, walkChs, endWalkCh)
+		}
+
+		tw.Lock()
+		if walks, ok := tw.pool[params]; ok {
+			if len(walks) > treeWalkSameEntryLimit {
+				t.Error("There aren't as many walks as were Set")
+			}
+		}
+		tw.Unlock()
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
+			tw.Lock()
+			if walks, ok := tw.pool[params]; ok {
+				// Before ith Release we should have n-i treeWalk go-routines.
+				if treeWalkSameEntryLimit-i != len(walks) {
 					t.Error("There aren't as many walks as were Set")
 				}
 			}

cmd/metrics.go

@@ -260,6 +260,27 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 		prometheus.CounterValue,
 		float64(cacheObjLayer.CacheStats().getBytesServed()),
 	)
+	for _, cdStats := range cacheObjLayer.CacheStats().GetDiskStats() {
+		// Cache disk usage percentage
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("cache", "usage", "percent"),
+				"Total percentage cache usage",
+				[]string{"disk"}, nil),
+			prometheus.GaugeValue,
+			float64(cdStats.UsagePercent),
+			cdStats.Dir,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("cache", "usage", "high"),
+				"Indicates cache usage is high or low, relative to current cache 'quota' settings",
+				[]string{"disk"}, nil),
+			prometheus.GaugeValue,
+			float64(cdStats.UsageState),
+			cdStats.Dir,
+		)
+	}
 }
 
 // collects http metrics for MinIO server in Prometheus specific format

cmd/namespace-lock.go

@@ -79,7 +79,7 @@ func (n *nsLockMap) lock(ctx context.Context, volume string, path string, lockSo
 	nsLk, found := n.lockMap[resource]
 	if !found {
 		nsLk = &nsLock{
-			LRWMutex: lsync.NewLRWMutex(ctx),
+			LRWMutex: lsync.NewLRWMutex(),
 		}
 		// Add a count to indicate that a parallel unlock doesn't clear this entry.
 	}
@@ -89,9 +89,9 @@ func (n *nsLockMap) lock(ctx context.Context, volume string, path string, lockSo
 
 	// Locking here will block (until timeout).
 	if readLock {
-		locked = nsLk.GetRLock(opsID, lockSource, timeout)
+		locked = nsLk.GetRLock(ctx, opsID, lockSource, timeout)
 	} else {
-		locked = nsLk.GetLock(opsID, lockSource, timeout)
+		locked = nsLk.GetLock(ctx, opsID, lockSource, timeout)
 	}
 
 	if !locked { // We failed to get the lock
@@ -139,6 +139,7 @@ func (n *nsLockMap) unlock(volume string, path string, readLock bool) {
 type distLockInstance struct {
 	rwMutex *dsync.DRWMutex
 	opsID   string
+	ctx     context.Context
 }
 
 // Lock - block until write lock is taken or timeout has occurred.
@@ -146,7 +147,7 @@ func (di *distLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error)
 	lockSource := getSource(2)
 	start := UTCNow()
 
-	if !di.rwMutex.GetLock(di.opsID, lockSource, timeout.Timeout()) {
+	if !di.rwMutex.GetLock(di.ctx, di.opsID, lockSource, timeout.Timeout()) {
 		timeout.LogFailure()
 		return OperationTimedOut{}
 	}
@@ -163,7 +164,7 @@ func (di *distLockInstance) Unlock() {
 func (di *distLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr error) {
 	lockSource := getSource(2)
 	start := UTCNow()
-	if !di.rwMutex.GetRLock(di.opsID, lockSource, timeout.Timeout()) {
+	if !di.rwMutex.GetRLock(di.ctx, di.opsID, lockSource, timeout.Timeout()) {
 		timeout.LogFailure()
 		return OperationTimedOut{}
 	}
@@ -191,10 +192,10 @@ type localLockInstance struct {
 func (n *nsLockMap) NewNSLock(ctx context.Context, lockersFn func() []dsync.NetLocker, volume string, paths ...string) RWLocker {
 	opsID := mustGetUUID()
 	if n.isDistXL {
-		drwmutex := dsync.NewDRWMutex(ctx, &dsync.Dsync{
+		drwmutex := dsync.NewDRWMutex(&dsync.Dsync{
 			GetLockersFn: lockersFn,
 		}, pathsJoinPrefix(volume, paths...)...)
-		return &distLockInstance{drwmutex, opsID}
+		return &distLockInstance{drwmutex, opsID, ctx}
 	}
 	sort.Strings(paths)
 	return &localLockInstance{ctx, n, volume, paths, opsID}

cmd/obdinfo.go

@@ -27,7 +27,6 @@ import (
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/madmin"
 	cpuhw "github.com/shirou/gopsutil/cpu"
-	"github.com/shirou/gopsutil/host"
 	memhw "github.com/shirou/gopsutil/mem"
 	"github.com/shirou/gopsutil/process"
 )
@@ -368,36 +367,3 @@ func getLocalProcOBD(ctx context.Context, r *http.Request) madmin.ServerProcOBDI
 		Processes: sysProcs,
 	}
 }
-
-func getLocalOsInfoOBD(ctx context.Context, r *http.Request) madmin.ServerOsOBDInfo {
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(globalEndpoints)
-	}
-
-	info, err := host.InfoWithContext(ctx)
-	if err != nil {
-		return madmin.ServerOsOBDInfo{
-			Addr:  addr,
-			Error: err.Error(),
-		}
-	}
-
-	sensors, err := host.SensorsTemperaturesWithContext(ctx)
-	if err != nil {
-		return madmin.ServerOsOBDInfo{
-			Addr:  addr,
-			Error: err.Error(),
-		}
-	}
-
-	// ignore user err, as it cannot be obtained reliably inside containers
-	users, _ := host.UsersWithContext(ctx)
-
-	return madmin.ServerOsOBDInfo{
-		Addr:    addr,
-		Info:    info,
-		Sensors: sensors,
-		Users:   users,
-	}
-}

cmd/obdinfo_other.go

@@ -1,4 +1,4 @@
-// +build !freebsd
+// +build !freebsd,!netbsd,!openbsd,!solaris
 
 /*
  * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
@@ -26,8 +26,42 @@ import (
 
 	"github.com/minio/minio/pkg/madmin"
 	diskhw "github.com/shirou/gopsutil/disk"
+	"github.com/shirou/gopsutil/host"
 )
 
+func getLocalOsInfoOBD(ctx context.Context, r *http.Request) madmin.ServerOsOBDInfo {
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(globalEndpoints)
+	}
+
+	info, err := host.InfoWithContext(ctx)
+	if err != nil {
+		return madmin.ServerOsOBDInfo{
+			Addr:  addr,
+			Error: err.Error(),
+		}
+	}
+
+	sensors, err := host.SensorsTemperaturesWithContext(ctx)
+	if err != nil {
+		return madmin.ServerOsOBDInfo{
+			Addr:  addr,
+			Error: err.Error(),
+		}
+	}
+
+	// ignore user err, as it cannot be obtained reliably inside containers
+	users, _ := host.UsersWithContext(ctx)
+
+	return madmin.ServerOsOBDInfo{
+		Addr:    addr,
+		Info:    info,
+		Sensors: sensors,
+		Users:   users,
+	}
+}
+
 func getLocalDiskHwOBD(ctx context.Context, r *http.Request) madmin.ServerDiskHwOBDInfo {
 	addr := r.Host
 	if globalIsDistXL {

cmd/obdinfo_unix.go

@@ -1,3 +1,5 @@
+// +build freebsd netbsd openbsd solaris
+
 /*
  * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
  *
@@ -20,6 +22,7 @@ package cmd
 import (
 	"context"
 	"net/http"
+	"runtime"
 
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -32,6 +35,18 @@ func getLocalDiskHwOBD(ctx context.Context, r *http.Request) madmin.ServerDiskHw
 
 	return madmin.ServerDiskHwOBDInfo{
 		Addr:  addr,
-		Error: "unsupported platform",
+		Error: "unsupported platform: " + runtime.GOOS,
+	}
+}
+
+func getLocalOsInfoOBD(ctx context.Context, r *http.Request) madmin.ServerOsOBDInfo {
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(globalEndpoints)
+	}
+
+	return madmin.ServerOsOBDInfo{
+		Addr:  addr,
+		Error: "unsupported platform: " + runtime.GOOS,
 	}
 }

cmd/object-api-listobjects_test.go

@@ -573,7 +573,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 
 	for i, testCase := range testCases {
 		testCase := testCase
-		t.Run(fmt.Sprintf("Test%d-%s", i+1, instanceType), func(t *testing.T) {
+		t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) {
 			result, err := obj.ListObjects(context.Background(), testCase.bucketName,
 				testCase.prefix, testCase.marker, testCase.delimeter, int(testCase.maxKeys))
 			if err != nil && testCase.shouldPass {

cmd/peer-rest-client.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/gob"
+	"errors"
 	"io"
 	"io/ioutil"
 	"math"
@@ -32,6 +33,7 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/http"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/event"
@@ -44,12 +46,6 @@ import (
 type peerRESTClient struct {
 	host       *xnet.Host
 	restClient *rest.Client
-	connected  int32
-}
-
-// Reconnect to a peer rest server.
-func (client *peerRESTClient) reConnect() {
-	atomic.StoreInt32(&client.connected, 1)
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
@@ -63,10 +59,6 @@ func (client *peerRESTClient) call(method string, values url.Values, body io.Rea
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	if !client.IsOnline() {
-		client.reConnect()
-	}
-
 	if values == nil {
 		values = make(url.Values)
 	}
@@ -76,10 +68,6 @@ func (client *peerRESTClient) callWithContext(ctx context.Context, method string
 		return respBody, nil
 	}
 
-	if isNetworkError(err) {
-		atomic.StoreInt32(&client.connected, 0)
-	}
-
 	return nil, err
 }
 
@@ -88,14 +76,8 @@ func (client *peerRESTClient) String() string {
 	return client.host.String()
 }
 
-// IsOnline - returns whether RPC client failed to connect or not.
-func (client *peerRESTClient) IsOnline() bool {
-	return atomic.LoadInt32(&client.connected) == 1
-}
-
 // Close - marks the client as closed.
 func (client *peerRESTClient) Close() error {
-	atomic.StoreInt32(&client.connected, 0)
 	client.restClient.Close()
 	return nil
 }
@@ -902,5 +884,15 @@ func newPeerRESTClient(peer *xnet.Host) (*peerRESTClient, error) {
 		return nil, err
 	}
 
-	return &peerRESTClient{host: peer, restClient: restClient, connected: 1}, nil
+	// Construct a new health function.
+	restClient.HealthCheckFn = func() bool {
+		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
+		respBody, err := restClient.CallWithContext(ctx, peerRESTMethodHealth, nil, nil, -1)
+		xhttp.DrainBody(respBody)
+		cancel()
+		var ne *rest.NetworkError
+		return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
+	}
+
+	return &peerRESTClient{host: peer, restClient: restClient}, nil
 }

cmd/peer-rest-common.go

@@ -24,6 +24,7 @@ const (
 )
 
 const (
+	peerRESTMethodHealth                = "/health"
 	peerRESTMethodServerInfo            = "/serverinfo"
 	peerRESTMethodDriveOBDInfo          = "/driveobdinfo"
 	peerRESTMethodNetOBDInfo            = "/netobdinfo"

cmd/peer-rest-server.go

@@ -606,6 +606,10 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
 	}
 
 	globalBucketMetadataSys.Set(bucketName, meta)
+
+	if meta.notificationConfig != nil {
+		globalNotificationSys.AddRulesMap(bucketName, meta.notificationConfig.ToRulesMap())
+	}
 }
 
 // ReloadFormatHandler - Reload Format.
@@ -729,6 +733,11 @@ func getLocalDiskIDs(z *xlZones) []string {
 	return ids
 }
 
+// HealthHandler - returns true of health
+func (s *peerRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {
+	s.IsValid(w, r)
+}
+
 // GetLocalDiskIDs - Return disk IDs of all the local disks.
 func (s *peerRESTServer) GetLocalDiskIDs(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -770,7 +779,7 @@ func (s *peerRESTServer) ServerUpdateHandler(w http.ResponseWriter, r *http.Requ
 	var latestReleaseTime time.Time
 	var err error
 	if latestRelease := vars[peerRESTLatestRelease]; latestRelease != "" {
-		latestReleaseTime, err = time.Parse(latestRelease, time.RFC3339)
+		latestReleaseTime, err = time.Parse(time.RFC3339, latestRelease)
 		if err != nil {
 			s.writeErrorResponse(w, err)
 			return
@@ -1020,6 +1029,7 @@ func (s *peerRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
 func registerPeerRESTHandlers(router *mux.Router) {
 	server := &peerRESTServer{}
 	subrouter := router.PathPrefix(peerRESTPrefix).Subrouter()
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodHealth).HandlerFunc(httpTraceHdrs(server.HealthHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcOBDInfo).HandlerFunc(httpTraceHdrs(server.ProcOBDInfoHandler))

cmd/posix.go

@@ -1341,7 +1341,10 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
 		return err
 	}
 
-	defer w.Close()
+	defer func() {
+		disk.Fdatasync(w) // Only interested in flushing the size_t not mtime/atime
+		w.Close()
+	}()
 
 	bufp := s.pool.Get().(*[]byte)
 	defer s.pool.Put(bufp)

cmd/rest/client.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"sync/atomic"
 	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
@@ -31,6 +32,12 @@ import (
 // DefaultRESTTimeout - default RPC timeout is one minute.
 const DefaultRESTTimeout = 1 * time.Minute
 
+const (
+	offline = iota
+	online
+	closed
+)
+
 // NetworkError - error type in case of errors related to http/transport
 // for ex. connection refused, connection reset, dns resolution failure etc.
 // All errors returned by storage-rest-server (ex errFileNotFound, errDiskNotFound) are not considered to be network errors.
@@ -42,12 +49,35 @@ func (n *NetworkError) Error() string {
 	return n.Err.Error()
 }
 
+// Unwrap returns the error wrapped in NetworkError.
+func (n *NetworkError) Unwrap() error {
+	return n.Err
+}
+
 // Client - http based RPC client.
 type Client struct {
+	// HealthCheckFn is the function set to test for health.
+	// If not set the client will not keep track of health.
+	// Calling this returns true or false if the target
+	// is online or offline.
+	HealthCheckFn func() bool
+
+	// HealthCheckInterval will be the duration between re-connection attempts
+	// when a call has failed with a network error.
+	HealthCheckInterval time.Duration
+
+	// HealthCheckTimeout determines timeout for each call.
+	HealthCheckTimeout time.Duration
+
+	// MaxErrResponseSize is the maximum expected response size.
+	// Should only be modified before any calls are made.
+	MaxErrResponseSize int64
+
 	httpClient          *http.Client
 	httpIdleConnsCloser func()
 	url                 *url.URL
 	newAuthToken        func(audience string) string
+	connected           int32
 }
 
 // URL query separator constants
@@ -57,6 +87,9 @@ const (
 
 // CallWithContext - make a REST call with context.
 func (c *Client) CallWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
+	if !c.IsOnline() {
+		return nil, &NetworkError{Err: errors.New("remote server offline")}
+	}
 	req, err := http.NewRequest(http.MethodPost, c.url.String()+method+querySep+values.Encode(), body)
 	if err != nil {
 		return nil, &NetworkError{err}
@@ -69,7 +102,11 @@ func (c *Client) CallWithContext(ctx context.Context, method string, values url.
 	}
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
-		c.httpClient.CloseIdleConnections()
+		// A canceled context doesn't always mean a network problem.
+		if !errors.Is(err, context.Canceled) {
+			// We are safe from recursion
+			c.MarkOffline()
+		}
 		return nil, &NetworkError{err}
 	}
 
@@ -80,9 +117,21 @@ func (c *Client) CallWithContext(ctx context.Context, method string, values url.
 	}
 
 	if resp.StatusCode != http.StatusOK {
+		// If server returns 412 pre-condition failed, it would
+		// mean that authentication succeeded, but another
+		// side-channel check has failed, we shall take
+		// the client offline in such situations.
+		// generally all implementations should simply return
+		// 403, but in situations where there is a dependency
+		// with the caller to take the client offline purpose
+		// fully it should make sure to respond with '412'
+		// instead, see cmd/storage-rest-server.go for ideas.
+		if resp.StatusCode == http.StatusPreconditionFailed {
+			c.MarkOffline()
+		}
 		defer xhttp.DrainBody(resp.Body)
 		// Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
-		b, err := ioutil.ReadAll(io.LimitReader(resp.Body, 4096))
+		b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize))
 		if err != nil {
 			return nil, err
 		}
@@ -102,6 +151,7 @@ func (c *Client) Call(method string, values url.Values, body io.Reader, length i
 
 // Close closes all idle connections of the underlying http client
 func (c *Client) Close() {
+	atomic.StoreInt32(&c.connected, closed)
 	if c.httpIdleConnsCloser != nil {
 		c.httpIdleConnsCloser()
 	}
@@ -117,5 +167,40 @@ func NewClient(url *url.URL, newCustomTransport func() *http.Transport, newAuthT
 		httpIdleConnsCloser: tr.CloseIdleConnections,
 		url:                 url,
 		newAuthToken:        newAuthToken,
+		connected:           online,
+
+		MaxErrResponseSize:  4096,
+		HealthCheckInterval: 200 * time.Millisecond,
+		HealthCheckTimeout:  time.Second,
 	}, nil
 }
+
+// IsOnline returns whether the client is likely to be online.
+func (c *Client) IsOnline() bool {
+	return atomic.LoadInt32(&c.connected) == online
+}
+
+// MarkOffline - will mark a client as being offline and spawns
+// a goroutine that will attempt to reconnect if HealthCheckFn is set.
+func (c *Client) MarkOffline() {
+	// Start goroutine that will attempt to reconnect.
+	// If server is already trying to reconnect this will have no effect.
+	if c.HealthCheckFn != nil && atomic.CompareAndSwapInt32(&c.connected, online, offline) {
+		if c.httpIdleConnsCloser != nil {
+			c.httpIdleConnsCloser()
+		}
+		go func() {
+			ticker := time.NewTicker(c.HealthCheckInterval)
+			defer ticker.Stop()
+			for range ticker.C {
+				if status := atomic.LoadInt32(&c.connected); status == closed {
+					return
+				}
+				if c.HealthCheckFn() {
+					atomic.CompareAndSwapInt32(&c.connected, offline, online)
+					return
+				}
+			}
+		}()
+	}
+}

cmd/server-main.go

@@ -204,6 +204,12 @@ func initSafeMode(ctx context.Context, newObject ObjectLayer) (err error) {
 		}
 	}(txnLk)
 
+	// Enable healing to heal drives if possible
+	if globalIsXL {
+		initBackgroundHealing(ctx, newObject)
+		initLocalDisksAutoHeal(ctx, newObject)
+	}
+
 	// ****  WARNING ****
 	// Migrating to encrypted backend should happen before initialization of any
 	// sub-systems, make sure that we do not move the above codeblock elsewhere.
@@ -419,9 +425,9 @@ func serverMain(ctx *cli.Context) {
 	setMaxResources()
 
 	if globalIsXL {
-		// Init global heal state
-		globalAllHealState = initHealState()
-		globalBackgroundHealState = initHealState()
+		// New global heal state
+		globalAllHealState = newHealState()
+		globalBackgroundHealState = newHealState()
 	}
 
 	// Configure server.
@@ -501,12 +507,6 @@ func serverMain(ctx *cli.Context) {
 
 	newAllSubsystems()
 
-	// Enable healing to heal drives if possible
-	if globalIsXL {
-		initBackgroundHealing(GlobalContext, newObject)
-		initLocalDisksAutoHeal(GlobalContext, newObject)
-	}
-
 	go startBackgroundOps(GlobalContext, newObject)
 
 	logger.FatalIf(initSafeMode(GlobalContext, newObject), "Unable to initialize server switching into safe-mode")

cmd/storage-rest-client.go

@@ -22,15 +22,17 @@ import (
 	"crypto/tls"
 	"encoding/gob"
 	"encoding/hex"
+	"errors"
 	"io"
 	"io/ioutil"
 	"net/url"
 	"path"
 	"strconv"
 	"strings"
-	"sync/atomic"
+	"time"
 
 	"github.com/minio/minio/cmd/http"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	xnet "github.com/minio/minio/pkg/net"
@@ -109,7 +111,6 @@ func toStorageErr(err error) error {
 type storageRESTClient struct {
 	endpoint   Endpoint
 	restClient *rest.Client
-	connected  int32
 	diskID     string
 }
 
@@ -130,9 +131,6 @@ func (client *storageRESTClient) call(method string, values url.Values, body io.
 	}
 
 	err = toStorageErr(err)
-	if err == errDiskNotFound {
-		atomic.StoreInt32(&client.connected, 0)
-	}
 
 	return nil, err
 }
@@ -144,7 +142,7 @@ func (client *storageRESTClient) String() string {
 
 // IsOnline - returns whether RPC client failed to connect or not.
 func (client *storageRESTClient) IsOnline() bool {
-	return atomic.LoadInt32(&client.connected) == 1
+	return client.restClient.IsOnline()
 }
 
 func (client *storageRESTClient) IsLocal() bool {
@@ -554,7 +552,6 @@ func (client *storageRESTClient) VerifyFile(volume, path string, size int64, alg
 
 // Close - marks the client as closed.
 func (client *storageRESTClient) Close() error {
-	atomic.StoreInt32(&client.connected, 0)
 	client.restClient.Close()
 	return nil
 }
@@ -579,8 +576,17 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
 	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: 0}
+		logger.Fatal(err, "Unable to initialize remote REST disks")
 	}
-	return &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}
+
+	restClient.HealthCheckInterval = 500 * time.Millisecond
+	restClient.HealthCheckFn = func() bool {
+		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
+		respBody, err := restClient.CallWithContext(ctx, storageRESTMethodHealth, nil, nil, -1)
+		xhttp.DrainBody(respBody)
+		cancel()
+		return !errors.Is(err, context.DeadlineExceeded) && toStorageErr(err) != errDiskNotFound
+	}
+
+	return &storageRESTClient{endpoint: endpoint, restClient: restClient}
 }

cmd/storage-rest-common.go

@@ -23,6 +23,7 @@ const (
 )
 
 const (
+	storageRESTMethodHealth               = "/health"
 	storageRESTMethodDiskInfo             = "/diskinfo"
 	storageRESTMethodCrawlAndGetDataUsage = "/crawlandgetdatausage"
 	storageRESTMethodMakeVol              = "/makevol"

cmd/storage-rest-server.go

@@ -47,7 +47,11 @@ type storageRESTServer struct {
 }
 
 func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
-	w.WriteHeader(http.StatusForbidden)
+	if errors.Is(err, errDiskStale) {
+		w.WriteHeader(http.StatusPreconditionFailed)
+	} else {
+		w.WriteHeader(http.StatusForbidden)
+	}
 	w.Write([]byte(err.Error()))
 	w.(http.Flusher).Flush()
 }
@@ -118,6 +122,11 @@ func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool
 	return false
 }
 
+// HealthHandler handler checks if disk is stale
+func (s *storageRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {
+	s.IsValid(w, r)
+}
+
 // DiskInfoHandler - returns disk info.
 func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -778,6 +787,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones
 
 			subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter()
 
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodHealth).HandlerFunc(httpTraceHdrs(server.HealthHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(httpTraceHdrs(server.DiskInfoHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCrawlAndGetDataUsage).HandlerFunc(httpTraceHdrs(server.CrawlAndGetDataUsageHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(httpTraceHdrs(server.MakeVolHandler)).Queries(restQueries(storageRESTVolume)...)

cmd/test-utils_test.go

@@ -404,7 +404,7 @@ func resetGlobalIsXL() {
 func resetGlobalHealState() {
 	// Init global heal state
 	if globalAllHealState == nil {
-		globalAllHealState = initHealState()
+		globalAllHealState = newHealState()
 	} else {
 		globalAllHealState.Lock()
 		for _, v := range globalAllHealState.healSeqMap {
@@ -417,7 +417,7 @@ func resetGlobalHealState() {
 
 	// Init background heal state
 	if globalBackgroundHealState == nil {
-		globalBackgroundHealState = initHealState()
+		globalBackgroundHealState = newHealState()
 	} else {
 		globalBackgroundHealState.Lock()
 		for _, v := range globalBackgroundHealState.healSeqMap {

cmd/tree-walk-pool.go

@@ -25,7 +25,9 @@ import (
 
 // Global lookup timeout.
 const (
-	globalLookupTimeout = time.Minute * 30 // 30minutes.
+	globalLookupTimeout    = time.Minute * 30 // 30minutes.
+	treeWalkEntryLimit     = 50
+	treeWalkSameEntryLimit = 4
 )
 
 // listParams - list object params used for list object map
@@ -44,6 +46,7 @@ var errWalkAbort = errors.New("treeWalk abort")
 
 // treeWalk - represents the go routine that does the file tree walk.
 type treeWalk struct {
+	added      time.Time
 	resultCh   chan TreeWalkResult
 	endWalkCh  chan struct{}   // To signal when treeWalk go-routine should end.
 	endTimerCh chan<- struct{} // To signal when timer go-routine should end.
@@ -72,7 +75,7 @@ func NewTreeWalkPool(timeout time.Duration) *TreeWalkPool {
 // Release - selects a treeWalk from the pool based on the input
 // listParams, removes it from the pool, and returns the TreeWalkResult
 // channel.
-// Returns nil if listParams does not have an asccociated treeWalk.
+// Returns nil if listParams does not have an associated treeWalk.
 func (t *TreeWalkPool) Release(params listParams) (resultCh chan TreeWalkResult, endWalkCh chan struct{}) {
 	t.Lock()
 	defer t.Unlock()
@@ -81,6 +84,7 @@ func (t *TreeWalkPool) Release(params listParams) (resultCh chan TreeWalkResult,
 		if len(walks) > 0 {
 			// Pop out the first valid walk entry.
 			walk := walks[0]
+			walks[0] = treeWalk{} // clear references.
 			walks = walks[1:]
 			if len(walks) > 0 {
 				t.pool[params] = walks
@@ -100,22 +104,59 @@ func (t *TreeWalkPool) Release(params listParams) (resultCh chan TreeWalkResult,
 // 1) time.After() expires after t.timeOut seconds.
 //    The expiration is needed so that the treeWalk go-routine resources are freed after a timeout
 //    if the S3 client does only partial listing of objects.
-// 2) Relase() signals the timer go-routine to end on endTimerCh.
+// 2) Release() signals the timer go-routine to end on endTimerCh.
 //    During listing the timer should not timeout and end the treeWalk go-routine, hence the
 //    timer go-routine should be ended.
 func (t *TreeWalkPool) Set(params listParams, resultCh chan TreeWalkResult, endWalkCh chan struct{}) {
 	t.Lock()
 	defer t.Unlock()
 
+	// If we are above the limit delete at least one entry from the pool.
+	if len(t.pool) > treeWalkEntryLimit {
+		age := time.Now()
+		var oldest listParams
+		for k, v := range t.pool {
+			if len(v) == 0 {
+				delete(t.pool, k)
+				continue
+			}
+			// The first element is the oldest, so we only check that.
+			if v[0].added.Before(age) {
+				oldest = k
+			}
+		}
+		// Invalidate and delete oldest.
+		if walks, ok := t.pool[oldest]; ok {
+			walk := walks[0]
+			walks[0] = treeWalk{} // clear references.
+			walks = walks[1:]
+			if len(walks) > 0 {
+				t.pool[params] = walks
+			} else {
+				delete(t.pool, params)
+			}
+			walk.endTimerCh <- struct{}{}
+		}
+	}
+
 	// Should be a buffered channel so that Release() never blocks.
 	endTimerCh := make(chan struct{}, 1)
 	walkInfo := treeWalk{
+		added:      UTCNow(),
 		resultCh:   resultCh,
 		endWalkCh:  endWalkCh,
 		endTimerCh: endTimerCh,
 	}
 	// Append new walk info.
-	t.pool[params] = append(t.pool[params], walkInfo)
+	walks := t.pool[params]
+	if len(walks) < treeWalkSameEntryLimit {
+		t.pool[params] = append(walks, walkInfo)
+	} else {
+		// We are at limit, invalidate oldest, move list down and add new as last.
+		walks[0].endTimerCh <- struct{}{}
+		copy(walks, walks[1:])
+		walks[len(walks)-1] = walkInfo
+	}
 
 	// Timer go-routine which times out after t.timeOut seconds.
 	go func(endTimerCh <-chan struct{}) {

cmd/tree-walk-pool_test.go

@@ -74,7 +74,7 @@ func TestManyWalksSameParam(t *testing.T) {
 		break
 	default:
 		// Create many treeWalk go-routines for the same params.
-		for i := 0; i < 10; i++ {
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
 			resultCh := make(chan TreeWalkResult)
 			endWalkCh := make(chan struct{})
 			tw.Set(params, resultCh, endWalkCh)
@@ -82,16 +82,61 @@ func TestManyWalksSameParam(t *testing.T) {
 
 		tw.Lock()
 		if walks, ok := tw.pool[params]; ok {
-			if len(walks) != 10 {
+			if len(walks) != treeWalkSameEntryLimit {
 				t.Error("There aren't as many walks as were Set")
 			}
 		}
 		tw.Unlock()
-		for i := 0; i < 10; i++ {
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
 			tw.Lock()
 			if walks, ok := tw.pool[params]; ok {
-				// Before ith Release we should have 10-i treeWalk go-routines.
-				if 10-i != len(walks) {
+				// Before ith Release we should have n-i treeWalk go-routines.
+				if treeWalkSameEntryLimit-i != len(walks) {
+					t.Error("There aren't as many walks as were Set")
+				}
+			}
+			tw.Unlock()
+			tw.Release(params)
+		}
+	}
+}
+
+// Test if multiple tree walkers for the same listParams are managed as expected by the pool
+// but that treeWalkSameEntryLimit is respected.
+func TestManyWalksSameParamPrune(t *testing.T) {
+	// Create a treeWalkPool.
+	tw := NewTreeWalkPool(5 * time.Second)
+
+	// Create sample params.
+	params := listParams{
+		bucket: "test-bucket",
+	}
+
+	select {
+	// This timeout is an upper-bound. This is started
+	// before the first treeWalk go-routine's timeout period starts.
+	case <-time.After(5 * time.Second):
+		break
+	default:
+		// Create many treeWalk go-routines for the same params.
+		for i := 0; i < treeWalkSameEntryLimit*4; i++ {
+			resultCh := make(chan TreeWalkResult)
+			endWalkCh := make(chan struct{})
+			tw.Set(params, resultCh, endWalkCh)
+		}
+
+		tw.Lock()
+		if walks, ok := tw.pool[params]; ok {
+			if len(walks) != treeWalkSameEntryLimit {
+				t.Error("There aren't as many walks as were Set")
+			}
+		}
+		tw.Unlock()
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
+			tw.Lock()
+			if walks, ok := tw.pool[params]; ok {
+				// Before ith Release we should have n-i treeWalk go-routines.
+				if treeWalkSameEntryLimit-i != len(walks) {
 					t.Error("There aren't as many walks as were Set")
 				}
 			}
@@ -99,5 +144,4 @@ func TestManyWalksSameParam(t *testing.T) {
 			tw.Release(params)
 		}
 	}
-
 }

cmd/web-handlers.go

@@ -336,7 +336,7 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
 		for _, bucket := range buckets {
 			if globalIAMSys.IsAllowed(iampolicy.Args{
 				AccountName:     claims.AccessKey,
-				Action:          iampolicy.ListAllMyBucketsAction,
+				Action:          iampolicy.ListBucketAction,
 				BucketName:      bucket.Name,
 				ConditionValues: getConditionValues(r, "", claims.AccessKey, claims.Map()),
 				IsOwner:         owner,

cmd/xl-sets.go

@@ -90,8 +90,8 @@ type xlSets struct {
 	pool       *MergeWalkPool
 	poolSplunk *MergeWalkPool
 
-	mrfMU      sync.Mutex
-	mrfUploads map[string]int
+	mrfMU         sync.Mutex
+	mrfOperations map[string]int
 }
 
 func isEndpointConnected(diskMap map[string]StorageAPI, endpoint string) bool {
@@ -303,7 +303,7 @@ func newXLSets(ctx context.Context, endpoints Endpoints, storageDisks []StorageA
 		distributionAlgo:   format.XL.DistributionAlgo,
 		pool:               NewMergeWalkPool(globalMergeLookupTimeout),
 		poolSplunk:         NewMergeWalkPool(globalMergeLookupTimeout),
-		mrfUploads:         make(map[string]int),
+		mrfOperations:      make(map[string]int),
 	}
 
 	mutex := newNSLock(globalIsDistXL)
@@ -348,7 +348,7 @@ func newXLSets(ctx context.Context, endpoints Endpoints, storageDisks []StorageA
 			getEndpoints: s.GetEndpoints(i),
 			nsMutex:      mutex,
 			bp:           bp,
-			mrfUploadCh:  make(chan partialUpload, 10000),
+			mrfOpCh:      make(chan partialOperation, 10000),
 		}
 
 		go s.sets[i].cleanupStaleMultipartUploads(ctx,
@@ -831,9 +831,17 @@ func (f *FileInfoCh) Push(fi FileInfo) {
 // if the caller wishes to list N entries to call lexicallySortedEntry
 // N times until this boolean is 'false'.
 func lexicallySortedEntry(entryChs []FileInfoCh, entries []FileInfo, entriesValid []bool) (FileInfo, int, bool) {
+	var wg sync.WaitGroup
 	for i := range entryChs {
-		entries[i], entriesValid[i] = entryChs[i].Pop()
+		i := i
+		wg.Add(1)
+		// Pop() entries in parallel for large drive setups.
+		go func() {
+			defer wg.Done()
+			entries[i], entriesValid[i] = entryChs[i].Pop()
+		}()
 	}
+	wg.Wait()
 
 	var isTruncated = false
 	for _, valid := range entriesValid {
@@ -915,9 +923,17 @@ func mergeEntriesCh(entryChs []FileInfoCh, maxKeys int, ndisks int) (entries Fil
 }
 
 func isTruncated(entryChs []FileInfoCh, entries []FileInfo, entriesValid []bool) bool {
+	var wg sync.WaitGroup
 	for i := range entryChs {
-		entries[i], entriesValid[i] = entryChs[i].Pop()
+		i := i
+		wg.Add(1)
+		// Pop() entries in parallel for large drive setups.
+		go func() {
+			defer wg.Done()
+			entries[i], entriesValid[i] = entryChs[i].Pop()
+		}()
 	}
+	wg.Wait()
 
 	var isTruncated = false
 	for _, valid := range entriesValid {
@@ -1749,9 +1765,9 @@ func (s *xlSets) GetMetrics(ctx context.Context) (*Metrics, error) {
 // from all underlying xl sets and puts them in a global map which
 // should not have more than 10000 entries.
 func (s *xlSets) maintainMRFList() {
-	var agg = make(chan partialUpload, 10000)
+	var agg = make(chan partialOperation, 10000)
 	for i, xl := range s.sets {
-		go func(c <-chan partialUpload, setIndex int) {
+		go func(c <-chan partialOperation, setIndex int) {
 			for msg := range c {
 				msg.failedSet = setIndex
 				select {
@@ -1759,16 +1775,16 @@ func (s *xlSets) maintainMRFList() {
 				default:
 				}
 			}
-		}(xl.mrfUploadCh, i)
+		}(xl.mrfOpCh, i)
 	}
 
-	for fUpload := range agg {
+	for fOp := range agg {
 		s.mrfMU.Lock()
-		if len(s.mrfUploads) > 10000 {
+		if len(s.mrfOperations) > 10000 {
 			s.mrfMU.Unlock()
 			continue
 		}
-		s.mrfUploads[pathJoin(fUpload.bucket, fUpload.object)] = fUpload.failedSet
+		s.mrfOperations[pathJoin(fOp.bucket, fOp.object)] = fOp.failedSet
 		s.mrfMU.Unlock()
 	}
 }
@@ -1794,17 +1810,17 @@ func (s *xlSets) healMRFRoutine() {
 	for e := range s.disksConnectEvent {
 		// Get the list of objects related the xl set
 		// to which the connected disk belongs.
-		var mrfUploads []string
+		var mrfOperations []string
 		s.mrfMU.Lock()
-		for k, v := range s.mrfUploads {
+		for k, v := range s.mrfOperations {
 			if v == e.setIndex {
-				mrfUploads = append(mrfUploads, k)
+				mrfOperations = append(mrfOperations, k)
 			}
 		}
 		s.mrfMU.Unlock()
 
 		// Heal objects
-		for _, u := range mrfUploads {
+		for _, u := range mrfOperations {
 			// Send an object to be healed with a timeout
 			select {
 			case bgSeq.sourceCh <- healSource{path: u}:
@@ -1812,7 +1828,7 @@ func (s *xlSets) healMRFRoutine() {
 			}
 
 			s.mrfMU.Lock()
-			delete(s.mrfUploads, u)
+			delete(s.mrfOperations, u)
 			s.mrfMU.Unlock()
 		}
 	}

cmd/xl-v1-multipart.go

@@ -731,9 +731,9 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Check if there is any offline disk and add it to the MRF list
-	for i := 0; i < len(onlineDisks); i++ {
-		if onlineDisks[i] == nil || storageDisks[i] == nil {
-			xl.addPartialUpload(bucket, object)
+	for i, disk := range onlineDisks {
+		if disk == nil || storageDisks[i] == nil {
+			xl.addPartial(bucket, object)
 			break
 		}
 	}

cmd/xl-v1-object.go

@@ -667,7 +667,7 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 	// during this upload, send it to the MRF list.
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] == nil || storageDisks[i] == nil {
-			xl.addPartialUpload(bucket, object)
+			xl.addPartial(bucket, object)
 			break
 		}
 	}
@@ -927,6 +927,21 @@ func (xl xlObjects) DeleteObjects(ctx context.Context, bucket string, objects []
 		}
 	}
 
+	// Check failed deletes across multiple objects
+	for i, object := range objects {
+		if deleteErrs[i] == nil {
+			// Check if there is any offline disk and add it to the MRF list
+			for _, disk := range xl.getDisks() {
+				if disk == nil {
+					// all other direct versionId references we should
+					// ensure no dangling file is left over.
+					xl.addPartial(bucket, object)
+					break
+				}
+			}
+		}
+	}
+
 	return deleteErrs, nil
 }
 
@@ -973,6 +988,13 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er
 		return toObjectErr(err, bucket, object)
 	}
 
+	for _, disk := range storageDisks {
+		if disk == nil {
+			xl.addPartial(bucket, object)
+			break
+		}
+	}
+
 	// Success.
 	return nil
 }
@@ -999,11 +1021,11 @@ func (xl xlObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	return listObjectsV2Info, err
 }
 
-// Send the successful but partial upload, however ignore
+// Send the successful but partial upload/delete, however ignore
 // if the channel is blocked by other items.
-func (xl xlObjects) addPartialUpload(bucket, key string) {
+func (xl xlObjects) addPartial(bucket, object string) {
 	select {
-	case xl.mrfUploadCh <- partialUpload{bucket: bucket, object: key}:
+	case xl.mrfOpCh <- partialOperation{bucket: bucket, object: object}:
 	default:
 	}
 }
@@ -1021,6 +1043,10 @@ func (xl xlObjects) PutObjectTags(ctx context.Context, bucket, object string, ta
 	}
 
 	for i, xlMeta := range metaArr {
+		if errs[i] != nil {
+			// Avoid disks where loading metadata fail
+			continue
+		}
 		// clean xlMeta.Meta of tag key, before updating the new tags
 		delete(xlMeta.Meta, xhttp.AmzObjectTagging)
 		// Don't update for empty tags

cmd/xl-v1.go

@@ -41,9 +41,9 @@ const (
 // OfflineDisk represents an unavailable disk.
 var OfflineDisk StorageAPI // zero value is nil
 
-// partialUpload is a successful upload of an object
-// but not written in all disks (having quorum)
-type partialUpload struct {
+// partialOperation is a successful upload/delete
+// of an object but not written in all disks (having quorum)
+type partialOperation struct {
 	bucket    string
 	object    string
 	failedSet int
@@ -69,7 +69,7 @@ type xlObjects struct {
 	// Byte pools used for temporary i/o buffers.
 	bp *bpool.BytePoolCap
 
-	mrfUploadCh chan partialUpload
+	mrfOpCh chan partialOperation
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
@@ -133,9 +133,6 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []DiskInfo,
 	errs = g.Wait()
 	// Wait for the routines.
 	for i, diskInfoErr := range errs {
-		if disks[i] == OfflineDisk {
-			continue
-		}
 		ep := endpoints[i]
 		if diskInfoErr != nil {
 			offlineDisks[ep]++
@@ -261,9 +258,11 @@ func (xl xlObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketIn
 	for _, b := range buckets {
 		e := oldCache.find(b.Name)
 		if e != nil {
-			if bf == nil || bf.containsDir(b.Name) {
+			cache.replace(b.Name, dataUsageRoot, *e)
+			lc, err := globalLifecycleSys.Get(b.Name)
+			activeLC := err == nil && lc.HasActiveRules("", true)
+			if activeLC || bf == nil || bf.containsDir(b.Name) {
 				bucketCh <- b
-				cache.replace(b.Name, dataUsageRoot, *e)
 			} else {
 				if intDataUpdateTracker.debug {
 					logger.Info(color.Green("crawlAndGetDataUsage:")+" Skipping bucket %v, not updated", b.Name)

cmd/xl-zones.go

@@ -859,9 +859,18 @@ func (z *xlZones) listObjects(ctx context.Context, bucket, prefix, marker, delim
 // N times until this boolean is 'false'.
 func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) (FileInfo, int, int, bool) {
 	for i, entryChs := range zoneEntryChs {
+		i := i
+		var wg sync.WaitGroup
 		for j := range entryChs {
-			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
+			j := j
+			wg.Add(1)
+			// Pop() entries in parallel for large drive setups.
+			go func() {
+				defer wg.Done()
+				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
+			}()
 		}
+		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -961,9 +970,18 @@ func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, ndisks int)
 
 func isTruncatedZones(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) bool {
 	for i, entryChs := range zoneEntryChs {
+		i := i
+		var wg sync.WaitGroup
 		for j := range entryChs {
-			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
+			j := j
+			wg.Add(1)
+			// Pop() entries in parallel for large drive setups.
+			go func() {
+				defer wg.Done()
+				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
+			}()
 		}
+		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -1608,7 +1626,7 @@ func (z *xlZones) getZoneAndSet(id string) (int, int, error) {
 			}
 		}
 	}
-	return 0, 0, errDiskNotFound
+	return 0, 0, fmt.Errorf("DiskID(%s) %w", id, errDiskNotFound)
 }
 
 // IsReady - Returns true all the erasure sets are writable.
@@ -1625,6 +1643,7 @@ func (z *xlZones) IsReady(ctx context.Context) bool {
 	for _, id := range diskIDs {
 		zoneIdx, setIdx, err := z.getZoneAndSet(id)
 		if err != nil {
+			logger.LogIf(ctx, err)
 			continue
 		}
 		erasureSetUpCount[zoneIdx][setIdx]++
@@ -1643,6 +1662,8 @@ func (z *xlZones) IsReady(ctx context.Context) bool {
 		}
 		for setIdx := range erasureSetUpCount[zoneIdx] {
 			if erasureSetUpCount[zoneIdx][setIdx] < writeQuorum {
+				logger.LogIf(ctx, fmt.Errorf("Write quorum lost on zone: %d, set: %d, expected write quorum: %d",
+					zoneIdx, setIdx, writeQuorum))
 				return false
 			}
 		}
@@ -1655,18 +1676,20 @@ func (z *xlZones) PutObjectTags(ctx context.Context, bucket, object string, tags
 	if z.SingleZone() {
 		return z.zones[0].PutObjectTags(ctx, bucket, object, tags)
 	}
+
 	for _, zone := range z.zones {
 		err := zone.PutObjectTags(ctx, bucket, object, tags)
 		if err != nil {
-			if isErrBucketNotFound(err) {
+			if isErrObjectNotFound(err) {
 				continue
 			}
 			return err
 		}
 		return nil
 	}
-	return BucketNotFound{
+	return ObjectNotFound{
 		Bucket: bucket,
+		Object: object,
 	}
 }
 
@@ -1678,15 +1701,16 @@ func (z *xlZones) DeleteObjectTags(ctx context.Context, bucket, object string) e
 	for _, zone := range z.zones {
 		err := zone.DeleteObjectTags(ctx, bucket, object)
 		if err != nil {
-			if isErrBucketNotFound(err) {
+			if isErrObjectNotFound(err) {
 				continue
 			}
 			return err
 		}
 		return nil
 	}
-	return BucketNotFound{
+	return ObjectNotFound{
 		Bucket: bucket,
+		Object: object,
 	}
 }
 
@@ -1698,14 +1722,15 @@ func (z *xlZones) GetObjectTags(ctx context.Context, bucket, object string) (*ta
 	for _, zone := range z.zones {
 		tags, err := zone.GetObjectTags(ctx, bucket, object)
 		if err != nil {
-			if isErrBucketNotFound(err) {
+			if isErrObjectNotFound(err) {
 				continue
 			}
 			return tags, err
 		}
 		return tags, nil
 	}
-	return nil, BucketNotFound{
+	return nil, ObjectNotFound{
 		Bucket: bucket,
+		Object: object,
 	}
 }

docs/config/README.md

@@ -190,7 +190,6 @@ or environment variables
 ```
 MINIO_API_REQUESTS_MAX       (number)    set the maximum number of concurrent requests, e.g. "1600"
 MINIO_API_REQUESTS_DEADLINE  (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
-MINIO_API_READY_DEADLINE     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
 MINIO_API_CORS_ALLOW_ORIGIN  (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
 ```
 

docs/disk-caching/DESIGN.md

@@ -15,6 +15,7 @@ minio gateway <name> -h
      MINIO_CACHE_AFTER: Minimum number of access before caching an object.
      MINIO_CACHE_WATERMARK_LOW: % of cache quota at which cache eviction stops
      MINIO_CACHE_WATERMARK_HIGH: % of cache quota at which cache eviction starts
+     MINIO_CACHE_RANGE: set to "on" or "off" caching of independent range requests per object, defaults to "on"
 
 
 ...
@@ -62,6 +63,7 @@ Disk caching caches objects for **downloaded** objects i.e
 - When an object is deleted, corresponding entry in cache if any is deleted as well.
 - Cache continues to work for read-only operations such as GET, HEAD when backend is offline.
 - Cache-Control and Expires headers can be used to control how long objects stay in the cache. ETag of cached objects are not validated with backend until expiry time as per the Cache-Control or Expires header is met.
+- All range GET requests are cached by default independently, this may be not desirable in all situations when cache storage is limited and where downloading an entire object at once might be more optimal. To optionally turn this feature off, and allow downloading entire object in the background `export MINIO_CACHE_RANGE=off`.
 - To ensure security guarantees, encrypted objects are normally not cached. However, if you wish to encrypt cached content on disk, you can set MINIO_CACHE_ENCRYPTION_MASTER_KEY environment variable to set a cache KMS
 master key to automatically encrypt all cached content.
 

docs/gateway/s3.md

@@ -22,6 +22,8 @@ minio gateway s3
 
 ### Using Binary in EC2
 Using IAM rotating credentials for AWS S3
+
+If you are using an S3 enabled IAM role on an EC2 instance for S3 access, MinIO will still require env vars MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be set for its internal use. These may be set to any value which meets the length requirements. Access key length should be at least 3, and secret key length at least 8 characters.
 ```
 export MINIO_ACCESS_KEY=custom_access_key
 export MINIO_SECRET_KEY=custom_secret_key

docs/metrics/healthcheck/README.md

@@ -1,35 +1,42 @@
 ## MinIO Healthcheck
 
-MinIO server exposes two un-authenticated, healthcheck endpoints - liveness probe and readiness probe at `/minio/health/live` and `/minio/health/ready` respectively.
+MinIO server exposes three un-authenticated, healthcheck endpoints liveness probe, readiness probe and a cluster probe at `/minio/health/live`, `/minio/health/ready` and `/minio/health/cluster` respectively.
 
 ### Liveness probe
 
-This probe is used to identify situations where the server is running but may not behave optimally, i.e. sluggish response or corrupt back-end. Such problems can be *only* fixed by a restart.
+This probe always responds with '200 OK'. When liveness probe fails, Kubernetes like platforms restart the container.
 
-Internally, MinIO liveness probe handler checks if backend is alive and in read quorum to take requests.
-
-When liveness probe fails, Kubernetes like platforms restart the container.
+```
+          livenessProbe:
+            httpGet:
+              path: /minio/health/live
+              port: 9000
+              scheme: HTTP
+            initialDelaySeconds: 3
+            periodSeconds: 1
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+```
 
 ### Readiness probe
 
-This probe is used to identify situations where the server is not ready to accept requests yet. In most cases, such conditions recover in some time such as quorum not available on drives due to load.
-
-Internally, MinIO readiness probe handler checks for backend is alive and in read quorum then the server returns 200 OK, otherwise 503 Service Unavailable.
-
-Platforms like Kubernetes *do not* forward traffic to a pod until its readiness probe is successful.
-
-### Configuration example
-
-Sample `liveness` and `readiness` probe configuration in a Kubernetes `yaml` file can be found [here](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes/minio-standalone-deployment.yaml).
-
-### Configure readiness deadline
-Readiness checks need to respond faster in orchestrated environments, to facilitate this you can use the following environment variable before starting MinIO
+This probe always responds with '200 OK'. When readiness probe fails, Kubernetes like platforms *do not* forward traffic to a pod.
 
 ```
-MINIO_API_READY_DEADLINE     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
+          readinessProbe:
+            httpGet:
+              path: /minio/health/ready
+              port: 9000
+              scheme: HTTP
+            initialDelaySeconds: 3
+            periodSeconds: 1
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+
 ```
 
-Set a *5s* deadline for MinIO to ensure readiness handler responds with-in 5seconds.
-```
-export MINIO_API_READY_DEADLINE=5s
-```
+### Cluster probe
+
+This probe is not useful in almost all cases, this is meant for administrators to see if quorum is available in any given cluster. The reply is '200 OK' if cluster has quorum if not it returns '503 Service Unavailable'.

docs/metrics/prometheus/README.md

@@ -110,7 +110,7 @@ These are the new set of metrics which will be in effect after `RELEASE.2019-10-
     - Metrics that records the http statistics and latencies are labeled to their respective APIs (putobject,getobject etc).
     - Disk usage metrics are distributed and labeled to the respective disk paths.
 
-For more details, please check the `Migration guide for the new set of metrics`. 
+For more details, please check the `Migration guide for the new set of metrics`.
 
 The list of metrics and its definition are as follows. (NOTE: instance here is one MinIO node)
 
@@ -178,14 +178,23 @@ All metrics are labeled by `bucket`, each metric is displayed per bucket. `bucke
 
 MinIO Gateway instances enabled with Disk-Caching expose caching related metrics.
 
-| name                 | description                             |
-|:---------------------|:----------------------------------------|
-| `cache_data_served`  | Total number of bytes served from cache |
-| `cache_hits_total`   | Total number of cache hits              |
-| `cache_misses_total` | Total number of cache misses            |
+#### Global cache metrics
+| name                 | description                                       |
+|:---------------------|:--------------------------------------------------|
+| `cache_hits_total`   | Total number of cache hits                        |
+| `cache_misses_total` | Total number of cache misses                      |
+| `cache_data_served`  | Total number of bytes served from cache           |
 
-### Gateway & Cache specific metrics
+#### Per disk cache metrics
+| `cache_usage_percent` | Total percentage cache usage                                                     |
+| `cache_usage_state`   | Indicates cache usage is high or low, relative to current cache 'quota' settings |
 
+`cache_usage_state` holds only two states
+
+- '1' indicates high disk usage
+- '0' indicates low disk usage
+
+### Gateway specific metrics
 MinIO Gateway instance exposes metrics related to Gateway communication with the cloud backend (S3, Azure & GCS Gateway).
 
 `<gateway_type>` changes based on the gateway in use can be 's3', 'gcs' or 'azure'. Other metrics are labeled with `method` that identifies HTTP GET, HEAD, PUT and POST requests to the backend.

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '3.7'
 # 9001 through 9004.
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
@@ -22,7 +22,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
@@ -39,7 +39,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-06-12T00-06-19Z
+    image: minio/minio:RELEASE.2020-06-22T03-12-50Z
     hostname: minio4
     volumes:
       - minio4-data:/export

go.mod

@@ -40,8 +40,8 @@ require (
 	github.com/google/btree v1.0.0 // indirect
 	github.com/google/uuid v1.1.1
 	github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f // indirect
-	github.com/gorilla/handlers v1.4.0
-	github.com/gorilla/mux v1.7.0
+	github.com/gorilla/handlers v1.4.2
+	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/rpc v1.2.0
 	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect

go.sum

@@ -96,6 +96,7 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga
 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.4.1 h1:Wv2VwvNn73pAdFIVUQRXYDFp31lXKbqblIXo/Q5GPSg=
 github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
 github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
@@ -135,6 +136,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -150,10 +152,10 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f h1:4Gslotqbs16iAg+1KR/XdabIfq8TlAWHdwS5QJFksLc=
 github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA=
-github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
-github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
-github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg=
+github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
+github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
 github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
 github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=

pkg/disk/directio_unsupported.go

@@ -1,4 +1,4 @@
-// +build !linux,!netbsd,!freebsd,!darwin
+// +build !linux,!netbsd,!freebsd,!darwin,!openbsd
 
 /*
  * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
@@ -47,16 +47,23 @@ import (
 // see this ZFS-on-Linux commit message:
 // https://github.com/openzfs/zfs/commit/a584ef26053065f486d46a7335bea222cb03eeea
 
+// OpenFileDirectIO wrapper around os.OpenFile nothing special
 func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) {
 	return os.OpenFile(filePath, flag, perm)
 }
 
+// DisableDirectIO is a no-op
 func DisableDirectIO(f *os.File) error {
 	return nil
 }
 
-// AlignedBlock simply returns an unaligned buffer for systems that do not
-// support DirectIO.
+// AlignedBlock simply returns an unaligned buffer
+// for systems that do not support DirectIO.
 func AlignedBlock(BlockSize int) []byte {
 	return make([]byte, BlockSize)
 }
+
+// Fdatasync is a no-op
+func Fdatasync(f *os.File) error {
+	return nil
+}

pkg/disk/fdatasync_linux.go

@@ -0,0 +1,38 @@
+// +build linux
+
+/*
+ * Minio Cloud Storage, (C) 2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"os"
+	"syscall"
+)
+
+// Fdatasync - fdatasync() is similar to fsync(), but does not flush modified metadata
+// unless that metadata is needed in order to allow a subsequent data retrieval
+// to  be  correctly  handled.   For example, changes to st_atime or st_mtime
+// (respectively, time of last access and time of last modification; see inode(7))
+// do not require flushing because they are not necessary for a subsequent data
+// read to be handled correctly. On the other hand, a change to the file size
+// (st_size, as made by say ftruncate(2)), would require a metadata flush.
+//
+// The aim of fdatasync() is to reduce disk activity for applications that
+// do not require all metadata to be synchronized with the disk.
+func Fdatasync(f *os.File) error {
+	return syscall.Fdatasync(int(f.Fd()))
+}

pkg/disk/fdatasync_unix.go

@@ -0,0 +1,29 @@
+// +build freebsd netbsd openbsd darwin
+
+/*
+ * Minio Cloud Storage, (C) 2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"os"
+	"syscall"
+)
+
+// Fdatasync is fsync on freebsd/darwin
+func Fdatasync(f *os.File) error {
+	return syscall.Fsync(int(f.Fd()))
+}

pkg/dsync/drwmutex.go

@@ -54,7 +54,6 @@ type DRWMutex struct {
 	readersLocks [][]string // Array of array of nodes that granted reader locks
 	m            sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
 	clnt         *Dsync
-	ctx          context.Context
 }
 
 // Granted - represents a structure of a granted lock.
@@ -72,12 +71,11 @@ func isLocked(uid string) bool {
 }
 
 // NewDRWMutex - initializes a new dsync RW mutex.
-func NewDRWMutex(ctx context.Context, clnt *Dsync, names ...string) *DRWMutex {
+func NewDRWMutex(clnt *Dsync, names ...string) *DRWMutex {
 	return &DRWMutex{
 		writeLocks: make([]string, len(clnt.GetLockersFn())),
 		Names:      names,
 		clnt:       clnt,
-		ctx:        ctx,
 	}
 }
 
@@ -88,7 +86,7 @@ func NewDRWMutex(ctx context.Context, clnt *Dsync, names ...string) *DRWMutex {
 func (dm *DRWMutex) Lock(id, source string) {
 
 	isReadLock := false
-	dm.lockBlocking(drwMutexInfinite, id, source, isReadLock)
+	dm.lockBlocking(context.Background(), drwMutexInfinite, id, source, isReadLock)
 }
 
 // GetLock tries to get a write lock on dm before the timeout elapses.
@@ -96,10 +94,10 @@ func (dm *DRWMutex) Lock(id, source string) {
 // If the lock is already in use, the calling go routine
 // blocks until either the mutex becomes available and return success or
 // more time has passed than the timeout value and return false.
-func (dm *DRWMutex) GetLock(id, source string, timeout time.Duration) (locked bool) {
+func (dm *DRWMutex) GetLock(ctx context.Context, id, source string, timeout time.Duration) (locked bool) {
 
 	isReadLock := false
-	return dm.lockBlocking(timeout, id, source, isReadLock)
+	return dm.lockBlocking(ctx, timeout, id, source, isReadLock)
 }
 
 // RLock holds a read lock on dm.
@@ -109,7 +107,7 @@ func (dm *DRWMutex) GetLock(id, source string, timeout time.Duration) (locked bo
 func (dm *DRWMutex) RLock(id, source string) {
 
 	isReadLock := true
-	dm.lockBlocking(drwMutexInfinite, id, source, isReadLock)
+	dm.lockBlocking(context.Background(), drwMutexInfinite, id, source, isReadLock)
 }
 
 // GetRLock tries to get a read lock on dm before the timeout elapses.
@@ -118,10 +116,10 @@ func (dm *DRWMutex) RLock(id, source string) {
 // Otherwise the calling go routine blocks until either the mutex becomes
 // available and return success or more time has passed than the timeout
 // value and return false.
-func (dm *DRWMutex) GetRLock(id, source string, timeout time.Duration) (locked bool) {
+func (dm *DRWMutex) GetRLock(ctx context.Context, id, source string, timeout time.Duration) (locked bool) {
 
 	isReadLock := true
-	return dm.lockBlocking(timeout, id, source, isReadLock)
+	return dm.lockBlocking(ctx, timeout, id, source, isReadLock)
 }
 
 // lockBlocking will try to acquire either a read or a write lock
@@ -129,10 +127,10 @@ func (dm *DRWMutex) GetRLock(id, source string, timeout time.Duration) (locked b
 // The function will loop using a built-in timing randomized back-off
 // algorithm until either the lock is acquired successfully or more
 // time has elapsed than the timeout value.
-func (dm *DRWMutex) lockBlocking(timeout time.Duration, id, source string, isReadLock bool) (locked bool) {
+func (dm *DRWMutex) lockBlocking(ctx context.Context, timeout time.Duration, id, source string, isReadLock bool) (locked bool) {
 	restClnts := dm.clnt.GetLockersFn()
 
-	retryCtx, cancel := context.WithTimeout(dm.ctx, timeout)
+	retryCtx, cancel := context.WithTimeout(ctx, timeout)
 
 	defer cancel()
 

pkg/dsync/drwmutex_test.go

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-// GOMAXPROCS=10 go test
-
 package dsync_test
 
 import (
@@ -36,14 +34,14 @@ const (
 
 func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 
-	drwm := NewDRWMutex(context.Background(), ds, "simplelock")
+	drwm := NewDRWMutex(ds, "simplelock")
 
-	if !drwm.GetRLock(id, source, time.Second) {
+	if !drwm.GetRLock(context.Background(), id, source, time.Second) {
 		panic("Failed to acquire read lock")
 	}
 	// fmt.Println("1st read lock acquired, waiting...")
 
-	if !drwm.GetRLock(id, source, time.Second) {
+	if !drwm.GetRLock(context.Background(), id, source, time.Second) {
 		panic("Failed to acquire read lock")
 	}
 	// fmt.Println("2nd read lock acquired, waiting...")
@@ -61,7 +59,7 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	}()
 
 	// fmt.Println("Trying to acquire write lock, waiting...")
-	locked = drwm.GetLock(id, source, duration)
+	locked = drwm.GetLock(context.Background(), id, source, duration)
 	if locked {
 		// fmt.Println("Write lock acquired, waiting...")
 		time.Sleep(time.Second)
@@ -92,10 +90,10 @@ func TestSimpleWriteLockTimedOut(t *testing.T) {
 
 func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 
-	drwm := NewDRWMutex(context.Background(), ds, "duallock")
+	drwm := NewDRWMutex(ds, "duallock")
 
 	// fmt.Println("Getting initial write lock")
-	if !drwm.GetLock(id, source, time.Second) {
+	if !drwm.GetLock(context.Background(), id, source, time.Second) {
 		panic("Failed to acquire initial write lock")
 	}
 
@@ -106,7 +104,7 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	}()
 
 	// fmt.Println("Trying to acquire 2nd write lock, waiting...")
-	locked = drwm.GetLock(id, source, duration)
+	locked = drwm.GetLock(context.Background(), id, source, duration)
 	if locked {
 		// fmt.Println("2nd write lock acquired, waiting...")
 		time.Sleep(time.Second)
@@ -140,8 +138,8 @@ func TestDualWriteLockTimedOut(t *testing.T) {
 // Test cases below are copied 1 to 1 from sync/rwmutex_test.go (adapted to use DRWMutex)
 
 // Borrowed from rwmutex_test.go
-func parallelReader(m *DRWMutex, clocked, cunlock, cdone chan bool) {
-	if m.GetRLock(id, source, time.Second) {
+func parallelReader(ctx context.Context, m *DRWMutex, clocked, cunlock, cdone chan bool) {
+	if m.GetRLock(ctx, id, source, time.Second) {
 		clocked <- true
 		<-cunlock
 		m.RUnlock()
@@ -152,13 +150,13 @@ func parallelReader(m *DRWMutex, clocked, cunlock, cdone chan bool) {
 // Borrowed from rwmutex_test.go
 func doTestParallelReaders(numReaders, gomaxprocs int) {
 	runtime.GOMAXPROCS(gomaxprocs)
-	m := NewDRWMutex(context.Background(), ds, "test-parallel")
+	m := NewDRWMutex(ds, "test-parallel")
 
 	clocked := make(chan bool)
 	cunlock := make(chan bool)
 	cdone := make(chan bool)
 	for i := 0; i < numReaders; i++ {
-		go parallelReader(m, clocked, cunlock, cdone)
+		go parallelReader(context.Background(), m, clocked, cunlock, cdone)
 	}
 	// Wait for all parallel RLock()s to succeed.
 	for i := 0; i < numReaders; i++ {
@@ -184,7 +182,7 @@ func TestParallelReaders(t *testing.T) {
 // Borrowed from rwmutex_test.go
 func reader(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool) {
 	for i := 0; i < numIterations; i++ {
-		if rwm.GetRLock(id, source, time.Second) {
+		if rwm.GetRLock(context.Background(), id, source, time.Second) {
 			n := atomic.AddInt32(activity, 1)
 			if n < 1 || n >= 10000 {
 				panic(fmt.Sprintf("wlock(%d)\n", n))
@@ -201,7 +199,7 @@ func reader(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool)
 // Borrowed from rwmutex_test.go
 func writer(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool) {
 	for i := 0; i < numIterations; i++ {
-		if rwm.GetLock(id, source, time.Second) {
+		if rwm.GetLock(context.Background(), id, source, time.Second) {
 			n := atomic.AddInt32(activity, 10000)
 			if n != 10000 {
 				panic(fmt.Sprintf("wlock(%d)\n", n))
@@ -220,7 +218,7 @@ func HammerRWMutex(gomaxprocs, numReaders, numIterations int) {
 	runtime.GOMAXPROCS(gomaxprocs)
 	// Number of active readers + 10000 * number of active writers.
 	var activity int32
-	rwm := NewDRWMutex(context.Background(), ds, "test")
+	rwm := NewDRWMutex(ds, "test")
 	cdone := make(chan bool)
 	go writer(rwm, numIterations, &activity, cdone)
 	var i int
@@ -263,7 +261,7 @@ func TestUnlockPanic(t *testing.T) {
 			t.Fatalf("unlock of unlocked RWMutex did not panic")
 		}
 	}()
-	mu := NewDRWMutex(context.Background(), ds, "test")
+	mu := NewDRWMutex(ds, "test")
 	mu.Unlock()
 }
 
@@ -274,7 +272,7 @@ func TestUnlockPanic2(t *testing.T) {
 			t.Fatalf("unlock of unlocked RWMutex did not panic")
 		}
 	}()
-	mu := NewDRWMutex(context.Background(), ds, "test-unlock-panic-2")
+	mu := NewDRWMutex(ds, "test-unlock-panic-2")
 	mu.RLock(id, source)
 	mu.Unlock()
 }
@@ -286,7 +284,7 @@ func TestRUnlockPanic(t *testing.T) {
 			t.Fatalf("read unlock of unlocked RWMutex did not panic")
 		}
 	}()
-	mu := NewDRWMutex(context.Background(), ds, "test")
+	mu := NewDRWMutex(ds, "test")
 	mu.RUnlock()
 }
 
@@ -297,14 +295,14 @@ func TestRUnlockPanic2(t *testing.T) {
 			t.Fatalf("read unlock of unlocked RWMutex did not panic")
 		}
 	}()
-	mu := NewDRWMutex(context.Background(), ds, "test-runlock-panic-2")
+	mu := NewDRWMutex(ds, "test-runlock-panic-2")
 	mu.Lock(id, source)
 	mu.RUnlock()
 }
 
 // Borrowed from rwmutex_test.go
 func benchmarkRWMutex(b *testing.B, localWork, writeRatio int) {
-	rwm := NewDRWMutex(context.Background(), ds, "test")
+	rwm := NewDRWMutex(ds, "test")
 	b.RunParallel(func(pb *testing.PB) {
 		foo := 0
 		for pb.Next() {

pkg/dsync/dsync_test.go

@@ -19,7 +19,6 @@
 package dsync_test
 
 import (
-	"context"
 	"fmt"
 	"log"
 	"math/rand"
@@ -89,7 +88,7 @@ func TestMain(m *testing.M) {
 
 func TestSimpleLock(t *testing.T) {
 
-	dm := NewDRWMutex(context.Background(), ds, "test")
+	dm := NewDRWMutex(ds, "test")
 
 	dm.Lock(id, source)
 
@@ -101,7 +100,7 @@ func TestSimpleLock(t *testing.T) {
 
 func TestSimpleLockUnlockMultipleTimes(t *testing.T) {
 
-	dm := NewDRWMutex(context.Background(), ds, "test")
+	dm := NewDRWMutex(ds, "test")
 
 	dm.Lock(id, source)
 	time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond)
@@ -127,8 +126,8 @@ func TestSimpleLockUnlockMultipleTimes(t *testing.T) {
 // Test two locks for same resource, one succeeds, one fails (after timeout)
 func TestTwoSimultaneousLocksForSameResource(t *testing.T) {
 
-	dm1st := NewDRWMutex(context.Background(), ds, "aap")
-	dm2nd := NewDRWMutex(context.Background(), ds, "aap")
+	dm1st := NewDRWMutex(ds, "aap")
+	dm2nd := NewDRWMutex(ds, "aap")
 
 	dm1st.Lock(id, source)
 
@@ -151,9 +150,9 @@ func TestTwoSimultaneousLocksForSameResource(t *testing.T) {
 // Test three locks for same resource, one succeeds, one fails (after timeout)
 func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 
-	dm1st := NewDRWMutex(context.Background(), ds, "aap")
-	dm2nd := NewDRWMutex(context.Background(), ds, "aap")
-	dm3rd := NewDRWMutex(context.Background(), ds, "aap")
+	dm1st := NewDRWMutex(ds, "aap")
+	dm2nd := NewDRWMutex(ds, "aap")
+	dm3rd := NewDRWMutex(ds, "aap")
 
 	dm1st.Lock(id, source)
 
@@ -216,8 +215,8 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 // Test two locks for different resources, both succeed
 func TestTwoSimultaneousLocksForDifferentResources(t *testing.T) {
 
-	dm1 := NewDRWMutex(context.Background(), ds, "aap")
-	dm2 := NewDRWMutex(context.Background(), ds, "noot")
+	dm1 := NewDRWMutex(ds, "aap")
+	dm2 := NewDRWMutex(ds, "noot")
 
 	dm1.Lock(id, source)
 	dm2.Lock(id, source)
@@ -247,7 +246,7 @@ func TestMutex(t *testing.T) {
 		loops = 5
 	}
 	c := make(chan bool)
-	m := NewDRWMutex(context.Background(), ds, "test")
+	m := NewDRWMutex(ds, "test")
 	for i := 0; i < 10; i++ {
 		go HammerMutex(m, loops, c)
 	}
@@ -261,7 +260,7 @@ func BenchmarkMutexUncontended(b *testing.B) {
 		*DRWMutex
 	}
 	b.RunParallel(func(pb *testing.PB) {
-		var mu = PaddedMutex{NewDRWMutex(context.Background(), ds, "")}
+		var mu = PaddedMutex{NewDRWMutex(ds, "")}
 		for pb.Next() {
 			mu.Lock(id, source)
 			mu.Unlock()
@@ -270,7 +269,7 @@ func BenchmarkMutexUncontended(b *testing.B) {
 }
 
 func benchmarkMutex(b *testing.B, slack, work bool) {
-	mu := NewDRWMutex(context.Background(), ds, "")
+	mu := NewDRWMutex(ds, "")
 	if slack {
 		b.SetParallelism(10)
 	}
@@ -313,7 +312,7 @@ func BenchmarkMutexNoSpin(b *testing.B) {
 	// These goroutines yield during local work, so that switching from
 	// a blocked goroutine to other goroutines is profitable.
 	// As a matter of fact, this benchmark still triggers some spinning in the mutex.
-	m := NewDRWMutex(context.Background(), ds, "")
+	m := NewDRWMutex(ds, "")
 	var acc0, acc1 uint64
 	b.SetParallelism(4)
 	b.RunParallel(func(pb *testing.PB) {
@@ -345,7 +344,7 @@ func BenchmarkMutexSpin(b *testing.B) {
 	// profitable. To achieve this we create a goroutine per-proc.
 	// These goroutines access considerable amount of local data so that
 	// unnecessary rescheduling is penalized by cache misses.
-	m := NewDRWMutex(context.Background(), ds, "")
+	m := NewDRWMutex(ds, "")
 	var acc0, acc1 uint64
 	b.RunParallel(func(pb *testing.PB) {
 		var data [16 << 10]uint64

pkg/event/target/redis.go

@@ -267,11 +267,17 @@ func NewRedisTarget(id string, args RedisArgs, doneCh <-chan struct{}, loggerOnc
 				return nil, err
 			}
 
-			if args.Password == "" {
-				return conn, nil
+			if args.Password != "" {
+				if _, err = conn.Do("AUTH", args.Password); err != nil {
+					cErr := conn.Close()
+					targetID := event.TargetID{ID: id, Name: "redis"}
+					loggerOnce(context.Background(), cErr, targetID)
+					return nil, err
+				}
 			}
 
-			if _, err = conn.Do("AUTH", args.Password); err != nil {
+			// Must be done after AUTH
+			if _, err = conn.Do("CLIENT", "SETNAME", "MinIO"); err != nil {
 				cErr := conn.Close()
 				targetID := event.TargetID{ID: id, Name: "redis"}
 				loggerOnce(context.Background(), cErr, targetID)

pkg/iam/policy/action.go

@@ -78,10 +78,10 @@ const (
 	ListMultipartUploadPartsAction = "s3:ListMultipartUploadParts"
 
 	// PutBucketLifecycleAction - PutBucketLifecycle Rest API action.
-	PutBucketLifecycleAction = "s3:PutBucketLifecycle"
+	PutBucketLifecycleAction = "s3:PutLifecycleConfiguration"
 
 	// GetBucketLifecycleAction - GetBucketLifecycle Rest API action.
-	GetBucketLifecycleAction = "s3:GetBucketLifecycle"
+	GetBucketLifecycleAction = "s3:GetLifecycleConfiguration"
 
 	// PutBucketNotificationAction - PutObjectNotification Rest API action.
 	PutBucketNotificationAction = "s3:PutBucketNotification"

pkg/ioutil/ioutil.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

pkg/lsync/lrwmutex.go

@@ -32,12 +32,11 @@ type LRWMutex struct {
 	isWriteLock bool
 	ref         int
 	m           sync.Mutex // Mutex to prevent multiple simultaneous locks
-	ctx         context.Context
 }
 
 // NewLRWMutex - initializes a new lsync RW mutex.
-func NewLRWMutex(ctx context.Context) *LRWMutex {
-	return &LRWMutex{ctx: ctx}
+func NewLRWMutex() *LRWMutex {
+	return &LRWMutex{}
 }
 
 // Lock holds a write lock on lm.
@@ -47,14 +46,14 @@ func NewLRWMutex(ctx context.Context) *LRWMutex {
 func (lm *LRWMutex) Lock() {
 
 	const isWriteLock = true
-	lm.lockLoop(lm.id, lm.source, time.Duration(math.MaxInt64), isWriteLock)
+	lm.lockLoop(context.Background(), lm.id, lm.source, time.Duration(math.MaxInt64), isWriteLock)
 }
 
 // GetLock tries to get a write lock on lm before the timeout occurs.
-func (lm *LRWMutex) GetLock(id string, source string, timeout time.Duration) (locked bool) {
+func (lm *LRWMutex) GetLock(ctx context.Context, id string, source string, timeout time.Duration) (locked bool) {
 
 	const isWriteLock = true
-	return lm.lockLoop(id, source, timeout, isWriteLock)
+	return lm.lockLoop(ctx, id, source, timeout, isWriteLock)
 }
 
 // RLock holds a read lock on lm.
@@ -64,60 +63,55 @@ func (lm *LRWMutex) GetLock(id string, source string, timeout time.Duration) (lo
 func (lm *LRWMutex) RLock() {
 
 	const isWriteLock = false
-	lm.lockLoop(lm.id, lm.source, time.Duration(1<<63-1), isWriteLock)
+	lm.lockLoop(context.Background(), lm.id, lm.source, time.Duration(1<<63-1), isWriteLock)
 }
 
 // GetRLock tries to get a read lock on lm before the timeout occurs.
-func (lm *LRWMutex) GetRLock(id string, source string, timeout time.Duration) (locked bool) {
+func (lm *LRWMutex) GetRLock(ctx context.Context, id string, source string, timeout time.Duration) (locked bool) {
 
 	const isWriteLock = false
-	return lm.lockLoop(id, source, timeout, isWriteLock)
+	return lm.lockLoop(ctx, id, source, timeout, isWriteLock)
+}
+
+func (lm *LRWMutex) lock(id, source string, isWriteLock bool) (locked bool) {
+	lm.m.Lock()
+	lm.id = id
+	lm.source = source
+	if isWriteLock {
+		if lm.ref == 0 && !lm.isWriteLock {
+			lm.ref = 1
+			lm.isWriteLock = true
+			locked = true
+		}
+	} else {
+		if !lm.isWriteLock {
+			lm.ref++
+			locked = true
+		}
+	}
+	lm.m.Unlock()
+
+	return locked
 }
 
 // lockLoop will acquire either a read or a write lock
 //
 // The call will block until the lock is granted using a built-in
 // timing randomized back-off algorithm to try again until successful
-func (lm *LRWMutex) lockLoop(id, source string, timeout time.Duration, isWriteLock bool) bool {
-	retryCtx, cancel := context.WithTimeout(lm.ctx, timeout)
-
+func (lm *LRWMutex) lockLoop(ctx context.Context, id, source string, timeout time.Duration, isWriteLock bool) (locked bool) {
+	retryCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
 	// We timed out on the previous lock, incrementally wait
 	// for a longer back-off time and try again afterwards.
 	for range retry.NewTimer(retryCtx) {
-		// Try to acquire the lock.
-		var success bool
-		{
-			lm.m.Lock()
-
-			lm.id = id
-			lm.source = source
-
-			if isWriteLock {
-				if lm.ref == 0 && !lm.isWriteLock {
-					lm.ref = 1
-					lm.isWriteLock = true
-					success = true
-				}
-			} else {
-				if !lm.isWriteLock {
-					lm.ref++
-					success = true
-				}
-			}
-
-			lm.m.Unlock()
-		}
-
-		if success {
+		if lm.lock(id, source, isWriteLock) {
 			return true
 		}
 	}
 
 	// We timed out on the previous lock, incrementally wait
 	// for a longer back-off time and try again afterwards.
-
 	return false
 }
 

pkg/lsync/lrwmutex_test.go

@@ -33,14 +33,15 @@ import (
 
 func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 
-	lrwm := NewLRWMutex(context.Background())
+	ctx := context.Background()
+	lrwm := NewLRWMutex()
 
-	if !lrwm.GetRLock("", "object1", time.Second) {
+	if !lrwm.GetRLock(ctx, "", "object1", time.Second) {
 		panic("Failed to acquire read lock")
 	}
 	// fmt.Println("1st read lock acquired, waiting...")
 
-	if !lrwm.GetRLock("", "object1", time.Second) {
+	if !lrwm.GetRLock(ctx, "", "object1", time.Second) {
 		panic("Failed to acquire read lock")
 	}
 	// fmt.Println("2nd read lock acquired, waiting...")
@@ -58,7 +59,7 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	}()
 
 	// fmt.Println("Trying to acquire write lock, waiting...")
-	locked = lrwm.GetLock("", "", duration)
+	locked = lrwm.GetLock(ctx, "", "", duration)
 	if locked {
 		// fmt.Println("Write lock acquired, waiting...")
 		time.Sleep(1 * time.Second)
@@ -90,10 +91,11 @@ func TestSimpleWriteLockTimedOut(t *testing.T) {
 
 func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 
-	lrwm := NewLRWMutex(context.Background())
+	ctx := context.Background()
+	lrwm := NewLRWMutex()
 
 	// fmt.Println("Getting initial write lock")
-	if !lrwm.GetLock("", "", time.Second) {
+	if !lrwm.GetLock(ctx, "", "", time.Second) {
 		panic("Failed to acquire initial write lock")
 	}
 
@@ -104,7 +106,7 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	}()
 
 	// fmt.Println("Trying to acquire 2nd write lock, waiting...")
-	locked = lrwm.GetLock("", "", duration)
+	locked = lrwm.GetLock(ctx, "", "", duration)
 	if locked {
 		// fmt.Println("2nd write lock acquired, waiting...")
 		time.Sleep(time.Second)
@@ -139,8 +141,8 @@ func TestDualWriteLockTimedOut(t *testing.T) {
 // Test cases below are copied 1 to 1 from sync/rwmutex_test.go (adapted to use LRWMutex)
 
 // Borrowed from rwmutex_test.go
-func parallelReader(m *LRWMutex, clocked, cunlock, cdone chan bool) {
-	if m.GetRLock("", "", time.Second) {
+func parallelReader(ctx context.Context, m *LRWMutex, clocked, cunlock, cdone chan bool) {
+	if m.GetRLock(ctx, "", "", time.Second) {
 		clocked <- true
 		<-cunlock
 		m.RUnlock()
@@ -151,13 +153,13 @@ func parallelReader(m *LRWMutex, clocked, cunlock, cdone chan bool) {
 // Borrowed from rwmutex_test.go
 func doTestParallelReaders(numReaders, gomaxprocs int) {
 	runtime.GOMAXPROCS(gomaxprocs)
-	m := NewLRWMutex(context.Background())
+	m := NewLRWMutex()
 
 	clocked := make(chan bool)
 	cunlock := make(chan bool)
 	cdone := make(chan bool)
 	for i := 0; i < numReaders; i++ {
-		go parallelReader(m, clocked, cunlock, cdone)
+		go parallelReader(context.Background(), m, clocked, cunlock, cdone)
 	}
 	// Wait for all parallel RLock()s to succeed.
 	for i := 0; i < numReaders; i++ {
@@ -183,7 +185,7 @@ func TestParallelReaders(t *testing.T) {
 // Borrowed from rwmutex_test.go
 func reader(rwm *LRWMutex, numIterations int, activity *int32, cdone chan bool) {
 	for i := 0; i < numIterations; i++ {
-		if rwm.GetRLock("", "", time.Second) {
+		if rwm.GetRLock(context.Background(), "", "", time.Second) {
 			n := atomic.AddInt32(activity, 1)
 			if n < 1 || n >= 10000 {
 				panic(fmt.Sprintf("wlock(%d)\n", n))
@@ -200,7 +202,7 @@ func reader(rwm *LRWMutex, numIterations int, activity *int32, cdone chan bool)
 // Borrowed from rwmutex_test.go
 func writer(rwm *LRWMutex, numIterations int, activity *int32, cdone chan bool) {
 	for i := 0; i < numIterations; i++ {
-		if rwm.GetLock("", "", time.Second) {
+		if rwm.GetLock(context.Background(), "", "", time.Second) {
 			n := atomic.AddInt32(activity, 10000)
 			if n != 10000 {
 				panic(fmt.Sprintf("wlock(%d)\n", n))
@@ -219,7 +221,7 @@ func HammerRWMutex(gomaxprocs, numReaders, numIterations int) {
 	runtime.GOMAXPROCS(gomaxprocs)
 	// Number of active readers + 10000 * number of active writers.
 	var activity int32
-	rwm := NewLRWMutex(context.Background())
+	rwm := NewLRWMutex()
 	cdone := make(chan bool)
 	go writer(rwm, numIterations, &activity, cdone)
 	var i int
@@ -257,7 +259,7 @@ func TestRWMutex(t *testing.T) {
 
 // Borrowed from rwmutex_test.go
 func TestDRLocker(t *testing.T) {
-	wl := NewLRWMutex(context.Background())
+	wl := NewLRWMutex()
 	var rl sync.Locker
 	wlocked := make(chan bool, 1)
 	rlocked := make(chan bool, 1)
@@ -298,7 +300,7 @@ func TestUnlockPanic(t *testing.T) {
 			t.Fatalf("unlock of unlocked RWMutex did not panic")
 		}
 	}()
-	mu := NewLRWMutex(context.Background())
+	mu := NewLRWMutex()
 	mu.Unlock()
 }
 
@@ -309,7 +311,7 @@ func TestUnlockPanic2(t *testing.T) {
 			t.Fatalf("unlock of unlocked RWMutex did not panic")
 		}
 	}()
-	mu := NewLRWMutex(context.Background())
+	mu := NewLRWMutex()
 	mu.RLock()
 	mu.Unlock()
 }
@@ -321,7 +323,7 @@ func TestRUnlockPanic(t *testing.T) {
 			t.Fatalf("read unlock of unlocked RWMutex did not panic")
 		}
 	}()
-	mu := NewLRWMutex(context.Background())
+	mu := NewLRWMutex()
 	mu.RUnlock()
 }
 
@@ -332,7 +334,7 @@ func TestRUnlockPanic2(t *testing.T) {
 			t.Fatalf("read unlock of unlocked RWMutex did not panic")
 		}
 	}()
-	mu := NewLRWMutex(context.Background())
+	mu := NewLRWMutex()
 	mu.Lock()
 	mu.RUnlock()
 }

pkg/madmin/README.md

@@ -43,11 +43,11 @@ func main() {
 
 ```
 
-| Service operations                  | Info operations               | Healing operations | Config operations         |
-|:------------------------------------|:------------------------------|:-------------------|:--------------------------|
-| [`ServiceTrace`](#ServiceTrace)     | [`ServerInfo`](#ServerInfo)   | [`Heal`](#Heal)    | [`GetConfig`](#GetConfig) |
-| [`ServiceStop`](#ServiceStop)       | [`StorageInfo`](#StorageInfo) |                    | [`SetConfig`](#SetConfig) |
-| [`ServiceRestart`](#ServiceRestart) |                               |                    |                           |
+| Service operations                  | Info operations                          | Healing operations | Config operations         |
+|:------------------------------------|:-----------------------------------------|:-------------------|:--------------------------|
+| [`ServiceTrace`](#ServiceTrace)     | [`ServerInfo`](#ServerInfo)              | [`Heal`](#Heal)    | [`GetConfig`](#GetConfig) |
+| [`ServiceStop`](#ServiceStop)       | [`StorageInfo`](#StorageInfo)            |                    | [`SetConfig`](#SetConfig) |
+| [`ServiceRestart`](#ServiceRestart) | [`AccountUsageInfo`](#AccountUsageInfo)  |                    |                           |
 
 
 
@@ -251,6 +251,47 @@ __Example__
 
  ```
 
+<a name="AccountUsageInfo"></a>
+
+### AccountUsageInfo(ctx context.Context) (AccountUsageInfo, error)
+
+Fetches accounting usage information for the current authenticated user
+
+| Param                          | Type                 | Description             |
+|--------------------------------|----------------------|-------------------------|
+| `AccountUsageInfo.AccountName` | _string_             | Account name.           |
+| `AccountUsageInfo.Buckets`     | _[]BucketUsageInfo_  | Bucket usage info.      |
+
+
+| Param                      | Type            | Description                             |
+|----------------------------|-----------------|-----------------------------------------|
+| `BucketUsageInfo.Name`     | _string_        | The name of the current bucket
+| `BucketUsageInfo.Size`     | _uint64_        | The total size of the current bucket
+| `BucketUsageInfo.Created`  | _time.Time_     | Bucket creation time
+| `BucketUsageInfo.Access`   | _AccountAccess_ | Type of access of the current account
+
+
+| Param                  | Type    | Description                                                                                                              |
+|------------------------|---------|------------------------------------------------------------------|
+| `AccountAccess.Read`   | _bool_  | Indicate if the bucket is readable by the current account name.  |
+| `AccountAccess.Write`  | _bool_  | Indocate if the bucket is writable by the current account name.  |
+
+
+__Example__
+
+```go
+
+   accountUsageInfo, err := madmClnt.AccountUsageInfo(context.Background())
+   if err != nil {
+	log.Fatalln(err)
+   }
+
+   log.Println(accountUsageInfo)
+
+```
+
+
+
 ## 5. Heal operations
 
 <a name="Heal"></a>

pkg/madmin/examples/accounting-usage-info.go

@@ -37,10 +37,10 @@ func main() {
 		log.Fatalln(err)
 	}
 
-	accountingUsageInfo, err := madmClnt.AccountingUsageInfo(context.Background())
+	accountUsageInfo, err := madmClnt.AccountUsageInfo(context.Background())
 	if err != nil {
 		log.Fatalln(err)
 	}
 
-	log.Println(accountingUsageInfo)
+	log.Println(accountUsageInfo)
 }

pkg/madmin/heal-commands.go

@@ -77,7 +77,6 @@ type HealTaskStatus struct {
 	FailureDetail string    `json:"detail"`
 	StartTime     time.Time `json:"startTime"`
 	HealSettings  HealOpts  `json:"settings"`
-	NumDisks      int       `json:"numDisks"`
 
 	Items []HealResultItem `json:"items,omitempty"`
 }

pkg/madmin/utils.go

@@ -67,12 +67,22 @@ func getEndpointURL(endpoint string, secure bool) (*url.URL, error) {
 			return nil, ErrInvalidArgument(msg)
 		}
 	}
+
 	// If secure is false, use 'http' scheme.
 	scheme := "https"
 	if !secure {
 		scheme = "http"
 	}
 
+	// Strip the obvious :443 and :80 from the endpoint
+	// to avoid the signature mismatch error.
+	if secure && strings.HasSuffix(endpoint, ":443") {
+		endpoint = strings.TrimSuffix(endpoint, ":443")
+	}
+	if !secure && strings.HasSuffix(endpoint, ":80") {
+		endpoint = strings.TrimSuffix(endpoint, ":80")
+	}
+
 	// Construct a secured endpoint URL.
 	endpointURLStr := scheme + "://" + endpoint
 	endpointURL, err := url.Parse(endpointURLStr)

pkg/sys/rlimit-file_nix.go

@@ -18,7 +18,10 @@
 
 package sys
 
-import "syscall"
+import (
+	"runtime"
+	"syscall"
+)
 
 // GetMaxOpenFileLimit - returns maximum file descriptor number that can be opened by this process.
 func GetMaxOpenFileLimit() (curLimit, maxLimit uint64, err error) {
@@ -33,6 +36,13 @@ func GetMaxOpenFileLimit() (curLimit, maxLimit uint64, err error) {
 
 // SetMaxOpenFileLimit - sets maximum file descriptor number that can be opened by this process.
 func SetMaxOpenFileLimit(curLimit, maxLimit uint64) error {
+	if runtime.GOOS == "darwin" && curLimit > 10240 {
+		// The max file limit is 10240, even though
+		// the max returned by Getrlimit is 1<<63-1.
+		// This is OPEN_MAX in sys/syslimits.h.
+		// refer https://github.com/golang/go/issues/30401
+		curLimit = 10240
+	}
 	rlimit := syscall.Rlimit{Cur: curLimit, Max: maxLimit}
 	return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)
 }