diff --git a/cmd/config-current.go b/cmd/config-current.go index 9dc033a69..dd22f72c5 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -600,6 +600,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf return fmt.Errorf("Unable to apply scanner config: %w", err) } // update dynamic scanner values. + scannerIdleMode.Store(scannerCfg.IdleMode) scannerCycle.Store(scannerCfg.Cycle) logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait)) case config.LoggerWebhookSubSys: diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 051a8d92a..5d6d3a048 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -65,8 +65,9 @@ var ( globalHealConfig heal.Config // Sleeper values are updated when config is loaded. - scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults - scannerCycle = uatomic.NewDuration(dataScannerStartDelay) + scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults + scannerCycle = uatomic.NewDuration(dataScannerStartDelay) + scannerIdleMode = uatomic.NewInt32(0) // default is throttled when idle ) // initDataScanner will start the scanner in the background. diff --git a/cmd/xl-storage-disk-id-check.go b/cmd/xl-storage-disk-id-check.go index aa365ff10..fae3e7413 100644 --- a/cmd/xl-storage-disk-id-check.go +++ b/cmd/xl-storage-disk-id-check.go @@ -286,8 +286,7 @@ func (p *xlStorageDiskIDCheck) NSScanner(ctx context.Context, cache dataUsageCac } weSleep := func() bool { - // Entire queue is full, so we sleep. - return cap(p.health.tokens) == len(p.health.tokens) + return scannerIdleMode.Load() == 0 } return p.storage.NSScanner(ctx, cache, updates, scanMode, weSleep) diff --git a/internal/config/api/api.go b/internal/config/api/api.go index 8e665ab23..dd8508416 100644 --- a/internal/config/api/api.go +++ b/internal/config/api/api.go @@ -130,9 +130,9 @@ var ( Value: "5m", }, config.KV{ - Key: apiDisableODirect, - Value: "", - Deprecated: true, + Key: apiDisableODirect, + Value: "", + HiddenIfEmpty: true, }, config.KV{ Key: apiODirect, diff --git a/internal/config/config.go b/internal/config/config.go index d329e0014..be16e1f88 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -274,7 +274,7 @@ type KV struct { Key string `json:"key"` Value string `json:"value"` - Deprecated bool `json:"-"` + HiddenIfEmpty bool `json:"-"` } func (kv KV) String() string { @@ -1447,7 +1447,7 @@ func (cs *SubsysInfo) WriteTo(b *strings.Builder, off bool) { continue } // Ignore empty and deprecated values - if dkv.Deprecated && kv.Value == "" { + if dkv.HiddenIfEmpty && kv.Value == "" { continue } // Do not need to print if state is on diff --git a/internal/config/scanner/scanner.go b/internal/config/scanner/scanner.go index 6f4bc7c0e..530a87abc 100644 --- a/internal/config/scanner/scanner.go +++ b/internal/config/scanner/scanner.go @@ -31,6 +31,9 @@ const ( Speed = "speed" EnvSpeed = "MINIO_SCANNER_SPEED" + IdleSpeed = "idle_speed" + EnvIdleSpeed = "MINIO_SCANNER_IDLE_SPEED" + // All below are deprecated in October 2022 and // replaced them with a single speed parameter Delay = "delay" @@ -47,6 +50,8 @@ const ( type Config struct { // Delay is the sleep multiplier. Delay float64 `json:"delay"` + // Behavior of the scanner when there is no other parallel S3 requests + IdleMode int32 // 0 => throttling, 1 => full speed // MaxWait is maximum wait time between operations MaxWait time.Duration // Cycle is the time.Duration between each scanner cycles @@ -59,23 +64,28 @@ var DefaultKVS = config.KVS{ Key: Speed, Value: "default", }, - // Deprecated Oct 2022 config.KV{ - Key: Delay, - Value: "", - Deprecated: true, + Key: IdleSpeed, + Value: "", + HiddenIfEmpty: true, }, // Deprecated Oct 2022 config.KV{ - Key: MaxWait, - Value: "", - Deprecated: true, + Key: Delay, + Value: "", + HiddenIfEmpty: true, }, // Deprecated Oct 2022 config.KV{ - Key: Cycle, - Value: "", - Deprecated: true, + Key: MaxWait, + Value: "", + HiddenIfEmpty: true, + }, + // Deprecated Oct 2022 + config.KV{ + Key: Cycle, + Value: "", + HiddenIfEmpty: true, }, } @@ -107,6 +117,16 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) { default: return cfg, fmt.Errorf("unknown '%s' value", speed) } + + switch idleSpeed := env.Get(EnvIdleSpeed, kvs.GetWithDefault(IdleSpeed, DefaultKVS)); idleSpeed { + case "", "throttled": // Empty is the default mode; + cfg.IdleMode = 0 + case "full": + cfg.IdleMode = 1 + default: + return cfg, fmt.Errorf("unknown value: '%s'", idleSpeed) + } + return }