From 21831b3fe29a6ae4d81c4e16132460b879fa872e Mon Sep 17 00:00:00 2001 From: Aditya Manthramurthy Date: Thu, 18 Aug 2022 15:55:17 -0700 Subject: [PATCH] fix: cleanup config KV parsing using madmin helpers (#15552) --- go.mod | 12 +++++++----- go.sum | 25 ++++++++++++------------- internal/config/config.go | 29 ++++++++++++++++++++++++++++- internal/config/config_test.go | 4 +--- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index cf8a7421c..b43571765 100644 --- a/go.mod +++ b/go.mod @@ -42,13 +42,13 @@ require ( github.com/klauspost/reedsolomon v1.10.0 github.com/lib/pq v1.10.4 github.com/miekg/dns v1.1.49 - github.com/minio/cli v1.22.0 + github.com/minio/cli v1.23.0 github.com/minio/console v0.19.5 github.com/minio/csvparser v1.0.0 github.com/minio/dperf v0.4.2 github.com/minio/highwayhash v1.0.2 github.com/minio/kes v0.20.0 - github.com/minio/madmin-go v1.4.20 + github.com/minio/madmin-go v1.4.23 github.com/minio/minio-go/v7 v7.0.34 github.com/minio/pkg v1.3.0 github.com/minio/selfupdate v0.5.0 @@ -72,7 +72,7 @@ require ( github.com/rs/cors v1.7.0 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/secure-io/sio-go v0.3.1 - github.com/shirou/gopsutil/v3 v3.22.6 + github.com/shirou/gopsutil/v3 v3.22.7 github.com/streadway/amqp v1.0.0 github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e github.com/valyala/bytebufferpool v1.0.0 @@ -164,7 +164,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/colorjson v1.0.2 // indirect github.com/minio/filepath v1.0.0 // indirect - github.com/minio/mc v0.0.0-20220805080128-351d021b924b // indirect + github.com/minio/mc v0.0.0-20220818165341-8c239d16aa37 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -202,7 +202,7 @@ require ( github.com/xdg/stringprep v1.0.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect - go.mongodb.org/mongo-driver v1.8.4 // indirect + go.mongodb.org/mongo-driver v1.9.1 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.8.0 // indirect @@ -221,3 +221,5 @@ require ( gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect ) + +replace github.com/minio/console => github.com/donatello/console v0.12.6-0.20220818165657-3d0aec775032 diff --git a/go.sum b/go.sum index 33bf0a686..08d3097ff 100644 --- a/go.sum +++ b/go.sum @@ -195,6 +195,8 @@ github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/donatello/console v0.12.6-0.20220818165657-3d0aec775032 h1:9hRMVUVhdpsVC5T2zn01urSzbCVzlxl0JlnjCxjD91o= +github.com/donatello/console v0.12.6-0.20220818165657-3d0aec775032/go.mod h1:EQRE16zX6DOTsvH4RUFKQOF3wpNf5I+MJj1NYG/j+KI= github.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d h1:QK8IYltsNy+5QZcDFbVkyInrs98/wHy1tfUTGG91sps= github.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d/go.mod h1:apXo4PA/BgBPrt66j0N45O2stlBTRowdip2igwcUWVc= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -605,12 +607,10 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8= github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/argon2 v1.0.0/go.mod h1:XtOGJ7MjwUJDPtCqqrisx5QwVB/jDx+adQHigJVsQHQ= -github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68= -github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= +github.com/minio/cli v1.23.0 h1:hNuf8xi/JU5EJRX+T38e2zcnh2EpdVqs8aH4lNXWe3w= +github.com/minio/cli v1.23.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/minio/colorjson v1.0.2 h1:Em3IM68MTm3h+Oxa0nxrV9VQqDgbxvC5iq5A+pqzDeI= github.com/minio/colorjson v1.0.2/go.mod h1:JWxcL2n8T8JVf+NY6awl6kn5nK49aAzHOeQEM33dL0k= -github.com/minio/console v0.19.5 h1:YbIorVtQ3s3lAG/yVyOPJz1ZG3VZq4vPrHlbREHzXQo= -github.com/minio/console v0.19.5/go.mod h1:1qeA/u15v0TE/uC27o/jVK6MCpGpK97avKxp+SKBsuI= github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU= github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM= github.com/minio/dperf v0.4.2 h1:u2fcdXViX3+alZcEYSOh1crNbGtVU5drQ24lP3Xx6j8= @@ -623,10 +623,10 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT github.com/minio/kes v0.20.0 h1:1tyC51Rr8zTregTESuT/QN/iebNMX7B9t7d3xLNMEpE= github.com/minio/kes v0.20.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc= github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc= -github.com/minio/madmin-go v1.4.20 h1:OpPxc8uIaevJMpBGSZ2TXJvFGvPSDn0IT3VJPh7w90M= -github.com/minio/madmin-go v1.4.20/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98= -github.com/minio/mc v0.0.0-20220805080128-351d021b924b h1:ikMXncKqNE/0acH6us6yy3v+gJBP7nGv/3Rc9F7vRio= -github.com/minio/mc v0.0.0-20220805080128-351d021b924b/go.mod h1:YUXIqqgGfFknByv0eeJSMBQl/WGuEN0XkpW68/ghBm0= +github.com/minio/madmin-go v1.4.23 h1:t36fg3htwB9RRa4a8I+47OZn2NgfA9T83zOjR9LuFsc= +github.com/minio/madmin-go v1.4.23/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98= +github.com/minio/mc v0.0.0-20220818165341-8c239d16aa37 h1:UE9kHgdza2HB4DDx0nIn0jHRl6YASUomVoKjciMfqjM= +github.com/minio/mc v0.0.0-20220818165341-8c239d16aa37/go.mod h1:O3pvs5/n57bZ0mpcG/skBuD4pHX6m+wVbW7lw8LO5go= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= @@ -806,8 +806,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc= github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs= github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88= -github.com/shirou/gopsutil/v3 v3.22.6 h1:FnHOFOh+cYAM0C30P+zysPISzlknLC5Z1G4EAElznfQ= -github.com/shirou/gopsutil/v3 v3.22.6/go.mod h1:EdIubSnZhbAvBS1yJ7Xi+AShB/hxwLHOMz4MCYz7yMs= +github.com/shirou/gopsutil/v3 v3.22.7 h1:flKnuCMfUUrO+oAvwAd6GKZgnPzr098VA/UJ14nhJd4= +github.com/shirou/gopsutil/v3 v3.22.7/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -851,7 +851,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -919,8 +918,8 @@ go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/ go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= -go.mongodb.org/mongo-driver v1.8.4 h1:NruvZPPL0PBcRJKmbswoWSrmHeUvzdxA3GCPfD/NEOA= -go.mongodb.org/mongo-driver v1.8.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c= +go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/internal/config/config.go b/internal/config/config.go index 706684702..9575ed9b1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -820,6 +820,33 @@ func GetSubSys(s string) (subSys string, inputs []string, tgt string, e error) { return subSys, inputs, tgt, e } +// kvFields - converts an input string of form "k1=v1 k2=v2" into +// fields of ["k1=v1", "k2=v2"], the tokenization of each `k=v` +// happens with the right number of input keys, if keys +// input is empty returned value is empty slice as well. +func kvFields(input string, keys []string) []string { + valueIndexes := make([]int, 0, len(keys)) + for _, key := range keys { + i := strings.Index(input, key+KvSeparator) + if i == -1 { + continue + } + valueIndexes = append(valueIndexes, i) + } + + sort.Ints(valueIndexes) + fields := make([]string, len(valueIndexes)) + for i := range valueIndexes { + j := i + 1 + if j < len(valueIndexes) { + fields[i] = strings.TrimSpace(input[valueIndexes[i]:valueIndexes[j]]) + } else { + fields[i] = strings.TrimSpace(input[valueIndexes[i]:]) + } + } + return fields +} + // SetKVS - set specific key values per sub-system. func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err error) { subSys, inputs, tgt, err := GetSubSys(s) @@ -829,7 +856,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err e dynamic = SubSystemsDynamic.Contains(subSys) - fields := madmin.KvFields(inputs[1], defaultKVS[subSys].Keys()) + fields := kvFields(inputs[1], defaultKVS[subSys].Keys()) if len(fields) == 0 { return false, Errorf("sub-system '%s' cannot have empty keys", subSys) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 1238cebf9..9a0a3f65e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -19,8 +19,6 @@ package config import ( "testing" - - "github.com/minio/madmin-go" ) func TestKVFields(t *testing.T) { @@ -92,7 +90,7 @@ func TestKVFields(t *testing.T) { for _, test := range tests { test := test t.Run("", func(t *testing.T) { - gotFields := madmin.KvFields(test.input, test.keys) + gotFields := kvFields(test.input, test.keys) if len(gotFields) != len(test.expectedFields) { t.Errorf("Expected keys %d, found %d", len(test.expectedFields), len(gotFields)) }