From e542084c37e7eeb1b380717e3c750bda394911b0 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 25 Nov 2019 16:33:34 -0800 Subject: [PATCH] Add etcd path prefix for all IAM assets (#8569) Currently, we use the top-level prefix "config/" for all our IAM assets, instead of to provide tenant-level separation bring 'path_prefix' to namespace the access properly. Fixes #8567 --- cmd/api-errors.go | 2 +- cmd/bucket-handlers.go | 2 +- cmd/config-current.go | 9 ++++--- {pkg => cmd/config/etcd}/dns/etcd_dns.go | 14 +++++----- {pkg => cmd/config/etcd}/dns/types.go | 10 ------- cmd/config/etcd/etcd.go | 33 +++++++++++++++++------- cmd/config/etcd/help.go | 7 ++++- cmd/generic-handlers.go | 2 +- cmd/globals.go | 4 +-- cmd/iam-etcd-store.go | 1 + cmd/object-api-utils.go | 2 +- cmd/object-handlers.go | 2 +- cmd/web-handlers.go | 2 +- 13 files changed, 51 insertions(+), 39 deletions(-) rename {pkg => cmd/config/etcd}/dns/etcd_dns.go (96%) rename {pkg => cmd/config/etcd}/dns/types.go (86%) diff --git a/cmd/api-errors.go b/cmd/api-errors.go index df9ef33ff..b81a0a04a 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -28,10 +28,10 @@ import ( "google.golang.org/api/googleapi" minio "github.com/minio/minio-go/v6" + "github.com/minio/minio/cmd/config/etcd/dns" "github.com/minio/minio/cmd/crypto" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/dns" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/hash" ) diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 44ae1ebe2..fd7513f50 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -32,10 +32,10 @@ import ( "github.com/gorilla/mux" "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio/cmd/config/etcd/dns" "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" - "github.com/minio/minio/pkg/dns" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/hash" diff --git a/cmd/config-current.go b/cmd/config-current.go index 960dc26ba..37f54b682 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -25,6 +25,8 @@ import ( "github.com/minio/minio/cmd/config/cache" "github.com/minio/minio/cmd/config/compress" "github.com/minio/minio/cmd/config/etcd" + xetcd "github.com/minio/minio/cmd/config/etcd" + "github.com/minio/minio/cmd/config/etcd/dns" xldap "github.com/minio/minio/cmd/config/identity/ldap" "github.com/minio/minio/cmd/config/identity/openid" "github.com/minio/minio/cmd/config/notify" @@ -34,7 +36,6 @@ import ( xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger/target/http" - "github.com/minio/minio/pkg/dns" "github.com/minio/minio/pkg/env" ) @@ -134,18 +135,18 @@ func lookupConfigs(s config.Config) (err error) { } } - etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs) + etcdCfg, err := xetcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs) if err != nil { return config.Errorf("Unable to initialize etcd config: %s", err) } - globalEtcdClient, err = etcd.New(etcdCfg) + globalEtcdClient, err = xetcd.New(etcdCfg) if err != nil { return config.Errorf("Unable to initialize etcd config: %s", err) } if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil { - globalDNSConfig, err = dns.NewCoreDNS(globalEtcdClient, + globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config, dns.DomainNames(globalDomainNames), dns.DomainIPs(globalDomainIPs), dns.DomainPort(globalMinioPort), diff --git a/pkg/dns/etcd_dns.go b/cmd/config/etcd/dns/etcd_dns.go similarity index 96% rename from pkg/dns/etcd_dns.go rename to cmd/config/etcd/dns/etcd_dns.go index 725c58acd..610ed3261 100644 --- a/pkg/dns/etcd_dns.go +++ b/cmd/config/etcd/dns/etcd_dns.go @@ -29,7 +29,7 @@ import ( "github.com/minio/minio-go/v6/pkg/set" "github.com/coredns/coredns/plugin/etcd/msg" - etcd "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/clientv3" ) // ErrNoEntriesFound - Indicates no entries were found for the given key (directory) @@ -106,7 +106,7 @@ func msgUnPath(s string) string { // Note that this method fetches entries upto only two levels deep. func (c *CoreDNS) list(key string) ([]SrvRecord, error) { ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout) - r, err := c.etcdClient.Get(ctx, key, etcd.WithPrefix()) + r, err := c.etcdClient.Get(ctx, key, clientv3.WithPrefix()) defer cancel() if err != nil { return nil, err @@ -217,7 +217,7 @@ type CoreDNS struct { domainIPs set.StringSet domainPort string prefixPath string - etcdClient *etcd.Client + etcdClient *clientv3.Client } // Option - functional options pattern style @@ -257,14 +257,14 @@ func CoreDNSPath(prefix string) Option { } // NewCoreDNS - initialize a new coreDNS set/unset values. -func NewCoreDNS(etcdClient *etcd.Client, setters ...Option) (Config, error) { - if etcdClient == nil { - return nil, errors.New("invalid argument") +func NewCoreDNS(cfg clientv3.Config, setters ...Option) (*CoreDNS, error) { + etcdClient, err := clientv3.New(cfg) + if err != nil { + return nil, err } args := &CoreDNS{ etcdClient: etcdClient, - prefixPath: defaultPrefixPath, } for _, setter := range setters { diff --git a/pkg/dns/types.go b/cmd/config/etcd/dns/types.go similarity index 86% rename from pkg/dns/types.go rename to cmd/config/etcd/dns/types.go index 63a7006de..30d586dab 100644 --- a/pkg/dns/types.go +++ b/cmd/config/etcd/dns/types.go @@ -55,13 +55,3 @@ type SrvRecord struct { // Key carries the original key used during Put(). Key string `json:"-"` } - -// Config - represents dns put, get interface. This interface can be -// used to implement various backends as needed. -type Config interface { - Put(key string) error - List() ([]SrvRecord, error) - Get(key string) ([]SrvRecord, error) - Delete(key string) error - DeleteRecord(record SrvRecord) error -} diff --git a/cmd/config/etcd/etcd.go b/cmd/config/etcd/etcd.go index 859af3503..6b04f0d12 100644 --- a/cmd/config/etcd/etcd.go +++ b/cmd/config/etcd/etcd.go @@ -24,6 +24,7 @@ import ( "time" "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/clientv3/namespace" "github.com/minio/minio/cmd/config" "github.com/minio/minio/pkg/env" xnet "github.com/minio/minio/pkg/net" @@ -38,12 +39,14 @@ const ( // etcd environment values const ( Endpoints = "endpoints" + PathPrefix = "path_prefix" CoreDNSPath = "coredns_path" ClientCert = "client_cert" ClientCertKey = "client_cert_key" EnvEtcdState = "MINIO_ETCD_STATE" EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS" + EnvEtcdPathPrefix = "MINIO_ETCD_PATH_PREFIX" EnvEtcdCoreDNSPath = "MINIO_ETCD_COREDNS_PATH" EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT" EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY" @@ -60,6 +63,10 @@ var ( Key: Endpoints, Value: "", }, + config.KV{ + Key: PathPrefix, + Value: "", + }, config.KV{ Key: CoreDNSPath, Value: "/skydns", @@ -78,6 +85,7 @@ var ( // Config - server etcd config. type Config struct { Enabled bool `json:"enabled"` + PathPrefix string `json:"pathPrefix"` CoreDNSPath string `json:"coreDNSPath"` clientv3.Config } @@ -87,7 +95,14 @@ func New(cfg Config) (*clientv3.Client, error) { if !cfg.Enabled { return nil, nil } - return clientv3.New(cfg.Config) + cli, err := clientv3.New(cfg.Config) + if err != nil { + return nil, err + } + cli.KV = namespace.NewKV(cli.KV, cfg.PathPrefix) + cli.Watcher = namespace.NewWatcher(cli.Watcher, cfg.PathPrefix) + cli.Lease = namespace.NewLease(cli.Lease, cfg.PathPrefix) + return cli, nil } func parseEndpoints(endpoints string) ([]string, bool, error) { @@ -123,7 +138,9 @@ func lookupLegacyConfig(rootCAs *x509.CertPool) (Config, error) { cfg.DialTimeout = defaultDialTimeout cfg.DialKeepAliveTime = defaultDialKeepAlive cfg.Endpoints = etcdEndpoints - cfg.CoreDNSPath = "/skydns" + cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, "/skydns") + // Default path prefix for all keys on etcd, other than CoreDNSPath. + cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, "") if etcdSecure { cfg.TLS = &tls.Config{ RootCAs: rootCAs, @@ -149,13 +166,9 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) { return cfg, err } - stateBool, err := config.ParseBool(env.Get(EnvEtcdState, config.StateOn)) - if err != nil { - return cfg, err - } - - if stateBool { + if env.Get(EnvEtcdEndpoints, "") != "" && env.Get(EnvEtcdState, "") == "" { // By default state is 'on' to honor legacy config. + var err error cfg, err = lookupLegacyConfig(rootCAs) if err != nil { return cfg, err @@ -166,7 +179,7 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) { } } - stateBool, err = config.ParseBool(env.Get(EnvEtcdState, kvs.Get(config.State))) + stateBool, err := config.ParseBool(env.Get(EnvEtcdState, kvs.Get(config.State))) if err != nil { if kvs.Empty() { return cfg, nil @@ -193,6 +206,8 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) { cfg.DialKeepAliveTime = defaultDialKeepAlive cfg.Endpoints = etcdEndpoints cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath)) + // Default path prefix for all keys on etcd, other than CoreDNSPath. + cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, kvs.Get(PathPrefix)) if etcdSecure { cfg.TLS = &tls.Config{ RootCAs: rootCAs, diff --git a/cmd/config/etcd/help.go b/cmd/config/etcd/help.go index a0a4ba1ed..c582ffe5d 100644 --- a/cmd/config/etcd/help.go +++ b/cmd/config/etcd/help.go @@ -26,9 +26,14 @@ var ( Description: `Comma separated list of etcd endpoints eg: "http://localhost:2379"`, Type: "csv", }, + config.HelpKV{ + Key: PathPrefix, + Description: `Default etcd path prefix to populate all IAM assets eg: "customer/"`, + Type: "path", + }, config.HelpKV{ Key: CoreDNSPath, - Description: `CoreDNS etcd path location to populate DNS srv records eg: "/skydns"`, + Description: `Default etcd path location to populate bucket DNS srv records eg: "/skydns"`, Optional: true, Type: "path", }, diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index a45389f64..5ae721fbc 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -25,10 +25,10 @@ import ( "github.com/minio/minio-go/v6/pkg/set" humanize "github.com/dustin/go-humanize" + "github.com/minio/minio/cmd/config/etcd/dns" "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" - "github.com/minio/minio/pkg/dns" "github.com/minio/minio/pkg/handlers" "github.com/rs/cors" ) diff --git a/cmd/globals.go b/cmd/globals.go index 211599935..4ca725992 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -27,6 +27,7 @@ import ( humanize "github.com/dustin/go-humanize" "github.com/minio/minio/cmd/config/cache" "github.com/minio/minio/cmd/config/compress" + "github.com/minio/minio/cmd/config/etcd/dns" xldap "github.com/minio/minio/cmd/config/identity/ldap" "github.com/minio/minio/cmd/config/identity/openid" "github.com/minio/minio/cmd/config/policy/opa" @@ -35,7 +36,6 @@ import ( xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/certs" - "github.com/minio/minio/pkg/dns" "github.com/minio/minio/pkg/pubsub" ) @@ -208,7 +208,7 @@ var ( globalEtcdClient *etcd.Client // Allocated DNS config wrapper over etcd client. - globalDNSConfig dns.Config + globalDNSConfig *dns.CoreDNS // Default usage check interval value. globalDefaultUsageCheckInterval = 12 * time.Hour // 12 hours diff --git a/cmd/iam-etcd-store.go b/cmd/iam-etcd-store.go index d3933c760..d133b6270 100644 --- a/cmd/iam-etcd-store.go +++ b/cmd/iam-etcd-store.go @@ -549,6 +549,7 @@ func (ies *IAMEtcdStore) watch(sys *IAMSys) { // Refresh IAMSys with etcd watch. watchCh := ies.client.Watch(context.Background(), iamConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly()) + for { select { case <-GlobalServiceDoneCh: diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index abe528eba..77e9453b4 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -37,11 +37,11 @@ import ( "github.com/klauspost/readahead" "github.com/minio/minio-go/v6/pkg/s3utils" "github.com/minio/minio/cmd/config/compress" + "github.com/minio/minio/cmd/config/etcd/dns" "github.com/minio/minio/cmd/config/storageclass" "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" - "github.com/minio/minio/pkg/dns" "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/ioutil" "github.com/minio/minio/pkg/wildcard" diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 9fc8b9c0c..a5576aaea 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -35,11 +35,11 @@ import ( "github.com/gorilla/mux" miniogo "github.com/minio/minio-go/v6" "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio/cmd/config/etcd/dns" "github.com/minio/minio/cmd/config/storageclass" "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" - "github.com/minio/minio/pkg/dns" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/hash" diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 351f50b86..298f0c93f 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -39,12 +39,12 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" "github.com/minio/minio-go/v6/pkg/set" "github.com/minio/minio/browser" + "github.com/minio/minio/cmd/config/etcd/dns" "github.com/minio/minio/cmd/config/identity/openid" "github.com/minio/minio/cmd/crypto" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/dns" "github.com/minio/minio/pkg/event" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/hash"