From b8f05b147185a99982a8690dd2561c384d540184 Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 15 Apr 2024 00:42:50 -0700 Subject: [PATCH] Keep an up-to-date copy of the KMS master key (#19492) --- cmd/common-main.go | 15 +++++---- cmd/logging.go | 8 +++++ internal/kms/config.go | 19 ++++++----- internal/kms/kes.go | 75 ++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 99 insertions(+), 18 deletions(-) diff --git a/cmd/common-main.go b/cmd/common-main.go index d24a6c7d7..b57dcf7d7 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -958,16 +958,19 @@ func handleKMSConfig() { } } - KMS, err := kms.NewWithConfig(kmsConf) + kmsLogger := Logger{} + KMS, err := kms.NewWithConfig(kmsConf, kmsLogger) if err != nil { logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment") } - // We check that the default key ID exists or try to create it otherwise. - // This implicitly checks that we can communicate to KES. We don't treat - // a policy error as failure condition since MinIO may not have the permission + // Try to generate a data encryption key. Only try to create key if this fails. + // This implicitly checks that we can communicate to KES. + // We don't treat a policy error as failure condition since MinIO may not have the permission // to create keys - just to generate/decrypt data encryption keys. - if err = KMS.CreateKey(context.Background(), env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) { - logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment") + if _, err = KMS.GenerateKey(GlobalContext, env.Get(kms.EnvKESKeyName, ""), kms.Context{}); err != nil && errors.Is(err, kes.ErrKeyNotFound) { + if err = KMS.CreateKey(context.Background(), env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) { + logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment") + } } GlobalKMS = KMS } diff --git a/cmd/logging.go b/cmd/logging.go index 490a8235a..79c3e93eb 100644 --- a/cmd/logging.go +++ b/cmd/logging.go @@ -193,3 +193,11 @@ func tierLogIf(ctx context.Context, err error, errKind ...interface{}) { func kmsLogIf(ctx context.Context, err error, errKind ...interface{}) { logger.LogIf(ctx, "kms", err, errKind...) } + +// Logger permits access to module specific logging +type Logger struct{} + +// LogOnceIf is the implementation of LogOnceIf, accessible using the Logger interface +func (l Logger) LogOnceIf(ctx context.Context, subsystem string, err error, id string, errKind ...interface{}) { + logger.LogOnceIf(ctx, subsystem, err, id, errKind...) +} diff --git a/internal/kms/config.go b/internal/kms/config.go index f19352ac3..ba38aea9a 100644 --- a/internal/kms/config.go +++ b/internal/kms/config.go @@ -19,13 +19,14 @@ package kms // Top level config constants for KMS const ( - EnvKMSSecretKey = "MINIO_KMS_SECRET_KEY" - EnvKMSSecretKeyFile = "MINIO_KMS_SECRET_KEY_FILE" - EnvKESEndpoint = "MINIO_KMS_KES_ENDPOINT" // One or multiple KES endpoints, separated by ',' - EnvKESKeyName = "MINIO_KMS_KES_KEY_NAME" // The default key name used for IAM data and when no key ID is specified on a bucket - EnvKESAPIKey = "MINIO_KMS_KES_API_KEY" // Access credential for KES - API keys and private key / certificate are mutually exclusive - EnvKESClientKey = "MINIO_KMS_KES_KEY_FILE" // Path to TLS private key for authenticating to KES with mTLS - usually prefer API keys - EnvKESClientPassword = "MINIO_KMS_KES_KEY_PASSWORD" // Optional password to decrypt an encrypt TLS private key - EnvKESClientCert = "MINIO_KMS_KES_CERT_FILE" // Path to TLS certificate for authenticating to KES with mTLS - usually prefer API keys - EnvKESServerCA = "MINIO_KMS_KES_CAPATH" // Path to file/directory containing CA certificates to verify the KES server certificate + EnvKMSSecretKey = "MINIO_KMS_SECRET_KEY" + EnvKMSSecretKeyFile = "MINIO_KMS_SECRET_KEY_FILE" + EnvKESEndpoint = "MINIO_KMS_KES_ENDPOINT" // One or multiple KES endpoints, separated by ',' + EnvKESKeyName = "MINIO_KMS_KES_KEY_NAME" // The default key name used for IAM data and when no key ID is specified on a bucket + EnvKESAPIKey = "MINIO_KMS_KES_API_KEY" // Access credential for KES - API keys and private key / certificate are mutually exclusive + EnvKESClientKey = "MINIO_KMS_KES_KEY_FILE" // Path to TLS private key for authenticating to KES with mTLS - usually prefer API keys + EnvKESClientPassword = "MINIO_KMS_KES_KEY_PASSWORD" // Optional password to decrypt an encrypt TLS private key + EnvKESClientCert = "MINIO_KMS_KES_CERT_FILE" // Path to TLS certificate for authenticating to KES with mTLS - usually prefer API keys + EnvKESServerCA = "MINIO_KMS_KES_CAPATH" // Path to file/directory containing CA certificates to verify the KES server certificate + EnvKESKeyCacheInterval = "MINIO_KMS_KEY_CACHE_INTERVAL" // Period between polls of the KES KMS Master Key cache, to prevent it from being unused and purged ) diff --git a/internal/kms/kes.go b/internal/kms/kes.go index 8a9d94300..3ab564dc8 100644 --- a/internal/kms/kes.go +++ b/internal/kms/kes.go @@ -27,10 +27,12 @@ import ( "fmt" "strings" "sync" + "time" + + "github.com/minio/pkg/v2/env" "github.com/minio/kms-go/kes" "github.com/minio/pkg/v2/certs" - "github.com/minio/pkg/v2/env" ) const ( @@ -70,7 +72,7 @@ type Config struct { // NewWithConfig returns a new KMS using the given // configuration. -func NewWithConfig(config Config) (KMS, error) { +func NewWithConfig(config Config, kmsLogger Logger) (KMS, error) { if len(config.Endpoints) == 0 { return nil, errors.New("kms: no server endpoints") } @@ -138,9 +140,38 @@ func NewWithConfig(config Config) (KMS, error) { } } }() + + go c.refreshKMSMasterKeyCache(kmsLogger) return c, nil } +// Request KES keep an up-to-date copy of the KMS master key to allow minio to start up even if KMS is down. The +// cached key may still be evicted if the period of this function is longer than that of KES .cache.expiry.unused +func (c *kesClient) refreshKMSMasterKeyCache(logger Logger) { + ctx := context.Background() + + defaultCacheInterval := 10 + cacheInterval, err := env.GetInt("EnvKESKeyCacheInterval", defaultCacheInterval) + if err != nil { + cacheInterval = defaultCacheInterval + } + + timer := time.NewTimer(time.Duration(cacheInterval) * time.Second) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + c.RefreshKey(ctx, logger) + + // Reset for the next interval + timer.Reset(time.Duration(cacheInterval) * time.Second) + } + } +} + type kesClient struct { lock sync.RWMutex defaultKeyID string @@ -393,7 +424,7 @@ func (c *kesClient) DescribeSelfIdentity(ctx context.Context) (*kes.IdentityInfo return c.client.DescribeSelf(ctx) } -// ListPolicies returns an iterator over all identities. +// ListIdentities returns an iterator over all identities. func (c *kesClient) ListIdentities(ctx context.Context) (*kes.ListIter[kes.Identity], error) { c.lock.RLock() defer c.lock.RUnlock() @@ -450,3 +481,41 @@ func (c *kesClient) Verify(ctx context.Context) []VerifyResult { } return results } + +// Logger interface permits access to module specific logging, in this case, for KMS +type Logger interface { + LogOnceIf(ctx context.Context, subsystem string, err error, id string, errKind ...interface{}) +} + +// RefreshKey checks the validity of the KMS Master Key +func (c *kesClient) RefreshKey(ctx context.Context, logger Logger) bool { + c.lock.RLock() + defer c.lock.RUnlock() + + validKey := false + kmsContext := Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation + for _, endpoint := range c.client.Endpoints { + client := kes.Client{ + Endpoints: []string{endpoint}, + HTTPClient: c.client.HTTPClient, + } + + // 1. Generate a new key using the KMS. + kmsCtx, err := kmsContext.MarshalText() + if err != nil { + logger.LogOnceIf(ctx, "kms", err, "refresh-kms-master-key") + validKey = false + break + } + _, err = client.GenerateKey(ctx, env.Get(EnvKESKeyName, ""), kmsCtx) + if err != nil { + logger.LogOnceIf(ctx, "kms", err, "refresh-kms-master-key") + validKey = false + break + } + if !validKey { + validKey = true + } + } + return validKey +}