From 468a9fae83e965ecefa1c1fdc2fc57b84ece95b0 Mon Sep 17 00:00:00 2001 From: Shubhendu Date: Thu, 28 Mar 2024 23:14:56 +0530 Subject: [PATCH] Enable replication of SSE-C objects (#19107) If site replication enabled across sites, replicate the SSE-C objects as well. These objects could be read from target sites using the same client encryption keys. Signed-off-by: Shubhendu Ram Tripathi --- Makefile | 6 + cmd/bucket-handlers.go | 5 - cmd/bucket-replication.go | 72 ++++-- cmd/erasure-multipart.go | 6 +- cmd/erasure-object.go | 5 + cmd/generic-handlers.go | 5 + cmd/generic-handlers_test.go | 6 +- cmd/handler-utils.go | 33 ++- cmd/object-api-options.go | 5 +- cmd/object-handlers.go | 8 +- cmd/object-multipart-handlers.go | 115 +++++---- docs/bucket/versioning/versioning-tests.sh | 40 +-- .../run-sse-kms-object-replication.sh | 230 ++++++++++++++++++ ...sec-object-replication-with-compression.sh | 193 +++++++++++++++ .../run-ssec-object-replication.sh | 219 +++++++++++++++++ internal/bucket/replication/replication.go | 3 - .../bucket/replication/replication_test.go | 14 +- internal/crypto/error.go | 2 + internal/http/headers.go | 3 + 19 files changed, 854 insertions(+), 116 deletions(-) create mode 100755 docs/site-replication/run-sse-kms-object-replication.sh create mode 100755 docs/site-replication/run-ssec-object-replication-with-compression.sh create mode 100755 docs/site-replication/run-ssec-object-replication.sh diff --git a/Makefile b/Makefile index 729f4e197..eac387c50 100644 --- a/Makefile +++ b/Makefile @@ -107,6 +107,12 @@ test-site-replication-oidc: install-race ## verify automatic site replication test-site-replication-minio: install-race ## verify automatic site replication @echo "Running tests for automatic site replication of IAM (with MinIO IDP)" @(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh) + @echo "Running tests for automatic site replication of SSE-C objects" + @(env bash $(PWD)/docs/site-replication/run-ssec-object-replication.sh) + @echo "Running tests for automatic site replication of SSE-C objects with SSE-KMS enabled for bucket" + @(env bash $(PWD)/docs/site-replication/run-sse-kms-object-replication.sh) + @echo "Running tests for automatic site replication of SSE-C objects with compression enabled for site" + @(env bash $(PWD)/docs/site-replication/run-ssec-object-replication-with-compression.sh) verify: ## verify minio various setups @echo "Verifying build with race" diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index ab46893d8..13c53511a 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -1218,11 +1218,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } - if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) { - writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL) - return - } - var ( reader io.Reader keyID string diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index 573b9e9c3..932ca251a 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -51,6 +51,8 @@ import ( "github.com/minio/minio/internal/logger" "github.com/tinylib/msgp/msgp" "github.com/zeebo/xxh3" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" ) const ( @@ -76,11 +78,6 @@ const ( ReplicationWorkerMultiplier = 1.5 ) -func isReplicationEnabled(ctx context.Context, bucketName string) bool { - rc, _ := getReplicationConfig(ctx, bucketName) - return rc != nil -} - // gets replication config associated to a given bucket name. func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) { rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName) @@ -764,13 +761,20 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) { func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, err error) { meta := make(map[string]string) for k, v := range objInfo.UserDefined { - if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { - continue + // In case of SSE-C objects copy the allowed internal headers as well + if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) { + if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { + continue + } + if isStandardHeader(k) { + continue + } } - if isStandardHeader(k) { - continue + if slices.Contains(maps.Keys(validSSEReplicationHeaders), k) { + meta[validSSEReplicationHeaders[k]] = v + } else { + meta[k] = v } - meta[k] = v } if sc == "" && (objInfo.StorageClass == storageclass.STANDARD || objInfo.StorageClass == storageclass.RRS) { @@ -1166,9 +1170,10 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object) gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{ - VersionID: ri.VersionID, - Versioned: versioned, - VersionSuspended: versionSuspended, + VersionID: ri.VersionID, + Versioned: versioned, + VersionSuspended: versionSuspended, + ReplicationRequest: true, }) if err != nil { if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { @@ -1322,11 +1327,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object) versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object) - gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{ - VersionID: ri.VersionID, - Versioned: versioned, - VersionSuspended: versionSuspended, - }) + gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, + ObjectOptions{ + VersionID: ri.VersionID, + Versioned: versioned, + VersionSuspended: versionSuspended, + ReplicationRequest: true, + }) if err != nil { if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { objInfo := ri.ToObjectInfo() @@ -1344,6 +1351,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object defer gr.Close() objInfo := gr.ObjInfo + // make sure we have the latest metadata for metrics calculation rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN) @@ -1367,6 +1375,11 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object return } + // Set the encrypted size for SSE-C objects + if crypto.SSEC.IsEncrypted(objInfo.UserDefined) { + size = objInfo.Size + } + if tgt.Bucket == "" { logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL())) sendEvent(eventArgs{ @@ -1599,21 +1612,35 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob pInfo minio.ObjectPart ) + var objectSize int64 for _, partInfo := range objInfo.Parts { - hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize) + if crypto.SSEC.IsEncrypted(objInfo.UserDefined) { + hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.Size), partInfo.Size, "", "", partInfo.ActualSize) + } else { + hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize) + } if err != nil { return err } + cHeader := http.Header{} + cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true") popts := minio.PutObjectPartOptions{ - SSE: opts.ServerSideEncryption, + SSE: opts.ServerSideEncryption, + CustomHeader: cHeader, } - pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts) + if crypto.SSEC.IsEncrypted(objInfo.UserDefined) { + objectSize += partInfo.Size + pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, popts) + } else { + objectSize += partInfo.ActualSize + pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts) + } if err != nil { return err } - if pInfo.Size != partInfo.ActualSize { + if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) && pInfo.Size != partInfo.ActualSize { return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize) } uploadedParts = append(uploadedParts, minio.CompletePart{ @@ -1624,6 +1651,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute) defer ccancel() _, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{ + UserMetadata: map[string]string{validSSEReplicationHeaders[ReservedMetadataPrefix+"Actual-Object-Size"]: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"]}, Internal: minio.AdvancedPutOptions{ SourceMTime: objInfo.ModTime, // always set this to distinguish between `mc mirror` replication and serverside diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 604004561..1179a2d7e 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -1255,7 +1255,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } // Save the consolidated actual size. - fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10) + if opts.ReplicationRequest { + fi.Metadata[ReservedMetadataPrefix+"actual-size"] = opts.UserDefined["X-Minio-Internal-Actual-Object-Size"] + } else { + fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10) + } if opts.DataMovement { fi.SetDataMov() diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 5e2599eef..e53a4b3c7 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -245,6 +245,11 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri }, toObjectErr(errMethodNotAllowed, bucket, object) } + // Set NoDecryption for SSE-C objects and if replication request + if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && opts.ReplicationRequest { + opts.NoDecryption = true + } + if objInfo.IsRemote() { gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts) if err != nil { diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 47cd09e2a..951a637e5 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -33,6 +33,8 @@ import ( "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio/internal/grid" xnet "github.com/minio/pkg/v2/net" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" "github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/config/dns" @@ -73,6 +75,9 @@ const ( // and must not set by clients func containsReservedMetadata(header http.Header) bool { for key := range header { + if slices.Contains(maps.Keys(validSSEReplicationHeaders), key) { + return false + } if stringsHasPrefixFold(key, ReservedMetadataPrefix) { return true } diff --git a/cmd/generic-handlers_test.go b/cmd/generic-handlers_test.go index b76ec2910..303e42d2f 100644 --- a/cmd/generic-handlers_test.go +++ b/cmd/generic-handlers_test.go @@ -108,15 +108,15 @@ var containsReservedMetadataTests = []struct { }, { header: http.Header{crypto.MetaIV: []string{"iv"}}, - shouldFail: true, + shouldFail: false, }, { header: http.Header{crypto.MetaAlgorithm: []string{crypto.InsecureSealAlgorithm}}, - shouldFail: true, + shouldFail: false, }, { header: http.Header{crypto.MetaSealedKeySSEC: []string{"mac"}}, - shouldFail: true, + shouldFail: false, }, { header: http.Header{ReservedMetadataPrefix + "Key": []string{"value"}}, diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go index 79a06a94e..67d183f56 100644 --- a/cmd/handler-utils.go +++ b/cmd/handler-utils.go @@ -33,6 +33,8 @@ import ( "github.com/minio/minio/internal/logger" "github.com/minio/minio/internal/mcontext" xnet "github.com/minio/pkg/v2/net" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" ) const ( @@ -82,6 +84,31 @@ var supportedHeaders = []string{ xhttp.AmzObjectTagging, "expires", xhttp.AmzBucketReplicationStatus, + "X-Minio-Replication-Server-Side-Encryption-Sealed-Key", + "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm", + "X-Minio-Replication-Server-Side-Encryption-Iv", + "X-Minio-Replication-Encrypted-Multipart", + "X-Minio-Replication-Actual-Object-Size", + // Add more supported headers here. +} + +// mapping of internal headers to allowed replication headers +var validSSEReplicationHeaders = map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "X-Minio-Replication-Server-Side-Encryption-Sealed-Key", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm", + "X-Minio-Internal-Server-Side-Encryption-Iv": "X-Minio-Replication-Server-Side-Encryption-Iv", + "X-Minio-Internal-Encrypted-Multipart": "X-Minio-Replication-Encrypted-Multipart", + "X-Minio-Internal-Actual-Object-Size": "X-Minio-Replication-Actual-Object-Size", + // Add more supported headers here. +} + +// mapping of replication headers to internal headers +var replicationToInternalHeaders = map[string]string{ + "X-Minio-Replication-Server-Side-Encryption-Sealed-Key": "X-Minio-Internal-Server-Side-Encryption-Sealed-Key", + "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm", + "X-Minio-Replication-Server-Side-Encryption-Iv": "X-Minio-Internal-Server-Side-Encryption-Iv", + "X-Minio-Replication-Encrypted-Multipart": "X-Minio-Internal-Encrypted-Multipart", + "X-Minio-Replication-Actual-Object-Size": "X-Minio-Internal-Actual-Object-Size", // Add more supported headers here. } @@ -178,7 +205,11 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[ for _, supportedHeader := range supportedHeaders { value, ok := nv[http.CanonicalHeaderKey(supportedHeader)] if ok { - m[supportedHeader] = strings.Join(value, ",") + if slices.Contains(maps.Keys(replicationToInternalHeaders), supportedHeader) { + m[replicationToInternalHeaders[supportedHeader]] = strings.Join(value, ",") + } else { + m[supportedHeader] = strings.Join(value, ",") + } } } diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go index 3d501c7b9..08c64bb81 100644 --- a/cmd/object-api-options.go +++ b/cmd/object-api-options.go @@ -480,7 +480,7 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object } opts.MTime = mtime opts.UserDefined = make(map[string]string) - + opts.UserDefined[ReservedMetadataPrefix+"Actual-Object-Size"] = r.Header.Get(xhttp.MinIOReplicationActualObjectSize) // Transfer SSEC key in opts.EncryptFn if crypto.SSEC.IsRequested(r.Header) { key, err := ParseSSECustomerRequest(r) @@ -491,5 +491,8 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object } } } + if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok { + opts.ReplicationRequest = true + } return opts, nil } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 66b308d0d..8d8ebeacb 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -2234,8 +2234,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req return } - if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) { - writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL) + if crypto.SSEC.IsRequested(r.Header) && isCompressible(r.Header, object) { + writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionWithCompression), r.URL) return } @@ -2649,10 +2649,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h return errInvalidEncryptionParameters } - if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) { - return errInvalidEncryptionParametersSSEC - } - reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata) if err != nil { return err diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go index d7fb86213..073d6206d 100644 --- a/cmd/object-multipart-handlers.go +++ b/cmd/object-multipart-handlers.go @@ -116,14 +116,29 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r return } - if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) { - writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL) + if crypto.SSEC.IsRequested(r.Header) && isCompressible(r.Header, object) { + writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionWithCompression), r.URL) return } - if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) - return + _, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest] + ssecRepHeaders := []string{ + "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm", + "X-Minio-Replication-Server-Side-Encryption-Sealed-Key", + "X-Minio-Replication-Server-Side-Encryption-Iv", + } + ssecRep := false + for _, header := range ssecRepHeaders { + if val := r.Header.Get(header); val != "" { + ssecRep = true + break + } + } + if !(ssecRep && sourceReplReq) { + if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } } // Set this for multipart only operations, we need to differentiate during // decryption if the file was actually multipart or not. @@ -757,9 +772,10 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http pReader := NewPutObjReader(hashReader) _, isEncrypted := crypto.IsEncrypted(mi.UserDefined) + _, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus] var objectEncryptionKey crypto.ObjectKey if isEncrypted { - if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) { + if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) && !replicationStatus { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL) return } @@ -779,52 +795,55 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http } } - // Calculating object encryption key - key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) - return - } - copy(objectEncryptionKey[:], key) + _, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest] + if !(sourceReplReq && crypto.SSEC.IsEncrypted(mi.UserDefined)) { + // Calculating object encryption key + key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + copy(objectEncryptionKey[:], key) - partEncryptionKey := objectEncryptionKey.DerivePartKey(uint32(partID)) - in := io.Reader(hashReader) - if size > encryptBufferThreshold { - // The encryption reads in blocks of 64KB. - // We add a buffer on bigger files to reduce the number of syscalls upstream. - in = bufio.NewReaderSize(hashReader, encryptBufferSize) - } - reader, err = sio.EncryptReader(in, sio.Config{Key: partEncryptionKey[:], CipherSuites: fips.DARECiphers()}) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) - return - } - wantSize := int64(-1) - if size >= 0 { - info := ObjectInfo{Size: size} - wantSize = info.EncryptedSize() - } - // do not try to verify encrypted content - hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) - return - } - if err := hashReader.AddChecksum(r, true); err != nil { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) - return - } + partEncryptionKey := objectEncryptionKey.DerivePartKey(uint32(partID)) + in := io.Reader(hashReader) + if size > encryptBufferThreshold { + // The encryption reads in blocks of 64KB. + // We add a buffer on bigger files to reduce the number of syscalls upstream. + in = bufio.NewReaderSize(hashReader, encryptBufferSize) + } + reader, err = sio.EncryptReader(in, sio.Config{Key: partEncryptionKey[:], CipherSuites: fips.DARECiphers()}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + wantSize := int64(-1) + if size >= 0 { + info := ObjectInfo{Size: size} + wantSize = info.EncryptedSize() + } + // do not try to verify encrypted content + hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if err := hashReader.AddChecksum(r, true); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } - pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) - return - } + pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } - if idxCb != nil { - idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb) + if idxCb != nil { + idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb) + } + opts.EncryptFn = metadataEncrypter(objectEncryptionKey) } - opts.EncryptFn = metadataEncrypter(objectEncryptionKey) } opts.IndexCB = idxCb diff --git a/docs/bucket/versioning/versioning-tests.sh b/docs/bucket/versioning/versioning-tests.sh index 584734b6f..5ea127f4a 100755 --- a/docs/bucket/versioning/versioning-tests.sh +++ b/docs/bucket/versioning/versioning-tests.sh @@ -10,10 +10,10 @@ trap 'catch $LINENO' ERR catch() { if [ $# -ne 0 ]; then echo "error on line $1" - echo "$site server logs =========" - cat "/tmp/${site}_1.log" + echo "server logs =========" + cat "/tmp/sitea_1.log" echo "===========================" - cat "/tmp/${site}_2.log" + cat "/tmp/sitea_2.log" fi echo "Cleaning up instances of MinIO" @@ -42,32 +42,34 @@ if [ ! -f ./mc ]; then chmod +x mc fi -minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & -minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ - "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & +minio server --address ":9001" "https://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "https://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address ":9002" "https://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "https://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & -export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001 +sleep 60 -./mc mb sitea/delissue +export MC_HOST_sitea=https://minio:minio123@localhost:9001 -./mc version enable sitea/delissue +./mc mb sitea/delissue --insecure -echo hello | ./mc pipe sitea/delissue/hello +./mc version enable sitea/delissue --insecure -./mc version suspend sitea/delissue +echo hello | ./mc pipe sitea/delissue/hello --insecure -./mc rm sitea/delissue/hello +./mc version suspend sitea/delissue --insecure -./mc version enable sitea/delissue +./mc rm sitea/delissue/hello --insecure -echo hello | ./mc pipe sitea/delissue/hello +./mc version enable sitea/delissue --insecure -./mc version suspend sitea/delissue +echo hello | ./mc pipe sitea/delissue/hello --insecure -./mc rm sitea/delissue/hello +./mc version suspend sitea/delissue --insecure -count=$(./mc ls --versions sitea/delissue | wc -l) +./mc rm sitea/delissue/hello --insecure + +count=$(./mc ls --versions sitea/delissue --insecure | wc -l) if [ ${count} -ne 3 ]; then echo "BUG: expected number of versions to be '3' found ${count}" @@ -76,6 +78,6 @@ if [ ${count} -ne 3 ]; then fi echo "SUCCESS:" -./mc ls --versions sitea/delissue +./mc ls --versions sitea/delissue --insecure catch diff --git a/docs/site-replication/run-sse-kms-object-replication.sh b/docs/site-replication/run-sse-kms-object-replication.sh new file mode 100755 index 000000000..76444e6d0 --- /dev/null +++ b/docs/site-replication/run-sse-kms-object-replication.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + + exit 1 +} + +cleanup() { + echo -n "Cleaning up instances of MinIO ..." + pkill -9 minio || sudo pkill -9 minio + pkill -9 kes || sudo pkill -9 kes + rm -rf ${PWD}/keys + rm -rf /tmp/minio{1,2} + echo "done" +} + +cleanup + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" + +# Create certificates for TLS enabled MinIO +echo -n "Setup certs for MinIO instances ..." +wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen +./certgen --host localhost +mkdir -p ~/.minio/certs +mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs +mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs +echo "done" + +# Start MinIO instances +echo -n "Starting MinIO instances ..." +CI=on MINIO_KMS_SECRET_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 & +CI=on MINIO_KMS_SECRET_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 & +echo "done" + +if [ ! -f ./mc ]; then + echo -n "Downloading MinIO client ..." + wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc && + chmod +x mc + echo "done" +fi + +sleep 10 + +export MC_HOST_minio1=https://minio:minio123@localhost:9001 +export MC_HOST_minio2=https://minio:minio123@localhost:9002 + +# Prepare data for tests +echo -n "Preparing test data ..." +mkdir -p /tmp/data +echo "Hello from encrypted world" >/tmp/data/encrypted +touch /tmp/data/mpartobj +shred -s 500M /tmp/data/mpartobj +touch /tmp/data/defpartsize +shred -s 500M /tmp/data/defpartsize +touch /tmp/data/custpartsize +shred -s 500M /tmp/data/custpartsize +echo "done" + +# Add replication site +./mc admin replicate add minio1 minio2 --insecure +# sleep for replication to complete +sleep 30 + +# Create bucket in source cluster +echo "Create bucket in source MinIO instance" +./mc mb minio1/test-bucket --insecure + +# Enable SSE KMS for the bucket +./mc encrypt set sse-kms minio-default-key minio1/test-bucket --insecure + +# Load objects to source site +echo "Loading objects to source MinIO instance" +./mc cp /tmp/data/encrypted minio1/test-bucket --insecure +./mc cp /tmp/data/mpartobj minio1/test-bucket --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure +./mc cp /tmp/data/defpartsize minio1/test-bucket --insecure +./mc put /tmp/data/custpartsize minio1/test-bucket --insecure --part-size 50MiB +sleep 120 + +# List the objects from source site +echo "Objects from source instance" +./mc ls minio1/test-bucket --insecure +count1=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l) +if [ "${count1}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/encrypted not found" + exit_1 +fi +count2=$(./mc ls minio1/test-bucket/mpartobj --insecure | wc -l) +if [ "${count2}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/mpartobj not found" + exit_1 +fi +count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l) +if [ "${count3}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/defpartsize not found" + exit_1 +fi +count4=$(./mc ls minio1/test-bucket/custpartsize --insecure | wc -l) +if [ "${count4}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/custpartsize not found" + exit_1 +fi + +# List the objects from replicated site +echo "Objects from replicated instance" +./mc ls minio2/test-bucket --insecure +repcount1=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l) +if [ "${repcount1}" -ne 1 ]; then + echo "BUG: object test-bucket/encrypted not replicated" + exit_1 +fi +repcount2=$(./mc ls minio2/test-bucket/mpartobj --insecure | wc -l) +if [ "${repcount2}" -ne 1 ]; then + echo "BUG: object test-bucket/mpartobj not replicated" + exit_1 +fi +repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l) +if [ "${repcount3}" -ne 1 ]; then + echo "BUG: object test-bucket/defpartsize not replicated" + exit_1 +fi +repcount4=$(./mc ls minio2/test-bucket/custpartsize --insecure | wc -l) +if [ "${repcount4}" -ne 1 ]; then + echo "BUG: object test-bucket/custpartsize not replicated" + exit_1 +fi + +# Stat the objects from source site +echo "Stat minio1/test-bucket/encrypted" +./mc stat minio1/test-bucket/encrypted --insecure --json +stat_out1=$(./mc stat minio1/test-bucket/encrypted --insecure --json) +src_obj1_algo=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +src_obj1_keyid=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio1/test-bucket/defpartsize" +./mc stat minio1/test-bucket/defpartsize --insecure --json +stat_out2=$(./mc stat minio1/test-bucket/defpartsize --insecure --json) +src_obj2_algo=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +src_obj2_keyid=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio1/test-bucket/custpartsize" +./mc stat minio1/test-bucket/custpartsize --insecure --json +stat_out3=$(./mc stat minio1/test-bucket/custpartsize --insecure --json) +src_obj3_algo=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +src_obj3_keyid=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio1/test-bucket/mpartobj" +./mc stat minio1/test-bucket/mpartobj --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out4=$(./mc stat minio1/test-bucket/mpartobj --encrypt-key "minio1/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json) +src_obj4_etag=$(echo "${stat_out4}" | jq '.etag') +src_obj4_size=$(echo "${stat_out4}" | jq '.size') +src_obj4_md5=$(echo "${stat_out4}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Stat the objects from replicated site +echo "Stat minio2/test-bucket/encrypted" +./mc stat minio2/test-bucket/encrypted --insecure --json +stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --insecure --json) +rep_obj1_algo=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +rep_obj1_keyid=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio2/test-bucket/defpartsize" +./mc stat minio2/test-bucket/defpartsize --insecure --json +stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --insecure --json) +rep_obj2_algo=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +rep_obj2_keyid=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio2/test-bucket/custpartsize" +./mc stat minio2/test-bucket/custpartsize --insecure --json +stat_out3_rep=$(./mc stat minio2/test-bucket/custpartsize --insecure --json) +rep_obj3_algo=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +rep_obj3_keyid=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio2/test-bucket/mpartobj" +./mc stat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out4_rep=$(./mc stat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure --json) +rep_obj4_etag=$(echo "${stat_out4}" | jq '.etag') +rep_obj4_size=$(echo "${stat_out4}" | jq '.size') +rep_obj4_md5=$(echo "${stat_out4}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Check the algo and keyId of replicated objects +if [ "${rep_obj1_algo}" != "${src_obj1_algo}" ]; then + echo "BUG: Algorithm: '${rep_obj1_algo}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_algo}'" + exit_1 +fi +if [ "${rep_obj1_keyid}" != "${src_obj1_keyid}" ]; then + echo "BUG: KeyId: '${rep_obj1_keyid}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_keyid}'" + exit_1 +fi +if [ "${rep_obj2_algo}" != "${src_obj2_algo}" ]; then + echo "BUG: Algorithm: '${rep_obj2_algo}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_algo}'" + exit_1 +fi +if [ "${rep_obj2_keyid}" != "${src_obj2_keyid}" ]; then + echo "BUG: KeyId: '${rep_obj2_keyid}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_keyid}'" + exit_1 +fi +if [ "${rep_obj3_algo}" != "${src_obj3_algo}" ]; then + echo "BUG: Algorithm: '${rep_obj3_algo}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_algo}'" + exit_1 +fi +if [ "${rep_obj3_keyid}" != "${src_obj3_keyid}" ]; then + echo "BUG: KeyId: '${rep_obj3_keyid}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_keyid}'" + exit_1 +fi + +# Check the etag, size and md5 of replicated SSEC object +if [ "${rep_obj4_etag}" != "${src_obj4_etag}" ]; then + echo "BUG: Etag: '${rep_obj4_etag}' of replicated object: 'minio2/test-bucket/mpartobj' doesn't match with source value: '${src_obj4_etag}'" + exit_1 +fi +if [ "${rep_obj4_size}" != "${src_obj4_size}" ]; then + echo "BUG: Size: '${rep_obj4_size}' of replicated object: 'minio2/test-bucket/mpartobj' doesn't match with source value: '${src_obj4_size}'" + exit_1 +fi +if [ "${src_obj4_md5}" != "${rep_obj4_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/mpartobj' doesn't match with source. Expected: '${src_obj4_md5}', Found: '${rep_obj4_md5}'" + exit_1 +fi + +# Check content of replicated objects +./mc cat minio2/test-bucket/encrypted --insecure +./mc cat minio2/test-bucket/mpartobj --encrypt-key "minio2/test-bucket/mpartobj=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1 +./mc cat minio2/test-bucket/defpartsize --insecure >/dev/null || exit_1 +./mc cat minio2/test-bucket/custpartsize --insecure >/dev/null || exit_1 + +cleanup diff --git a/docs/site-replication/run-ssec-object-replication-with-compression.sh b/docs/site-replication/run-ssec-object-replication-with-compression.sh new file mode 100755 index 000000000..d88ec70f1 --- /dev/null +++ b/docs/site-replication/run-ssec-object-replication-with-compression.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + + exit 1 +} + +cleanup() { + echo -n "Cleaning up instances of MinIO ..." + pkill minio || sudo pkill minio + pkill -9 minio || sudo pkill -9 minio + rm -rf /tmp/minio{1,2} + echo "done" +} + +cleanup + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" + +# Create certificates for TLS enabled MinIO +echo -n "Setup certs for MinIO instances ..." +wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen +./certgen --host localhost +mkdir -p ~/.minio/certs +mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs +mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs +echo "done" + +# Start MinIO instances +echo -n "Starting MinIO instances ..." +minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 & +minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 & +echo "done" + +if [ ! -f ./mc ]; then + echo -n "Downloading MinIO client ..." + wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc && + chmod +x mc + echo "done" +fi + +sleep 10 + +export MC_HOST_minio1=https://minio:minio123@localhost:9001 +export MC_HOST_minio2=https://minio:minio123@localhost:9002 + +# Prepare data for tests +echo -n "Preparing test data ..." +mkdir -p /tmp/data +echo "Hello world" >/tmp/data/plainfile +echo "Hello from encrypted world" >/tmp/data/encrypted +touch /tmp/data/defpartsize +shred -s 500M /tmp/data/defpartsize +touch /tmp/data/mpartobj.txt +shred -s 500M /tmp/data/mpartobj.txt +echo "done" + +# Enable compression for site minio1 +./mc admin config set minio1 compression enable=on extensions=".txt" --insecure +./mc admin config set minio1 compression allow_encryption=on --insecure + +# Create bucket in source cluster +echo "Create bucket in source MinIO instance" +./mc mb minio1/test-bucket --insecure + +# Load objects to source site +echo "Loading objects to source MinIO instance" +./mc cp /tmp/data/plainfile minio1/test-bucket --insecure +./mc cp /tmp/data/encrypted minio1/test-bucket --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure +./mc cp /tmp/data/defpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure + +# Below should fail as compression and SSEC used at the same time +RESULT=$({ ./mc put /tmp/data/mpartobj.txt minio1/test-bucket --encrypt-key "minio1/test-bucket/mpartobj.txt=iliketobecrazybutnotsomuchreally" --insecure; } 2>&1) +if [[ ${RESULT} != *"Server side encryption specified with SSE-C with compression not allowed"* ]]; then + echo "BUG: Loading an SSE-C object to site with compression should fail. Succeeded though." + exit_1 +fi + +# Add replication site +./mc admin replicate add minio1 minio2 --insecure +# sleep for replication to complete +sleep 30 + +# List the objects from source site +echo "Objects from source instance" +./mc ls minio1/test-bucket --insecure +count1=$(./mc ls minio1/test-bucket/plainfile --insecure | wc -l) +if [ "${count1}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/plainfile not found" + exit_1 +fi +count2=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l) +if [ "${count2}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/encrypted not found" + exit_1 +fi +count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l) +if [ "${count3}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/defpartsize not found" + exit_1 +fi +sleep 120 + +# List the objects from replicated site +echo "Objects from replicated instance" +./mc ls minio2/test-bucket --insecure +repcount1=$(./mc ls minio2/test-bucket/plainfile --insecure | wc -l) +if [ "${repcount1}" -ne 1 ]; then + echo "BUG: object test-bucket/plainfile not replicated" + exit_1 +fi +repcount2=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l) +if [ "${repcount2}" -ne 1 ]; then + echo "BUG: object test-bucket/encrypted not replicated" + exit_1 +fi +repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l) +if [ "${repcount3}" -ne 1 ]; then + echo "BUG: object test-bucket/defpartsize not replicated" + exit_1 +fi + +# Stat the SSEC objects from source site +echo "Stat minio1/test-bucket/encrypted" +./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out1=$(./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json) +src_obj1_etag=$(echo "${stat_out1}" | jq '.etag') +src_obj1_size=$(echo "${stat_out1}" | jq '.size') +src_obj1_md5=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio1/test-bucket/defpartsize" +./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out2=$(./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json) +src_obj2_etag=$(echo "${stat_out2}" | jq '.etag') +src_obj2_size=$(echo "${stat_out2}" | jq '.size') +src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Stat the SSEC objects from replicated site +echo "Stat minio2/test-bucket/encrypted" +./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json) +rep_obj1_etag=$(echo "${stat_out1_rep}" | jq '.etag') +rep_obj1_size=$(echo "${stat_out1_rep}" | jq '.size') +rep_obj1_md5=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio2/test-bucket/defpartsize" +./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json) +rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag') +rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size') +rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Check the etag and size of replicated SSEC objects +if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then + echo "BUG: Etag: '${rep_obj1_etag}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_etag}'" + exit_1 +fi +if [ "${rep_obj1_size}" != "${src_obj1_size}" ]; then + echo "BUG: Size: '${rep_obj1_size}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_size}'" + exit_1 +fi +if [ "${rep_obj2_etag}" != "${src_obj2_etag}" ]; then + echo "BUG: Etag: '${rep_obj2_etag}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_etag}'" + exit_1 +fi +if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then + echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'" + exit_1 +fi + +# Check content of replicated SSEC objects +./mc cat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure +./mc cat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1 + +# Check the MD5 checksums of encrypted objects from source and target +if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/encrypted' doesn't match with source. Expected: '${src_obj1_md5}', Found: '${rep_obj1_md5}'" + exit_1 +fi +if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'" + exit_1 +fi + +cleanup diff --git a/docs/site-replication/run-ssec-object-replication.sh b/docs/site-replication/run-ssec-object-replication.sh new file mode 100755 index 000000000..188e441db --- /dev/null +++ b/docs/site-replication/run-ssec-object-replication.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + + exit 1 +} + +cleanup() { + echo -n "Cleaning up instances of MinIO ..." + pkill minio || sudo pkill minio + pkill -9 minio || sudo pkill -9 minio + rm -rf /tmp/minio{1,2} + echo "done" +} + +cleanup + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" + +# Create certificates for TLS enabled MinIO +echo -n "Setup certs for MinIO instances ..." +wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen +./certgen --host localhost +mkdir -p ~/.minio/certs +mv public.crt ~/.minio/certs || sudo mv public.crt ~/.minio/certs +mv private.key ~/.minio/certs || sudo mv private.key ~/.minio/certs +echo "done" + +# Start MinIO instances +echo -n "Starting MinIO instances ..." +minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 & +minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 & +echo "done" + +if [ ! -f ./mc ]; then + echo -n "Downloading MinIO client ..." + wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc && + chmod +x mc + echo "done" +fi + +sleep 10 + +export MC_HOST_minio1=https://minio:minio123@localhost:9001 +export MC_HOST_minio2=https://minio:minio123@localhost:9002 + +# Prepare data for tests +echo -n "Preparing test data ..." +mkdir -p /tmp/data +echo "Hello world" >/tmp/data/plainfile +echo "Hello from encrypted world" >/tmp/data/encrypted +touch /tmp/data/defpartsize +shred -s 500M /tmp/data/defpartsize +touch /tmp/data/custpartsize +shred -s 500M /tmp/data/custpartsize +echo "done" + +# Add replication site +./mc admin replicate add minio1 minio2 --insecure +# sleep for replication to complete +sleep 30 + +# Create bucket in source cluster +echo "Create bucket in source MinIO instance" +./mc mb minio1/test-bucket --insecure + +# Load objects to source site +echo "Loading objects to source MinIO instance" +./mc cp /tmp/data/plainfile minio1/test-bucket --insecure +./mc cp /tmp/data/encrypted minio1/test-bucket --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure +./mc cp /tmp/data/defpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure +./mc put /tmp/data/custpartsize minio1/test-bucket --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --part-size 50MiB +sleep 120 + +# List the objects from source site +echo "Objects from source instance" +./mc ls minio1/test-bucket --insecure +count1=$(./mc ls minio1/test-bucket/plainfile --insecure | wc -l) +if [ "${count1}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/plainfile not found" + exit_1 +fi +count2=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l) +if [ "${count2}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/encrypted not found" + exit_1 +fi +count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l) +if [ "${count3}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/defpartsize not found" + exit_1 +fi +count4=$(./mc ls minio1/test-bucket/custpartsize --insecure | wc -l) +if [ "${count4}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/custpartsize not found" + exit_1 +fi + +# List the objects from replicated site +echo "Objects from replicated instance" +./mc ls minio2/test-bucket --insecure +repcount1=$(./mc ls minio2/test-bucket/plainfile --insecure | wc -l) +if [ "${repcount1}" -ne 1 ]; then + echo "BUG: object test-bucket/plainfile not replicated" + exit_1 +fi +repcount2=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l) +if [ "${repcount2}" -ne 1 ]; then + echo "BUG: object test-bucket/encrypted not replicated" + exit_1 +fi +repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l) +if [ "${repcount3}" -ne 1 ]; then + echo "BUG: object test-bucket/defpartsize not replicated" + exit_1 +fi + +repcount4=$(./mc ls minio2/test-bucket/custpartsize --insecure | wc -l) +if [ "${repcount4}" -ne 1 ]; then + echo "BUG: object test-bucket/custpartsize not replicated" + exit_1 +fi + +# Stat the SSEC objects from source site +echo "Stat minio1/test-bucket/encrypted" +./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out1=$(./mc stat minio1/test-bucket/encrypted --encrypt-key "minio1/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json) +src_obj1_etag=$(echo "${stat_out1}" | jq '.etag') +src_obj1_size=$(echo "${stat_out1}" | jq '.size') +src_obj1_md5=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio1/test-bucket/defpartsize" +./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out2=$(./mc stat minio1/test-bucket/defpartsize --encrypt-key "minio1/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json) +src_obj2_etag=$(echo "${stat_out2}" | jq '.etag') +src_obj2_size=$(echo "${stat_out2}" | jq '.size') +src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio1/test-bucket/custpartsize" +./mc stat minio1/test-bucket/custpartsize --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out3=$(./mc stat minio1/test-bucket/custpartsize --encrypt-key "minio1/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json) +src_obj3_etag=$(echo "${stat_out3}" | jq '.etag') +src_obj3_size=$(echo "${stat_out3}" | jq '.size') +src_obj3_md5=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Stat the SSEC objects from replicated site +echo "Stat minio2/test-bucket/encrypted" +./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out1_rep=$(./mc stat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure --json) +rep_obj1_etag=$(echo "${stat_out1_rep}" | jq '.etag') +rep_obj1_size=$(echo "${stat_out1_rep}" | jq '.size') +rep_obj1_md5=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio2/test-bucket/defpartsize" +./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out2_rep=$(./mc stat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure --json) +rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag') +rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size') +rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio2/test-bucket/custpartsize" +./mc stat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json +stat_out3_rep=$(./mc stat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure --json) +rep_obj3_etag=$(echo "${stat_out3_rep}" | jq '.etag') +rep_obj3_size=$(echo "${stat_out3_rep}" | jq '.size') +rep_obj3_md5=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Check the etag and size of replicated SSEC objects +if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then + echo "BUG: Etag: '${rep_obj1_etag}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_etag}'" + exit_1 +fi +if [ "${rep_obj1_size}" != "${src_obj1_size}" ]; then + echo "BUG: Size: '${rep_obj1_size}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_size}'" + exit_1 +fi +if [ "${rep_obj2_etag}" != "${src_obj2_etag}" ]; then + echo "BUG: Etag: '${rep_obj2_etag}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_etag}'" + exit_1 +fi +if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then + echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'" + exit_1 +fi +if [ "${rep_obj3_etag}" != "${src_obj3_etag}" ]; then + echo "BUG: Etag: '${rep_obj3_etag}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_etag}'" + exit_1 +fi +if [ "${rep_obj3_size}" != "${src_obj3_size}" ]; then + echo "BUG: Size: '${rep_obj3_size}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_size}'" + exit_1 +fi + +# Check content of replicated SSEC objects +./mc cat minio2/test-bucket/encrypted --encrypt-key "minio2/test-bucket/encrypted=iliketobecrazybutnotsomuchreally" --insecure +./mc cat minio2/test-bucket/defpartsize --encrypt-key "minio2/test-bucket/defpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1 +./mc cat minio2/test-bucket/custpartsize --encrypt-key "minio2/test-bucket/custpartsize=iliketobecrazybutnotsomuchreally" --insecure >/dev/null || exit_1 + +# Check the MD5 checksums of encrypted objects from source and target +if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/encrypted' doesn't match with source. Expected: '${src_obj1_md5}', Found: '${rep_obj1_md5}'" + exit_1 +fi +if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'" + exit_1 +fi +if [ "${src_obj3_md5}" != "${rep_obj3_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/custpartsize' doesn't match with source. Expected: '${src_obj3_md5}', Found: '${rep_obj3_md5}'" + exit_1 +fi + +cleanup diff --git a/internal/bucket/replication/replication.go b/internal/bucket/replication/replication.go index c95b15452..8a179ebdd 100644 --- a/internal/bucket/replication/replication.go +++ b/internal/bucket/replication/replication.go @@ -220,9 +220,6 @@ func (c Config) GetDestination() Destination { // Replicate returns true if the object should be replicated. func (c Config) Replicate(obj ObjectOpts) bool { - if obj.SSEC { - return false - } for _, rule := range c.FilterActionableRules(obj) { if rule.Status == Disabled { continue diff --git a/internal/bucket/replication/replication_test.go b/internal/bucket/replication/replication_test.go index 4e872fd66..8fae740e1 100644 --- a/internal/bucket/replication/replication_test.go +++ b/internal/bucket/replication/replication_test.go @@ -250,12 +250,12 @@ func TestReplicate(t *testing.T) { {ObjectOpts{Name: "c1test"}, cfgs[0], true}, // 2. valid ObjectOpts passing empty Filter {ObjectOpts{Name: "c1test", VersionID: "vid"}, cfgs[0], true}, // 3. valid ObjectOpts passing empty Filter - {ObjectOpts{Name: "c1test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 4. DeleteMarker version replication valid case - matches DeleteMarkerReplication status - {ObjectOpts{Name: "c1test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[0], true}, // 5. permanent delete of version, matches DeleteReplication status - valid case - {ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 6. permanent delete of version, matches DeleteReplication status - {ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], false}, // 7. permanent delete of version, disqualified by SSE-C - {ObjectOpts{Name: "c1test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], false}, // 8. setting DeleteMarker on SSE-C encrypted object, disqualified by SSE-C - {ObjectOpts{Name: "c1test", SSEC: true}, cfgs[0], false}, // 9. replication of SSE-C encrypted object, disqualified + {ObjectOpts{Name: "c1test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 4. DeleteMarker version replication valid case - matches DeleteMarkerReplication status + {ObjectOpts{Name: "c1test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[0], true}, // 5. permanent delete of version, matches DeleteReplication status - valid case + {ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 6. permanent delete of version, matches DeleteReplication status + {ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 7. permanent delete of version + {ObjectOpts{Name: "c1test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 8. setting DeleteMarker on SSE-C encrypted object + {ObjectOpts{Name: "c1test", SSEC: true}, cfgs[0], true}, // 9. replication of SSE-C encrypted object // using config 2 - no filters, only replication of object, metadata enabled {ObjectOpts{Name: "c2test"}, cfgs[1], true}, // 10. valid ObjectOpts passing empty Filter @@ -264,7 +264,7 @@ func TestReplicate(t *testing.T) { {ObjectOpts{Name: "c2test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 13. permanent delete of DeleteMarker version, disallowed by DeleteReplication status {ObjectOpts{Name: "c2test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 14. permanent delete of version, disqualified by SSE-C & DeleteReplication status {ObjectOpts{Name: "c2test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 15. setting DeleteMarker on SSE-C encrypted object, disqualified by SSE-C & DeleteMarkerReplication status - {ObjectOpts{Name: "c2test", SSEC: true}, cfgs[1], false}, // 16. replication of SSE-C encrypted object, disqualified by default + {ObjectOpts{Name: "c2test", SSEC: true}, cfgs[1], true}, // 16. replication of SSE-C encrypted object // using config 2 - has more than one rule with overlapping prefixes {ObjectOpts{Name: "xy/c3test", UserTags: "k1=v1"}, cfgs[2], true}, // 17. matches rule 1 for replication of content/metadata {ObjectOpts{Name: "xyz/c3test", UserTags: "k1=v1"}, cfgs[2], true}, // 18. matches rule 1 for replication of content/metadata diff --git a/internal/crypto/error.go b/internal/crypto/error.go index eca05a9f9..72fb4674c 100644 --- a/internal/crypto/error.go +++ b/internal/crypto/error.go @@ -82,6 +82,8 @@ var ( // ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible // The client needs to remove the SSE-S3 header or the SSE-C headers ErrIncompatibleEncryptionMethod = Errorf("Server side encryption specified with both SSE-C and SSE-S3 headers") + // ErrIncompatibleEncryptionWithCompression indicates that both data compression and SSE-C not allowed at the same time + ErrIncompatibleEncryptionWithCompression = Errorf("Server side encryption specified with SSE-C with compression not allowed") // ErrInvalidEncryptionKeyID returns error when KMS key id contains invalid characters ErrInvalidEncryptionKeyID = Errorf("KMS KeyID contains unsupported characters") diff --git a/internal/http/headers.go b/internal/http/headers.go index 2c0fa1efc..9b8054e80 100644 --- a/internal/http/headers.go +++ b/internal/http/headers.go @@ -233,6 +233,9 @@ const ( // Header indicates a Tag operation was performed on one/more peers successfully, though the // current cluster does not have the object yet. This is in a site/bucket replication scenario. MinIOTaggingProxied = "X-Minio-Tagging-Proxied" + // Header indicates the actual replicated object size + // In case of SSEC objects getting replicated (multipart) actual size would be needed at target + MinIOReplicationActualObjectSize = "X-Minio-Replication-Actual-Object-Size" // predicted date/time of transition MinIOTransition = "X-Minio-Transition"