From 690434514d1b861109837679d8a24994e410020a Mon Sep 17 00:00:00 2001 From: Poorna Krishnamoorthy Date: Wed, 3 Mar 2021 11:13:31 -0800 Subject: [PATCH] Avoid notification event for replicas (#11683) Creating notification events for replica creation is not particularly useful to send as the notification event generated at source already includes replication completion events. For applications using replica cluster as failover, avoiding duplicate notifications for replica event will allow seamless failover. --- cmd/bucket-replication.go | 16 +++++++++++----- cmd/handler-utils.go | 6 +++++- cmd/http/headers.go | 2 ++ cmd/notification.go | 5 +++++ go.mod | 3 +-- go.sum | 4 ++++ 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index 7e3fd6ac3..3658cba38 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -268,6 +268,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "", ReplicationMTime: dobj.DeleteMarkerMTime.Time, ReplicationStatus: miniogo.ReplicationStatusReplica, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside }, }) @@ -412,10 +413,11 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn ContentEncoding: objInfo.ContentEncoding, StorageClass: sc, Internal: miniogo.AdvancedPutOptions{ - SourceVersionID: objInfo.VersionID, - ReplicationStatus: miniogo.ReplicationStatusReplica, - SourceMTime: objInfo.ModTime, - SourceETag: objInfo.ETag, + SourceVersionID: objInfo.VersionID, + ReplicationStatus: miniogo.ReplicationStatusReplica, + SourceMTime: objInfo.ModTime, + SourceETag: objInfo.ETag, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside }, } if objInfo.UserTags != "" { @@ -653,7 +655,11 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa Bucket: dest.Bucket, Object: object, VersionID: objInfo.VersionID} - dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}} + dstOpts := miniogo.PutObjectOptions{ + Internal: miniogo.AdvancedPutOptions{ + SourceVersionID: objInfo.VersionID, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside + }} if _, err = c.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), srcOpts, dstOpts); err != nil { replicationStatus = replication.Failed logger.LogIf(ctx, fmt.Errorf("Unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err)) diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go index aabd8e697..8f193324e 100644 --- a/cmd/handler-utils.go +++ b/cmd/handler-utils.go @@ -234,12 +234,16 @@ func extractReqParams(r *http.Request) map[string]string { cred := getReqAccessCred(r, region) // Success. - return map[string]string{ + m := map[string]string{ "region": region, "accessKey": cred.AccessKey, "sourceIPAddress": handlers.GetSourceIP(r), // Add more fields here. } + if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok { + m[xhttp.MinIOSourceReplicationRequest] = "" + } + return m } // Extract response elements to be sent with event notifiation. diff --git a/cmd/http/headers.go b/cmd/http/headers.go index 882f9a397..3a04cd15e 100644 --- a/cmd/http/headers.go +++ b/cmd/http/headers.go @@ -165,6 +165,8 @@ const ( MinIODeleteMarkerReplicationStatus = "X-Minio-Replication-DeleteMarker-Status" // Header indicates if its a GET/HEAD proxy request for active-active replication MinIOSourceProxyRequest = "X-Minio-Source-Proxy-Request" + // Header indicates that this request is a replication request to create a REPLICA + MinIOSourceReplicationRequest = "X-Minio-Source-Replication-Request" ) // Common http query params S3 API diff --git a/cmd/notification.go b/cmd/notification.go index 70d084bdf..8f1020cd0 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -33,6 +33,7 @@ import ( "github.com/klauspost/compress/zip" "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio/cmd/crypto" + xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" bandwidth "github.com/minio/minio/pkg/bandwidth" bucketBandwidth "github.com/minio/minio/pkg/bucket/bandwidth" @@ -1387,6 +1388,10 @@ func (args eventArgs) ToEvent(escape bool) event.Event { func sendEvent(args eventArgs) { args.Object.Size, _ = args.Object.GetActualSize() + // avoid generating a notification for REPLICA creation event. + if _, ok := args.ReqParams[xhttp.MinIOSourceReplicationRequest]; ok { + return + } // remove sensitive encryption entries in metadata. crypto.RemoveSensitiveEntries(args.Object.UserDefined) crypto.RemoveInternalEntries(args.Object.UserDefined) diff --git a/go.mod b/go.mod index 3f1429399..0cc0dbe0f 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/minio/cli v1.22.0 github.com/minio/highwayhash v1.0.1 github.com/minio/md5-simd v1.1.1 // indirect - github.com/minio/minio-go/v7 v7.0.10 + github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 github.com/minio/selfupdate v0.3.1 github.com/minio/sha256-simd v1.0.0 github.com/minio/simdjson-go v0.2.1 @@ -72,7 +72,6 @@ require ( github.com/rs/cors v1.7.0 github.com/secure-io/sio-go v0.3.1 github.com/shirou/gopsutil/v3 v3.21.1 - github.com/sirupsen/logrus v1.7.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/streadway/amqp v1.0.0 github.com/tidwall/gjson v1.6.7 diff --git a/go.sum b/go.sum index 240f22663..add85a34b 100644 --- a/go.sum +++ b/go.sum @@ -364,6 +364,7 @@ github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -397,6 +398,8 @@ github.com/minio/md5-simd v1.1.1 h1:9ojcLbuZ4gXbB2sX53MKn8JUZ0sB/2wfwsEcRw+I08U= github.com/minio/md5-simd v1.1.1/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.10 h1:1oUKe4EOPUEhw2qnPQaPsJ0lmVTYLFu03SiItauXs94= github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo= +github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 h1:v7OMbUnWkyRlO2MZ5AuYioELhwXF/BgZEznrQ1drBEM= +github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw= github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -555,6 +558,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.1 h1:T/YLemO5Yp7KPzS+lVtu+WsHn8yoSwTfItdAd1r3cck= github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=