From 21b62046924114b23b35dab18431d91a20d2c746 Mon Sep 17 00:00:00 2001 From: Shubhendu Date: Tue, 4 Jun 2024 17:08:26 +0530 Subject: [PATCH] Test proxying of DEL marker for bucket replication (#19870) Make sure to avoid proxying for DEL markers Signed-off-by: Shubhendu Ram Tripathi --- Makefile | 5 +- cmd/api-response.go | 2 +- cmd/object-handlers_test.go | 2 +- cmd/object-multipart-handlers.go | 2 +- .../replication/test_del_marker_proxying.sh | 73 +++++++++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) create mode 100755 docs/bucket/replication/test_del_marker_proxying.sh diff --git a/Makefile b/Makefile index 2a9d9cdee..e80797a66 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,10 @@ test-replication-3site: test-delete-replication: @(env bash $(PWD)/docs/bucket/replication/delete-replication.sh) -test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication +test-delete-marker-proxying: + @(env bash $(PWD)/docs/bucket/replication/test_del_marker_proxying.sh) + +test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error test-delete-marker-proxying ## verify multi site replication @echo "Running tests for replicating three sites" test-site-replication-ldap: install-race ## verify automatic site replication diff --git a/cmd/api-response.go b/cmd/api-response.go index 01a96cac4..5caa76bb5 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -789,7 +789,7 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi } // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag. -func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse { +func generateCompleteMultipartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse { cs := oi.decryptChecksums(0) c := CompleteMultipartUploadResponse{ Location: location, diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index 0925c38c8..33de77680 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -2914,7 +2914,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s s3MD5 := getCompleteMultipartMD5(inputParts[3].parts) // generating the response body content for the success case. - successResponse := generateCompleteMultpartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), ObjectInfo{ETag: s3MD5}) + successResponse := generateCompleteMultipartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), ObjectInfo{ETag: s3MD5}) encodedSuccessResponse := encodeResponse(successResponse) ctx := context.Background() diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go index 328494efc..ed6d3d56e 100644 --- a/cmd/object-multipart-handlers.go +++ b/cmd/object-multipart-handlers.go @@ -1040,7 +1040,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite // Get object location. location := getObjectLocation(r, globalDomainNames, bucket, object) // Generate complete multipart response. - response := generateCompleteMultpartUploadResponse(bucket, object, location, objInfo) + response := generateCompleteMultipartUploadResponse(bucket, object, location, objInfo) encodedSuccessResponse := encodeResponse(response) // Write success response. diff --git a/docs/bucket/replication/test_del_marker_proxying.sh b/docs/bucket/replication/test_del_marker_proxying.sh new file mode 100755 index 000000000..c64c9d154 --- /dev/null +++ b/docs/bucket/replication/test_del_marker_proxying.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + for site in sitea siteb; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + + exit 1 +} + +cleanup() { + echo -n "Cleaning up instances of MinIO ..." + pkill -9 minio || sudo pkill -9 minio + rm -rf /tmp/sitea + rm -rf /tmp/siteb + echo "done" +} + +cleanup + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" + +# Start MinIO instances +echo -n "Starting MinIO instances ..." +minio server --address ":9001" --console-address ":10000" /tmp/sitea/{1...4}/disk{1...4} /tmp/sitea/{5...8}/disk{1...4} >/tmp/sitea_1.log 2>&1 & +minio server --address ":9002" --console-address ":11000" /tmp/siteb/{1...4}/disk{1...4} /tmp/siteb/{5...8}/disk{1...4} >/tmp/siteb_1.log 2>&1 & +echo "done" + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001 +export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9002 + +./mc ready sitea +./mc ready siteb + +./mc mb sitea/bucket +./mc version enable sitea/bucket +./mc mb siteb/bucket +./mc version enable siteb/bucket + +# Set bucket replication +./mc replicate add sitea/bucket --remote-bucket siteb/bucket + +# Run the test to make sure proxying of DEL marker doesn't happen +loop_count=0 +while true; do + if [ $loop_count -eq 100 ]; then + break + fi + echo "Hello World" | ./mc pipe sitea/bucket/obj$loop_count + ./mc rm sitea/bucket/obj$loop_count + RESULT=$({ ./mc stat sitea/bucket/obj$loop_count; } 2>&1) + if [[ ${RESULT} != *"Object does not exist"* ]]; then + echo "BUG: stat should fail. succeeded." + exit_1 + fi + loop_count=$((loop_count + 1)) +done + +cleanup