s3: Fix early listing stopping when ILM is enabled (#472) (#21246)

S3 listing call is usually sent with a 'max-keys' parameter. This
'max-keys' will also be passed to WalkDir() call. However, when ILM is
enabled in a bucket and some objects are skipped, the listing can
return IsTruncated set to false even if there are more entries in
the drives.

The reason is that drives stop feeding the listing code because it has
max-keys parameter and the listing code thinks listing is finished
because it is being fed anymore.

Ask the drives to not stop listing and relies on the context
cancellation to stop listing in the drives as fast as possible.

cmd/object-api-listobjects_test.go

@@ -26,6 +26,9 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
 )
 
 func TestListObjectsVersionedFolders(t *testing.T) {
@@ -1929,3 +1932,112 @@ func BenchmarkListObjects(b *testing.B) {
 		}
 	}
 }
+
+func TestListObjectsWithILM(t *testing.T) {
+	ExecObjectLayerTest(t, testListObjectsWithILM)
+}
+
+func testListObjectsWithILM(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
+	t, _ := t1.(*testing.T)
+
+	objContent := "test-content"
+	objMd5 := md5.Sum([]byte(objContent))
+
+	uploads := []struct {
+		bucket     string
+		expired    int
+		notExpired int
+	}{
+		{"test-list-ilm-nothing-expired", 0, 6},
+		{"test-list-ilm-all-expired", 6, 0},
+		{"test-list-ilm-all-half-expired", 3, 3},
+	}
+
+	oneWeekAgo := time.Now().Add(-7 * 24 * time.Hour)
+
+	lifecycleBytes := []byte(`
+<LifecycleConfiguration>
+	<Rule>
+		<Status>Enabled</Status>
+		<Expiration>
+			<Days>1</Days>
+		</Expiration>
+	</Rule>
+</LifecycleConfiguration>
+`)
+
+	lifecycleConfig, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(lifecycleBytes))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i, upload := range uploads {
+		err := obj.MakeBucket(context.Background(), upload.bucket, MakeBucketOptions{})
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+
+		globalBucketMetadataSys.Set(upload.bucket, BucketMetadata{lifecycleConfig: lifecycleConfig})
+		defer globalBucketMetadataSys.Remove(upload.bucket)
+
+		// Upload objects which modtime as one week ago, supposed to be expired by ILM
+		for range upload.expired {
+			_, err := obj.PutObject(context.Background(), upload.bucket, randString(32),
+				mustGetPutObjReader(t,
+					bytes.NewBufferString(objContent),
+					int64(len(objContent)),
+					hex.EncodeToString(objMd5[:]),
+					""),
+				ObjectOptions{MTime: oneWeekAgo},
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// Upload objects which current time as modtime, not expired by ILM
+		for range upload.notExpired {
+			_, err := obj.PutObject(context.Background(), upload.bucket, randString(32),
+				mustGetPutObjReader(t,
+					bytes.NewBufferString(objContent),
+					int64(len(objContent)),
+					hex.EncodeToString(objMd5[:]),
+					""),
+				ObjectOptions{},
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		for _, maxKeys := range []int{1, 10, 49} {
+			// Test ListObjects V2
+			totalObjs, didRuns := 0, 0
+			marker := ""
+			for {
+				didRuns++
+				if didRuns > 1000 {
+					t.Fatal("too many runs")
+					return
+				}
+				result, err := obj.ListObjectsV2(context.Background(), upload.bucket, "", marker, "", maxKeys, false, "")
+				if err != nil {
+					t.Fatalf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i, instanceType, err.Error())
+				}
+				totalObjs += len(result.Objects)
+				if !result.IsTruncated {
+					break
+				}
+				if marker != "" && marker == result.NextContinuationToken {
+					t.Fatalf("infinite loop marker: %s", result.NextContinuationToken)
+				}
+				marker = result.NextContinuationToken
+			}
+
+			if totalObjs != upload.notExpired {
+				t.Fatalf("Test %d: %s: max-keys=%d, %d objects are expected to be seen, but %d found instead (didRuns=%d)",
+					i+1, instanceType, maxKeys, upload.notExpired, totalObjs, didRuns)
+			}
+		}
+	}
+}