scott/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2026-02-04 18:00:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

merge object lifecycle checks into usage crawler (#9579)

Browse Source
This commit is contained in:
Klaus Post
2020-06-12 10:28:21 -07:00
committed by GitHub
parent 225b812b5e
commit 43d6e3ae06
18 changed files with 1400 additions and 852 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
cmd/data-usage_test.go
View File

@@ -17,9 +17,12 @@
package cmd
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"testing"
)
@@ -34,6 +37,7 @@ func TestDataUsageUpdate(t *testing.T) {
if err != nil {
t.Skip(err)
}
const bucket = "bucket"
defer os.RemoveAll(base)
var files = []usageTestFile{
{name: "rootfile", size: 10000},
@@ -45,9 +49,9 @@ func TestDataUsageUpdate(t *testing.T) {
{name: "dir1/dira/dirasub/dcfile", size: 1000000},
{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
}
createUsageTestFiles(t, base, files)
createUsageTestFiles(t, base, bucket, files)
getSize := func(item Item) (i int64, err error) {
getSize := func(item crawlItem) (i int64, err error) {
if item.Typ&os.ModeDir == 0 {
s, err := os.Stat(item.Path)
if err != nil {
@@ -58,7 +62,7 @@ func TestDataUsageUpdate(t *testing.T) {
return 0, nil
}
got, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
if err != nil {
t.Fatal(err)
}
@@ -105,16 +109,17 @@ func TestDataUsageUpdate(t *testing.T) {
oSizes: sizeHistogram{0: 1, 1: 3},
},
{
path: "/dir1/dira",
size: 300000,
objs: 2,
oSizes: sizeHistogram{0: 0, 1: 2},
path: "/dir1",
size: 2000,
objs: 1,
oSizes: sizeHistogram{0: 0, 1: 1},
},
{
// Children are flattened
path: "/dir1/dira/",
size: 300000,
objs: 2,
oSizes: sizeHistogram{0: 0, 1: 2},
size: 1300010,
objs: 4,
oSizes: sizeHistogram{0: 1, 1: 3},
},
{
path: "/nonexistying",
@@ -123,8 +128,9 @@ func TestDataUsageUpdate(t *testing.T) {
}
for _, w := range want {
t.Run(w.path, func(t *testing.T) {
e := got.find(w.path)
p := path.Join(bucket, w.path)
t.Run(p, func(t *testing.T) {
e := got.find(p)
if w.isNil {
if e != nil {
t.Error("want nil, got", e)
@@ -134,6 +140,7 @@ func TestDataUsageUpdate(t *testing.T) {
if e == nil {
t.Fatal("got nil result")
}
t.Log(e.Children)
if w.flatten {
*e = got.flatten(*e)
}
@@ -175,8 +182,8 @@ func TestDataUsageUpdate(t *testing.T) {
size: 1000,
},
}
createUsageTestFiles(t, base, files)
got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
createUsageTestFiles(t, base, bucket, files)
got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
if err != nil {
t.Fatal(err)
}
@@ -222,7 +229,7 @@ func TestDataUsageUpdate(t *testing.T) {
for _, w := range want {
t.Run(w.path, func(t *testing.T) {
e := got.find(w.path)
e := got.find(path.Join(bucket, w.path))
if w.isNil {
if e != nil {
t.Error("want nil, got", e)
@@ -233,6 +240,7 @@ func TestDataUsageUpdate(t *testing.T) {
t.Fatal("got nil result")
}
if w.flatten {
t.Log(e.Children)
*e = got.flatten(*e)
}
if e.Size != int64(w.size) {
@@ -254,14 +262,14 @@ func TestDataUsageUpdate(t *testing.T) {
},
}
createUsageTestFiles(t, base, files)
err = os.RemoveAll(filepath.Join(base, "dir1/dira/dirasub/dcfile"))
createUsageTestFiles(t, base, bucket, files)
err = os.RemoveAll(filepath.Join(base, bucket, "dir1/dira/dirasub/dcfile"))
if err != nil {
t.Fatal(err)
}
// Changed dir must be picked up in this many cycles.
for i := 0; i < dataUsageUpdateDirCycles; i++ {
got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
if err != nil {
t.Fatal(err)
}
@@ -291,8 +299,9 @@ func TestDataUsageUpdate(t *testing.T) {
}
for _, w := range want {
t.Run(w.path, func(t *testing.T) {
e := got.find(w.path)
p := path.Join(bucket, w.path)
t.Run(p, func(t *testing.T) {
e := got.find(p)
if w.isNil {
if e != nil {
t.Error("want nil, got", e)
@@ -335,9 +344,9 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
{name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000},
{name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
}
createUsageTestFiles(t, base, files)
createUsageTestFiles(t, base, "", files)
getSize := func(item Item) (i int64, err error) {
getSize := func(item crawlItem) (i int64, err error) {
if item.Typ&os.ModeDir == 0 {
s, err := os.Stat(item.Path)
if err != nil {
@@ -347,7 +356,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
}
return 0, nil
}
got, err := updateUsage(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
if err != nil {
t.Fatal(err)
}
@@ -449,8 +458,8 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
size: 1000,
},
}
createUsageTestFiles(t, base, files)
got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
createUsageTestFiles(t, base, "", files)
got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
if err != nil {
t.Fatal(err)
}
@@ -526,14 +535,14 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
},
}
createUsageTestFiles(t, base, files)
createUsageTestFiles(t, base, "", files)
err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile"))
if err != nil {
t.Fatal(err)
}
// Changed dir must be picked up in this many cycles.
for i := 0; i < dataUsageUpdateDirCycles; i++ {
got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
if err != nil {
t.Fatal(err)
}
@@ -588,13 +597,13 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
}
}
func createUsageTestFiles(t *testing.T, base string, files []usageTestFile) {
func createUsageTestFiles(t *testing.T, base, bucket string, files []usageTestFile) {
for _, f := range files {
err := os.MkdirAll(filepath.Dir(filepath.Join(base, f.name)), os.ModePerm)
err := os.MkdirAll(filepath.Dir(filepath.Join(base, bucket, f.name)), os.ModePerm)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(base, f.name), make([]byte, f.size), os.ModePerm)
err = ioutil.WriteFile(filepath.Join(base, bucket, f.name), make([]byte, f.size), os.ModePerm)
if err != nil {
t.Fatal(err)
}
@@ -606,20 +615,28 @@ func TestDataUsageCacheSerialize(t *testing.T) {
if err != nil {
t.Skip(err)
}
const bucket = "abucket"
defer os.RemoveAll(base)
var files = []usageTestFile{
{name: "rootfile", size: 10000},
{name: "rootfile2", size: 10000},
{name: "dir1/d1file", size: 2000},
{name: "dir2/d2file", size: 300},
{name: "dir2/d2file2", size: 300},
{name: "dir2/d2file3/", size: 300},
{name: "dir2/d2file4/", size: 300},
{name: "dir2/d2file5", size: 300},
{name: "dir1/dira/dafile", size: 100000},
{name: "dir1/dira/dbfile", size: 200000},
{name: "dir1/dira/dirasub/dcfile", size: 1000000},
{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
{name: "dir1/dira/dirasub/sublevel3/dccccfile20", size: 20},
{name: "dir1/dira/dirasub/sublevel3/dccccfile30", size: 30},
{name: "dir1/dira/dirasub/sublevel3/dccccfile40", size: 40},
}
createUsageTestFiles(t, base, files)
createUsageTestFiles(t, base, bucket, files)
getSize := func(item Item) (i int64, err error) {
getSize := func(item crawlItem) (i int64, err error) {
if item.Typ&os.ModeDir == 0 {
s, err := os.Stat(item.Path)
if err != nil {
@@ -629,23 +646,33 @@ func TestDataUsageCacheSerialize(t *testing.T) {
}
return 0, nil
}
want, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
if err != nil {
t.Fatal(err)
}
b := want.serialize()
var got dataUsageCache
err = got.deserialize(b)
err = got.deserialize(bytes.NewBuffer(b))
if err != nil {
t.Fatal(err)
}
t.Log("serialized size:", len(b), "bytes")
if got.Info.LastUpdate.IsZero() {
t.Error("lastupdate not set")
}
if !want.Info.LastUpdate.Equal(got.Info.LastUpdate) {
t.Fatalf("deserialize mismatch\nwant: %+v\ngot: %+v", want, got)
t.Fatalf("deserialize LastUpdate mismatch\nwant: %+v\ngot: %+v", want, got)
}
if len(want.Cache) != len(got.Cache) {
t.Errorf("deserialize mismatch length\nwant: %+v\ngot: %+v", len(want.Cache), len(got.Cache))
}
for wkey, wval := range want.Cache {
gotv := got.Cache[wkey]
if fmt.Sprint(gotv) != fmt.Sprint(wval) {
t.Errorf("deserialize mismatch, key %v\nwant: %+v\ngot: %+v", wkey, wval, gotv)
}
}
}