scott/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2026-02-04 18:00:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

xl: Implement posix.DeletePrefixes to enhance delete perf (#9100)

Browse Source 
Bulk delete API was using cleanupObjectsBulk() which calls posix
listing and delete API to remove objects internal files in the
backend (xl.json and parts) one by one.

Add DeletePrefixes in the storage API to remove the content
of a directory in a single call.

Also use a remove goroutine for each disk to accelerate removal.
This commit is contained in:
Anis Elleuch
2020-03-11 08:56:36 -07:00
committed by GitHub
parent 7c32f3f554
commit 0af62d35a0
10 changed files with 226 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
cmd/object-api-common.go
View File

@@ -147,78 +147,6 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
return err
}
// Cleanup objects in bulk and recursively: each object will have a list of sub-files to delete in the backend
func cleanupObjectsBulk(storage StorageAPI, volume string, objsPaths []string, errs []error) ([]error, error) {
// The list of files in disk to delete
var filesToDelete []string
// Map files to delete to the passed objsPaths
var filesToDeleteObjsIndexes []int
// Traverse and return the list of sub entries
var traverse func(string) ([]string, error)
traverse = func(entryPath string) ([]string, error) {
var output = make([]string, 0)
if !HasSuffix(entryPath, SlashSeparator) {
output = append(output, entryPath)
return output, nil
}
entries, err := storage.ListDir(volume, entryPath, -1, "")
if err != nil {
if err == errFileNotFound {
return nil, nil
}
return nil, err
}
for _, entry := range entries {
subEntries, err := traverse(pathJoin(entryPath, entry))
if err != nil {
return nil, err
}
output = append(output, subEntries...)
}
return output, nil
}
// Find and collect the list of files to remove associated
// to the passed objects paths
for idx, objPath := range objsPaths {
if errs[idx] != nil {
continue
}
output, err := traverse(retainSlash(pathJoin(objPath)))
if err != nil {
errs[idx] = err
continue
} else {
errs[idx] = nil
}
filesToDelete = append(filesToDelete, output...)
for i := 0; i < len(output); i++ {
filesToDeleteObjsIndexes = append(filesToDeleteObjsIndexes, idx)
}
}
// Reverse the list so remove can succeed
reverseStringSlice(filesToDelete)
dErrs, err := storage.DeleteFileBulk(volume, filesToDelete)
if err != nil {
return nil, err
}
// Map files deletion errors to the correspondent objects
for i := range dErrs {
if dErrs[i] != nil {
if errs[filesToDeleteObjsIndexes[i]] != nil {
errs[filesToDeleteObjsIndexes[i]] = dErrs[i]
}
}
}
return errs, nil
}
// Removes notification.xml for a given bucket, only used during DeleteBucket.
func removeNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
// Verify bucket is valid.