scott/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2026-02-04 18:00:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

optimize mkdir calls to avoid base-dir Mkdir attempts (#18021)

Browse Source 
Currently we have IOPs of these patterns

```
[OS] os.Mkdir play.min.io:9000 /disk1 2.718µs
[OS] os.Mkdir play.min.io:9000 /disk1/data 2.406µs
[OS] os.Mkdir play.min.io:9000 /disk1/data/.minio.sys 4.068µs
[OS] os.Mkdir play.min.io:9000 /disk1/data/.minio.sys/tmp 2.843µs
[OS] os.Mkdir play.min.io:9000 /disk1/data/.minio.sys/tmp/d89c8ceb-f8d1-4cc6-b483-280f87c4719f 20.152µs
```

It can be seen that we can save quite Nx levels such as
if your drive is mounted at `/disk1/minio` you can simply
skip sending an `Mkdir /disk1/` and `Mkdir /disk1/minio`.

Since they are expected to exist already, this PR adds a way
for us to ignore all paths upto the mount or a directory which
ever has been provided to MinIO setup.
This commit is contained in:
Harshavardhana
2023-09-13 08:14:36 -07:00
committed by GitHub
parent 96fbf18201
commit 8b8be2695f
11 changed files with 60 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/os-reliable.go
View File

@@ -73,7 +73,7 @@ func reliableRemoveAll(dirPath string) (err error) {
// Wrapper functions to os.MkdirAll, which calls reliableMkdirAll
// this is to ensure that if there is a racy parent directory
// delete in between we can simply retry the operation.
func mkdirAll(dirPath string, mode os.FileMode) (err error) {
func mkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) {
if dirPath == "" {
return errInvalidArgument
}
@@ -82,7 +82,7 @@ func mkdirAll(dirPath string, mode os.FileMode) (err error) {
return err
}
if err = reliableMkdirAll(dirPath, mode); err != nil {
if err = reliableMkdirAll(dirPath, mode, baseDir); err != nil {
// File path cannot be verified since one of the parents is a file.
if isSysErrNotDir(err) {
return errFileAccessDenied
@@ -100,11 +100,11 @@ func mkdirAll(dirPath string, mode os.FileMode) (err error) {
// Reliably retries os.MkdirAll if for some reason os.MkdirAll returns
// syscall.ENOENT (parent does not exist).
func reliableMkdirAll(dirPath string, mode os.FileMode) (err error) {
func reliableMkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) {
i := 0
for {
// Creates all the parent directories, with mode 0777 mkdir honors system umask.
if err = osMkdirAll(dirPath, mode); err != nil {
if err = osMkdirAll(dirPath, mode, baseDir); err != nil {
// Retry only for the first retryable error.
if osIsNotExist(err) && i == 0 {
i++
@@ -120,7 +120,7 @@ func reliableMkdirAll(dirPath string, mode os.FileMode) (err error) {
// and reliableRenameAll. This is to ensure that if there is a
// racy parent directory delete in between we can simply retry
// the operation.
func renameAll(srcFilePath, dstFilePath string) (err error) {
func renameAll(srcFilePath, dstFilePath, baseDir string) (err error) {
if srcFilePath == "" || dstFilePath == "" {
return errInvalidArgument
}
@@ -132,7 +132,7 @@ func renameAll(srcFilePath, dstFilePath string) (err error) {
return err
}
if err = reliableRename(srcFilePath, dstFilePath); err != nil {
if err = reliableRename(srcFilePath, dstFilePath, baseDir); err != nil {
switch {
case isSysErrNotDir(err) && !osIsNotExist(err):
// Windows can have both isSysErrNotDir(err) and osIsNotExist(err) returning
@@ -162,8 +162,8 @@ func renameAll(srcFilePath, dstFilePath string) (err error) {
// Reliably retries os.RenameAll if for some reason os.RenameAll returns
// syscall.ENOENT (parent does not exist).
func reliableRename(srcFilePath, dstFilePath string) (err error) {
if err = reliableMkdirAll(path.Dir(dstFilePath), 0o777); err != nil {
func reliableRename(srcFilePath, dstFilePath, baseDir string) (err error) {
if err = reliableMkdirAll(path.Dir(dstFilePath), 0o777, baseDir); err != nil {
return err
}