From b517c791e9f164e7b2c27d7bb0c31de4ae23c3c9 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 24 Feb 2021 00:14:16 -0800 Subject: [PATCH] [feat]: use DSYNC for xl.meta writes and NOATIME for reads (#11615) Instead of using O_SYNC, we are better off using O_DSYNC instead since we are only ever interested in data to be persisted to disk not the associated filesystem metadata. For reads we ask customers to turn off noatime, but instead we can proactively use O_NOATIME flag to avoid atime updates upon reads. --- cmd/bitrot_test.go | 19 ++-- cmd/format-erasure.go | 7 +- cmd/fs-v1-multipart.go | 10 +- cmd/fs-v1.go | 3 +- cmd/metacache-walk.go | 12 +-- cmd/os-readdir_other.go | 5 +- cmd/os-readdir_unix.go | 6 +- cmd/os-readdir_windows.go | 6 +- cmd/xl-storage.go | 100 ++++++++++--------- cmd/xl-storage_noatime_notsupported.go | 31 ++++++ cmd/xl-storage_noatime_supported.go | 31 ++++++ pkg/ioutil/read_file.go | 74 ++++++++++++++ pkg/ioutil/read_file_noatime_notsupported.go | 23 +++++ pkg/ioutil/read_file_noatime_supported.go | 25 +++++ 14 files changed, 278 insertions(+), 74 deletions(-) create mode 100644 cmd/xl-storage_noatime_notsupported.go create mode 100644 cmd/xl-storage_noatime_supported.go create mode 100644 pkg/ioutil/read_file.go create mode 100644 pkg/ioutil/read_file_noatime_notsupported.go create mode 100644 pkg/ioutil/read_file_noatime_supported.go diff --git a/cmd/bitrot_test.go b/cmd/bitrot_test.go index 17cfc3813..86d3a0b9b 100644 --- a/cmd/bitrot_test.go +++ b/cmd/bitrot_test.go @@ -20,7 +20,6 @@ import ( "context" "io" "io/ioutil" - "log" "os" "testing" ) @@ -28,7 +27,7 @@ import ( func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) { tmpDir, err := ioutil.TempDir("", "") if err != nil { - log.Fatal(err) + t.Fatal(err) } defer os.RemoveAll(tmpDir) @@ -46,35 +45,35 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) { _, err = writer.Write([]byte("aaaaaaaaaa")) if err != nil { - log.Fatal(err) + t.Fatal(err) } _, err = writer.Write([]byte("aaaaaaaaaa")) if err != nil { - log.Fatal(err) + t.Fatal(err) } _, err = writer.Write([]byte("aaaaaaaaaa")) if err != nil { - log.Fatal(err) + t.Fatal(err) } _, err = writer.Write([]byte("aaaaa")) if err != nil { - log.Fatal(err) + t.Fatal(err) } writer.(io.Closer).Close() reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10) b := make([]byte, 10) if _, err = reader.ReadAt(b, 0); err != nil { - log.Fatal(err) + t.Fatal(err) } if _, err = reader.ReadAt(b, 10); err != nil { - log.Fatal(err) + t.Fatal(err) } if _, err = reader.ReadAt(b, 20); err != nil { - log.Fatal(err) + t.Fatal(err) } if _, err = reader.ReadAt(b[:5], 30); err != nil { - log.Fatal(err) + t.Fatal(err) } } diff --git a/cmd/format-erasure.go b/cmd/format-erasure.go index 94a3ce295..8e28739f7 100644 --- a/cmd/format-erasure.go +++ b/cmd/format-erasure.go @@ -31,6 +31,7 @@ import ( "github.com/minio/minio/cmd/config/storageclass" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/color" + xioutil "github.com/minio/minio/pkg/ioutil" "github.com/minio/minio/pkg/sync/errgroup" sha256 "github.com/minio/sha256-simd" ) @@ -156,7 +157,7 @@ func newFormatErasureV3(numSets int, setLen int) *formatErasureV3 { // successfully the version only if the backend is Erasure. func formatGetBackendErasureVersion(formatPath string) (string, error) { meta := &formatMetaV1{} - b, err := ioutil.ReadFile(formatPath) + b, err := xioutil.ReadFile(formatPath) if err != nil { return "", err } @@ -218,7 +219,7 @@ func formatErasureMigrateV1ToV2(export, version string) error { formatPath := pathJoin(export, minioMetaBucket, formatConfigFile) formatV1 := &formatErasureV1{} - b, err := ioutil.ReadFile(formatPath) + b, err := xioutil.ReadFile(formatPath) if err != nil { return err } @@ -251,7 +252,7 @@ func formatErasureMigrateV2ToV3(export, version string) error { formatPath := pathJoin(export, minioMetaBucket, formatConfigFile) formatV2 := &formatErasureV2{} - b, err := ioutil.ReadFile(formatPath) + b, err := xioutil.ReadFile(formatPath) if err != nil { return err } diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index 9b8e7b146..e5b064879 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -31,7 +31,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/minio/minio/cmd/logger" - mioutil "github.com/minio/minio/pkg/ioutil" + xioutil "github.com/minio/minio/pkg/ioutil" "github.com/minio/minio/pkg/trie" ) @@ -114,7 +114,7 @@ func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploa } partPath := pathJoin(uploadIDDir, entry) - err = mioutil.AppendFile(file.filePath, partPath, globalFSOSync) + err = xioutil.AppendFile(file.filePath, partPath, globalFSOSync) if err != nil { reqInfo := logger.GetReqInfo(ctx).AppendTags("partPath", partPath) reqInfo.AppendTags("filepath", file.filePath) @@ -390,7 +390,7 @@ func (fs *FSObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploa return minfo, toObjectErr(err, bucket, object) } - fsMetaBytes, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile)) + fsMetaBytes, err := xioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile)) if err != nil { logger.LogIf(ctx, err) return minfo, toObjectErr(err, bucket, object) @@ -700,7 +700,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, GotETag: part.ETag, } } - if err = mioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partFile), globalFSOSync); err != nil { + if err = xioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partFile), globalFSOSync); err != nil { logger.LogIf(ctx, err) return oi, toObjectErr(err) } @@ -744,7 +744,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, }() // Read saved fs metadata for ongoing multipart. - fsMetaBuf, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile)) + fsMetaBuf, err := xioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile)) if err != nil { logger.LogIf(ctx, err) return oi, toObjectErr(err, bucket, object) diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 93427c717..37112130d 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -40,6 +40,7 @@ import ( "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/bucket/policy" "github.com/minio/minio/pkg/color" + xioutil "github.com/minio/minio/pkg/ioutil" "github.com/minio/minio/pkg/lock" "github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/mimedb" @@ -323,7 +324,7 @@ func (fs *FSObjects) crawlBucket(ctx context.Context, bucket string, cache dataU // Load bucket info. cache, err = crawlDataFolder(ctx, fs.fsPath, cache, func(item crawlItem) (sizeSummary, error) { bucket, object := item.bucket, item.objectPath() - fsMetaBytes, err := ioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)) + fsMetaBytes, err := xioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)) if err != nil && !osIsNotExist(err) { if intDataUpdateTracker.debug { logger.Info(color.Green("crawlBucket:")+" object return unexpected error: %v/%v: %w", item.bucket, item.objectPath(), err) diff --git a/cmd/metacache-walk.go b/cmd/metacache-walk.go index 990b63581..f5f8ddd31 100644 --- a/cmd/metacache-walk.go +++ b/cmd/metacache-walk.go @@ -19,7 +19,6 @@ package cmd import ( "context" "io" - "io/ioutil" "net/http" "net/url" "os" @@ -30,6 +29,7 @@ import ( "github.com/gorilla/mux" "github.com/minio/minio/cmd/logger" + xioutil "github.com/minio/minio/pkg/ioutil" ) // WalkDirOptions provides options for WalkDir operations. @@ -91,7 +91,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // Fast exit track to check if we are listing an object with // a trailing slash, this will avoid to list the object content. if HasSuffix(opts.BaseDir, SlashSeparator) { - metadata, err := ioutil.ReadFile(pathJoin(volumeDir, + metadata, err := xioutil.ReadFile(pathJoin(volumeDir, opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix, xlStorageFormatFile)) if err == nil { @@ -151,7 +151,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // If root was an object return it as such. if HasSuffix(entry, xlStorageFormatFile) { var meta metaCacheEntry - meta.metadata, err = ioutil.ReadFile(pathJoin(volumeDir, current, entry)) + meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, current, entry)) if err != nil { logger.LogIf(ctx, err) continue @@ -166,7 +166,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // Check legacy. if HasSuffix(entry, xlStorageFormatFileV1) { var meta metaCacheEntry - meta.metadata, err = ioutil.ReadFile(pathJoin(volumeDir, current, entry)) + meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, current, entry)) if err != nil { logger.LogIf(ctx, err) continue @@ -213,7 +213,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash } - meta.metadata, err = ioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile)) + meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile)) switch { case err == nil: // It was an object @@ -222,7 +222,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ } out <- meta case osIsNotExist(err): - meta.metadata, err = ioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1)) + meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1)) if err == nil { // Maybe rename? Would make it inconsistent across disks though. // os.Rename(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1), pathJoin(volumeDir, meta.name, xlStorageFormatFile)) diff --git a/cmd/os-readdir_other.go b/cmd/os-readdir_other.go index 39a457d45..ae6db23b2 100644 --- a/cmd/os-readdir_other.go +++ b/cmd/os-readdir_other.go @@ -50,10 +50,11 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) if err == io.EOF { break } - if osErrToFileErr(err) == errFileNotFound { + err = osErrToFileErr(err) + if err == errFileNotFound { return nil } - return osErrToFileErr(err) + return err } for _, fi := range fis { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { diff --git a/cmd/os-readdir_unix.go b/cmd/os-readdir_unix.go index a328ce064..a94c74d73 100644 --- a/cmd/os-readdir_unix.go +++ b/cmd/os-readdir_unix.go @@ -109,6 +109,10 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro if isSysErrNotDir(err) { return nil } + err = osErrToFileErr(err) + if err == errFileNotFound { + return nil + } return err } if nbuf <= 0 { @@ -183,7 +187,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) { if isSysErrNotDir(err) { return nil, errFileNotFound } - return nil, err + return nil, osErrToFileErr(err) } if nbuf <= 0 { break diff --git a/cmd/os-readdir_windows.go b/cmd/os-readdir_windows.go index 99119d532..7199d6768 100644 --- a/cmd/os-readdir_windows.go +++ b/cmd/os-readdir_windows.go @@ -61,11 +61,15 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) if isSysErrPathNotFound(e) { return nil } - return osErrToFileErr(&os.PathError{ + err = osErrToFileErr(&os.PathError{ Op: "FindNextFile", Path: dirPath, Err: e, }) + if err == errFileNotFound { + return nil + } + return err } } name := syscall.UTF16ToString(data.FileName[0:]) diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 41f57cf49..11874da98 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -264,9 +264,10 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) { return &b }, }, - globalSync: env.Get(config.EnvFSOSync, config.EnableOff) == config.EnableOn, - ctx: GlobalContext, - rootDisk: rootDisk, + globalSync: env.Get(config.EnvFSOSync, config.EnableOff) == config.EnableOn, + ctx: GlobalContext, + rootDisk: rootDisk, + readODirectSupported: true, } // Create all necessary bucket folders if possible. @@ -382,7 +383,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac return sizeSummary{}, errSkipFile } - buf, err := ioutil.ReadFile(item.Path) + buf, err := xioutil.ReadFile(item.Path) if err != nil { if intDataUpdateTracker.debug { console.Debugf(color.Green("crawlBucket:")+" object path missing: %v: %w\n", item.Path, err) @@ -539,7 +540,7 @@ func (s *xlStorage) GetDiskID() (string, error) { return diskID, nil } - b, err := ioutil.ReadFile(formatFile) + b, err := xioutil.ReadFile(formatFile) if err != nil { // If the disk is still not initialized. if osIsNotExist(err) { @@ -827,7 +828,7 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st }, } } else { - xlMetaBuf, err := ioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, xlStorageFormatFile)) + xlMetaBuf, err := xioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, xlStorageFormatFile)) if err != nil { continue } @@ -1150,9 +1151,9 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str func (s *xlStorage) readAllData(volumeDir string, filePath string, requireDirectIO bool) (buf []byte, err error) { var f *os.File if requireDirectIO { - f, err = disk.OpenFileDirectIO(filePath, os.O_RDONLY, 0666) + f, err = disk.OpenFileDirectIO(filePath, readMode, 0666) } else { - f, err = os.Open(filePath) + f, err = os.OpenFile(filePath, readMode, 0) } if err != nil { if osIsNotExist(err) { @@ -1175,6 +1176,13 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string, requireDirect } else if isSysErrTooManyFiles(err) { return nil, errTooManyOpenFiles } else if isSysErrInvalidArg(err) { + st, _ := os.Lstat(filePath) + if st != nil && st.IsDir() { + // Linux returns InvalidArg for directory O_DIRECT + // we need to keep this fallback code to return correct + // errors upwards. + return nil, errFileNotFound + } return nil, errUnsupportedDisk } return nil, err @@ -1328,42 +1336,23 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err return nil, err } - // Stat a volume entry. - _, err = os.Lstat(volumeDir) - if err != nil { - if osIsNotExist(err) { - return nil, errVolumeNotFound - } else if isSysErrIO(err) { - return nil, errFaultyDisk - } - return nil, err - } - filePath := pathJoin(volumeDir, path) if err = checkPathLength(filePath); err != nil { return nil, err } - // Verify if the file already exists and is not of regular type. - var st os.FileInfo - if st, err = os.Lstat(filePath); err == nil { - if !st.Mode().IsRegular() { - return nil, errIsNotRegular - } - } else { - // Create top level directories if they don't exist. - // with mode 0777 mkdir honors system umask. - if err = mkdirAll(pathutil.Dir(filePath), 0777); err != nil { - return nil, err - } + // Create top level directories if they don't exist. + // with mode 0777 mkdir honors system umask. + if err = mkdirAll(pathutil.Dir(filePath), 0777); err != nil { + return nil, err } - w, err := os.OpenFile(filePath, mode, 0666) + w, err := os.OpenFile(filePath, mode|writeMode, 0666) if err != nil { // File path cannot be verified since one of the parents is a file. switch { - case isSysErrNotDir(err): - return nil, errFileAccessDenied + case isSysErrIsDir(err): + return nil, errIsNotRegular case osIsPermission(err): return nil, errFileAccessDenied case isSysErrIO(err): @@ -1668,19 +1657,21 @@ func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, b atomic.AddInt32(&s.activeIOCount, -1) }() - w, err := s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_WRONLY) + w, err := s.openFile(volume, path, os.O_CREATE|os.O_WRONLY) if err != nil { return err } - defer w.Close() + n, err := w.Write(b) if err != nil { return err } + if n != len(b) { return io.ErrShortWrite } + return nil } @@ -1692,19 +1683,38 @@ func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string, atomic.AddInt32(&s.activeIOCount, -1) }() - var w *os.File - // Create file if not found. Not doing O_DIRECT here to avoid the code that does buffer aligned writes. - // AppendFile() is only used by healing code to heal objects written in old format. - w, err = s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_APPEND|os.O_WRONLY) + volumeDir, err := s.getVolDir(volume) if err != nil { return err } - if _, err = w.Write(buf); err != nil { + // Stat a volume entry. + if _, err = os.Lstat(volumeDir); err != nil { + if osIsNotExist(err) { + return errVolumeNotFound + } return err } - return w.Close() + var w *os.File + // Create file if not found. Not doing O_DIRECT here to avoid the code that does buffer aligned writes. + // AppendFile() is only used by healing code to heal objects written in old format. + w, err = s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY) + if err != nil { + return err + } + defer w.Close() + + n, err := w.Write(buf) + if err != nil { + return err + } + + if n != len(buf) { + return io.ErrShortWrite + } + + return nil } // CheckParts check if path has necessary parts available. @@ -2000,7 +2010,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, return err } - srcBuf, err := ioutil.ReadFile(srcFilePath) + srcBuf, err := xioutil.ReadFile(srcFilePath) if err != nil { return osErrToFileErr(err) } @@ -2010,7 +2020,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, return err } - dstBuf, err := ioutil.ReadFile(dstFilePath) + dstBuf, err := xioutil.ReadFile(dstFilePath) if err != nil { if !osIsNotExist(err) { return osErrToFileErr(err) @@ -2021,7 +2031,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, return err } if err == nil { - dstBuf, err = ioutil.ReadFile(dstFilePath) + dstBuf, err = xioutil.ReadFile(dstFilePath) if err != nil && !osIsNotExist(err) { return osErrToFileErr(err) } diff --git a/cmd/xl-storage_noatime_notsupported.go b/cmd/xl-storage_noatime_notsupported.go new file mode 100644 index 000000000..ce590005d --- /dev/null +++ b/cmd/xl-storage_noatime_notsupported.go @@ -0,0 +1,31 @@ +// +build windows darwin + +/* + * MinIO Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "os" +) + +var ( + // No special option for reads on windows + readMode = os.O_RDONLY + + // Write with sync no buffering only used only for `xl.meta` writes + writeMode = os.O_SYNC +) diff --git a/cmd/xl-storage_noatime_supported.go b/cmd/xl-storage_noatime_supported.go new file mode 100644 index 000000000..b286b3b17 --- /dev/null +++ b/cmd/xl-storage_noatime_supported.go @@ -0,0 +1,31 @@ +// +build !windows,!darwin + +/* + * MinIO Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "os" +) + +var ( + // Disallow updating access times + readMode = os.O_RDONLY | 0x40000 // O_NOATIME + + // Write with data sync only used only for `xl.meta` writes + writeMode = 0x1000 // O_DSYNC +) diff --git a/pkg/ioutil/read_file.go b/pkg/ioutil/read_file.go new file mode 100644 index 000000000..50e67349c --- /dev/null +++ b/pkg/ioutil/read_file.go @@ -0,0 +1,74 @@ +/* + * MinIO Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Forked from golang.org/pkg/os.ReadFile with NOATIME support. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ioutil + +import ( + "io" + "os" +) + +// ReadFile reads the named file and returns the contents. +// A successful call returns err == nil, not err == EOF. +// Because ReadFile reads the whole file, it does not treat an EOF from Read +// as an error to be reported. +// +// passes NOATIME flag for reads on Unix systems to avoid atime updates. +func ReadFile(name string) ([]byte, error) { + f, err := os.OpenFile(name, readMode, 0) + if err != nil { + return nil, err + } + defer f.Close() + + var size int + if info, err := f.Stat(); err == nil { + size64 := info.Size() + if int64(int(size64)) == size64 { + size = int(size64) + } + } + size++ // one byte for final read at EOF + + // If a file claims a small size, read at least 512 bytes. + // In particular, files in Linux's /proc claim size 0 but + // then do not work right if read in small pieces, + // so an initial read of 1 byte would not work correctly. + if size < 512 { + size = 512 + } + + data := make([]byte, 0, size) + for { + if len(data) >= cap(data) { + d := append(data[:cap(data)], 0) + data = d[:len(data)] + } + n, err := f.Read(data[len(data):cap(data)]) + data = data[:len(data)+n] + if err != nil { + if err == io.EOF { + err = nil + } + return data, err + } + } +} diff --git a/pkg/ioutil/read_file_noatime_notsupported.go b/pkg/ioutil/read_file_noatime_notsupported.go new file mode 100644 index 000000000..29391f9e9 --- /dev/null +++ b/pkg/ioutil/read_file_noatime_notsupported.go @@ -0,0 +1,23 @@ +// +build windows darwin + +/* + * MinIO Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ioutil + +import "os" + +var readMode = os.O_RDONLY diff --git a/pkg/ioutil/read_file_noatime_supported.go b/pkg/ioutil/read_file_noatime_supported.go new file mode 100644 index 000000000..58819185d --- /dev/null +++ b/pkg/ioutil/read_file_noatime_supported.go @@ -0,0 +1,25 @@ +// +build !windows,!darwin + +/* + * MinIO Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ioutil + +import ( + "os" +) + +var readMode = os.O_RDONLY | 0x40000 // read with O_NOATIME