From 4e34e03dd443f0997a0bab4f3434567edc4f8f89 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 29 Apr 2016 14:24:10 -0700 Subject: [PATCH] xl/fs: Split object layer into interface. (#1415) --- access-key.go | 22 +- accesslog-handler.go | 20 +- api-router.go | 2 +- auth-handler.go | 9 +- bucket-handlers.go | 123 ++-- bucket-policy-handlers.go | 39 +- bucket-policy.go | 79 +-- certs.go | 20 +- config-migrate.go | 41 +- config-old.go | 37 +- config-v4.go | 36 +- config.go | 30 +- ...pi-multipart.go => fs-objects-multipart.go | 331 ++++++----- fs-objects.go | 333 +++++++++++ httprange.go | 28 +- logger-console-hook.go | 5 +- logger-file-hook.go | 9 +- logger-syslog-hook_nix.go | 15 +- logger-syslog-hook_windows.go | 4 +- logger.go | 27 +- logger_test.go | 3 +- main.go | 2 +- notifier.go | 5 +- object-api-getobjectinfo_test.go | 42 +- object-api-listobjects_test.go | 51 +- object-api-multipart_test.go | 120 ++-- object-api_test.go | 11 +- object-handlers.go | 118 ++-- object-interface.go | 43 ++ object-utils.go | 50 ++ object_api_suite_test.go | 44 +- pkg/quick/quick.go | 162 +++--- fs-dir-common.go => posix-dir-common.go | 0 fs-dir-nix.go => posix-dir-nix.go | 0 fs-dir-others.go => posix-dir-others.go | 0 fs-dirent-fileno.go => posix-dirent-fileno.go | 0 fs-dirent-ino.go => posix-dirent-ino.go | 0 fs-utils.go => posix-utils.go | 0 fs.go => posix.go | 2 +- routers.go | 32 +- network-fs.go => rpc-client.go | 4 +- ...pc-datatypes.go => rpc-server-datatypes.go | 0 storage-rpc-server.go => rpc-server.go | 11 +- server-main.go | 62 +- signature-jwt.go | 9 +- signature-v4-postpolicyform.go | 31 +- update-main.go | 53 +- utils.go | 10 +- web-handlers.go | 30 +- web-router.go | 2 +- xl-v1-common.go => xl-erasure-v1-common.go | 0 ...eatefile.go => xl-erasure-v1-createfile.go | 0 xl-v1-errors.go => xl-erasure-v1-errors.go | 0 ...1-healfile.go => xl-erasure-v1-healfile.go | 0 ...1-metadata.go => xl-erasure-v1-metadata.go | 0 ...1-readfile.go => xl-erasure-v1-readfile.go | 0 xl-v1-utils.go => xl-erasure-v1-utils.go | 0 ...itcloser.go => xl-erasure-v1-waitcloser.go | 0 xl-v1.go => xl-erasure-v1.go | 9 +- xl-objects-multipart.go | 531 ++++++++++++++++++ object-api.go => xl-objects.go | 205 +++---- 61 files changed, 1815 insertions(+), 1037 deletions(-) rename object-api-multipart.go => fs-objects-multipart.go (55%) create mode 100644 fs-objects.go create mode 100644 object-interface.go rename fs-dir-common.go => posix-dir-common.go (100%) rename fs-dir-nix.go => posix-dir-nix.go (100%) rename fs-dir-others.go => posix-dir-others.go (100%) rename fs-dirent-fileno.go => posix-dirent-fileno.go (100%) rename fs-dirent-ino.go => posix-dirent-ino.go (100%) rename fs-utils.go => posix-utils.go (100%) rename fs.go => posix.go (99%) rename network-fs.go => rpc-client.go (98%) rename storage-rpc-datatypes.go => rpc-server-datatypes.go (100%) rename storage-rpc-server.go => rpc-server.go (96%) rename xl-v1-common.go => xl-erasure-v1-common.go (100%) rename xl-v1-createfile.go => xl-erasure-v1-createfile.go (100%) rename xl-v1-errors.go => xl-erasure-v1-errors.go (100%) rename xl-v1-healfile.go => xl-erasure-v1-healfile.go (100%) rename xl-v1-metadata.go => xl-erasure-v1-metadata.go (100%) rename xl-v1-readfile.go => xl-erasure-v1-readfile.go (100%) rename xl-v1-utils.go => xl-erasure-v1-utils.go (100%) rename xl-v1-waitcloser.go => xl-erasure-v1-waitcloser.go (100%) rename xl-v1.go => xl-erasure-v1.go (98%) create mode 100644 xl-objects-multipart.go rename object-api.go => xl-objects.go (53%) diff --git a/access-key.go b/access-key.go index a4d2a1aea..0782836df 100644 --- a/access-key.go +++ b/access-key.go @@ -21,8 +21,6 @@ import ( "encoding/base64" "fmt" "regexp" - - "github.com/minio/minio/pkg/probe" ) // credential container for access and secret keys. @@ -52,19 +50,19 @@ var isValidAccessKey = regexp.MustCompile(`^[a-zA-Z0-9\\-\\.\\_\\~]{5,20}$`) // mustGenAccessKeys - must generate access credentials. func mustGenAccessKeys() (creds credential) { creds, err := genAccessKeys() - fatalIf(err.Trace(), "Unable to generate access keys.", nil) + fatalIf(err, "Unable to generate access keys.", nil) return creds } // genAccessKeys - generate access credentials. -func genAccessKeys() (credential, *probe.Error) { +func genAccessKeys() (credential, error) { accessKeyID, err := genAccessKeyID() if err != nil { - return credential{}, err.Trace() + return credential{}, err } secretAccessKey, err := genSecretAccessKey() if err != nil { - return credential{}, err.Trace() + return credential{}, err } creds := credential{ AccessKeyID: string(accessKeyID), @@ -75,10 +73,10 @@ func genAccessKeys() (credential, *probe.Error) { // genAccessKeyID - generate random alpha numeric value using only uppercase characters // takes input as size in integer -func genAccessKeyID() ([]byte, *probe.Error) { +func genAccessKeyID() ([]byte, error) { alpha := make([]byte, minioAccessID) - if _, e := rand.Read(alpha); e != nil { - return nil, probe.NewError(e) + if _, err := rand.Read(alpha); err != nil { + return nil, err } for i := 0; i < minioAccessID; i++ { alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))] @@ -87,10 +85,10 @@ func genAccessKeyID() ([]byte, *probe.Error) { } // genSecretAccessKey - generate random base64 numeric value from a random seed. -func genSecretAccessKey() ([]byte, *probe.Error) { +func genSecretAccessKey() ([]byte, error) { rb := make([]byte, minioSecretID) - if _, e := rand.Read(rb); e != nil { - return nil, probe.NewError(e) + if _, err := rand.Read(rb); err != nil { + return nil, err } return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil } diff --git a/accesslog-handler.go b/accesslog-handler.go index 7a0211009..36812f5c8 100644 --- a/accesslog-handler.go +++ b/accesslog-handler.go @@ -22,8 +22,6 @@ import ( "net/url" "os" "time" - - "github.com/minio/minio/pkg/probe" ) type accessLogHandler struct { @@ -60,14 +58,14 @@ type LogMessage struct { func (h *accessLogHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { message, err := getLogMessage(w, req) - fatalIf(err.Trace(), "Unable to extract http message.", nil) - _, e := h.accessLogFile.Write(message) - fatalIf(probe.NewError(e), "Writing to log file failed.", nil) + fatalIf(err, "Unable to extract http message.", nil) + _, err = h.accessLogFile.Write(message) + fatalIf(err, "Writing to log file failed.", nil) h.Handler.ServeHTTP(w, req) } -func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Error) { +func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, error) { logMessage := &LogMessage{ StartTime: time.Now().UTC(), } @@ -103,9 +101,9 @@ func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Err // logMessage.HTTP.Request = req logMessage.Duration = time.Now().UTC().Sub(logMessage.StartTime) - js, e := json.Marshal(logMessage) - if e != nil { - return nil, probe.NewError(e) + js, err := json.Marshal(logMessage) + if err != nil { + return nil, err } js = append(js, byte('\n')) // append a new line return js, nil @@ -113,8 +111,8 @@ func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Err // setAccessLogHandler logs requests func setAccessLogHandler(h http.Handler) http.Handler { - file, e := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - fatalIf(probe.NewError(e), "Unable to open access log.", nil) + file, err := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + fatalIf(err, "Unable to open access log.", nil) return &accessLogHandler{Handler: h, accessLogFile: file} } diff --git a/api-router.go b/api-router.go index 987cf2d46..ec19fd504 100644 --- a/api-router.go +++ b/api-router.go @@ -20,7 +20,7 @@ import router "github.com/gorilla/mux" // objectAPIHandler implements and provides http handlers for S3 API. type objectAPIHandlers struct { - ObjectAPI objectAPI + ObjectAPI ObjectLayer } // registerAPIRouter - registers S3 compatible APIs. diff --git a/auth-handler.go b/auth-handler.go index 3133615bc..cc602a26b 100644 --- a/auth-handler.go +++ b/auth-handler.go @@ -26,7 +26,6 @@ import ( "strings" fastSha256 "github.com/minio/minio/pkg/crypto/sha256" - "github.com/minio/minio/pkg/probe" ) // Verify if request has JWT. @@ -113,12 +112,12 @@ func sumMD5(data []byte) []byte { // Verify if request has valid AWS Signature Version '4'. func isReqAuthenticated(r *http.Request) (s3Error APIErrorCode) { if r == nil { - errorIf(probe.NewError(errInvalidArgument), "HTTP request cannot be empty.", nil) + errorIf(errInvalidArgument, "HTTP request cannot be empty.", nil) return ErrInternalError } - payload, e := ioutil.ReadAll(r.Body) - if e != nil { - errorIf(probe.NewError(e), "Unable to read HTTP body.", nil) + payload, err := ioutil.ReadAll(r.Body) + if err != nil { + errorIf(err, "Unable to read HTTP body.", nil) return ErrInternalError } // Verify Content-Md5, if payload is set. diff --git a/bucket-handlers.go b/bucket-handlers.go index d0a9fbbc5..48d8b535c 100644 --- a/bucket-handlers.go +++ b/bucket-handlers.go @@ -29,7 +29,6 @@ import ( "strings" mux "github.com/gorilla/mux" - "github.com/minio/minio/pkg/probe" ) // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html @@ -37,8 +36,8 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error // Read saved bucket policy. policy, err := readBucketPolicy(bucket) if err != nil { - errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "GetBucketPolicy failed.", nil) + switch err.(type) { case BucketNotFound: return ErrNoSuchBucket case BucketNameInvalid: @@ -49,9 +48,9 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error } } // Parse the saved policy. - bucketPolicy, e := parseBucketPolicy(policy) - if e != nil { - errorIf(probe.NewError(e), "Parse policy failed.", nil) + bucketPolicy, err := parseBucketPolicy(policy) + if err != nil { + errorIf(err, "Parse policy failed.", nil) return ErrAccessDenied } @@ -90,8 +89,8 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * return } case authTypeSigned, authTypePresigned: - payload, e := ioutil.ReadAll(r.Body) - if e != nil { + payload, err := ioutil.ReadAll(r.Body) + if err != nil { writeErrorResponse(w, r, ErrInternalError, r.URL.Path) return } @@ -117,10 +116,9 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * } } - _, err := api.ObjectAPI.GetBucketInfo(bucket) - if err != nil { - errorIf(err.Trace(), "GetBucketInfo failed.", nil) - switch err.ToGoError().(type) { + if _, err := api.ObjectAPI.GetBucketInfo(bucket); err != nil { + errorIf(err, "GetBucketInfo failed.", nil) + switch err.(type) { case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) case BucketNameInvalid: @@ -181,26 +179,24 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, } if keyMarker != "" { // Unescape keyMarker string - keyMarkerUnescaped, e := url.QueryUnescape(keyMarker) - if e != nil { - if e != nil { - // Return 'NoSuchKey' to indicate invalid marker key. - writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) - return - } - keyMarker = keyMarkerUnescaped - // Marker not common with prefix is not implemented. - if !strings.HasPrefix(keyMarker, prefix) { - writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path) - return - } + keyMarkerUnescaped, err := url.QueryUnescape(keyMarker) + if err != nil { + // Return 'NoSuchKey' to indicate invalid marker key. + writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) + return + } + keyMarker = keyMarkerUnescaped + // Marker not common with prefix is not implemented. + if !strings.HasPrefix(keyMarker, prefix) { + writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path) + return } } listMultipartsInfo, err := api.ObjectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) if err != nil { - errorIf(err.Trace(), "ListMultipartUploads failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "ListMultipartUploads failed.", nil) + switch err.(type) { case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) default: @@ -259,8 +255,8 @@ func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.R // If marker is set unescape. if marker != "" { // Try to unescape marker. - markerUnescaped, e := url.QueryUnescape(marker) - if e != nil { + markerUnescaped, err := url.QueryUnescape(marker) + if err != nil { // Return 'NoSuchKey' to indicate invalid marker key. writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) return @@ -284,7 +280,8 @@ func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.R writeSuccessResponse(w, encodedSuccessResponse) return } - switch err.ToGoError().(type) { + errorIf(err, "ListObjects failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketNotFound: @@ -292,7 +289,6 @@ func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.R case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) default: - errorIf(err.Trace(), "ListObjects failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } } @@ -347,8 +343,8 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R writeSuccessResponse(w, encodedSuccessResponse) return } - errorIf(err.Trace(), "ListBuckets failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "ListBuckets failed.", nil) + switch err.(type) { case StorageInsufficientReadResources: writeErrorResponse(w, r, ErrInsufficientReadResources, r.URL.Path) default: @@ -398,16 +394,16 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, deleteXMLBytes := make([]byte, r.ContentLength) // Read incoming body XML bytes. - _, e := io.ReadFull(r.Body, deleteXMLBytes) - if e != nil { - errorIf(probe.NewError(e), "DeleteMultipleObjects failed.", nil) + if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil { + errorIf(err, "DeleteMultipleObjects failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) return } // Unmarshal list of keys to be deleted. deleteObjects := &DeleteObjectsRequest{} - if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil { + if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil { + errorIf(err, "DeleteMultipartObjects xml decoding failed.", nil) writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path) return } @@ -422,8 +418,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, ObjectName: object.ObjectName, }) } else { - errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "DeleteObject failed.", nil) + switch err.(type) { case BucketNameInvalid: deleteErrors = append(deleteErrors, DeleteError{ Code: errorCodeResponse[ErrInvalidBucketName].Code, @@ -498,8 +494,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req // Make bucket. err := api.ObjectAPI.MakeBucket(bucket) if err != nil { - errorIf(err.Trace(), "MakeBucket failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "MakeBucket failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketExists: @@ -514,24 +510,25 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req writeSuccessResponse(w, nil) } -func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, *probe.Error) { +func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, error) { /// HTML Form values formValues := make(map[string]string) filePart := new(bytes.Buffer) - var e error - for e == nil { + var err error + for err == nil { var part *multipart.Part - part, e = reader.NextPart() + part, err = reader.NextPart() if part != nil { if part.FileName() == "" { - buffer, e := ioutil.ReadAll(part) - if e != nil { - return nil, nil, probe.NewError(e) + var buffer []byte + buffer, err = ioutil.ReadAll(part) + if err != nil { + return nil, nil, err } formValues[http.CanonicalHeaderKey(part.FormName())] = string(buffer) } else { - if _, e := io.Copy(filePart, part); e != nil { - return nil, nil, probe.NewError(e) + if _, err = io.Copy(filePart, part); err != nil { + return nil, nil, err } } } @@ -546,16 +543,16 @@ func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]stri func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { // Here the parameter is the size of the form data that should // be loaded in memory, the remaining being put in temporary files. - reader, e := r.MultipartReader() - if e != nil { - errorIf(probe.NewError(e), "Unable to initialize multipart reader.", nil) + reader, err := r.MultipartReader() + if err != nil { + errorIf(err, "Unable to initialize multipart reader.", nil) writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path) return } fileBody, formValues, err := extractHTTPFormValues(reader) if err != nil { - errorIf(err.Trace(), "Unable to parse form values.", nil) + errorIf(err, "Unable to parse form values.", nil) writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path) return } @@ -575,8 +572,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h } md5Sum, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil) if err != nil { - errorIf(err.Trace(), "PutObject failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "PutObject failed.", nil) + switch err.(type) { case StorageFull: writeErrorResponse(w, r, ErrStorageFull, r.URL.Path) case BucketNotFound: @@ -626,10 +623,9 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re } } - _, err := api.ObjectAPI.GetBucketInfo(bucket) - if err != nil { - errorIf(err.Trace(), "GetBucketInfo failed.", nil) - switch err.ToGoError().(type) { + if _, err := api.ObjectAPI.GetBucketInfo(bucket); err != nil { + errorIf(err, "GetBucketInfo failed.", nil) + switch err.(type) { case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) case BucketNameInvalid: @@ -661,10 +657,9 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http. } } - err := api.ObjectAPI.DeleteBucket(bucket) - if err != nil { - errorIf(err.Trace(), "DeleteBucket failed.", nil) - switch err.ToGoError().(type) { + if err := api.ObjectAPI.DeleteBucket(bucket); err != nil { + errorIf(err, "DeleteBucket failed.", nil) + switch err.(type) { case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) case BucketNotEmpty: diff --git a/bucket-policy-handlers.go b/bucket-policy-handlers.go index 35f674d81..cd9302f06 100644 --- a/bucket-policy-handlers.go +++ b/bucket-policy-handlers.go @@ -25,7 +25,6 @@ import ( "strings" mux "github.com/gorilla/mux" - "github.com/minio/minio/pkg/probe" ) // maximum supported access policy size. @@ -67,8 +66,8 @@ func bucketPolicyMatchStatement(action string, resource string, conditions map[s func bucketPolicyActionMatch(action string, statement policyStatement) bool { for _, policyAction := range statement.Actions { // Policy action can be a regex, validate the action with matching string. - matched, e := regexp.MatchString(policyAction, action) - fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil) + matched, err := regexp.MatchString(policyAction, action) + fatalIf(err, "Invalid pattern, please verify the pattern string.", nil) if matched { return true } @@ -79,8 +78,8 @@ func bucketPolicyActionMatch(action string, statement policyStatement) bool { // Verify if given resource matches with policy statement. func bucketPolicyResourceMatch(resource string, statement policyStatement) bool { for _, presource := range statement.Resources { - matched, e := regexp.MatchString(presource, strings.TrimPrefix(resource, "/")) - fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil) + matched, err := regexp.MatchString(presource, strings.TrimPrefix(resource, "/")) + fatalIf(err, "Invalid pattern, please verify the pattern string.", nil) // For any path matches, we return quickly and the let the caller continue. if matched { return true @@ -161,17 +160,17 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht // Read access policy up to maxAccessPolicySize. // http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html // bucket policies are limited to 20KB in size, using a limit reader. - bucketPolicyBuf, e := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize)) - if e != nil { - errorIf(probe.NewError(e).Trace(bucket), "Reading policy failed.", nil) + bucketPolicyBuf, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize)) + if err != nil { + errorIf(err, "Reading policy failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) return } // Parse bucket policy. - bucketPolicy, e := parseBucketPolicy(bucketPolicyBuf) - if e != nil { - errorIf(probe.NewError(e), "Unable to parse bucket policy.", nil) + bucketPolicy, err := parseBucketPolicy(bucketPolicyBuf) + if err != nil { + errorIf(err, "Unable to parse bucket policy.", nil) writeErrorResponse(w, r, ErrInvalidPolicyDocument, r.URL.Path) return } @@ -183,10 +182,9 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht } // Save bucket policy. - err := writeBucketPolicy(bucket, bucketPolicyBuf) - if err != nil { - errorIf(err.Trace(bucket, string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil) - switch err.ToGoError().(type) { + if err := writeBucketPolicy(bucket, bucketPolicyBuf); err != nil { + errorIf(err, "SaveBucketPolicy failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) default: @@ -218,10 +216,9 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r } // Delete bucket access policy. - err := removeBucketPolicy(bucket) - if err != nil { - errorIf(err.Trace(bucket), "DeleteBucketPolicy failed.", nil) - switch err.ToGoError().(type) { + if err := removeBucketPolicy(bucket); err != nil { + errorIf(err, "DeleteBucketPolicy failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketPolicyNotFound: @@ -257,8 +254,8 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht // Read bucket access policy. p, err := readBucketPolicy(bucket) if err != nil { - errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "GetBucketPolicy failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketPolicyNotFound: diff --git a/bucket-policy.go b/bucket-policy.go index 3791321a8..d9186878b 100644 --- a/bucket-policy.go +++ b/bucket-policy.go @@ -20,132 +20,115 @@ import ( "io/ioutil" "os" "path/filepath" - - "github.com/minio/minio/pkg/probe" ) // getBucketsConfigPath - get buckets path. -func getBucketsConfigPath() (string, *probe.Error) { +func getBucketsConfigPath() (string, error) { configPath, err := getConfigPath() if err != nil { - return "", err.Trace() + return "", err } return filepath.Join(configPath, "buckets"), nil } // createBucketsConfigPath - create buckets directory. -func createBucketsConfigPath() *probe.Error { +func createBucketsConfigPath() error { bucketsConfigPath, err := getBucketsConfigPath() if err != nil { return err } - if e := os.MkdirAll(bucketsConfigPath, 0700); e != nil { - return probe.NewError(e) - } - return nil + return os.MkdirAll(bucketsConfigPath, 0700) } // getBucketConfigPath - get bucket config path. -func getBucketConfigPath(bucket string) (string, *probe.Error) { +func getBucketConfigPath(bucket string) (string, error) { bucketsConfigPath, err := getBucketsConfigPath() if err != nil { - return "", err.Trace() + return "", err } return filepath.Join(bucketsConfigPath, bucket), nil } // createBucketConfigPath - create bucket config directory. -func createBucketConfigPath(bucket string) *probe.Error { +func createBucketConfigPath(bucket string) error { bucketConfigPath, err := getBucketConfigPath(bucket) if err != nil { return err } - if e := os.MkdirAll(bucketConfigPath, 0700); e != nil { - return probe.NewError(e) - } - return nil + return os.MkdirAll(bucketConfigPath, 0700) } // readBucketPolicy - read bucket policy. -func readBucketPolicy(bucket string) ([]byte, *probe.Error) { +func readBucketPolicy(bucket string) ([]byte, error) { // Verify bucket is valid. if !IsValidBucketName(bucket) { - return nil, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return nil, BucketNameInvalid{Bucket: bucket} } bucketConfigPath, err := getBucketConfigPath(bucket) if err != nil { - return nil, err.Trace() + return nil, err } // Get policy file. bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") - if _, e := os.Stat(bucketPolicyFile); e != nil { - if os.IsNotExist(e) { - return nil, probe.NewError(BucketPolicyNotFound{Bucket: bucket}) + if _, err = os.Stat(bucketPolicyFile); err != nil { + if os.IsNotExist(err) { + return nil, BucketPolicyNotFound{Bucket: bucket} } - return nil, probe.NewError(e) + return nil, err } - - accessPolicyBytes, e := ioutil.ReadFile(bucketPolicyFile) - if e != nil { - return nil, probe.NewError(e) - } - return accessPolicyBytes, nil + return ioutil.ReadFile(bucketPolicyFile) } // removeBucketPolicy - remove bucket policy. -func removeBucketPolicy(bucket string) *probe.Error { +func removeBucketPolicy(bucket string) error { // Verify bucket is valid. if !IsValidBucketName(bucket) { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) + return BucketNameInvalid{Bucket: bucket} } bucketConfigPath, err := getBucketConfigPath(bucket) if err != nil { - return err.Trace(bucket) + return err } // Get policy file. bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") - if _, e := os.Stat(bucketPolicyFile); e != nil { - if os.IsNotExist(e) { - return probe.NewError(BucketPolicyNotFound{Bucket: bucket}) + if _, err = os.Stat(bucketPolicyFile); err != nil { + if os.IsNotExist(err) { + return BucketPolicyNotFound{Bucket: bucket} } - return probe.NewError(e) + return err } return nil } // writeBucketPolicy - save bucket policy. -func writeBucketPolicy(bucket string, accessPolicyBytes []byte) *probe.Error { +func writeBucketPolicy(bucket string, accessPolicyBytes []byte) error { // Verify if bucket path legal if !IsValidBucketName(bucket) { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) + return BucketNameInvalid{Bucket: bucket} } // Create bucket config path. if err := createBucketConfigPath(bucket); err != nil { - return err.Trace() + return err } bucketConfigPath, err := getBucketConfigPath(bucket) if err != nil { - return err.Trace() + return err } // Get policy file. bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") - if _, e := os.Stat(bucketPolicyFile); e != nil { - if !os.IsNotExist(e) { - return probe.NewError(e) + if _, err := os.Stat(bucketPolicyFile); err != nil { + if !os.IsNotExist(err) { + return err } } // Write bucket policy. - if e := ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600); e != nil { - return probe.NewError(e) - } - - return nil + return ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600) } diff --git a/certs.go b/certs.go index 90fb5885b..9897dfd9d 100644 --- a/certs.go +++ b/certs.go @@ -21,26 +21,22 @@ import ( "path/filepath" "github.com/minio/go-homedir" - "github.com/minio/minio/pkg/probe" ) // createCertsPath create certs path. -func createCertsPath() *probe.Error { +func createCertsPath() error { certsPath, err := getCertsPath() if err != nil { - return err.Trace() + return err } - if err := os.MkdirAll(certsPath, 0700); err != nil { - return probe.NewError(err) - } - return nil + return os.MkdirAll(certsPath, 0700) } // getCertsPath get certs path. -func getCertsPath() (string, *probe.Error) { - homeDir, e := homedir.Dir() - if e != nil { - return "", probe.NewError(e) +func getCertsPath() (string, error) { + homeDir, err := homedir.Dir() + if err != nil { + return "", err } certsPath := filepath.Join(homeDir, globalMinioCertsDir) return certsPath, nil @@ -49,7 +45,7 @@ func getCertsPath() (string, *probe.Error) { // mustGetCertsPath must get certs path. func mustGetCertsPath() string { certsPath, err := getCertsPath() - fatalIf(err.Trace(), "Unable to retrieve certs path.", nil) + fatalIf(err, "Unable to retrieve certs path.", nil) return certsPath } diff --git a/config-migrate.go b/config-migrate.go index e198c0f17..b0b92f50e 100644 --- a/config-migrate.go +++ b/config-migrate.go @@ -22,7 +22,6 @@ import ( "path/filepath" "github.com/minio/mc/pkg/console" - "github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/quick" ) @@ -38,35 +37,31 @@ func migrateConfig() { // Version '1' is not supported anymore and deprecated, safe to delete. func purgeV1() { cv1, err := loadConfigV1() - if err != nil { - if os.IsNotExist(err.ToGoError()) { - return - } + if err != nil && os.IsNotExist(err) { + return } - fatalIf(err.Trace(), "Unable to load config version ‘1’.", nil) + fatalIf(err, "Unable to load config version ‘1’.", nil) if cv1.Version == "1" { console.Println("Unsupported config version ‘1’ found, removed successfully.") /// Purge old fsUsers.json file configPath, err := getConfigPath() - fatalIf(err.Trace(), "Unable to retrieve config path.", nil) + fatalIf(err, "Unable to retrieve config path.", nil) configFile := filepath.Join(configPath, "fsUsers.json") os.RemoveAll(configFile) } - fatalIf(probe.NewError(errors.New("")), "Unexpected version found ‘"+cv1.Version+"’, cannot migrate.", nil) + fatalIf(errors.New(""), "Unexpected version found ‘"+cv1.Version+"’, cannot migrate.", nil) } // Version '2' to '3' config migration adds new fields and re-orders // previous fields. Simplifies config for future additions. func migrateV2ToV3() { cv2, err := loadConfigV2() - if err != nil { - if os.IsNotExist(err.ToGoError()) { - return - } + if err != nil && os.IsNotExist(err) { + return } - fatalIf(err.Trace(), "Unable to load config version ‘2’.", nil) + fatalIf(err, "Unable to load config version ‘2’.", nil) if cv2.Version != "2" { return } @@ -99,14 +94,14 @@ func migrateV2ToV3() { srvConfig.Logger.Syslog = slogger qc, err := quick.New(srvConfig) - fatalIf(err.Trace(), "Unable to initialize config.", nil) + fatalIf(err, "Unable to initialize config.", nil) configFile, err := getConfigFile() - fatalIf(err.Trace(), "Unable to get config file.", nil) + fatalIf(err, "Unable to get config file.", nil) // Migrate the config. err = qc.Save(configFile) - fatalIf(err.Trace(), "Migrating from version ‘"+cv2.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil) + fatalIf(err, "Migrating from version ‘"+cv2.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil) console.Println("Migration from version ‘" + cv2.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.") } @@ -116,12 +111,10 @@ func migrateV2ToV3() { // the config for future additions. func migrateV3ToV4() { cv3, err := loadConfigV3() - if err != nil { - if os.IsNotExist(err.ToGoError()) { - return - } + if err != nil && os.IsNotExist(err) { + return } - fatalIf(err.Trace(), "Unable to load config version ‘3’.", nil) + fatalIf(err, "Unable to load config version ‘3’.", nil) if cv3.Version != "3" { return } @@ -136,12 +129,12 @@ func migrateV3ToV4() { srvConfig.Logger.Syslog = cv3.Logger.Syslog qc, err := quick.New(srvConfig) - fatalIf(err.Trace(), "Unable to initialize the quick config.", nil) + fatalIf(err, "Unable to initialize the quick config.", nil) configFile, err := getConfigFile() - fatalIf(err.Trace(), "Unable to get config file.", nil) + fatalIf(err, "Unable to get config file.", nil) err = qc.Save(configFile) - fatalIf(err.Trace(), "Migrating from version ‘"+cv3.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil) + fatalIf(err, "Migrating from version ‘"+cv3.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil) console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.") } diff --git a/config-old.go b/config-old.go index ca0157eb2..be618e98c 100644 --- a/config-old.go +++ b/config-old.go @@ -4,7 +4,6 @@ import ( "os" "path/filepath" - "github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/quick" ) @@ -16,23 +15,23 @@ type configV1 struct { } // loadConfigV1 load config -func loadConfigV1() (*configV1, *probe.Error) { +func loadConfigV1() (*configV1, error) { configPath, err := getConfigPath() if err != nil { - return nil, err.Trace() + return nil, err } configFile := filepath.Join(configPath, "fsUsers.json") - if _, err := os.Stat(configFile); err != nil { - return nil, probe.NewError(err) + if _, err = os.Stat(configFile); err != nil { + return nil, err } a := &configV1{} a.Version = "1" qc, err := quick.New(a) if err != nil { - return nil, err.Trace() + return nil, err } if err := qc.Load(configFile); err != nil { - return nil, err.Trace() + return nil, err } return qc.Data().(*configV1), nil } @@ -60,22 +59,22 @@ type configV2 struct { } // loadConfigV2 load config version '2'. -func loadConfigV2() (*configV2, *probe.Error) { +func loadConfigV2() (*configV2, error) { configFile, err := getConfigFile() if err != nil { - return nil, err.Trace() + return nil, err } - if _, err := os.Stat(configFile); err != nil { - return nil, probe.NewError(err) + if _, err = os.Stat(configFile); err != nil { + return nil, err } a := &configV2{} a.Version = "2" qc, err := quick.New(a) if err != nil { - return nil, err.Trace() + return nil, err } if err := qc.Load(configFile); err != nil { - return nil, err.Trace() + return nil, err } return qc.Data().(*configV2), nil } @@ -127,22 +126,22 @@ type configV3 struct { } // loadConfigV3 load config version '3'. -func loadConfigV3() (*configV3, *probe.Error) { +func loadConfigV3() (*configV3, error) { configFile, err := getConfigFile() if err != nil { - return nil, err.Trace() + return nil, err } - if _, err := os.Stat(configFile); err != nil { - return nil, probe.NewError(err) + if _, err = os.Stat(configFile); err != nil { + return nil, err } a := &configV3{} a.Version = "3" qc, err := quick.New(a) if err != nil { - return nil, err.Trace() + return nil, err } if err := qc.Load(configFile); err != nil { - return nil, err.Trace() + return nil, err } return qc.Data().(*configV3), nil } diff --git a/config-v4.go b/config-v4.go index 8a6254259..1985a1227 100644 --- a/config-v4.go +++ b/config-v4.go @@ -20,7 +20,6 @@ import ( "os" "sync" - "github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/quick" ) @@ -40,7 +39,7 @@ type serverConfigV4 struct { } // initConfig - initialize server config. config version (called only once). -func initConfig() *probe.Error { +func initConfig() error { if !isConfigFileExists() { srvCfg := &serverConfigV4{} srvCfg.Version = globalMinioConfigVersion @@ -55,41 +54,37 @@ func initConfig() *probe.Error { // Create config path. err := createConfigPath() if err != nil { - return err.Trace() + return err } // Create certs path. err = createCertsPath() if err != nil { - return err.Trace() + return err } // Save the new config globally. serverConfig = srvCfg // Save config into file. - err = serverConfig.Save() - if err != nil { - return err.Trace() - } - return nil + return serverConfig.Save() } configFile, err := getConfigFile() if err != nil { - return err.Trace() + return err } - if _, e := os.Stat(configFile); err != nil { - return probe.NewError(e) + if _, err = os.Stat(configFile); err != nil { + return err } srvCfg := &serverConfigV4{} srvCfg.Version = globalMinioConfigVersion srvCfg.rwMutex = &sync.RWMutex{} qc, err := quick.New(srvCfg) if err != nil { - return err.Trace() + return err } if err := qc.Load(configFile); err != nil { - return err.Trace() + return err } // Save the loaded config globally. serverConfig = qc.Data().(*serverConfigV4) @@ -181,27 +176,22 @@ func (s serverConfigV4) GetCredential() credential { } // Save config. -func (s serverConfigV4) Save() *probe.Error { +func (s serverConfigV4) Save() error { s.rwMutex.RLock() defer s.rwMutex.RUnlock() // get config file. configFile, err := getConfigFile() if err != nil { - return err.Trace() + return err } // initialize quick. qc, err := quick.New(&s) if err != nil { - return err.Trace() + return err } // Save config file. - if err := qc.Save(configFile); err != nil { - return err.Trace() - } - - // Return success. - return nil + return qc.Save(configFile) } diff --git a/config.go b/config.go index 83439cbdc..be7203d36 100644 --- a/config.go +++ b/config.go @@ -21,7 +21,6 @@ import ( "path/filepath" "github.com/minio/go-homedir" - "github.com/minio/minio/pkg/probe" ) // configPath for custom config path only for testing purposes @@ -33,13 +32,13 @@ func setGlobalConfigPath(configPath string) { } // getConfigPath get server config path -func getConfigPath() (string, *probe.Error) { +func getConfigPath() (string, error) { if customConfigPath != "" { return customConfigPath, nil } - homeDir, e := homedir.Dir() - if e != nil { - return "", probe.NewError(e) + homeDir, err := homedir.Dir() + if err != nil { + return "", err } configPath := filepath.Join(homeDir, globalMinioConfigDir) return configPath, nil @@ -48,27 +47,24 @@ func getConfigPath() (string, *probe.Error) { // mustGetConfigPath must get server config path. func mustGetConfigPath() string { configPath, err := getConfigPath() - fatalIf(err.Trace(), "Unable to get config path.", nil) + fatalIf(err, "Unable to get config path.", nil) return configPath } // createConfigPath create server config path. -func createConfigPath() *probe.Error { +func createConfigPath() error { configPath, err := getConfigPath() if err != nil { - return err.Trace() + return err } - if err := os.MkdirAll(configPath, 0700); err != nil { - return probe.NewError(err) - } - return nil + return os.MkdirAll(configPath, 0700) } // isConfigFileExists - returns true if config file exists. func isConfigFileExists() bool { - st, e := os.Stat(mustGetConfigFile()) + st, err := os.Stat(mustGetConfigFile()) // If file exists and is regular return true. - if e == nil && st.Mode().IsRegular() { + if err == nil && st.Mode().IsRegular() { return true } return false @@ -77,16 +73,16 @@ func isConfigFileExists() bool { // mustGetConfigFile must get server config file. func mustGetConfigFile() string { configFile, err := getConfigFile() - fatalIf(err.Trace(), "Unable to get config file.", nil) + fatalIf(err, "Unable to get config file.", nil) return configFile } // getConfigFile get server config file. -func getConfigFile() (string, *probe.Error) { +func getConfigFile() (string, error) { configPath, err := getConfigPath() if err != nil { - return "", err.Trace() + return "", err } return filepath.Join(configPath, globalMinioConfigFile), nil } diff --git a/object-api-multipart.go b/fs-objects-multipart.go similarity index 55% rename from object-api-multipart.go rename to fs-objects-multipart.go index 860c2fa74..a341cc4d5 100644 --- a/object-api-multipart.go +++ b/fs-objects-multipart.go @@ -27,28 +27,22 @@ import ( "strings" "github.com/Sirupsen/logrus" - "github.com/minio/minio/pkg/probe" "github.com/skyrings/skyring-common/tools/uuid" ) -const ( - // Minio meta volume. - minioMetaVolume = ".minio" -) - // listLeafEntries - lists all entries if a given prefixPath is a leaf // directory, returns error if any - returns empty list if prefixPath // is not a leaf directory. -func (o objectAPI) listLeafEntries(prefixPath string) (entries []FileInfo, e error) { +func (fs fsObjects) listLeafEntries(prefixPath string) (entries []FileInfo, e error) { var markerPath string for { - fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, markerPath, false, 1000) - if e != nil { + fileInfos, eof, err := fs.storage.ListFiles(minioMetaVolume, prefixPath, markerPath, false, 1000) + if err != nil { log.WithFields(logrus.Fields{ "prefixPath": prefixPath, "markerPath": markerPath, - }).Errorf("%s", e) - return nil, e + }).Errorf("%s", err) + return nil, err } for _, fileInfo := range fileInfos { // Set marker for next batch of ListFiles. @@ -72,7 +66,7 @@ func (o objectAPI) listLeafEntries(prefixPath string) (entries []FileInfo, e err } // listMetaVolumeFiles - list all files at a given prefix inside minioMetaVolume. -func (o objectAPI) listMetaVolumeFiles(prefixPath string, markerPath string, recursive bool, maxKeys int) (allFileInfos []FileInfo, eof bool, e error) { +func (fs fsObjects) listMetaVolumeFiles(prefixPath string, markerPath string, recursive bool, maxKeys int) (allFileInfos []FileInfo, eof bool, err error) { // newMaxKeys tracks the size of entries which are going to be // returned back. var newMaxKeys int @@ -82,15 +76,15 @@ func (o objectAPI) listMetaVolumeFiles(prefixPath string, markerPath string, rec for { var fileInfos []FileInfo // List files up to maxKeys-newMaxKeys, since we are skipping entries for special files. - fileInfos, eof, e = o.storage.ListFiles(minioMetaVolume, prefixPath, markerPath, recursive, maxKeys-newMaxKeys) - if e != nil { + fileInfos, eof, err = fs.storage.ListFiles(minioMetaVolume, prefixPath, markerPath, recursive, maxKeys-newMaxKeys) + if err != nil { log.WithFields(logrus.Fields{ "prefixPath": prefixPath, "markerPath": markerPath, "recursive": recursive, "maxKeys": maxKeys, - }).Errorf("%s", e) - return nil, true, e + }).Errorf("%s", err) + return nil, true, err } // Loop through and validate individual file. for _, fi := range fileInfos { @@ -99,20 +93,18 @@ func (o objectAPI) listMetaVolumeFiles(prefixPath string, markerPath string, rec // List all the entries if fi.Name is a leaf directory, if // fi.Name is not a leaf directory then the resulting // entries are empty. - entries, e = o.listLeafEntries(fi.Name) - if e != nil { + entries, err = fs.listLeafEntries(fi.Name) + if err != nil { log.WithFields(logrus.Fields{ "prefixPath": fi.Name, - }).Errorf("%s", e) - return nil, false, e + }).Errorf("%s", err) + return nil, false, err } } // Set markerPath for next batch of listing. markerPath = fi.Name if len(entries) > 0 { - // We reach here for non-recursive case and a leaf entry. - for _, entry := range entries { allFileInfos = append(allFileInfos, entry) newMaxKeys++ @@ -121,24 +113,22 @@ func (o objectAPI) listMetaVolumeFiles(prefixPath string, markerPath string, rec if newMaxKeys == maxKeys { // Return values: // allFileInfos : "maxKeys" number of entries. - // eof : eof returned by o.storage.ListFiles() + // eof : eof returned by fs.storage.ListFiles() // error : nil return } } - continue - } - - // We reach here for a non-recursive case non-leaf entry - // OR recursive case with fi.Name matching pattern bucket/object/uploadID[.partNum.md5sum] - - if !fi.Mode.IsDir() { // Do not skip non-recursive case directory entries. - // Skip files matching pattern bucket/object/uploadID.partNum.md5sum - // and retain files matching pattern bucket/object/uploadID - specialFile := path.Base(fi.Name) - if strings.Contains(specialFile, ".") { - // Contains partnumber and md5sum info, skip this. - continue + } else { + // We reach here for a non-recursive case non-leaf entry + // OR recursive case with fi.Name matching pattern bucket/object/uploadID[.partNum.md5sum] + if !fi.Mode.IsDir() { // Do not skip non-recursive case directory entries. + // Skip files matching pattern bucket/object/uploadID.partNum.md5sum + // and retain files matching pattern bucket/object/uploadID + specialFile := path.Base(fi.Name) + if strings.Contains(specialFile, ".") { + // Contains partnumber and md5sum info, skip this. + continue + } } } allFileInfos = append(allFileInfos, fi) @@ -148,7 +138,7 @@ func (o objectAPI) listMetaVolumeFiles(prefixPath string, markerPath string, rec if newMaxKeys == maxKeys { // Return values: // allFileInfos : "maxKeys" number of entries. - // eof : eof returned by o.storage.ListFiles() + // eof : eof returned by fs.storage.ListFiles() // error : nil return } @@ -164,41 +154,41 @@ func (o objectAPI) listMetaVolumeFiles(prefixPath string, markerPath string, rec } // ListMultipartUploads - list multipart uploads. -func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error) { +func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) { result := ListMultipartsInfo{} // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListMultipartsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return ListMultipartsInfo{}, (BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectPrefix(prefix) { - return ListMultipartsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) + return ListMultipartsInfo{}, (ObjectNameInvalid{Bucket: bucket, Object: prefix}) } // Verify if delimiter is anything other than '/', which we do not support. if delimiter != "" && delimiter != slashSeparator { - return ListMultipartsInfo{}, probe.NewError(UnsupportedDelimiter{ + return ListMultipartsInfo{}, (UnsupportedDelimiter{ Delimiter: delimiter, }) } // Verify if marker has prefix. if keyMarker != "" && !strings.HasPrefix(keyMarker, prefix) { - return ListMultipartsInfo{}, probe.NewError(InvalidMarkerPrefixCombination{ + return ListMultipartsInfo{}, (InvalidMarkerPrefixCombination{ Marker: keyMarker, Prefix: prefix, }) } if uploadIDMarker != "" { if strings.HasSuffix(keyMarker, slashSeparator) { - return result, probe.NewError(InvalidUploadIDKeyCombination{ + return result, (InvalidUploadIDKeyCombination{ UploadIDMarker: uploadIDMarker, KeyMarker: keyMarker, }) } - id, e := uuid.Parse(uploadIDMarker) - if e != nil { - return result, probe.NewError(e) + id, err := uuid.Parse(uploadIDMarker) + if err != nil { + return result, err } if id.IsZero() { - return result, probe.NewError(MalformedUploadID{ + return result, (MalformedUploadID{ UploadID: uploadIDMarker, }) } @@ -220,15 +210,15 @@ func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarke keyMarkerPath = pathJoin(pathJoin(bucket, keyMarker), uploadIDMarker) } // List all the multipart files at prefixPath, starting with marker keyMarkerPath. - fileInfos, eof, e := o.listMetaVolumeFiles(prefixPath, keyMarkerPath, recursive, maxUploads) - if e != nil { + fileInfos, eof, err := fs.listMetaVolumeFiles(prefixPath, keyMarkerPath, recursive, maxUploads) + if err != nil { log.WithFields(logrus.Fields{ "prefixPath": prefixPath, "markerPath": keyMarkerPath, "recursive": recursive, "maxUploads": maxUploads, - }).Errorf("listMetaVolumeFiles failed with %s", e) - return ListMultipartsInfo{}, probe.NewError(e) + }).Errorf("listMetaVolumeFiles failed with %s", err) + return ListMultipartsInfo{}, err } // Loop through all the received files fill in the multiparts result. @@ -260,55 +250,50 @@ func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarke return result, nil } -func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Error) { +func (fs fsObjects) NewMultipartUpload(bucket, object string) (string, error) { // Verify if bucket name is valid. if !IsValidBucketName(bucket) { - return "", probe.NewError(BucketNameInvalid{Bucket: bucket}) + return "", (BucketNameInvalid{Bucket: bucket}) } // Verify if object name is valid. if !IsValidObjectName(object) { - return "", probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return "", ObjectNameInvalid{Bucket: bucket, Object: object} } // Verify whether the bucket exists. - isExist, err := o.isBucketExist(bucket) - if err != nil { - return "", probe.NewError(err) - } - if !isExist { - return "", probe.NewError(BucketNotFound{Bucket: bucket}) + if isExist, err := isBucketExist(fs.storage, bucket); err != nil { + return "", err + } else if !isExist { + return "", BucketNotFound{Bucket: bucket} } - if _, e := o.storage.StatVol(minioMetaVolume); e != nil { - if e == errVolumeNotFound { - e = o.storage.MakeVol(minioMetaVolume) - if e != nil { - if e == errDiskFull { - return "", probe.NewError(StorageFull{}) - } - return "", probe.NewError(e) + if _, err := fs.storage.StatVol(minioMetaVolume); err != nil { + if err == errVolumeNotFound { + err = fs.storage.MakeVol(minioMetaVolume) + if err != nil { + return "", toObjectErr(err) } } } for { - uuid, e := uuid.New() - if e != nil { - return "", probe.NewError(e) + uuid, err := uuid.New() + if err != nil { + return "", err } uploadID := uuid.String() uploadIDPath := path.Join(bucket, object, uploadID) - if _, e = o.storage.StatFile(minioMetaVolume, uploadIDPath); e != nil { - if e != errFileNotFound { - return "", probe.NewError(toObjectErr(e, minioMetaVolume, uploadIDPath)) + if _, err = fs.storage.StatFile(minioMetaVolume, uploadIDPath); err != nil { + if err != errFileNotFound { + return "", (toObjectErr(err, minioMetaVolume, uploadIDPath)) } // uploadIDPath doesn't exist, so create empty file to reserve the name var w io.WriteCloser - if w, e = o.storage.CreateFile(minioMetaVolume, uploadIDPath); e == nil { + if w, err = fs.storage.CreateFile(minioMetaVolume, uploadIDPath); err == nil { // Close the writer. - if e = w.Close(); e != nil { - return "", probe.NewError(e) + if err = w.Close(); err != nil { + return "", err } } else { - return "", probe.NewError(toObjectErr(e, minioMetaVolume, uploadIDPath)) + return "", toObjectErr(err, minioMetaVolume, uploadIDPath) } return uploadID, nil } @@ -318,48 +303,46 @@ func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Err } // isUploadIDExists - verify if a given uploadID exists and is valid. -func (o objectAPI) isUploadIDExists(bucket, object, uploadID string) (bool, error) { +func isUploadIDExists(storage StorageAPI, bucket, object, uploadID string) (bool, error) { uploadIDPath := path.Join(bucket, object, uploadID) - st, e := o.storage.StatFile(minioMetaVolume, uploadIDPath) - if e != nil { + st, err := storage.StatFile(minioMetaVolume, uploadIDPath) + if err != nil { // Upload id does not exist. - if e == errFileNotFound { + if err == errFileNotFound { return false, nil } - return false, e + return false, err } // Upload id exists and is a regular file. return st.Mode.IsRegular(), nil } // PutObjectPart - writes the multipart upload chunks. -func (o objectAPI) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) { +func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", probe.NewError(BucketNameInvalid{Bucket: bucket}) + return "", BucketNameInvalid{Bucket: bucket} } if !IsValidObjectName(object) { - return "", probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return "", ObjectNameInvalid{Bucket: bucket, Object: object} } // Verify whether the bucket exists. - isExist, err := o.isBucketExist(bucket) - if err != nil { - return "", probe.NewError(err) - } - if !isExist { - return "", probe.NewError(BucketNotFound{Bucket: bucket}) + if isExist, err := isBucketExist(fs.storage, bucket); err != nil { + return "", err + } else if !isExist { + return "", BucketNotFound{Bucket: bucket} } - if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil { - return "", probe.NewError(e) + if status, err := isUploadIDExists(fs.storage, bucket, object, uploadID); err != nil { + return "", err } else if !status { - return "", probe.NewError(InvalidUploadID{UploadID: uploadID}) + return "", InvalidUploadID{UploadID: uploadID} } partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partID, md5Hex) - fileWriter, e := o.storage.CreateFile(minioMetaVolume, path.Join(bucket, object, partSuffix)) - if e != nil { - return "", probe.NewError(toObjectErr(e, bucket, object)) + fileWriter, err := fs.storage.CreateFile(minioMetaVolume, path.Join(bucket, object, partSuffix)) + if err != nil { + return "", toObjectErr(err, bucket, object) } // Initialize md5 writer. @@ -370,21 +353,21 @@ func (o objectAPI) PutObjectPart(bucket, object, uploadID string, partID int, si // Instantiate checksum hashers and create a multiwriter. if size > 0 { - if _, e = io.CopyN(multiWriter, data, size); e != nil { + if _, err = io.CopyN(multiWriter, data, size); err != nil { safeCloseAndRemove(fileWriter) - return "", probe.NewError(toObjectErr(e)) + return "", (toObjectErr(err)) } // Reader shouldn't have more data what mentioned in size argument. // reading one more byte from the reader to validate it. // expected to fail, success validates existence of more data in the reader. - if _, e = io.CopyN(ioutil.Discard, data, 1); e == nil { + if _, err = io.CopyN(ioutil.Discard, data, 1); err == nil { safeCloseAndRemove(fileWriter) - return "", probe.NewError(UnExpectedDataSize{Size: int(size)}) + return "", (UnExpectedDataSize{Size: int(size)}) } } else { - if _, e = io.Copy(multiWriter, data); e != nil { + if _, err = io.Copy(multiWriter, data); err != nil { safeCloseAndRemove(fileWriter) - return "", probe.NewError(toObjectErr(e)) + return "", (toObjectErr(err)) } } @@ -392,28 +375,28 @@ func (o objectAPI) PutObjectPart(bucket, object, uploadID string, partID int, si if md5Hex != "" { if newMD5Hex != md5Hex { safeCloseAndRemove(fileWriter) - return "", probe.NewError(BadDigest{md5Hex, newMD5Hex}) + return "", (BadDigest{md5Hex, newMD5Hex}) } } - e = fileWriter.Close() - if e != nil { - return "", probe.NewError(e) + err = fileWriter.Close() + if err != nil { + return "", err } return newMD5Hex, nil } -func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error) { +func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListPartsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return ListPartsInfo{}, (BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return ListPartsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return ListPartsInfo{}, (ObjectNameInvalid{Bucket: bucket, Object: object}) } - if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil { - return ListPartsInfo{}, probe.NewError(e) + if status, err := isUploadIDExists(fs.storage, bucket, object, uploadID); err != nil { + return ListPartsInfo{}, err } else if !status { - return ListPartsInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID}) + return ListPartsInfo{}, (InvalidUploadID{UploadID: uploadID}) } result := ListPartsInfo{} var markerPath string @@ -423,26 +406,26 @@ func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMa // partNumberMarker is already set. if partNumberMarker > 0 { partNumberMarkerPath := uploadIDPath + "." + strconv.Itoa(partNumberMarker) + "." - fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, partNumberMarkerPath, "", false, 1) - if e != nil { - return result, probe.NewError(toObjectErr(e, minioMetaVolume, partNumberMarkerPath)) + fileInfos, _, err := fs.storage.ListFiles(minioMetaVolume, partNumberMarkerPath, "", false, 1) + if err != nil { + return result, toObjectErr(err, minioMetaVolume, partNumberMarkerPath) } if len(fileInfos) == 0 { - return result, probe.NewError(InvalidPart{}) + return result, (InvalidPart{}) } markerPath = fileInfos[0].Name } uploadIDPrefix := uploadIDPath + "." - fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDPrefix, markerPath, false, maxParts) - if e != nil { - return result, probe.NewError(InvalidPart{}) + fileInfos, eof, err := fs.storage.ListFiles(minioMetaVolume, uploadIDPrefix, markerPath, false, maxParts) + if err != nil { + return result, InvalidPart{} } for _, fileInfo := range fileInfos { fileName := path.Base(fileInfo.Name) splitResult := strings.Split(fileName, ".") - partNum, e := strconv.Atoi(splitResult[1]) - if e != nil { - return result, probe.NewError(e) + partNum, err := strconv.Atoi(splitResult[1]) + if err != nil { + return result, err } md5sum := splitResult[2] result.Parts = append(result.Parts, partInfo{ @@ -463,86 +446,90 @@ func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMa return result, nil } -// Create an s3 compatible MD5sum for complete multipart transaction. -func makeS3MD5(md5Strs ...string) (string, *probe.Error) { - var finalMD5Bytes []byte - for _, md5Str := range md5Strs { - md5Bytes, e := hex.DecodeString(md5Str) - if e != nil { - return "", probe.NewError(e) - } - finalMD5Bytes = append(finalMD5Bytes, md5Bytes...) - } - md5Hasher := md5.New() - md5Hasher.Write(finalMD5Bytes) - s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(md5Strs)) - return s3MD5, nil -} - -func (o objectAPI) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, *probe.Error) { +func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", probe.NewError(BucketNameInvalid{Bucket: bucket}) + return "", (BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return "", probe.NewError(ObjectNameInvalid{ + return "", (ObjectNameInvalid{ Bucket: bucket, Object: object, }) } - if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil { - return "", probe.NewError(e) + if status, err := isUploadIDExists(fs.storage, bucket, object, uploadID); err != nil { + return "", err } else if !status { - return "", probe.NewError(InvalidUploadID{UploadID: uploadID}) + return "", (InvalidUploadID{UploadID: uploadID}) + } + + fileWriter, err := fs.storage.CreateFile(bucket, object) + if err != nil { + return "", toObjectErr(err, bucket, object) } var md5Sums []string for _, part := range parts { // Construct part suffix. partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, part.PartNumber, part.ETag) - e := o.storage.RenameFile(minioMetaVolume, path.Join(bucket, object, partSuffix), bucket, path.Join(object, fmt.Sprint(part.PartNumber))) - if e != nil { - return "", probe.NewError(e) + var fileReader io.ReadCloser + fileReader, err = fs.storage.ReadFile(minioMetaVolume, path.Join(bucket, object, partSuffix), 0) + if err != nil { + if err == errFileNotFound { + return "", (InvalidPart{}) + } + return "", err + } + _, err = io.Copy(fileWriter, fileReader) + if err != nil { + return "", err + } + err = fileReader.Close() + if err != nil { + return "", err } md5Sums = append(md5Sums, part.ETag) } - fileWriter, e := o.storage.CreateFile(bucket, path.Join(object, "multipart.json")) - if e != nil { - return "", probe.NewError(e) + + err = fileWriter.Close() + if err != nil { + return "", err } - fileWriter.Close() + // Save the s3 md5. s3MD5, err := makeS3MD5(md5Sums...) if err != nil { - return "", err.Trace(md5Sums...) + return "", err } // Cleanup all the parts. - // o.removeMultipartUpload(bucket, object, uploadID) + fs.removeMultipartUpload(bucket, object, uploadID) // Return md5sum. return s3MD5, nil } -func (o objectAPI) removeMultipartUpload(bucket, object, uploadID string) *probe.Error { +func (fs fsObjects) removeMultipartUpload(bucket, object, uploadID string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) + return (BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return (ObjectNameInvalid{Bucket: bucket, Object: object}) } marker := "" for { uploadIDPath := path.Join(bucket, object, uploadID) - fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDPath, marker, false, 1000) - if e != nil { - - return probe.NewError(InvalidUploadID{UploadID: uploadID}) + fileInfos, eof, err := fs.storage.ListFiles(minioMetaVolume, uploadIDPath, marker, false, 1000) + if err != nil { + if err == errFileNotFound { + return (InvalidUploadID{UploadID: uploadID}) + } + return toObjectErr(err) } for _, fileInfo := range fileInfos { - o.storage.DeleteFile(minioMetaVolume, fileInfo.Name) + fs.storage.DeleteFile(minioMetaVolume, fileInfo.Name) marker = fileInfo.Name } if eof { @@ -552,22 +539,18 @@ func (o objectAPI) removeMultipartUpload(bucket, object, uploadID string) *probe return nil } -func (o objectAPI) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error { +func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) + return (BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return (ObjectNameInvalid{Bucket: bucket, Object: object}) } - if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil { - return probe.NewError(e) + if status, err := isUploadIDExists(fs.storage, bucket, object, uploadID); err != nil { + return err } else if !status { - return probe.NewError(InvalidUploadID{UploadID: uploadID}) + return (InvalidUploadID{UploadID: uploadID}) } - err := o.removeMultipartUpload(bucket, object, uploadID) - if err != nil { - return err.Trace(bucket, object, uploadID) - } - return nil + return fs.removeMultipartUpload(bucket, object, uploadID) } diff --git a/fs-objects.go b/fs-objects.go new file mode 100644 index 000000000..35171ebf7 --- /dev/null +++ b/fs-objects.go @@ -0,0 +1,333 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "crypto/md5" + "encoding/hex" + "io" + "path/filepath" + "sort" + "strings" + + "github.com/minio/minio/pkg/mimedb" +) + +// fsObjects - Implements fs object layer. +type fsObjects struct { + storage StorageAPI +} + +// newFSObjects - initialize new fs object layer. +func newFSObjects(exportPath string) (ObjectLayer, error) { + var storage StorageAPI + var err error + if !strings.ContainsRune(exportPath, ':') || filepath.VolumeName(exportPath) != "" { + // Initialize filesystem storage API. + storage, err = newPosix(exportPath) + if err != nil { + return nil, err + } + } else { + // Initialize rpc client storage API. + storage, err = newRPCClient(exportPath) + if err != nil { + return nil, err + } + } + return fsObjects{storage}, nil +} + +// checks whether bucket exists. +func isBucketExist(storage StorageAPI, bucketName string) (bool, error) { + // Check whether bucket exists. + if _, e := storage.StatVol(bucketName); e != nil { + if e == errVolumeNotFound { + return false, nil + } + return false, e + } + return true, nil +} + +/// Bucket operations + +// MakeBucket - make a bucket. +func (fs fsObjects) MakeBucket(bucket string) error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return BucketNameInvalid{Bucket: bucket} + } + if err := fs.storage.MakeVol(bucket); err != nil { + return toObjectErr(err, bucket) + } + // This happens for the first time, but keep this here since this + // is the only place where it can be made expensive optimizing all + // other calls. + // Create minio meta volume, if it doesn't exist yet. + if err := fs.storage.MakeVol(minioMetaVolume); err != nil { + if err != errVolumeExists { + return toObjectErr(err, minioMetaVolume) + } + } + return nil +} + +// GetBucketInfo - get bucket info. +func (fs fsObjects) GetBucketInfo(bucket string) (BucketInfo, error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return BucketInfo{}, BucketNameInvalid{Bucket: bucket} + } + vi, err := fs.storage.StatVol(bucket) + if err != nil { + return BucketInfo{}, toObjectErr(err, bucket) + } + return BucketInfo{ + Name: bucket, + Created: vi.Created, + Total: vi.Total, + Free: vi.Free, + }, nil +} + +// ListBuckets - list buckets. +func (fs fsObjects) ListBuckets() ([]BucketInfo, error) { + var bucketInfos []BucketInfo + vols, err := fs.storage.ListVols() + if err != nil { + return nil, toObjectErr(err) + } + for _, vol := range vols { + // StorageAPI can send volume names which are incompatible + // with buckets, handle it and skip them. + if !IsValidBucketName(vol.Name) { + continue + } + bucketInfos = append(bucketInfos, BucketInfo{ + Name: vol.Name, + Created: vol.Created, + Total: vol.Total, + Free: vol.Free, + }) + } + sort.Sort(byBucketName(bucketInfos)) + return bucketInfos, nil +} + +// DeleteBucket - delete a bucket. +func (fs fsObjects) DeleteBucket(bucket string) error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return BucketNameInvalid{Bucket: bucket} + } + if err := fs.storage.DeleteVol(bucket); err != nil { + return toObjectErr(err) + } + return nil +} + +/// Object Operations + +// GetObject - get an object. +func (fs fsObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return nil, (BucketNameInvalid{Bucket: bucket}) + } + // Verify if object is valid. + if !IsValidObjectName(object) { + return nil, (ObjectNameInvalid{Bucket: bucket, Object: object}) + } + fileReader, err := fs.storage.ReadFile(bucket, object, startOffset) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + return fileReader, nil +} + +// GetObjectInfo - get object info. +func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ObjectInfo{}, (BucketNameInvalid{Bucket: bucket}) + } + // Verify if object is valid. + if !IsValidObjectName(object) { + return ObjectInfo{}, (ObjectNameInvalid{Bucket: bucket, Object: object}) + } + fi, err := fs.storage.StatFile(bucket, object) + if err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + contentType := "application/octet-stream" + if objectExt := filepath.Ext(object); objectExt != "" { + content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))] + if ok { + contentType = content.ContentType + } + } + return ObjectInfo{ + Bucket: bucket, + Name: object, + ModTime: fi.ModTime, + Size: fi.Size, + IsDir: fi.Mode.IsDir(), + ContentType: contentType, + MD5Sum: "", // Read from metadata. + }, nil +} + +func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return "", (BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return "", (ObjectNameInvalid{ + Bucket: bucket, + Object: object, + }) + } + // Check whether the bucket exists. + if isExist, err := isBucketExist(fs.storage, bucket); err != nil { + return "", err + } else if !isExist { + return "", BucketNotFound{Bucket: bucket} + } + + fileWriter, err := fs.storage.CreateFile(bucket, object) + if err != nil { + return "", toObjectErr(err, bucket, object) + } + + // Initialize md5 writer. + md5Writer := md5.New() + + // Instantiate a new multi writer. + multiWriter := io.MultiWriter(md5Writer, fileWriter) + + // Instantiate checksum hashers and create a multiwriter. + if size > 0 { + if _, err = io.CopyN(multiWriter, data, size); err != nil { + if clErr := safeCloseAndRemove(fileWriter); clErr != nil { + return "", clErr + } + return "", toObjectErr(err) + } + } else { + if _, err = io.Copy(multiWriter, data); err != nil { + if clErr := safeCloseAndRemove(fileWriter); clErr != nil { + return "", clErr + } + return "", err + } + } + + newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + // md5Hex representation. + var md5Hex string + if len(metadata) != 0 { + md5Hex = metadata["md5Sum"] + } + if md5Hex != "" { + if newMD5Hex != md5Hex { + if err = safeCloseAndRemove(fileWriter); err != nil { + return "", err + } + return "", BadDigest{md5Hex, newMD5Hex} + } + } + err = fileWriter.Close() + if err != nil { + return "", err + } + + // Return md5sum, successfully wrote object. + return newMD5Hex, nil +} + +func (fs fsObjects) DeleteObject(bucket, object string) error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return BucketNameInvalid{Bucket: bucket} + } + if !IsValidObjectName(object) { + return ObjectNameInvalid{Bucket: bucket, Object: object} + } + if err := fs.storage.DeleteFile(bucket, object); err != nil { + return toObjectErr(err, bucket, object) + } + return nil +} + +func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket} + } + if !IsValidObjectPrefix(prefix) { + return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix} + } + // Verify if delimiter is anything other than '/', which we do not support. + if delimiter != "" && delimiter != slashSeparator { + return ListObjectsInfo{}, UnsupportedDelimiter{ + Delimiter: delimiter, + } + } + // Verify if marker has prefix. + if marker != "" { + if !strings.HasPrefix(marker, prefix) { + return ListObjectsInfo{}, InvalidMarkerPrefixCombination{ + Marker: marker, + Prefix: prefix, + } + } + } + + // Default is recursive, if delimiter is set then list non recursive. + recursive := true + if delimiter == slashSeparator { + recursive = false + } + fileInfos, eof, err := fs.storage.ListFiles(bucket, prefix, marker, recursive, maxKeys) + if err != nil { + return ListObjectsInfo{}, toObjectErr(err, bucket) + } + if maxKeys == 0 { + return ListObjectsInfo{}, nil + } + + result := ListObjectsInfo{IsTruncated: !eof} + for _, fileInfo := range fileInfos { + // With delimiter set we fill in NextMarker and Prefixes. + if delimiter == slashSeparator { + result.NextMarker = fileInfo.Name + if fileInfo.Mode.IsDir() { + result.Prefixes = append(result.Prefixes, fileInfo.Name) + continue + } + } + result.Objects = append(result.Objects, ObjectInfo{ + Name: fileInfo.Name, + ModTime: fileInfo.ModTime, + Size: fileInfo.Size, + IsDir: false, + }) + } + return result, nil +} diff --git a/httprange.go b/httprange.go index cc32a946f..b00a37771 100644 --- a/httprange.go +++ b/httprange.go @@ -21,8 +21,6 @@ import ( "fmt" "strconv" "strings" - - "github.com/minio/minio/pkg/probe" ) const ( @@ -50,7 +48,7 @@ func (r *httpRange) String() string { } // Grab new range from request header -func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) { +func getRequestedRange(hrange string, size int64) (*httpRange, error) { r := &httpRange{ start: 0, length: 0, @@ -60,16 +58,16 @@ func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) { if hrange != "" { err := r.parseRange(hrange) if err != nil { - return nil, err.Trace() + return nil, err } } return r, nil } -func (r *httpRange) parse(ra string) *probe.Error { +func (r *httpRange) parse(ra string) error { i := strings.Index(ra, "-") if i < 0 { - return probe.NewError(InvalidRange{}) + return InvalidRange{} } start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:]) if start == "" { @@ -77,7 +75,7 @@ func (r *httpRange) parse(ra string) *probe.Error { // range start relative to the end of the file. i, err := strconv.ParseInt(end, 10, 64) if err != nil { - return probe.NewError(InvalidRange{}) + return InvalidRange{} } if i > r.size { i = r.size @@ -87,7 +85,7 @@ func (r *httpRange) parse(ra string) *probe.Error { } else { i, err := strconv.ParseInt(start, 10, 64) if err != nil || i > r.size || i < 0 { - return probe.NewError(InvalidRange{}) + return InvalidRange{} } r.start = i if end == "" { @@ -96,7 +94,7 @@ func (r *httpRange) parse(ra string) *probe.Error { } else { i, err := strconv.ParseInt(end, 10, 64) if err != nil || r.start > i { - return probe.NewError(InvalidRange{}) + return InvalidRange{} } if i >= r.size { i = r.size - 1 @@ -108,26 +106,26 @@ func (r *httpRange) parse(ra string) *probe.Error { } // parseRange parses a Range header string as per RFC 2616. -func (r *httpRange) parseRange(s string) *probe.Error { +func (r *httpRange) parseRange(s string) error { if s == "" { - return probe.NewError(errors.New("header not present")) + return errors.New("header not present") } if !strings.HasPrefix(s, b) { - return probe.NewError(InvalidRange{}) + return InvalidRange{} } ras := strings.Split(s[len(b):], ",") if len(ras) == 0 { - return probe.NewError(errors.New("invalid request")) + return errors.New("invalid request") } // Just pick the first one and ignore the rest, we only support one range per object if len(ras) > 1 { - return probe.NewError(errors.New("multiple ranges specified")) + return errors.New("multiple ranges specified") } ra := strings.TrimSpace(ras[0]) if ra == "" { - return probe.NewError(InvalidRange{}) + return InvalidRange{} } return r.parse(ra) } diff --git a/logger-console-hook.go b/logger-console-hook.go index f1455a54d..506bb23ea 100644 --- a/logger-console-hook.go +++ b/logger-console-hook.go @@ -20,7 +20,6 @@ import ( "io/ioutil" "github.com/Sirupsen/logrus" - "github.com/minio/minio/pkg/probe" ) // consoleLogger - default logger if not other logging is enabled. @@ -39,8 +38,8 @@ func enableConsoleLogger() { } // log.Out and log.Formatter use the default versions. // Only set specific log level. - lvl, e := logrus.ParseLevel(clogger.Level) - fatalIf(probe.NewError(e), "Unknown log level detected, please fix your console logger configuration.", nil) + lvl, err := logrus.ParseLevel(clogger.Level) + fatalIf(err, "Unknown log level detected, please fix your console logger configuration.", nil) log.Level = lvl } diff --git a/logger-file-hook.go b/logger-file-hook.go index aaedf74a6..ed84f2841 100644 --- a/logger-file-hook.go +++ b/logger-file-hook.go @@ -21,7 +21,6 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/minio/minio/pkg/probe" ) type fileLogger struct { @@ -40,14 +39,14 @@ func enableFileLogger() { return } - file, e := os.OpenFile(flogger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - fatalIf(probe.NewError(e), "Unable to open log file.", nil) + file, err := os.OpenFile(flogger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + fatalIf(err, "Unable to open log file.", nil) // Add a local file hook. log.Hooks.Add(&localFile{file}) - lvl, e := logrus.ParseLevel(flogger.Level) - fatalIf(probe.NewError(e), "Unknown log level detected, please fix your console logger configuration.", nil) + lvl, err := logrus.ParseLevel(flogger.Level) + fatalIf(err, "Unknown log level detected, please fix your console logger configuration.", nil) // Set default JSON formatter. log.Formatter = new(logrus.JSONFormatter) diff --git a/logger-syslog-hook_nix.go b/logger-syslog-hook_nix.go index 8423c73c9..aeac4eefc 100644 --- a/logger-syslog-hook_nix.go +++ b/logger-syslog-hook_nix.go @@ -23,7 +23,6 @@ import ( "log/syslog" "github.com/Sirupsen/logrus" - "github.com/minio/minio/pkg/probe" ) type syslogLogger struct { @@ -41,8 +40,8 @@ type syslogHook struct { // enableSyslogLogger - enable logger at raddr. func enableSyslogLogger(raddr string) { - syslogHook, e := newSyslog("udp", raddr, syslog.LOG_ERR, "MINIO") - fatalIf(probe.NewError(e), "Unable to instantiate syslog.", nil) + syslogHook, err := newSyslog("udp", raddr, syslog.LOG_ERR, "MINIO") + fatalIf(err, "Unable to instantiate syslog.", nil) log.Hooks.Add(syslogHook) // Add syslog hook. log.Formatter = &logrus.JSONFormatter{} // JSON formatted log. @@ -51,15 +50,15 @@ func enableSyslogLogger(raddr string) { // newSyslog - Creates a hook to be added to an instance of logger. func newSyslog(network, raddr string, priority syslog.Priority, tag string) (*syslogHook, error) { - w, e := syslog.Dial(network, raddr, priority, tag) - return &syslogHook{w, network, raddr}, e + w, err := syslog.Dial(network, raddr, priority, tag) + return &syslogHook{w, network, raddr}, err } // Fire - fire the log event func (hook *syslogHook) Fire(entry *logrus.Entry) error { - line, e := entry.String() - if e != nil { - return fmt.Errorf("Unable to read entry, %v", e) + line, err := entry.String() + if err != nil { + return fmt.Errorf("Unable to read entry, %v", err) } switch entry.Level { case logrus.PanicLevel: diff --git a/logger-syslog-hook_windows.go b/logger-syslog-hook_windows.go index 9013c1b80..3628c4248 100644 --- a/logger-syslog-hook_windows.go +++ b/logger-syslog-hook_windows.go @@ -18,8 +18,6 @@ package main -import "github.com/minio/minio/pkg/probe" - type syslogLogger struct { Enable bool `json:"enable"` Addr string `json:"address"` @@ -28,5 +26,5 @@ type syslogLogger struct { // enableSyslogLogger - unsupported on windows. func enableSyslogLogger(raddr string) { - fatalIf(probe.NewError(errSyslogNotSupported), "Unable to enable syslog.", nil) + fatalIf(errSyslogNotSupported, "Unable to enable syslog.", nil) } diff --git a/logger.go b/logger.go index 1cd2ab49d..4683e6b6c 100644 --- a/logger.go +++ b/logger.go @@ -17,11 +17,9 @@ package main import ( - "encoding/json" "reflect" "github.com/Sirupsen/logrus" - "github.com/minio/minio/pkg/probe" ) type fields map[string]interface{} @@ -43,7 +41,7 @@ type logger struct { } // errorIf synonymous with fatalIf but doesn't exit on error != nil -func errorIf(err *probe.Error, msg string, fields logrus.Fields) { +func errorIf(err error, msg string, fields logrus.Fields) { if err == nil { return } @@ -51,21 +49,17 @@ func errorIf(err *probe.Error, msg string, fields logrus.Fields) { fields = make(logrus.Fields) } fields["Error"] = struct { - Cause string `json:"cause,omitempty"` - Type string `json:"type,omitempty"` - CallTrace []probe.TracePoint `json:"trace,omitempty"` - SysInfo map[string]string `json:"sysinfo,omitempty"` + Cause string `json:"cause,omitempty"` + Type string `json:"type,omitempty"` }{ - err.Cause.Error(), - reflect.TypeOf(err.Cause).String(), - err.CallTrace, - err.SysInfo, + err.Error(), + reflect.TypeOf(err).String(), } log.WithFields(fields).Error(msg) } // fatalIf wrapper function which takes error and prints jsonic error messages. -func fatalIf(err *probe.Error, msg string, fields logrus.Fields) { +func fatalIf(err error, msg string, fields logrus.Fields) { if err == nil { return } @@ -73,9 +67,12 @@ func fatalIf(err *probe.Error, msg string, fields logrus.Fields) { fields = make(logrus.Fields) } - fields["error"] = err.ToGoError() - if jsonErr, e := json.Marshal(err); e == nil { - fields["probe"] = string(jsonErr) + fields["Error"] = struct { + Cause string `json:"cause,omitempty"` + Type string `json:"type,omitempty"` + }{ + err.Error(), + reflect.TypeOf(err).String(), } log.WithFields(fields).Fatal(msg) } diff --git a/logger_test.go b/logger_test.go index 09c654218..6c96fef19 100644 --- a/logger_test.go +++ b/logger_test.go @@ -22,7 +22,6 @@ import ( "errors" "github.com/Sirupsen/logrus" - "github.com/minio/minio/pkg/probe" . "gopkg.in/check.v1" ) @@ -37,7 +36,7 @@ func (s *LoggerSuite) TestLogger(c *C) { log.Out = &buffer log.Formatter = new(logrus.JSONFormatter) - errorIf(probe.NewError(errors.New("Fake error")), "Failed with error.", nil) + errorIf(errors.New("Fake error"), "Failed with error.", nil) err := json.Unmarshal(buffer.Bytes(), &fields) c.Assert(err, IsNil) c.Assert(fields["level"], Equals, "error") diff --git a/main.go b/main.go index e732e6c95..ebaa2f709 100644 --- a/main.go +++ b/main.go @@ -197,7 +197,7 @@ func main() { // Initialize config. err := initConfig() - fatalIf(err.Trace(), "Unable to initialize minio config.", nil) + fatalIf(err, "Unable to initialize minio config.", nil) // Enable all loggers by now. enableLoggers() diff --git a/notifier.go b/notifier.go index 91cf54e14..5943ec524 100644 --- a/notifier.go +++ b/notifier.go @@ -22,12 +22,11 @@ import ( "strings" "github.com/fatih/color" - "github.com/minio/minio/pkg/probe" "github.com/olekukonko/ts" ) // colorizeUpdateMessage - inspired from Yeoman project npm package https://github.com/yeoman/update-notifier -func colorizeUpdateMessage(updateString string) (string, *probe.Error) { +func colorizeUpdateMessage(updateString string) (string, error) { // Initialize coloring. cyan := color.New(color.FgCyan, color.Bold).SprintFunc() yellow := color.New(color.FgYellow, color.Bold).SprintfFunc() @@ -47,7 +46,7 @@ func colorizeUpdateMessage(updateString string) (string, *probe.Error) { terminal, err := ts.GetSize() if err != nil { - return "", probe.NewError(err) + return "", err } var message string diff --git a/object-api-getobjectinfo_test.go b/object-api-getobjectinfo_test.go index 82a473e89..cccf8ebf1 100644 --- a/object-api-getobjectinfo_test.go +++ b/object-api-getobjectinfo_test.go @@ -25,27 +25,22 @@ import ( "os" "strconv" "testing" - - "github.com/minio/minio/pkg/probe" ) // Testing GetObjectInfo(). func TestGetObjectInfo(t *testing.T) { - directory, e := ioutil.TempDir("", "minio-get-objinfo-test") - if e != nil { - t.Fatal(e) + directory, err := ioutil.TempDir("", "minio-get-objinfo-test") + if err != nil { + t.Fatal(err) } defer os.RemoveAll(directory) // Create the obj. - fs, e := newFS(directory) - if e != nil { - t.Fatal(e) + obj, err := newFSObjects(directory) + if err != nil { + t.Fatal(err) } - obj := newObjectLayer(fs) - var err *probe.Error - // This bucket is used for testing getObjectInfo operations. err = obj.MakeBucket("test-getobjectinfo") if err != nil { @@ -93,15 +88,15 @@ func TestGetObjectInfo(t *testing.T) { for i, testCase := range testCases { result, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName) if err != nil && testCase.shouldPass { - t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Cause.Error()) + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { - if testCase.err.Error() != err.Cause.Error() { - t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error()) + if testCase.err.Error() != err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } @@ -125,21 +120,18 @@ func TestGetObjectInfo(t *testing.T) { func BenchmarkGetObject(b *testing.B) { // Make a temporary directory to use as the obj. - directory, e := ioutil.TempDir("", "minio-benchmark-getobject") - if e != nil { - b.Fatal(e) + directory, err := ioutil.TempDir("", "minio-benchmark-getobject") + if err != nil { + b.Fatal(err) } defer os.RemoveAll(directory) // Create the obj. - fs, e := newFS(directory) - if e != nil { - b.Fatal(e) + obj, err := newFSObjects(directory) + if err != nil { + b.Fatal(err) } - obj := newObjectLayer(fs) - var err *probe.Error - // Make a bucket and put in a few objects. err = obj.MakeBucket("bucket") if err != nil { @@ -165,8 +157,8 @@ func BenchmarkGetObject(b *testing.B) { if err != nil { b.Error(err) } - if _, e := io.Copy(buffer, r); e != nil { - b.Error(e) + if _, err := io.Copy(buffer, r); err != nil { + b.Error(err) } if buffer.Len() != len(text) { b.Errorf("GetObject returned incorrect length %d (should be %d)\n", buffer.Len(), len(text)) diff --git a/object-api-listobjects_test.go b/object-api-listobjects_test.go index bb8079a16..90310f467 100644 --- a/object-api-listobjects_test.go +++ b/object-api-listobjects_test.go @@ -24,26 +24,22 @@ import ( "strconv" "strings" "testing" - - "github.com/minio/minio/pkg/probe" ) func TestListObjects(t *testing.T) { // Make a temporary directory to use as the obj. - directory, e := ioutil.TempDir("", "minio-list-object-test") - if e != nil { - t.Fatal(e) + directory, err := ioutil.TempDir("", "minio-list-object-test") + if err != nil { + t.Fatal(err) } defer os.RemoveAll(directory) // Create the obj. - fs, e := newFS(directory) - if e != nil { - t.Fatal(e) + obj, err := newFSObjects(directory) + if err != nil { + t.Fatal(err) } - obj := newObjectLayer(fs) - var err *probe.Error // This bucket is used for testing ListObject operations. err = obj.MakeBucket("test-bucket-list-object") if err != nil { @@ -56,25 +52,25 @@ func TestListObjects(t *testing.T) { t.Fatal(err) } - tmpfile, e := ioutil.TempFile("", "simple-file.txt") - if e != nil { - t.Fatal(e) + tmpfile, err := ioutil.TempFile("", "simple-file.txt") + if err != nil { + t.Fatal(err) } defer os.Remove(tmpfile.Name()) // clean up _, err = obj.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) if err != nil { - t.Fatal(e) + t.Fatal(err) } _, err = obj.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { - t.Fatal(e) + t.Fatal(err) } _, err = obj.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { - t.Fatal(e) + t.Fatal(err) } for i := 0; i < 2; i++ { @@ -86,7 +82,7 @@ func TestListObjects(t *testing.T) { } _, err = obj.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) if err != nil { - t.Fatal(e) + t.Fatal(err) } for i := 0; i < 3; i++ { @@ -536,15 +532,15 @@ func TestListObjects(t *testing.T) { for i, testCase := range testCases { result, err := obj.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys) if err != nil && testCase.shouldPass { - t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Cause.Error()) + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { - if !strings.Contains(err.Cause.Error(), testCase.err.Error()) { - t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error()) + if !strings.Contains(err.Error(), testCase.err.Error()) { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } // Since there are cases for which ListObjects fails, this is necessary. @@ -571,21 +567,18 @@ func TestListObjects(t *testing.T) { func BenchmarkListObjects(b *testing.B) { // Make a temporary directory to use as the obj. - directory, e := ioutil.TempDir("", "minio-list-benchmark") - if e != nil { - b.Fatal(e) + directory, err := ioutil.TempDir("", "minio-list-benchmark") + if err != nil { + b.Fatal(err) } defer os.RemoveAll(directory) // Create the obj. - fs, e := newFS(directory) - if e != nil { - b.Fatal(e) + obj, err := newFSObjects(directory) + if err != nil { + b.Fatal(err) } - obj := newObjectLayer(fs) - var err *probe.Error - // Create a bucket. err = obj.MakeBucket("ls-benchmark-bucket") if err != nil { diff --git a/object-api-multipart_test.go b/object-api-multipart_test.go index be0b6e3cd..708921cc0 100644 --- a/object-api-multipart_test.go +++ b/object-api-multipart_test.go @@ -21,27 +21,23 @@ import ( "fmt" "io/ioutil" "os" - "path" "testing" ) // Tests validate creation of new multipart upload instance. func TestObjectNewMultipartUpload(t *testing.T) { - directory, e := ioutil.TempDir("", "minio-multipart-1-test") - if e != nil { - t.Fatal(e) + directory, err := ioutil.TempDir("", "minio-multipart-1-test") + if err != nil { + t.Fatal(err) } defer os.RemoveAll(directory) - // Initialize fs layer. - fs, e := newFS(directory) - if e != nil { - t.Fatal(e) + // Initialize fs object layer. + obj, err := newFSObjects(directory) + if err != nil { + t.Fatal(err) } - // Initialize object layer. - obj := newObjectLayer(fs) - bucket := "minio-bucket" object := "minio-object" @@ -49,77 +45,68 @@ func TestObjectNewMultipartUpload(t *testing.T) { // opearation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist. uploadID, err := obj.NewMultipartUpload(bucket, object) if err == nil { - t.Fatalf("Expcected to fail since the NewMultipartUpload is intialized on a non-existant bucket.") + t.Fatalf("Expected to fail since the NewMultipartUpload is intialized on a non-existant bucket.") } - if errMsg != err.ToGoError().Error() { - t.Errorf("Expected to fail with Error \"%s\", but instead found \"%s\".", errMsg, err.ToGoError().Error()) + if errMsg != err.Error() { + t.Errorf("Expected to fail with Error \"%s\", but instead found \"%s\".", errMsg, err.Error()) } // Create bucket before intiating NewMultipartUpload. err = obj.MakeBucket(bucket) if err != nil { // failed to create newbucket, abort. - t.Fatal(err.ToGoError()) + t.Fatal(err) } uploadID, err = obj.NewMultipartUpload(bucket, object) if err != nil { - t.Fatal(err.ToGoError()) + t.Fatal(err) } - uploadIDPath := path.Join(bucket, object, uploadID) - _, e = obj.storage.StatFile(minioMetaVolume, uploadIDPath) - if e != nil { - if e == errFileNotFound { + err = obj.AbortMultipartUpload(bucket, object, uploadID) + if err != nil { + switch err.(type) { + case InvalidUploadID: t.Fatalf("New Multipart upload failed to create uuid file.") + default: + t.Fatalf(err.Error()) } - t.Fatalf(e.Error()) } } // Tests validates the validator for existence of uploadID. func TestObjectAPIIsUploadIDExists(t *testing.T) { - directory, e := ioutil.TempDir("", "minio-multipart-2-test") - if e != nil { - t.Fatal(e) + directory, err := ioutil.TempDir("", "minio-multipart-2-test") + if err != nil { + t.Fatal(err) } defer os.RemoveAll(directory) - // Initialize fs layer. - fs, e := newFS(directory) - if e != nil { - t.Fatal(e) + // Initialize fs object layer. + obj, err := newFSObjects(directory) + if err != nil { + t.Fatal(err) } - // Initialize object layer. - obj := newObjectLayer(fs) - bucket := "minio-bucket" object := "minio-object" // Create bucket before intiating NewMultipartUpload. - err := obj.MakeBucket(bucket) + err = obj.MakeBucket(bucket) if err != nil { // Failed to create newbucket, abort. - t.Fatal(err.ToGoError()) + t.Fatal(err) } - // UploadID file shouldn't exist. - isExists, e := obj.isUploadIDExists(bucket, object, "abc") - if e == nil && isExists { - t.Fatal("Expected uploadIDPath to not to exist.") - } - - uploadID, err := obj.NewMultipartUpload(bucket, object) + _, err = obj.NewMultipartUpload(bucket, object) if err != nil { - t.Fatal(err.ToGoError()) + t.Fatal(err) } - // UploadID file should exist. - isExists, e = obj.isUploadIDExists(bucket, object, uploadID) - if e != nil { - t.Fatal(e.Error()) - } - if !isExists { + + err = obj.AbortMultipartUpload(bucket, object, "abc") + switch err.(type) { + case InvalidUploadID: + default: t.Fatal("Expected uploadIDPath to exist.") } } @@ -127,40 +114,38 @@ func TestObjectAPIIsUploadIDExists(t *testing.T) { // Tests validate correctness of PutObjectPart. func TestObjectAPIPutObjectPart(t *testing.T) { // Generating cases for which the PutObjectPart fails. - directory, e := ioutil.TempDir("", "minio-multipart-3-test") - if e != nil { - t.Fatal(e) + directory, err := ioutil.TempDir("", "minio-multipart-3-test") + if err != nil { + t.Fatal(err) } defer os.RemoveAll(directory) - // Initializing fs layer. - fs, e := newFS(directory) - if e != nil { - t.Fatal(e) + // Initializing fs object layer. + obj, err := newFSObjects(directory) + if err != nil { + t.Fatal(err) } + bucket := "minio-bucket" object := "minio-object" - // Initializing object layer. - obj := newObjectLayer(fs) - // Create bucket before intiating NewMultipartUpload. - err := obj.MakeBucket(bucket) + err = obj.MakeBucket(bucket) if err != nil { // Failed to create newbucket, abort. - t.Fatal(err.ToGoError()) + t.Fatal(err) } // Initiate Multipart Upload on the above created bucket. uploadID, err := obj.NewMultipartUpload(bucket, object) if err != nil { // Failed to create NewMultipartUpload, abort. - t.Fatal(err.ToGoError()) + t.Fatal(err) } // Creating a dummy bucket for tests. err = obj.MakeBucket("unused-bucket") if err != nil { // Failed to create newbucket, abort. - t.Fatal(err.ToGoError()) + t.Fatal(err) } failCases := []struct { @@ -235,31 +220,24 @@ func TestObjectAPIPutObjectPart(t *testing.T) { // All are test cases above are expected to fail. if actualErr != nil && testCase.shouldPass { - t.Errorf("Test %d: Expected to pass, but failed with: %s.", i+1, actualErr.ToGoError().Error()) + t.Errorf("Test %d: Expected to pass, but failed with: %s.", i+1, actualErr.Error()) } if actualErr == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead.", i+1, testCase.expectedError.Error()) } // Failed as expected, but does it fail for the expected reason. if actualErr != nil && !testCase.shouldPass { - if testCase.expectedError.Error() != actualErr.ToGoError().Error() { + if testCase.expectedError.Error() != actualErr.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, - testCase.expectedError.Error(), actualErr.ToGoError().Error()) + testCase.expectedError.Error(), actualErr.Error()) } } - // Since there are cases for which ListObjects fails, this is necessary. // Test passes as expected, but the output values are verified for correctness here. if actualErr == nil && testCase.shouldPass { // Asserting whether the md5 output is correct. if testCase.inputMd5 != actualMd5Hex { t.Errorf("Test %d: Calculated Md5 different from the actual one %s.", i+1, actualMd5Hex) } - partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, testCase.PartID, testCase.inputMd5) - // Verifying whether the part file is created. - _, e := obj.storage.StatFile(minioMetaVolume, path.Join(bucket, object, partSuffix)) - if e != nil { - t.Errorf("Test %d: Failed to create the Part file.", i+1) - } } } } diff --git a/object-api_test.go b/object-api_test.go index 6f4a97756..32678529b 100644 --- a/object-api_test.go +++ b/object-api_test.go @@ -29,12 +29,11 @@ var _ = Suite(&MySuite{}) func (s *MySuite) TestFSAPISuite(c *C) { var storageList []string - create := func() objectAPI { + create := func() ObjectLayer { path, err := ioutil.TempDir(os.TempDir(), "minio-") c.Check(err, IsNil) - storageAPI, err := newStorageAPI(path) + objAPI, err := newFSObjects(path) c.Check(err, IsNil) - objAPI := newObjectLayer(storageAPI) storageList = append(storageList, path) return objAPI } @@ -48,7 +47,7 @@ func (s *MySuite) TestXLAPISuite(c *C) { // Initialize name space lock. initNSLock() - create := func() objectAPI { + create := func() ObjectLayer { var nDisks = 16 // Maximum disks. var erasureDisks []string for i := 0; i < nDisks; i++ { @@ -56,10 +55,8 @@ func (s *MySuite) TestXLAPISuite(c *C) { c.Check(err, IsNil) erasureDisks = append(erasureDisks, path) } - storageList = append(storageList, erasureDisks...) - storageAPI, err := newStorageAPI(erasureDisks...) + objAPI, err := newXLObjects(erasureDisks...) c.Check(err, IsNil) - objAPI := newObjectLayer(storageAPI) return objAPI } APITestSuite(c, create) diff --git a/object-handlers.go b/object-handlers.go index 4eff72d38..93d6f0b0e 100644 --- a/object-handlers.go +++ b/object-handlers.go @@ -32,7 +32,6 @@ import ( fastSha256 "github.com/minio/minio/pkg/crypto/sha256" mux "github.com/gorilla/mux" - "github.com/minio/minio/pkg/probe" ) // supportedGetReqParams - supported request parameters for GET presigned request. @@ -100,7 +99,8 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req // Fetch object stat info. objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) if err != nil { - switch err.ToGoError().(type) { + errorIf(err, "GetObjectInfo failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketNotFound: @@ -110,7 +110,6 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) default: - errorIf(err.Trace(), "GetObjectInfo failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } return @@ -137,13 +136,13 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req startOffset := hrange.start readCloser, err := api.ObjectAPI.GetObject(bucket, object, startOffset) if err != nil { - switch err.ToGoError().(type) { + switch err.(type) { case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) case ObjectNotFound: writeErrorResponse(w, r, errAllowableObjectNotFound(bucket, r), r.URL.Path) default: - errorIf(err.Trace(), "GetObject failed.", nil) + errorIf(err, "GetObject failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } return @@ -157,14 +156,14 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req setGetRespHeaders(w, r.URL.Query()) if hrange.length > 0 { - if _, e := io.CopyN(w, readCloser, hrange.length); e != nil { - errorIf(probe.NewError(e), "Writing to client failed", nil) + if _, err := io.CopyN(w, readCloser, hrange.length); err != nil { + errorIf(err, "Writing to client failed", nil) // Do not send error response here, since client could have died. return } } else { - if _, e := io.Copy(w, readCloser); e != nil { - errorIf(probe.NewError(e), "Writing to client failed", nil) + if _, err := io.Copy(w, readCloser); err != nil { + errorIf(err, "Writing to client failed", nil) // Do not send error response here, since client could have died. return } @@ -294,8 +293,8 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) if err != nil { - errorIf(err.Trace(bucket, object), "GetObjectInfo failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "GetObjectInfo failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketNotFound: @@ -387,8 +386,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re objInfo, err := api.ObjectAPI.GetObjectInfo(sourceBucket, sourceObject) if err != nil { - errorIf(err.Trace(), "GetObjectInfo failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "GetObjectInfo failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, objectSource) case BucketNotFound: @@ -425,10 +424,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re var md5Bytes []byte if objInfo.MD5Sum != "" { - var e error - md5Bytes, e = hex.DecodeString(objInfo.MD5Sum) - if e != nil { - errorIf(probe.NewError(e), "Decoding md5 failed.", nil) + md5Bytes, err = hex.DecodeString(objInfo.MD5Sum) + if err != nil { + errorIf(err, "Decoding md5 failed.", nil) writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) return } @@ -436,10 +434,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re startOffset := int64(0) // Read the whole file. // Get the object. - readCloser, getErr := api.ObjectAPI.GetObject(sourceBucket, sourceObject, startOffset) - if getErr != nil { - errorIf(getErr.Trace(sourceBucket, sourceObject), "Reading "+objectSource+" failed.", nil) - switch err.ToGoError().(type) { + readCloser, err := api.ObjectAPI.GetObject(sourceBucket, sourceObject, startOffset) + if err != nil { + errorIf(err, "Reading "+objectSource+" failed.", nil) + switch err.(type) { case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, objectSource) case ObjectNotFound: @@ -459,7 +457,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // Create the object. md5Sum, err := api.ObjectAPI.PutObject(bucket, object, size, readCloser, metadata) if err != nil { - switch err.ToGoError().(type) { + errorIf(err, "PutObject failed.", nil) + switch err.(type) { case StorageFull: writeErrorResponse(w, r, ErrStorageFull, r.URL.Path) case BucketNotFound: @@ -473,7 +472,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re case ObjectExistsAsPrefix: writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path) default: - errorIf(err.Trace(), "PutObject failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } return @@ -481,7 +479,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re objInfo, err = api.ObjectAPI.GetObjectInfo(bucket, object) if err != nil { - errorIf(err.Trace(), "GetObjectInfo failed.", nil) + errorIf(err, "GetObjectInfo failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) return } @@ -605,7 +603,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req // Get Content-Md5 sent by client and verify if valid md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5")) if err != nil { - errorIf(err.Trace(r.Header.Get("Content-Md5")), "Decoding md5 failed.", nil) + errorIf(err, "Decoding md5 failed.", nil) writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) return } @@ -643,9 +641,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req go func() { shaWriter := fastSha256.New() multiWriter := io.MultiWriter(shaWriter, writer) - if _, e := io.CopyN(multiWriter, r.Body, size); e != nil { - errorIf(probe.NewError(e), "Unable to read HTTP body.", nil) - writer.CloseWithError(e) + if _, cerr := io.CopyN(multiWriter, r.Body, size); cerr != nil { + errorIf(cerr, "Unable to read HTTP body.", nil) + writer.CloseWithError(err) return } shaPayload := shaWriter.Sum(nil) @@ -676,14 +674,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata) } if err != nil { - errorIf(err.Trace(), "PutObject failed.", nil) - e := err.ToGoError() + errorIf(err, "PutObject failed.", nil) // Verify if the underlying error is signature mismatch. - if e == errSignatureMismatch { + if err == errSignatureMismatch { writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) return } - switch e.(type) { + switch err.(type) { case StorageFull: writeErrorResponse(w, r, ErrStorageFull, r.URL.Path) case BucketNotFound: @@ -736,8 +733,8 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r uploadID, err := api.ObjectAPI.NewMultipartUpload(bucket, object) if err != nil { - errorIf(err.Trace(), "NewMultipartUpload failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "NewMultipartUpload failed.", nil) + switch err.(type) { case StorageFull: writeErrorResponse(w, r, ErrStorageFull, r.URL.Path) case BucketNameInvalid: @@ -771,7 +768,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http // get Content-Md5 sent by client and verify if valid md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5")) if err != nil { - errorIf(err.Trace(r.Header.Get("Content-Md5")), "Decoding md5 failed.", nil) + errorIf(err, "Decoding md5 failed.", nil) writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path) return } @@ -792,8 +789,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http uploadID := r.URL.Query().Get("uploadId") partIDString := r.URL.Query().Get("partNumber") - partID, e := strconv.Atoi(partIDString) - if e != nil { + partID, err := strconv.Atoi(partIDString) + if err != nil { writeErrorResponse(w, r, ErrInvalidPart, r.URL.Path) return } @@ -822,9 +819,9 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http go func() { shaWriter := fastSha256.New() multiWriter := io.MultiWriter(shaWriter, writer) - if _, e := io.CopyN(multiWriter, r.Body, size); e != nil { - errorIf(probe.NewError(e), "Unable to read HTTP body.", nil) - writer.CloseWithError(e) + if _, err = io.CopyN(multiWriter, r.Body, size); err != nil { + errorIf(err, "Unable to read HTTP body.", nil) + writer.CloseWithError(err) return } shaPayload := shaWriter.Sum(nil) @@ -848,14 +845,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, hex.EncodeToString(md5Bytes)) } if err != nil { - errorIf(err.Trace(), "PutObjectPart failed.", nil) - e := err.ToGoError() + errorIf(err, "PutObjectPart failed.", nil) // Verify if the underlying error is signature mismatch. - if e == errSignatureMismatch { + if err == errSignatureMismatch { writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path) return } - switch e.(type) { + switch err.(type) { case StorageFull: writeErrorResponse(w, r, ErrStorageFull, r.URL.Path) case InvalidUploadID: @@ -900,10 +896,9 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, } uploadID, _, _, _ := getObjectResources(r.URL.Query()) - err := api.ObjectAPI.AbortMultipartUpload(bucket, object, uploadID) - if err != nil { - errorIf(err.Trace(), "AbortMutlipartUpload failed.", nil) - switch err.ToGoError().(type) { + if err := api.ObjectAPI.AbortMultipartUpload(bucket, object, uploadID); err != nil { + errorIf(err, "AbortMutlipartUpload failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketNotFound: @@ -961,8 +956,8 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht listPartsInfo, err := api.ObjectAPI.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts) if err != nil { - errorIf(err.Trace(), "ListObjectParts failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "ListObjectParts failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketNotFound: @@ -998,7 +993,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite uploadID, _, _, _ := getObjectResources(r.URL.Query()) var md5Sum string - var err *probe.Error + var err error switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -1016,15 +1011,15 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite return } } - completeMultipartBytes, e := ioutil.ReadAll(r.Body) - if e != nil { - errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil) + completeMultipartBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + errorIf(err, "CompleteMultipartUpload failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) return } complMultipartUpload := &completeMultipartUpload{} - if e = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); e != nil { - errorIf(probe.NewError(e), "XML Unmarshal failed", nil) + if err = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); err != nil { + errorIf(err, "XML Unmarshal failed", nil) writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path) return } @@ -1042,8 +1037,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite // Complete multipart upload. md5Sum, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts) if err != nil { - errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil) - switch err.ToGoError().(type) { + errorIf(err, "CompleteMultipartUpload failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketNotFound: @@ -1099,10 +1094,9 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. return } } - err := api.ObjectAPI.DeleteObject(bucket, object) - if err != nil { - errorIf(err.Trace(), "DeleteObject failed.", nil) - switch err.ToGoError().(type) { + if err := api.ObjectAPI.DeleteObject(bucket, object); err != nil { + errorIf(err, "DeleteObject failed.", nil) + switch err.(type) { case BucketNameInvalid: writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketNotFound: diff --git a/object-interface.go b/object-interface.go new file mode 100644 index 000000000..553b44a5f --- /dev/null +++ b/object-interface.go @@ -0,0 +1,43 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import "io" + +// ObjectLayer implements primitives for object API layer. +type ObjectLayer interface { + // Bucket operations. + MakeBucket(bucket string) error + GetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) + ListBuckets() (buckets []BucketInfo, err error) + DeleteBucket(bucket string) error + ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) + + // Object operations. + GetObject(bucket, object string, startOffset int64) (reader io.ReadCloser, err error) + GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) + PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string) (md5 string, err error) + DeleteObject(bucket, object string) error + + // Multipart operations. + ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) + NewMultipartUpload(bucket, object string) (uploadID string, err error) + PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (md5 string, err error) + ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) + AbortMultipartUpload(bucket, object, uploadID string) error + CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (md5 string, err error) +} diff --git a/object-utils.go b/object-utils.go index 63d3e0627..aa2de285e 100644 --- a/object-utils.go +++ b/object-utils.go @@ -17,9 +17,21 @@ package main import ( + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "io" "regexp" "strings" "unicode/utf8" + + "github.com/minio/minio/pkg/safe" +) + +const ( + // Minio meta volume. + minioMetaVolume = ".minio" ) // validBucket regexp. @@ -96,3 +108,41 @@ func retainSlash(s string) string { func pathJoin(s1 string, s2 string) string { return retainSlash(s1) + s2 } + +// Create an s3 compatible MD5sum for complete multipart transaction. +func makeS3MD5(md5Strs ...string) (string, error) { + var finalMD5Bytes []byte + for _, md5Str := range md5Strs { + md5Bytes, err := hex.DecodeString(md5Str) + if err != nil { + return "", err + } + finalMD5Bytes = append(finalMD5Bytes, md5Bytes...) + } + md5Hasher := md5.New() + md5Hasher.Write(finalMD5Bytes) + s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(md5Strs)) + return s3MD5, nil +} + +// byBucketName is a collection satisfying sort.Interface. +type byBucketName []BucketInfo + +func (d byBucketName) Len() int { return len(d) } +func (d byBucketName) Swap(i, j int) { d[i], d[j] = d[j], d[i] } +func (d byBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name } + +// safeCloseAndRemove - safely closes and removes underlying temporary +// file writer if possible. +func safeCloseAndRemove(writer io.WriteCloser) error { + // If writer is a safe file, Attempt to close and remove. + safeWriter, ok := writer.(*safe.File) + if ok { + return safeWriter.CloseAndRemove() + } + pipeWriter, ok := writer.(*io.PipeWriter) + if ok { + return pipeWriter.CloseWithError(errors.New("Close and error out.")) + } + return nil +} diff --git a/object_api_suite_test.go b/object_api_suite_test.go index a322c0e61..75e256550 100644 --- a/object_api_suite_test.go +++ b/object_api_suite_test.go @@ -31,7 +31,7 @@ import ( // TODO - enable all the commented tests. // APITestSuite - collection of API tests. -func APITestSuite(c *check.C, create func() objectAPI) { +func APITestSuite(c *check.C, create func() ObjectLayer) { testMakeBucket(c, create) testMultipleObjectCreation(c, create) testPaging(c, create) @@ -50,7 +50,7 @@ func APITestSuite(c *check.C, create func() objectAPI) { } // Tests validate bucket creation. -func testMakeBucket(c *check.C, create func() objectAPI) { +func testMakeBucket(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("bucket-unknown") c.Assert(err, check.IsNil) @@ -58,7 +58,7 @@ func testMakeBucket(c *check.C, create func() objectAPI) { } // Tests validate creation of part files during Multipart operation. -func testMultipartObjectCreation(c *check.C, create func() objectAPI) { +func testMultipartObjectCreation(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) @@ -83,7 +83,7 @@ func testMultipartObjectCreation(c *check.C, create func() objectAPI) { } // Tests validate abortion of Multipart operation. -func testMultipartObjectAbort(c *check.C, create func() objectAPI) { +func testMultipartObjectAbort(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) @@ -115,7 +115,7 @@ func testMultipartObjectAbort(c *check.C, create func() objectAPI) { } // Tests validate object creation. -func testMultipleObjectCreation(c *check.C, create func() objectAPI) { +func testMultipleObjectCreation(c *check.C, create func() ObjectLayer) { objects := make(map[string][]byte) obj := create() err := obj.MakeBucket("bucket") @@ -157,7 +157,7 @@ func testMultipleObjectCreation(c *check.C, create func() objectAPI) { } // Tests validate creation of objects and the order of listing using various filters for ListObjects operation. -func testPaging(c *check.C, create func() objectAPI) { +func testPaging(c *check.C, create func() ObjectLayer) { obj := create() obj.MakeBucket("bucket") result, err := obj.ListObjects("bucket", "", "", "", 0) @@ -261,7 +261,7 @@ func testPaging(c *check.C, create func() objectAPI) { } // Tests validate overwriting of an existing object. -func testObjectOverwriteWorks(c *check.C, create func() objectAPI) { +func testObjectOverwriteWorks(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) @@ -284,25 +284,25 @@ func testObjectOverwriteWorks(c *check.C, create func() objectAPI) { } // Tests validate that bucket operation on non-existent bucket fails. -func testNonExistantBucketOperations(c *check.C, create func() objectAPI) { +func testNonExistantBucketOperations(c *check.C, create func() ObjectLayer) { obj := create() _, err := obj.PutObject("bucket1", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.Not(check.IsNil)) - c.Assert(err.ToGoError().Error(), check.Equals, "Bucket not found: bucket1") + c.Assert(err.Error(), check.Equals, "Bucket not found: bucket1") } // Tests validate that recreation of the bucket fails. -func testBucketRecreateFails(c *check.C, create func() objectAPI) { +func testBucketRecreateFails(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("string") c.Assert(err, check.IsNil) err = obj.MakeBucket("string") c.Assert(err, check.Not(check.IsNil)) - c.Assert(err.ToGoError().Error(), check.Equals, "Bucket exists: string") + c.Assert(err.Error(), check.Equals, "Bucket exists: string") } // Tests validate PutObject with subdirectory prefix. -func testPutObjectInSubdir(c *check.C, create func() objectAPI) { +func testPutObjectInSubdir(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) @@ -321,7 +321,7 @@ func testPutObjectInSubdir(c *check.C, create func() objectAPI) { } // Tests validate ListBuckets. -func testListBuckets(c *check.C, create func() objectAPI) { +func testListBuckets(c *check.C, create func() ObjectLayer) { obj := create() // test empty list. @@ -354,7 +354,7 @@ func testListBuckets(c *check.C, create func() objectAPI) { } // Tests validate the order of result of ListBuckets. -func testListBucketsOrder(c *check.C, create func() objectAPI) { +func testListBucketsOrder(c *check.C, create func() ObjectLayer) { // if implementation contains a map, order of map keys will vary. // this ensures they return in the same order each time. for i := 0; i < 10; i++ { @@ -373,24 +373,24 @@ func testListBucketsOrder(c *check.C, create func() objectAPI) { } // Tests validate that ListObjects operation on a non-existent bucket fails as expected. -func testListObjectsTestsForNonExistantBucket(c *check.C, create func() objectAPI) { +func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectLayer) { obj := create() result, err := obj.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.Not(check.IsNil)) c.Assert(result.IsTruncated, check.Equals, false) c.Assert(len(result.Objects), check.Equals, 0) - c.Assert(err.ToGoError().Error(), check.Equals, "Bucket not found: bucket") + c.Assert(err.Error(), check.Equals, "Bucket not found: bucket") } // Tests validate that GetObject fails on a non-existent bucket as expected. -func testNonExistantObjectInBucket(c *check.C, create func() objectAPI) { +func testNonExistantObjectInBucket(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) _, err = obj.GetObject("bucket", "dir1", 0) c.Assert(err, check.Not(check.IsNil)) - switch err := err.ToGoError().(type) { + switch err := err.(type) { case ObjectNotFound: c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1") default: @@ -399,7 +399,7 @@ func testNonExistantObjectInBucket(c *check.C, create func() objectAPI) { } // Tests validate that GetObject on an existing directory fails as expected. -func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() objectAPI) { +func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) @@ -408,7 +408,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() objectAPI) c.Assert(err, check.IsNil) _, err = obj.GetObject("bucket", "dir1", 0) - switch err := err.ToGoError().(type) { + switch err := err.(type) { case ObjectNotFound: c.Assert(err.Bucket, check.Equals, "bucket") c.Assert(err.Object, check.Equals, "dir1") @@ -418,7 +418,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() objectAPI) } _, err = obj.GetObject("bucket", "dir1/", 0) - switch err := err.ToGoError().(type) { + switch err := err.(type) { case ObjectNotFound: c.Assert(err.Bucket, check.Equals, "bucket") c.Assert(err.Object, check.Equals, "dir1/") @@ -429,7 +429,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() objectAPI) } // Tests valdiate the default ContentType. -func testDefaultContentType(c *check.C, create func() objectAPI) { +func testDefaultContentType(c *check.C, create func() ObjectLayer) { obj := create() err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) diff --git a/pkg/quick/quick.go b/pkg/quick/quick.go index 1ef27f755..691dca569 100644 --- a/pkg/quick/quick.go +++ b/pkg/quick/quick.go @@ -30,7 +30,6 @@ import ( "sync" "github.com/fatih/structs" - "github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/safe" ) @@ -38,11 +37,11 @@ import ( type Config interface { String() string Version() string - Save(string) *probe.Error - Load(string) *probe.Error + Save(string) error + Load(string) error Data() interface{} - Diff(Config) ([]structs.Field, *probe.Error) - DeepDiff(Config) ([]structs.Field, *probe.Error) + Diff(Config) ([]structs.Field, error) + DeepDiff(Config) ([]structs.Field, error) } // config - implements quick.Config interface @@ -53,28 +52,28 @@ type config struct { // CheckData - checks the validity of config data. Data should be of // type struct and contain a string type field called "Version". -func CheckData(data interface{}) *probe.Error { +func CheckData(data interface{}) error { if !structs.IsStruct(data) { - return probe.NewError(fmt.Errorf("Invalid argument type. Expecing \"struct\" type.")) + return fmt.Errorf("Invalid argument type. Expecing \"struct\" type.") } st := structs.New(data) f, ok := st.FieldOk("Version") if !ok { - return probe.NewError(fmt.Errorf("Invalid type of struct argument. No [%s.Version] field found.", st.Name())) + return fmt.Errorf("Invalid type of struct argument. No [%s.Version] field found.", st.Name()) } if f.Kind() != reflect.String { - return probe.NewError(fmt.Errorf("Invalid type of struct argument. Expecting \"string\" type [%s.Version] field.", st.Name())) + return fmt.Errorf("Invalid type of struct argument. Expecting \"string\" type [%s.Version] field.", st.Name()) } return nil } // New - instantiate a new config -func New(data interface{}) (Config, *probe.Error) { +func New(data interface{}) (Config, error) { if err := CheckData(data); err != nil { - return nil, err.Trace() + return nil, err } d := new(config) @@ -85,15 +84,15 @@ func New(data interface{}) (Config, *probe.Error) { // CheckVersion - loads json and compares the version number provided returns back true or false - any failure // is returned as error. -func CheckVersion(filename string, version string) (bool, *probe.Error) { - _, e := os.Stat(filename) - if e != nil { - return false, probe.NewError(e) +func CheckVersion(filename string, version string) (bool, error) { + _, err := os.Stat(filename) + if err != nil { + return false, err } - fileData, e := ioutil.ReadFile(filename) - if e != nil { - return false, probe.NewError(e) + fileData, err := ioutil.ReadFile(filename) + if err != nil { + return false, err } if runtime.GOOS == "windows" { @@ -104,18 +103,18 @@ func CheckVersion(filename string, version string) (bool, *probe.Error) { }{ Version: "", } - e = json.Unmarshal(fileData, &data) - if e != nil { - switch e := e.(type) { + err = json.Unmarshal(fileData, &data) + if err != nil { + switch err := err.(type) { case *json.SyntaxError: - return false, probe.NewError(FormatJSONSyntaxError(bytes.NewReader(fileData), e)) + return false, FormatJSONSyntaxError(bytes.NewReader(fileData), err) default: - return false, probe.NewError(e) + return false, err } } config, err := New(data) if err != nil { - return false, err.Trace() + return false, err } if config.Version() != version { return false, nil @@ -124,34 +123,34 @@ func CheckVersion(filename string, version string) (bool, *probe.Error) { } // Load - loads json config from filename for the a given struct data -func Load(filename string, data interface{}) (Config, *probe.Error) { - _, e := os.Stat(filename) - if e != nil { - return nil, probe.NewError(e) +func Load(filename string, data interface{}) (Config, error) { + _, err := os.Stat(filename) + if err != nil { + return nil, err } - fileData, e := ioutil.ReadFile(filename) - if e != nil { - return nil, probe.NewError(e) + fileData, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err } if runtime.GOOS == "windows" { fileData = []byte(strings.Replace(string(fileData), "\r\n", "\n", -1)) } - e = json.Unmarshal(fileData, &data) - if e != nil { - switch e := e.(type) { + err = json.Unmarshal(fileData, &data) + if err != nil { + switch err := err.(type) { case *json.SyntaxError: - return nil, probe.NewError(FormatJSONSyntaxError(bytes.NewReader(fileData), e)) + return nil, FormatJSONSyntaxError(bytes.NewReader(fileData), err) default: - return nil, probe.NewError(e) + return nil, err } } config, err := New(data) if err != nil { - return nil, err.Trace() + return nil, err } return config, nil @@ -177,20 +176,16 @@ func (d config) Version() string { // writeFile writes data to a file named by filename. // If the file does not exist, writeFile creates it; // otherwise writeFile truncates it before writing. -func writeFile(filename string, data []byte) *probe.Error { - safeFile, e := safe.CreateFile(filename) - if e != nil { - return probe.NewError(e) +func writeFile(filename string, data []byte) error { + safeFile, err := safe.CreateFile(filename) + if err != nil { + return err } - _, e = safeFile.Write(data) - if e != nil { - return probe.NewError(e) + _, err = safeFile.Write(data) + if err != nil { + return err } - e = safeFile.Close() - if e != nil { - return probe.NewError(e) - } - return nil + return safeFile.Close() } // String converts JSON config to printable string @@ -200,37 +195,37 @@ func (d config) String() string { } // Save writes config data in JSON format to a file. -func (d config) Save(filename string) *probe.Error { +func (d config) Save(filename string) error { d.lock.Lock() defer d.lock.Unlock() // Check for existing file, if yes create a backup. - st, e := os.Stat(filename) + st, err := os.Stat(filename) // If file exists and stat failed return here. - if e != nil && !os.IsNotExist(e) { - return probe.NewError(e) + if err != nil && !os.IsNotExist(err) { + return err } // File exists and proceed to take backup. - if e == nil { + if err == nil { // File exists and is not a regular file return error. if !st.Mode().IsRegular() { - return probe.NewError(fmt.Errorf("%s is not a regular file", filename)) + return fmt.Errorf("%s is not a regular file", filename) } // Read old data. var oldData []byte - oldData, e = ioutil.ReadFile(filename) - if e != nil { - return probe.NewError(e) + oldData, err = ioutil.ReadFile(filename) + if err != nil { + return err } // Save read data to the backup file. - if err := writeFile(filename+".old", oldData); err != nil { - return err.Trace(filename + ".old") + if err = writeFile(filename+".old", oldData); err != nil { + return err } } // Proceed to create or overwrite file. - jsonData, e := json.MarshalIndent(d.data, "", "\t") - if e != nil { - return probe.NewError(e) + jsonData, err := json.MarshalIndent(d.data, "", "\t") + if err != nil { + return err } if runtime.GOOS == "windows" { @@ -238,23 +233,22 @@ func (d config) Save(filename string) *probe.Error { } // Save data. - err := writeFile(filename, jsonData) - return err.Trace(filename) + return writeFile(filename, jsonData) } // Load - loads JSON config from file and merge with currently set values -func (d *config) Load(filename string) *probe.Error { +func (d *config) Load(filename string) error { d.lock.Lock() defer d.lock.Unlock() - _, e := os.Stat(filename) - if e != nil { - return probe.NewError(e) + _, err := os.Stat(filename) + if err != nil { + return err } - fileData, e := ioutil.ReadFile(filename) - if e != nil { - return probe.NewError(e) + fileData, err := ioutil.ReadFile(filename) + if err != nil { + return err } if runtime.GOOS == "windows" { @@ -264,25 +258,25 @@ func (d *config) Load(filename string) *probe.Error { st := structs.New(d.data) f, ok := st.FieldOk("Version") if !ok { - return probe.NewError(fmt.Errorf("Argument struct [%s] does not contain field \"Version\".", st.Name())) + return fmt.Errorf("Argument struct [%s] does not contain field \"Version\".", st.Name()) } - e = json.Unmarshal(fileData, d.data) - if e != nil { - switch e := e.(type) { + err = json.Unmarshal(fileData, d.data) + if err != nil { + switch err := err.(type) { case *json.SyntaxError: - return probe.NewError(FormatJSONSyntaxError(bytes.NewReader(fileData), e)) + return FormatJSONSyntaxError(bytes.NewReader(fileData), err) default: - return probe.NewError(e) + return err } } if err := CheckData(d.data); err != nil { - return err.Trace(filename) + return err } if (*d).Version() != f.Value() { - return probe.NewError(fmt.Errorf("Version mismatch")) + return fmt.Errorf("Version mismatch") } return nil @@ -294,11 +288,11 @@ func (d config) Data() interface{} { } //Diff - list fields that are in A but not in B -func (d config) Diff(c Config) ([]structs.Field, *probe.Error) { +func (d config) Diff(c Config) ([]structs.Field, error) { var fields []structs.Field err := CheckData(c.Data()) if err != nil { - return []structs.Field{}, err.Trace() + return []structs.Field{}, err } currFields := structs.Fields(d.Data()) @@ -320,11 +314,11 @@ func (d config) Diff(c Config) ([]structs.Field, *probe.Error) { } //DeepDiff - list fields in A that are missing or not equal to fields in B -func (d config) DeepDiff(c Config) ([]structs.Field, *probe.Error) { +func (d config) DeepDiff(c Config) ([]structs.Field, error) { var fields []structs.Field err := CheckData(c.Data()) if err != nil { - return []structs.Field{}, err.Trace() + return []structs.Field{}, err } currFields := structs.Fields(d.Data()) diff --git a/fs-dir-common.go b/posix-dir-common.go similarity index 100% rename from fs-dir-common.go rename to posix-dir-common.go diff --git a/fs-dir-nix.go b/posix-dir-nix.go similarity index 100% rename from fs-dir-nix.go rename to posix-dir-nix.go diff --git a/fs-dir-others.go b/posix-dir-others.go similarity index 100% rename from fs-dir-others.go rename to posix-dir-others.go diff --git a/fs-dirent-fileno.go b/posix-dirent-fileno.go similarity index 100% rename from fs-dirent-fileno.go rename to posix-dirent-fileno.go diff --git a/fs-dirent-ino.go b/posix-dirent-ino.go similarity index 100% rename from fs-dirent-ino.go rename to posix-dirent-ino.go diff --git a/fs-utils.go b/posix-utils.go similarity index 100% rename from fs-utils.go rename to posix-utils.go diff --git a/fs.go b/posix.go similarity index 99% rename from fs.go rename to posix.go index 1a9cadb96..e1628d53f 100644 --- a/fs.go +++ b/posix.go @@ -78,7 +78,7 @@ func isDirExist(dirname string) (bool, error) { } // Initialize a new storage disk. -func newFS(diskPath string) (StorageAPI, error) { +func newPosix(diskPath string) (StorageAPI, error) { if diskPath == "" { log.Debug("Disk cannot be empty") return nil, errInvalidArgument diff --git a/routers.go b/routers.go index 6a9e6f73a..39afda7f7 100644 --- a/routers.go +++ b/routers.go @@ -18,38 +18,30 @@ package main import ( "net/http" - "path/filepath" - "strings" router "github.com/gorilla/mux" - "github.com/minio/minio/pkg/probe" ) -// newStorageAPI - initialize any storage API depending on the export path style. -func newStorageAPI(exportPaths ...string) (StorageAPI, error) { +// newObjectLayer - initialize any object layer depending on the +// number of export paths. +func newObjectLayer(exportPaths ...string) (ObjectLayer, error) { if len(exportPaths) == 1 { exportPath := exportPaths[0] - if !strings.ContainsRune(exportPath, ':') || filepath.VolumeName(exportPath) != "" { - // Initialize filesystem storage API. - return newFS(exportPath) - } - // Initialize network storage API. - return newNetworkFS(exportPath) + // Initialize FS object layer. + return newFSObjects(exportPath) } - // Initialize XL storage API. - return newXL(exportPaths...) + // Initialize XL object layer. + return newXLObjects(exportPaths...) } // configureServer handler returns final handler for the http server. func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler { - storageAPI, e := newStorageAPI(srvCmdConfig.exportPaths...) - fatalIf(probe.NewError(e), "Initializing storage API failed.", nil) + objAPI, err := newObjectLayer(srvCmdConfig.exportPaths...) + fatalIf(err, "Initializing object layer failed.", nil) - // Initialize object layer. - objAPI := newObjectLayer(storageAPI) - - // Initialize storage rpc. - storageRPC := newStorageRPC(storageAPI) + // Initialize storage rpc server. + storageRPC, err := newRPCServer(srvCmdConfig.exportPaths[0]) // FIXME: should only have one path. + fatalIf(err, "Initializing storage rpc server failed.", nil) // Initialize API. apiHandlers := objectAPIHandlers{ diff --git a/network-fs.go b/rpc-client.go similarity index 98% rename from network-fs.go rename to rpc-client.go index 7c5dca2b5..5f9904e39 100644 --- a/network-fs.go +++ b/rpc-client.go @@ -76,8 +76,8 @@ func toStorageErr(err error) error { return err } -// Initialize new network file system. -func newNetworkFS(networkPath string) (StorageAPI, error) { +// Initialize new rpc client. +func newRPCClient(networkPath string) (StorageAPI, error) { // Input validation. if networkPath == "" || strings.LastIndex(networkPath, ":") == -1 { log.WithFields(logrus.Fields{ diff --git a/storage-rpc-datatypes.go b/rpc-server-datatypes.go similarity index 100% rename from storage-rpc-datatypes.go rename to rpc-server-datatypes.go diff --git a/storage-rpc-server.go b/rpc-server.go similarity index 96% rename from storage-rpc-server.go rename to rpc-server.go index 3329924be..4d1f05d74 100644 --- a/storage-rpc-server.go +++ b/rpc-server.go @@ -119,10 +119,15 @@ func (s *storageServer) DeleteFileHandler(arg *DeleteFileArgs, reply *GenericRep } // Initialize new storage rpc. -func newStorageRPC(storageAPI StorageAPI) *storageServer { - return &storageServer{ - storage: storageAPI, +func newRPCServer(exportPath string) (*storageServer, error) { + // Initialize posix storage API. + storage, err := newPosix(exportPath) + if err != nil { + return nil, err } + return &storageServer{ + storage: storage, + }, nil } // registerStorageRPCRouter - register storage rpc router. diff --git a/server-main.go b/server-main.go index 028087606..4f5d63771 100644 --- a/server-main.go +++ b/server-main.go @@ -31,7 +31,6 @@ import ( "github.com/minio/cli" "github.com/minio/mc/pkg/console" "github.com/minio/minio/pkg/minhttp" - "github.com/minio/minio/pkg/probe" ) var serverCmd = cli.Command{ @@ -89,11 +88,11 @@ func configureServer(srvCmdConfig serverCmdConfig) *http.Server { // Configure TLS if certs are available. if isSSL() { - var e error + var err error apiServer.TLSConfig = &tls.Config{} apiServer.TLSConfig.Certificates = make([]tls.Certificate, 1) - apiServer.TLSConfig.Certificates[0], e = tls.LoadX509KeyPair(mustGetCertFile(), mustGetKeyFile()) - fatalIf(probe.NewError(e), "Unable to load certificates.", nil) + apiServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(mustGetCertFile(), mustGetKeyFile()) + fatalIf(err, "Unable to load certificates.", nil) } // Returns configured HTTP server. @@ -102,16 +101,16 @@ func configureServer(srvCmdConfig serverCmdConfig) *http.Server { // Print listen ips. func printListenIPs(httpServerConf *http.Server) { - host, port, e := net.SplitHostPort(httpServerConf.Addr) - fatalIf(probe.NewError(e), "Unable to split host port.", nil) + host, port, err := net.SplitHostPort(httpServerConf.Addr) + fatalIf(err, "Unable to split host port.", nil) var hosts []string switch { case host != "": hosts = append(hosts, host) default: - addrs, e := net.InterfaceAddrs() - fatalIf(probe.NewError(e), "Unable to get interface address.", nil) + addrs, err := net.InterfaceAddrs() + fatalIf(err, "Unable to get interface address.", nil) for _, addr := range addrs { if addr.Network() == "ip+net" { host := strings.Split(addr.String(), "/")[0] @@ -134,7 +133,7 @@ func printListenIPs(httpServerConf *http.Server) { func initServerConfig(c *cli.Context) { // Save new config. err := serverConfig.Save() - fatalIf(err.Trace(), "Unable to save config.", nil) + fatalIf(err, "Unable to save config.", nil) // Fetch access keys from environment variables if any and update the config. accessKey := os.Getenv("MINIO_ACCESS_KEY") @@ -143,10 +142,10 @@ func initServerConfig(c *cli.Context) { // Validate if both keys are specified and they are valid save them. if accessKey != "" && secretKey != "" { if !isValidAccessKey.MatchString(accessKey) { - fatalIf(probe.NewError(errInvalidArgument), "Access key does not have required length", nil) + fatalIf(errInvalidArgument, "Access key does not have required length", nil) } if !isValidSecretKey.MatchString(secretKey) { - fatalIf(probe.NewError(errInvalidArgument), "Secret key does not have required length", nil) + fatalIf(errInvalidArgument, "Secret key does not have required length", nil) } serverConfig.SetCredential(credential{ AccessKeyID: accessKey, @@ -169,10 +168,10 @@ func checkServerSyntax(c *cli.Context) { // Extract port number from address address should be of the form host:port. func getPort(address string) int { - _, portStr, e := net.SplitHostPort(address) - fatalIf(probe.NewError(e), "Unable to split host port.", nil) - portInt, e := strconv.Atoi(portStr) - fatalIf(probe.NewError(e), "Invalid port number.", nil) + _, portStr, err := net.SplitHostPort(address) + fatalIf(err, "Unable to split host port.", nil) + portInt, err := strconv.Atoi(portStr) + fatalIf(err, "Invalid port number.", nil) return portInt } @@ -186,11 +185,11 @@ func getPort(address string) int { // on 127.0.0.1 even though minio server is running. So before we start // the minio server we make sure that the port is free on all the IPs. func checkPortAvailability(port int) { - isAddrInUse := func(e error) bool { + isAddrInUse := func(err error) bool { // Check if the syscall error is EADDRINUSE. // EADDRINUSE is the system call error if another process is // already listening at the specified port. - neterr, ok := e.(*net.OpError) + neterr, ok := err.(*net.OpError) if !ok { return false } @@ -207,19 +206,19 @@ func checkPortAvailability(port int) { } return true } - ifcs, e := net.Interfaces() - if e != nil { - fatalIf(probe.NewError(e), "Unable to list interfaces.", nil) + ifcs, err := net.Interfaces() + if err != nil { + fatalIf(err, "Unable to list interfaces.", nil) } for _, ifc := range ifcs { - addrs, e := ifc.Addrs() - if e != nil { - fatalIf(probe.NewError(e), fmt.Sprintf("Unable to list addresses on interface %s.", ifc.Name), nil) + addrs, err := ifc.Addrs() + if err != nil { + fatalIf(err, fmt.Sprintf("Unable to list addresses on interface %s.", ifc.Name), nil) } for _, addr := range addrs { ipnet, ok := addr.(*net.IPNet) if !ok { - errorIf(probe.NewError(errors.New("")), "Interface type assertion to (*net.IPNet) failed.", nil) + errorIf(errors.New(""), "Interface type assertion to (*net.IPNet) failed.", nil) continue } ip := ipnet.IP @@ -228,19 +227,18 @@ func checkPortAvailability(port int) { network = "tcp6" } tcpAddr := net.TCPAddr{IP: ip, Port: port, Zone: ifc.Name} - l, e := net.ListenTCP(network, &tcpAddr) - if e != nil { - if isAddrInUse(e) { + l, err := net.ListenTCP(network, &tcpAddr) + if err != nil { + if isAddrInUse(err) { // Fail if port is already in use. - fatalIf(probe.NewError(e), fmt.Sprintf("Unable to listen on IP %s, port %.d", tcpAddr.IP, tcpAddr.Port), nil) + fatalIf(err, fmt.Sprintf("Unable to listen on IP %s, port %.d", tcpAddr.IP, tcpAddr.Port), nil) } else { // Ignore other errors. continue } } - e = l.Close() - if e != nil { - fatalIf(probe.NewError(e), fmt.Sprintf("Unable to close listener on IP %s, port %.d", tcpAddr.IP, tcpAddr.Port), nil) + if err = l.Close(); err != nil { + fatalIf(err, fmt.Sprintf("Unable to close listener on IP %s, port %.d", tcpAddr.IP, tcpAddr.Port), nil) } } } @@ -308,5 +306,5 @@ func serverMain(c *cli.Context) { // Start server. err := minhttp.ListenAndServe(apiServer) - errorIf(err.Trace(), "Failed to start the minio server.", nil) + errorIf(err.Cause, "Failed to start the minio server.", nil) } diff --git a/signature-jwt.go b/signature-jwt.go index 7d7336921..932174465 100644 --- a/signature-jwt.go +++ b/signature-jwt.go @@ -21,7 +21,6 @@ import ( "time" jwtgo "github.com/dgrijalva/jwt-go" - "github.com/minio/minio/pkg/probe" "golang.org/x/crypto/bcrypt" ) @@ -49,17 +48,13 @@ func initJWT() *JWT { } // GenerateToken - generates a new Json Web Token based on the incoming user id. -func (jwt *JWT) GenerateToken(userName string) (string, *probe.Error) { +func (jwt *JWT) GenerateToken(userName string) (string, error) { token := jwtgo.New(jwtgo.SigningMethodHS512) // Token expires in 10hrs. token.Claims["exp"] = time.Now().Add(time.Hour * tokenExpires).Unix() token.Claims["iat"] = time.Now().Unix() token.Claims["sub"] = userName - tokenString, e := token.SignedString([]byte(jwt.SecretAccessKey)) - if e != nil { - return "", probe.NewError(e) - } - return tokenString, nil + return token.SignedString([]byte(jwt.SecretAccessKey)) } // Authenticate - authenticates incoming username and password. diff --git a/signature-v4-postpolicyform.go b/signature-v4-postpolicyform.go index 20ea45dc7..0abf54d6b 100644 --- a/signature-v4-postpolicyform.go +++ b/signature-v4-postpolicyform.go @@ -23,8 +23,6 @@ import ( "reflect" "strings" "time" - - "github.com/minio/minio/pkg/probe" ) // toString - Safely convert interface to string without causing panic. @@ -70,7 +68,7 @@ type PostPolicyForm struct { } // parsePostPolicyFormV4 - Parse JSON policy string into typed POostPolicyForm structure. -func parsePostPolicyFormV4(policy string) (PostPolicyForm, *probe.Error) { +func parsePostPolicyFormV4(policy string) (PostPolicyForm, error) { // Convert po into interfaces and // perform strict type conversion using reflection. var rawPolicy struct { @@ -78,17 +76,17 @@ func parsePostPolicyFormV4(policy string) (PostPolicyForm, *probe.Error) { Conditions []interface{} `json:"conditions"` } - e := json.Unmarshal([]byte(policy), &rawPolicy) - if e != nil { - return PostPolicyForm{}, probe.NewError(e) + err := json.Unmarshal([]byte(policy), &rawPolicy) + if err != nil { + return PostPolicyForm{}, err } parsedPolicy := PostPolicyForm{} // Parse expiry time. - parsedPolicy.Expiration, e = time.Parse(time.RFC3339Nano, rawPolicy.Expiration) - if e != nil { - return PostPolicyForm{}, probe.NewError(e) + parsedPolicy.Expiration, err = time.Parse(time.RFC3339Nano, rawPolicy.Expiration) + if err != nil { + return PostPolicyForm{}, err } parsedPolicy.Conditions.Policies = make(map[string]struct { Operator string @@ -102,8 +100,7 @@ func parsePostPolicyFormV4(policy string) (PostPolicyForm, *probe.Error) { for k, v := range condt { if !isString(v) { // Pre-check value type. // All values must be of type string. - return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.", - reflect.TypeOf(condt).String(), condt)) + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.", reflect.TypeOf(condt).String(), condt) } // {"acl": "public-read" } is an alternate way to indicate - [ "eq", "$acl", "public-read" ] // In this case we will just collapse this into "eq" for all use cases. @@ -117,16 +114,14 @@ func parsePostPolicyFormV4(policy string) (PostPolicyForm, *probe.Error) { } case []interface{}: // Handle array types. if len(condt) != 3 { // Return error if we have insufficient elements. - return parsedPolicy, probe.NewError(fmt.Errorf("Malformed conditional fields %s of type %s found in POST policy form.", - condt, reflect.TypeOf(condt).String())) + return parsedPolicy, fmt.Errorf("Malformed conditional fields %s of type %s found in POST policy form.", condt, reflect.TypeOf(condt).String()) } switch toString(condt[0]) { case "eq", "starts-with": for _, v := range condt { // Pre-check all values for type. if !isString(v) { // All values must be of type string. - return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.", - reflect.TypeOf(condt).String(), condt)) + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.", reflect.TypeOf(condt).String(), condt) } } operator, matchType, value := toString(condt[0]), toString(condt[1]), toString(condt[2]) @@ -147,12 +142,10 @@ func parsePostPolicyFormV4(policy string) (PostPolicyForm, *probe.Error) { } default: // Condition should be valid. - return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.", - reflect.TypeOf(condt).String(), condt)) + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.", reflect.TypeOf(condt).String(), condt) } default: - return parsedPolicy, probe.NewError(fmt.Errorf("Unknown field %s of type %s found in POST policy form.", - condt, reflect.TypeOf(condt).String())) + return parsedPolicy, fmt.Errorf("Unknown field %s of type %s found in POST policy form.", condt, reflect.TypeOf(condt).String()) } } return parsedPolicy, nil diff --git a/update-main.go b/update-main.go index 617c5c9b3..90cad8452 100644 --- a/update-main.go +++ b/update-main.go @@ -28,7 +28,6 @@ import ( "github.com/fatih/color" "github.com/minio/cli" "github.com/minio/mc/pkg/console" - "github.com/minio/minio/pkg/probe" ) // command specific flags. @@ -90,7 +89,7 @@ func (u updateMessage) String() string { return updateMessage("You are already running the most recent version of ‘minio’.") } msg, err := colorizeUpdateMessage(u.Download) - fatalIf(err.Trace(msg), "Unable to colorize experimental update notification string ‘"+msg+"’.", nil) + fatalIf(err, "Unable to colorize experimental update notification string ‘"+msg+"’.", nil) return msg } @@ -98,38 +97,38 @@ func (u updateMessage) String() string { func (u updateMessage) JSON() string { u.Status = "success" updateMessageJSONBytes, err := json.Marshal(u) - fatalIf(probe.NewError(err), "Unable to marshal into JSON.", nil) + fatalIf((err), "Unable to marshal into JSON.", nil) return string(updateMessageJSONBytes) } -func parseReleaseData(data string) (time.Time, *probe.Error) { +func parseReleaseData(data string) (time.Time, error) { releaseStr := strings.Fields(data) if len(releaseStr) < 2 { - return time.Time{}, probe.NewError(errors.New("Update data malformed")) + return time.Time{}, errors.New("Update data malformed") } releaseDate := releaseStr[1] releaseDateSplits := strings.SplitN(releaseDate, ".", 3) if len(releaseDateSplits) < 3 { - return time.Time{}, probe.NewError(errors.New("Update data malformed")) + return time.Time{}, (errors.New("Update data malformed")) } if releaseDateSplits[0] != "minio" { - return time.Time{}, probe.NewError(errors.New("Update data malformed, missing minio tag")) + return time.Time{}, (errors.New("Update data malformed, missing minio tag")) } // "OFFICIAL" tag is still kept for backward compatibility, we should remove this for the next release. if releaseDateSplits[1] != "RELEASE" && releaseDateSplits[1] != "OFFICIAL" { - return time.Time{}, probe.NewError(errors.New("Update data malformed, missing RELEASE tag")) + return time.Time{}, (errors.New("Update data malformed, missing RELEASE tag")) } dateSplits := strings.SplitN(releaseDateSplits[2], "T", 2) if len(dateSplits) < 2 { - return time.Time{}, probe.NewError(errors.New("Update data malformed, not in modified RFC3359 form")) + return time.Time{}, (errors.New("Update data malformed, not in modified RFC3359 form")) } dateSplits[1] = strings.Replace(dateSplits[1], "-", ":", -1) date := strings.Join(dateSplits, "T") - parsedDate, e := time.Parse(time.RFC3339, date) - if e != nil { - return time.Time{}, probe.NewError(e) + parsedDate, err := time.Parse(time.RFC3339, date) + if err != nil { + return time.Time{}, err } return parsedDate, nil } @@ -159,32 +158,32 @@ func getReleaseUpdate(updateURL string, noError bool) updateMessage { // Instantiate a new client with 1 sec timeout. client := &http.Client{ - Timeout: 500 * time.Millisecond, + Timeout: 1 * time.Millisecond, } // Fetch new update. - data, e := client.Get(newUpdateURL) - if e != nil && noError { + data, err := client.Get(newUpdateURL) + if err != nil && noError { return updateMsg } - fatalIf(probe.NewError(e), "Unable to read from update URL ‘"+newUpdateURL+"’.", nil) + fatalIf((err), "Unable to read from update URL ‘"+newUpdateURL+"’.", nil) // Error out if 'update' command is issued for development based builds. if minioVersion == "DEVELOPMENT.GOGET" && !noError { - fatalIf(probe.NewError(errors.New("")), + fatalIf((errors.New("")), "Update mechanism is not supported for ‘go get’ based binary builds. Please download official releases from https://minio.io/#minio", nil) } // Parse current minio version into RFC3339. - current, e := time.Parse(time.RFC3339, minioVersion) - if e != nil && noError { + current, err := time.Parse(time.RFC3339, minioVersion) + if err != nil && noError { return updateMsg } - fatalIf(probe.NewError(e), "Unable to parse version string as time.", nil) + fatalIf((err), "Unable to parse version string as time.", nil) // Verify if current minio version is zero. if current.IsZero() && !noError { - fatalIf(probe.NewError(errors.New("")), + fatalIf((errors.New("")), "Updates not supported for custom builds. Version field is empty. Please download official releases from https://minio.io/#minio", nil) } @@ -195,27 +194,27 @@ func getReleaseUpdate(updateURL string, noError bool) updateMessage { if noError { return updateMsg } - fatalIf(probe.NewError(errors.New("")), "Update server responsed with "+data.Status, nil) + fatalIf((errors.New("")), "Update server responsed with "+data.Status, nil) } } // Read the response body. - updateBody, e := ioutil.ReadAll(data.Body) - if e != nil && noError { + updateBody, err := ioutil.ReadAll(data.Body) + if err != nil && noError { return updateMsg } - fatalIf(probe.NewError(e), "Fetching updates failed. Please try again.", nil) + fatalIf((err), "Fetching updates failed. Please try again.", nil) // Parse the date if its valid. latest, err := parseReleaseData(string(updateBody)) if err != nil && noError { return updateMsg } - fatalIf(err.Trace(updateURL), "Please report this issue at https://github.com/minio/minio/issues.", nil) + fatalIf(err, "Please report this issue at https://github.com/minio/minio/issues.", nil) // Verify if the date is not zero. if latest.IsZero() && !noError { - fatalIf(probe.NewError(errors.New("")), + fatalIf((errors.New("")), "Unable to validate any update available at this time. Please open an issue at https://github.com/minio/minio/issues", nil) } diff --git a/utils.go b/utils.go index d4de6fac5..aea79e3b6 100644 --- a/utils.go +++ b/utils.go @@ -19,7 +19,6 @@ package main import ( "encoding/base64" "encoding/xml" - "github.com/minio/minio/pkg/probe" "io" "strings" ) @@ -31,12 +30,8 @@ func xmlDecoder(body io.Reader, v interface{}) error { } // checkValidMD5 - verify if valid md5, returns md5 in bytes. -func checkValidMD5(md5 string) ([]byte, *probe.Error) { - md5Bytes, e := base64.StdEncoding.DecodeString(strings.TrimSpace(md5)) - if e != nil { - return nil, probe.NewError(e) - } - return md5Bytes, nil +func checkValidMD5(md5 string) ([]byte, error) { + return base64.StdEncoding.DecodeString(strings.TrimSpace(md5)) } /// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html @@ -56,6 +51,5 @@ func contains(stringList []string, element string) bool { return true } } - return false } diff --git a/web-handlers.go b/web-handlers.go index 1b297961e..cf2fbe55c 100644 --- a/web-handlers.go +++ b/web-handlers.go @@ -109,9 +109,8 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep return &json2.Error{Message: "Unauthorized request"} } reply.UIVersion = miniobrowser.UIVersion - e := web.ObjectAPI.MakeBucket(args.BucketName) - if e != nil { - return &json2.Error{Message: e.Cause.Error()} + if err := web.ObjectAPI.MakeBucket(args.BucketName); err != nil { + return &json2.Error{Message: err.Error()} } return nil } @@ -139,9 +138,9 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } - buckets, e := web.ObjectAPI.ListBuckets() - if e != nil { - return &json2.Error{Message: e.Cause.Error()} + buckets, err := web.ObjectAPI.ListBuckets() + if err != nil { + return &json2.Error{Message: err.Error()} } for _, bucket := range buckets { // List all buckets which are not private. @@ -191,7 +190,7 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r for { lo, err := web.ObjectAPI.ListObjects(args.BucketName, args.Prefix, marker, "/", 1000) if err != nil { - return &json2.Error{Message: err.Cause.Error()} + return &json2.Error{Message: err.Error()} } marker = lo.NextMarker for _, obj := range lo.Objects { @@ -227,9 +226,8 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, return &json2.Error{Message: "Unauthorized request"} } reply.UIVersion = miniobrowser.UIVersion - e := web.ObjectAPI.DeleteObject(args.BucketName, args.ObjectName) - if e != nil { - return &json2.Error{Message: e.Cause.Error()} + if err := web.ObjectAPI.DeleteObject(args.BucketName, args.ObjectName); err != nil { + return &json2.Error{Message: err.Error()} } return nil } @@ -252,7 +250,7 @@ func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginR if jwt.Authenticate(args.Username, args.Password) { token, err := jwt.GenerateToken(args.Username) if err != nil { - return &json2.Error{Message: err.Cause.Error(), Data: err.String()} + return &json2.Error{Message: err.Error()} } reply.Token = token reply.UIVersion = miniobrowser.UIVersion @@ -305,7 +303,7 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se cred := credential{args.AccessKey, args.SecretKey} serverConfig.SetCredential(cred) if err := serverConfig.Save(); err != nil { - return &json2.Error{Message: err.Cause.Error()} + return &json2.Error{Message: err.Error()} } jwt := initJWT() @@ -314,7 +312,7 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se } token, err := jwt.GenerateToken(args.AccessKey) if err != nil { - return &json2.Error{Message: err.Cause.Error()} + return &json2.Error{Message: err.Error()} } reply.Token = token reply.UIVersion = miniobrowser.UIVersion @@ -350,7 +348,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { bucket := vars["bucket"] object := vars["object"] if _, err := web.ObjectAPI.PutObject(bucket, object, -1, r.Body, nil); err != nil { - writeWebErrorResponse(w, err.ToGoError()) + writeWebErrorResponse(w, err) } } @@ -377,10 +375,10 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { objReader, err := web.ObjectAPI.GetObject(bucket, object, 0) if err != nil { - writeWebErrorResponse(w, err.ToGoError()) + writeWebErrorResponse(w, err) return } - if _, e := io.Copy(w, objReader); e != nil { + if _, err := io.Copy(w, objReader); err != nil { /// No need to print error, response writer already written to. return } diff --git a/web-router.go b/web-router.go index 00d16221d..6e063a43a 100644 --- a/web-router.go +++ b/web-router.go @@ -30,7 +30,7 @@ import ( // webAPI container for Web API. type webAPIHandlers struct { - ObjectAPI objectAPI + ObjectAPI ObjectLayer } // indexHandler - Handler to serve index.html diff --git a/xl-v1-common.go b/xl-erasure-v1-common.go similarity index 100% rename from xl-v1-common.go rename to xl-erasure-v1-common.go diff --git a/xl-v1-createfile.go b/xl-erasure-v1-createfile.go similarity index 100% rename from xl-v1-createfile.go rename to xl-erasure-v1-createfile.go diff --git a/xl-v1-errors.go b/xl-erasure-v1-errors.go similarity index 100% rename from xl-v1-errors.go rename to xl-erasure-v1-errors.go diff --git a/xl-v1-healfile.go b/xl-erasure-v1-healfile.go similarity index 100% rename from xl-v1-healfile.go rename to xl-erasure-v1-healfile.go diff --git a/xl-v1-metadata.go b/xl-erasure-v1-metadata.go similarity index 100% rename from xl-v1-metadata.go rename to xl-erasure-v1-metadata.go diff --git a/xl-v1-readfile.go b/xl-erasure-v1-readfile.go similarity index 100% rename from xl-v1-readfile.go rename to xl-erasure-v1-readfile.go diff --git a/xl-v1-utils.go b/xl-erasure-v1-utils.go similarity index 100% rename from xl-v1-utils.go rename to xl-erasure-v1-utils.go diff --git a/xl-v1-waitcloser.go b/xl-erasure-v1-waitcloser.go similarity index 100% rename from xl-v1-waitcloser.go rename to xl-erasure-v1-waitcloser.go diff --git a/xl-v1.go b/xl-erasure-v1.go similarity index 98% rename from xl-v1.go rename to xl-erasure-v1.go index c70ae3caf..82e82f3ee 100644 --- a/xl-v1.go +++ b/xl-erasure-v1.go @@ -83,7 +83,7 @@ func newXL(disks ...string) (StorageAPI, error) { storageDisks := make([]StorageAPI, len(disks)) for index, disk := range disks { var err error - storageDisks[index], err = newFS(disk) + storageDisks[index], err = newPosix(disk) if err != nil { return nil, err } @@ -573,6 +573,7 @@ func (xl XL) DeleteFile(volume, path string) error { // RenameFile - rename file. func (xl XL) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error { + // Validate inputs. if !isValidVolname(srcVolume) { return errInvalidArgument } @@ -587,6 +588,12 @@ func (xl XL) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error { } for _, disk := range xl.storageDisks { if err := disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath); err != nil { + log.WithFields(logrus.Fields{ + "srcVolume": srcVolume, + "srcPath": srcPath, + "dstVolume": dstVolume, + "dstPath": dstPath, + }).Errorf("RenameFile failed with %s", err) return err } } diff --git a/xl-objects-multipart.go b/xl-objects-multipart.go new file mode 100644 index 000000000..b69cb3516 --- /dev/null +++ b/xl-objects-multipart.go @@ -0,0 +1,531 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "path" + "strconv" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/skyrings/skyring-common/tools/uuid" +) + +// listLeafEntries - lists all entries if a given prefixPath is a leaf +// directory, returns error if any - returns empty list if prefixPath +// is not a leaf directory. +func (xl xlObjects) listLeafEntries(prefixPath string) (entries []FileInfo, e error) { + var markerPath string + for { + fileInfos, eof, err := xl.storage.ListFiles(minioMetaVolume, prefixPath, markerPath, false, 1000) + if err != nil { + log.WithFields(logrus.Fields{ + "prefixPath": prefixPath, + "markerPath": markerPath, + }).Errorf("%s", err) + return nil, err + } + for _, fileInfo := range fileInfos { + // Set marker for next batch of ListFiles. + markerPath = fileInfo.Name + if fileInfo.Mode.IsDir() { + // If a directory is found, doesn't return anything. + return nil, nil + } + fileName := path.Base(fileInfo.Name) + if !strings.Contains(fileName, ".") { + // Skip the entry if it is of the pattern bucket/object/uploadID.partNum.md5sum + // and retain entries of the pattern bucket/object/uploadID + entries = append(entries, fileInfo) + } + } + if eof { + break + } + } + return entries, nil +} + +// listMetaVolumeFiles - list all files at a given prefix inside minioMetaVolume. +func (xl xlObjects) listMetaVolumeFiles(prefixPath string, markerPath string, recursive bool, maxKeys int) (allFileInfos []FileInfo, eof bool, err error) { + // newMaxKeys tracks the size of entries which are going to be + // returned back. + var newMaxKeys int + + // Following loop gathers and filters out special files inside + // minio meta volume. + for { + var fileInfos []FileInfo + // List files up to maxKeys-newMaxKeys, since we are skipping entries for special files. + fileInfos, eof, err = xl.storage.ListFiles(minioMetaVolume, prefixPath, markerPath, recursive, maxKeys-newMaxKeys) + if err != nil { + log.WithFields(logrus.Fields{ + "prefixPath": prefixPath, + "markerPath": markerPath, + "recursive": recursive, + "maxKeys": maxKeys, + }).Errorf("%s", err) + return nil, true, err + } + // Loop through and validate individual file. + for _, fi := range fileInfos { + var entries []FileInfo + if fi.Mode.IsDir() { + // List all the entries if fi.Name is a leaf directory, if + // fi.Name is not a leaf directory then the resulting + // entries are empty. + entries, err = xl.listLeafEntries(fi.Name) + if err != nil { + log.WithFields(logrus.Fields{ + "prefixPath": fi.Name, + }).Errorf("%s", err) + return nil, false, err + } + } + // Set markerPath for next batch of listing. + markerPath = fi.Name + if len(entries) > 0 { + // We reach here for non-recursive case and a leaf entry. + for _, entry := range entries { + allFileInfos = append(allFileInfos, entry) + newMaxKeys++ + // If we have reached the maxKeys, it means we have listed + // everything that was requested. Return right here. + if newMaxKeys == maxKeys { + // Return values: + // allFileInfos : "maxKeys" number of entries. + // eof : eof returned by xl.storage.ListFiles() + // error : nil + return + } + } + } else { + // We reach here for a non-recursive case non-leaf entry + // OR recursive case with fi.Name matching pattern bucket/object/uploadID[.partNum.md5sum] + if !fi.Mode.IsDir() { // Do not skip non-recursive case directory entries. + // Skip files matching pattern bucket/object/uploadID.partNum.md5sum + // and retain files matching pattern bucket/object/uploadID + specialFile := path.Base(fi.Name) + if strings.Contains(specialFile, ".") { + // Contains partnumber and md5sum info, skip this. + continue + } + } + } + allFileInfos = append(allFileInfos, fi) + newMaxKeys++ + // If we have reached the maxKeys, it means we have listed + // everything that was requested. Return right here. + if newMaxKeys == maxKeys { + // Return values: + // allFileInfos : "maxKeys" number of entries. + // eof : eof returned by xl.storage.ListFiles() + // error : nil + return + } + } + // If we have reached eof then we break out. + if eof { + break + } + } + + // Return entries here. + return allFileInfos, eof, nil +} + +// ListMultipartUploads - list multipart uploads. +func (xl xlObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) { + result := ListMultipartsInfo{} + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ListMultipartsInfo{}, (BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectPrefix(prefix) { + return ListMultipartsInfo{}, (ObjectNameInvalid{Bucket: bucket, Object: prefix}) + } + // Verify if delimiter is anything other than '/', which we do not support. + if delimiter != "" && delimiter != slashSeparator { + return ListMultipartsInfo{}, (UnsupportedDelimiter{ + Delimiter: delimiter, + }) + } + // Verify if marker has prefix. + if keyMarker != "" && !strings.HasPrefix(keyMarker, prefix) { + return ListMultipartsInfo{}, (InvalidMarkerPrefixCombination{ + Marker: keyMarker, + Prefix: prefix, + }) + } + if uploadIDMarker != "" { + if strings.HasSuffix(keyMarker, slashSeparator) { + return result, (InvalidUploadIDKeyCombination{ + UploadIDMarker: uploadIDMarker, + KeyMarker: keyMarker, + }) + } + id, err := uuid.Parse(uploadIDMarker) + if err != nil { + return result, err + } + if id.IsZero() { + return result, (MalformedUploadID{ + UploadID: uploadIDMarker, + }) + } + } + + recursive := true + if delimiter == slashSeparator { + recursive = false + } + + result.IsTruncated = true + result.MaxUploads = maxUploads + + // Not using path.Join() as it strips off the trailing '/'. + // Also bucket should always be followed by '/' even if prefix is empty. + prefixPath := pathJoin(bucket, prefix) + keyMarkerPath := "" + if keyMarker != "" { + keyMarkerPath = pathJoin(pathJoin(bucket, keyMarker), uploadIDMarker) + } + // List all the multipart files at prefixPath, starting with marker keyMarkerPath. + fileInfos, eof, err := xl.listMetaVolumeFiles(prefixPath, keyMarkerPath, recursive, maxUploads) + if err != nil { + log.WithFields(logrus.Fields{ + "prefixPath": prefixPath, + "markerPath": keyMarkerPath, + "recursive": recursive, + "maxUploads": maxUploads, + }).Errorf("listMetaVolumeFiles failed with %s", err) + return ListMultipartsInfo{}, err + } + + // Loop through all the received files fill in the multiparts result. + for _, fi := range fileInfos { + var objectName string + var uploadID string + if fi.Mode.IsDir() { + // All directory entries are common prefixes. + uploadID = "" // Upload ids are empty for CommonPrefixes. + objectName = strings.TrimPrefix(fi.Name, retainSlash(bucket)) + result.CommonPrefixes = append(result.CommonPrefixes, objectName) + } else { + uploadID = path.Base(fi.Name) + objectName = strings.TrimPrefix(path.Dir(fi.Name), retainSlash(bucket)) + result.Uploads = append(result.Uploads, uploadMetadata{ + Object: objectName, + UploadID: uploadID, + Initiated: fi.ModTime, + }) + } + result.NextKeyMarker = objectName + result.NextUploadIDMarker = uploadID + } + result.IsTruncated = !eof + if !result.IsTruncated { + result.NextKeyMarker = "" + result.NextUploadIDMarker = "" + } + return result, nil +} + +func (xl xlObjects) NewMultipartUpload(bucket, object string) (string, error) { + // Verify if bucket name is valid. + if !IsValidBucketName(bucket) { + return "", (BucketNameInvalid{Bucket: bucket}) + } + // Verify if object name is valid. + if !IsValidObjectName(object) { + return "", ObjectNameInvalid{Bucket: bucket, Object: object} + } + // Verify whether the bucket exists. + if isExist, err := xl.isBucketExist(bucket); err != nil { + return "", err + } else if !isExist { + return "", BucketNotFound{Bucket: bucket} + } + + if _, err := xl.storage.StatVol(minioMetaVolume); err != nil { + if err == errVolumeNotFound { + err = xl.storage.MakeVol(minioMetaVolume) + if err != nil { + return "", toObjectErr(err) + } + } + } + for { + uuid, err := uuid.New() + if err != nil { + return "", err + } + uploadID := uuid.String() + uploadIDPath := path.Join(bucket, object, uploadID) + if _, err = xl.storage.StatFile(minioMetaVolume, uploadIDPath); err != nil { + if err != errFileNotFound { + return "", (toObjectErr(err, minioMetaVolume, uploadIDPath)) + } + // uploadIDPath doesn't exist, so create empty file to reserve the name + var w io.WriteCloser + if w, err = xl.storage.CreateFile(minioMetaVolume, uploadIDPath); err == nil { + // Close the writer. + if err = w.Close(); err != nil { + return "", err + } + } else { + return "", toObjectErr(err, minioMetaVolume, uploadIDPath) + } + return uploadID, nil + } + // uploadIDPath already exists. + // loop again to try with different uuid generated. + } +} + +// isUploadIDExists - verify if a given uploadID exists and is valid. +func (xl xlObjects) isUploadIDExists(bucket, object, uploadID string) (bool, error) { + uploadIDPath := path.Join(bucket, object, uploadID) + st, err := xl.storage.StatFile(minioMetaVolume, uploadIDPath) + if err != nil { + // Upload id does not exist. + if err == errFileNotFound { + return false, nil + } + return false, err + } + // Upload id exists and is a regular file. + return st.Mode.IsRegular(), nil +} + +// PutObjectPart - writes the multipart upload chunks. +func (xl xlObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return "", BucketNameInvalid{Bucket: bucket} + } + if !IsValidObjectName(object) { + return "", ObjectNameInvalid{Bucket: bucket, Object: object} + } + // Verify whether the bucket exists. + if isExist, err := xl.isBucketExist(bucket); err != nil { + return "", err + } else if !isExist { + return "", BucketNotFound{Bucket: bucket} + } + + if status, err := xl.isUploadIDExists(bucket, object, uploadID); err != nil { + return "", err + } else if !status { + return "", InvalidUploadID{UploadID: uploadID} + } + + partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partID, md5Hex) + fileWriter, err := xl.storage.CreateFile(minioMetaVolume, path.Join(bucket, object, partSuffix)) + if err != nil { + return "", toObjectErr(err, bucket, object) + } + + // Initialize md5 writer. + md5Writer := md5.New() + + // Instantiate a new multi writer. + multiWriter := io.MultiWriter(md5Writer, fileWriter) + + // Instantiate checksum hashers and create a multiwriter. + if size > 0 { + if _, err = io.CopyN(multiWriter, data, size); err != nil { + safeCloseAndRemove(fileWriter) + return "", (toObjectErr(err)) + } + // Reader shouldn't have more data what mentioned in size argument. + // reading one more byte from the reader to validate it. + // expected to fail, success validates existence of more data in the reader. + if _, err = io.CopyN(ioutil.Discard, data, 1); err == nil { + safeCloseAndRemove(fileWriter) + return "", (UnExpectedDataSize{Size: int(size)}) + } + } else { + if _, err = io.Copy(multiWriter, data); err != nil { + safeCloseAndRemove(fileWriter) + return "", (toObjectErr(err)) + } + } + + newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + if md5Hex != "" { + if newMD5Hex != md5Hex { + safeCloseAndRemove(fileWriter) + return "", (BadDigest{md5Hex, newMD5Hex}) + } + } + err = fileWriter.Close() + if err != nil { + return "", err + } + return newMD5Hex, nil +} + +func (xl xlObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ListPartsInfo{}, (BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return ListPartsInfo{}, (ObjectNameInvalid{Bucket: bucket, Object: object}) + } + if status, err := xl.isUploadIDExists(bucket, object, uploadID); err != nil { + return ListPartsInfo{}, err + } else if !status { + return ListPartsInfo{}, (InvalidUploadID{UploadID: uploadID}) + } + result := ListPartsInfo{} + var markerPath string + nextPartNumberMarker := 0 + uploadIDPath := path.Join(bucket, object, uploadID) + // Figure out the marker for the next subsequent calls, if the + // partNumberMarker is already set. + if partNumberMarker > 0 { + partNumberMarkerPath := uploadIDPath + "." + strconv.Itoa(partNumberMarker) + "." + fileInfos, _, err := xl.storage.ListFiles(minioMetaVolume, partNumberMarkerPath, "", false, 1) + if err != nil { + return result, toObjectErr(err, minioMetaVolume, partNumberMarkerPath) + } + if len(fileInfos) == 0 { + return result, (InvalidPart{}) + } + markerPath = fileInfos[0].Name + } + uploadIDPrefix := uploadIDPath + "." + fileInfos, eof, err := xl.storage.ListFiles(minioMetaVolume, uploadIDPrefix, markerPath, false, maxParts) + if err != nil { + return result, InvalidPart{} + } + for _, fileInfo := range fileInfos { + fileName := path.Base(fileInfo.Name) + splitResult := strings.Split(fileName, ".") + partNum, err := strconv.Atoi(splitResult[1]) + if err != nil { + return result, err + } + md5sum := splitResult[2] + result.Parts = append(result.Parts, partInfo{ + PartNumber: partNum, + LastModified: fileInfo.ModTime, + ETag: md5sum, + Size: fileInfo.Size, + }) + nextPartNumberMarker = partNum + } + result.Bucket = bucket + result.Object = object + result.UploadID = uploadID + result.PartNumberMarker = partNumberMarker + result.NextPartNumberMarker = nextPartNumberMarker + result.MaxParts = maxParts + result.IsTruncated = !eof + return result, nil +} + +func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return "", (BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return "", (ObjectNameInvalid{ + Bucket: bucket, + Object: object, + }) + } + if status, err := xl.isUploadIDExists(bucket, object, uploadID); err != nil { + return "", err + } else if !status { + return "", (InvalidUploadID{UploadID: uploadID}) + } + + fileWriter, err := xl.storage.CreateFile(bucket, object) + if err != nil { + return "", toObjectErr(err, bucket, object) + } + + var md5Sums []string + for _, part := range parts { + // Construct part suffix. + partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, part.PartNumber, part.ETag) + err = xl.storage.RenameFile(minioMetaVolume, path.Join(bucket, object, partSuffix), bucket, path.Join(object, fmt.Sprint(part.PartNumber))) + if err != nil { + return "", err + } + md5Sums = append(md5Sums, part.ETag) + } + + err = fileWriter.Close() + if err != nil { + return "", err + } + + // Save the s3 md5. + s3MD5, err := makeS3MD5(md5Sums...) + if err != nil { + return "", err + } + + // Return md5sum. + return s3MD5, nil +} + +// AbortMultipartUpload - abort multipart upload. +func (xl xlObjects) AbortMultipartUpload(bucket, object, uploadID string) error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return (BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return (ObjectNameInvalid{Bucket: bucket, Object: object}) + } + if status, err := xl.isUploadIDExists(bucket, object, uploadID); err != nil { + return err + } else if !status { + return (InvalidUploadID{UploadID: uploadID}) + } + + markerPath := "" + for { + uploadIDPath := path.Join(bucket, object, uploadID) + fileInfos, eof, err := xl.storage.ListFiles(minioMetaVolume, uploadIDPath, markerPath, false, 1000) + if err != nil { + if err == errFileNotFound { + return (InvalidUploadID{UploadID: uploadID}) + } + return toObjectErr(err) + } + for _, fileInfo := range fileInfos { + xl.storage.DeleteFile(minioMetaVolume, fileInfo.Name) + markerPath = fileInfo.Name + } + if eof { + break + } + } + return nil +} diff --git a/object-api.go b/xl-objects.go similarity index 53% rename from object-api.go rename to xl-objects.go index 2027207d8..5f88707d9 100644 --- a/object-api.go +++ b/xl-objects.go @@ -27,26 +27,30 @@ import ( "strings" "github.com/minio/minio/pkg/mimedb" - "github.com/minio/minio/pkg/probe" - "github.com/minio/minio/pkg/safe" ) const ( multipartMetaFile = "multipart.json" ) -type objectAPI struct { +// xlObjects - Implements fs object layer. +type xlObjects struct { storage StorageAPI } -func newObjectLayer(storage StorageAPI) objectAPI { - return objectAPI{storage} +// newXLObjects - initialize new xl object layer. +func newXLObjects(exportPaths ...string) (ObjectLayer, error) { + storage, err := newXL(exportPaths...) + if err != nil { + return nil, err + } + return xlObjects{storage}, nil } // checks whether bucket exists. -func (o objectAPI) isBucketExist(bucketName string) (bool, error) { +func (xl xlObjects) isBucketExist(bucketName string) (bool, error) { // Check whether bucket exists. - if _, e := o.storage.StatVol(bucketName); e != nil { + if _, e := xl.storage.StatVol(bucketName); e != nil { if e == errVolumeNotFound { return false, nil } @@ -58,35 +62,35 @@ func (o objectAPI) isBucketExist(bucketName string) (bool, error) { /// Bucket operations // MakeBucket - make a bucket. -func (o objectAPI) MakeBucket(bucket string) *probe.Error { +func (xl xlObjects) MakeBucket(bucket string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) + return BucketNameInvalid{Bucket: bucket} } - if e := o.storage.MakeVol(bucket); e != nil { - return probe.NewError(toObjectErr(e, bucket)) + if err := xl.storage.MakeVol(bucket); err != nil { + return toObjectErr(err, bucket) } // This happens for the first time, but keep this here since this // is the only place where it can be made expensive optimizing all // other calls. // Create minio meta volume, if it doesn't exist yet. - if e := o.storage.MakeVol(minioMetaVolume); e != nil { - if e != errVolumeExists { - return probe.NewError(toObjectErr(e, minioMetaVolume)) + if err := xl.storage.MakeVol(minioMetaVolume); err != nil { + if err != errVolumeExists { + return toObjectErr(err, minioMetaVolume) } } return nil } // GetBucketInfo - get bucket info. -func (o objectAPI) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) { +func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return BucketInfo{}, BucketNameInvalid{Bucket: bucket} } - vi, e := o.storage.StatVol(bucket) - if e != nil { - return BucketInfo{}, probe.NewError(toObjectErr(e, bucket)) + vi, err := xl.storage.StatVol(bucket) + if err != nil { + return BucketInfo{}, toObjectErr(err, bucket) } return BucketInfo{ Name: bucket, @@ -96,19 +100,12 @@ func (o objectAPI) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) { }, nil } -// byBucketName is a collection satisfying sort.Interface. -type byBucketName []BucketInfo - -func (d byBucketName) Len() int { return len(d) } -func (d byBucketName) Swap(i, j int) { d[i], d[j] = d[j], d[i] } -func (d byBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name } - // ListBuckets - list buckets. -func (o objectAPI) ListBuckets() ([]BucketInfo, *probe.Error) { +func (xl xlObjects) ListBuckets() ([]BucketInfo, error) { var bucketInfos []BucketInfo - vols, e := o.storage.ListVols() - if e != nil { - return nil, probe.NewError(toObjectErr(e)) + vols, err := xl.storage.ListVols() + if err != nil { + return nil, toObjectErr(err) } for _, vol := range vols { // StorageAPI can send volume names which are incompatible @@ -128,13 +125,13 @@ func (o objectAPI) ListBuckets() ([]BucketInfo, *probe.Error) { } // DeleteBucket - delete a bucket. -func (o objectAPI) DeleteBucket(bucket string) *probe.Error { +func (xl xlObjects) DeleteBucket(bucket string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) + return BucketNameInvalid{Bucket: bucket} } - if e := o.storage.DeleteVol(bucket); e != nil { - return probe.NewError(toObjectErr(e)) + if err := xl.storage.DeleteVol(bucket); err != nil { + return toObjectErr(err) } return nil } @@ -142,12 +139,12 @@ func (o objectAPI) DeleteBucket(bucket string) *probe.Error { /// Object Operations // GetObject - get an object. -func (o objectAPI) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) { +func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) { findPathOffset := func() (i int, partOffset int64, err error) { partOffset = startOffset for i = 1; i < 10000; i++ { var fileInfo FileInfo - fileInfo, err = o.storage.StatFile(bucket, pathJoin(object, fmt.Sprint(i))) + fileInfo, err = xl.storage.StatFile(bucket, pathJoin(object, fmt.Sprint(i))) if err != nil { if err == errFileNotFound { continue @@ -167,36 +164,31 @@ func (o objectAPI) GetObject(bucket, object string, startOffset int64) (io.ReadC // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return nil, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return nil, BucketNameInvalid{Bucket: bucket} } // Verify if object is valid. if !IsValidObjectName(object) { - return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return nil, ObjectNameInvalid{Bucket: bucket, Object: object} } - _, err := o.storage.StatFile(bucket, object) - if err == nil { - fmt.Println("1", err) - r, e := o.storage.ReadFile(bucket, object, startOffset) - if e != nil { - fmt.Println("1.5", err) - return nil, probe.NewError(toObjectErr(e, bucket, object)) + if _, err := xl.storage.StatFile(bucket, pathJoin(object, multipartMetaFile)); err != nil { + if _, err = xl.storage.StatFile(bucket, object); err == nil { + var reader io.ReadCloser + reader, err = xl.storage.ReadFile(bucket, object, startOffset) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + return reader, nil } - return r, nil - } - _, err = o.storage.StatFile(bucket, pathJoin(object, multipartMetaFile)) - if err != nil { - fmt.Println("2", err) - return nil, probe.NewError(toObjectErr(err, bucket, object)) + return nil, toObjectErr(err, bucket, object) } fileReader, fileWriter := io.Pipe() partNum, offset, err := findPathOffset() if err != nil { - fmt.Println("3", err) - return nil, probe.NewError(toObjectErr(err, bucket, object)) + return nil, toObjectErr(err, bucket, object) } go func() { for ; partNum < 10000; partNum++ { - r, err := o.storage.ReadFile(bucket, pathJoin(object, fmt.Sprint(partNum)), offset) + r, err := xl.storage.ReadFile(bucket, pathJoin(object, fmt.Sprint(partNum)), offset) if err != nil { if err == errFileNotFound { continue @@ -215,10 +207,10 @@ func (o objectAPI) GetObject(bucket, object string, startOffset int64) (io.ReadC } // GetObjectInfo - get object info. -func (o objectAPI) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) { +func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { getMultpartFileSize := func() (size int64) { for i := 0; i < 10000; i++ { - fi, err := o.storage.StatFile(bucket, pathJoin(object, fmt.Sprint(i))) + fi, err := xl.storage.StatFile(bucket, pathJoin(object, fmt.Sprint(i))) if err != nil { continue } @@ -228,17 +220,17 @@ func (o objectAPI) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Erro } // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return ObjectInfo{}, BucketNameInvalid{Bucket: bucket} } // Verify if object is valid. if !IsValidObjectName(object) { - return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object} } - fi, e := o.storage.StatFile(bucket, object) - if e != nil { - fi, e = o.storage.StatFile(bucket, pathJoin(object, multipartMetaFile)) - if e != nil { - return ObjectInfo{}, probe.NewError(toObjectErr(e, bucket, object)) + fi, err := xl.storage.StatFile(bucket, object) + if err != nil { + fi, err = xl.storage.StatFile(bucket, pathJoin(object, multipartMetaFile)) + if err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) } fi.Size = getMultpartFileSize() } @@ -260,44 +252,29 @@ func (o objectAPI) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Erro }, nil } -// safeCloseAndRemove - safely closes and removes underlying temporary -// file writer if possible. -func safeCloseAndRemove(writer io.WriteCloser) error { - // If writer is a safe file, Attempt to close and remove. - safeWriter, ok := writer.(*safe.File) - if ok { - return safeWriter.CloseAndRemove() - } - pipeWriter, ok := writer.(*io.PipeWriter) - if ok { - return pipeWriter.CloseWithError(errors.New("Close and error out.")) - } - return nil -} - -func (o objectAPI) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, *probe.Error) { +func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return "", probe.NewError(BucketNameInvalid{Bucket: bucket}) + return "", (BucketNameInvalid{Bucket: bucket}) } if !IsValidObjectName(object) { - return "", probe.NewError(ObjectNameInvalid{ + return "", (ObjectNameInvalid{ Bucket: bucket, Object: object, }) } // Check whether the bucket exists. - isExist, err := o.isBucketExist(bucket) + isExist, err := xl.isBucketExist(bucket) if err != nil { - return "", probe.NewError(err) + return "", err } if !isExist { - return "", probe.NewError(BucketNotFound{Bucket: bucket}) + return "", BucketNotFound{Bucket: bucket} } - fileWriter, e := o.storage.CreateFile(bucket, object) - if e != nil { - return "", probe.NewError(toObjectErr(e, bucket, object)) + fileWriter, err := xl.storage.CreateFile(bucket, object) + if err != nil { + return "", toObjectErr(err, bucket, object) } // Initialize md5 writer. @@ -308,18 +285,18 @@ func (o objectAPI) PutObject(bucket string, object string, size int64, data io.R // Instantiate checksum hashers and create a multiwriter. if size > 0 { - if _, e = io.CopyN(multiWriter, data, size); e != nil { + if _, err = io.CopyN(multiWriter, data, size); err != nil { if clErr := safeCloseAndRemove(fileWriter); clErr != nil { - return "", probe.NewError(clErr) + return "", clErr } - return "", probe.NewError(toObjectErr(e)) + return "", toObjectErr(err) } } else { - if _, e = io.Copy(multiWriter, data); e != nil { + if _, err = io.Copy(multiWriter, data); err != nil { if clErr := safeCloseAndRemove(fileWriter); clErr != nil { - return "", probe.NewError(clErr) + return "", clErr } - return "", probe.NewError(e) + return "", err } } @@ -331,56 +308,56 @@ func (o objectAPI) PutObject(bucket string, object string, size int64, data io.R } if md5Hex != "" { if newMD5Hex != md5Hex { - if e = safeCloseAndRemove(fileWriter); e != nil { - return "", probe.NewError(e) + if err = safeCloseAndRemove(fileWriter); err != nil { + return "", err } - return "", probe.NewError(BadDigest{md5Hex, newMD5Hex}) + return "", BadDigest{md5Hex, newMD5Hex} } } - e = fileWriter.Close() - if e != nil { - return "", probe.NewError(e) + err = fileWriter.Close() + if err != nil { + return "", err } // Return md5sum, successfully wrote object. return newMD5Hex, nil } -func (o objectAPI) DeleteObject(bucket, object string) *probe.Error { +func (xl xlObjects) DeleteObject(bucket, object string) error { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) + return BucketNameInvalid{Bucket: bucket} } if !IsValidObjectName(object) { - return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + return ObjectNameInvalid{Bucket: bucket, Object: object} } - if e := o.storage.DeleteFile(bucket, object); e != nil { - return probe.NewError(toObjectErr(e, bucket, object)) + if err := xl.storage.DeleteFile(bucket, object); err != nil { + return toObjectErr(err, bucket, object) } return nil } -func (o objectAPI) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error) { +func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { - return ListObjectsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket} } if !IsValidObjectPrefix(prefix) { - return ListObjectsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) + return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix} } // Verify if delimiter is anything other than '/', which we do not support. if delimiter != "" && delimiter != slashSeparator { - return ListObjectsInfo{}, probe.NewError(UnsupportedDelimiter{ + return ListObjectsInfo{}, UnsupportedDelimiter{ Delimiter: delimiter, - }) + } } // Verify if marker has prefix. if marker != "" { if !strings.HasPrefix(marker, prefix) { - return ListObjectsInfo{}, probe.NewError(InvalidMarkerPrefixCombination{ + return ListObjectsInfo{}, InvalidMarkerPrefixCombination{ Marker: marker, Prefix: prefix, - }) + } } } @@ -389,9 +366,9 @@ func (o objectAPI) ListObjects(bucket, prefix, marker, delimiter string, maxKeys if delimiter == slashSeparator { recursive = false } - fileInfos, eof, e := o.storage.ListFiles(bucket, prefix, marker, recursive, maxKeys) - if e != nil { - return ListObjectsInfo{}, probe.NewError(toObjectErr(e, bucket)) + fileInfos, eof, err := xl.storage.ListFiles(bucket, prefix, marker, recursive, maxKeys) + if err != nil { + return ListObjectsInfo{}, toObjectErr(err, bucket) } if maxKeys == 0 { return ListObjectsInfo{}, nil