From 3ddd8b04d170a7d3cdf98694648a99f8802b5678 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Tue, 30 Mar 2021 23:19:36 -0700 Subject: [PATCH] fix: handle unsupported APIs more granularly (#11674) --- cmd/admin-handlers.go | 14 ++- cmd/api-router.go | 232 ++++++++++++++++++++++++++++------------ cmd/auth-handler.go | 2 + cmd/generic-handlers.go | 106 ++---------------- cmd/http-stats.go | 18 +++- cmd/metrics-v2.go | 121 +++++++++++++++------ cmd/routers.go | 3 - 7 files changed, 285 insertions(+), 211 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 349a77334..4a585554c 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -259,11 +259,15 @@ type ServerHTTPAPIStats struct { // ServerHTTPStats holds all type of http operations performed to/from the server // including their average execution time. type ServerHTTPStats struct { - S3RequestsInQueue int32 `json:"s3RequestsInQueue"` - CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"` - TotalS3Requests ServerHTTPAPIStats `json:"totalS3Requests"` - TotalS3Errors ServerHTTPAPIStats `json:"totalS3Errors"` - TotalS3Canceled ServerHTTPAPIStats `json:"totalS3Canceled"` + S3RequestsInQueue int32 `json:"s3RequestsInQueue"` + CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"` + TotalS3Requests ServerHTTPAPIStats `json:"totalS3Requests"` + TotalS3Errors ServerHTTPAPIStats `json:"totalS3Errors"` + TotalS3Canceled ServerHTTPAPIStats `json:"totalS3Canceled"` + TotalS3RejectedAuth uint64 `json:"totalS3RejectedAuth"` + TotalS3RejectedTime uint64 `json:"totalS3RejectedTime"` + TotalS3RejectedHeader uint64 `json:"totalS3RejectedHeader"` + TotalS3RejectedInvalid uint64 `json:"totalS3RejectedInvalid"` } // ServerInfoData holds storage, connections and other diff --git a/cmd/api-router.go b/cmd/api-router.go index 88ba6e16f..870b72b12 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -78,6 +78,103 @@ func getHost(r *http.Request) string { return r.Host } +func notImplementedHandler(w http.ResponseWriter, r *http.Request) { + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) +} + +type rejectedAPI struct { + api string + methods []string + queries []string + path string +} + +var rejectedAPIs = []rejectedAPI{ + { + api: "inventory", + methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete}, + queries: []string{"inventory", ""}, + }, + { + api: "cors", + methods: []string{http.MethodPut, http.MethodDelete}, + queries: []string{"cors", ""}, + }, + { + api: "metrics", + methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete}, + queries: []string{"metrics", ""}, + }, + { + api: "website", + methods: []string{http.MethodPut}, + queries: []string{"website", ""}, + }, + { + api: "logging", + methods: []string{http.MethodPut, http.MethodDelete}, + queries: []string{"logging", ""}, + }, + { + api: "accelerate", + methods: []string{http.MethodPut, http.MethodDelete}, + queries: []string{"accelerate", ""}, + }, + { + api: "requestPayment", + methods: []string{http.MethodPut, http.MethodDelete}, + queries: []string{"requestPayment", ""}, + }, + { + api: "torrent", + methods: []string{http.MethodPut, http.MethodDelete, http.MethodGet}, + queries: []string{"torrent", ""}, + path: "/{object:.+}", + }, + { + api: "acl", + methods: []string{http.MethodDelete}, + queries: []string{"acl", ""}, + path: "/{object:.+}", + }, + { + api: "acl", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodHead}, + queries: []string{"acl", ""}, + }, + { + api: "publicAccessBlock", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodGet}, + queries: []string{"publicAccessBlock", ""}, + }, + { + api: "ownershipControls", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodGet}, + queries: []string{"ownershipControls", ""}, + }, + { + api: "intelligent-tiering", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodGet}, + queries: []string{"intelligent-tiering", ""}, + }, + { + api: "analytics", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodGet}, + queries: []string{"analytics", ""}, + }, +} + +func rejectUnsupportedAPIs(router *mux.Router) { + for _, r := range rejectedAPIs { + t := router.Methods(r.methods...). + HandlerFunc(collectAPIStats(r.api, httpTraceAll(notImplementedHandler))). + Queries(r.queries...) + if r.path != "" { + t.Path(r.path) + } + } +} + // registerAPIRouter - registers S3 compatible APIs. func registerAPIRouter(router *mux.Router) { // Initialize API. @@ -116,221 +213,222 @@ func registerAPIRouter(router *mux.Router) { } routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) - for _, bucket := range routers { + for _, router := range routers { + rejectUnsupportedAPIs(router) // Object operations // HeadObject - bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc( collectAPIStats("headobject", maxClients(httpTraceAll(api.HeadObjectHandler)))) // CopyObjectPart - bucket.Methods(http.MethodPut).Path("/{object:.+}"). + router.Methods(http.MethodPut).Path("/{object:.+}"). HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?"). HandlerFunc(collectAPIStats("copyobjectpart", maxClients(httpTraceAll(api.CopyObjectPartHandler)))). Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // PutObjectPart - bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( collectAPIStats("putobjectpart", maxClients(httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // ListObjectParts - bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( collectAPIStats("listobjectparts", maxClients(httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}") // CompleteMultipartUpload - bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( collectAPIStats("completemutipartupload", maxClients(httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload - bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( collectAPIStats("newmultipartupload", maxClients(httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "") // AbortMultipartUpload - bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( collectAPIStats("abortmultipartupload", maxClients(httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}") // GetObjectACL - this is a dummy call. - bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( collectAPIStats("getobjectacl", maxClients(httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "") // PutObjectACL - this is a dummy call. - bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( collectAPIStats("putobjectacl", maxClients(httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "") // GetObjectTagging - bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( collectAPIStats("getobjecttagging", maxClients(httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "") // PutObjectTagging - bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( collectAPIStats("putobjecttagging", maxClients(httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "") // DeleteObjectTagging - bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( collectAPIStats("deleteobjecttagging", maxClients(httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "") // SelectObjectContent - bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( collectAPIStats("selectobjectcontent", maxClients(httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2") // GetObjectRetention - bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( collectAPIStats("getobjectretention", maxClients(httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "") // GetObjectLegalHold - bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( collectAPIStats("getobjectlegalhold", maxClients(httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "") // GetObject - bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( collectAPIStats("getobject", maxClients(httpTraceHdrs(api.GetObjectHandler)))) // CopyObject - bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc( + router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc( collectAPIStats("copyobject", maxClients(httpTraceAll(api.CopyObjectHandler)))) // PutObjectRetention - bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( collectAPIStats("putobjectretention", maxClients(httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "") // PutObjectLegalHold - bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( collectAPIStats("putobjectlegalhold", maxClients(httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "") // PutObject with auto-extract support for zip - bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc( + router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc( collectAPIStats("putobject", maxClients(httpTraceHdrs(api.PutObjectExtractHandler)))) // PutObject - bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( collectAPIStats("putobject", maxClients(httpTraceHdrs(api.PutObjectHandler)))) // DeleteObject - bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( collectAPIStats("deleteobject", maxClients(httpTraceAll(api.DeleteObjectHandler)))) // PostRestoreObject - bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( + router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( collectAPIStats("restoreobject", maxClients(httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "") /// Bucket operations // GetBucketLocation - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketlocation", maxClients(httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "") // GetBucketPolicy - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketpolicy", maxClients(httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "") // GetBucketLifecycle - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "") // GetBucketEncryption - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketencryption", maxClients(httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "") // GetBucketObjectLockConfig - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketobjectlockconfiguration", maxClients(httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "") // GetBucketReplicationConfig - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketreplicationconfiguration", maxClients(httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "") // GetBucketVersioning - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketversioning", maxClients(httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "") // GetBucketNotification - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketnotification", maxClients(httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "") // ListenNotification - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}") // Dummy Bucket Calls // GetBucketACL -- this is a dummy call. - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketacl", maxClients(httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "") // PutBucketACL -- this is a dummy call. - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucketacl", maxClients(httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "") // GetBucketCors - this is a dummy call. - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketcors", maxClients(httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "") // GetBucketWebsiteHandler - this is a dummy call. - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketwebsite", maxClients(httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "") // GetBucketAccelerateHandler - this is a dummy call. - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketaccelerate", maxClients(httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "") // GetBucketRequestPaymentHandler - this is a dummy call. - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketrequestpayment", maxClients(httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "") // GetBucketLoggingHandler - this is a dummy call. - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketlogging", maxClients(httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "") // GetBucketLifecycleHandler - this is a dummy call. - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "") // GetBucketTaggingHandler - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getbuckettagging", maxClients(httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "") //DeleteBucketWebsiteHandler - bucket.Methods(http.MethodDelete).HandlerFunc( + router.Methods(http.MethodDelete).HandlerFunc( collectAPIStats("deletebucketwebsite", maxClients(httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "") // DeleteBucketTaggingHandler - bucket.Methods(http.MethodDelete).HandlerFunc( + router.Methods(http.MethodDelete).HandlerFunc( collectAPIStats("deletebuckettagging", maxClients(httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "") // ListMultipartUploads - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("listmultipartuploads", maxClients(httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "") // ListObjectsV2M - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("listobjectsv2M", maxClients(httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true") // ListObjectsV2 - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("listobjectsv2", maxClients(httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2") // ListObjectVersions - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("listobjectversions", maxClients(httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "") // GetBucketPolicyStatus - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("getpolicystatus", maxClients(httpTraceAll(api.GetBucketPolicyStatusHandler)))).Queries("policyStatus", "") // PutBucketLifecycle - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucketlifecycle", maxClients(httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "") // PutBucketReplicationConfig - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucketreplicationconfiguration", maxClients(httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "") // GetObjectRetention // PutBucketEncryption - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucketencryption", maxClients(httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "") // PutBucketPolicy - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucketpolicy", maxClients(httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "") // PutBucketObjectLockConfig - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucketobjectlockconfig", maxClients(httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "") // PutBucketTaggingHandler - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbuckettagging", maxClients(httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "") // PutBucketVersioning - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucketversioning", maxClients(httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "") // PutBucketNotification - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucketnotification", maxClients(httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "") // PutBucket - bucket.Methods(http.MethodPut).HandlerFunc( + router.Methods(http.MethodPut).HandlerFunc( collectAPIStats("putbucket", maxClients(httpTraceAll(api.PutBucketHandler)))) // HeadBucket - bucket.Methods(http.MethodHead).HandlerFunc( + router.Methods(http.MethodHead).HandlerFunc( collectAPIStats("headbucket", maxClients(httpTraceAll(api.HeadBucketHandler)))) // PostPolicy - bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc( + router.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc( collectAPIStats("postpolicybucket", maxClients(httpTraceHdrs(api.PostPolicyBucketHandler)))) // DeleteMultipleObjects - bucket.Methods(http.MethodPost).HandlerFunc( + router.Methods(http.MethodPost).HandlerFunc( collectAPIStats("deletemultipleobjects", maxClients(httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "") // DeleteBucketPolicy - bucket.Methods(http.MethodDelete).HandlerFunc( + router.Methods(http.MethodDelete).HandlerFunc( collectAPIStats("deletebucketpolicy", maxClients(httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "") // DeleteBucketReplication - bucket.Methods(http.MethodDelete).HandlerFunc( + router.Methods(http.MethodDelete).HandlerFunc( collectAPIStats("deletebucketreplicationconfiguration", maxClients(httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "") // DeleteBucketLifecycle - bucket.Methods(http.MethodDelete).HandlerFunc( + router.Methods(http.MethodDelete).HandlerFunc( collectAPIStats("deletebucketlifecycle", maxClients(httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "") // DeleteBucketEncryption - bucket.Methods(http.MethodDelete).HandlerFunc( + router.Methods(http.MethodDelete).HandlerFunc( collectAPIStats("deletebucketencryption", maxClients(httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "") // DeleteBucket - bucket.Methods(http.MethodDelete).HandlerFunc( + router.Methods(http.MethodDelete).HandlerFunc( collectAPIStats("deletebucket", maxClients(httpTraceAll(api.DeleteBucketHandler)))) // ListObjectsV1 (Legacy) - bucket.Methods(http.MethodGet).HandlerFunc( + router.Methods(http.MethodGet).HandlerFunc( collectAPIStats("listobjectsv1", maxClients(httpTraceAll(api.ListObjectsV1Handler)))) } diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index 52735286b..d7112cc08 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -28,6 +28,7 @@ import ( "net/http" "strconv" "strings" + "sync/atomic" "time" xhttp "github.com/minio/minio/cmd/http" @@ -505,6 +506,7 @@ func setAuthHandler(h http.Handler) http.Handler { return } writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r)) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1) }) } diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index e32bcbeb2..a3e67ba95 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -20,6 +20,7 @@ import ( "context" "net/http" "strings" + "sync/atomic" "time" "github.com/minio/minio-go/v7/pkg/set" @@ -63,6 +64,7 @@ func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if isHTTPHeaderSizeTooLarge(r.Header) { writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL, guessIsBrowserReq(r)) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1) return } h.ServeHTTP(w, r) @@ -344,6 +346,7 @@ func setTimeValidityHandler(h http.Handler) http.Handler { // header, for all requests where Date header is not // present we will reject such clients. writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r)) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1) return } // Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past @@ -351,6 +354,7 @@ func setTimeValidityHandler(h http.Handler) http.Handler { curTime := UTCNow() if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime { writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL, guessIsBrowserReq(r)) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1) return } } @@ -358,105 +362,6 @@ func setTimeValidityHandler(h http.Handler) http.Handler { }) } -var supportedDummyBucketAPIs = map[string][]string{ - "acl": {http.MethodPut, http.MethodGet}, - "cors": {http.MethodGet}, - "website": {http.MethodGet, http.MethodDelete}, - "logging": {http.MethodGet}, - "accelerate": {http.MethodGet}, - "requestPayment": {http.MethodGet}, -} - -// List of not implemented bucket queries -var notImplementedBucketResourceNames = map[string]struct{}{ - "cors": {}, - "metrics": {}, - "website": {}, - "logging": {}, - "inventory": {}, - "accelerate": {}, - "requestPayment": {}, - "analytics": {}, - "intelligent-tiering": {}, - "ownershipControls": {}, - "publicAccessBlock": {}, -} - -// Checks requests for not implemented Bucket resources -func ignoreNotImplementedBucketResources(req *http.Request) bool { - for name := range req.URL.Query() { - methods, ok := supportedDummyBucketAPIs[name] - if ok { - for _, method := range methods { - if method == req.Method { - return false - } - } - } - - if _, ok := notImplementedBucketResourceNames[name]; ok { - return true - } - } - return false -} - -var supportedDummyObjectAPIs = map[string][]string{ - "acl": {http.MethodGet, http.MethodPut}, -} - -// List of not implemented object APIs -var notImplementedObjectResourceNames = map[string]struct{}{ - "torrent": {}, -} - -// Checks requests for not implemented Object resources -func ignoreNotImplementedObjectResources(req *http.Request) bool { - for name := range req.URL.Query() { - methods, ok := supportedDummyObjectAPIs[name] - if ok { - for _, method := range methods { - if method == req.Method { - return false - } - } - } - if _, ok := notImplementedObjectResourceNames[name]; ok { - return true - } - } - return false -} - -// setIgnoreResourcesHandler - -// Ignore resources handler is wrapper handler used for API request resource validation -// Since we do not support all the S3 queries, it is necessary for us to throw back a -// valid error message indicating that requested feature is not implemented. -func setIgnoreResourcesHandler(h http.Handler) http.Handler { - // Resource handler ServeHTTP() wrapper - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - bucketName, objectName := request2BucketObjectName(r) - - // If bucketName is present and not objectName check for bucket level resource queries. - if bucketName != "" && objectName == "" { - if ignoreNotImplementedBucketResources(r) { - writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) - return - } - } - // If bucketName and objectName are present check for its resource queries. - if bucketName != "" && objectName != "" { - if ignoreNotImplementedObjectResources(r) { - writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) - return - } - } - - // Serve HTTP. - h.ServeHTTP(w, r) - }) -} - // setHttpStatsHandler sets a http Stats handler to gather HTTP statistics func setHTTPStatsHandler(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -517,6 +422,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler { // Check for bad components in URL path. if hasBadPathComponent(r.URL.Path) { writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL, guessIsBrowserReq(r)) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1) return } // Check for bad components in URL query values. @@ -524,12 +430,14 @@ func setRequestValidityHandler(h http.Handler) http.Handler { for _, v := range vv { if hasBadPathComponent(v) { writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL, guessIsBrowserReq(r)) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1) return } } } if hasMultipleAuth(r) { writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r)) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1) return } h.ServeHTTP(w, r) diff --git a/cmd/http-stats.go b/cmd/http-stats.go index bcef1141e..3d55a07c9 100644 --- a/cmd/http-stats.go +++ b/cmd/http-stats.go @@ -137,11 +137,15 @@ func (stats *HTTPAPIStats) Load() map[string]int { // HTTPStats holds statistics information about // HTTP requests made by all clients type HTTPStats struct { - s3RequestsInQueue int32 - currentS3Requests HTTPAPIStats - totalS3Requests HTTPAPIStats - totalS3Errors HTTPAPIStats - totalS3Canceled HTTPAPIStats + s3RequestsInQueue int32 + currentS3Requests HTTPAPIStats + totalS3Requests HTTPAPIStats + totalS3Errors HTTPAPIStats + totalS3Canceled HTTPAPIStats + rejectedRequestsAuth uint64 + rejectedRequestsTime uint64 + rejectedRequestsHeader uint64 + rejectedRequestsInvalid uint64 } func (st *HTTPStats) addRequestsInQueue(i int32) { @@ -152,6 +156,10 @@ func (st *HTTPStats) addRequestsInQueue(i int32) { func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats { serverStats := ServerHTTPStats{} serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue) + serverStats.TotalS3RejectedAuth = atomic.LoadUint64(&st.rejectedRequestsAuth) + serverStats.TotalS3RejectedTime = atomic.LoadUint64(&st.rejectedRequestsTime) + serverStats.TotalS3RejectedHeader = atomic.LoadUint64(&st.rejectedRequestsHeader) + serverStats.TotalS3RejectedInvalid = atomic.LoadUint64(&st.rejectedRequestsInvalid) serverStats.CurrentS3Requests = ServerHTTPAPIStats{ APIStats: st.currentS3Requests.Load(), } diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 5436e48ef..21bc7c02e 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -49,44 +49,49 @@ const ( ) const ( - cacheSubsystem MetricSubsystem = "cache" - capacityRawSubsystem MetricSubsystem = "capacity_raw" - capacityUsableSubsystem MetricSubsystem = "capacity_usable" - diskSubsystem MetricSubsystem = "disk" - fileDescriptorSubsystem MetricSubsystem = "file_descriptor" - goRoutines MetricSubsystem = "go_routine" - ioSubsystem MetricSubsystem = "io" - nodesSubsystem MetricSubsystem = "nodes" - objectsSubsystem MetricSubsystem = "objects" - processSubsystem MetricSubsystem = "process" - replicationSubsystem MetricSubsystem = "replication" - requestsSubsystem MetricSubsystem = "requests" - timeSubsystem MetricSubsystem = "time" - trafficSubsystem MetricSubsystem = "traffic" - softwareSubsystem MetricSubsystem = "software" - sysCallSubsystem MetricSubsystem = "syscall" - usageSubsystem MetricSubsystem = "usage" + cacheSubsystem MetricSubsystem = "cache" + capacityRawSubsystem MetricSubsystem = "capacity_raw" + capacityUsableSubsystem MetricSubsystem = "capacity_usable" + diskSubsystem MetricSubsystem = "disk" + fileDescriptorSubsystem MetricSubsystem = "file_descriptor" + goRoutines MetricSubsystem = "go_routine" + ioSubsystem MetricSubsystem = "io" + nodesSubsystem MetricSubsystem = "nodes" + objectsSubsystem MetricSubsystem = "objects" + processSubsystem MetricSubsystem = "process" + replicationSubsystem MetricSubsystem = "replication" + requestsSubsystem MetricSubsystem = "requests" + requestsRejectedSubsystem MetricSubsystem = "requests_rejected" + timeSubsystem MetricSubsystem = "time" + trafficSubsystem MetricSubsystem = "traffic" + softwareSubsystem MetricSubsystem = "software" + sysCallSubsystem MetricSubsystem = "syscall" + usageSubsystem MetricSubsystem = "usage" ) // MetricName are the individual names for the metric. type MetricName string const ( - total MetricName = "total" - errorsTotal MetricName = "error_total" - canceledTotal MetricName = "canceled_total" - healTotal MetricName = "heal_total" - hitsTotal MetricName = "hits_total" - inflightTotal MetricName = "inflight_total" - limitTotal MetricName = "limit_total" - missedTotal MetricName = "missed_total" - waitingTotal MetricName = "waiting_total" - objectTotal MetricName = "object_total" - offlineTotal MetricName = "offline_total" - onlineTotal MetricName = "online_total" - openTotal MetricName = "open_total" - readTotal MetricName = "read_total" - writeTotal MetricName = "write_total" + authTotal MetricName = "auth_total" + canceledTotal MetricName = "canceled_total" + errorsTotal MetricName = "errors_total" + headerTotal MetricName = "header_total" + healTotal MetricName = "heal_total" + hitsTotal MetricName = "hits_total" + inflightTotal MetricName = "inflight_total" + invalidTotal MetricName = "invalid_total" + limitTotal MetricName = "limit_total" + missedTotal MetricName = "missed_total" + waitingTotal MetricName = "waiting_total" + objectTotal MetricName = "object_total" + offlineTotal MetricName = "offline_total" + onlineTotal MetricName = "online_total" + openTotal MetricName = "open_total" + readTotal MetricName = "read_total" + timestampTotal MetricName = "timestamp_total" + writeTotal MetricName = "write_total" + total MetricName = "total" failedBytes MetricName = "failed_bytes" freeBytes MetricName = "free_bytes" @@ -505,6 +510,42 @@ func getS3RequestsCanceledMD() MetricDescription { Type: counterMetric, } } +func getS3RejectedAuthRequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsRejectedSubsystem, + Name: authTotal, + Help: "Total number S3 requests rejected for auth failure.", + Type: counterMetric, + } +} +func getS3RejectedHeaderRequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsRejectedSubsystem, + Name: headerTotal, + Help: "Total number S3 requests rejected for invalid header.", + Type: counterMetric, + } +} +func getS3RejectedTimestampRequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsRejectedSubsystem, + Name: timestampTotal, + Help: "Total number S3 requests rejected for invalid timestamp.", + Type: counterMetric, + } +} +func getS3RejectedInvalidRequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsRejectedSubsystem, + Name: invalidTotal, + Help: "Total number S3 invalid requests.", + Type: counterMetric, + } +} func getCacheHitsTotalMD() MetricDescription { return MetricDescription{ Namespace: minioNamespace, @@ -1072,6 +1113,22 @@ func getHTTPMetrics() MetricsGroup { len(httpStats.CurrentS3Requests.APIStats)+ len(httpStats.TotalS3Requests.APIStats)+ len(httpStats.TotalS3Errors.APIStats)) + metrics = append(metrics, Metric{ + Description: getS3RejectedAuthRequestsTotalMD(), + Value: float64(httpStats.TotalS3RejectedAuth), + }) + metrics = append(metrics, Metric{ + Description: getS3RejectedTimestampRequestsTotalMD(), + Value: float64(httpStats.TotalS3RejectedTime), + }) + metrics = append(metrics, Metric{ + Description: getS3RejectedHeaderRequestsTotalMD(), + Value: float64(httpStats.TotalS3RejectedHeader), + }) + metrics = append(metrics, Metric{ + Description: getS3RejectedInvalidRequestsTotalMD(), + Value: float64(httpStats.TotalS3RejectedInvalid), + }) metrics = append(metrics, Metric{ Description: getS3RequestsInQueueMD(), Value: float64(httpStats.S3RequestsInQueue), diff --git a/cmd/routers.go b/cmd/routers.go index 174b534c6..f4c510429 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -48,9 +48,6 @@ var globalHandlers = []mux.MiddlewareFunc{ // routes them accordingly. Client receives a HTTP error for // invalid/unsupported signatures. setAuthHandler, - // Validates all incoming URL resources, for invalid/unsupported - // resources client receives a HTTP error. - setIgnoreResourcesHandler, // Validates all incoming requests to have a valid date header. setTimeValidityHandler, // Adds cache control for all browser requests.