scott/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2026-02-04 18:00:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

migrate all bucket metadata into a single file (#9586)

Browse Source 
this is a major overhaul by migrating off all
bucket metadata related configs into a single
object '.metadata.bin' this allows us for faster
bootups across 1000's of buckets and as well
as keeps the code simple enough for future
work and additions.

Additionally also fixes #9396, #9394
This commit is contained in:
Harshavardhana
2020-05-19 13:53:54 -07:00
committed by GitHub
parent 730775bb4e
commit bd032d13ff
48 changed files with 1648 additions and 2845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
cmd/bucket-notification-handlers.go
View File

@@ -19,10 +19,8 @@ package cmd
import (
"encoding/json"
"encoding/xml"
"errors"
"io"
"net/http"
"path"
"reflect"
"time"
@@ -38,8 +36,6 @@ const (
bucketNotificationConfig = "notification.xml"
)
var errNoSuchNotifications = errors.New("The specified bucket does not have bucket notifications")
// GetBucketNotificationHandler - This HTTP handler returns event notification configuration
// as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
// It returns empty configuration if its not set.
@@ -73,64 +69,49 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
return
}
// Construct path to notification.xml for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
var config = event.Config{
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
}
config.SetRegion(globalServerRegion)
var config = event.Config{}
configData, err := readConfig(ctx, objAPI, configFile)
configData, err := globalBucketMetadataSys.GetConfig(bucketName, bucketNotificationConfig)
if err != nil {
if err != errConfigNotFound {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
config.SetRegion(globalServerRegion)
config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
notificationBytes, err := xml.Marshal(config)
if err != nil {
} else {
if err = xml.Unmarshal(configData, &config); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
writeSuccessResponseXML(w, notificationBytes)
return
}
config.SetRegion(globalServerRegion)
if err = xml.Unmarshal(configData, &config); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
arnErr, ok := err.(*event.ErrARNNotFound)
if !ok {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
for i, queue := range config.QueueList {
// Remove ARN not found queues, because we previously allowed
// adding unexpected entries into the config.
//
// With newer config disallowing changing / turning off
// notification targets without removing ARN in notification
// configuration we won't see this problem anymore.
if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
config.QueueList = append(config.QueueList[:i],
config.QueueList[i+1:]...)
if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
arnErr, ok := err.(*event.ErrARNNotFound)
if ok {
for i, queue := range config.QueueList {
// Remove ARN not found queues, because we previously allowed
// adding unexpected entries into the config.
//
// With newer config disallowing changing / turning off
// notification targets without removing ARN in notification
// configuration we won't see this problem anymore.
if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
config.QueueList = append(config.QueueList[:i],
config.QueueList[i+1:]...)
}
// This is a one time activity we shall do this
// here and allow stale ARN to be removed. We shall
// never reach a stage where we will have stale
// notification configs.
}
} else {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
// This is a one time activity we shall do this
// here and allow stale ARN to be removed. We shall
// never reach a stage where we will have stale
// notification configs.
}
}
// If xml namespace is empty, set a default value before returning.
if config.XMLNS == "" {
config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
}
notificationBytes, err := xml.Marshal(config)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -178,8 +159,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
return
}
lreader := io.LimitReader(r.Body, r.ContentLength)
config, err := event.ParseConfig(lreader, globalServerRegion, globalNotificationSys.targetList)
config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerRegion, globalNotificationSys.targetList)
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
if event.IsEventError(err) {
@@ -189,14 +169,19 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
return
}
if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
configData, err := xml.Marshal(config)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
if err = globalBucketMetadataSys.Update(bucketName, bucketNotificationConfig, configData); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
rulesMap := config.ToRulesMap()
globalNotificationSys.AddRulesMap(bucketName, rulesMap)
globalNotificationSys.PutBucketNotification(ctx, bucketName, rulesMap)
writeSuccessResponseHeadersOnly(w)
}