Vendorize all recent changes to minio-go (#7135)

- Default support for S3 dualstack endpoints (IPv6 support)
- Support granular policy conditionals in List operations
- Support proxy cookies for stickiness
This commit is contained in:
Harshavardhana
2019-01-23 19:22:09 +05:30
committed by Nitish Tiwari
parent dc2348daa5
commit 55ef51a99d
13 changed files with 11069 additions and 182 deletions

View File

@@ -192,18 +192,16 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s
// Always set list-type in ListObjects V2
urlValues.Set("list-type", "2")
// Set object prefix.
if objectPrefix != "" {
urlValues.Set("prefix", objectPrefix)
}
// Set object prefix, prefix value to be set to empty is okay.
urlValues.Set("prefix", objectPrefix)
// Set delimiter, delimiter value to be set to empty is okay.
urlValues.Set("delimiter", delimiter)
// Set continuation token
if continuationToken != "" {
urlValues.Set("continuation-token", continuationToken)
}
// Set delimiter.
if delimiter != "" {
urlValues.Set("delimiter", delimiter)
}
// Fetch owner when listing
if fetchOwner {
@@ -380,18 +378,17 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
// Set object prefix.
if objectPrefix != "" {
urlValues.Set("prefix", objectPrefix)
}
// Set object prefix, prefix value to be set to empty is okay.
urlValues.Set("prefix", objectPrefix)
// Set delimiter, delimiter value to be set to empty is okay.
urlValues.Set("delimiter", delimiter)
// Set object marker.
if objectMarker != "" {
urlValues.Set("marker", objectMarker)
}
// Set delimiter.
if delimiter != "" {
urlValues.Set("delimiter", delimiter)
}
// maxkeys should default to 1000 or less.
if maxkeys == 0 || maxkeys > 1000 {
@@ -563,14 +560,12 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker,
if uploadIDMarker != "" {
urlValues.Set("upload-id-marker", uploadIDMarker)
}
// Set prefix marker.
if prefix != "" {
urlValues.Set("prefix", prefix)
}
// Set delimiter.
if delimiter != "" {
urlValues.Set("delimiter", delimiter)
}
// Set object prefix, prefix value to be set to empty is okay.
urlValues.Set("prefix", prefix)
// Set delimiter, delimiter value to be set to empty is okay.
urlValues.Set("delimiter", delimiter)
// maxUploads should be 1000 or less.
if maxUploads == 0 || maxUploads > 1000 {

View File

@@ -191,13 +191,20 @@ type StatsMessage struct {
BytesReturned int64
}
// messageType represents the type of message.
type messageType string
const (
errorMsg messageType = "error"
commonMsg = "event"
)
// eventType represents the type of event.
type eventType string
// list of event-types returned by Select API.
const (
endEvent eventType = "End"
errorEvent = "Error"
recordsEvent = "Records"
progressEvent = "Progress"
statsEvent = "Stats"
@@ -314,53 +321,58 @@ func (s *SelectResults) start(pipeWriter *io.PipeWriter) {
// bytes can be read or parsed.
payloadLen := prelude.PayloadLen()
// Get content-type of the payload.
c := contentType(headers.Get("content-type"))
m := messageType(headers.Get("message-type"))
// Get event type of the payload.
e := eventType(headers.Get("event-type"))
// Handle all supported events.
switch e {
case endEvent:
pipeWriter.Close()
closeResponse(s.resp)
return
case errorEvent:
switch m {
case errorMsg:
pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message")))
closeResponse(s.resp)
return
case recordsEvent:
if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil {
pipeWriter.CloseWithError(err)
case commonMsg:
// Get content-type of the payload.
c := contentType(headers.Get("content-type"))
// Get event type of the payload.
e := eventType(headers.Get("event-type"))
// Handle all supported events.
switch e {
case endEvent:
pipeWriter.Close()
closeResponse(s.resp)
return
}
case progressEvent:
switch c {
case xmlContent:
if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil {
case recordsEvent:
if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
default:
pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent))
closeResponse(s.resp)
return
}
case statsEvent:
switch c {
case xmlContent:
if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil {
pipeWriter.CloseWithError(err)
case progressEvent:
switch c {
case xmlContent:
if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
default:
pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent))
closeResponse(s.resp)
return
}
case statsEvent:
switch c {
case xmlContent:
if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
default:
pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent))
closeResponse(s.resp)
return
}
default:
pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent))
closeResponse(s.resp)
return
}
}

View File

@@ -30,6 +30,7 @@ import (
"math/rand"
"net"
"net/http"
"net/http/cookiejar"
"net/http/httputil"
"net/url"
"os"
@@ -38,6 +39,8 @@ import (
"sync"
"time"
"golang.org/x/net/publicsuffix"
"github.com/minio/minio-go/pkg/credentials"
"github.com/minio/minio-go/pkg/s3signer"
"github.com/minio/minio-go/pkg/s3utils"
@@ -99,7 +102,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v6.0.11"
libraryVersion = "v6.0.14"
)
// User Agent should always following the below style.
@@ -273,6 +276,13 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
return nil, err
}
// Initialize cookies to preserve server sent cookies if any and replay
// them upon each request.
jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
if err != nil {
return nil, err
}
// instantiate new Client.
clnt := new(Client)
@@ -287,6 +297,7 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
// Instantiate http client and bucket location cache.
clnt.httpClient = &http.Client{
Jar: jar,
Transport: DefaultTransport,
CheckRedirect: clnt.redirectHeaders,
}

View File

@@ -67,9 +67,7 @@ func getEndpoint(endpoint string) (string, bool) {
return defaultIAMRoleEndpoint, false
}
// NewIAM returns a pointer to a new Credentials object wrapping
// the IAM. Takes a ConfigProvider to create a EC2Metadata client.
// The ConfigProvider is satisfied by the session.Session type.
// NewIAM returns a pointer to a new Credentials object wrapping the IAM.
func NewIAM(endpoint string) *Credentials {
p := &IAM{
Client: &http.Client{

View File

@@ -0,0 +1,165 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2019 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package credentials
import (
"encoding/xml"
"errors"
"fmt"
"net/http"
"net/url"
"time"
)
// AssumedRoleUser - The identifiers for the temporary security credentials that
// the operation returns. Please also see https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumedRoleUser
type AssumedRoleUser struct {
Arn string
AssumedRoleID string `xml:"AssumeRoleId"`
}
// AssumeRoleWithClientGrantsResponse contains the result of successful AssumeRoleWithClientGrants request.
type AssumeRoleWithClientGrantsResponse struct {
XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithClientGrantsResponse" json:"-"`
Result ClientGrantsResult `xml:"AssumeRoleWithClientGrantsResult"`
ResponseMetadata struct {
RequestID string `xml:"RequestId,omitempty"`
} `xml:"ResponseMetadata,omitempty"`
}
// ClientGrantsResult - Contains the response to a successful AssumeRoleWithClientGrants
// request, including temporary credentials that can be used to make Minio API requests.
type ClientGrantsResult struct {
AssumedRoleUser AssumedRoleUser `xml:",omitempty"`
Audience string `xml:",omitempty"`
Credentials struct {
AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"`
SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"`
Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"`
SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"`
} `xml:",omitempty"`
PackedPolicySize int `xml:",omitempty"`
Provider string `xml:",omitempty"`
SubjectFromClientGrantsToken string `xml:",omitempty"`
}
// ClientGrantsToken - client grants token with expiry.
type ClientGrantsToken struct {
// access token returned after authenticating client grants
Token string
// expiry for the access token returned after authenticating
// client grants.
Expiry int
}
// A STSClientGrants retrieves credentials from Minio service, and keeps track if
// those credentials are expired.
type STSClientGrants struct {
Expiry
// Required http Client to use when connecting to Minio STS service.
Client *http.Client
// Minio endpoint to fetch STS credentials.
stsEndpoint string
// getClientGrantsTokenExpiry function to retrieve tokens
// from IDP This function should return two values one is
// accessToken which is a self contained access token (JWT)
// and second return value is the expiry associated with
// this token. This is a customer provided function and
// is mandatory.
getClientGrantsTokenExpiry func() (*ClientGrantsToken, error)
}
// NewSTSClientGrants returns a pointer to a new
// Credentials object wrapping the STSClientGrants.
func NewSTSClientGrants(stsEndpoint string, getClientGrantsTokenExpiry func() (*ClientGrantsToken, error)) (*Credentials, error) {
if stsEndpoint == "" {
return nil, errors.New("STS endpoint cannot be empty")
}
if getClientGrantsTokenExpiry == nil {
return nil, errors.New("Client grants access token and expiry retrieval function should be defined")
}
return New(&STSClientGrants{
Client: &http.Client{
Transport: http.DefaultTransport,
},
stsEndpoint: stsEndpoint,
getClientGrantsTokenExpiry: getClientGrantsTokenExpiry,
}), nil
}
func getClientGrantsCredentials(clnt *http.Client, endpoint string,
getClientGrantsTokenExpiry func() (*ClientGrantsToken, error)) (AssumeRoleWithClientGrantsResponse, error) {
accessToken, err := getClientGrantsTokenExpiry()
if err != nil {
return AssumeRoleWithClientGrantsResponse{}, err
}
v := url.Values{}
v.Set("Action", "AssumeRoleWithClientGrants")
v.Set("Token", accessToken.Token)
v.Set("DurationSeconds", fmt.Sprintf("%d", accessToken.Expiry))
v.Set("Version", "2011-06-15")
u, err := url.Parse(endpoint)
if err != nil {
return AssumeRoleWithClientGrantsResponse{}, err
}
u.RawQuery = v.Encode()
req, err := http.NewRequest("POST", u.String(), nil)
if err != nil {
return AssumeRoleWithClientGrantsResponse{}, err
}
resp, err := clnt.Do(req)
if err != nil {
return AssumeRoleWithClientGrantsResponse{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return AssumeRoleWithClientGrantsResponse{}, errors.New(resp.Status)
}
a := AssumeRoleWithClientGrantsResponse{}
if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil {
return AssumeRoleWithClientGrantsResponse{}, err
}
return a, nil
}
// Retrieve retrieves credentials from the Minio service.
// Error will be returned if the request fails.
func (m *STSClientGrants) Retrieve() (Value, error) {
a, err := getClientGrantsCredentials(m.Client, m.stsEndpoint, m.getClientGrantsTokenExpiry)
if err != nil {
return Value{}, err
}
// Expiry window is set to 10secs.
m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow)
return Value{
AccessKeyID: a.Result.Credentials.AccessKey,
SecretAccessKey: a.Result.Credentials.SecretKey,
SessionToken: a.Result.Credentials.SessionToken,
SignerType: SignatureV4,
}, nil
}

View File

@@ -0,0 +1,161 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2019 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package credentials
import (
"encoding/xml"
"errors"
"fmt"
"net/http"
"net/url"
"time"
)
// AssumeRoleWithWebIdentityResponse contains the result of successful AssumeRoleWithWebIdentity request.
type AssumeRoleWithWebIdentityResponse struct {
XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithWebIdentityResponse" json:"-"`
Result WebIdentityResult `xml:"AssumeRoleWithWebIdentityResult"`
ResponseMetadata struct {
RequestID string `xml:"RequestId,omitempty"`
} `xml:"ResponseMetadata,omitempty"`
}
// WebIdentityResult - Contains the response to a successful AssumeRoleWithWebIdentity
// request, including temporary credentials that can be used to make Minio API requests.
type WebIdentityResult struct {
AssumedRoleUser AssumedRoleUser `xml:",omitempty"`
Audience string `xml:",omitempty"`
Credentials struct {
AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"`
SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"`
Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"`
SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"`
} `xml:",omitempty"`
PackedPolicySize int `xml:",omitempty"`
Provider string `xml:",omitempty"`
SubjectFromWebIdentityToken string `xml:",omitempty"`
}
// WebIdentityToken - web identity token with expiry.
type WebIdentityToken struct {
// access token returned after authenticating web identity.
Token string
// expiry for the access token returned after authenticating
// web identity.
Expiry int
}
// A STSWebIdentity retrieves credentials from Minio service, and keeps track if
// those credentials are expired.
type STSWebIdentity struct {
Expiry
// Required http Client to use when connecting to Minio STS service.
Client *http.Client
// Minio endpoint to fetch STS credentials.
stsEndpoint string
// getWebIDTokenExpiry function which returns ID tokens
// from IDP. This function should return two values one
// is ID token which is a self contained ID token (JWT)
// and second return value is the expiry associated with
// this token.
// This is a customer provided function and is mandatory.
getWebIDTokenExpiry func() (*WebIdentityToken, error)
}
// NewSTSWebIdentity returns a pointer to a new
// Credentials object wrapping the STSWebIdentity.
func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdentityToken, error)) (*Credentials, error) {
if stsEndpoint == "" {
return nil, errors.New("STS endpoint cannot be empty")
}
if getWebIDTokenExpiry == nil {
return nil, errors.New("Web ID token and expiry retrieval function should be defined")
}
return New(&STSWebIdentity{
Client: &http.Client{
Transport: http.DefaultTransport,
},
stsEndpoint: stsEndpoint,
getWebIDTokenExpiry: getWebIDTokenExpiry,
}), nil
}
func getWebIdentityCredentials(clnt *http.Client, endpoint string,
getWebIDTokenExpiry func() (*WebIdentityToken, error)) (AssumeRoleWithWebIdentityResponse, error) {
idToken, err := getWebIDTokenExpiry()
if err != nil {
return AssumeRoleWithWebIdentityResponse{}, err
}
v := url.Values{}
v.Set("Action", "AssumeRoleWithWebIdentity")
v.Set("WebIdentityToken", idToken.Token)
v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry))
v.Set("Version", "2011-06-15")
u, err := url.Parse(endpoint)
if err != nil {
return AssumeRoleWithWebIdentityResponse{}, err
}
u.RawQuery = v.Encode()
req, err := http.NewRequest("POST", u.String(), nil)
if err != nil {
return AssumeRoleWithWebIdentityResponse{}, err
}
resp, err := clnt.Do(req)
if err != nil {
return AssumeRoleWithWebIdentityResponse{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return AssumeRoleWithWebIdentityResponse{}, errors.New(resp.Status)
}
a := AssumeRoleWithWebIdentityResponse{}
if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil {
return AssumeRoleWithWebIdentityResponse{}, err
}
return a, nil
}
// Retrieve retrieves credentials from the Minio service.
// Error will be returned if the request fails.
func (m *STSWebIdentity) Retrieve() (Value, error) {
a, err := getWebIdentityCredentials(m.Client, m.stsEndpoint, m.getWebIDTokenExpiry)
if err != nil {
return Value{}, err
}
// Expiry window is set to 10secs.
m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow)
return Value{
AccessKeyID: a.Result.Credentials.AccessKey,
SecretAccessKey: a.Result.Credentials.SecretKey,
SessionToken: a.Result.Credentials.SessionToken,
SignerType: SignatureV4,
}, nil
}

View File

@@ -139,7 +139,7 @@ func isS3CodeRetryable(s3Code string) (ok bool) {
// List of HTTP status codes which are retryable.
var retryableHTTPStatusCodes = map[int]struct{}{
429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
http.StatusInternalServerError: {},
http.StatusBadGateway: {},
http.StatusServiceUnavailable: {},

View File

@@ -19,22 +19,24 @@ package minio
// awsS3EndpointMap Amazon S3 endpoint map.
var awsS3EndpointMap = map[string]string{
"us-east-1": "s3.amazonaws.com",
"us-east-2": "s3-us-east-2.amazonaws.com",
"us-west-2": "s3-us-west-2.amazonaws.com",
"us-west-1": "s3-us-west-1.amazonaws.com",
"ca-central-1": "s3-ca-central-1.amazonaws.com",
"eu-west-1": "s3-eu-west-1.amazonaws.com",
"eu-west-2": "s3-eu-west-2.amazonaws.com",
"eu-west-3": "s3-eu-west-3.amazonaws.com",
"eu-central-1": "s3-eu-central-1.amazonaws.com",
"ap-south-1": "s3-ap-south-1.amazonaws.com",
"ap-southeast-1": "s3-ap-southeast-1.amazonaws.com",
"ap-southeast-2": "s3-ap-southeast-2.amazonaws.com",
"ap-northeast-1": "s3-ap-northeast-1.amazonaws.com",
"ap-northeast-2": "s3-ap-northeast-2.amazonaws.com",
"sa-east-1": "s3-sa-east-1.amazonaws.com",
"us-gov-west-1": "s3-us-gov-west-1.amazonaws.com",
"us-east-1": "s3.dualstack.us-east-1.amazonaws.com",
"us-east-2": "s3.dualstack.us-east-2.amazonaws.com",
"us-west-2": "s3.dualstack.us-west-2.amazonaws.com",
"us-west-1": "s3.dualstack.us-west-1.amazonaws.com",
"ca-central-1": "s3.dualstack.ca-central-1.amazonaws.com",
"eu-west-1": "s3.dualstack.eu-west-1.amazonaws.com",
"eu-west-2": "s3.dualstack.eu-west-2.amazonaws.com",
"eu-west-3": "s3.dualstack.eu-west-3.amazonaws.com",
"eu-central-1": "s3.dualstack.eu-central-1.amazonaws.com",
"eu-north-1": "s3.dualstack.eu-north-1.amazonaws.com",
"ap-south-1": "s3.dualstack.ap-south-1.amazonaws.com",
"ap-southeast-1": "s3.dualstack.ap-southeast-1.amazonaws.com",
"ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com",
"ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com",
"ap-northeast-2": "s3.dualstack.ap-northeast-2.amazonaws.com",
"sa-east-1": "s3.dualstack.sa-east-1.amazonaws.com",
"us-gov-west-1": "s3.dualstack.us-gov-west-1.amazonaws.com",
"us-gov-east-1": "s3.dualstack.us-gov-east-1.amazonaws.com",
"cn-north-1": "s3.cn-north-1.amazonaws.com.cn",
"cn-northwest-1": "s3.cn-northwest-1.amazonaws.com.cn",
}
@@ -43,8 +45,8 @@ var awsS3EndpointMap = map[string]string{
func getS3Endpoint(bucketLocation string) (s3Endpoint string) {
s3Endpoint, ok := awsS3EndpointMap[bucketLocation]
if !ok {
// Default to 's3.amazonaws.com' endpoint.
s3Endpoint = "s3.amazonaws.com"
// Default to 's3.dualstack.us-east-1.amazonaws.com' endpoint.
s3Endpoint = "s3.dualstack.us-east-1.amazonaws.com"
}
return s3Endpoint
}