GCP Cloud Storage - Golang aws sdk2 - Upload file with s3 INTEROPERABILITY Creds

I'm trying to implements download/upload a file from/to a bucket in cloud storage via the s3 go sdk aws-sdk-go-v2 using the Interoperability feature

The download is working as expected, but the upload isnt working, with this error message: SDK 2022/09/14 11:24:43 DEBUG request failed with unretryable error https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.

As I use same access_key and secret_key for both download and upload, it does not seems to be a credentials problem.
Plus, the service account behind the hmac keys has the Storage object Admin Role.

Here the code:

main.go

package main
import (
"context"
"fmt"
"os"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
var BUCKET_NAME = ""
func main() {
//prepare gcp resolver
gcpResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL:               "https://storage.googleapis.com",
SigningRegion:     "auto",
Source:            aws.EndpointSourceCustom,
HostnameImmutable: true,
}, nil
})
//file with fornat : $accessKey:$secretKey
file, _ := os.ReadFile("/home/bapt/creds/amz-keys-gcp-2")
keys := strings.Split(string(file), ":")
//init the config options
optConfig := []func(*config.LoadOptions) error{
config.WithRegion("auto"),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(keys[0], strings.TrimRight(keys[1], "\n"), "")),
config.WithClientLogMode(aws.LogRetries | aws.LogRequestWithBody | aws.LogResponseWithBody | aws.LogRequestEventMessage | aws.LogResponseEventMessage | aws.LogSigning),
config.WithEndpointResolverWithOptions(gcpResolver),
}
//init config
cfg, _ := config.LoadDefaultConfig(context.TODO(), optConfig...)
//init service
svc := s3.NewFromConfig(cfg)
tempFile, _ := os.CreateTemp("/tmp", "test-gcp-*")
defer tempFile.Close()
downloader := manager.NewDownloader(svc)
downloader.Download(context.TODO(),tempFile, &s3.GetObjectInput{
Bucket: aws.String(BUCKET_NAME),
Key:    aws.String("file-test.txt"),
})
//init uploader ( no multipart)
uploader := manager.NewUploader(svc, func(u *manager.Uploader) {
u.Concurrency = 1
u.MaxUploadParts = 1
})
//upload
_, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(BUCKET_NAME),
Key:    aws.String("file-test.txt"),
Body:   strings.NewReader("HELLO"),
})
fmt.Println(err)
}

go.mod

module gcps3/v2
go 1.18
require (
github.com/aws/aws-sdk-go-v2 v1.16.14
github.com/aws/aws-sdk-go-v2/config v1.17.5
github.com/aws/aws-sdk-go-v2/credentials v1.12.18
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.31
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.9
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.21 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.17 // indirect
github.com/aws/smithy-go v1.13.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
)

And here the debug trace of the PUT ( my bucket name is replaced by BUCKET, and access_key with GOOG1ID:

SDK 2022/09/14 14:52:37 DEBUG Request Signature:
---[ CANONICAL STRING  ]-----------------------------
PUT
/BUCKET/file-test.txt
x-id=PutObject
accept-encoding:identity
amz-sdk-invocation-id:d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
amz-sdk-request:attempt=1; max=3
content-length:5
content-type:application/octet-stream
host:storage.googleapis.com
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:20220914T125237Z
accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD
---[ STRING TO SIGN ]--------------------------------
AWS4-HMAC-SHA256
20220914T125237Z
20220914/auto/s3/aws4_request
bc09.....daf520
-----------------------------------------------------
SDK 2022/09/14 14:52:37 DEBUG Request
PUT /BUCKET/file-test.txt?x-id=PutObject HTTP/1.1
Host: storage.googleapis.com
User-Agent: aws-sdk-go-v2/1.16.14 os/linux lang/go/1.18.1 md/GOOS/linux md/GOARCH/amd64 api/s3/1.27.9 ft/s3-transfer
Content-Length: 5
Accept-Encoding: identity
Amz-Sdk-Invocation-Id: d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
Amz-Sdk-Request: attempt=1; max=3
Authorization: AWS4-HMAC-SHA256 Credential=GOOG1ID/20220914/auto/s3/aws4_request, SignedHeaders=accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date, Signature=c994....37d0
Content-Type: application/octet-stream
X-Amz-Content-Sha256: UNSIGNED-PAYLOAD
X-Amz-Date: 20220914T125237Z
HELLO
SDK 2022/09/14 14:52:37 DEBUG Response
HTTP/2.0 403 Forbidden
Content-Length: 883
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Content-Type: application/xml; charset=UTF-8
Date: Wed, 14 Sep 2022 12:52:38 GMT
Server: UploadServer
X-Guploader-Uploadid: ADPycdt0aCu6BTmzWQl2Ehc4q2sP8rtexDb4Keyn6cQL_GigREvc8T1CzX0HH-ZXgw_6XWLJPPYXufwRCr0Sl7uSsiIi0Q
<?xml version='1.0' encoding='UTF-8'?><Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.</Message><StringToSign>AWS4-HMAC-SHA256
20220914T125237Z
20220914/auto/s3/aws4_request
0a10....d63e</StringToSign><CanonicalRequest>PUT
/BUCKET/file-test.txt
x-id=PutObject
accept-encoding:identity,gzip(gfe)
amz-sdk-invocation-id:d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
amz-sdk-request:attempt=1; max=3
content-length:5
content-type:application/octet-stream
host:storage.googleapis.com
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:20220914T125237Z
accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD</CanonicalRequest></Error>
SDK 2022/09/14 14:52:37 DEBUG request failed with unretryable error https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.
operation error S3: PutObject, https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.

I tested the upload, with this python script, and it works (same creds)

boto3.set_stream_logger('', logging.DEBUG)
GCP_BUCKET = True
FILE_TO_BE_UPLOADED = '/tmp/toto'
if GCP_BUCKET:
ACCESS_KEY = "ACESS"
SECRET_KEY = "SECRET"
bucket_name = "BUCKET"    
region_name="auto"
endpoint_url="https://storage.googleapis.com"
def makeS3Client():
s3 = boto3.client("s3", 
region_name=region_name,
endpoint_url=endpoint_url,
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
)
return s3
def upload_file(s3,bucket_name,fname):
"""
Uploads file to S3 bucket using S3 client object
:return: None
"""
object_name = os.path.basename(fname)
file_name = os.path.abspath(fname)
#file_name = os.path.join(pathlib.Path(__file__).parent.resolve(), fname)
response = s3.upload_file(file_name, bucket_name, object_name)
#print(response)  # prints None

The only different between go and python i saw in the https requests made is the signedHeaders used.

But the go code with an s3 aws bucket is working fine...

Am I missing an option ?

Thanks for your help.

The issue that @h3yduck mentioned, it seems that accept-encoding needs to be excluded from the signature signing process in the v2 library. Since the signature is dependent on the headers mentioned in SignedHeaders, we will have to recalculate the signature ourselves.

The AWS Configuration object exposes a variable for a custom http client to be used for all services. We can use this by defining our own RoundTripper that modifies the request's signature to one that doesn't account for accept-encoding.

Example:

package main

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws"
	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/credentials"
	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
	"github.com/aws/aws-sdk-go-v2/service/s3"
)

var BUCKET_NAME = "test"

type RecalculateV4Signature struct {
	next   http.RoundTripper
	signer *v4.Signer
	cfg    aws.Config
}

func (lt *RecalculateV4Signature) RoundTrip(req *http.Request) (*http.Response, error) {
	// store for later use
	val := req.Header.Get("Accept-Encoding")

	// delete the header so the header doesn't account for in the signature
	req.Header.Del("Accept-Encoding")

	// sign with the same date
	timeString := req.Header.Get("X-Amz-Date")
	timeDate, _ := time.Parse("20060102T150405Z", timeString)

	creds, _ := lt.cfg.Credentials.Retrieve(req.Context())
	err := lt.signer.SignHTTP(req.Context(), creds, req, v4.GetPayloadHash(req.Context()), "s3", lt.cfg.Region, timeDate)
	if err != nil {
		return nil, err
	}
	// Reset Accept-Encoding if desired
	req.Header.Set("Accept-Encoding", val)

	fmt.Println("AfterAdjustment")
	rrr, _ := httputil.DumpRequest(req, false)
	fmt.Println(string(rrr))

	// follows up the original round tripper
	return lt.next.RoundTrip(req)
}

func main() {

	//prepare gcp resolver
	gcpResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
		return aws.Endpoint{
			URL:               "https://storage.googleapis.com",
			SigningRegion:     "auto",
			Source:            aws.EndpointSourceCustom,
			HostnameImmutable: true,
		}, nil
	})
	//file with format : $accessKey:$secretKey

	//init the config options
	optConfig := []func(*config.LoadOptions) error{
		config.WithRegion("auto"),
		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "session")),
		config.WithClientLogMode(aws.LogRetries | aws.LogRequestWithBody | aws.LogResponseWithBody | aws.LogRequestEventMessage | aws.LogResponseEventMessage | aws.LogSigning),
		config.WithEndpointResolverWithOptions(gcpResolver),
	}

	//init config
	cfg, _ := config.LoadDefaultConfig(context.TODO(), optConfig...)
	// Assign custom client with our own transport
	cfg.HTTPClient = &http.Client{Transport: &RecalculateV4Signature{http.DefaultTransport, v4.NewSigner(), cfg}}
	//init service
	svc := s3.NewFromConfig(cfg)
	uploader := manager.NewUploader(svc, func(u *manager.Uploader) {
		u.Concurrency = 1
		u.MaxUploadParts = 1
	})
	//upload
	_, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{
		Bucket: aws.String(BUCKET_NAME),
		Key:    aws.String("file-test.txt"),
		Body:   strings.NewReader("HELLO"),
	})
	fmt.Println(err)
}

P.S. the client should ideally be built on BuildableClient from aws as they provide sensible defaults for their services, I will leave that up to the reader

According to https://github.com/aws/aws-sdk-go-v2/issues/1816 the v2 SDK doesn't support GCP GCS.

Removing Accept-Encoding from the sighed headers solved the issue for me, but I'm not sure if it's an advised thing to do. Also, I didn't find a way to remove this header from the list of signed headers (there might be a middleware that supports this), other than modifying the lib's source code (adding it to the IgnoredHeaders global variable).