I tried to do the multipart upload with the S3 methods as follows using AWS.ji library, but every time it gives me the following error, appreciate if someone can point out the mistake that I’ve made.
<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><AWSAccessKeyId>AKIAWMRX3SPXW54A3657</AWSAccessKeyId><StringToSign>AWS4-HMAC-SHA256
And following is the Code:
NOTE: I’ve used (-------) to show the different parts of the code
using AWS
@service S3 use_response_type = true
using Base64
using MD5
---------
""
const GLOBAL_AWS_CONFIG = Ref{AWS.AWSConfig}()
"""
set_global_config(aws_secret_key_id::String, aws_secret_access_key::String, aws_region::String, aws_profile::String)
Sets the `GLOBAL_AWS_CONFIG[]` reference object with the credentials provided.
This extends the AWS Credential chain sequencing. Order of auth options:
1. Use the profile name if provided
2. Usage of the access and secret keys
2a. If credentials are not provided explicitly, `GLOBAL_AWS_CONFIG[]` is set with the environment variables:
- `AWS_ACCESS_KEY_ID`
- `AWS_SECRET_ACCESS_KEY`
- `AWS_DEFAULT_REGION`
3. empty inputs resolve to the AWS credential chain, including machine roles
"""
function set_global_config(aws_secret_key_id::String=get(ENV, "AWS_ACCESS_KEY_ID", ""),
aws_secret_access_key::String=get(ENV, "AWS_SECRET_ACCESS_KEY", ""),
aws_region::String=get(ENV, "AWS_DEFAULT_REGION", "ap-southeast-2"),
aws_profile::String="")
if !isempty(aws_profile)
GLOBAL_AWS_CONFIG[] = global_aws_config(profile = aws_profile)
elseif !isempty(aws_secret_access_key)
creds = AWSCredentials(aws_secret_key_id, aws_secret_access_key)
GLOBAL_AWS_CONFIG[] = global_aws_config(region=aws_region, creds=creds)
else
GLOBAL_AWS_CONFIG[] = global_aws_config(region=aws_region)
end
end
set_global_config(aws_credentials::AWSCredentials, aws_region::String=get(ENV, "AWS_DEFAULT_REGION", "ap-southeast-2")) = set_global_config(aws_credentials.access_key_id, aws_credentials.secret_key, aws_region)
"""
get_global_config()
Returns the `GLOBAL_AWS_CONFIG[]` reference object.
"""
get_global_config() = isassigned(GLOBAL_AWS_CONFIG) ? GLOBAL_AWS_CONFIG[] : set_global_config()
----------
function s3_multipart_upload(
bucket,
key,
data::IO;
aws=global_aws_config(),
part_size_mb=50,
parse_response::Bool=true,
tags::AbstractDict=Dict(),
metadata::AbstractDict=Dict(),
)
@info "========== Starting S3 Multi-part Upload"
part_size = part_size_mb * 1024 * 1024
headers = Dict{String,Any}()
if !isempty(tags)
headers["x-amz-tagging"] = escapeuri(tags)
headers["x-amz-tagging-directive"] = "REPLACE"
end
if !isempty(metadata)
merge!(headers, Dict("x-amz-meta-$k" => v for (k, v) in metadata))
headers["x-amz-metadata-directive"] = "REPLACE"
end
headers["Content-Type"] = "application/json"
@info "+++ Creating Multipart Upload"
upload = S3.create_multipart_upload(bucket, key, headers; aws_config=aws)
upload_id = upload["UploadId"]
@info "upload : $(upload)"
@info "bucket", upload["Bucket"]
@info "upload_id", upload["UploadId"]
tags = Vector{String}()
buf = Vector{UInt8}(undef, part_size)
i = 0
while (n = readbytes!(data, buf, part_size)) > 0
if n < part_size
resize!(buf, n)
end
@info "+++ Uploading Part $(i)"
part = S3.upload_part(
bucket,
key,
(i += 1),
upload_id,
Dict{String,Any}(
"Body" => buf,
"Content-MD5" => generate_content_md5(data, part_size),
"Content-Type" => "application/json"
),
aws_config=aws,
)
@info "Part response $(part)"
# push!(tags, s3_upload_part(aws, upload, (i += 1), buf))
push!(tags, part)
end
@info "All tags : $(tags)"
@info "+++ Completing Multipart Upload"
return s3_complete_multipart_upload(aws, upload, tags; parse_response)
end
------
function generate_content_md5(data, part_size)
# Calculate the MD5 digest of the data
md5_digest = md5(data, part_size)
@info "md5_digest : $(md5_digest)"
# Encode the MD5 digest in base64 format
base64_md5_digest = base64encode(md5_digest)
@info "base64_md5_digest : $(base64_md5_digest)"
return base64_md5_digest
end
----------
Usage of the above function:
function test_s3_upload()
set_global_config("KEY", "SECRET", "ap-southeast-2")
tags = Dict(
"version" => "0.0.0"
)
metadata = Dict(
"version" => "0.0.0"
)
open("./small.json", "r") do file_io::IO
S3Interface.s3_multipart_upload(
App.AWS_S3_MM_BUCKET[],
"map_exporter/osm_export/latest/new.json",
file_io;
tags=tags,
metadata=metadata,
);
end
end
-----
Following is the above small.json context
{
"version": "2.2",
"changeset_id": "15cdde2b-0a19-428e-9ebe-5c3518d3cf63",
"source_ingestion_date": "O2023-12-05T23:27:21.722"
}