From da5692abb7f58ecabd12357a0c615ab43e490fef Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Mon, 17 Nov 2025 12:22:11 -0800 Subject: [PATCH 01/30] Update checksum plugin to not load body into memory --- .../lib/aws-sdk-core/plugins/checksum_algorithm.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index c5b665e428e..145c07891a7 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -346,16 +346,16 @@ def calculate_request_checksum(context, checksum_properties) def apply_request_checksum(context, headers, checksum_properties) header_name = checksum_properties[:name] - body = context.http_request.body_contents headers[header_name] = calculate_checksum( checksum_properties[:algorithm], - body + context.http_request.body ) end def calculate_checksum(algorithm, body) digest = ChecksumAlgorithm.digest_for_algorithm(algorithm) if body.respond_to?(:read) + body.rewind update_in_chunks(digest, body) else digest.update(body) From 20bd2b89983c5b392ab4b8060e89f1baedd69434 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Mon, 17 Nov 2025 12:59:16 -0800 Subject: [PATCH 02/30] Add customization --- build_tools/customizations.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build_tools/customizations.rb b/build_tools/customizations.rb index e61761224fd..3ace3397415 100644 --- a/build_tools/customizations.rb +++ b/build_tools/customizations.rb @@ -153,13 +153,18 @@ def dynamodb_example_deep_transform(subsegment, keys) api['metadata'].delete('signatureVersion') # handled by endpoints 2.0 - api['operations'].each do |_key, operation| + api['operations'].each do |key, operation| # requestUri should always exist. Remove bucket from path # and preserve request uri as / if operation['http'] && operation['http']['requestUri'] operation['http']['requestUri'].gsub!('/{Bucket}', '/') operation['http']['requestUri'].gsub!('//', '/') end + + next unless %w[PutObject UploadPart].include?(key) + + operation['authType'] = 'v4-unsigned-body' + operation['unsignedPayload'] = true end # Ensure Expires is a timestamp regardless of model to be backwards From 1a761ec7121f0fe1295029ac92fbc026b82def8d Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Fri, 21 Nov 2025 10:20:37 -0800 Subject: [PATCH 03/30] Fix trailer impl to include overhead bytes --- .../plugins/checksum_algorithm.rb | 92 ++++++++++--------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 145c07891a7..6ebf27071ef 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -162,9 +162,7 @@ def call(context) context[:http_checksum] ||= {} # Set validation mode to enabled when supported. - if context.config.response_checksum_validation == 'when_supported' - enable_request_validation_mode(context) - end + enable_request_validation_mode(context) if context.config.response_checksum_validation == 'when_supported' @handler.call(context) end @@ -194,9 +192,7 @@ def call(context) calculate_request_checksum(context, request_algorithm) end - if should_verify_response_checksum?(context) - add_verify_response_checksum_handlers(context) - end + add_verify_response_checksum_handlers(context) if should_verify_response_checksum?(context) with_metrics(context.config, algorithm) { @handler.call(context) } end @@ -388,13 +384,15 @@ def apply_request_trailer_checksum(context, headers, checksum_properties) unless context.http_request.body.respond_to?(:size) raise Aws::Errors::ChecksumError, 'Could not determine length of the body' end + headers['X-Amz-Decoded-Content-Length'] = context.http_request.body.size - context.http_request.body = AwsChunkedTrailerDigestIO.new( - context.http_request.body, - checksum_properties[:algorithm], - location_name - ) + context.http_request.body = + AwsChunkedTrailerDigestIO.new( + io: context.http_request.body, + algorithm: checksum_properties[:algorithm], + location_name: location_name + ) end def should_verify_response_checksum?(context) @@ -417,10 +415,7 @@ def add_verify_response_headers_handler(context, checksum_context) context[:http_checksum][:validation_list] = validation_list context.http_response.on_headers do |_status, headers| - header_name, algorithm = response_header_to_verify( - headers, - validation_list - ) + header_name, algorithm = response_header_to_verify(headers, validation_list) next unless header_name expected = headers[header_name] @@ -466,52 +461,67 @@ def response_header_to_verify(headers, validation_list) # Wrapper for request body that implements application-layer # chunking with Digest computed on chunks + added as a trailer class AwsChunkedTrailerDigestIO - CHUNK_SIZE = 16_384 + DEFAULT_CHUNK_SIZE = 16_384 - def initialize(io, algorithm, location_name) - @io = io - @location_name = location_name - @algorithm = algorithm - @digest = ChecksumAlgorithm.digest_for_algorithm(algorithm) - @trailer_io = nil + def initialize(options = {}) + @io = options.delete(:io) + @location_name = options.delete(:location_name) + @algorithm = options.delete(:algorithm) + @digest = ChecksumAlgorithm.digest_for_algorithm(@algorithm) + + @chunk_size = options.delete(:chunk_size) || DEFAULT_CHUNK_SIZE + @overhead_bytes = calculate_overhead(@chunk_size) + @max_chunk_size = @chunk_size - @overhead_bytes + @current_chunk = ''.b + @eof = false end # the size of the application layer aws-chunked + trailer body def size - # compute the number of chunks - # a full chunk has 4 + 4 bytes overhead, a partial chunk is len.to_s(16).size + 4 orig_body_size = @io.size - n_full_chunks = orig_body_size / CHUNK_SIZE - partial_bytes = orig_body_size % CHUNK_SIZE - chunked_body_size = n_full_chunks * (CHUNK_SIZE + 8) - chunked_body_size += partial_bytes.to_s(16).size + partial_bytes + 4 unless partial_bytes.zero? + n_full_chunks = orig_body_size / @max_chunk_size + partial_bytes = orig_body_size % @max_chunk_size + + chunked_body_size = n_full_chunks * (@max_chunk_size + @max_chunk_size.to_s(16).size + 4) + chunked_body_size += partial_bytes.to_s(16).size + partial_bytes + 4 unless partial_bytes.zero? trailer_size = ChecksumAlgorithm.trailer_length(@algorithm, @location_name) chunked_body_size + trailer_size end def rewind @io.rewind + @current_chunk = ''.b + @eof = false end def read(length, buf = nil) - # account for possible leftover bytes at the end, if we have trailer bytes, send them - if @trailer_io - return @trailer_io.read(length, buf) - end + return if @eof + + buf&.clear + output_buffer = buf || ''.b + fill_chunk(length) if @current_chunk.empty? && !@eof + + output_buffer << @current_chunk + @current_chunk.clear + output_buffer + end + + private + + def calculate_overhead(chunk_size) + chunk_size.to_s(16).size + 4 # hex_length + "\r\n\r\n" + end - chunk = @io.read(length) + def fill_chunk(_length) + chunk = @io.read(@max_chunk_size) if chunk @digest.update(chunk) - application_chunked = "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n" - return StringIO.new(application_chunked).read(application_chunked.size, buf) + @current_chunk << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n" else - trailers = {} - trailers[@location_name] = @digest.base64digest - trailers = trailers.map { |k,v| "#{k}:#{v}" }.join("\r\n") - @trailer_io = StringIO.new("0\r\n#{trailers}\r\n\r\n") - chunk = @trailer_io.read(length, buf) + trailer_str = { @location_name => @digest.base64digest }.map { |k, v| "#{k}:#{v}" }.join("\r\n") + @current_chunk << "0\r\n#{trailer_str}\r\n\r\n" + @eof = true end - chunk end end end From dc9b6037edb897ee60c8d75b5450cce1b4362e7f Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Wed, 26 Nov 2025 09:09:38 -0800 Subject: [PATCH 04/30] Update net http patches --- .../lib/seahorse/client/net_http/patches.rb | 55 +++++++++++++++---- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/gems/aws-sdk-core/lib/seahorse/client/net_http/patches.rb b/gems/aws-sdk-core/lib/seahorse/client/net_http/patches.rb index d47e41254b3..ec2028d8fb6 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/net_http/patches.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/net_http/patches.rb @@ -6,28 +6,61 @@ module Seahorse module Client # @api private module NetHttp - # @api private module Patches - def self.apply! - Net::HTTPGenericRequest.prepend(PatchDefaultContentType) + Net::HTTPGenericRequest.prepend(RequestPatches) end - # For requests with bodies, Net::HTTP sets a default content type of: - # 'application/x-www-form-urlencoded' - # There are cases where we should not send content type at all. - # Even when no body is supplied, Net::HTTP uses a default empty body - # and sets it anyway. This patch disables the behavior when a Thread - # local variable is set. - module PatchDefaultContentType + # Patches intended to override Net::HTTP functionality + module RequestPatches + # For requests with bodies, Net::HTTP sets a default content type of: + # 'application/x-www-form-urlencoded' + # There are cases where we should not send content type at all. + # Even when no body is supplied, Net::HTTP uses a default empty body + # and sets it anyway. This patch disables the behavior when a Thread + # local variable is set. def supply_default_content_type return if Thread.current[:net_http_skip_default_content_type] super end - end + # IO.copy_stream is capped at 16KB buffer so this patch intends to + # increase its chunk size for better performance. + # Only intended to use for S3 TM implementation. + def send_request_with_body_stream(sock, ver, path, f) + return super unless Thread.current[:net_http_override_body_stream_chunk] + + unless content_length || chunked? + raise ArgumentError, 'Content-Length not given and Transfer-Encoding is not `chunked`' + end + + supply_default_content_type + write_header(sock, ver, path) + wait_for_continue sock, ver if sock.continue_timeout + chunk_size = Thread.current[:net_http_override_body_stream_chunk] + if chunked? + chunker = Chunker.new(sock) + RequestIO.custom_stream(f, chunker, chunk_size) # replaces IO.copy_stream + chunker.finish + else + RequestIO.custom_stream(f, sock, chunk_size) + end + end + + class RequestIO + def self.custom_stream(src, dst, chunk_size) + copied = 0 + puts chunk_size + while (chunk = src.read(chunk_size)) + dst.write(chunk) + copied += chunk.bytesize + end + copied + end + end + end end end end From 5d453b3320b6cb0ab0db4f75cba6df2715c74894 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Sat, 6 Dec 2025 12:11:20 -0800 Subject: [PATCH 05/30] Update trailer impl with patch --- .../aws-sdk-core/plugins/checksum_algorithm.rb | 16 ++++++++-------- .../lib/seahorse/client/net_http/patches.rb | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 6ebf27071ef..89cda6460e7 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -461,15 +461,14 @@ def response_header_to_verify(headers, validation_list) # Wrapper for request body that implements application-layer # chunking with Digest computed on chunks + added as a trailer class AwsChunkedTrailerDigestIO - DEFAULT_CHUNK_SIZE = 16_384 + MIN_CHUNK_SIZE = 16_384 def initialize(options = {}) @io = options.delete(:io) @location_name = options.delete(:location_name) @algorithm = options.delete(:algorithm) @digest = ChecksumAlgorithm.digest_for_algorithm(@algorithm) - - @chunk_size = options.delete(:chunk_size) || DEFAULT_CHUNK_SIZE + @chunk_size = Thread.current[:net_http_override_body_stream_chunk] || MIN_CHUNK_SIZE @overhead_bytes = calculate_overhead(@chunk_size) @max_chunk_size = @chunk_size - @overhead_bytes @current_chunk = ''.b @@ -494,12 +493,12 @@ def rewind @eof = false end - def read(length, buf = nil) + def read(_length = nil, buf = nil) return if @eof buf&.clear output_buffer = buf || ''.b - fill_chunk(length) if @current_chunk.empty? && !@eof + fill_chunk if @current_chunk.empty? && !@eof output_buffer << @current_chunk @current_chunk.clear @@ -512,14 +511,15 @@ def calculate_overhead(chunk_size) chunk_size.to_s(16).size + 4 # hex_length + "\r\n\r\n" end - def fill_chunk(_length) + def fill_chunk chunk = @io.read(@max_chunk_size) if chunk + chunk.force_encoding('ASCII-8BIT') @digest.update(chunk) - @current_chunk << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n" + @current_chunk << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n".b else trailer_str = { @location_name => @digest.base64digest }.map { |k, v| "#{k}:#{v}" }.join("\r\n") - @current_chunk << "0\r\n#{trailer_str}\r\n\r\n" + @current_chunk << "0\r\n#{trailer_str}\r\n\r\n".b @eof = true end end diff --git a/gems/aws-sdk-core/lib/seahorse/client/net_http/patches.rb b/gems/aws-sdk-core/lib/seahorse/client/net_http/patches.rb index ec2028d8fb6..113c8c62eb6 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/net_http/patches.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/net_http/patches.rb @@ -20,6 +20,7 @@ module RequestPatches # Even when no body is supplied, Net::HTTP uses a default empty body # and sets it anyway. This patch disables the behavior when a Thread # local variable is set. + # See: https://github.com/ruby/net-http/issues/205 def supply_default_content_type return if Thread.current[:net_http_skip_default_content_type] @@ -29,8 +30,9 @@ def supply_default_content_type # IO.copy_stream is capped at 16KB buffer so this patch intends to # increase its chunk size for better performance. # Only intended to use for S3 TM implementation. + # See: https://github.com/ruby/net-http/blob/master/lib/net/http/generic_request.rb#L292 def send_request_with_body_stream(sock, ver, path, f) - return super unless Thread.current[:net_http_override_body_stream_chunk] + return super unless (chunk_size = Thread.current[:net_http_override_body_stream_chunk]) unless content_length || chunked? raise ArgumentError, 'Content-Length not given and Transfer-Encoding is not `chunked`' @@ -39,10 +41,9 @@ def send_request_with_body_stream(sock, ver, path, f) supply_default_content_type write_header(sock, ver, path) wait_for_continue sock, ver if sock.continue_timeout - chunk_size = Thread.current[:net_http_override_body_stream_chunk] if chunked? chunker = Chunker.new(sock) - RequestIO.custom_stream(f, chunker, chunk_size) # replaces IO.copy_stream + RequestIO.custom_stream(f, chunker, chunk_size) chunker.finish else RequestIO.custom_stream(f, sock, chunk_size) @@ -52,7 +53,6 @@ def send_request_with_body_stream(sock, ver, path, f) class RequestIO def self.custom_stream(src, dst, chunk_size) copied = 0 - puts chunk_size while (chunk = src.read(chunk_size)) dst.write(chunk) copied += chunk.bytesize From 81dfafd20724522a3e7b3483e9cc47b116ac8ad4 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Sat, 6 Dec 2025 12:38:49 -0800 Subject: [PATCH 06/30] Update MPU with custom chunk size --- .../lib/aws-sdk-core/plugins/checksum_algorithm.rb | 5 ++--- gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb | 10 +++++++++- .../lib/aws-sdk-s3/multipart_file_uploader.rb | 3 +++ gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb | 11 +++++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 89cda6460e7..42129776f3c 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -5,6 +5,7 @@ module Plugins # @api private class ChecksumAlgorithm < Seahorse::Client::Plugin CHUNK_SIZE = 1 * 1024 * 1024 # one MB + MIN_CHUNK_SIZE = 16_384 # 16 KB # determine the set of supported client side checksum algorithms # CRC32c requires aws-crt (optional sdk dependency) for support @@ -461,8 +462,6 @@ def response_header_to_verify(headers, validation_list) # Wrapper for request body that implements application-layer # chunking with Digest computed on chunks + added as a trailer class AwsChunkedTrailerDigestIO - MIN_CHUNK_SIZE = 16_384 - def initialize(options = {}) @io = options.delete(:io) @location_name = options.delete(:location_name) @@ -489,7 +488,7 @@ def size def rewind @io.rewind - @current_chunk = ''.b + @current_chunk.clear @eof = false end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb index 62dbb07f8c3..8a87c478fc2 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb @@ -15,6 +15,7 @@ class FileUploader def initialize(options = {}) @client = options[:client] || Client.new @executor = options[:executor] + @http_chunk_size = options[:http_chunk_size] @multipart_threshold = options[:multipart_threshold] || DEFAULT_MULTIPART_THRESHOLD end @@ -37,7 +38,11 @@ def initialize(options = {}) def upload(source, options = {}) Aws::Plugins::UserAgent.metric('S3_TRANSFER') do if File.size(source) >= @multipart_threshold - MultipartFileUploader.new(client: @client, executor: @executor).upload(source, options) + MultipartFileUploader.new( + client: @client, + executor: @executor, + http_chunk_size: @http_chunk_size + ).upload(source, options) else put_object(source, options) end @@ -59,7 +64,10 @@ def put_object(source, options) options[:on_chunk_sent] = single_part_progress(callback) end open_file(source) do |file| + Thread.current[:net_http_override_body_stream_chunk] = @http_chunk_size if @http_chunk_size @client.put_object(options.merge(body: file)) + ensure + Thread.current[:net_http_override_body_stream_chunk] = nil end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb index aac37851af7..f46a3e510fb 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb @@ -22,6 +22,7 @@ class MultipartFileUploader def initialize(options = {}) @client = options[:client] || Client.new @executor = options[:executor] + @http_chunk_size = options[:http_chunk_size] end # @return [Client] @@ -150,6 +151,7 @@ def upload_with_executor(pending, completed, options) upload_attempts += 1 @executor.post(part) do |p| + Thread.current[:net_http_override_body_stream_chunk] = @http_chunk_size if @http_chunk_size update_progress(progress, p) resp = @client.upload_part(p) p[:body].close @@ -160,6 +162,7 @@ def upload_with_executor(pending, completed, options) abort_upload = true errors << e ensure + Thread.current[:net_http_override_body_stream_chunk] = nil if @http_chunk_size completion_queue << :done end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb index 94ab1077c4c..a088dfe6bea 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb @@ -205,6 +205,11 @@ def download_file(destination, bucket:, key:, **options) # @option options [Integer] :thread_count (10) Customize threads used in the multipart upload. # Only used when no custom executor is provided (creates {DefaultExecutor} with the given thread count). # + # @option option [Integer] :http_chunk_size (16384) Sizes in bytes for HTTP request body processing. + # Controls how much data is processed at once during S3 uploads. Default value is 16384 bytes (16KB). + # Larger values use more memory but may be faster due to fewer I/O operations. + # Custom values must be at least 16KB. + # # @option options [Proc] :progress_callback (nil) # A Proc that will be called when each chunk of the upload is sent. # It will be invoked with `[bytes_read]` and `[total_sizes]`. @@ -221,9 +226,15 @@ def download_file(destination, bucket:, key:, **options) # @see Client#upload_part def upload_file(source, bucket:, key:, **options) upload_opts = options.merge(bucket: bucket, key: key) + http_chunk_size = upload_opts.delete(:http_chunk_size) + if http_chunk_size && http_chunk_size < Aws::Plugins::ChecksumAlgorithm::MIN_CHUNK_SIZE + raise ArgumentError, ':http_chunk_size must be at least 16384 bytes (16KB)' + end + executor = @executor || DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count)) uploader = FileUploader.new( multipart_threshold: upload_opts.delete(:multipart_threshold), + http_chunk_size: http_chunk_size, client: @client, executor: executor ) From 14b7f347d7d3fb755b30ddc9035906028ba85eca Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Sat, 6 Dec 2025 12:41:18 -0800 Subject: [PATCH 07/30] Generated S3 gem --- gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb | 4 ++-- gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb index d8b8a7ba4b1..1438ec0dd4b 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb @@ -17659,7 +17659,7 @@ def put_bucket_website(params = {}, options = {}) # [3]: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-using-rest-api.html # [4]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/about-object-ownership.html # - # @option params [String, StringIO, File] :body + # @option params [String, IO] :body # Object data. # # @option params [required, String] :bucket @@ -20968,7 +20968,7 @@ def update_bucket_metadata_journal_table_configuration(params = {}, options = {} # [16]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html # [17]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html # - # @option params [String, StringIO, File] :body + # @option params [String, IO] :body # Object data. # # @option params [required, String] :bucket diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb index 2ef351a0b09..6a771a29e4a 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb @@ -4121,6 +4121,7 @@ module ClientApi "requestAlgorithmMember" => "checksum_algorithm", "requestChecksumRequired" => false, } + o['unsignedPayload'] = true o.input = Shapes::ShapeRef.new(shape: PutObjectRequest) o.output = Shapes::ShapeRef.new(shape: PutObjectOutput) o.errors << Shapes::ShapeRef.new(shape: InvalidRequest) @@ -4309,6 +4310,7 @@ module ClientApi "requestAlgorithmMember" => "checksum_algorithm", "requestChecksumRequired" => false, } + o['unsignedPayload'] = true o.input = Shapes::ShapeRef.new(shape: UploadPartRequest) o.output = Shapes::ShapeRef.new(shape: UploadPartOutput) end) From 06e08f2a7bef22d8f01d43719e4ece837c121371 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Sat, 6 Dec 2025 13:38:38 -0800 Subject: [PATCH 08/30] Fix failing tests --- gems/aws-sdk-s3/spec/client_spec.rb | 19 +-- .../aws-sdk-s3/spec/encryption/client_spec.rb | 127 +++++++++--------- .../spec/plugins/express_session_auth_spec.rb | 2 +- 3 files changed, 69 insertions(+), 79 deletions(-) diff --git a/gems/aws-sdk-s3/spec/client_spec.rb b/gems/aws-sdk-s3/spec/client_spec.rb index 1fabee78a0f..18faa626788 100644 --- a/gems/aws-sdk-s3/spec/client_spec.rb +++ b/gems/aws-sdk-s3/spec/client_spec.rb @@ -167,7 +167,8 @@ module S3 tmpfile.unlink s3 = Client.new(stub_responses: true) resp = s3.put_object(bucket: 'bucket', key: 'key', body: tmpfile) - expect(resp.context.http_request.body_contents).to eq(data) + # Need to discuss + expect(resp.context.http_request.body.instance_variable_get(:@io).read).to eq(data) end end @@ -176,14 +177,7 @@ module S3 closed_file = File.open(__FILE__, 'rb') closed_file.close client = Client.new(stub_responses: true) - resp = client.put_object( - bucket: 'aws-sdk', key: 'key', body: closed_file - ) - body = resp.context.http_request.body - expect(body).to be_kind_of(File) - expect(body.path).to eq(__FILE__) - expect(body).not_to be(closed_file) - expect(body.closed?).to be(true) + expect { client.put_object(bucket: 'aws-sdk', key: 'key', body: closed_file) }.not_to raise_error end it 'accepts closed Tempfile objects' do @@ -191,12 +185,7 @@ module S3 tmpfile.write('abc') tmpfile.close client = Client.new(stub_responses: true) - resp = client.put_object(bucket: 'aws-sdk', key: 'key', body: tmpfile) - body = resp.context.http_request.body - expect(body).to be_kind_of(File) - expect(body.path).to eq(tmpfile.path) - expect(body).not_to be(tmpfile) - expect(body.closed?).to be(true) + expect { client.put_object(bucket: 'aws-sdk', key: 'key', body: tmpfile) }.not_to raise_error end end diff --git a/gems/aws-sdk-s3/spec/encryption/client_spec.rb b/gems/aws-sdk-s3/spec/encryption/client_spec.rb index f1662582b08..1e6be90073d 100644 --- a/gems/aws-sdk-s3/spec/encryption/client_spec.rb +++ b/gems/aws-sdk-s3/spec/encryption/client_spec.rb @@ -121,6 +121,19 @@ module Encryption end describe 'encryption methods' do + def extract_chunked_content(chunked_body) + lines = chunked_body.split("\r\n") + return nil unless lines.length >= 2 + + hex_size = lines[0] + content = lines[1] + + # Verify the hex size matches content length + return content if hex_size.to_i(16) == content.bytesize + + nil + end + # this is the encrypted string "secret" using the fixed envelope # keys defined below in the before(:each) block let(:encrypted_body) { Base64.decode64('JIgXCTXpeQerPLiU6dVL4Q==') } @@ -138,28 +151,24 @@ module Encryption describe '#put_object' do it 'encrypts the data client-side' do - stub_request( - :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' - ) + stub_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') client.put_object(bucket: 'bucket', key: 'key', body: 'secret') - expect( - a_request( - :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' - ).with( - body: encrypted_body, + expect(a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') + .with( + body: lambda { |body| + actual_content = extract_chunked_content(body) + actual_content == encrypted_body + }, headers: { - 'Content-Length' => '16', - # key is encrypted here with the master encryption key, - # then base64 encoded - 'X-Amz-Meta-X-Amz-Key' => 'gX+a4JQYj7FP0y5TAAvxTz4e'\ - '2l0DvOItbXByml/NPtKQcUls'\ - 'oGHoYR/T0TuYHcNj', + 'Content-Length' => '58', + # key is encrypted here with the master encryption key, then base64 encoded + 'X-Amz-Meta-X-Amz-Key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOItbXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', 'X-Amz-Meta-X-Amz-Iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', 'X-Amz-Meta-X-Amz-Matdesc' => '{}', 'X-Amz-Meta-X-Amz-Unencrypted-Content-Length' => '6' } - ) - ).to have_been_made.once + )) + .to have_been_made.once end it 'encrypts an empty or missing body' do @@ -175,45 +184,37 @@ module Encryption end it 'can store the encryption envelope in a separate object' do - stub_request( - :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' - ) - stub_request( - :put, - 'https://bucket.s3.us-west-1.amazonaws.com/key.instruction' - ) + stub_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') + stub_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key.instruction') options[:envelope_location] = :instruction_file client.put_object(bucket: 'bucket', key: 'key', body: 'secret') - # first request stores the encryption materials in the - # instruction file - expect( - a_request( - :put, - 'https://bucket.s3.us-west-1.amazonaws.com/key.instruction' - ).with( - body: Json.dump( - 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOIt'\ - 'bXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', - 'x-amz-iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', - 'x-amz-matdesc' => '{}' - ) - ) - ).to have_been_made.once + # first request stores the encryption materials in the instruction file + expect(a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key.instruction') + .with( + body: lambda { |body| + actual_content = extract_chunked_content(body) + expected_content = Json.dump( + 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOItbXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', + 'x-amz-iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', + 'x-amz-matdesc' => '{}' + ) + actual_content == expected_content + } + )) + .to have_been_made.once # second request stores teh encrypted object - expect( - a_request( - :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' - ).with( - body: encrypted_body, - headers: { - 'Content-Length' => '16', - 'X-Amz-Meta-X-Amz-Unencrypted-Content-Length' => '6' - } - ) - ).to have_been_made.once + expect(a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') + .with( + body: lambda { |body| + actual_content = extract_chunked_content(body) + actual_content == encrypted_body + }, + headers: { 'Content-Length' => '58', 'X-Amz-Meta-X-Amz-Unencrypted-Content-Length' => '6' } + )) + .to have_been_made.once end it 'accpets a custom instruction file suffix' do @@ -233,20 +234,17 @@ module Encryption end it 'does not set the un-encrypted md5 header' do - stub_request( - :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' - ) + stub_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') expect_any_instance_of(EncryptHandler).to receive(:warn) - client.put_object( - bucket: 'bucket', key: 'key', body: 'secret', content_md5: 'MD5' - ) - expect( - a_request( - :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' - ).with( - body: encrypted_body - ) - ).to have_been_made.once + client.put_object(bucket: 'bucket', key: 'key', body: 'secret', content_md5: 'MD5') + expect(a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') + .with( + body: lambda { |body| + actual_content = extract_chunked_content(body) + actual_content == encrypted_body + } + )) + .to have_been_made.once end it 'supports encryption with an asymmetric key pair' do @@ -583,9 +581,12 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') envelope.each do |key, value| expect(headers["x-amz-meta-#{key}"]).to eq(value) end + # TODO: fails due to encoding issues + # Our trailer implementation uses US-ASCII encoding + # but the value below is based on UTF-8 expect( Base64.encode64(resp.context.http_request.body_contents) - ).to eq("4FAj3kTOIisQ+9b8/kia8g==\n") + ).to eq("MTANCuBQI95EziIrEPvW/P5ImvINCg==\n") end it 'supports decryption via KMS w/ CBC' do diff --git a/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb b/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb index 60d165a3237..eeffe8dc150 100644 --- a/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb +++ b/gems/aws-sdk-s3/spec/plugins/express_session_auth_spec.rb @@ -147,7 +147,7 @@ module S3 key: 'key', body: 'body' ) - expect(resp.context.http_request.headers['x-amz-checksum-crc32']).to_not be_nil + expect(resp.context.http_request.headers['x-amz-sdk-checksum-algorithm']).to eq('CRC32') end end From ad14d42b68c890a063dc23ca9217520317974a21 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Sat, 6 Dec 2025 13:40:24 -0800 Subject: [PATCH 09/30] Add temp changelog --- build_tools/services.rb | 4 ++-- gems/aws-sdk-core/CHANGELOG.md | 2 ++ gems/aws-sdk-s3/CHANGELOG.md | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build_tools/services.rb b/build_tools/services.rb index 0905296c500..fcfaa613138 100644 --- a/build_tools/services.rb +++ b/build_tools/services.rb @@ -9,10 +9,10 @@ class ServiceEnumerator MANIFEST_PATH = File.expand_path('../../services.json', __FILE__) # Minimum `aws-sdk-core` version for new gem builds - MINIMUM_CORE_VERSION = "3.239.1" + MINIMUM_CORE_VERSION = "3.240.0" # Minimum `aws-sdk-core` version for new S3 gem builds - MINIMUM_CORE_VERSION_S3 = "3.234.0" + MINIMUM_CORE_VERSION_S3 = "3.240.0" EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration" diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 10908bdd6c9..e8227d1f976 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - TBD + 3.239.2 (2025-11-25) ------------------ diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index 4eb221cd331..e9bb4cb86bf 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - TBD + 1.206.0 (2025-12-02) ------------------ From de72cf12b167dbd640e770533b9add6c01648724 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Sun, 7 Dec 2025 14:14:53 -0800 Subject: [PATCH 10/30] Add http_chunk_size spec --- gems/aws-sdk-s3/spec/transfer_manager_spec.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb index a752b35f0f6..28e5db63ee5 100644 --- a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -95,6 +95,22 @@ module S3 expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: large_file }) subject.upload_file(large_file, bucket: 'bucket', key: 'key', multipart_threshold: 200 * one_mb_size) end + + context ':http_check_size' do + it 'sets chunk size greater than 16KB' do + subject.upload_file(file, bucket: 'bucket', key: 'key', http_chunk_size: 32_768) do |resp| + expect(resp.context.http_request.body).to be_a(Aws::Plugins::ChecksumAlgorithm::AwsChunkedTrailerDigestIO) + expect(resp.context.http_request.body.instance_variable_get(:@chunk_size)).to eq(32_768) + end + end + + it 'raises error when less than 16KB' do + expect do + subject.upload_file(large_file, bucket: 'bucket', key: 'key', http_chunk_size: 100) + end.to raise_error(ArgumentError, /:http_chunk_size must be at least 16384 bytes/) + end + end + end describe '#upload_stream', :jruby_flaky do From 4abf3f8381c1c4cbeafaac6ff1bc2c8e3b7fc5a6 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Sun, 7 Dec 2025 14:32:02 -0800 Subject: [PATCH 11/30] Improve documentation --- gems/aws-sdk-core/CHANGELOG.md | 2 +- gems/aws-sdk-s3/CHANGELOG.md | 5 ++++- gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb | 7 +++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index e8227d1f976..c220b126f4d 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,7 +1,7 @@ Unreleased Changes ------------------ -* Feature - TBD +* Feature - Added `:http_chunk_size` parameter to `TransferManager#upload_file` to control the buffer size when streaming request bodies over HTTP. Larger chunk sizes may improve network throughput at the cost of higher memory usage. 3.239.2 (2025-11-25) ------------------ diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index e9bb4cb86bf..675b98edab8 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,7 +1,10 @@ Unreleased Changes ------------------ -* Feature - TBD +* Feature - `Aws::S3::TransferManager#upload_file` supports a new parameter, `:http_chunk_size` to control +chunk size + +* Feature - Improved memory efficiency when calculating request checksums for large file uploads. 1.206.0 (2025-12-02) ------------------ diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb index a088dfe6bea..e0f945c1dc4 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb @@ -205,10 +205,9 @@ def download_file(destination, bucket:, key:, **options) # @option options [Integer] :thread_count (10) Customize threads used in the multipart upload. # Only used when no custom executor is provided (creates {DefaultExecutor} with the given thread count). # - # @option option [Integer] :http_chunk_size (16384) Sizes in bytes for HTTP request body processing. - # Controls how much data is processed at once during S3 uploads. Default value is 16384 bytes (16KB). - # Larger values use more memory but may be faster due to fewer I/O operations. - # Custom values must be at least 16KB. + # @option option [Integer] :http_chunk_size (16384) Size in bytes for each chunk when streaming request bodies + # over HTTP. Controls the buffer size used when sending data to S3. Larger values may improve throughput by + # reducing the number of network writes, but use more memory. Custom values must be at least 16KB. # # @option options [Proc] :progress_callback (nil) # A Proc that will be called when each chunk of the upload is sent. From 4fbff2cf20913ea3cff8395c38944c11bb3a6f2b Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Sun, 7 Dec 2025 14:42:55 -0800 Subject: [PATCH 12/30] Fix changelog entries --- gems/aws-sdk-core/CHANGELOG.md | 2 +- gems/aws-sdk-s3/CHANGELOG.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index c220b126f4d..044304b3595 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,7 +1,7 @@ Unreleased Changes ------------------ -* Feature - Added `:http_chunk_size` parameter to `TransferManager#upload_file` to control the buffer size when streaming request bodies over HTTP. Larger chunk sizes may improve network throughput at the cost of higher memory usage. +* Feature - Improved memory efficiency when calculating request checksums. 3.239.2 (2025-11-25) ------------------ diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index 675b98edab8..1ba155b05a3 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,8 +1,7 @@ Unreleased Changes ------------------ -* Feature - `Aws::S3::TransferManager#upload_file` supports a new parameter, `:http_chunk_size` to control -chunk size +* Feature - Added `:http_chunk_size` parameter to `TransferManager#upload_file` to control the buffer size when streaming request bodies over HTTP. Larger chunk sizes may improve network throughput at the cost of higher memory usage. * Feature - Improved memory efficiency when calculating request checksums for large file uploads. From 41ae00ca531f2d450b6e4d750462d566b9ed50fc Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Thu, 11 Dec 2025 10:09:04 -0800 Subject: [PATCH 13/30] Add chunk testing --- gems/aws-sdk-s3/spec/transfer_manager_spec.rb | 85 ++++++++++++++++++- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb index 28e5db63ee5..d77eaf4de89 100644 --- a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -97,11 +97,88 @@ module S3 end context ':http_check_size' do - it 'sets chunk size greater than 16KB' do - subject.upload_file(file, bucket: 'bucket', key: 'key', http_chunk_size: 32_768) do |resp| - expect(resp.context.http_request.body).to be_a(Aws::Plugins::ChecksumAlgorithm::AwsChunkedTrailerDigestIO) - expect(resp.context.http_request.body.instance_variable_get(:@chunk_size)).to eq(32_768) + before do + WebMock.disable! + end + + after do + WebMock.enable! + end + + let(:port) { 1234 } + let(:test_file) do + file = '/tmp/test_upload_file' + File.write(file, 'x' * 65_536) + file + end + + let(:tm) do + client = Aws::S3::Client.new( + endpoint: "http://localhost:#{port}", + region: 'us-east-1', + access_key_id: 't', + secret_access_key: 't' + ) + Aws::S3::TransferManager.new(client: client) + end + + def start_mirror_server(port, chunk_size) + server = TCPServer.new(port) + chunks = [] + + server_thread = Thread.new do + client = server.accept + + headers = '' + while (line = client.gets) + headers += line + break if line.strip.empty? + end + + if headers.include?('Expect: 100-continue') + client.write("HTTP/1.1 100 Continue\r\n\r\n") + + loop do + sleep(0.001) # needs wait between reads + data = client.read_nonblock(chunk_size, exception: false) + break if data == :wait_readable || data.nil? + + chunks << data.size + end + end + + chunks << headers.length + client.write("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + ensure + client.close end + [server, server_thread, chunks] + end + + it 'uses the given chunk size when uploading' do + chunk_size = 32_768 + server, server_thread, chunks = start_mirror_server(port, chunk_size) + tm.upload_file(test_file, bucket: 'test-bucket', key: 'test-key', http_chunk_size: chunk_size) + + server_thread.join + expect(chunks.first).to eq(chunk_size) + expect(chunks.sum).to eq(66_514) # includes trailing bytes + ensure + File.delete(test_file) + server.close + end + + it 'uses default chunk size' do + chunk_size = 16_384 + server, server_thread, chunks = start_mirror_server(port, chunk_size) + tm.upload_file(test_file, bucket: 'test-bucket', key: 'test-key') + + server_thread.join + expect(chunks.first).to eq(chunk_size) + expect(chunks.sum).to eq(66_530) # includes trailing bytes + ensure + File.delete(test_file) + server.close end it 'raises error when less than 16KB' do From c935ea7c5a8e5a51e055e84c4e89e74ab8fd3d55 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Thu, 11 Dec 2025 10:11:05 -0800 Subject: [PATCH 14/30] Remove todo --- gems/aws-sdk-s3/spec/client_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/gems/aws-sdk-s3/spec/client_spec.rb b/gems/aws-sdk-s3/spec/client_spec.rb index 18faa626788..b903ad2d5f4 100644 --- a/gems/aws-sdk-s3/spec/client_spec.rb +++ b/gems/aws-sdk-s3/spec/client_spec.rb @@ -167,7 +167,6 @@ module S3 tmpfile.unlink s3 = Client.new(stub_responses: true) resp = s3.put_object(bucket: 'bucket', key: 'key', body: tmpfile) - # Need to discuss expect(resp.context.http_request.body.instance_variable_get(:@io).read).to eq(data) end end From cc98a3668753496d0c883103a949967d132b7f7f Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Thu, 11 Dec 2025 12:57:25 -0800 Subject: [PATCH 15/30] Address feedbacks --- .../plugins/checksum_algorithm.rb | 11 ++-- gems/aws-sdk-s3/spec/transfer_manager_spec.rb | 51 ++++++++++--------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 42129776f3c..4aedff7d088 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -354,6 +354,7 @@ def calculate_checksum(algorithm, body) if body.respond_to?(:read) body.rewind update_in_chunks(digest, body) + body.rewind else digest.update(body) end @@ -469,7 +470,7 @@ def initialize(options = {}) @digest = ChecksumAlgorithm.digest_for_algorithm(@algorithm) @chunk_size = Thread.current[:net_http_override_body_stream_chunk] || MIN_CHUNK_SIZE @overhead_bytes = calculate_overhead(@chunk_size) - @max_chunk_size = @chunk_size - @overhead_bytes + @base_chunk_size = @chunk_size - @overhead_bytes @current_chunk = ''.b @eof = false end @@ -477,10 +478,10 @@ def initialize(options = {}) # the size of the application layer aws-chunked + trailer body def size orig_body_size = @io.size - n_full_chunks = orig_body_size / @max_chunk_size - partial_bytes = orig_body_size % @max_chunk_size + n_full_chunks = orig_body_size / @base_chunk_size + partial_bytes = orig_body_size % @base_chunk_size - chunked_body_size = n_full_chunks * (@max_chunk_size + @max_chunk_size.to_s(16).size + 4) + chunked_body_size = n_full_chunks * (@base_chunk_size + @base_chunk_size.to_s(16).size + 4) chunked_body_size += partial_bytes.to_s(16).size + partial_bytes + 4 unless partial_bytes.zero? trailer_size = ChecksumAlgorithm.trailer_length(@algorithm, @location_name) chunked_body_size + trailer_size @@ -511,7 +512,7 @@ def calculate_overhead(chunk_size) end def fill_chunk - chunk = @io.read(@max_chunk_size) + chunk = @io.read(@base_chunk_size) if chunk chunk.force_encoding('ASCII-8BIT') @digest.update(chunk) diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb index d77eaf4de89..2b579fc7c4b 100644 --- a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative 'spec_helper' +require 'socket' require 'tempfile' module Aws @@ -105,30 +106,20 @@ module S3 WebMock.enable! end - let(:port) { 1234 } let(:test_file) do - file = '/tmp/test_upload_file' - File.write(file, 'x' * 65_536) - file - end - - let(:tm) do - client = Aws::S3::Client.new( - endpoint: "http://localhost:#{port}", - region: 'us-east-1', - access_key_id: 't', - secret_access_key: 't' - ) - Aws::S3::TransferManager.new(client: client) + Tempfile.new('test_upload_file').tap do |f| + f.write('x' * 65_536) + f.rewind + end end - def start_mirror_server(port, chunk_size) - server = TCPServer.new(port) + def start_mirror_server(chunk_size) + server = TCPServer.new('localhost', 0) + port = server.addr[1] chunks = [] server_thread = Thread.new do client = server.accept - headers = '' while (line = client.gets) headers += line @@ -152,32 +143,44 @@ def start_mirror_server(port, chunk_size) ensure client.close end - [server, server_thread, chunks] + [server, server_thread, chunks, port] end it 'uses the given chunk size when uploading' do chunk_size = 32_768 - server, server_thread, chunks = start_mirror_server(port, chunk_size) + server, server_thread, chunks, port = start_mirror_server(chunk_size) + client = Aws::S3::Client.new( + endpoint: "http://localhost:#{port}", + region: 'us-east-1', + access_key_id: 't', + secret_access_key: 't' + ) + tm = Aws::S3::TransferManager.new(client: client) tm.upload_file(test_file, bucket: 'test-bucket', key: 'test-key', http_chunk_size: chunk_size) server_thread.join expect(chunks.first).to eq(chunk_size) - expect(chunks.sum).to eq(66_514) # includes trailing bytes + expect(chunks.sum).to eq(66_515) # includes trailing bytes ensure - File.delete(test_file) server.close end it 'uses default chunk size' do chunk_size = 16_384 - server, server_thread, chunks = start_mirror_server(port, chunk_size) + server, server_thread, chunks, port = start_mirror_server(chunk_size) + client = Aws::S3::Client.new( + endpoint: "http://localhost:#{port}", + region: 'us-east-1', + access_key_id: 't', + secret_access_key: 't' + ) + tm = Aws::S3::TransferManager.new(client: client) tm.upload_file(test_file, bucket: 'test-bucket', key: 'test-key') server_thread.join expect(chunks.first).to eq(chunk_size) - expect(chunks.sum).to eq(66_530) # includes trailing bytes + expect(chunks.sum).to eq(66_531) ensure - File.delete(test_file) server.close end From 81b6be5313afd1e95fc89bc7eb026e4fd844a130 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Thu, 11 Dec 2025 13:03:06 -0800 Subject: [PATCH 16/30] Add fix --- gems/aws-sdk-s3/spec/transfer_manager_spec.rb | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb index 2b579fc7c4b..0f81d9600b1 100644 --- a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -98,14 +98,6 @@ module S3 end context ':http_check_size' do - before do - WebMock.disable! - end - - after do - WebMock.enable! - end - let(:test_file) do Tempfile.new('test_upload_file').tap do |f| f.write('x' * 65_536) @@ -114,6 +106,7 @@ module S3 end def start_mirror_server(chunk_size) + server = TCPServer.new('localhost', 0) port = server.addr[1] chunks = [] @@ -147,6 +140,7 @@ def start_mirror_server(chunk_size) end it 'uses the given chunk size when uploading' do + WebMock.disable! chunk_size = 32_768 server, server_thread, chunks, port = start_mirror_server(chunk_size) client = Aws::S3::Client.new( @@ -163,9 +157,11 @@ def start_mirror_server(chunk_size) expect(chunks.sum).to eq(66_515) # includes trailing bytes ensure server.close + WebMock.enable! end it 'uses default chunk size' do + WebMock.disable! chunk_size = 16_384 server, server_thread, chunks, port = start_mirror_server(chunk_size) client = Aws::S3::Client.new( @@ -182,6 +178,7 @@ def start_mirror_server(chunk_size) expect(chunks.sum).to eq(66_531) ensure server.close + WebMock.enable! end it 'raises error when less than 16KB' do From 78554ac86e2f2f68c31cb9896f470fc76ff60380 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Thu, 11 Dec 2025 13:12:16 -0800 Subject: [PATCH 17/30] Fix changelog entry --- gems/aws-sdk-s3/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index 1ba155b05a3..b7910b4b7ec 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -3,7 +3,7 @@ Unreleased Changes * Feature - Added `:http_chunk_size` parameter to `TransferManager#upload_file` to control the buffer size when streaming request bodies over HTTP. Larger chunk sizes may improve network throughput at the cost of higher memory usage. -* Feature - Improved memory efficiency when calculating request checksums for large file uploads. +* Feature - Improved memory efficiency when calculating request checksums for large file uploads. 1.206.0 (2025-12-02) ------------------ From cf84f0200bb63c473db9e132e42d397f52acb6d5 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Thu, 11 Dec 2025 13:23:47 -0800 Subject: [PATCH 18/30] Fix overhead tests --- gems/aws-sdk-s3/spec/transfer_manager_spec.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb index 0f81d9600b1..9e60d05e825 100644 --- a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -130,8 +130,6 @@ def start_mirror_server(chunk_size) chunks << data.size end end - - chunks << headers.length client.write("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") ensure client.close @@ -154,7 +152,6 @@ def start_mirror_server(chunk_size) server_thread.join expect(chunks.first).to eq(chunk_size) - expect(chunks.sum).to eq(66_515) # includes trailing bytes ensure server.close WebMock.enable! @@ -175,7 +172,6 @@ def start_mirror_server(chunk_size) server_thread.join expect(chunks.first).to eq(chunk_size) - expect(chunks.sum).to eq(66_531) ensure server.close WebMock.enable! From 1bfa7ec0637785ef535daadd6f566ce503c35a10 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Thu, 11 Dec 2025 14:06:48 -0800 Subject: [PATCH 19/30] More changes --- gems/aws-sdk-s3/spec/transfer_manager_spec.rb | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb index 9e60d05e825..4be82621188 100644 --- a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -134,13 +134,13 @@ def start_mirror_server(chunk_size) ensure client.close end - [server, server_thread, chunks, port] + [server, server_thread, port] end it 'uses the given chunk size when uploading' do WebMock.disable! chunk_size = 32_768 - server, server_thread, chunks, port = start_mirror_server(chunk_size) + server, server_thread, port = start_mirror_server(chunk_size) client = Aws::S3::Client.new( endpoint: "http://localhost:#{port}", region: 'us-east-1', @@ -148,19 +148,28 @@ def start_mirror_server(chunk_size) secret_access_key: 't' ) tm = Aws::S3::TransferManager.new(client: client) - tm.upload_file(test_file, bucket: 'test-bucket', key: 'test-key', http_chunk_size: chunk_size) + read_sizes = [] + + expect(Seahorse::Client::NetHttp::Patches::RequestPatches::RequestIO) + .to receive(:custom_stream).and_call_original + allow_any_instance_of(Aws::Plugins::ChecksumAlgorithm::AwsChunkedTrailerDigestIO) + .to receive(:read).and_wrap_original do |method, size| + read_sizes << size + method.call(size) + end + tm.upload_file(test_file, bucket: 'test-bucket', key: 'test-key', http_chunk_size: chunk_size) server_thread.join - expect(chunks.first).to eq(chunk_size) + expect(read_sizes).to all(eq(chunk_size)) ensure - server.close + server&.close WebMock.enable! end it 'uses default chunk size' do WebMock.disable! chunk_size = 16_384 - server, server_thread, chunks, port = start_mirror_server(chunk_size) + server, server_thread, port = start_mirror_server(chunk_size) client = Aws::S3::Client.new( endpoint: "http://localhost:#{port}", region: 'us-east-1', @@ -168,10 +177,16 @@ def start_mirror_server(chunk_size) secret_access_key: 't' ) tm = Aws::S3::TransferManager.new(client: client) - tm.upload_file(test_file, bucket: 'test-bucket', key: 'test-key') + read_sizes = [] + allow_any_instance_of(Aws::Plugins::ChecksumAlgorithm::AwsChunkedTrailerDigestIO) + .to receive(:read).and_wrap_original do |method, size| + read_sizes << size + method.call(size) + end + tm.upload_file(test_file, bucket: 'test-bucket', key: 'test-key') server_thread.join - expect(chunks.first).to eq(chunk_size) + expect(read_sizes).to all(eq(chunk_size)) ensure server.close WebMock.enable! @@ -183,7 +198,6 @@ def start_mirror_server(chunk_size) end.to raise_error(ArgumentError, /:http_chunk_size must be at least 16384 bytes/) end end - end describe '#upload_stream', :jruby_flaky do From 104cf0358f27d5aadd7e90bd08f8e1dfe4cb2b0f Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Mon, 15 Dec 2025 12:44:42 -0800 Subject: [PATCH 20/30] Fix failing specs --- .../aws-sdk-s3/spec/encryption/client_spec.rb | 140 ++++++++---------- 1 file changed, 61 insertions(+), 79 deletions(-) diff --git a/gems/aws-sdk-s3/spec/encryption/client_spec.rb b/gems/aws-sdk-s3/spec/encryption/client_spec.rb index 1e6be90073d..57f9c17c397 100644 --- a/gems/aws-sdk-s3/spec/encryption/client_spec.rb +++ b/gems/aws-sdk-s3/spec/encryption/client_spec.rb @@ -63,9 +63,7 @@ module Encryption expect do options.delete(:encryption_key) Encryption::Client.new(options) - end.to raise_error( - ArgumentError, /:kms_key_id, :key_provider, or :encryption_key/ - ) + end.to raise_error(ArgumentError, /:kms_key_id, :key_provider, or :encryption_key/) expect do Encryption::Client.new(options.merge(encryption_key: master_key)) @@ -121,19 +119,6 @@ module Encryption end describe 'encryption methods' do - def extract_chunked_content(chunked_body) - lines = chunked_body.split("\r\n") - return nil unless lines.length >= 2 - - hex_size = lines[0] - content = lines[1] - - # Verify the hex size matches content length - return content if hex_size.to_i(16) == content.bytesize - - nil - end - # this is the encrypted string "secret" using the fixed envelope # keys defined below in the before(:each) block let(:encrypted_body) { Base64.decode64('JIgXCTXpeQerPLiU6dVL4Q==') } @@ -153,22 +138,24 @@ def extract_chunked_content(chunked_body) it 'encrypts the data client-side' do stub_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') client.put_object(bucket: 'bucket', key: 'key', body: 'secret') - expect(a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') - .with( - body: lambda { |body| - actual_content = extract_chunked_content(body) - actual_content == encrypted_body - }, + expect( + a_request( + :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' + ).with( + body: ->(b) { b == encrypted_body }, headers: { 'Content-Length' => '58', - # key is encrypted here with the master encryption key, then base64 encoded - 'X-Amz-Meta-X-Amz-Key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOItbXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', + # key is encrypted here with the master encryption key, + # then base64 encoded + 'X-Amz-Meta-X-Amz-Key' => 'gX+a4JQYj7FP0y5TAAvxTz4e'\ + '2l0DvOItbXByml/NPtKQcUls'\ + 'oGHoYR/T0TuYHcNj', 'X-Amz-Meta-X-Amz-Iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', 'X-Amz-Meta-X-Amz-Matdesc' => '{}', 'X-Amz-Meta-X-Amz-Unencrypted-Content-Length' => '6' } - )) - .to have_been_made.once + ) + ).to have_been_made.once end it 'encrypts an empty or missing body' do @@ -190,31 +177,34 @@ def extract_chunked_content(chunked_body) options[:envelope_location] = :instruction_file client.put_object(bucket: 'bucket', key: 'key', body: 'secret') - # first request stores the encryption materials in the instruction file - expect(a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key.instruction') - .with( - body: lambda { |body| - actual_content = extract_chunked_content(body) - expected_content = Json.dump( - 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOItbXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', - 'x-amz-iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', - 'x-amz-matdesc' => '{}' - ) - actual_content == expected_content - } - )) - .to have_been_made.once + # first request stores the encryption materials in the + # instruction file + expect( + a_request( + :put, + 'https://bucket.s3.us-west-1.amazonaws.com/key.instruction' + ).with( + body: Json.dump( + 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOIt'\ + 'bXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', + 'x-amz-iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', + 'x-amz-matdesc' => '{}' + ) + ) + ).to have_been_made.once # second request stores teh encrypted object - expect(a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') - .with( - body: lambda { |body| - actual_content = extract_chunked_content(body) - actual_content == encrypted_body - }, - headers: { 'Content-Length' => '58', 'X-Amz-Meta-X-Amz-Unencrypted-Content-Length' => '6' } - )) - .to have_been_made.once + expect( + a_request( + :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' + ).with( + body: ->(b) { b == encrypted_body }, + headers: { + 'Content-Length' => '58', + 'X-Amz-Meta-X-Amz-Unencrypted-Content-Length' => '6' + } + ) + ).to have_been_made.once end it 'accpets a custom instruction file suffix' do @@ -237,14 +227,9 @@ def extract_chunked_content(chunked_body) stub_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') expect_any_instance_of(EncryptHandler).to receive(:warn) client.put_object(bucket: 'bucket', key: 'key', body: 'secret', content_md5: 'MD5') - expect(a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') - .with( - body: lambda { |body| - actual_content = extract_chunked_content(body) - actual_content == encrypted_body - } - )) - .to have_been_made.once + expect( + a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key').with(body: encrypted_body) + ).to have_been_made.once end it 'supports encryption with an asymmetric key pair' do @@ -277,8 +262,8 @@ def stub_encrypted_get(matdesc = '{}') body: encrypted_body, headers: { 'X-Amz-Meta-X-Amz-Key' => 'gX+a4JQYj7FP0y5TAAvxTz4e'\ - '2l0DvOItbXByml/NPtKQcUls'\ - 'oGHoYR/T0TuYHcNj', + '2l0DvOItbXByml/NPtKQcUls'\ + 'oGHoYR/T0TuYHcNj', 'X-Amz-Meta-X-Amz-Iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', 'X-Amz-Meta-X-Amz-Matdesc' => matdesc } @@ -299,8 +284,8 @@ def stub_encrypted_get_chunked body: encrypted_body, headers: { 'X-Amz-Meta-X-Amz-Key' => 'gX+a4JQYj7FP0y5TAAvxTz4e'\ - '2l0DvOItbXByml/NPtKQcUls'\ - 'oGHoYR/T0TuYHcNj', + '2l0DvOItbXByml/NPtKQcUls'\ + 'oGHoYR/T0TuYHcNj', 'X-Amz-Meta-X-Amz-Iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', 'X-Amz-Meta-X-Amz-Matdesc' => '{}' } @@ -316,7 +301,7 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') ).to_return( body: Json.dump( 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOIt'\ - 'bXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', + 'bXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', 'x-amz-iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', 'x-amz-matdesc' => '{}' ) @@ -333,7 +318,7 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') stub_encrypted_get_chunked allow_any_instance_of(DecryptHandler) .to receive(:attach_http_event_listeners) - .and_wrap_original do |m, context| + .and_wrap_original do |m, context| m.call(context) context.http_response.on_data do |_chunk| if context.retries.zero? @@ -421,7 +406,7 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') stub_encrypted_get('MATERIALS-DESC') key_provider = double('key-provider') expect(key_provider).to receive(:key_for) - .with('MATERIALS-DESC').and_return(master_key) + .with('MATERIALS-DESC').and_return(master_key) options[:key_provider] = key_provider resp = client.get_object(bucket: 'bucket', key: 'key') expect(resp.body.read).to eq('secret') @@ -436,8 +421,8 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') expect do client.get_object(bucket: 'bucket', key: 'key') end.to raise_error( - Errors::DecryptionError, 'unable to locate encryption envelope' - ) + Errors::DecryptionError, 'unable to locate encryption envelope' + ) end it 'resets the cipher during decryption on error' do @@ -446,8 +431,8 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') http_resp = context.http_response headers = { 'X-Amz-Meta-X-Amz-Key' => 'gX+a4JQYj7FP0y5TAAvxTz4e'\ - '2l0DvOItbXByml/NPtKQcUls'\ - 'oGHoYR/T0TuYHcNj', + '2l0DvOItbXByml/NPtKQcUls'\ + 'oGHoYR/T0TuYHcNj', 'X-Amz-Meta-X-Amz-Iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', 'X-Amz-Meta-X-Amz-Matdesc' => '{}' } @@ -478,16 +463,16 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') expect do client.get_object(bucket: 'bucket', key: 'key') end.to raise_error( - Errors::DecryptionError, - 'decryption failed, possible incorrect key' - ) + Errors::DecryptionError, + 'decryption failed, possible incorrect key' + ) end it 'validates the key length' do stub_encrypted_get options[:encryption_key] = '.' * 31 msg = 'invalid key, symmetric key required to be 16, 24, or 32 '\ - 'bytes in length, saw length 31' + 'bytes in length, saw length 31' expect do client.get_object(bucket: 'bucket', key: 'key') end.to raise_error(ArgumentError, msg) @@ -554,7 +539,7 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') let(:plaintext_object_key) do "\xE4^\xE3\xE0v@\x8Aq\xAF\xE7y\x10\x18\xD4X"\ - "\xC2\xDC&\xF6\xDB\xCCM\x03\xAF3DD\xFF\xDA\x0Flj" + "\xC2\xDC&\xF6\xDB\xCCM\x03\xAF3DD\xFF\xDA\x0Flj" end let(:encrypted_object_key) { 'encrypted-object-key' } @@ -564,7 +549,7 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') before(:each) do allow_any_instance_of(OpenSSL::Cipher).to( receive(:random_iv) - .and_return(random_iv) + .and_return(random_iv) ) end @@ -581,12 +566,9 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') envelope.each do |key, value| expect(headers["x-amz-meta-#{key}"]).to eq(value) end - # TODO: fails due to encoding issues - # Our trailer implementation uses US-ASCII encoding - # but the value below is based on UTF-8 expect( Base64.encode64(resp.context.http_request.body_contents) - ).to eq("MTANCuBQI95EziIrEPvW/P5ImvINCg==\n") + ).to eq("4FAj3kTOIisQ+9b8/kia8g==\n") end it 'supports decryption via KMS w/ CBC' do @@ -629,7 +611,7 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') let(:plaintext_object_key) do "\xACb.\xEB\x16\x19(\x9AJ\xE0uCA\x034z\xF6&\x7F"\ - "\x8E\x0E\xC0\xD5\x1A\x88\xAF2\xB1\xEEg#\x15" + "\x8E\x0E\xC0\xD5\x1A\x88\xAF2\xB1\xEEg#\x15" end if OpenSSL::Cipher.ciphers.include?('aes-256-gcm') From 0663037b22ef1773196d56c343ea769f771a72ee Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Mon, 15 Dec 2025 12:52:05 -0800 Subject: [PATCH 21/30] More fixes --- .../plugins/checksum_algorithm.rb | 50 ++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 4aedff7d088..fdcfb8e399e 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -331,6 +331,9 @@ def calculate_request_checksum(context, checksum_properties) if (algorithm_header = checksum_properties[:request_algorithm_header]) headers[algorithm_header] = checksum_properties[:algorithm] end + + return apply_request_checksum(context, headers, checksum_properties) if defined?(JRUBY_VERSION) + case checksum_properties[:in] when 'header' apply_request_checksum(context, headers, checksum_properties) @@ -388,7 +391,6 @@ def apply_request_trailer_checksum(context, headers, checksum_properties) end headers['X-Amz-Decoded-Content-Length'] = context.http_request.body.size - context.http_request.body = AwsChunkedTrailerDigestIO.new( io: context.http_request.body, @@ -471,7 +473,8 @@ def initialize(options = {}) @chunk_size = Thread.current[:net_http_override_body_stream_chunk] || MIN_CHUNK_SIZE @overhead_bytes = calculate_overhead(@chunk_size) @base_chunk_size = @chunk_size - @overhead_bytes - @current_chunk = ''.b + @buffer = +'' + @current_chunk = +'' @eof = false end @@ -489,19 +492,41 @@ def size def rewind @io.rewind - @current_chunk.clear + @buffer = +'' + @current_chunk = +'' @eof = false end - def read(_length = nil, buf = nil) - return if @eof + def read(length = nil, buf = nil) + return @io.read unless length + + return if @eof && @buffer.empty? && @current_chunk.empty? buf&.clear - output_buffer = buf || ''.b - fill_chunk if @current_chunk.empty? && !@eof + output_buffer = buf || +'' + + while output_buffer.bytesize < length # fill until output buffer is ready + unless @buffer.empty? + take = [length - output_buffer.bytesize, @buffer.bytesize].min + slice_data = @buffer.slice!(0, take) + output_buffer << slice_data + next + end + + unless @current_chunk.empty? + take = [length - output_buffer.bytesize, @current_chunk.bytesize].min + slice_data = @current_chunk.slice!(0, take) + output_buffer << slice_data + next + end + + if !@eof + fill_chunk + else + break + end + end - output_buffer << @current_chunk - @current_chunk.clear output_buffer end @@ -513,13 +538,12 @@ def calculate_overhead(chunk_size) def fill_chunk chunk = @io.read(@base_chunk_size) - if chunk - chunk.force_encoding('ASCII-8BIT') + if chunk && !chunk.empty? @digest.update(chunk) - @current_chunk << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n".b + @current_chunk << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n" else trailer_str = { @location_name => @digest.base64digest }.map { |k, v| "#{k}:#{v}" }.join("\r\n") - @current_chunk << "0\r\n#{trailer_str}\r\n\r\n".b + @current_chunk << "0\r\n#{trailer_str}\r\n\r\n" @eof = true end end From 98f450cfb0fea0a6894638a93a88211074ba3e0d Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Mon, 15 Dec 2025 13:00:03 -0800 Subject: [PATCH 22/30] Other updates --- gems/aws-sdk-s3/CHANGELOG.md | 4 ++-- gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb | 1 + gems/aws-sdk-s3/spec/transfer_manager_spec.rb | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index b7910b4b7ec..7a63a097f82 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,9 +1,9 @@ Unreleased Changes ------------------ -* Feature - Added `:http_chunk_size` parameter to `TransferManager#upload_file` to control the buffer size when streaming request bodies over HTTP. Larger chunk sizes may improve network throughput at the cost of higher memory usage. +* Feature - Added `:http_chunk_size` parameter to `TransferManager#upload_file` to control the buffer size when streaming request bodies over HTTP. Larger chunk sizes may improve network throughput at the cost of higher memory usage (Ruby MRI only). -* Feature - Improved memory efficiency when calculating request checksums for large file uploads. +* Feature - Improved memory efficiency when calculating request checksums for large file uploads (Ruby MRI only). 1.206.0 (2025-12-02) ------------------ diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb index e0f945c1dc4..f670185cc02 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb @@ -208,6 +208,7 @@ def download_file(destination, bucket:, key:, **options) # @option option [Integer] :http_chunk_size (16384) Size in bytes for each chunk when streaming request bodies # over HTTP. Controls the buffer size used when sending data to S3. Larger values may improve throughput by # reducing the number of network writes, but use more memory. Custom values must be at least 16KB. + # Only Ruby MRI is supported. # # @option options [Proc] :progress_callback (nil) # A Proc that will be called when each chunk of the upload is sent. diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb index 4be82621188..e85c07cd169 100644 --- a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -97,7 +97,7 @@ module S3 subject.upload_file(large_file, bucket: 'bucket', key: 'key', multipart_threshold: 200 * one_mb_size) end - context ':http_check_size' do + context ':http_check_size', skip: defined?(JRUBY_VERSION) do let(:test_file) do Tempfile.new('test_upload_file').tap do |f| f.write('x' * 65_536) @@ -106,7 +106,6 @@ module S3 end def start_mirror_server(chunk_size) - server = TCPServer.new('localhost', 0) port = server.addr[1] chunks = [] From 599be962f63c192b05c39a1c92c5775eeac8c7db Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Mon, 15 Dec 2025 13:29:20 -0800 Subject: [PATCH 23/30] Update specs --- .../spec/aws/plugins/checksum_algorithm_spec.rb | 7 +++++-- gems/aws-sdk-s3/spec/client_spec.rb | 7 ++++++- gems/aws-sdk-s3/spec/encryption/client_spec.rb | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index 8e8f17b419e..991b3eef466 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -259,8 +259,11 @@ module Plugins client.stub_responses(:http_checksum_streaming_operation, lambda do |context| headers = context.http_request.headers - expect(headers['x-amz-content-sha256']) - .to eq('STREAMING-UNSIGNED-PAYLOAD-TRAILER') + unless defined?(JRUBY_VERSION) + expect(headers['x-amz-content-sha256']).to eq('STREAMING-UNSIGNED-PAYLOAD-TRAILER') + end + + test_case['expectHeaders'].each do |key, value| expect(headers[key]).to eq(value) end diff --git a/gems/aws-sdk-s3/spec/client_spec.rb b/gems/aws-sdk-s3/spec/client_spec.rb index b903ad2d5f4..7ca10759590 100644 --- a/gems/aws-sdk-s3/spec/client_spec.rb +++ b/gems/aws-sdk-s3/spec/client_spec.rb @@ -167,7 +167,12 @@ module S3 tmpfile.unlink s3 = Client.new(stub_responses: true) resp = s3.put_object(bucket: 'bucket', key: 'key', body: tmpfile) - expect(resp.context.http_request.body.instance_variable_get(:@io).read).to eq(data) + + if defined?(JRUBY_VERSION) + expect(resp.context.http_request.body_contents).to eq(data) + else + expect(resp.context.http_request.body.instance_variable_get(:@io).read).to eq(data) + end end end diff --git a/gems/aws-sdk-s3/spec/encryption/client_spec.rb b/gems/aws-sdk-s3/spec/encryption/client_spec.rb index 57f9c17c397..a05d53aab44 100644 --- a/gems/aws-sdk-s3/spec/encryption/client_spec.rb +++ b/gems/aws-sdk-s3/spec/encryption/client_spec.rb @@ -142,7 +142,13 @@ module Encryption a_request( :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' ).with( - body: ->(b) { b == encrypted_body }, + body: lambda { |b| + if defined?(JRUBY_VERSION) + encrypted_body + else + b == encrypted_body + end + }, headers: { 'Content-Length' => '58', # key is encrypted here with the master encryption key, @@ -194,11 +200,18 @@ module Encryption ).to have_been_made.once # second request stores teh encrypted object + expect( a_request( :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' ).with( - body: ->(b) { b == encrypted_body }, + body: lambda { |b| + if defined?(JRUBY_VERSION) + encrypted_body + else + b == encrypted_body + end + }, headers: { 'Content-Length' => '58', 'X-Amz-Meta-X-Amz-Unencrypted-Content-Length' => '6' From aacb57ab4373d404f17bbf8ab5f2fa1707afb2e9 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Mon, 15 Dec 2025 13:46:03 -0800 Subject: [PATCH 24/30] Update to fix --- .../lib/aws-sdk-core/plugins/checksum_algorithm.rb | 2 ++ gems/aws-sdk-s3/spec/encryption/client_spec.rb | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index fdcfb8e399e..abd6bfb42c6 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -332,6 +332,8 @@ def calculate_request_checksum(context, checksum_properties) headers[algorithm_header] = checksum_properties[:algorithm] end + # Trailer implementation within JRUBY environment is facing some + # network issues that will need further investigation return apply_request_checksum(context, headers, checksum_properties) if defined?(JRUBY_VERSION) case checksum_properties[:in] diff --git a/gems/aws-sdk-s3/spec/encryption/client_spec.rb b/gems/aws-sdk-s3/spec/encryption/client_spec.rb index a05d53aab44..4343f4afb8d 100644 --- a/gems/aws-sdk-s3/spec/encryption/client_spec.rb +++ b/gems/aws-sdk-s3/spec/encryption/client_spec.rb @@ -150,7 +150,7 @@ module Encryption end }, headers: { - 'Content-Length' => '58', + 'Content-Length' => defined?(JRUBY_VERSION) ? '16' : '58', # key is encrypted here with the master encryption key, # then base64 encoded 'X-Amz-Meta-X-Amz-Key' => 'gX+a4JQYj7FP0y5TAAvxTz4e'\ @@ -183,16 +183,14 @@ module Encryption options[:envelope_location] = :instruction_file client.put_object(bucket: 'bucket', key: 'key', body: 'secret') - # first request stores the encryption materials in the - # instruction file + # first request stores the encryption materials in the instruction file expect( a_request( :put, 'https://bucket.s3.us-west-1.amazonaws.com/key.instruction' ).with( body: Json.dump( - 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOIt'\ - 'bXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', + 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOItbXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', 'x-amz-iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', 'x-amz-matdesc' => '{}' ) @@ -213,7 +211,7 @@ module Encryption end }, headers: { - 'Content-Length' => '58', + 'Content-Length' => defined?(JRUBY_VERSION) ? '16' : '58', 'X-Amz-Meta-X-Amz-Unencrypted-Content-Length' => '6' } ) From c083d2759944678b74f1246cbdfa0dcfc81a94c3 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Tue, 16 Dec 2025 07:50:51 -0800 Subject: [PATCH 25/30] Update to skip unsigned body tests for JRuby --- .../spec/aws/plugins/checksum_algorithm_spec.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index 991b3eef466..1c4209a74d3 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -239,7 +239,8 @@ module Plugins end end - context 'request streaming checksum calculation' do + # JRuby live testing against service is not reliable. We plan to investigate deeper before enabling + context 'request streaming checksum calculation', skip: defined?(JRUBY_VERSION) do file = File.expand_path('checksum_streaming_request.json', __dir__) test_cases = JSON.load_file(file) @@ -259,11 +260,7 @@ module Plugins client.stub_responses(:http_checksum_streaming_operation, lambda do |context| headers = context.http_request.headers - unless defined?(JRUBY_VERSION) - expect(headers['x-amz-content-sha256']).to eq('STREAMING-UNSIGNED-PAYLOAD-TRAILER') - end - - + expect(headers['x-amz-content-sha256']).to eq('STREAMING-UNSIGNED-PAYLOAD-TRAILER') test_case['expectHeaders'].each do |key, value| expect(headers[key]).to eq(value) end From 15a4f12e5338599d8fb82dc0ba745ec2b37568b8 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Tue, 16 Dec 2025 08:14:00 -0800 Subject: [PATCH 26/30] Update flakey test --- gems/aws-sdk-s3/spec/transfer_manager_spec.rb | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb index e85c07cd169..495416f1ba8 100644 --- a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -111,27 +111,29 @@ def start_mirror_server(chunk_size) chunks = [] server_thread = Thread.new do - client = server.accept - headers = '' - while (line = client.gets) - headers += line - break if line.strip.empty? - end + Timeout.timeout(10) do + client = server.accept + headers = '' + while (line = client.gets) + headers += line + break if line.strip.empty? + end - if headers.include?('Expect: 100-continue') - client.write("HTTP/1.1 100 Continue\r\n\r\n") + if headers.include?('Expect: 100-continue') + client.write("HTTP/1.1 100 Continue\r\n\r\n") - loop do - sleep(0.001) # needs wait between reads - data = client.read_nonblock(chunk_size, exception: false) - break if data == :wait_readable || data.nil? + loop do + sleep(0.01) # needs wait between reads + data = client.read_nonblock(chunk_size, exception: false) + break if data == :wait_readable || data.nil? - chunks << data.size + chunks << data.size + end end + client.write("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + ensure + client.close end - client.write("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") - ensure - client.close end [server, server_thread, port] end From da32853901d7d636d9eb0d4126e447e843851777 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Wed, 17 Dec 2025 07:57:08 -0800 Subject: [PATCH 27/30] Bump min core --- build_tools/services.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/services.rb b/build_tools/services.rb index fcfaa613138..d95b7938193 100644 --- a/build_tools/services.rb +++ b/build_tools/services.rb @@ -9,10 +9,10 @@ class ServiceEnumerator MANIFEST_PATH = File.expand_path('../../services.json', __FILE__) # Minimum `aws-sdk-core` version for new gem builds - MINIMUM_CORE_VERSION = "3.240.0" + MINIMUM_CORE_VERSION = "3.241.0" # Minimum `aws-sdk-core` version for new S3 gem builds - MINIMUM_CORE_VERSION_S3 = "3.240.0" + MINIMUM_CORE_VERSION_S3 = "3.241.0" EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration" From ad6849ebdecf146cdf2bc4b030fe1e08f3ef832f Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Wed, 17 Dec 2025 08:08:08 -0800 Subject: [PATCH 28/30] Feedback - link open issues --- .../lib/aws-sdk-core/plugins/checksum_algorithm.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index abd6bfb42c6..8a8e967e116 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -332,8 +332,10 @@ def calculate_request_checksum(context, checksum_properties) headers[algorithm_header] = checksum_properties[:algorithm] end - # Trailer implementation within JRUBY environment is facing some - # network issues that will need further investigation + # Trailer implementation within Mac/JRUBY environment is facing some + # network issues that will need further investigation: + # * https://github.com/jruby/jruby-openssl/issues/271 + # * https://github.com/jruby/jruby-openssl/issues/317 return apply_request_checksum(context, headers, checksum_properties) if defined?(JRUBY_VERSION) case checksum_properties[:in] From d3c9d4e04427327984ce0103b1c4f3fff5a71233 Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Wed, 17 Dec 2025 08:45:45 -0800 Subject: [PATCH 29/30] Feedback - fix impl and associated tests --- .../plugins/checksum_algorithm.rb | 2 +- .../aws-sdk-s3/spec/encryption/client_spec.rb | 47 +++++++++++++------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 8a8e967e116..9384c415dee 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -502,7 +502,7 @@ def rewind end def read(length = nil, buf = nil) - return @io.read unless length + length ||= MIN_CHUNK_SIZE return if @eof && @buffer.empty? && @current_chunk.empty? diff --git a/gems/aws-sdk-s3/spec/encryption/client_spec.rb b/gems/aws-sdk-s3/spec/encryption/client_spec.rb index 4343f4afb8d..e3dc6c36c57 100644 --- a/gems/aws-sdk-s3/spec/encryption/client_spec.rb +++ b/gems/aws-sdk-s3/spec/encryption/client_spec.rb @@ -146,7 +146,7 @@ module Encryption if defined?(JRUBY_VERSION) encrypted_body else - b == encrypted_body + b.include?(encrypted_body) end }, headers: { @@ -182,6 +182,11 @@ module Encryption options[:envelope_location] = :instruction_file client.put_object(bucket: 'bucket', key: 'key', body: 'secret') + expected_body = Json.dump( + 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOItbXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', + 'x-amz-iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', + 'x-amz-matdesc' => '{}' + ) # first request stores the encryption materials in the instruction file expect( @@ -189,16 +194,17 @@ module Encryption :put, 'https://bucket.s3.us-west-1.amazonaws.com/key.instruction' ).with( - body: Json.dump( - 'x-amz-key' => 'gX+a4JQYj7FP0y5TAAvxTz4e2l0DvOItbXByml/NPtKQcUlsoGHoYR/T0TuYHcNj', - 'x-amz-iv' => 'TO5mQgtOzWkTfoX4RE5tsA==', - 'x-amz-matdesc' => '{}' - ) + body: lambda { |b| + if defined?(JRUBY_VERSION) + expected_body + else + b.include?(expected_body) + end + } ) ).to have_been_made.once # second request stores teh encrypted object - expect( a_request( :put, 'https://bucket.s3.us-west-1.amazonaws.com/key' @@ -207,7 +213,7 @@ module Encryption if defined?(JRUBY_VERSION) encrypted_body else - b == encrypted_body + b.include?(encrypted_body) end }, headers: { @@ -239,7 +245,16 @@ module Encryption expect_any_instance_of(EncryptHandler).to receive(:warn) client.put_object(bucket: 'bucket', key: 'key', body: 'secret', content_md5: 'MD5') expect( - a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key').with(body: encrypted_body) + a_request(:put, 'https://bucket.s3.us-west-1.amazonaws.com/key') + .with( + body: lambda { |b| + if defined?(JRUBY_VERSION) + encrypted_body + else + b.include?(encrypted_body) + end + } + ) ).to have_been_made.once end @@ -570,16 +585,18 @@ def stub_encrypted_get_with_instruction_file(sfx = '.instruction') plaintext: plaintext_object_key, ciphertext_blob: encrypted_object_key ) - resp = client.put_object( - bucket: 'aws-sdk', key: 'foo', body: 'plain-text' - ) + resp = client.put_object(bucket: 'aws-sdk', key: 'foo', body: 'plain-text') headers = resp.context.http_request.headers envelope.each do |key, value| expect(headers["x-amz-meta-#{key}"]).to eq(value) end - expect( - Base64.encode64(resp.context.http_request.body_contents) - ).to eq("4FAj3kTOIisQ+9b8/kia8g==\n") + result = + if defined?(JRUBY_VERSION) + resp.context.http_request.body_contents + else + resp.context.http_request.body.instance_variable_get('@io').read + end + expect(Base64.encode64(result)).to eq("4FAj3kTOIisQ+9b8/kia8g==\n") end it 'supports decryption via KMS w/ CBC' do From 652a6dbf56d4073cf4d337b9ffd482ad8da6448b Mon Sep 17 00:00:00 2001 From: Juli Tera Date: Wed, 17 Dec 2025 09:01:14 -0800 Subject: [PATCH 30/30] Feedback - remove setting thread local for JRUBY --- .../aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb index f670185cc02..df4c221c807 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb @@ -226,10 +226,17 @@ def download_file(destination, bucket:, key:, **options) # @see Client#upload_part def upload_file(source, bucket:, key:, **options) upload_opts = options.merge(bucket: bucket, key: key) - http_chunk_size = upload_opts.delete(:http_chunk_size) - if http_chunk_size && http_chunk_size < Aws::Plugins::ChecksumAlgorithm::MIN_CHUNK_SIZE - raise ArgumentError, ':http_chunk_size must be at least 16384 bytes (16KB)' - end + http_chunk_size = + if defined?(JRUBY_VERSION) + nil + else + chunk = upload_opts.delete(:http_chunk_size) + if chunk && chunk < Aws::Plugins::ChecksumAlgorithm::MIN_CHUNK_SIZE + raise ArgumentError, ':http_chunk_size must be at least 16384 bytes (16KB)' + end + + chunk + end executor = @executor || DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count)) uploader = FileUploader.new(