diff --git a/src/native/crt.c b/src/native/crt.c index abecf61bf..ba1ce0246 100644 --- a/src/native/crt.c +++ b/src/native/crt.c @@ -438,6 +438,10 @@ jobject aws_jni_direct_byte_buffer_from_raw_ptr(JNIEnv *env, const void *dst, si if (jByteBuf) { aws_jni_byte_buffer_set_limit(env, jByteBuf, (jint)capacity); aws_jni_byte_buffer_set_position(env, jByteBuf, 0); + } else { + if (aws_jni_check_and_clear_exception(env)) { + (void)aws_raise_error(AWS_ERROR_HTTP_CALLBACK_FAILURE); + } } return jByteBuf; diff --git a/src/native/http_request_utils.c b/src/native/http_request_utils.c index e3274a22b..a2782031b 100644 --- a/src/native/http_request_utils.c +++ b/src/native/http_request_utils.c @@ -94,8 +94,17 @@ static int s_aws_input_stream_read(struct aws_input_stream *stream, struct aws_b } size_t out_remaining = dest->capacity - dest->len; + size_t chunk_size = 1024; + /* Newer updates allow part sizes up to 5GB. Since number of bytes required for a 5GB part is + greater than INT_MAX, it would cause a bug where the java does not allocate memory and return a null buffer + since Java natively does not support direct allocation of buffers of capacity > Integer.MAX_VALUE. + Since C handles recursively calling for more data, we read up to chunk size or out_remaining (whichever is lower) + and return the C. */ + if (out_remaining <= chunk_size) { + chunk_size = out_remaining; + } - jobject direct_buffer = aws_jni_direct_byte_buffer_from_raw_ptr(env, dest->buffer + dest->len, out_remaining); + jobject direct_buffer = aws_jni_direct_byte_buffer_from_raw_ptr(env, dest->buffer + dest->len, chunk_size); impl->body_done = (*env)->CallBooleanMethod( env, impl->http_request_body_stream, http_request_body_stream_properties.send_outgoing_body, direct_buffer); diff --git a/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java b/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java index c63c29985..e48c3643c 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java @@ -84,13 +84,13 @@ private S3Client createS3Client(S3ClientOptions options, int numThreads, int cpu private S3Client createS3Client(S3ClientOptions options, EventLoopGroup elg) { try (HostResolver hostResolver = new HostResolver(elg); - ClientBootstrap clientBootstrap = new ClientBootstrap(elg, hostResolver);) { + ClientBootstrap clientBootstrap = new ClientBootstrap(elg, hostResolver);) { Assert.assertNotNull(clientBootstrap); try (DefaultChainCredentialsProvider credentialsProvider = new DefaultChainCredentialsProvider.DefaultChainCredentialsProviderBuilder() .withClientBootstrap(clientBootstrap).build(); - AwsSigningConfig signingConfig = AwsSigningConfig.getDefaultS3SigningConfig(REGION, - credentialsProvider);) { + AwsSigningConfig signingConfig = AwsSigningConfig.getDefaultS3SigningConfig(REGION, + credentialsProvider);) { Assert.assertNotNull(credentialsProvider); options.withClientBootstrap(clientBootstrap).withSigningConfig(signingConfig); return new S3Client(options); @@ -134,7 +134,7 @@ public void testS3ClientCreateDestroyWithTLS() { skipIfNetworkUnavailable(); try (TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient(); - TlsContext tlsContext = new TlsContext(tlsContextOptions);) { + TlsContext tlsContext = new TlsContext(tlsContextOptions);) { S3ClientOptions clientOptions = new S3ClientOptions() .withRegion(REGION) .withTlsContext(tlsContext); @@ -149,10 +149,10 @@ public void testS3ClientCreateDestroyWithCredentialsProvider() { skipIfNetworkUnavailable(); try (EventLoopGroup elg = new EventLoopGroup(0, 1); - HostResolver hostResolver = new HostResolver(elg); - ClientBootstrap clientBootstrap = new ClientBootstrap(elg, hostResolver); - DefaultChainCredentialsProvider credentialsProvider = new DefaultChainCredentialsProvider.DefaultChainCredentialsProviderBuilder() - .withClientBootstrap(clientBootstrap).build();) { + HostResolver hostResolver = new HostResolver(elg); + ClientBootstrap clientBootstrap = new ClientBootstrap(elg, hostResolver); + DefaultChainCredentialsProvider credentialsProvider = new DefaultChainCredentialsProvider.DefaultChainCredentialsProviderBuilder() + .withClientBootstrap(clientBootstrap).build();) { S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION) .withClientBootstrap(clientBootstrap).withCredentialsProvider(credentialsProvider); try (S3Client client = new S3Client(clientOptions)) { @@ -166,8 +166,8 @@ public void testS3ClientCreateDestroyWithoutSigningConfig() throws Exception { skipIfAndroid(); skipIfNetworkUnavailable(); try (EventLoopGroup elg = new EventLoopGroup(0, 1); - HostResolver hostResolver = new HostResolver(elg); - ClientBootstrap clientBootstrap = new ClientBootstrap(elg, hostResolver);) { + HostResolver hostResolver = new HostResolver(elg); + ClientBootstrap clientBootstrap = new ClientBootstrap(elg, hostResolver);) { S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION) .withClientBootstrap(clientBootstrap); try (S3Client client = new S3Client(clientOptions)) { @@ -265,9 +265,9 @@ public void testS3ClientCreateDestroyHttpProxyOptions() { skipIfAndroid(); skipIfNetworkUnavailable(); try (EventLoopGroup elg = new EventLoopGroup(0, 1); - EventLoopGroup retry_elg = new EventLoopGroup(0, 1); - TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient(); - TlsContext tlsContext = new TlsContext(tlsContextOptions);) { + EventLoopGroup retry_elg = new EventLoopGroup(0, 1); + TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient(); + TlsContext tlsContext = new TlsContext(tlsContextOptions);) { HttpProxyOptions proxyOptions = new HttpProxyOptions(); proxyOptions.setHost("localhost"); proxyOptions.setConnectionType(HttpProxyOptions.HttpProxyConnectionType.Tunneling); @@ -292,10 +292,10 @@ public void testS3ClientCreateDestroyHttpProxyEnvironmentVariableSetting() { skipIfAndroid(); skipIfNetworkUnavailable(); try (EventLoopGroup elg = new EventLoopGroup(0, 1); - EventLoopGroup retry_elg = new EventLoopGroup(0, 1); - TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient(); - TlsContext tlsContext = new TlsContext(tlsContextOptions); - TlsConnectionOptions tlsConnectionOptions = new TlsConnectionOptions(tlsContext);) { + EventLoopGroup retry_elg = new EventLoopGroup(0, 1); + TlsContextOptions tlsContextOptions = TlsContextOptions.createDefaultClient(); + TlsContext tlsContext = new TlsContext(tlsContextOptions); + TlsConnectionOptions tlsConnectionOptions = new TlsConnectionOptions(tlsContext);) { HttpProxyEnvironmentVariableSetting environmentVariableSetting = new HttpProxyEnvironmentVariableSetting(); environmentVariableSetting.setConnectionType(HttpProxyOptions.HttpProxyConnectionType.Tunneling); environmentVariableSetting.setEnvironmentVariableType( @@ -337,7 +337,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -374,7 +374,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -421,7 +421,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -474,7 +474,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", "/key_does_not_exist", headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -509,7 +509,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -548,7 +548,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -593,7 +593,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT + ":443") }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT + ":443")}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -659,7 +659,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -682,7 +682,7 @@ public void onFinished(S3FinishedResponseContext context) { } Duration timeSinceSomethingHappened = Duration.between(lastTimeSomethingHappened, currentTime); - Assert.assertTrue(accumulated_data_size <= max_data_allowed); + Assert.assertTrue(accumulated_data_size <= max_data_allowed); // If it seems like data has stopped flowing, then we know a stall happened due // to backpressure. @@ -701,36 +701,36 @@ public void onFinished(S3FinishedResponseContext context) { // Assert that download stalled due to backpressure at some point Assert.assertTrue( - String.format("Download never stalled. Stall count: %d, File size: %d bytes, " + - "Total data received: %d bytes, Total window increments: %d bytes", - stallCount, fileSize, accumulated_data_size, accumulated_window_increments), - stallCount > 0); + String.format("Download never stalled. Stall count: %d, File size: %d bytes, " + + "Total data received: %d bytes, Total window increments: %d bytes", + stallCount, fileSize, accumulated_data_size, accumulated_window_increments), + stallCount > 0); Integer result = onFinishedFuture.get(); Assert.assertEquals( - String.format("S3 request failed. Expected error code 0 but got: %s. " + - "Download stats - Total bytes: %d, Stall count: %d", - result, accumulated_data_size, stallCount), - Integer.valueOf(0), result); + String.format("S3 request failed. Expected error code 0 but got: %s. " + + "Download stats - Total bytes: %d, Stall count: %d", + result, accumulated_data_size, stallCount), + Integer.valueOf(0), result); } } catch (InterruptedException | ExecutionException ex) { String detailedError = String.format( - "testS3GetWithBackpressure failed with exception: %s\n" + - "Exception type: %s\n" + - "Cause: %s\n" + - "Download progress: %d bytes received\n" + - "Window increments: %d bytes\n" + - "Stall count: %d\n" + - "Initial window: %d bytes\n" + - "File size: %d bytes", - ex.getMessage() != null ? ex.getMessage() : "(null message)", - ex.getClass().getName(), - ex.getCause() != null ? ex.getCause().toString() : "(no cause)", - accumulated_data_size, - accumulated_window_increments, - stallCount, - initialReadWindowSize, - fileSize); + "testS3GetWithBackpressure failed with exception: %s\n" + + "Exception type: %s\n" + + "Cause: %s\n" + + "Download progress: %d bytes received\n" + + "Window increments: %d bytes\n" + + "Stall count: %d\n" + + "Initial window: %d bytes\n" + + "File size: %d bytes", + ex.getMessage() != null ? ex.getMessage() : "(null message)", + ex.getClass().getName(), + ex.getCause() != null ? ex.getCause().toString() : "(no cause)", + accumulated_data_size, + accumulated_window_increments, + stallCount, + initialReadWindowSize, + fileSize); Log.log(Log.LogLevel.Error, Log.LogSubject.JavaCrtS3, detailedError); Assert.fail(detailedError); @@ -771,7 +771,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -799,9 +799,9 @@ public void testS3OverrideRequestCredentials() { StaticCredentialsProvider.StaticCredentialsProviderBuilder builder = new StaticCredentialsProvider.StaticCredentialsProviderBuilder() .withAccessKeyId(madeUpCredentials).withSecretAccessKey(madeUpCredentials); try (S3Client client = createS3Client(clientOptions); - CredentialsProvider emptyCredentialsProvider = builder.build(); - AwsSigningConfig signingConfig = AwsSigningConfig.getDefaultS3SigningConfig(REGION, - emptyCredentialsProvider);) { + CredentialsProvider emptyCredentialsProvider = builder.build(); + AwsSigningConfig signingConfig = AwsSigningConfig.getDefaultS3SigningConfig(REGION, + emptyCredentialsProvider);) { CompletableFuture onFinishedFuture = new CompletableFuture<>(); S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() { @Override @@ -816,7 +816,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() .withMetaRequestType(MetaRequestType.GET_OBJECT).withHttpRequest(httpRequest) @@ -846,13 +846,13 @@ public void testS3GetWithSignConfigShouldSignHeader() throws Exception { S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION); Predicate shouldSignHeader = name -> !name.equalsIgnoreCase("DoNotSignThis"); try (S3Client client = createS3Client(clientOptions); - EventLoopGroup elg = new EventLoopGroup(0, 1); - HostResolver hostResolver = new HostResolver(elg); - ClientBootstrap clientBootstrap = new ClientBootstrap(elg, hostResolver); - DefaultChainCredentialsProvider credentialsProvider = new DefaultChainCredentialsProvider.DefaultChainCredentialsProviderBuilder() - .withClientBootstrap(clientBootstrap).build(); - AwsSigningConfig signingConfig = AwsSigningConfig.getDefaultS3SigningConfig(REGION, - credentialsProvider);) { + EventLoopGroup elg = new EventLoopGroup(0, 1); + HostResolver hostResolver = new HostResolver(elg); + ClientBootstrap clientBootstrap = new ClientBootstrap(elg, hostResolver); + DefaultChainCredentialsProvider credentialsProvider = new DefaultChainCredentialsProvider.DefaultChainCredentialsProviderBuilder() + .withClientBootstrap(clientBootstrap).build(); + AwsSigningConfig signingConfig = AwsSigningConfig.getDefaultS3SigningConfig(REGION, + credentialsProvider);) { CompletableFuture onFinishedFuture = new CompletableFuture<>(); S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() { @Override @@ -867,7 +867,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); signingConfig.setShouldSignHeader(shouldSignHeader); @@ -906,7 +906,7 @@ private String uploadObjectPathInit(String objectPath) { } private void testS3PutHelper(boolean useFile, boolean unknownContentLength, String objectPath, boolean s3express, - int contentLength, boolean contentMD5) throws IOException { + int contentLength, boolean contentMD5) throws IOException { /* * Give a default file options, which should have no affect on non-file path * tests and also works for every file-based tests. @@ -1094,7 +1094,7 @@ public void testS3PutNonexistentFilePath() throws IOException { } private S3MetaRequestResponseHandler createTestPutPauseResumeHandler(CompletableFuture onFinishedFuture, - CompletableFuture onProgressFuture) { + CompletableFuture onProgressFuture) { return new S3MetaRequestResponseHandler() { @Override public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) { @@ -1154,8 +1154,8 @@ public long getLength() { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT), - new HttpHeader("Content-Length", Integer.valueOf(payload.capacity()).toString()), }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT), + new HttpHeader("Content-Length", Integer.valueOf(payload.capacity()).toString()),}; HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_object_test_128MB"), headers, payloadStream); @@ -1198,8 +1198,8 @@ public long getLength() { } }; - HttpHeader[] headersResume = { new HttpHeader("Host", ENDPOINT), - new HttpHeader("Content-Length", Integer.valueOf(payloadResume.capacity()).toString()), }; + HttpHeader[] headersResume = {new HttpHeader("Host", ENDPOINT), + new HttpHeader("Content-Length", Integer.valueOf(payloadResume.capacity()).toString()),}; HttpRequest httpRequestResume = new HttpRequest("PUT", uploadObjectPathInit("/put_object_test_128MB"), headersResume, payloadStreamResume); @@ -1230,7 +1230,7 @@ public long getLength() { } private void testS3RoundTripWithChecksumHelper(ChecksumAlgorithm algo, ChecksumLocation location, boolean MPU, - boolean provide_full_object_checksum) throws IOException { + boolean provide_full_object_checksum) throws IOException { S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION); if (MPU) { @@ -1325,7 +1325,7 @@ public long getLength() { // Get request! - HttpHeader[] getHeaders = { new HttpHeader("Host", ENDPOINT), }; + HttpHeader[] getHeaders = {new HttpHeader("Host", ENDPOINT),}; HttpRequest httpGetRequest = new HttpRequest("GET", objectPath, getHeaders, null); @@ -1487,7 +1487,7 @@ public void onFinished(S3FinishedResponseContext context) { } }; - HttpHeader[] headers = { new HttpHeader("Host", S3EXPRESS_ENDPOINT_USW2_AZ1) }; + HttpHeader[] headers = {new HttpHeader("Host", S3EXPRESS_ENDPOINT_USW2_AZ1)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_10MB_PATH, headers, null); AwsSigningConfig config = new AwsSigningConfig(); @@ -1671,8 +1671,8 @@ public void onProgress(S3MetaRequestProgress progress) { // x-amz-copy-source-header is composed of {source_bucket}/{source_key} final String copySource = COPY_SOURCE_BUCKET + "/" + COPY_SOURCE_KEY; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT), - new HttpHeader(X_AMZ_COPY_SOURCE_HEADER, copySource) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT), + new HttpHeader(X_AMZ_COPY_SOURCE_HEADER, copySource)}; HttpRequest httpRequest = new HttpRequest("PUT", "/copy_object_test_5GB.txt", headers, null); @@ -1708,7 +1708,7 @@ private static class CapturedMetrics { public long serviceCallDurationNs; public int retryCount; public String ipAddress; - + public void captureFrom(S3RequestMetrics metrics) { try { this.apiCallDurationNs = metrics.getApiCallDurationNs(); @@ -1758,7 +1758,7 @@ public void captureFrom(S3RequestMetrics metrics) { this.retryCount = metrics.getRetryCount(); this.ipAddress = metrics.getIpAddress(); } - + public void validateMetrics() { Assert.assertTrue("API Call duration should be >= 0", apiCallDurationNs >= 0); Assert.assertTrue("API call should be successful", apiCallSuccessful); @@ -1843,7 +1843,7 @@ public void onTelemetry(S3RequestMetrics metrics) { } }; - HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpHeader[] headers = {new HttpHeader("Host", ENDPOINT)}; HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() @@ -1855,7 +1855,7 @@ public void onTelemetry(S3RequestMetrics metrics) { Assert.assertEquals(Integer.valueOf(0), onFinishedFuture.get()); Assert.assertTrue("Telemetry callback should have been called at least once", telemetryCallbackCount.get() > 0); - + // Validate captured metrics on main thread capturedMetrics.validateMetrics(); } @@ -1864,6 +1864,92 @@ public void onTelemetry(S3RequestMetrics metrics) { } } + @Test + public void testS3PutLargePartViaStream() { + skipIfAndroid(); + skipIfNetworkUnavailable(); + Assume.assumeTrue(hasAwsCredentials()); + + long contentLength = 3L * 1024; + + S3ClientOptions clientOptions = new S3ClientOptions() + .withRegion(REGION) + .withPartSize(contentLength); + + try (S3Client client = createS3Client(clientOptions)) { + CompletableFuture onFinishedFuture = new CompletableFuture<>(); + + S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() { + @Override + public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) { + return 0; + } + + @Override + public void onFinished(S3FinishedResponseContext context) { + if (context.getErrorCode() != 0) { + onFinishedFuture.completeExceptionally(makeExceptionFromFinishedResponseContext(context)); + return; + } + onFinishedFuture.complete(Integer.valueOf(context.getErrorCode())); + } + }; + + HttpHeader[] headers = { + new HttpHeader("Host", ENDPOINT), + new HttpHeader("Content-Length", String.valueOf(contentLength)) + }; + + String path = uploadObjectPathInit("/put_object_test_3GB_stream.dat"); + String encodedPath = Uri.encodeUriPath(path); + + // Create a stream that generates 3GB of data + HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() { + private long bytesWritten = 0; + private final byte[] chunk = new byte[8 * 1024 * 1024]; // 8MB chunks + + @Override + public boolean sendRequestBody(ByteBuffer outBuffer) { + while (outBuffer.hasRemaining() && bytesWritten < contentLength) { + int toWrite = (int)Math.min(chunk.length, + Math.min(outBuffer.remaining(), contentLength - bytesWritten)); + outBuffer.put(chunk, 0, toWrite); + bytesWritten += toWrite; + } + return bytesWritten >= contentLength; + } + + @Override + public boolean resetPosition() { + bytesWritten = 0; + return true; + } + + @Override + public long getLength() { + return contentLength; + } + }; + + HttpRequest httpRequest = new HttpRequest("PUT", encodedPath, headers, payloadStream); + + S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() + .withMetaRequestType(MetaRequestType.PUT_OBJECT) + .withHttpRequest(httpRequest) + .withResponseHandler(responseHandler); + + try (S3MetaRequest metaRequest = client.makeMetaRequest(metaRequestOptions)) { + // Without the fix, this will throw: + // java.lang.IllegalArgumentException: JNI NewDirectByteBuffer passed capacity > Integer.MAX_VALUE + Assert.assertEquals(Integer.valueOf(0), onFinishedFuture.get()); + } + } catch (Exception ex) { + // Without fix: expect IllegalArgumentException about ByteBuffer capacity + // With fix: should succeed + Assert.fail("Upload failed: " + ex.getMessage()); + } + } + static class TransferStats { static final double GBPS = 1000 * 1000 * 1000; @@ -1995,7 +2081,7 @@ public void benchmarkS3Get() { .withThroughputTargetGbps(expectedGbps).withTlsContext(useTls ? tlsCtx : null); try (S3Client client = createS3Client(clientOptions, threadCount)) { - HttpHeader[] headers = { new HttpHeader("Host", endpoint) }; + HttpHeader[] headers = {new HttpHeader("Host", endpoint)}; HttpRequest httpRequest = new HttpRequest("GET", String.format("/%s", objectName), headers, null); List> requestFutures = new LinkedList<>(); @@ -2216,8 +2302,8 @@ public long getLength() { } }; - HttpHeader[] headers = { new HttpHeader("Host", endpoint), - new HttpHeader("Content-Length", Long.valueOf(payloadSize).toString()), }; + HttpHeader[] headers = {new HttpHeader("Host", endpoint), + new HttpHeader("Content-Length", Long.valueOf(payloadSize).toString()),}; HttpRequest httpRequest = new HttpRequest("PUT", String.format("/put_object_test_5GB_%d.txt", myIdx + 1), headers, payloadStream);