feat(libstore): curl-based s3

doc/manual/rl-next/s3-curl-implementation.md

@@ -0,0 +1,36 @@
+---
+synopsis: "Improved S3 binary cache support via HTTP"
+prs: [13752]
+issues: [13084, 12671, 11748, 12403, 5947]
+---
+
+S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native AWS
+SigV4 authentication instead of the AWS C++ SDK, providing significant
+improvements:
+
+- **Reduced memory usage**: Eliminates memory buffering issues that caused
+  segfaults with large files (>3.5GB)
+- **Fixed upload reliability**: Resolves AWS SDK chunking errors
+  (`InvalidChunkSizeError`)
+- **Resolved OpenSSL conflicts**: No more S2N engine override issues in
+  sandboxed builds
+- **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full
+  `aws-cpp-sdk`, reducing build complexity
+
+The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential
+management.
+
+All existing S3 URL formats and parameters remain supported.
+
+## Breaking changes
+
+The legacy `S3BinaryCacheStore` implementation has been removed in favor of the
+new curl-based approach, as a result multipart uploads are no longer supported. They may be reimplemented in the future.
+
+**Migration**: No action required for most users. S3 URLs continue to work
+with the same syntax. Users directly using `S3BinaryCacheStore` class
+should migrate to standard HTTP binary cache stores with S3 endpoints.
+
+**Build requirement**: S3 support now requires curl >= 7.75.0 for AWS SigV4
+authentication. Build configuration will warn if `aws-crt-cpp` is available
+but S3 support is disabled due to an insufficient curl version.

packaging/components.nix

@@ -440,7 +440,7 @@ in
 
       Example:
       ```
-      overrideScope (finalScope: prevScope: { aws-sdk-cpp = null; })
+      overrideScope (finalScope: prevScope: { aws-crt-cpp = null; })
       ```
     */
     overrideScope = f: (scope.overrideScope f).nix-everything;

packaging/dependencies.nix

@@ -16,21 +16,6 @@ in
 scope: {
   inherit stdenv;
 
-  aws-sdk-cpp =
-    (pkgs.aws-sdk-cpp.override {
-      apis = [
-        "identity-management"
-        "s3"
-        "transfer"
-      ];
-      customMemoryManagement = false;
-    }).overrideAttrs
-      {
-        # only a stripped down version is built, which takes a lot less resources
-        # to build, so we don't need a "big-parallel" machine.
-        requiredSystemFeatures = [ ];
-      };
-
   boehmgc =
     (pkgs.boehmgc.override {
       enableLargeConfig = true;

src/libstore-tests/filetransfer-s3.cc

@@ -0,0 +1,95 @@
+#include "nix/store/filetransfer.hh"
+#include "nix/store/config.hh"
+#include "nix/store/s3-binary-cache-store.hh"
+#include "nix/store/s3-url.hh"
+
+#if NIX_WITH_S3_SUPPORT
+
+#  include <gtest/gtest.h>
+#  include <gmock/gmock.h>
+
+namespace nix {
+
+class S3FileTransferTest : public ::testing::Test
+{
+protected:
+    void SetUp() override
+    {
+        // Clean environment for predictable tests
+        unsetenv("AWS_ACCESS_KEY_ID");
+        unsetenv("AWS_SECRET_ACCESS_KEY");
+        unsetenv("AWS_SESSION_TOKEN");
+        unsetenv("AWS_PROFILE");
+    }
+};
+
+/**
+ * Regression test: Verify that S3 uploads are supported (not rejected with "not supported" error)
+ * This test ensures the S3 upload functionality is properly implemented
+ */
+TEST_F(S3FileTransferTest, s3UploadsAreSupported)
+{
+    auto ft = makeFileTransfer();
+
+    // Parse S3 URL and convert to HTTPS for curl-based transfer
+    auto s3Url = ParsedS3URL::parse(parseURL("s3://test-bucket/test-file"));
+    auto httpsUrl = s3Url.toHttpsUrl();
+
+    // Create a mock upload request
+    FileTransferRequest uploadReq(httpsUrl);
+    uploadReq.data = std::string("test data");
+
+    // This should not throw "uploading to '...' is not supported"
+    // The request should be accepted (though it may fail later due to auth/network)
+    bool gotNotSupportedError = false;
+    try {
+        ft->upload(uploadReq);
+    } catch (const Error & e) {
+        std::string msg = e.what();
+        if (msg.find("is not supported") != std::string::npos) {
+            gotNotSupportedError = true;
+        }
+    } catch (...) {
+        // Other errors are expected (no credentials, network issues, etc.)
+    }
+
+    EXPECT_FALSE(gotNotSupportedError) << "S3 uploads should be supported";
+}
+
+/**
+ * Test that S3BinaryCacheStoreConfig properly stores region parameters
+ */
+TEST_F(S3FileTransferTest, regionParametersArePreserved)
+{
+    StringMap params;
+    params["region"] = "us-west-2";
+
+    S3BinaryCacheStoreConfig config("s3", "test-bucket", params);
+
+    EXPECT_EQ(config.cacheUri.query["region"], "us-west-2");
+
+    // Test with a different region
+    StringMap params2;
+    params2["region"] = "eu-central-1";
+
+    S3BinaryCacheStoreConfig config2("s3", "another-bucket", params2);
+    EXPECT_EQ(config2.cacheUri.query["region"], "eu-central-1");
+}
+
+/**
+ * Test that S3 scheme registration works correctly
+ */
+TEST_F(S3FileTransferTest, s3SchemeIsRegistered)
+{
+    // S3 should be registered in S3BinaryCacheStoreConfig
+    auto s3Schemes = S3BinaryCacheStoreConfig::uriSchemes();
+    EXPECT_TRUE(s3Schemes.count("s3") > 0) << "S3 scheme should be registered";
+
+    // S3 should NOT be in HttpBinaryCacheStoreConfig
+    auto httpSchemes = HttpBinaryCacheStoreConfig::uriSchemes();
+    EXPECT_FALSE(httpSchemes.count("s3") > 0) << "S3 should not be in HTTP config schemes";
+}
+
+} // namespace nix
+
+#endif

src/libstore-tests/meson.build

@@ -61,6 +61,7 @@ sources = files(
   'derivation.cc',
   'derived-path.cc',
   'downstream-placeholder.cc',
+  'filetransfer-s3.cc',
   'http-binary-cache-store.cc',
   'legacy-ssh-store.cc',
   'local-binary-cache-store.cc',
@@ -76,7 +77,7 @@ sources = files(
   'path.cc',
   'references.cc',
   's3-binary-cache-store.cc',
-  's3.cc',
+  's3-url.cc',
   'serve-protocol.cc',
   'ssh-store.cc',
   'store-reference.cc',

src/libstore-tests/s3-binary-cache-store.cc

@@ -1,4 +1,7 @@
 #include "nix/store/s3-binary-cache-store.hh"
+#include "nix/store/http-binary-cache-store.hh"
+#include "nix/store/filetransfer.hh"
+#include "nix/store/s3-url.hh"
 
 #if NIX_WITH_S3_SUPPORT
 
@@ -10,7 +13,91 @@ TEST(S3BinaryCacheStore, constructConfig)
 {
     S3BinaryCacheStoreConfig config{"s3", "foobar", {}};
 
-    EXPECT_EQ(config.bucketName, "foobar");
+    // The bucket name is stored as the host part of the authority in cacheUri
+    ASSERT_TRUE(config.cacheUri.authority.has_value());
+    EXPECT_EQ(config.cacheUri.authority->host, "foobar");
+    EXPECT_EQ(config.cacheUri.scheme, "s3");
+}
+
+TEST(S3BinaryCacheStore, constructConfigWithRegion)
+{
+    Store::Config::Params params{{"region", "eu-west-1"}};
+    S3BinaryCacheStoreConfig config{"s3", "my-bucket", params};
+
+    ASSERT_TRUE(config.cacheUri.authority.has_value());
+    EXPECT_EQ(config.cacheUri.authority->host, "my-bucket");
+    EXPECT_EQ(config.region.get(), "eu-west-1");
+}
+
+TEST(S3BinaryCacheStore, defaultSettings)
+{
+    S3BinaryCacheStoreConfig config{"s3", "test-bucket", {}};
+
+    // Check default values
+    EXPECT_EQ(config.region.get(), "us-east-1");
+    EXPECT_EQ(config.profile.get(), "");
+    EXPECT_EQ(config.endpoint.get(), "");
+}
+
+// =============================================================================
+// S3BinaryCacheStore Integration Tests
+// =============================================================================
+
+/**
+ * Test that S3BinaryCacheStore properly preserves S3-specific parameters
+ */
+TEST(S3BinaryCacheStore, s3StoreConfigPreservesParameters)
+{
+    StringMap params;
+    params["region"] = "eu-west-1";
+    params["endpoint"] = "custom.s3.com";
+
+    S3BinaryCacheStoreConfig config("s3", "test-bucket", params);
+
+    // The config should preserve S3-specific parameters
+    EXPECT_EQ(config.cacheUri.scheme, "s3");
+    EXPECT_EQ(config.cacheUri.authority->host, "test-bucket");
+    EXPECT_FALSE(config.cacheUri.query.empty());
+    EXPECT_EQ(config.cacheUri.query["region"], "eu-west-1");
+    EXPECT_EQ(config.cacheUri.query["endpoint"], "custom.s3.com");
+}
+
+/**
+ * Test that S3 store scheme is properly registered
+ */
+TEST(S3BinaryCacheStore, s3SchemeRegistration)
+{
+    auto schemes = S3BinaryCacheStoreConfig::uriSchemes();
+    EXPECT_TRUE(schemes.count("s3") > 0) << "S3 scheme should be supported";
+
+    // Verify HttpBinaryCacheStoreConfig doesn't directly list S3
+    auto httpSchemes = HttpBinaryCacheStoreConfig::uriSchemes();
+    EXPECT_FALSE(httpSchemes.count("s3") > 0) << "HTTP store shouldn't directly list S3 scheme";
+}
+
+// =============================================================================
+// FileTransferRequest Tests (moved from s3-url.cc)
+// =============================================================================
+
+/**
+ * Test that S3 upload requests are properly configured
+ */
+TEST(S3BinaryCacheStore, s3UploadRequestConfiguration)
+{
+    auto s3Url = ParsedS3URL{
+        .bucket = "test-bucket",
+        .key = {"test-file"},
+    };
+    auto httpsUrl = s3Url.toHttpsUrl();
+
+    FileTransferRequest uploadReq(httpsUrl);
+    uploadReq.data = std::string("test data");
+
+    // Verify upload request is properly configured
+    EXPECT_EQ(uploadReq.uri.scheme(), "https");
+    EXPECT_TRUE(uploadReq.data.has_value());
+    EXPECT_EQ(*uploadReq.data, "test data");
+    EXPECT_TRUE(uploadReq.uri.to_string().find("test-bucket") != std::string::npos);
 }
 
 } // namespace nix

src/libstore-tests/s3.cc → src/libstore-tests/s3-url.cc

@@ -1,4 +1,4 @@
-#include "nix/store/s3.hh"
+#include "nix/store/s3-url.hh"
 #include "nix/util/tests/gmock-matchers.hh"
 
 #if NIX_WITH_S3_SUPPORT