fix(libstore): misc fixes for curl-based s3

1. Added S3-specific settings (region, endpoint, profile, scheme) to
   HttpBinaryCacheStoreConfig to prevent 'unknown setting' warnings

2. Fixed URL construction in makeRequest() to properly handle:
   - S3 store URLs with query parameters (like ?region=us-east-2)
   - Paths that contain their own query parameters (like nar/file.nar?hash=abc)
   - Proper placement of path before query parameters in URLs

3. Fixed parseS3Url() to allow empty keys for store-level operations
   The key is filled in by specific operations (e.g., 'nix-cache-info')

Author: lovesegfault

src/libstore/filetransfer.cc

@@ -918,8 +918,23 @@ struct curlFileTransfer : public FileTransfer
 
             if (!endpoint.empty()) {
                 // Custom endpoint (e.g., MinIO, custom S3-compatible service)
-                httpsUrl.authority = ParsedURL::Authority{.host = endpoint};
-                httpsUrl.path = "/" + bucket + "/" + key;
+                // Parse the endpoint if it's a full URL
+                if (hasPrefix(endpoint, "http://") || hasPrefix(endpoint, "https://")) {
+                    try {
+                        auto endpointUrl = parseURL(endpoint);
+                        httpsUrl.scheme = endpointUrl.scheme;
+                        httpsUrl.authority = endpointUrl.authority;
+                        httpsUrl.path = "/" + bucket + "/" + key;
+                    } catch (BadURL & e) {
+                        // If parsing fails, treat it as a hostname
+                        httpsUrl.authority = ParsedURL::Authority{.host = endpoint};
+                        httpsUrl.path = "/" + bucket + "/" + key;
+                    }
+                } else {
+                    // Endpoint is just a hostname
+                    httpsUrl.authority = ParsedURL::Authority{.host = endpoint};
+                    httpsUrl.path = "/" + bucket + "/" + key;
+                }
             } else {
                 // Standard AWS S3 endpoint
                 httpsUrl.authority = ParsedURL::Authority{.host = "s3." + region + ".amazonaws.com"};
@@ -952,8 +967,8 @@ struct curlFileTransfer : public FileTransfer
                 key = key.substr(1);
             }
 
-            if (key.empty())
-                throw nix::Error("S3 URI missing object key");
+            // Allow empty keys for store-level operations
+            // The key will be filled in by the specific operation (e.g., "nix-cache-info")
 
             return S3Url{.bucket = bucket, .key = key, .params = parsed.query};
         } catch (BadURL & e) {

src/libstore/http-binary-cache-store.cc

@@ -87,7 +87,11 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore
     void init() override
     {
         // FIXME: do this lazily?
-        if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri.to_string())) {
+        // For consistent cache key handling, use the reference without parameters
+        // This matches what's used in Store::queryPathInfo() lookups
+        auto cacheKey = config->getReference().render(/*withParams=*/false);
+
+        if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) {
             config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
             config->priority.setDefault(cacheInfo->priority);
         } else {
@@ -96,8 +100,7 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore
             } catch (UploadToHTTP &) {
                 throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string());
             }
-            diskCache->createCache(
-                config->cacheUri.to_string(), config->storeDir, config->wantMassQuery, config->priority);
+            diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority);
         }
     }
 
@@ -179,10 +182,24 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore
         if (hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")) {
             return FileTransferRequest(path);
         } else {
-            // Properly construct URL preserving query parameters
-            auto baseUrl = config->cacheUri;
-            baseUrl.path = baseUrl.path + "/" + path;
-            return FileTransferRequest(baseUrl.to_string());
+            // For S3 URLs, we need to properly construct the URL with path before query parameters
+            if (config->cacheUri.scheme == "s3") {
+                auto s3Url = config->cacheUri;
+
+                // Check if path contains its own query parameters
+                auto questionPos = path.find('?');
+                if (questionPos != std::string::npos) {
+                    // Path has query params - don't include base query params
+                    s3Url.query.clear();
+                }
+
+                // Append the path and reconstruct with query parameters in the right place
+                s3Url.path = s3Url.path + "/" + path;
+                return FileTransferRequest(s3Url.to_string());
+            } else {
+                // For non-S3 URLs, use the original simpler approach
+                return FileTransferRequest(config->cacheUri.to_string() + "/" + path);
+            }
         }
     }
 

src/libstore/include/nix/store/http-binary-cache-store.hh

@@ -14,6 +14,47 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCache
 
     ParsedURL cacheUri;
 
+#if NIX_WITH_AWS_CRT_SUPPORT
+    // S3-specific settings that are passed as query parameters
+    // These are used by the curl-based S3 implementation
+    const Setting<std::string> region{
+        this,
+        "us-east-1",
+        "region",
+        R"(
+          The region of the S3 bucket. If your bucket is not in
+          `us-east-1`, you should always explicitly specify the region
+          parameter.
+        )"};
+
+    const Setting<std::string> endpoint{
+        this,
+        "",
+        "endpoint",
+        R"(
+          The S3 endpoint to use. By default, Nix uses the standard
+          AWS S3 endpoint.
+        )"};
+
+    const Setting<std::string> profile{
+        this,
+        "",
+        "profile",
+        R"(
+          The name of the AWS configuration profile to use. By default
+          Nix uses the `default` profile.
+        )"};
+
+    const Setting<std::string> scheme{
+        this,
+        "",
+        "scheme",
+        R"(
+          The scheme to use for S3 requests (http or https). By default,
+          https is used.
+        )"};
+#endif
+
     static const std::string name()
     {
         return "HTTP Binary Cache Store";