CONNECT/CONNECT-UDP Encapsulation Support

README.md

@@ -194,6 +194,9 @@ bazel-bin/nighthawk_client  [--user-defined-plugin-config <string>] ...
 |experimental_fortio_pedantic|csv
 |prometheus>] [-v <trace|debug|info|warn
 |error|critical>] [--concurrency <string>]
+[--tunnel-tls-context <string>]
+[--tunnel-uri <string>] [--tunnel-protocol
+<http1|http2|http3>]
 [--http3-protocol-options <string>] [-p
 <http1|http2|http3>] [--h2] [--timeout
 <uint32_t>] [--duration <uint32_t>]
@@ -397,6 +400,24 @@ Nighthawk process. Note that increasing this results in an effective
 load multiplier combined with the configured --rps and --connections
 values. Default: 1.
 
+--tunnel-tls-context <string>
+Upstream TlS context configuration in json. Required to encapsulate in
+HTTP3 Example (json):
+{common_tls_context:{tls_params:{cipher_suites:["-ALL:ECDHE-RSA-AES128
+-SHA"]}}}
+
+--tunnel-uri <string>
+The address of the proxy. Possible values: [http1, http2, http3]. The
+default protocol is 'http1'
+
+--tunnel-protocol <http1|http2|http3>
+The protocol for setting up tunnel encapsulation. Possible values:
+[http1, http2, http3]. The default protocol is 'http1' Combinations
+not supported currently are protocol = HTTP3 and tunnel_protocol =
+HTTP1. and protocol = HTTP3 and tunnel_protocol = HTTP3. When protocol
+is set to HTTP3 and tunneling is enabled, the CONNECT-UDP method is
+used Otherwise, the HTTP CONNECT method is used
+
 --http3-protocol-options <string>
 HTTP3 protocol options (envoy::config::core::v3::Http3ProtocolOptions)
 in json. If specified, Nighthawk uses these HTTP3 protocol options

api/client/options.proto

@@ -136,7 +136,7 @@ message Protocol {
 
 // TODO(oschaaf): Ultimately this will be a load test specification. The fact that it
 // can arrive via CLI is just a concrete detail. Change this to reflect that.
-// Next unused number is 114.
+// Next unused number is 120.
 message CommandLineOptions {
   // The target requests-per-second rate. Default: 5.
   google.protobuf.UInt32Value requests_per_second = 1
@@ -163,6 +163,25 @@ message CommandLineOptions {
     Protocol protocol = 107;
   }
 
+  // Options for routing requests via a proxy. if set, requests
+  // are encapsulated and forwarded to a terminating proxy running at
+  // tunnel_uri.
+  // When the oneof_protocol field is set to H1 or H2, an HTTP CONNECT
+  // tunnel is established with the proxy
+  // When the oneof_protocol field is set to H3, a CONNECT-UDP
+  // upgrade is used instead
+  message TunnelOptions {
+    // URI to the proxy.
+    string tunnel_uri = 1;
+    // the top level protocol.
+    Protocol tunnel_protocol = 2;
+    // TLS context for the proxy.
+    // TLS configuration is required for HTTP/3 tunnels.
+    envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext tunnel_tls_context = 3;
+  }
+
+  TunnelOptions tunnel_options = 114;
+
   // Allows user to set specific HTTP3 protocol options.
   // Only valid when protocol is set to HTTP3.
   // Is exclusive with any other command line option that would modify the
@@ -173,6 +192,8 @@ message CommandLineOptions {
   // Nighthawk leverage all vCPUs that have affinity to the Nighthawk process. Note that
   // increasing this results in an effective load multiplier combined with the configured
   // --rps and --connections values. Default: 1.
+  // When tunneling is enabled using tunnel_options, the tunnel has the same
+  // concurrency.
   google.protobuf.StringValue concurrency =
       6; // [(validate.rules).string = {pattern: "^([0-9]*|auto)$"}];
   // Verbosity of the output. Possible values: [trace, debug, info, warn,

include/nighthawk/client/options.h

@@ -49,6 +49,13 @@ class Options {
   virtual const absl::optional<envoy::config::core::v3::Http3ProtocolOptions>&
   http3ProtocolOptions() const PURE;
 
+  // HTTP CONNECT/CONNECT-UDP Tunneling related options.
+  virtual Envoy::Http::Protocol tunnelProtocol() const PURE;
+  virtual std::string tunnelUri() const PURE;
+  virtual uint32_t encapPort() const PURE;
+  virtual const absl::optional<envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext>
+  tunnelTlsContext() const PURE;
+
   virtual std::string concurrency() const PURE;
   virtual nighthawk::client::Verbosity::VerbosityOptions verbosity() const PURE;
   virtual nighthawk::client::OutputFormat::OutputFormatOptions outputFormat() const PURE;

source/client/BUILD

@@ -50,7 +50,16 @@ envoy_cc_library(
         "//include/nighthawk/client:options_lib",
         "//source/common:nighthawk_common_lib",
         "@envoy//source/common/common:statusor_lib_with_external_headers",
+        "@envoy//source/common/formatter:formatter_extension_lib",
+        "@envoy//source/extensions/filters/network/tcp_proxy:config",
+        "@envoy//source/extensions/filters/udp/udp_proxy:config",
+        "@envoy//source/extensions/filters/udp/udp_proxy:udp_proxy_filter_lib",
+        "@envoy//source/extensions/filters/udp/udp_proxy/session_filters/http_capsule:config",
         "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/udp/udp_proxy/v3:pkg_cc_proto",
     ],
 )
 
@@ -135,6 +144,7 @@ envoy_cc_library(
         "@envoy//source/common/tracing:tracer_lib_with_external_headers",
         "@envoy//source/common/upstream:cluster_manager_lib_with_external_headers",
         "@envoy//source/exe:all_extensions_lib_with_external_headers",
+        "@envoy//source/exe:main_common_lib_with_external_headers",
         "@envoy//source/exe:platform_header_lib_with_external_headers",
         "@envoy//source/exe:platform_impl_lib",
         "@envoy//source/exe:process_wide_lib_with_external_headers",

source/client/options_impl.cc

@@ -1,5 +1,9 @@
 #include "source/client/options_impl.h"
 
+#include <cerrno>
+#include <cstdint>
+#include <exception>
+
 #include "external/envoy/source/common/protobuf/message_validator_impl.h"
 #include "external/envoy/source/common/protobuf/protobuf.h"
 #include "external/envoy/source/common/protobuf/utility.h"
@@ -12,6 +16,7 @@
 #include "source/common/version_info.h"
 
 #include "absl/strings/numbers.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_split.h"
 #include "absl/types/optional.h"
 #include "fmt/ranges.h"
@@ -85,6 +90,35 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) {
       "{quic_protocol_options:{max_concurrent_streams:1}}",
       false, "", "string", cmd);
 
+  std::vector<std::string> tunnel_protocols = {"http1", "http2", "http3"};
+  TCLAP::ValuesConstraint<std::string> tunnel_protocols_allowed(tunnel_protocols);
+  TCLAP::ValueArg<std::string> tunnel_protocol(
+      "", "tunnel-protocol",
+      fmt::format(
+          "The protocol for setting up tunnel encapsulation. Possible values: [http1, http2, "
+          "http3]. The default protocol is '{}' "
+          "Combinations not supported currently are protocol = HTTP3 and tunnel_protocol = HTTP1."
+          " and protocol = HTTP3 and tunnel_protocol = HTTP3."
+          " When protocol is set to HTTP3 and tunneling is enabled, the CONNECT-UDP method is used"
+          " Otherwise, the HTTP CONNECT method is used",
+          absl::AsciiStrToLower(
+              nighthawk::client::Protocol_ProtocolOptions_Name(tunnel_protocol_))),
+      false, "", &tunnel_protocols_allowed, cmd);
+  TCLAP::ValueArg<std::string> tunnel_uri(
+      "", "tunnel-uri",
+      fmt::format(
+          "The address of the proxy. Possible values: [http1, http2, "
+          "http3]. The default protocol is '{}' ",
+          absl::AsciiStrToLower(nighthawk::client::Protocol_ProtocolOptions_Name(protocol_))),
+      false, "", "string", cmd);
+  TCLAP::ValueArg<std::string> tunnel_tls_context(
+      "", "tunnel-tls-context",
+      "Upstream TlS context configuration in json."
+      " Required to encapsulate in HTTP3"
+      " Example (json): "
+      " {common_tls_context:{tls_params:{cipher_suites:[\"-ALL:ECDHE-RSA-AES128-SHA\"]}}}",
+      false, "", "string", cmd);
+
   TCLAP::ValueArg<std::string> concurrency(
       "", "concurrency",
       fmt::format(
@@ -677,6 +711,46 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) {
     }
   }
 
+  if (tunnel_protocol.isSet()) {
+    std::string upper_cased = tunnel_protocol.getValue();
+    absl::AsciiStrToUpper(&upper_cased);
+    RELEASE_ASSERT(
+        nighthawk::client::Protocol::ProtocolOptions_Parse(upper_cased, &tunnel_protocol_),
+        "Failed to parse tunnel protocol");
+    if (!tunnel_uri.isSet()) {
+      throw MalformedArgvException("--tunnel-protocol requires --tunnel-uri");
+    }
+    tunnel_uri_ = tunnel_uri.getValue();
+    encap_port_ = Utility::GetAvailablePort(/*udp=*/protocol_ == Protocol::HTTP3, address_family_);
+
+  } else if (tunnel_uri.isSet() || tunnel_tls_context.isSet()) {
+    throw MalformedArgvException("tunnel flags require --tunnel-protocol");
+  }
+
+  if (!tunnel_tls_context.getValue().empty()) {
+    try {
+      tunnel_tls_context_.emplace(
+          envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext());
+      Envoy::MessageUtil::loadFromJson(tunnel_tls_context.getValue(), tunnel_tls_context_.value(),
+                                       Envoy::ProtobufMessage::getStrictValidationVisitor());
+    } catch (const Envoy::EnvoyException& e) {
+      throw MalformedArgvException(e.what());
+    }
+  } else if (tunnel_protocol_ == Protocol::HTTP3) {
+    throw MalformedArgvException("--tunnel-tls-context is required to use --tunnel-protocol http3");
+  }
+
+  if (tunnel_protocol.isSet()) {
+    if (tunnel_protocol_ == Protocol::HTTP3 && protocol_ == Protocol::HTTP3) {
+      throw MalformedArgvException(
+          "--protocol HTTP3 over --tunnel-protocol HTTP3 is not supported");
+    }
+    if (tunnel_protocol_ == Protocol::HTTP1 && protocol_ == Protocol::HTTP3) {
+      throw MalformedArgvException(
+          "--protocol HTTP3 over --tunnel-protocol HTTP1 is not supported");
+    }
+  }
+
   validate();
 }
 
@@ -690,6 +764,16 @@ Envoy::Http::Protocol OptionsImpl::protocol() const {
   }
 }
 
+Envoy::Http::Protocol OptionsImpl::tunnelProtocol() const {
+  if (tunnel_protocol_ == Protocol::HTTP2) {
+    return Envoy::Http::Protocol::Http2;
+  } else if (tunnel_protocol_ == Protocol::HTTP3) {
+    return Envoy::Http::Protocol::Http3;
+  } else {
+    return Envoy::Http::Protocol::Http11;
+  }
+}
+
 void OptionsImpl::parsePredicates(const TCLAP::MultiArg<std::string>& arg,
                                   TerminationPredicateMap& predicates) {
   if (arg.isSet()) {
@@ -753,6 +837,18 @@ OptionsImpl::OptionsImpl(const nighthawk::client::CommandLineOptions& options) {
   burst_size_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, burst_size, burst_size_);
   address_family_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, address_family, address_family_);
 
+  if (options.has_tunnel_options()) {
+    tunnel_protocol_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(options.tunnel_options(), tunnel_protocol,
+                                                       tunnel_protocol_);
+    tunnel_uri_ = options.tunnel_options().tunnel_uri();
+
+    // we must find an available port for the encap listener
+    encap_port_ =
+        Utility::GetAvailablePort(/*is_udp=*/protocol_ == Protocol::HTTP3, address_family_);
+
+    tunnel_tls_context_->MergeFrom(options.tunnel_options().tunnel_tls_context());
+  }
+
   if (options.has_request_options()) {
     const auto& request_options = options.request_options();
     for (const auto& header : request_options.request_headers()) {

source/client/options_impl.h

@@ -39,6 +39,15 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable<Envoy::Logger
   absl::optional<std::string> uri() const override { return uri_; }
 
   Envoy::Http::Protocol protocol() const override;
+
+  Envoy::Http::Protocol tunnelProtocol() const override;
+  std::string tunnelUri() const override { return tunnel_uri_; }
+  uint32_t encapPort() const override { return encap_port_; }
+  virtual const absl::optional<envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext>
+  tunnelTlsContext() const override {
+    return tunnel_tls_context_;
+  }
+
   const absl::optional<envoy::config::core::v3::Http3ProtocolOptions>&
   http3ProtocolOptions() const override {
     return http3_protocol_options_;
@@ -138,6 +147,14 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable<Envoy::Logger
   absl::optional<envoy::config::core::v3::Http3ProtocolOptions> http3_protocol_options_;
 
   std::string concurrency_;
+
+  // Tunnel related options.
+  nighthawk::client::Protocol::ProtocolOptions tunnel_protocol_{nighthawk::client::Protocol::HTTP1};
+  std::string tunnel_uri_;
+  uint32_t encap_port_{0};
+  absl::optional<envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext>
+      tunnel_tls_context_;
+
   nighthawk::client::Verbosity::VerbosityOptions verbosity_{nighthawk::client::Verbosity::WARN};
   nighthawk::client::OutputFormat::OutputFormatOptions output_format_{
       nighthawk::client::OutputFormat::JSON};