diff --git a/.bazelrc b/.bazelrc index 9213a4bfe..85873b0b4 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,7 +3,7 @@ # the Envoy repository contents via Bazel. # unique # unique build:asan --test_timeout=900 # unique -build:tsan --test_timeout=900 # unique +build:tsan --test_timeout=3600 # unique # See https://github.com/envoyproxy/nighthawk/issues/405 # unique build:macos --copt -UDEBUG # unique # unique @@ -217,7 +217,7 @@ build:tsan --copt -DEVENT__DISABLE_DEBUG_MODE # https://github.com/abseil/abseil-cpp/issues/760 # https://github.com/google/sanitizers/issues/953 build:tsan --test_env="TSAN_OPTIONS=report_atomic_races=0" -build:tsan --test_timeout=120,600,1500,4800 +#build:tsan --test_timeout=120,600,1500,4800 # unique # Base MSAN config build:msan --action_env=ENVOY_MSAN=1 diff --git a/README.md b/README.md index a7adf0e73..715b342cc 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,9 @@ bazel-bin/nighthawk_client [--user-defined-plugin-config ] ... |experimental_fortio_pedantic|csv |prometheus>] [-v ] [--concurrency ] +[--tunnel-tls-context ] +[--tunnel-uri ] [--tunnel-protocol +] [--http3-protocol-options ] [-p ] [--h2] [--timeout ] [--duration ] @@ -397,6 +400,24 @@ Nighthawk process. Note that increasing this results in an effective load multiplier combined with the configured --rps and --connections values. Default: 1. +--tunnel-tls-context +Upstream TlS context configuration in json. Required to encapsulate in +HTTP3 Example (json): +{common_tls_context:{tls_params:{cipher_suites:["-ALL:ECDHE-RSA-AES128 +-SHA"]}}} + +--tunnel-uri +The address of the proxy. Possible values: [http1, http2, http3]. The +default protocol is 'http1' + +--tunnel-protocol +The protocol for setting up tunnel encapsulation. Possible values: +[http1, http2, http3]. The default protocol is 'http1' Combinations +not supported currently are protocol = HTTP3 and tunnel_protocol = +HTTP1. and protocol = HTTP3 and tunnel_protocol = HTTP3. When protocol +is set to HTTP3 and tunneling is enabled, the CONNECT-UDP method is +used Otherwise, the HTTP CONNECT method is used + --http3-protocol-options HTTP3 protocol options (envoy::config::core::v3::Http3ProtocolOptions) in json. If specified, Nighthawk uses these HTTP3 protocol options diff --git a/api/client/options.proto b/api/client/options.proto index d77b80f97..ccfdca2a0 100644 --- a/api/client/options.proto +++ b/api/client/options.proto @@ -136,7 +136,7 @@ message Protocol { // TODO(oschaaf): Ultimately this will be a load test specification. The fact that it // can arrive via CLI is just a concrete detail. Change this to reflect that. -// Next unused number is 114. +// Next unused number is 120. message CommandLineOptions { // The target requests-per-second rate. Default: 5. google.protobuf.UInt32Value requests_per_second = 1 @@ -163,6 +163,25 @@ message CommandLineOptions { Protocol protocol = 107; } + // Options for routing requests via a proxy. if set, requests + // are encapsulated and forwarded to a terminating proxy running at + // tunnel_uri. + // When the oneof_protocol field is set to H1 or H2, an HTTP CONNECT + // tunnel is established with the proxy + // When the oneof_protocol field is set to H3, a CONNECT-UDP + // upgrade is used instead + message TunnelOptions { + // URI to the proxy. + string tunnel_uri = 1; + // the top level protocol. + Protocol tunnel_protocol = 2; + // TLS context for the proxy. + // TLS configuration is required for HTTP/3 tunnels. + envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext tunnel_tls_context = 3; + } + + TunnelOptions tunnel_options = 114; + // Allows user to set specific HTTP3 protocol options. // Only valid when protocol is set to HTTP3. // Is exclusive with any other command line option that would modify the @@ -173,6 +192,8 @@ message CommandLineOptions { // Nighthawk leverage all vCPUs that have affinity to the Nighthawk process. Note that // increasing this results in an effective load multiplier combined with the configured // --rps and --connections values. Default: 1. + // When tunneling is enabled using tunnel_options, the tunnel has the same + // concurrency. google.protobuf.StringValue concurrency = 6; // [(validate.rules).string = {pattern: "^([0-9]*|auto)$"}]; // Verbosity of the output. Possible values: [trace, debug, info, warn, diff --git a/include/nighthawk/client/options.h b/include/nighthawk/client/options.h index 7e5f52589..791bd873f 100644 --- a/include/nighthawk/client/options.h +++ b/include/nighthawk/client/options.h @@ -49,6 +49,13 @@ class Options { virtual const absl::optional& http3ProtocolOptions() const PURE; + // HTTP CONNECT/CONNECT-UDP Tunneling related options. + virtual Envoy::Http::Protocol tunnelProtocol() const PURE; + virtual std::string tunnelUri() const PURE; + virtual uint32_t encapPort() const PURE; + virtual const absl::optional + tunnelTlsContext() const PURE; + virtual std::string concurrency() const PURE; virtual nighthawk::client::Verbosity::VerbosityOptions verbosity() const PURE; virtual nighthawk::client::OutputFormat::OutputFormatOptions outputFormat() const PURE; diff --git a/source/client/BUILD b/source/client/BUILD index bfd46925a..35efabe9c 100644 --- a/source/client/BUILD +++ b/source/client/BUILD @@ -50,7 +50,16 @@ envoy_cc_library( "//include/nighthawk/client:options_lib", "//source/common:nighthawk_common_lib", "@envoy//source/common/common:statusor_lib_with_external_headers", + "@envoy//source/common/formatter:formatter_extension_lib", + "@envoy//source/extensions/filters/network/tcp_proxy:config", + "@envoy//source/extensions/filters/udp/udp_proxy:config", + "@envoy//source/extensions/filters/udp/udp_proxy:udp_proxy_filter_lib", + "@envoy//source/extensions/filters/udp/udp_proxy/session_filters/http_capsule:config", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/udp/udp_proxy/v3:pkg_cc_proto", ], ) @@ -135,6 +144,7 @@ envoy_cc_library( "@envoy//source/common/tracing:tracer_lib_with_external_headers", "@envoy//source/common/upstream:cluster_manager_lib_with_external_headers", "@envoy//source/exe:all_extensions_lib_with_external_headers", + "@envoy//source/exe:main_common_lib_with_external_headers", "@envoy//source/exe:platform_header_lib_with_external_headers", "@envoy//source/exe:platform_impl_lib", "@envoy//source/exe:process_wide_lib_with_external_headers", diff --git a/source/client/options_impl.cc b/source/client/options_impl.cc index f5de1f259..4861bcdba 100644 --- a/source/client/options_impl.cc +++ b/source/client/options_impl.cc @@ -1,5 +1,9 @@ #include "source/client/options_impl.h" +#include +#include +#include + #include "external/envoy/source/common/protobuf/message_validator_impl.h" #include "external/envoy/source/common/protobuf/protobuf.h" #include "external/envoy/source/common/protobuf/utility.h" @@ -12,6 +16,7 @@ #include "source/common/version_info.h" #include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" #include "absl/types/optional.h" #include "fmt/ranges.h" @@ -85,6 +90,35 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) { "{quic_protocol_options:{max_concurrent_streams:1}}", false, "", "string", cmd); + std::vector tunnel_protocols = {"http1", "http2", "http3"}; + TCLAP::ValuesConstraint tunnel_protocols_allowed(tunnel_protocols); + TCLAP::ValueArg tunnel_protocol( + "", "tunnel-protocol", + fmt::format( + "The protocol for setting up tunnel encapsulation. Possible values: [http1, http2, " + "http3]. The default protocol is '{}' " + "Combinations not supported currently are protocol = HTTP3 and tunnel_protocol = HTTP1." + " and protocol = HTTP3 and tunnel_protocol = HTTP3." + " When protocol is set to HTTP3 and tunneling is enabled, the CONNECT-UDP method is used" + " Otherwise, the HTTP CONNECT method is used", + absl::AsciiStrToLower( + nighthawk::client::Protocol_ProtocolOptions_Name(tunnel_protocol_))), + false, "", &tunnel_protocols_allowed, cmd); + TCLAP::ValueArg tunnel_uri( + "", "tunnel-uri", + fmt::format( + "The address of the proxy. Possible values: [http1, http2, " + "http3]. The default protocol is '{}' ", + absl::AsciiStrToLower(nighthawk::client::Protocol_ProtocolOptions_Name(protocol_))), + false, "", "string", cmd); + TCLAP::ValueArg tunnel_tls_context( + "", "tunnel-tls-context", + "Upstream TlS context configuration in json." + " Required to encapsulate in HTTP3" + " Example (json): " + " {common_tls_context:{tls_params:{cipher_suites:[\"-ALL:ECDHE-RSA-AES128-SHA\"]}}}", + false, "", "string", cmd); + TCLAP::ValueArg concurrency( "", "concurrency", fmt::format( @@ -677,6 +711,46 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv) { } } + if (tunnel_protocol.isSet()) { + std::string upper_cased = tunnel_protocol.getValue(); + absl::AsciiStrToUpper(&upper_cased); + RELEASE_ASSERT( + nighthawk::client::Protocol::ProtocolOptions_Parse(upper_cased, &tunnel_protocol_), + "Failed to parse tunnel protocol"); + if (!tunnel_uri.isSet()) { + throw MalformedArgvException("--tunnel-protocol requires --tunnel-uri"); + } + tunnel_uri_ = tunnel_uri.getValue(); + encap_port_ = Utility::GetAvailablePort(/*udp=*/protocol_ == Protocol::HTTP3, address_family_); + + } else if (tunnel_uri.isSet() || tunnel_tls_context.isSet()) { + throw MalformedArgvException("tunnel flags require --tunnel-protocol"); + } + + if (!tunnel_tls_context.getValue().empty()) { + try { + tunnel_tls_context_.emplace( + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext()); + Envoy::MessageUtil::loadFromJson(tunnel_tls_context.getValue(), tunnel_tls_context_.value(), + Envoy::ProtobufMessage::getStrictValidationVisitor()); + } catch (const Envoy::EnvoyException& e) { + throw MalformedArgvException(e.what()); + } + } else if (tunnel_protocol_ == Protocol::HTTP3) { + throw MalformedArgvException("--tunnel-tls-context is required to use --tunnel-protocol http3"); + } + + if (tunnel_protocol.isSet()) { + if (tunnel_protocol_ == Protocol::HTTP3 && protocol_ == Protocol::HTTP3) { + throw MalformedArgvException( + "--protocol HTTP3 over --tunnel-protocol HTTP3 is not supported"); + } + if (tunnel_protocol_ == Protocol::HTTP1 && protocol_ == Protocol::HTTP3) { + throw MalformedArgvException( + "--protocol HTTP3 over --tunnel-protocol HTTP1 is not supported"); + } + } + validate(); } @@ -690,6 +764,16 @@ Envoy::Http::Protocol OptionsImpl::protocol() const { } } +Envoy::Http::Protocol OptionsImpl::tunnelProtocol() const { + if (tunnel_protocol_ == Protocol::HTTP2) { + return Envoy::Http::Protocol::Http2; + } else if (tunnel_protocol_ == Protocol::HTTP3) { + return Envoy::Http::Protocol::Http3; + } else { + return Envoy::Http::Protocol::Http11; + } +} + void OptionsImpl::parsePredicates(const TCLAP::MultiArg& arg, TerminationPredicateMap& predicates) { if (arg.isSet()) { @@ -753,6 +837,18 @@ OptionsImpl::OptionsImpl(const nighthawk::client::CommandLineOptions& options) { burst_size_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, burst_size, burst_size_); address_family_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(options, address_family, address_family_); + if (options.has_tunnel_options()) { + tunnel_protocol_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(options.tunnel_options(), tunnel_protocol, + tunnel_protocol_); + tunnel_uri_ = options.tunnel_options().tunnel_uri(); + + // we must find an available port for the encap listener + encap_port_ = + Utility::GetAvailablePort(/*is_udp=*/protocol_ == Protocol::HTTP3, address_family_); + + tunnel_tls_context_->MergeFrom(options.tunnel_options().tunnel_tls_context()); + } + if (options.has_request_options()) { const auto& request_options = options.request_options(); for (const auto& header : request_options.request_headers()) { diff --git a/source/client/options_impl.h b/source/client/options_impl.h index ac0c900b0..8c07890db 100644 --- a/source/client/options_impl.h +++ b/source/client/options_impl.h @@ -39,6 +39,15 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable uri() const override { return uri_; } Envoy::Http::Protocol protocol() const override; + + Envoy::Http::Protocol tunnelProtocol() const override; + std::string tunnelUri() const override { return tunnel_uri_; } + uint32_t encapPort() const override { return encap_port_; } + virtual const absl::optional + tunnelTlsContext() const override { + return tunnel_tls_context_; + } + const absl::optional& http3ProtocolOptions() const override { return http3_protocol_options_; @@ -138,6 +147,14 @@ class OptionsImpl : public Options, public Envoy::Logger::Loggable http3_protocol_options_; std::string concurrency_; + + // Tunnel related options. + nighthawk::client::Protocol::ProtocolOptions tunnel_protocol_{nighthawk::client::Protocol::HTTP1}; + std::string tunnel_uri_; + uint32_t encap_port_{0}; + absl::optional + tunnel_tls_context_; + nighthawk::client::Verbosity::VerbosityOptions verbosity_{nighthawk::client::Verbosity::WARN}; nighthawk::client::OutputFormat::OutputFormatOptions output_format_{ nighthawk::client::OutputFormat::JSON}; diff --git a/source/client/process_bootstrap.cc b/source/client/process_bootstrap.cc index 857c3f3ba..2e68e5eb0 100644 --- a/source/client/process_bootstrap.cc +++ b/source/client/process_bootstrap.cc @@ -1,5 +1,7 @@ #include "source/client/process_bootstrap.h" +#include + #include #include @@ -8,6 +10,10 @@ #include "external/envoy/source/common/common/statusor.h" #include "external/envoy_api/envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "external/envoy_api/envoy/extensions/filters/network/tcp_proxy/v3/tcp_proxy.pb.h" +#include "external/envoy_api/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3/http_capsule.pb.h" +#include "external/envoy_api/envoy/extensions/filters/udp/udp_proxy/v3/route.pb.h" +#include "external/envoy_api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.pb.h" #include "external/envoy_api/envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.h" #include "external/envoy_api/envoy/extensions/upstreams/http/v3/http_protocol_options.pb.h" @@ -188,7 +194,7 @@ Cluster createNighthawkClusterForWorker(const Client::Options& options, absl::Status extractAndResolveUrisFromOptions(Envoy::Event::Dispatcher& dispatcher, const Client::Options& options, Envoy::Network::DnsResolver& dns_resolver, - std::vector* uris, + UriPtr* encap_uri, std::vector* uris, UriPtr* request_source_uri) { try { if (options.uri().has_value()) { @@ -205,6 +211,13 @@ absl::Status extractAndResolveUrisFromOptions(Envoy::Event::Dispatcher& dispatch uri->resolve(dispatcher, dns_resolver, Utility::translateFamilyOptionString(options.addressFamily())); } + if (!options.tunnelUri().empty()) { + *encap_uri = + std::make_unique(fmt::format("https://localhost:{}", options.encapPort())); + (*encap_uri) + ->resolve(dispatcher, dns_resolver, + Utility::translateFamilyOptionString(options.addressFamily())); + } if (options.requestSource() != "") { *request_source_uri = std::make_unique(options.requestSource()); (*request_source_uri) @@ -232,17 +245,28 @@ absl::StatusOr createBootstrapConfiguration( if (!dns_resolver.ok()) { return dns_resolver.status(); } - std::vector uris; - UriPtr request_source_uri; + // resolve targets and encapsulation + std::vector uris, encap_uris; + UriPtr request_source_uri, encap_uri; absl::Status uri_status = extractAndResolveUrisFromOptions( - dispatcher, options, *dns_resolver.value(), &uris, &request_source_uri); + dispatcher, options, *dns_resolver.value(), &encap_uri, &uris, &request_source_uri); if (!uri_status.ok()) { return uri_status; } - + if (encap_uri != nullptr) { + encap_uris.push_back(std::move(encap_uri)); + } Bootstrap bootstrap; for (int worker_number = 0; worker_number < number_of_workers; worker_number++) { - Cluster nighthawk_cluster = createNighthawkClusterForWorker(options, uris, worker_number); + bool is_tunneling = !options.tunnelUri().empty(); + // if we're tunneling, redirect traffic to the encap listener + // while maintaining the host value + if (is_tunneling && encap_uris.empty()) { + return absl::InvalidArgumentError("No encapsulation URI for tunneling"); + } + Cluster nighthawk_cluster = + is_tunneling ? createNighthawkClusterForWorker(options, encap_uris, worker_number) + : createNighthawkClusterForWorker(options, uris, worker_number); if (needTransportSocket(options, uris)) { absl::StatusOr transport_socket = createTransportSocket(options, uris); @@ -278,4 +302,165 @@ absl::StatusOr createBootstrapConfiguration( return bootstrap; } +absl::StatusOr +createEncapBootstrap(const Client::Options& options, UriImpl& tunnel_uri, + Envoy::Event::Dispatcher& dispatcher, + const Envoy::Network::DnsResolverSharedPtr& dns_resolver) { + envoy::config::bootstrap::v3::Bootstrap encap_bootstrap; + encap_bootstrap.mutable_stats_server_version_override()->set_value(1); + + // CONNECT-UDP for HTTP3. + bool is_udp = options.protocol() == Envoy::Http::Protocol::Http3; + auto tunnel_protocol = options.tunnelProtocol(); + + // Create encap bootstrap. + auto* listener = encap_bootstrap.mutable_static_resources()->add_listeners(); + listener->set_name("encap_listener"); + auto* address = listener->mutable_address(); + auto* socket_address = address->mutable_socket_address(); + + UriImpl encap_uri(fmt::format("http://localhost:{}", options.encapPort())); + encap_uri.resolve(dispatcher, *dns_resolver, + Utility::translateFamilyOptionString(options.addressFamily())); + + socket_address->set_address(encap_uri.address()->ip()->addressAsString()); + socket_address->set_protocol(is_udp ? envoy::config::core::v3::SocketAddress::UDP + : envoy::config::core::v3::SocketAddress::TCP); + socket_address->set_port_value(encap_uri.port()); + + if (is_udp) { + address->mutable_socket_address()->set_protocol(envoy::config::core::v3::SocketAddress::UDP); + auto* filter = listener->add_listener_filters(); + filter->set_name("udp_proxy"); + filter->mutable_typed_config()->set_type_url( + "type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig"); + envoy::extensions::filters::udp::udp_proxy::v3::UdpProxyConfig udp_proxy_config; + *udp_proxy_config.mutable_stat_prefix() = "udp_proxy"; + auto* action = udp_proxy_config.mutable_matcher()->mutable_on_no_match()->mutable_action(); + action->set_name("route"); + action->mutable_typed_config()->set_type_url( + "type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.Route"); + envoy::extensions::filters::udp::udp_proxy::v3::Route route_config; + route_config.set_cluster("cluster_0"); + action->mutable_typed_config()->PackFrom(route_config); + + auto* session_filter = udp_proxy_config.mutable_session_filters()->Add(); + session_filter->set_name("envoy.filters.udp.session.http_capsule"); + session_filter->mutable_typed_config()->set_type_url( + "type.googleapis.com/" + "envoy.extensions.filters.udp.udp_proxy.session.http_capsule.v3.FilterConfig"); + envoy::extensions::filters::udp::udp_proxy::session::http_capsule::v3::FilterConfig + session_filter_config; + session_filter->mutable_typed_config()->PackFrom(session_filter_config); + + auto* tunneling_config = udp_proxy_config.mutable_tunneling_config(); + *tunneling_config->mutable_proxy_host() = "%FILTER_STATE(proxy.host.key:PLAIN)%"; + *tunneling_config->mutable_target_host() = "%FILTER_STATE(target.host.key:PLAIN)%"; + tunneling_config->set_default_target_port(443); + auto* retry_options = tunneling_config->mutable_retry_options(); + retry_options->mutable_max_connect_attempts()->set_value(2); + auto* buffer_options = tunneling_config->mutable_buffer_options(); + buffer_options->mutable_max_buffered_datagrams()->set_value(1024); + buffer_options->mutable_max_buffered_bytes()->set_value(16384); + auto* headers_to_add = tunneling_config->mutable_headers_to_add()->Add(); + headers_to_add->mutable_header()->set_key("original_dst_port"); + headers_to_add->mutable_header()->set_value("%DOWNSTREAM_LOCAL_PORT%"); + + filter->mutable_typed_config()->PackFrom(udp_proxy_config); + + } else { + address->mutable_socket_address()->set_protocol(envoy::config::core::v3::SocketAddress::TCP); + auto* filter = listener->add_filter_chains()->add_filters(); + filter->set_name("envoy.filters.network.tcp_proxy"); + filter->mutable_typed_config()->set_type_url( + "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy"); + envoy::extensions::filters::network::tcp_proxy::v3::TcpProxy tcp_proxy_config; + tcp_proxy_config.set_stat_prefix("tcp_proxy"); + *tcp_proxy_config.mutable_cluster() = "cluster_0"; + auto* tunneling_config = tcp_proxy_config.mutable_tunneling_config(); + *tunneling_config->mutable_hostname() = "host.com:443"; + auto* header_to_add = tunneling_config->add_headers_to_add(); + header_to_add->mutable_header()->set_key("original_dst_port"); + header_to_add->mutable_header()->set_value("%DOWNSTREAM_LOCAL_PORT%"); + filter->mutable_typed_config()->PackFrom(tcp_proxy_config); + } + + auto* cluster = encap_bootstrap.mutable_static_resources()->add_clusters(); + cluster->set_name("cluster_0"); + cluster->mutable_connect_timeout()->set_seconds(5); + + envoy::extensions::upstreams::http::v3::HttpProtocolOptions protocol_options; + if (tunnel_protocol == Envoy::Http::Protocol::Http3) { + protocol_options.mutable_explicit_http_config()->mutable_http3_protocol_options(); + auto* transport_socket = cluster->mutable_transport_socket(); + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext upstream_tls_context = + *options.tunnelTlsContext(); + transport_socket->set_name("envoy.transport_sockets.quic"); + envoy::extensions::transport_sockets::quic::v3::QuicUpstreamTransport quic_upstream_transport; + *quic_upstream_transport.mutable_upstream_tls_context() = upstream_tls_context; + transport_socket->mutable_typed_config()->PackFrom(quic_upstream_transport); + + } else if (tunnel_protocol == Envoy::Http::Protocol::Http2) { + protocol_options.mutable_explicit_http_config()->mutable_http2_protocol_options(); + if (options.tunnelTlsContext().has_value()) { + auto* transport_socket = cluster->mutable_transport_socket(); + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext upstream_tls_context = + *options.tunnelTlsContext(); + transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); + transport_socket->set_name("envoy.transport_sockets.tls"); + } + } else { + protocol_options.mutable_explicit_http_config()->mutable_http_protocol_options(); + if (options.tunnelTlsContext().has_value()) { + auto* transport_socket = cluster->mutable_transport_socket(); + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext upstream_tls_context = + *options.tunnelTlsContext(); + transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); + transport_socket->set_name("envoy.transport_sockets.tls"); + } + } + + (*cluster->mutable_typed_extension_protocol_options()) + ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] + .PackFrom(protocol_options); + + *cluster->mutable_load_assignment()->mutable_cluster_name() = "cluster_0"; + auto* endpoint = cluster->mutable_load_assignment() + ->mutable_endpoints() + ->Add() + ->add_lb_endpoints() + ->mutable_endpoint(); + + tunnel_uri.resolve(dispatcher, *dns_resolver, + Utility::translateFamilyOptionString(options.addressFamily())); + + auto endpoint_socket = endpoint->mutable_address()->mutable_socket_address(); + endpoint_socket->set_address(tunnel_uri.address()->ip()->addressAsString()); + endpoint_socket->set_port_value(tunnel_uri.port()); + + return encap_bootstrap; +} + +absl::Status +EncapsulationSubProcessRunner::RunWithSubprocess(std::function nighthawk_fn, + std::function envoy_fn) { + + pid_t pid_ = fork(); + if (pid_ == -1) { + return absl::InternalError("fork failed"); + } + if (pid_ == 0) { + envoy_fn(*nighthawk_control_sem_); + exit(0); + } else { + // wait for envoy to start and signal nighthawk to start + sem_wait(nighthawk_control_sem_); + // start nighthawk + nighthawk_fn(); + // signal envoy to shutdown + return TerminateEncapSubProcess(); + } + return absl::OkStatus(); +} + } // namespace Nighthawk diff --git a/source/client/process_bootstrap.h b/source/client/process_bootstrap.h index f83cd03d8..8a63ccd1f 100644 --- a/source/client/process_bootstrap.h +++ b/source/client/process_bootstrap.h @@ -1,15 +1,21 @@ #pragma once +#include + +#include #include #include "nighthawk/client/options.h" #include "nighthawk/common/uri.h" +#include "external/envoy/source/common/common/posix/thread_impl.h" #include "external/envoy/source/common/common/statusor.h" #include "external/envoy/source/common/event/dispatcher_impl.h" #include "external/envoy/source/common/network/dns_resolver/dns_factory_util.h" #include "external/envoy_api/envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "source/common/uri_impl.h" + namespace Nighthawk { /** @@ -36,4 +42,114 @@ absl::StatusOr createBootstrapConfigura const envoy::config::core::v3::TypedExtensionConfig& typed_dns_resolver_config, int number_of_workers); +/** + * Creates Encapsulation envoy bootstrap configuration. + * + * This envoy receives traffic and encapsulates it HTTP + * + * @param options are the options this Nighthawk execution was triggered with. + * @param tunnel_uri URI to the terminating proxy. + * @param dispatcher is used when resolving hostnames to IP addresses in the + * bootstrap. + * @param resolver bootstrap resolver + * + * @return the created bootstrap configuration. + */ +absl::StatusOr +createEncapBootstrap(const Client::Options& options, UriImpl& tunnel_uri, + Envoy::Event::Dispatcher& dispatcher, + const Envoy::Network::DnsResolverSharedPtr& resolver); + +class EncapsulationSubProcessRunner { +public: + /** + * Forks a separate process for Envoy. Both nighthawk and envoy are required to be their own + * processes + * + * @param nighthawk_runner executes nighthawk's workers in current process + * @param encap_envoy_runner starts up Encapsulation Envoy in a child process. + * This takes a blocked semaphore which it is responsible for signalling and allowing + * nighthawk_runner to execute once envoy is ready to serve. + * Once nighthawk_runner finishes executing, encap_envoy_runner receives a SIGTERM + * + */ + EncapsulationSubProcessRunner(std::function nighthawk_runner, + std::function encap_envoy_runner) + : nighthawk_runner_(nighthawk_runner), encap_envoy_runner_(encap_envoy_runner) { + nighthawk_control_sem_ = static_cast( + mmap(NULL, sizeof(sem_t), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0)); + + // create blocked semaphore for nighthawk to wait on + int ret = sem_init(nighthawk_control_sem_, /*pshared=*/1, /*count=*/0); + if (ret != 0) { + throw NighthawkException("Could not initialise semaphore"); + } + }; + + ~EncapsulationSubProcessRunner() { + auto status = TerminateEncapSubProcess(); + if (!status.ok()) { + ENVOY_LOG_MISC(warn, status.ToString()); + } + if (pid_ == 0) { + // Have only parent process destroy semaphore + sem_destroy(nighthawk_control_sem_); + munmap(nighthawk_control_sem_, sizeof(sem_t)); + } + } + /** + * Run functions in parent and child processes. It blocks until nighthawk_runner + * returns. + * + * @return error status for processes + **/ + absl::Status Run() { return RunWithSubprocess(nighthawk_runner_, encap_envoy_runner_); } + + /** + * Sends a SIGTERM to Encap Envoy subprocess and blocks till exit + * + **/ + absl::Status TerminateEncapSubProcess() { + Envoy::Thread::LockGuard guard(terminate_mutex_); + if (pid_ == -1 || pid_ == 0) { + return absl::OkStatus(); + } + + if (kill(pid_, SIGTERM) == -1 && errno != ESRCH) { + return absl::InternalError("Failed to kill encapsulation subprocess"); + } + + int status; + waitpid(pid_, &status, 0); + pid_ = -1; + + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { + // Child process did not crash. + return absl::OkStatus(); + } + + // Child process crashed. + return absl::InternalError(absl::StrCat("Envoy crashed with code: ", status)); + } + +private: + absl::Status RunWithSubprocess(std::function nighthawk_runner, + std::function encap_envoy_runner); + + std::function nighthawk_runner_; + std::function encap_envoy_runner_; + pid_t pid_ = -1; + sem_t* nighthawk_control_sem_; + Envoy::Thread::MutexBasicLockable terminate_mutex_; +}; + +/** + * Spins function into thread + * + * @param thread_routine executes nighthawk's workers + * + * @return thread pointer + */ +Envoy::Thread::PosixThreadPtr createThread(std::function thread_routine); + } // namespace Nighthawk diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc index a73593b37..2b0e64215 100644 --- a/source/client/process_impl.cc +++ b/source/client/process_impl.cc @@ -38,6 +38,7 @@ #include "external/envoy/source/common/singleton/manager_impl.h" #include "external/envoy/source/common/stats/tag_producer_impl.h" #include "external/envoy/source/common/thread_local/thread_local_impl.h" +#include "external/envoy/source/exe/main_common.h" #include "external/envoy/source/server/null_overload_manager.h" #include "external/envoy/source/server/server.h" #include "external/envoy_api/envoy/config/core/v3/resolver.pb.h" @@ -654,6 +655,12 @@ ProcessImpl::~ProcessImpl() { } void ProcessImpl::shutdown() { + if (encap_runner_ != nullptr) { + auto status = encap_runner_->TerminateEncapSubProcess(); + if (status != absl::OkStatus()) { + ENVOY_LOG(error, status); + } + } // Before we shut down the worker threads, stop threading. tls_.shutdownGlobalThreading(); store_root_.shutdownThreading(); @@ -683,11 +690,17 @@ void ProcessImpl::shutdown() { } bool ProcessImpl::requestExecutionCancellation() { - ENVOY_LOG(debug, "Requesting workers to cancel execution"); auto guard = std::make_unique(workers_lock_); for (auto& worker : workers_) { worker->requestExecutionCancellation(); } + if (encap_runner_ != nullptr) { + auto status = encap_runner_->TerminateEncapSubProcess(); + if (status != absl::OkStatus()) { + ENVOY_LOG(error, status); + return false; + } + } cancelled_ = true; return true; } @@ -764,7 +777,10 @@ ProcessImpl::mergeWorkerStatistics(const std::vector& workers) // (We always have at least one worker, and all workers have the same number of Statistic // instances associated to them, in the same order). std::vector merged_statistics; - StatisticPtrMap w0_statistics = workers[0]->statistics(); + StatisticPtrMap w0_statistics; + if (!workers.empty()) { + w0_statistics = workers[0]->statistics(); + } for (const auto& w0_statistic : w0_statistics) { auto new_statistic = w0_statistic.second->createNewInstanceOfSameType(); new_statistic->setId(w0_statistic.first); @@ -869,116 +885,206 @@ bool ProcessImpl::runInternal(OutputCollector& collector, const UriPtr& tracing_ const Envoy::Network::DnsResolverSharedPtr& dns_resolver, const absl::optional& scheduled_start) { const Envoy::SystemTime now = time_system_.systemTime(); + std::shared_ptr encap_main_common = nullptr; + if (scheduled_start.value_or(now) < now) { ENVOY_LOG(error, "Scheduled execution date already transpired."); return false; } - { - auto guard = std::make_unique(workers_lock_); - if (cancelled_) { - return true; - } - shutdown_ = false; - - // Needs to happen as early as possible (before createWorkers()) in the instantiation to preempt - // the objects that require stats. - if (!options_.statsSinks().empty()) { - absl::StatusOr producer_or_error = - Envoy::Stats::TagProducerImpl::createTagProducer(bootstrap_.stats_config(), - envoy_options_.statsTags()); - if (!producer_or_error.ok()) { - ENVOY_LOG(error, "createTagProducer failed. Received bad status: {}", - producer_or_error.status()); - return false; - } - store_root_.setTagProducer(std::move(producer_or_error.value())); - } - absl::Status workers_status = createWorkers(number_of_workers_, scheduled_start); - if (!workers_status.ok()) { - ENVOY_LOG(error, "createWorkers failed. Received bad status: {}", workers_status.message()); - return false; - } - tls_.registerThread(*dispatcher_, true); - store_root_.initializeThreading(*dispatcher_, tls_); + Bootstrap encap_bootstrap; - absl::StatusOr loader = Envoy::Runtime::LoaderImpl::create( - *dispatcher_, tls_, {}, *local_info_, store_root_, generator_, - Envoy::ProtobufMessage::getStrictValidationVisitor(), *api_); + if (!options_.tunnelUri().empty()) { + // Spin up an envoy for tunnel encapsulation. - if (!loader.ok()) { - ENVOY_LOG(error, "create runtime loader failed. Received bad status: {}", loader.status()); - return false; - } + UriImpl tunnel_uri(options_.tunnelUri()); - runtime_loader_ = *std::move(loader); - - server_ = std::make_unique( - admin_, *api_, *dispatcher_, access_log_manager_, envoy_options_, *runtime_loader_.get(), - *singleton_manager_, tls_, *local_info_, validation_context_, grpc_context_, http_context_, - router_context_, store_root_, secret_manager_); - ssl_context_manager_ = - std::make_unique( - server_->serverFactoryContext()); - dynamic_cast(&server_->serverFactoryContext()) - ->setSslContextManager(*ssl_context_manager_); - cluster_manager_factory_ = std::make_unique( - server_->serverFactoryContext(), - [dns_resolver]() -> Envoy::Network::DnsResolverSharedPtr { return dns_resolver; }, - quic_stat_names_); - cluster_manager_factory_->setConnectionReuseStrategy( - options_.h1ConnectionReuseStrategy() == nighthawk::client::H1ConnectionReuseStrategy::LRU - ? Http1PoolImpl::ConnectionReuseStrategy::LRU - : Http1PoolImpl::ConnectionReuseStrategy::MRU); - cluster_manager_factory_->setPrefetchConnections(options_.prefetchConnections()); - if (tracing_uri != nullptr) { - setupTracingImplementation(bootstrap_, *tracing_uri); - addTracingCluster(bootstrap_, *tracing_uri); - } - ENVOY_LOG(debug, "Computed configuration: {}", absl::StrCat(bootstrap_)); - absl::StatusOr cluster_manager = - cluster_manager_factory_->clusterManagerFromProto(bootstrap_); - if (!cluster_manager.ok()) { - ENVOY_LOG(error, "clusterManagerFromProto failed. Received bad status: {}", - cluster_manager.status().message()); - return false; - } - cluster_manager_ = std::move(*cluster_manager); - dynamic_cast(&server_->serverFactoryContext()) - ->setClusterManager(*cluster_manager_); - absl::Status status = cluster_manager_->initialize(bootstrap_); - if (!status.ok()) { - ENVOY_LOG(error, "cluster_manager initialize failed. Received bad status: {}", - status.message()); + auto status_or_bootstrap = + createEncapBootstrap(options_, tunnel_uri, *dispatcher_.get(), dns_resolver); + if (!status_or_bootstrap.ok()) { + ENVOY_LOG(error, status_or_bootstrap.status().ToString()); return false; } - maybeCreateTracingDriver(bootstrap_.tracing()); - cluster_manager_->setInitializedCb( - [this]() -> void { init_manager_.initialize(init_watcher_); }); - - absl::Status initialize_status = runtime_loader_->initialize(*cluster_manager_); - if (!initialize_status.ok()) { - ENVOY_LOG(error, "runtime_loader initialize failed. Received bad status: {}", - initialize_status.message()); - return false; + encap_bootstrap = *status_or_bootstrap; + } + + std::function envoy_routine = [this, &encap_main_common, + &encap_bootstrap](sem_t& nighthawk_control_sem) { + const Envoy::OptionsImpl::HotRestartVersionCb hot_restart_version_cb = [](bool) { + return "disabled"; + }; + + std::string lower = absl::AsciiStrToLower( + nighthawk::client::Verbosity::VerbosityOptions_Name(options_.verbosity())); + + Envoy::OptionsImpl envoy_options({"encap_envoy"}, hot_restart_version_cb, + spdlog::level::from_str(lower)); + + ENVOY_LOG(info, encap_bootstrap.DebugString()); + envoy_options.setConfigProto(encap_bootstrap); + // for now, match the concurrency of nighthawk + envoy_options.setConcurrency(number_of_workers_); + + Envoy::ProdComponentFactory prod_component_factory; + auto listener_test_hooks = std::make_unique(); + + if (!options_.tunnelUri().empty()) { + // Spin up an envoy for tunnel encapsulation. + try { + encap_main_common = std::make_shared( + envoy_options, time_system_, *listener_test_hooks, prod_component_factory, + std::make_unique(), + std::make_unique(), nullptr); + + // spin up envoy thread that first manages envoy. + auto startup_envoy_thread_ptr = + encap_main_common->server()->lifecycleNotifier().registerCallback( + NighthawkLifecycleNotifierImpl::Stage::PostInit, [&nighthawk_control_sem]() { + // signal nighthawk to start. + sem_post(&nighthawk_control_sem); + }); + encap_main_common->run(); + } catch (const Envoy::EnvoyException& ex) { + std::cout << "error caught by envoy " << ex.what() << std::endl; + ENVOY_LOG(error, ex.what()); + // let nighthawk start and close envoy process + sem_post(&nighthawk_control_sem); + } + } else { + sem_post(&nighthawk_control_sem); } + }; - std::list> stats_sinks; - setupStatsSinks(bootstrap_, stats_sinks); - std::chrono::milliseconds stats_flush_interval = std::chrono::milliseconds( - Envoy::DurationUtil::durationToMilliseconds(bootstrap_.stats_flush_interval())); + bool result = true; + std::function nigthawk_fn = [this, &result, &dns_resolver, &scheduled_start, + &tracing_uri]() { + { + auto guard = std::make_unique(workers_lock_); + if (cancelled_) { + return; + } + shutdown_ = false; + + // Needs to happen as early as possible (before createWorkers()) in the instantiation to + // preempt the objects that require stats. + if (!options_.statsSinks().empty()) { + absl::StatusOr producer_or_error = + Envoy::Stats::TagProducerImpl::createTagProducer(bootstrap_.stats_config(), + envoy_options_.statsTags()); + if (!producer_or_error.ok()) { + ENVOY_LOG(error, "createTagProducer failed. Received bad status: {}", + producer_or_error.status()); + result = false; + return; + } + store_root_.setTagProducer(std::move(producer_or_error.value())); + } - if (!options_.statsSinks().empty()) { - // There should be only a single live flush worker instance at any time. - flush_worker_ = std::make_unique( - stats_flush_interval, *api_, tls_, store_root_, stats_sinks, *cluster_manager_); - flush_worker_->start(); - } + absl::Status workers_status = createWorkers(number_of_workers_, scheduled_start); + if (!workers_status.ok()) { + ENVOY_LOG(error, "createWorkers failed. Received bad status: {}", workers_status.message()); + result = false; + return; + } + tls_.registerThread(*dispatcher_, true); + store_root_.initializeThreading(*dispatcher_, tls_); + + absl::StatusOr loader = Envoy::Runtime::LoaderImpl::create( + *dispatcher_, tls_, {}, *local_info_, store_root_, generator_, + Envoy::ProtobufMessage::getStrictValidationVisitor(), *api_); + + if (!loader.ok()) { + ENVOY_LOG(error, "create runtime loader failed. Received bad status: {}", loader.status()); + result = false; + return; + } + + runtime_loader_ = *std::move(loader); + + server_ = std::make_unique( + admin_, *api_, *dispatcher_, access_log_manager_, envoy_options_, *runtime_loader_.get(), + *singleton_manager_, tls_, *local_info_, validation_context_, grpc_context_, + http_context_, router_context_, store_root_, secret_manager_); + ssl_context_manager_ = + std::make_unique( + server_->serverFactoryContext()); + dynamic_cast(&server_->serverFactoryContext()) + ->setSslContextManager(*ssl_context_manager_); + cluster_manager_factory_ = std::make_unique( + server_->serverFactoryContext(), + [dns_resolver]() -> Envoy::Network::DnsResolverSharedPtr { return dns_resolver; }, + quic_stat_names_); + cluster_manager_factory_->setConnectionReuseStrategy( + options_.h1ConnectionReuseStrategy() == nighthawk::client::H1ConnectionReuseStrategy::LRU + ? Http1PoolImpl::ConnectionReuseStrategy::LRU + : Http1PoolImpl::ConnectionReuseStrategy::MRU); + cluster_manager_factory_->setPrefetchConnections(options_.prefetchConnections()); + if (tracing_uri != nullptr) { + setupTracingImplementation(bootstrap_, *tracing_uri); + addTracingCluster(bootstrap_, *tracing_uri); + } + ENVOY_LOG(debug, "Computed configuration: {}", absl::StrCat(bootstrap_)); + absl::StatusOr cluster_manager = + cluster_manager_factory_->clusterManagerFromProto(bootstrap_); + if (!cluster_manager.ok()) { + ENVOY_LOG(error, "clusterManagerFromProto failed. Received bad status: {}", + cluster_manager.status().message()); + result = false; + return; + } + cluster_manager_ = std::move(*cluster_manager); + dynamic_cast(&server_->serverFactoryContext()) + ->setClusterManager(*cluster_manager_); + absl::Status status = cluster_manager_->initialize(bootstrap_); + if (!status.ok()) { + ENVOY_LOG(error, "cluster_manager initialize failed. Received bad status: {}", + status.message()); + result = false; + return; + } + maybeCreateTracingDriver(bootstrap_.tracing()); + cluster_manager_->setInitializedCb( + [this]() -> void { init_manager_.initialize(init_watcher_); }); + + absl::Status initialize_status = runtime_loader_->initialize(*cluster_manager_); + if (!initialize_status.ok()) { + ENVOY_LOG(error, "runtime_loader initialize failed. Received bad status: {}", + initialize_status.message()); + result = false; + return; + } + + std::list> stats_sinks; + setupStatsSinks(bootstrap_, stats_sinks); + std::chrono::milliseconds stats_flush_interval = std::chrono::milliseconds( + Envoy::DurationUtil::durationToMilliseconds(bootstrap_.stats_flush_interval())); - for (auto& w : workers_) { - w->start(); + ENVOY_LOG(error, bootstrap_.DebugString()); + + if (!options_.statsSinks().empty()) { + // There should be only a single live flush worker instance at any time. + flush_worker_ = std::make_unique( + stats_flush_interval, *api_, tls_, store_root_, stats_sinks, *cluster_manager_); + flush_worker_->start(); + } + + for (auto& w : workers_) { + w->start(); + } } + }; + encap_runner_ = std::make_shared(nigthawk_fn, envoy_routine); + auto status = encap_runner_->Run(); + + if (!result) { + return result; } + + if (!status.ok()) { + ENVOY_LOG(error, status); + return false; + } + for (auto& w : workers_) { w->waitForCompletion(); } @@ -1034,9 +1140,11 @@ bool ProcessImpl::runInternal(OutputCollector& collector, const UriPtr& tracing_ std::vector global_user_defined_outputs = compileGlobalUserDefinedPluginOutputs(user_defined_outputs_by_plugin, user_defined_output_factories_); - collector.addResult("global", mergeWorkerStatistics(workers_), counters, - total_execution_duration / workers_.size(), first_acquisition_time, - global_user_defined_outputs); + if (workers_.size() > 0) { + collector.addResult("global", mergeWorkerStatistics(workers_), counters, + total_execution_duration / workers_.size(), first_acquisition_time, + global_user_defined_outputs); + } if (counters.find("sequencer.failed_terminations") == counters.end()) { return true; } else { diff --git a/source/client/process_impl.h b/source/client/process_impl.h index a81855a37..8a7e3bbff 100644 --- a/source/client/process_impl.h +++ b/source/client/process_impl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "envoy/api/api.h" #include "envoy/network/address.h" @@ -41,6 +42,7 @@ #include "source/client/benchmark_client_impl.h" #include "source/client/factories_impl.h" #include "source/client/flush_worker_impl.h" +#include "source/client/process_bootstrap.h" namespace Nighthawk { namespace Client { @@ -224,6 +226,8 @@ class ProcessImpl : public Process, public Envoy::Logger::Loggable user_defined_output_factories_{}; + // Tunnel Encapsulation envoy runner + std::shared_ptr encap_runner_; }; } // namespace Client diff --git a/source/common/utility.cc b/source/common/utility.cc index 92f1c750e..2f560f53c 100644 --- a/source/common/utility.cc +++ b/source/common/utility.cc @@ -1,5 +1,7 @@ #include "source/common/utility.h" +#include + #include "nighthawk/common/exception.h" #include "external/envoy/source/common/http/utility.h" @@ -82,4 +84,71 @@ bool Utility::parseHostPort(const std::string& host_port, std::string* address, RE2::FullMatch(host_port, R"(([-.0-9a-zA-Z]+):(\d+))", address, port); } +// Obtains an available TCP or UDP port. Throws an exception if one cannot be +// allocated. +uint16_t +Utility::GetAvailablePort(bool udp, + nighthawk::client::AddressFamily::AddressFamilyOptions address_family) { + int family = (address_family == nighthawk::client::AddressFamily::V4) ? AF_INET : AF_INET6; + int sock = socket(family, udp ? SOCK_DGRAM : SOCK_STREAM, udp ? 0 : IPPROTO_TCP); + if (sock < 0) { + throw NighthawkException(absl::StrCat("could not create socket: ", Envoy::errorDetails(errno))); + return 0; + } + + // Reuseaddr lets us start up a server immediately after it exits + int one = 1; + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &one, sizeof(one)) < 0) { + throw NighthawkException(absl::StrCat("setsockopt: ", Envoy::errorDetails(errno))); + close(sock); + return 0; + } + union { + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + } addr; + size_t size; + if (family == AF_INET) { + size = sizeof(sockaddr_in); + memset(&addr, 0, size); + addr.sin.sin_family = AF_INET; + addr.sin.sin_addr.s_addr = INADDR_ANY; + addr.sin.sin_port = 0; + } else { + size = sizeof(sockaddr_in6); + memset(&addr, 0, size); + addr.sin6.sin6_family = AF_INET6; + addr.sin6.sin6_addr = in6addr_any; + addr.sin6.sin6_port = 0; + } + + if (bind(sock, reinterpret_cast(&addr), size) < 0) { + if (errno == EADDRINUSE) { + throw NighthawkException(absl::StrCat("Port allocated already in use")); + } else { + throw NighthawkException( + absl::StrCat("Could not bind to process: ", Envoy::errorDetails(errno))); + } + return 0; + } + + socklen_t len = size; + if (getsockname(sock, reinterpret_cast(&addr), &len) == -1) { + throw NighthawkException(absl::StrCat("Could not get sock name: ", Envoy::errorDetails(errno))); + return 0; + } + + uint16_t port = + ntohs(family == AF_INET ? reinterpret_cast(&addr)->sin_port + : reinterpret_cast(&addr)->sin6_port); + + // close the socket, freeing the port to be used later. + if (close(sock) < 0) { + throw NighthawkException(absl::StrCat("Could not close socket: ", Envoy::errorDetails(errno))); + return 0; + } + + return port; +} + } // namespace Nighthawk diff --git a/source/common/utility.h b/source/common/utility.h index 94342b8ba..e7882e53e 100644 --- a/source/common/utility.h +++ b/source/common/utility.h @@ -64,6 +64,15 @@ class Utility { * @return bool true if the input could be parsed as host:port */ static bool parseHostPort(const std::string& host_port, std::string* host, int* port); + + // Obtains an available TCP or UDP port. Throws an exception if one cannot be + // allocated. + /** + * @param udp boolean true if a UDP port is requested, otherwise get a TCP port + * @return port number + */ + static uint16_t + GetAvailablePort(bool udp, nighthawk::client::AddressFamily::AddressFamilyOptions address_family); }; } // namespace Nighthawk diff --git a/test/integration/BUILD b/test/integration/BUILD index 2fb3063ce..0888067b5 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1,5 +1,6 @@ load( "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_binary", "envoy_package", ) load("@nh_pip3//:requirements.bzl", "requirement") @@ -28,9 +29,26 @@ py_library( ], ) +# envoy binary with Logical DNS support +envoy_cc_binary( + name = "envoy-static-testonly", + linkopts = [ + "-latomic", + "-lrt", + ], + repository = "@envoy", + deps = [ + "@envoy//source/exe:envoy_main_entry_lib", + "@envoy//source/extensions/clusters/logical_dns:logical_dns_cluster_lib", + "@envoy//source/extensions/clusters/original_dst:original_dst_cluster_lib", + "@envoy//source/extensions/load_balancing_policies/cluster_provided:config", + ], +) + py_library( name = "integration_test_base", data = [ + ":envoy-static-testonly", ":test_server_configs", "//:nighthawk_client_testonly", "//:nighthawk_output_transform", diff --git a/test/integration/configurations/terminating_http1_connect_envoy.yaml b/test/integration/configurations/terminating_http1_connect_envoy.yaml new file mode 100644 index 000000000..ce824a693 --- /dev/null +++ b/test/integration/configurations/terminating_http1_connect_envoy.yaml @@ -0,0 +1,58 @@ +admin: + address: + socket_address: + protocol: TCP + address: $server_ip + port_value: 0 +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: TCP + address: $server_ip + port_value: 0 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: + - "*" + routes: + - match: + connect_matcher: + {} + route: + cluster: localhost_routing + upgrade_configs: + - upgrade_type: CONNECT + connect_config: + {} + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + http_protocol_options: {} + upgrade_configs: + - upgrade_type: CONNECT + clusters: + - name: localhost_routing + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: AUTO + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: localhost_routing + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: $server_ip + port_value: $target_server_port diff --git a/test/integration/configurations/terminating_http2_connect_envoy.yaml b/test/integration/configurations/terminating_http2_connect_envoy.yaml new file mode 100644 index 000000000..d76bd7086 --- /dev/null +++ b/test/integration/configurations/terminating_http2_connect_envoy.yaml @@ -0,0 +1,56 @@ +admin: + address: + socket_address: { address: $server_ip, port_value: 0 } +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: TCP + address: $server_ip + port_value: 0 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: + - "*" + routes: + - match: + connect_matcher: + {} + route: + cluster: localhost_routing + upgrade_configs: + - upgrade_type: CONNECT + connect_config: + {} + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + http2_protocol_options: + allow_connect: true + upgrade_configs: + - upgrade_type: CONNECT + clusters: + - name: localhost_routing + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: AUTO + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: localhost_routing + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: $server_ip + port_value: $target_server_port \ No newline at end of file diff --git a/test/integration/configurations/terminating_http2_connect_udp_envoy.yaml b/test/integration/configurations/terminating_http2_connect_udp_envoy.yaml new file mode 100644 index 000000000..a9b4d18f4 --- /dev/null +++ b/test/integration/configurations/terminating_http2_connect_udp_envoy.yaml @@ -0,0 +1,56 @@ +admin: + address: + socket_address: { address: $server_ip, port_value: 0 } +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: TCP + address: $server_ip + port_value: 0 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: + - "*" + routes: + - match: + connect_matcher: + {} + route: + cluster: localhost_routing + upgrade_configs: + - upgrade_type: CONNECT-UDP + connect_config: + {} + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + http2_protocol_options: + allow_connect: true + upgrade_configs: + - upgrade_type: CONNECT-UDP + clusters: + - name: localhost_routing + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: AUTO + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: localhost_routing + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: $server_ip + port_value: $target_server_port diff --git a/test/integration/configurations/terminating_http3_connect_envoy.yaml b/test/integration/configurations/terminating_http3_connect_envoy.yaml new file mode 100644 index 000000000..8bc2e34d6 --- /dev/null +++ b/test/integration/configurations/terminating_http3_connect_envoy.yaml @@ -0,0 +1,73 @@ +admin: + address: + socket_address: { address: $server_ip, port_value: 0 } +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: UDP + address: $server_ip + port_value: 0 + udp_listener_config: + quic_options: {} + downstream_socket_config: + prefer_gro: true + filter_chains: + - transport_socket: + name: envoy.transport_sockets.quic + typed_config: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.quic.v3.QuicDownstreamTransport + downstream_tls_context: + common_tls_context: + tls_certificates: + - certificate_chain: + inline_string: | + @inject-runfile:nighthawk/external/envoy/test/config/integration/certs/upstreamlocalhostcert.pem + private_key: + inline_string: | + @inject-runfile:nighthawk/external/envoy/test/config/integration/certs/upstreamlocalhostkey.pem + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: HTTP3 + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: + - "*" + routes: + - match: + connect_matcher: + {} + route: + cluster: localhost_routing + upgrade_configs: + - upgrade_type: CONNECT + connect_config: + {} + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + http3_protocol_options: + allow_extended_connect: true + upgrade_configs: + - upgrade_type: CONNECT + clusters: + - name: localhost_routing + type: LOGICAL_DNS + dns_lookup_family: AUTO + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: localhost_routing + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: $server_ip + port_value: $target_server_port \ No newline at end of file diff --git a/test/integration/integration_test.py b/test/integration/integration_test.py index c1b8703f7..6d1eb84c0 100644 --- a/test/integration/integration_test.py +++ b/test/integration/integration_test.py @@ -21,14 +21,37 @@ "no:cacheprovider", # Avoid a bunch of warnings on readonly filesystems "-k", test_selection_arg, # Passed in via BUILD/py_test() + "-m" + "serial", "-x", path, "-n", - str(num_workers), + "1", # Run in serial "--log-level", "INFO", "--log-cli-level", "INFO", ], plugins=["xdist"]) + # if (r != 0): + # exit(r) + # r = pytest.main( + # [ + # "--rootdir=" + path, + # "-p", + # "no:cacheprovider", # Avoid a bunch of warnings on readonly filesystems + # "-k", + # test_selection_arg, # Passed in via BUILD/py_test() + # "-m" + # "not serial", + # "-x", + # path, + # "-n", + # str(num_workers), + # "--log-level", + # "INFO", + # "--log-cli-level", + # "INFO", + # ], + # plugins=["xdist"]) exit(r) diff --git a/test/integration/integration_test_fixtures.py b/test/integration/integration_test_fixtures.py index 30e9502e2..ce404dd90 100644 --- a/test/integration/integration_test_fixtures.py +++ b/test/integration/integration_test_fixtures.py @@ -268,6 +268,8 @@ def runNighthawkClient(self, args = [self.nighthawk_client_path] + args if self.ip_version == IpVersion.IPV6: args.append("--address-family v6") + else: + args.append("--address-family v4") if as_json: args.append("--output-format json") logging.info("Nighthawk client popen() args: %s" % str.join(" ", args)) @@ -380,6 +382,100 @@ def __init__(self, request, server_config_quic): self.server_ip = "::1" if self.ip_version == IpVersion.IPV6 else "127.0.0.1" +class TunnelingConnectUdpIntegrationTestBase(QuicIntegrationTestBase): + """Base class for HTTP CONNECT UDP based tunneling.""" + + def __init__(self, request, server_config, terminating_proxy_config): + """See base class.""" + super(TunnelingConnectUdpIntegrationTestBase, self).__init__(request, server_config) + self.server_ip = "::1" if self.ip_version == IpVersion.IPV6 else "127.0.0.1" + self._terminating_proxy_config_path = terminating_proxy_config + self._envoy_exe_path = "test/integration/envoy-static-testonly" + + def getTunnelProtocol(self): + """Get HTTP protocol used by tunnel.""" + return self._tunnel_protocol + + def getTunnelUri(self, https=False): + """Get the http://host:port/ for envoy to query the server we started in setUp().""" + uri_host = self.server_ip + if self.ip_version == IpVersion.IPV6: + uri_host = "[%s]" % self.server_ip + uri = "%s://%s:%s/" % ("https" if https else "http", uri_host, + self._terminating_envoy.server_port) + return uri + + def getTestServerRootUri(self): + """See base class.""" + return super(TunnelingConnectUdpIntegrationTestBase, self).getTestServerRootUri() + + def _tryStartTerminatingEnvoy(self): + self._terminating_envoy = NighthawkTestServer(self._envoy_exe_path, + self._terminating_proxy_config_path, + self.server_ip, + self.ip_version, + self.request, + parameters=self.parameters, + tag=self.tag + "envoy") + if not self._terminating_envoy.start(): + return False + return True + + def setUp(self): + """Set up the Terminating Envoy and target server.""" + super(TunnelingConnectUdpIntegrationTestBase, self).setUp() + # Terminating envoy's template needs listener port of the target webserver + self.parameters["target_server_port"] = self.test_server.server_port + assert self._tryStartTerminatingEnvoy(), "Tunneling envoy failed to start" + + +class TunnelingConnectIntegrationTestBase(HttpIntegrationTestBase): + """Base class for HTTP CONNECT based tunneling.""" + + def __init__(self, request, server_config, terminating_proxy_config): + """See base class.""" + super(TunnelingConnectIntegrationTestBase, self).__init__(request, server_config) + self.server_ip = "::1" if self.ip_version == IpVersion.IPV6 else "127.0.0.1" + self._terminating_proxy_config_path = terminating_proxy_config + self._envoy_exe_path = "test/integration/envoy-static-testonly" + + def getTunnelProtocol(self): + """Get Terminating envoy protocol.""" + return self._tunnel_protocol + + def getTunnelUri(self, https=False): + """Get the http://host:port/ for envoy to query the server we started in setUp().""" + uri_host = self.server_ip + if self.ip_version == IpVersion.IPV6: + uri_host = "[%s]" % self.server_ip + uri = "%s://%s:%s/" % ("https" if https else "http", uri_host, + self._terminating_envoy.server_port) + return uri + + def getTestServerRootUri(self): + """See base class.""" + return super(TunnelingConnectIntegrationTestBase, self).getTestServerRootUri() + + def _tryStartTerminatingEnvoy(self): + self._terminating_envoy = NighthawkTestServer(self._envoy_exe_path, + self._terminating_proxy_config_path, + self.server_ip, + self.ip_version, + self.request, + parameters=self.parameters, + tag=self.tag + "envoy") + if not self._terminating_envoy.start(): + return False + return True + + def setUp(self): + """Set up terminating envoy and target web server.""" + super(TunnelingConnectIntegrationTestBase, self).setUp() + # Terminating envoy's template needs listener port of the target webserver + self.parameters["target_server_port"] = self.test_server.server_port + assert self._tryStartTerminatingEnvoy(), "Tunneling envoy failed to start" + + class SniIntegrationTestBase(HttpsIntegrationTestBase): """Base for https/sni tests against the Nighthawk test server.""" @@ -428,6 +524,16 @@ def server_config_quic(): yield "nighthawk/test/integration/configurations/nighthawk_https_origin_quic.yaml" +@pytest.fixture() +def terminating_proxy_config(): + """Fixture which yields the path to an envoy terminating proxy configuration. + + Yields: + String: Path to the proxy configuration. + """ + yield "nighthawk/test/integration/configurations/terminating_http1_connect_envoy.yaml" + + @pytest.fixture(params=determineIpVersionsFromEnvironment()) def http_test_server_fixture(request, server_config, caplog): """Fixture for setting up a test environment with the stock http server configuration. @@ -467,6 +573,33 @@ def quic_test_server_fixture(request, server_config_quic, caplog): f.tearDown(caplog) +@pytest.fixture(params=determineIpVersionsFromEnvironment()) +def tunneling_connect_udp_test_server_fixture(request, server_config_quic, terminating_proxy_config, + caplog): + """Fixture for setting up a test environment with the stock https server and CONNECT UDP terminating proxy. + + Yields: + TunnelingConnectIntegrationUdpTestBase: A fully set up instance. Tear down will happen automatically. + """ + f = TunnelingConnectUdpIntegrationTestBase(request, server_config_quic, terminating_proxy_config) + f.setUp() + yield f + f.tearDown(caplog) + + +@pytest.fixture(params=determineIpVersionsFromEnvironment()) +def tunneling_connect_test_server_fixture(request, server_config, terminating_proxy_config, caplog): + """Fixture for setting up a test environment with the stock http server and a CONNECT terminating proxy. + + Yields: + TunnelingConnectIntegrationTestBase: A fully set up instance. Tear down will happen automatically. + """ + f = TunnelingConnectIntegrationTestBase(request, server_config, terminating_proxy_config) + f.setUp() + yield f + f.tearDown(caplog) + + @pytest.fixture(params=determineIpVersionsFromEnvironment()) def multi_http_test_server_fixture(request, server_config, caplog): """Fixture for setting up a test environment with multiple servers, using the stock http server configuration. diff --git a/test/integration/nighthawk_test_server.py b/test/integration/nighthawk_test_server.py index 8efddf081..36221c0e1 100644 --- a/test/integration/nighthawk_test_server.py +++ b/test/integration/nighthawk_test_server.py @@ -220,6 +220,7 @@ def _tryUpdateFromAdminInterface(self): try: listeners = self.fetchJsonFromAdminInterface("/listeners?format=json") # We assume the listeners all use the same address. + for listener in listeners["listener_statuses"]: port = listener["local_address"]["socket_address"]["port_value"] self.server_ports.append(port) @@ -275,7 +276,7 @@ def stop(self): class NighthawkTestServer(TestServerBase): - """Run the Nighthawk test server in a separate process. + """Run the Nighthawk test server or envoy in a separate process. Passes in the right cli-arg to point it to its configuration. For, say, NGINX this would be '-c' instead. @@ -289,7 +290,7 @@ def __init__(self, request, parameters=dict(), tag=""): - """Initialize a NighthawkTestServer instance. + """Initialize a NighthawkTestServer instance or an envoy instance. Args: server_binary_path (String): Path to the nighthawk test server binary. diff --git a/test/integration/test_integration_basics.py b/test/integration/test_integration_basics.py index 254441bbe..c7cf35bcc 100644 --- a/test/integration/test_integration_basics.py +++ b/test/integration/test_integration_basics.py @@ -14,7 +14,8 @@ from test.integration.integration_test_fixtures import ( http_test_server_fixture, https_test_server_fixture, https_test_server_fixture, multi_http_test_server_fixture, multi_https_test_server_fixture, quic_test_server_fixture, - server_config, server_config_quic) + server_config, server_config_quic, tunneling_connect_test_server_fixture, + tunneling_connect_udp_test_server_fixture) from test.integration import asserts from test.integration import utility @@ -194,6 +195,134 @@ def test_http_h2(http_test_server_fixture): asserts.assertGreaterEqual(len(counters), 12) +@pytest.mark.serial +@pytest.mark.parametrize( + 'terminating_proxy_config, tunnel_protocol', + [ + ("nighthawk/test/integration/configurations/terminating_http1_connect_envoy.yaml", "http1"), + # ("nighthawk/test/integration/configurations/terminating_http2_connect_envoy.yaml", "http2"), + # ("nighthawk/test/integration/configurations/terminating_http3_connect_envoy.yaml", "http3"), + ]) +def test_connect_tunneling(tunneling_connect_test_server_fixture, tunnel_protocol): + """Test h1, h2 over h1/2/3 CONNECT tunnels. + + Runs the CLI configured to use h2c against our test server, and sanity + checks statistics from both client and server. + """ + client_params = [ + "--tunnel-uri", + tunneling_connect_test_server_fixture.getTunnelUri(), "--tunnel-protocol", tunnel_protocol, + tunneling_connect_test_server_fixture.getTestServerRootUri(), "--max-active-requests", "1", + "--duration", "100", "--termination-predicate", "benchmark.http_2xx:24", "--rps", "100" + ] + path = os.path.join(os.environ["TEST_SRCDIR"], os.environ["TEST_WORKSPACE"], + "external/envoy/test/config/integration/certs/upstreamcacert.pem") + if (tunnel_protocol == "http3"): + client_params = client_params + [ + "--tunnel-tls-context", "{common_tls_context:{validation_context:{trusted_ca:{filename:\"" + + path + "\"},trust_chain_verification:\"ACCEPT_UNTRUSTED\"} }," + "sni:\"localhost\"}" + ] + # H2 as underlying protocol + parsed_json, _ = tunneling_connect_test_server_fixture.runNighthawkClient(client_params + + ["--protocol http2"]) + counters = tunneling_connect_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json) + asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25) + asserts.assertCounterEqual(counters, "upstream_cx_http2_total", 1) + asserts.assertCounterGreaterEqual(counters, "upstream_cx_rx_bytes_total", 900) + asserts.assertCounterEqual(counters, "upstream_cx_total", 1) + asserts.assertCounterGreaterEqual(counters, "upstream_cx_tx_bytes_total", 403) + asserts.assertCounterEqual(counters, "upstream_rq_pending_total", 1) + asserts.assertCounterEqual(counters, "upstream_rq_total", 25) + asserts.assertCounterEqual(counters, "default.total_match_count", 1) + asserts.assertGreaterEqual(len(counters), 12) + + # Do H1 as underlying protocol + + parsed_json, _ = tunneling_connect_test_server_fixture.runNighthawkClient(client_params + + ["--protocol http1"]) + counters = tunneling_connect_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json) + asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25) + asserts.assertCounterEqual(counters, "upstream_cx_rx_bytes_total", 3400) + # It is possible that the # of upstream_cx > # of backend connections for H1 + # as new connections will spawn if the existing clients cannot keep up with the RPS. + asserts.assertCounterGreaterEqual(counters, "upstream_cx_http1_total", 1) + asserts.assertCounterGreaterEqual(counters, "upstream_cx_total", 1) + asserts.assertCounterGreaterEqual(counters, "upstream_cx_tx_bytes_total", 500) + asserts.assertCounterGreaterEqual(counters, "upstream_rq_pending_total", 1) + asserts.assertCounterEqual(counters, "upstream_rq_total", 25) + asserts.assertCounterEqual(counters, "default.total_match_count", 1) + + global_histograms = tunneling_connect_test_server_fixture.getNighthawkGlobalHistogramsbyIdFromJson( + parsed_json) + asserts.assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["count"]), + 25) + asserts.assertEqual(int(global_histograms["benchmark_http_client.response_header_size"]["count"]), + 25) + asserts.assertEqual( + int(global_histograms["benchmark_http_client.response_body_size"]["raw_mean"]), 10) + asserts.assertEqual( + int(global_histograms["benchmark_http_client.response_header_size"]["raw_mean"]), 97) + asserts.assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["raw_min"]), + 10) + asserts.assertEqual( + int(global_histograms["benchmark_http_client.response_header_size"]["raw_min"]), 97) + asserts.assertEqual(int(global_histograms["benchmark_http_client.response_body_size"]["raw_max"]), + 10) + asserts.assertEqual( + int(global_histograms["benchmark_http_client.response_header_size"]["raw_max"]), 97) + asserts.assertEqual( + int(global_histograms["benchmark_http_client.response_body_size"]["raw_pstdev"]), 0) + asserts.assertEqual( + int(global_histograms["benchmark_http_client.response_header_size"]["raw_pstdev"]), 0) + + asserts.assertGreaterEqual(len(counters), 12) + + +# @pytest.mark.serial +@pytest.mark.parametrize('terminating_proxy_config', [ + ("nighthawk/test/integration/configurations/terminating_http2_connect_udp_envoy.yaml"), +]) +def test_connect_udp_tunneling(tunneling_connect_udp_test_server_fixture): + """Test h3 quic over h2 CONNECT-UDP tunnel. + + Runs the CLI configured to use HTTP/3 Quic against our test server, and sanity + checks statistics from both client and server. + """ + client_params = [ + "--protocol http3", + tunneling_connect_udp_test_server_fixture.getTestServerRootUri(), + "--rps", + "100", + "--duration", + "100", + "--termination-predicate", + "benchmark.http_2xx:24", + "--max-active-requests", + "1", + # Envoy doesn't support disabling certificate verification on Quic + # connections, so the host in our requests has to match the hostname in + # the leaf certificate. + "--request-header", + "Host:www.lyft.com", + "--tunnel-protocol", + "http2", + "--tunnel-uri", + tunneling_connect_udp_test_server_fixture.getTunnelUri(), + ] + parsed_json, _ = tunneling_connect_udp_test_server_fixture.runNighthawkClient(client_params) + + counters = tunneling_connect_udp_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json) + asserts.assertCounterEqual(counters, "benchmark.http_2xx", 25) + asserts.assertCounterEqual(counters, "upstream_cx_http3_total", 1) + asserts.assertCounterEqual(counters, "upstream_cx_total", 1) + asserts.assertCounterEqual(counters, "upstream_rq_pending_total", 1) + asserts.assertCounterEqual(counters, "upstream_rq_total", 25) + asserts.assertCounterEqual(counters, "default.total_match_count", 1) + + return + + def test_http_concurrency(http_test_server_fixture): """Test that concurrency acts like a multiplier.""" parsed_json, _ = http_test_server_fixture.runNighthawkClient([ diff --git a/test/mocks/client/mock_options.h b/test/mocks/client/mock_options.h index de2cbb591..f6ebf656e 100644 --- a/test/mocks/client/mock_options.h +++ b/test/mocks/client/mock_options.h @@ -20,6 +20,18 @@ class MockOptions : public Options { MOCK_METHOD(Envoy::Http::Protocol, protocol, (), (const, override)); MOCK_METHOD(absl::optional&, http3ProtocolOptions, (), (const, override)); + + // HTTP CONNECT/CONNECT-UDP Tunneling related options. + MOCK_METHOD(Envoy::Http::Protocol, tunnelProtocol, (), (const, override)); + MOCK_METHOD(std::string, tunnelUri, (), (const PURE)); + MOCK_METHOD(uint32_t, encapPort, (), (const PURE)); + MOCK_METHOD( + const absl::optional, + tunnelTlsContext, (), (const PURE)); + MOCK_METHOD(const absl::optional&, + tunnelHttp3ProtocolOptions, (), (const PURE)); + + MOCK_METHOD(std::string, tunnelConcurrency, (), (const PURE)); MOCK_METHOD(std::string, concurrency, (), (const, override)); MOCK_METHOD(nighthawk::client::Verbosity::VerbosityOptions, verbosity, (), (const, override)); MOCK_METHOD(nighthawk::client::OutputFormat::OutputFormatOptions, outputFormat, (), diff --git a/test/options_test.cc b/test/options_test.cc index 27cf18bb4..23072adbd 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -1203,5 +1203,98 @@ TEST_F(OptionsImplTest, ThrowsMalformedArgvExceptionForInvalidTypedExtensionConf MalformedArgvException, "UserDefinedPluginConfigs"); } +TEST_F(OptionsImplTest, TunnelModeHInvalidProtocolCombination) { + // not implemented in envoy. + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl( + fmt::format("{} {} --protocol http3 --tunnel-protocol http1 --tunnel-uri http://foo/", + client_name_, good_test_uri_)), + MalformedArgvException, "--protocol HTTP3 over --tunnel-protocol HTTP1 is not supported"); + + std::string tls_context = + "{sni:\"localhost\",common_tls_context:{validation_context:{trusted_ca:{filename:" + "\"fakeRootCA.pem\"},trust_chain_verification:\"ACCEPT_UNTRUSTED\"}}}"; + + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http3 --tunnel-protocol http3 " + "--tunnel-uri http://foo/ --tunnel-tls-context {}", + client_name_, good_test_uri_, tls_context)), + MalformedArgvException, "--protocol HTTP3 over --tunnel-protocol HTTP3 is not supported"); + + EXPECT_NO_THROW( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http1 --tunnel-protocol http3 " + "--tunnel-uri http://foo/ --tunnel-tls-context {}", + client_name_, good_test_uri_, tls_context))); + + EXPECT_NO_THROW( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http2 --tunnel-protocol http3 " + "--tunnel-uri http://foo/ --tunnel-tls-context {}", + client_name_, good_test_uri_, tls_context))); + + EXPECT_NO_THROW( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http1 --tunnel-protocol http1 " + "--tunnel-uri http://foo/", + client_name_, good_test_uri_, tls_context))); + + EXPECT_NO_THROW( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http2 --tunnel-protocol http1 " + "--tunnel-uri http://foo/", + client_name_, good_test_uri_, tls_context))); + + EXPECT_NO_THROW( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http2 --tunnel-protocol http2 " + "--tunnel-uri http://foo/", + client_name_, good_test_uri_, tls_context))); + + EXPECT_NO_THROW( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http1 --tunnel-protocol http2 " + "--tunnel-uri http://foo/", + client_name_, good_test_uri_, tls_context))); +} + +TEST_F(OptionsImplTest, TunnelModeMissingParams) { + // test missing tunnel URI + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http1 --tunnel-protocol http1", + client_name_, good_test_uri_)), + MalformedArgvException, "--tunnel-protocol requires --tunnel-uri"); + + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http2 --tunnel-protocol http1", + client_name_, good_test_uri_)), + MalformedArgvException, "--tunnel-protocol requires --tunnel-uri"); + + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http3 --tunnel-protocol http1", + client_name_, good_test_uri_)), + MalformedArgvException, "--tunnel-protocol requires --tunnel-uri"); + + std::string tls_context = + "{sni:\"localhost\",common_tls_context:{validation_context:{trusted_ca:{filename:" + "\"fakeRootCA.pem\"},trust_chain_verification:\"ACCEPT_UNTRUSTED\"}}}"; + + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http1 --tunnel-tls-context {}", + client_name_, good_test_uri_, tls_context)), + MalformedArgvException, "tunnel flags require --tunnel-protocol"); + + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http2 --tunnel-tls-context {}", + client_name_, good_test_uri_, tls_context)), + MalformedArgvException, "tunnel flags require --tunnel-protocol"); + + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl(fmt::format("{} {} --protocol http3 --tunnel-tls-context {}", + client_name_, good_test_uri_, tls_context)), + MalformedArgvException, "tunnel flags require --tunnel-protocol"); + + // test missing TLS context for H3 tunnel + EXPECT_THROW_WITH_REGEX( + TestUtility::createOptionsImpl( + fmt::format("{} {} --protocol http2 --tunnel-protocol http3 --tunnel-uri http://foo/", + client_name_, good_test_uri_)), + MalformedArgvException, "--tunnel-tls-context is required to use --tunnel-protocol http3"); +} + } // namespace Client } // namespace Nighthawk diff --git a/test/process_bootstrap_test.cc b/test/process_bootstrap_test.cc index 9f5e65ff0..bbc9ebb30 100644 --- a/test/process_bootstrap_test.cc +++ b/test/process_bootstrap_test.cc @@ -22,6 +22,7 @@ #include "test/client/utility.h" #include "test/test_common/proto_matchers.h" +#include "absl/strings/substitute.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -1952,5 +1953,202 @@ TEST_F(CreateBootstrapConfigurationTest, DnsResolverFactoryError) { ASSERT_THAT(bootstrap, StatusIs(absl::StatusCode::kInternal)); } +TEST_F(CreateBootstrapConfigurationTest, CreateEncapBootstrap) { + setupUriResolutionExpectations(); + + std::unique_ptr options = Client::TestUtility::createOptionsImpl( + "nighthawk_client http://www.example.org --address-family v4 --tunnel-protocol http2 " + "--tunnel-uri http://www.example.org"); + UriImpl tunnel_uri("www.example.org"); + tunnel_uri.resolve(mock_dispatcher_, *mock_resolver_, Envoy::Network::DnsLookupFamily::V4Only); + auto encap_bootstrap = + createEncapBootstrap(*options, tunnel_uri, mock_dispatcher_, mock_resolver_); + ASSERT_THAT(encap_bootstrap, StatusIs(absl::StatusCode::kOk)); + + uint16_t encap_port = options->encapPort(); + absl::StatusOr expected_bootstrap = + parseBootstrapFromText(absl::Substitute(R"pb( +static_resources { + listeners { + name: "encap_listener" + address { + socket_address { + address: "127.0.0.1" + port_value: $0 + } + } + filter_chains { + filters { + name: "envoy.filters.network.tcp_proxy" + typed_config { + [type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy] { + stat_prefix: "tcp_proxy" + cluster: "cluster_0" + tunneling_config { + hostname: "host.com:443" + headers_to_add { + header { + key: "original_dst_port" + value: "%DOWNSTREAM_LOCAL_PORT%" + } + } + } + } + } + } + } + } + clusters { + name: "cluster_0" + connect_timeout { + seconds: 5 + } + load_assignment { + cluster_name: "cluster_0" + endpoints { + lb_endpoints { + endpoint { + address { + socket_address { + address: "127.0.0.1" + port_value: 80 + } + } + } + } + } + } + typed_extension_protocol_options { + key: "envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + value { + [type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions] { + explicit_http_config { + http2_protocol_options { + } + } + } + } + } + } +} +stats_server_version_override { + value: 1 +} +)pb", + encap_port)); + ASSERT_THAT(expected_bootstrap, StatusIs(absl::StatusCode::kOk)); + EXPECT_THAT(*encap_bootstrap, EqualsProto(*expected_bootstrap)); +} + +TEST_F(CreateBootstrapConfigurationTest, CreateEncapBootstrapWithCustomTLSContextH3Options) { + setupUriResolutionExpectations(); + + std::unique_ptr options = Client::TestUtility::createOptionsImpl( + "nighthawk_client http://www.example.org --address-family v4" + " --tunnel-protocol http3 --tunnel-uri http://www.example.org --tunnel-tls-context" + " {sni:\"localhost\",common_tls_context:{validation_context:" + "{trusted_ca:{filename:\"fakeRootCA.pem\"},trust_chain_verification:\"ACCEPT_UNTRUSTED\"}}}" + + ); + + uint16_t encap_port = options->encapPort(); + UriImpl tunnel_uri("www.example.org"); + tunnel_uri.resolve(mock_dispatcher_, *mock_resolver_, Envoy::Network::DnsLookupFamily::V4Only); + auto encap_bootstrap = + createEncapBootstrap(*options, tunnel_uri, mock_dispatcher_, mock_resolver_); + ASSERT_THAT(encap_bootstrap, StatusIs(absl::StatusCode::kOk)); + + absl::StatusOr expected_bootstrap = + parseBootstrapFromText(absl::Substitute(R"pb( +static_resources { + listeners { + name: "encap_listener" + address { + socket_address { + address: "127.0.0.1" + port_value: $0 + } + } + filter_chains { + filters { + name: "envoy.filters.network.tcp_proxy" + typed_config { + [type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy] { + stat_prefix: "tcp_proxy" + cluster: "cluster_0" + tunneling_config { + hostname: "host.com:443" + headers_to_add { + header { + key: "original_dst_port" + value: "%DOWNSTREAM_LOCAL_PORT%" + } + } + } + } + } + } + } + } + clusters { + name: "cluster_0" + connect_timeout { + seconds: 5 + } + transport_socket { + name: "envoy.transport_sockets.quic" + typed_config { + [type.googleapis.com/envoy.extensions.transport_sockets.quic.v3.QuicUpstreamTransport] { + upstream_tls_context { + common_tls_context { + validation_context { + trusted_ca { + filename: "fakeRootCA.pem" + } + trust_chain_verification: ACCEPT_UNTRUSTED + } + } + sni: "localhost" + } + } + } + } + load_assignment { + cluster_name: "cluster_0" + endpoints { + lb_endpoints { + endpoint { + address { + socket_address { + address: "127.0.0.1" + port_value: 80 + } + } + } + } + } + } + typed_extension_protocol_options { + key: "envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + value { + [type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions] { + explicit_http_config { + http3_protocol_options { + } + } + } + } + } + } +} +stats_server_version_override { + value: 1 +} +)pb", + encap_port)); + ASSERT_THAT(expected_bootstrap, StatusIs(absl::StatusCode::kOk)); + EXPECT_THAT(*encap_bootstrap, EqualsProto(*expected_bootstrap)); +} + } // namespace } // namespace Nighthawk diff --git a/test/process_test.cc b/test/process_test.cc index 8384da2b6..dd2f983e9 100644 --- a/test/process_test.cc +++ b/test/process_test.cc @@ -365,6 +365,13 @@ TEST_P(ProcessTest, FailsWhenDnsResolverFactoryFails) { .ok()); } +TEST_P(ProcessTest, TestWithEncapsulation) { + options_ = TestUtility::createOptionsImpl( + fmt::format("foo --tunnel-uri https://{}/ --tunnel-protocol http1 --protocol http1 " + "--concurrency 2 https://{}/", + loopback_address_, loopback_address_)); + EXPECT_TRUE(runProcess(RunExpectation::EXPECT_FAILURE).ok()); +} /** * Fixture for executing the Nighthawk process with simulated time. */ diff --git a/test/utility_test.cc b/test/utility_test.cc index 3c435d41f..a1f0894e6 100644 --- a/test/utility_test.cc +++ b/test/utility_test.cc @@ -1,3 +1,4 @@ +#include #include #include "external/envoy/source/common/network/dns_resolver/dns_factory_util.h" @@ -197,4 +198,20 @@ TEST_F(UtilityTest, MultipleSemicolons) { EXPECT_THROW(UriImpl("HTTP://HTTP://a:111"), UriException); } +TEST_F(UtilityTest, GetAvailablePort) { + + for (auto ip_version : {nighthawk::client::AddressFamily::AddressFamilyOptions:: + AddressFamily_AddressFamilyOptions_V4, + nighthawk::client::AddressFamily::AddressFamilyOptions:: + AddressFamily_AddressFamilyOptions_V6}) { + uint16_t tcp_port = 0; + uint16_t udp_port = 0; + + EXPECT_NO_THROW({ tcp_port = Utility::GetAvailablePort(/*udp=*/false, ip_version); }); + EXPECT_NO_THROW({ udp_port = Utility::GetAvailablePort(/*udp=*/true, ip_version); }); + EXPECT_GT(tcp_port, 0); + EXPECT_GT(udp_port, 0); + } +} + } // namespace Nighthawk