httpx: add support for OTEL_PYTHON_EXCLUDED_URLS / OTEL_PYTHON_HTTPX_EXCLUDED_URLS

Author: xrmx

instrumentation/opentelemetry-instrumentation-httpx/src/opentelemetry/instrumentation/httpx/__init__.py

@@ -199,6 +199,24 @@ async def async_response_hook(span, request, response):
         response_hook=async_response_hook
     )
 
+
+Configuration
+-------------
+
+Exclude lists
+*************
+To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_HTTPX_EXCLUDED_URLS``
+(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the
+URLs.
+
+For example,
+
+::
+
+    export OTEL_PYTHON_HTTPX_EXCLUDED_URLS="client/.*/info,healthcheck"
+
+will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``.
+
 API
 ---
 """
@@ -259,7 +277,12 @@ async def async_response_hook(span, request, response):
 from opentelemetry.trace import SpanKind, Tracer, TracerProvider, get_tracer
 from opentelemetry.trace.span import Span
 from opentelemetry.trace.status import StatusCode
-from opentelemetry.util.http import redact_url, sanitize_method
+from opentelemetry.util.http import (
+    ExcludeList,
+    get_excluded_urls,
+    redact_url,
+    sanitize_method,
+)
 
 _logger = logging.getLogger(__name__)
 
@@ -304,7 +327,7 @@ def _extract_parameters(
     args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any]
 ) -> tuple[
     bytes,
-    httpx.URL,
+    httpx.URL | tuple[bytes, bytes, int | None, bytes],
     httpx.Headers | None,
     httpx.SyncByteStream | httpx.AsyncByteStream | None,
     dict[str, typing.Any],
@@ -330,6 +353,21 @@ def _extract_parameters(
     return method, url, headers, stream, extensions
 
 
+def _normalize_url(
+    url: httpx.URL | tuple[bytes, bytes, int | None, bytes],
+) -> str:
+    if isinstance(url, tuple):
+        scheme, host, port, path = [
+            part.decode() if isinstance(part, bytes) else part for part in url
+        ]
+        if port:
+            return f"{scheme}://{host}:{port}{path}"
+        else:
+            return f"{scheme}://{host}{path}"
+
+    return str(url)
+
+
 def _inject_propagation_headers(headers, args, kwargs):
     _headers = _prepare_headers(headers)
     inject(_headers)
@@ -485,6 +523,7 @@ class SyncOpenTelemetryTransport(httpx.BaseTransport):
             right after the span is created
         response_hook: A hook that receives the span, request, and response
             that is called right before the span ends
+        excluded_urls: an ExcludeList instance as returned by opentelemetry.util.http.get_excluded_urls
     """
 
     def __init__(
@@ -533,6 +572,7 @@ def __init__(
             )
         self._request_hook = request_hook
         self._response_hook = response_hook
+        self._excluded_urls = get_excluded_urls("HTTPX")
 
     def __enter__(self) -> SyncOpenTelemetryTransport:
         self._transport.__enter__()
@@ -562,6 +602,12 @@ def handle_request(
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if self._excluded_urls and self._excluded_urls.url_disabled(
+            _normalize_url(url)
+        ):
+            return self._transport.handle_request(*args, **kwargs)
+
         method_original = method.decode()
         span_name = _get_default_span_name(method_original)
         span_attributes = {}
@@ -676,6 +722,7 @@ class AsyncOpenTelemetryTransport(httpx.AsyncBaseTransport):
             right after the span is created
         response_hook: A hook that receives the span, request, and response
             that is called right before the span ends
+        excluded_urls: an ExcludeList instance as returned by opentelemetry.util.http.get_
     """
 
     def __init__(
@@ -685,6 +732,7 @@ def __init__(
         meter_provider: MeterProvider | None = None,
         request_hook: AsyncRequestHook | None = None,
         response_hook: AsyncResponseHook | None = None,
+        excluded_urls: ExcludeList | None = None,
     ):
         _OpenTelemetrySemanticConventionStability._initialize()
         self._sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode(
@@ -726,6 +774,7 @@ def __init__(
 
         self._request_hook = request_hook
         self._response_hook = response_hook
+        self._excluded_urls = excluded_urls
 
     async def __aenter__(self) -> "AsyncOpenTelemetryTransport":
         await self._transport.__aenter__()
@@ -753,6 +802,12 @@ async def handle_async_request(
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if self._excluded_urls and self._excluded_urls.url_disabled(
+            _normalize_url(url)
+        ):
+            return await self._transport.handle_async_request(*args, **kwargs)
+
         method_original = method.decode()
         span_name = _get_default_span_name(method_original)
         span_attributes = {}
@@ -900,6 +955,7 @@ def _instrument(self, **kwargs: typing.Any):
             if iscoroutinefunction(async_response_hook)
             else None
         )
+        excluded_urls = get_excluded_urls("HTTPX")
 
         _OpenTelemetrySemanticConventionStability._initialize()
         sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode(
@@ -948,6 +1004,7 @@ def _instrument(self, **kwargs: typing.Any):
                 sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                 request_hook=request_hook,
                 response_hook=response_hook,
+                excluded_urls=excluded_urls,
             ),
         )
         wrap_function_wrapper(
@@ -961,6 +1018,7 @@ def _instrument(self, **kwargs: typing.Any):
                 sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                 async_request_hook=async_request_hook,
                 async_response_hook=async_response_hook,
+                excluded_urls=excluded_urls,
             ),
         )
 
@@ -980,13 +1038,18 @@ def _handle_request_wrapper(  # pylint: disable=too-many-locals
         sem_conv_opt_in_mode: _StabilityMode,
         request_hook: RequestHook,
         response_hook: ResponseHook,
+        excluded_urls: ExcludeList | None,
     ):
         if not is_http_instrumentation_enabled():
             return wrapped(*args, **kwargs)
 
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if excluded_urls and excluded_urls.url_disabled(_normalize_url(url)):
+            return wrapped(*args, **kwargs)
+
         method_original = method.decode()
         span_name = _get_default_span_name(method_original)
         span_attributes = {}
@@ -1096,13 +1159,18 @@ async def _handle_async_request_wrapper(  # pylint: disable=too-many-locals
         sem_conv_opt_in_mode: _StabilityMode,
         async_request_hook: AsyncRequestHook,
         async_response_hook: AsyncResponseHook,
+        excluded_urls: ExcludeList | None,
     ):
         if not is_http_instrumentation_enabled():
             return await wrapped(*args, **kwargs)
 
         method, url, headers, stream, extensions = _extract_parameters(
             args, kwargs
         )
+
+        if excluded_urls and excluded_urls.url_disabled(_normalize_url(url)):
+            return await wrapped(*args, **kwargs)
+
         method_original = method.decode()
         span_name = _get_default_span_name(method_original)
         span_attributes = {}
@@ -1274,6 +1342,8 @@ def instrument_client(
             # response_hook already set
             async_response_hook = None
 
+        excluded_urls = get_excluded_urls("HTTPX")
+
         if hasattr(client._transport, "handle_request"):
             wrap_function_wrapper(
                 client._transport,
@@ -1286,6 +1356,7 @@ def instrument_client(
                     sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                     request_hook=request_hook,
                     response_hook=response_hook,
+                    excluded_urls=excluded_urls,
                 ),
             )
             for transport in client._mounts.values():
@@ -1301,6 +1372,7 @@ def instrument_client(
                             sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                             request_hook=request_hook,
                             response_hook=response_hook,
+                            excluded_urls=excluded_urls,
                         ),
                     )
             client._is_instrumented_by_opentelemetry = True
@@ -1316,6 +1388,7 @@ def instrument_client(
                     sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                     async_request_hook=async_request_hook,
                     async_response_hook=async_response_hook,
+                    excluded_urls=excluded_urls,
                 ),
             )
             for transport in client._mounts.values():
@@ -1331,6 +1404,7 @@ def instrument_client(
                             sem_conv_opt_in_mode=sem_conv_opt_in_mode,
                             async_request_hook=async_request_hook,
                             async_response_hook=async_response_hook,
+                            excluded_urls=excluded_urls,
                         ),
                     )
             client._is_instrumented_by_opentelemetry = True

instrumentation/opentelemetry-instrumentation-httpx/tests/test_httpx_integration.py

@@ -738,6 +738,17 @@ def test_if_headers_equals_none(self):
             self.assertEqual(result.text, "Hello!")
             self.assert_span()
 
+        def test_ignores_excluded_urls(self):
+            with mock.patch.dict(
+                "os.environ", {"OTEL_PYTHON_HTTPX_EXCLUDED_URLS": self.URL}
+            ):
+                client = self.create_client()
+                HTTPXClientInstrumentor().instrument_client(client=client)
+                self.perform_request(self.URL, client=client)
+            self.assert_span(num_spans=0)
+            self.assert_metrics(num_metrics=0)
+            HTTPXClientInstrumentor().uninstrument_client(client)
+
     class BaseManualTest(BaseTest, metaclass=abc.ABCMeta):
         @abc.abstractmethod
         def create_transport(
@@ -972,6 +983,17 @@ def test_client_mounts_with_instrumented_transport(self):
             self.assertEqual(spans[0].attributes[HTTP_URL], self.URL)
             self.assertEqual(spans[1].attributes[HTTP_URL], https_url)
 
+        def test_ignores_excluded_urls(self):
+            with mock.patch.dict(
+                "os.environ", {"OTEL_PYTHON_HTTPX_EXCLUDED_URLS": self.URL}
+            ):
+                client = self.create_client()
+                HTTPXClientInstrumentor().instrument_client(client=client)
+                self.perform_request(self.URL, client=client)
+            self.assert_span(num_spans=0)
+            self.assert_metrics(num_metrics=0)
+            HTTPXClientInstrumentor().uninstrument_client(client=client)
+
     @mock.patch.dict("os.environ", {"NO_PROXY": ""}, clear=True)
     class BaseInstrumentorTest(BaseTest, metaclass=abc.ABCMeta):
         @abc.abstractmethod
@@ -998,7 +1020,7 @@ def setUp(self):
             HTTPXClientInstrumentor().instrument_client(self.client)
 
         def tearDown(self):
-            HTTPXClientInstrumentor().uninstrument()
+            HTTPXClientInstrumentor().uninstrument_client(self.client)
 
         def create_proxy_mounts(self):
             return {
@@ -1329,6 +1351,17 @@ def test_uninstrument_client_with_proxy(self):
             self.assertEqual(result.text, "Hello!")
             self.assert_span()
 
+        def test_ignores_excluded_urls(self):
+            # need to instrument again for the environment variable
+            HTTPXClientInstrumentor().uninstrument_client(self.client)
+            with mock.patch.dict(
+                "os.environ", {"OTEL_PYTHON_HTTPX_EXCLUDED_URLS": self.URL}
+            ):
+                HTTPXClientInstrumentor().instrument_client(client=self.client)
+                self.perform_request(self.URL, client=self.client)
+            self.assert_span(num_spans=0)
+            self.assert_metrics(num_metrics=0)
+
 
 class TestSyncIntegration(BaseTestCases.BaseManualTest):
     def setUp(self):