Add Support for Detecting Synthetic Source (#3674)

Author: JacksonWeber

CHANGELOG.md

@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+- `opentelemetry-instrumentation-requests`, `opentelemetry-instrumentation-wsgi`, `opentelemetry-instrumentation-asgi` Detect synthetic sources on requests, ASGI, and WSGI.
+  ([#3674](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3674))
+
 ### Added 
 
 - `opentelemetry-instrumentation-aiohttp-client`: add support for url exclusions via `OTEL_PYTHON_EXCLUDED_URLS` / `OTEL_PYTHON_AIOHTTP_CLIENT_EXCLUDED_URLS`

instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py

@@ -258,6 +258,9 @@ def client_response_hook(span: Span, scope: Scope, message: dict[str, Any]):
 from opentelemetry.instrumentation.utils import _start_internal_or_server_span
 from opentelemetry.metrics import get_meter
 from opentelemetry.propagators.textmap import Getter, Setter
+from opentelemetry.semconv._incubating.attributes.user_agent_attributes import (
+    USER_AGENT_SYNTHETIC_TYPE,
+)
 from opentelemetry.semconv._incubating.metrics.http_metrics import (
     create_http_server_active_requests,
     create_http_server_request_body_size,
@@ -276,6 +279,7 @@ def client_response_hook(span: Span, scope: Scope, message: dict[str, Any]):
     ExcludeList,
     SanitizeValue,
     _parse_url_query,
+    detect_synthetic_user_agent,
     get_custom_headers,
     normalise_request_header_name,
     normalise_response_header_name,
@@ -397,7 +401,13 @@ def collect_request_attributes(
             )
     http_user_agent = asgi_getter.get(scope, "user-agent")
     if http_user_agent:
-        _set_http_user_agent(result, http_user_agent[0], sem_conv_opt_in_mode)
+        user_agent_value = http_user_agent[0]
+        _set_http_user_agent(result, user_agent_value, sem_conv_opt_in_mode)
+
+        # Check for synthetic user agent type
+        synthetic_type = detect_synthetic_user_agent(user_agent_value)
+        if synthetic_type:
+            result[USER_AGENT_SYNTHETIC_TYPE] = synthetic_type
 
     if "client" in scope and scope["client"] is not None:
         _set_http_peer_ip_server(

instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py

@@ -42,6 +42,9 @@
     HistogramDataPoint,
     NumberDataPoint,
 )
+from opentelemetry.semconv._incubating.attributes.user_agent_attributes import (
+    USER_AGENT_SYNTHETIC_TYPE,
+)
 from opentelemetry.semconv.attributes.client_attributes import (
     CLIENT_ADDRESS,
     CLIENT_PORT,
@@ -883,6 +886,145 @@ def update_expected_user_agent(expected):
             new_sem_conv=True,
         )
 
+    async def test_user_agent_synthetic_bot_detection(self):
+        """Test that bot user agents are detected as synthetic with type 'bot'"""
+        test_cases = [
+            b"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+            b"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+            b"googlebot/1.0",
+            b"bingbot/1.0",
+        ]
+
+        # Test each user agent case separately to avoid span accumulation
+        for user_agent in test_cases:
+            with self.subTest(user_agent=user_agent):
+                # Clear headers first
+                self.scope["headers"] = []
+
+                def update_expected_synthetic_bot(
+                    expected, ua: bytes = user_agent
+                ):
+                    expected[3]["attributes"].update(
+                        {
+                            SpanAttributes.HTTP_USER_AGENT: ua.decode("utf8"),
+                            USER_AGENT_SYNTHETIC_TYPE: "bot",
+                        }
+                    )
+                    return expected
+
+                self.scope["headers"].append([b"user-agent", user_agent])
+                app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+                self.seed_app(app)
+                await self.send_default_request()
+                outputs = await self.get_all_output()
+                self.validate_outputs(
+                    outputs, modifiers=[update_expected_synthetic_bot]
+                )
+
+                # Clear spans after each test case to prevent accumulation
+                self.memory_exporter.clear()
+
+    async def test_user_agent_synthetic_test_detection(self):
+        """Test that test user agents are detected as synthetic with type 'test'"""
+        test_cases = [
+            b"alwayson/1.0",
+            b"AlwaysOn/2.0",
+            b"test-alwayson-client",
+        ]
+
+        # Test each user agent case separately to avoid span accumulation
+        for user_agent in test_cases:
+            with self.subTest(user_agent=user_agent):
+                # Clear headers first
+                self.scope["headers"] = []
+
+                def update_expected_synthetic_test(
+                    expected, ua: bytes = user_agent
+                ):
+                    expected[3]["attributes"].update(
+                        {
+                            SpanAttributes.HTTP_USER_AGENT: ua.decode("utf8"),
+                            USER_AGENT_SYNTHETIC_TYPE: "test",
+                        }
+                    )
+                    return expected
+
+                self.scope["headers"].append([b"user-agent", user_agent])
+                app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+                self.seed_app(app)
+                await self.send_default_request()
+                outputs = await self.get_all_output()
+                self.validate_outputs(
+                    outputs, modifiers=[update_expected_synthetic_test]
+                )
+
+                # Clear spans after each test case to prevent accumulation
+                self.memory_exporter.clear()
+
+    async def test_user_agent_non_synthetic(self):
+        """Test that normal user agents are not marked as synthetic"""
+        test_cases = [
+            b"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            b"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
+            b"PostmanRuntime/7.28.4",
+            b"curl/7.68.0",
+        ]
+
+        # Test each user agent case separately to avoid span accumulation
+        for user_agent in test_cases:
+            with self.subTest(user_agent=user_agent):
+                # Clear headers first
+                self.scope["headers"] = []
+
+                def update_expected_non_synthetic(
+                    expected, ua: bytes = user_agent
+                ):
+                    # Should only have the user agent, not synthetic type
+                    expected[3]["attributes"].update(
+                        {
+                            SpanAttributes.HTTP_USER_AGENT: ua.decode("utf8"),
+                        }
+                    )
+                    return expected
+
+                self.scope["headers"].append([b"user-agent", user_agent])
+                app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+                self.seed_app(app)
+                await self.send_default_request()
+                outputs = await self.get_all_output()
+                self.validate_outputs(
+                    outputs, modifiers=[update_expected_non_synthetic]
+                )
+
+                # Clear spans after each test case to prevent accumulation
+                self.memory_exporter.clear()
+
+    async def test_user_agent_synthetic_new_semconv(self):
+        """Test synthetic user agent detection with new semantic conventions"""
+        user_agent = b"Mozilla/5.0 (compatible; Googlebot/2.1)"
+
+        def update_expected_synthetic_new_semconv(expected):
+            expected[3]["attributes"].update(
+                {
+                    USER_AGENT_ORIGINAL: user_agent.decode("utf8"),
+                    USER_AGENT_SYNTHETIC_TYPE: "bot",
+                }
+            )
+            return expected
+
+        self.scope["headers"] = []
+        self.scope["headers"].append([b"user-agent", user_agent])
+        app = otel_asgi.OpenTelemetryMiddleware(simple_asgi)
+        self.seed_app(app)
+        await self.send_default_request()
+        outputs = await self.get_all_output()
+        self.validate_outputs(
+            outputs,
+            modifiers=[update_expected_synthetic_new_semconv],
+            old_sem_conv=False,
+            new_sem_conv=True,
+        )
+
     async def test_traceresponse_header(self):
         """Test a traceresponse header is sent when a global propagator is set."""
 

instrumentation/opentelemetry-instrumentation-requests/src/opentelemetry/instrumentation/requests/__init__.py

@@ -132,11 +132,17 @@ def response_hook(span, request_obj, response):
 )
 from opentelemetry.metrics import Histogram, get_meter
 from opentelemetry.propagate import inject
+from opentelemetry.semconv._incubating.attributes.user_agent_attributes import (
+    USER_AGENT_SYNTHETIC_TYPE,
+)
 from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE
 from opentelemetry.semconv.attributes.network_attributes import (
     NETWORK_PEER_ADDRESS,
     NETWORK_PEER_PORT,
 )
+from opentelemetry.semconv.attributes.user_agent_attributes import (
+    USER_AGENT_ORIGINAL,
+)
 from opentelemetry.semconv.metrics import MetricInstruments
 from opentelemetry.semconv.metrics.http_metrics import (
     HTTP_CLIENT_REQUEST_DURATION,
@@ -145,6 +151,7 @@ def response_hook(span, request_obj, response):
 from opentelemetry.trace.span import Span
 from opentelemetry.util.http import (
     ExcludeList,
+    detect_synthetic_user_agent,
     get_excluded_urls,
     parse_excluded_urls,
     redact_url,
@@ -243,6 +250,15 @@ def get_or_create_headers():
         )
         _set_http_url(span_attributes, url, sem_conv_opt_in_mode)
 
+        # Check for synthetic user agent type
+        headers = get_or_create_headers()
+        user_agent = headers.get("User-Agent")
+        synthetic_type = detect_synthetic_user_agent(user_agent)
+        if synthetic_type:
+            span_attributes[USER_AGENT_SYNTHETIC_TYPE] = synthetic_type
+        if user_agent:
+            span_attributes[USER_AGENT_ORIGINAL] = user_agent
+
         metric_labels = {}
         _set_http_method(
             metric_labels,
@@ -297,7 +313,6 @@ def get_or_create_headers():
             if callable(request_hook):
                 request_hook(span, request)
 
-            headers = get_or_create_headers()
             inject(headers)
 
             with suppress_http_instrumentation():

instrumentation/opentelemetry-instrumentation-requests/tests/test_requests_integration.py

@@ -63,6 +63,9 @@
     SERVER_PORT,
 )
 from opentelemetry.semconv.attributes.url_attributes import URL_FULL
+from opentelemetry.semconv.attributes.user_agent_attributes import (
+    USER_AGENT_ORIGINAL,
+)
 from opentelemetry.test.mock_textmap import MockTextMapPropagator
 from opentelemetry.test.test_base import TestBase
 from opentelemetry.trace import StatusCode
@@ -175,6 +178,7 @@ def test_basic(self):
                 HTTP_METHOD: "GET",
                 HTTP_URL: self.URL,
                 HTTP_STATUS_CODE: 200,
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
 
@@ -211,6 +215,7 @@ def test_basic_new_semconv(self):
                 NETWORK_PROTOCOL_VERSION: "1.1",
                 SERVER_PORT: 80,
                 NETWORK_PEER_PORT: 80,
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
 
@@ -253,6 +258,7 @@ def test_basic_both_semconv(self):
                 NETWORK_PROTOCOL_VERSION: "1.1",
                 SERVER_PORT: 80,
                 NETWORK_PEER_PORT: 80,
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
 
@@ -276,6 +282,7 @@ def test_nonstandard_http_method(self):
                 HTTP_METHOD: "_OTHER",
                 HTTP_URL: self.URL,
                 HTTP_STATUS_CODE: 405,
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
 
@@ -300,6 +307,7 @@ def test_nonstandard_http_method_new_semconv(self):
                 NETWORK_PROTOCOL_VERSION: "1.1",
                 ERROR_TYPE: "405",
                 HTTP_REQUEST_METHOD_ORIGINAL: "NONSTANDARD",
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
         self.assertIs(span.status.status_code, trace.StatusCode.ERROR)
@@ -534,6 +542,7 @@ def response_hook(
                 HTTP_URL: self.URL,
                 HTTP_STATUS_CODE: 200,
                 "http.response.body": "Hello!",
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
 
@@ -564,6 +573,7 @@ def test_requests_exception_without_response(self, *_, **__):
             {
                 HTTP_METHOD: "GET",
                 HTTP_URL: self.URL,
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
         self.assertEqual(span.status.status_code, StatusCode.ERROR)
@@ -591,6 +601,7 @@ def test_requests_exception_new_semconv(self, *_, **__):
                 NETWORK_PEER_PORT: 80,
                 NETWORK_PEER_ADDRESS: "mock",
                 ERROR_TYPE: "RequestException",
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
         self.assertEqual(span.status.status_code, StatusCode.ERROR)
@@ -613,6 +624,7 @@ def test_requests_exception_without_proper_response_type(self, *_, **__):
             {
                 HTTP_METHOD: "GET",
                 HTTP_URL: self.URL,
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
         self.assertEqual(span.status.status_code, StatusCode.ERROR)
@@ -636,6 +648,7 @@ def test_requests_exception_with_response(self, *_, **__):
                 HTTP_METHOD: "GET",
                 HTTP_URL: self.URL,
                 HTTP_STATUS_CODE: 500,
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )
         self.assertEqual(span.status.status_code, StatusCode.ERROR)
@@ -675,6 +688,7 @@ def test_adapter_with_custom_response(self):
                 "http.method": "GET",
                 "http.url": self.URL,
                 "http.status_code": 210,
+                USER_AGENT_ORIGINAL: "python-requests/2.32.3",
             },
         )