diff --git a/.env-devel b/.env-devel index cd60f2e9d365..9980683c0637 100644 --- a/.env-devel +++ b/.env-devel @@ -377,7 +377,7 @@ TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE=2 TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT=http://opentelemetry-collector TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318 TRACING_OPENTELEMETRY_COLLECTOR_PORT=4318 -TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=100 +TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY=1.0 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack TRASH_RETENTION_DAYS=7 TWILIO_ACCOUNT_SID=DUMMY diff --git a/packages/service-library/src/servicelib/aiohttp/tracing.py b/packages/service-library/src/servicelib/aiohttp/tracing.py index 140a24852e4c..e2c75aa85de5 100644 --- a/packages/service-library/src/servicelib/aiohttp/tracing.py +++ b/packages/service-library/src/servicelib/aiohttp/tracing.py @@ -1,6 +1,7 @@ """Adds aiohttp middleware for tracing using opentelemetry instrumentation.""" import logging +import time from collections.abc import AsyncIterator, Callable from typing import Final @@ -13,18 +14,28 @@ AioHttpClientInstrumentor, ) from opentelemetry.instrumentation.aiohttp_server import ( - middleware as aiohttp_server_opentelemetry_middleware, # pylint:disable=no-name-in-module + _parse_active_request_count_attrs, + _parse_duration_attrs, + collect_request_attributes, + get_default_span_details, + getter, + meter, + set_status_code, ) -from opentelemetry.sdk.resources import Resource +from opentelemetry.propagate import extract from opentelemetry.sdk.trace import SpanProcessor, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.semconv.metrics import MetricInstruments from settings_library.tracing import TracingSettings from yarl import URL from ..logging_utils import log_context -from ..tracing import get_trace_id_header +from ..tracing import TracingConfig, get_trace_id_header _logger = logging.getLogger(__name__) + +TRACING_CONFIG_KEY: Final[str] = "tracing_config" + try: from opentelemetry.instrumentation.botocore import ( # type: ignore[import-not-found] BotocoreInstrumentor, @@ -65,6 +76,57 @@ ) +@web.middleware +async def aiohttp_server_opentelemetry_middleware(request: web.Request, handler): + """This middleware is extracted from https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/instrumentation/opentelemetry-instrumentation-aiohttp-server/src/opentelemetry/instrumentation/aiohttp_server/__init__.py + and adapted to allow passing the tracer provider via the app instead of using the global object. The original code for the function is licensed under https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/LICENSE. + FIXME: I have recorded this limitation in the official source here: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3801 and plan on providing a fix soon. + """ + + span_name, additional_attributes = get_default_span_details(request) + + req_attrs = collect_request_attributes(request) + duration_attrs = _parse_duration_attrs(req_attrs) + active_requests_count_attrs = _parse_active_request_count_attrs(req_attrs) + + duration_histogram = meter.create_histogram( + name=MetricInstruments.HTTP_SERVER_DURATION, + unit="ms", + description="Measures the duration of inbound HTTP requests.", + ) + + active_requests_counter = meter.create_up_down_counter( + name=MetricInstruments.HTTP_SERVER_ACTIVE_REQUESTS, + unit="requests", + description="measures the number of concurrent HTTP requests those are currently in flight", + ) + tracing_config = request.app[TRACING_CONFIG_KEY] + assert isinstance(tracing_config, TracingConfig) # nosec + assert tracing_config.tracer_provider # nosec + tracer = tracing_config.tracer_provider.get_tracer(__name__) + with tracer.start_as_current_span( + span_name, + context=extract(request, getter=getter), + kind=trace.SpanKind.SERVER, + ) as span: + attributes = collect_request_attributes(request) + attributes.update(additional_attributes) + span.set_attributes(attributes) + start = time.perf_counter() + active_requests_counter.add(1, active_requests_count_attrs) + try: + resp = await handler(request) + set_status_code(span, resp.status) + except web.HTTPException as ex: + set_status_code(span, ex.status_code) + raise + finally: + duration = max((time.perf_counter() - start) * 1000, 0) + duration_histogram.record(duration, duration_attrs) + active_requests_counter.add(-1, active_requests_count_attrs) + return resp + + def _create_span_processor(tracing_destination: str) -> SpanProcessor: otlp_exporter = OTLPSpanExporterHTTP( endpoint=tracing_destination, @@ -77,12 +139,12 @@ def _startup( app: web.Application, tracing_settings: TracingSettings, service_name: str, + tracer_provider: TracerProvider, add_response_trace_id_header: bool = False, ) -> None: """ Sets up this service for a distributed tracing system (opentelemetry) """ - _ = app opentelemetry_collector_endpoint = ( f"{tracing_settings.TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}" ) @@ -99,9 +161,6 @@ def _startup( "unset. Provide both or remove both." ) raise RuntimeError(msg) - resource = Resource(attributes={"service.name": service_name}) - trace.set_tracer_provider(TracerProvider(resource=resource)) - tracer_provider: trace.TracerProvider = trace.get_tracer_provider() tracing_destination: str = ( f"{URL(opentelemetry_collector_endpoint).with_port(opentelemetry_collector_port).with_path('/v1/traces')}" @@ -114,7 +173,7 @@ def _startup( ) # Add the span processor to the tracer provider - tracer_provider.add_span_processor(_create_span_processor(tracing_destination)) # type: ignore[attr-defined] # https://github.com/open-telemetry/opentelemetry-python/issues/3713 + tracer_provider.add_span_processor(_create_span_processor(tracing_destination)) # Instrument aiohttp server # Explanation for custom middleware call DK 10/2024: # OpenTelemetry Aiohttp autoinstrumentation is meant to be used by only calling `AioHttpServerInstrumentor().instrument()` @@ -135,35 +194,35 @@ def _startup( # - opentelemetry-instrumentation==0.48b0 # Instrument aiohttp client - AioHttpClientInstrumentor().instrument() + AioHttpClientInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_AIOPG: with log_context( _logger, logging.INFO, msg="Attempting to add aio-pg opentelemetry autoinstrumentation...", ): - AiopgInstrumentor().instrument() + AiopgInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_ASYNCPG: with log_context( _logger, logging.INFO, msg="Attempting to add asyncpg opentelemetry autoinstrumentation...", ): - AsyncPGInstrumentor().instrument() + AsyncPGInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_BOTOCORE: with log_context( _logger, logging.INFO, msg="Attempting to add botocore opentelemetry autoinstrumentation...", ): - BotocoreInstrumentor().instrument() + BotocoreInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_REQUESTS: with log_context( _logger, logging.INFO, msg="Attempting to add requests opentelemetry autoinstrumentation...", ): - RequestsInstrumentor().instrument() + RequestsInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_AIO_PIKA: with log_context( @@ -171,7 +230,7 @@ def _startup( logging.INFO, msg="Attempting to add aio_pika opentelemetry autoinstrumentation...", ): - AioPikaInstrumentor().instrument() + AioPikaInstrumentor().instrument(tracer_provider=tracer_provider) @web.middleware @@ -222,17 +281,24 @@ def _shutdown() -> None: _logger.exception("Failed to uninstrument AioPikaInstrumentor") -def get_tracing_lifespan( +def setup_tracing( *, app: web.Application, - tracing_settings: TracingSettings, - service_name: str, + tracing_config: TracingConfig, add_response_trace_id_header: bool = False, ) -> Callable[[web.Application], AsyncIterator]: + + if tracing_config.tracing_enabled is False: + msg = "Tracing is not enabled" + raise ValueError(msg) + assert tracing_config.tracer_provider # nosec + assert tracing_config.tracing_settings # nosec + _startup( app=app, - tracing_settings=tracing_settings, - service_name=service_name, + tracing_settings=tracing_config.tracing_settings, + tracer_provider=tracing_config.tracer_provider, + service_name=tracing_config.service_name, add_response_trace_id_header=add_response_trace_id_header, ) diff --git a/packages/service-library/src/servicelib/fastapi/client_session.py b/packages/service-library/src/servicelib/fastapi/client_session.py index f9c126272eec..287f974306a4 100644 --- a/packages/service-library/src/servicelib/fastapi/client_session.py +++ b/packages/service-library/src/servicelib/fastapi/client_session.py @@ -2,7 +2,7 @@ import httpx from fastapi import FastAPI -from settings_library.tracing import TracingSettings +from servicelib.tracing import TracingConfig from .tracing import setup_httpx_client_tracing @@ -12,7 +12,7 @@ def setup_client_session( *, default_timeout: datetime.timedelta = datetime.timedelta(seconds=20), max_keepalive_connections: int = 20, - tracing_settings: TracingSettings | None, + tracing_config: TracingConfig | None ) -> None: async def on_startup() -> None: session = httpx.AsyncClient( @@ -20,8 +20,8 @@ async def on_startup() -> None: limits=httpx.Limits(max_keepalive_connections=max_keepalive_connections), timeout=default_timeout.total_seconds(), ) - if tracing_settings: - setup_httpx_client_tracing(session) + if tracing_config: + setup_httpx_client_tracing(session, tracing_config=tracing_config) app.state.aiohttp_client_session = session async def on_shutdown() -> None: diff --git a/packages/service-library/src/servicelib/fastapi/http_client_thin.py b/packages/service-library/src/servicelib/fastapi/http_client_thin.py index a62461f00091..0a48f104d54d 100644 --- a/packages/service-library/src/servicelib/fastapi/http_client_thin.py +++ b/packages/service-library/src/servicelib/fastapi/http_client_thin.py @@ -8,7 +8,7 @@ from common_library.errors_classes import OsparcErrorMixin from httpx import AsyncClient, ConnectError, HTTPError, PoolTimeout, Response from httpx._types import TimeoutTypes, URLTypes -from settings_library.tracing import TracingSettings +from servicelib.tracing import TracingConfig from tenacity import RetryCallState from tenacity.asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log @@ -200,7 +200,7 @@ def __init__( self, *, total_retry_interval: float, - tracing_settings: TracingSettings | None, + tracing_config: TracingConfig, base_url: URLTypes | None = None, default_http_client_timeout: TimeoutTypes | None = None, extra_allowed_method_names: set[str] | None = None, @@ -224,8 +224,8 @@ def __init__( client_args["timeout"] = default_http_client_timeout client = AsyncClient(**client_args) - if tracing_settings: - setup_httpx_client_tracing(client) + if tracing_config.tracing_enabled: + setup_httpx_client_tracing(client, tracing_config=tracing_config) super().__init__(client=client) async def __aenter__(self): diff --git a/packages/service-library/src/servicelib/fastapi/logging_lifespan.py b/packages/service-library/src/servicelib/fastapi/logging_lifespan.py index 035d9bc10aa7..4cdcfb4a603a 100644 --- a/packages/service-library/src/servicelib/fastapi/logging_lifespan.py +++ b/packages/service-library/src/servicelib/fastapi/logging_lifespan.py @@ -4,7 +4,7 @@ from common_library.logging.logging_utils_filtering import LoggerName, MessageSubstring from fastapi import FastAPI -from settings_library.tracing import TracingSettings +from servicelib.tracing import TracingConfig from ..logging_utils import ( LogLevelInt, @@ -20,7 +20,7 @@ def create_logging_lifespan( *, log_format_local_dev_enabled: bool, logger_filter_mapping: dict[LoggerName, list[MessageSubstring]], - tracing_settings: TracingSettings | None, + tracing_config: TracingConfig, log_base_level: LogLevelInt, noisy_loggers: tuple[str, ...] | None, ) -> Lifespan: @@ -32,7 +32,7 @@ def create_logging_lifespan( noisy_loggers=noisy_loggers, log_format_local_dev_enabled=log_format_local_dev_enabled, logger_filter_mapping=logger_filter_mapping, - tracing_settings=tracing_settings, + tracing_config=tracing_config, ) ) @@ -49,7 +49,7 @@ def create_logging_shutdown_event( *, log_format_local_dev_enabled: bool, logger_filter_mapping: dict[LoggerName, list[MessageSubstring]], - tracing_settings: TracingSettings | None, + tracing_config: TracingConfig, log_base_level: LogLevelInt, noisy_loggers: tuple[str, ...] | None, ) -> Callable[[], Awaitable[None]]: @@ -67,7 +67,7 @@ def create_logging_shutdown_event( noisy_loggers=noisy_loggers, log_format_local_dev_enabled=log_format_local_dev_enabled, logger_filter_mapping=logger_filter_mapping, - tracing_settings=tracing_settings, + tracing_config=tracing_config, ) ) diff --git a/packages/service-library/src/servicelib/fastapi/tracing.py b/packages/service-library/src/servicelib/fastapi/tracing.py index 50c8aeab1d7d..2d6419a381e5 100644 --- a/packages/service-library/src/servicelib/fastapi/tracing.py +++ b/packages/service-library/src/servicelib/fastapi/tracing.py @@ -6,13 +6,11 @@ from fastapi import FastAPI, Request from fastapi_lifespan_manager import State from httpx import AsyncClient, Client -from opentelemetry import trace from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter as OTLPSpanExporterHTTP, ) from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor -from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import SpanProcessor, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from settings_library.tracing import TracingSettings @@ -20,7 +18,7 @@ from yarl import URL from ..logging_utils import log_context -from ..tracing import get_trace_id_header +from ..tracing import TracingConfig, get_trace_id_header _logger = logging.getLogger(__name__) @@ -78,18 +76,17 @@ def _create_span_processor(tracing_destination: str) -> SpanProcessor: return BatchSpanProcessor(otlp_exporter) -def _startup(tracing_settings: TracingSettings, service_name: str) -> None: +def _startup( + tracing_settings: TracingSettings, + service_name: str, + tracer_provider: TracerProvider, +) -> None: if ( not tracing_settings.TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT and not tracing_settings.TRACING_OPENTELEMETRY_COLLECTOR_PORT ): _logger.warning("Skipping opentelemetry tracing setup") return - # Set up the tracer provider - resource = Resource(attributes={"service.name": service_name}) - trace.set_tracer_provider(TracerProvider(resource=resource)) - global_tracer_provider = trace.get_tracer_provider() - assert isinstance(global_tracer_provider, TracerProvider) # nosec opentelemetry_collector_endpoint: str = ( f"{tracing_settings.TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}" @@ -105,9 +102,7 @@ def _startup(tracing_settings: TracingSettings, service_name: str) -> None: tracing_destination, ) # Add the span processor to the tracer provider - global_tracer_provider.add_span_processor( - _create_span_processor(tracing_destination) - ) + tracer_provider.add_span_processor(_create_span_processor(tracing_destination)) if HAS_AIOPG: with log_context( @@ -115,42 +110,42 @@ def _startup(tracing_settings: TracingSettings, service_name: str) -> None: logging.INFO, msg="Attempting to add asyncpg opentelemetry autoinstrumentation...", ): - AiopgInstrumentor().instrument() + AiopgInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_AIOPIKA_INSTRUMENTOR: with log_context( _logger, logging.INFO, msg="Attempting to add aio_pika opentelemetry autoinstrumentation...", ): - AioPikaInstrumentor().instrument() + AioPikaInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_ASYNCPG: with log_context( _logger, logging.INFO, msg="Attempting to add asyncpg opentelemetry autoinstrumentation...", ): - AsyncPGInstrumentor().instrument() + AsyncPGInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_REDIS: with log_context( _logger, logging.INFO, msg="Attempting to add redis opentelemetry autoinstrumentation...", ): - RedisInstrumentor().instrument() + RedisInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_BOTOCORE: with log_context( _logger, logging.INFO, msg="Attempting to add botocore opentelemetry autoinstrumentation...", ): - BotocoreInstrumentor().instrument() + BotocoreInstrumentor().instrument(tracer_provider=tracer_provider) if HAS_REQUESTS: with log_context( _logger, logging.INFO, msg="Attempting to add requests opentelemetry autoinstrumentation...", ): - RequestsInstrumentor().instrument() + RequestsInstrumentor().instrument(tracer_provider=tracer_provider) def _shutdown() -> None: @@ -189,22 +184,39 @@ def _shutdown() -> None: def initialize_fastapi_app_tracing( - app: FastAPI, *, add_response_trace_id_header: bool = False + app: FastAPI, + *, + tracing_config: TracingConfig, + add_response_trace_id_header: bool = False, ): if add_response_trace_id_header: app.add_middleware(ResponseTraceIdHeaderMiddleware) - FastAPIInstrumentor.instrument_app(app) + FastAPIInstrumentor.instrument_app( + app, tracer_provider=tracing_config.tracer_provider + ) -def setup_httpx_client_tracing(client: AsyncClient | Client): - HTTPXClientInstrumentor.instrument_client(client) +def setup_httpx_client_tracing( + client: AsyncClient | Client, tracing_config: TracingConfig +) -> None: + HTTPXClientInstrumentor.instrument_client( + client, tracer_provider=tracing_config.tracer_provider + ) -def setup_tracing( - app: FastAPI, tracing_settings: TracingSettings, service_name: str -) -> None: +def setup_tracing(app: FastAPI, tracing_config: TracingConfig) -> None: # NOTE: This does not instrument the app itself. Call setup_fastapi_app_tracing to do that. - _startup(tracing_settings=tracing_settings, service_name=service_name) + if not tracing_config.tracing_enabled: + msg = "Tracing is not enabled in tracing_config" + raise ValueError(msg) + assert tracing_config.tracing_settings # nosec + assert tracing_config.tracer_provider # nosec + + _startup( + tracing_settings=tracing_config.tracing_settings, + service_name=tracing_config.service_name, + tracer_provider=tracing_config.tracer_provider, + ) def _on_shutdown() -> None: _shutdown() @@ -212,11 +224,18 @@ def _on_shutdown() -> None: app.add_event_handler("shutdown", _on_shutdown) -def get_tracing_instrumentation_lifespan( - tracing_settings: TracingSettings, service_name: str -): +def get_tracing_instrumentation_lifespan(tracing_config: TracingConfig): # NOTE: This lifespan does not instrument the app itself. Call setup_fastapi_app_tracing to do that. - _startup(tracing_settings=tracing_settings, service_name=service_name) + if not tracing_config.tracing_enabled: + msg = "Tracing is not enabled in tracing_config" + raise ValueError(msg) + assert tracing_config.tracing_settings # nosec + assert tracing_config.tracer_provider # nosec + _startup( + tracing_settings=tracing_config.tracing_settings, + service_name=tracing_config.service_name, + tracer_provider=tracing_config.tracer_provider, + ) async def tracing_instrumentation_lifespan( app: FastAPI, @@ -238,3 +257,11 @@ async def dispatch(self, request: Request, call_next): if trace_id_header: response.headers.update(trace_id_header) return response + + +def get_tracing_config(app: FastAPI) -> TracingConfig: + assert hasattr( + app.state, "tracing_config" + ), "Tracing not setup for this app" # nosec + assert isinstance(app.state.tracing_config, TracingConfig) + return app.state.tracing_config diff --git a/packages/service-library/src/servicelib/logging_utils.py b/packages/service-library/src/servicelib/logging_utils.py index 27bd08dd87f6..45ead816e330 100644 --- a/packages/service-library/src/servicelib/logging_utils.py +++ b/packages/service-library/src/servicelib/logging_utils.py @@ -27,9 +27,8 @@ LoggerName, MessageSubstring, ) -from settings_library.tracing import TracingSettings -from .tracing import setup_log_tracing +from .tracing import TracingConfig, setup_log_tracing from .utils_secrets import mask_sensitive_data _logger = logging.getLogger(__name__) @@ -182,7 +181,7 @@ def _dampen_noisy_loggers( def _configure_common_logging_settings( *, log_format_local_dev_enabled: bool, - tracing_settings: TracingSettings | None, + tracing_config: TracingConfig, log_base_level: LogLevelInt, noisy_loggers: tuple[str, ...] | None, ) -> logging.Formatter: @@ -194,9 +193,7 @@ def _configure_common_logging_settings( _setup_base_logging_level(log_base_level) if noisy_loggers is not None: _dampen_noisy_loggers(noisy_loggers) - if tracing_settings is not None: - setup_log_tracing(tracing_settings=tracing_settings) - + setup_log_tracing(tracing_config=tracing_config) return _setup_logging_formatter( log_format_local_dev_enabled=log_format_local_dev_enabled, ) @@ -220,7 +217,7 @@ def setup_loggers( *, log_format_local_dev_enabled: bool, logger_filter_mapping: dict[LoggerName, list[MessageSubstring]], - tracing_settings: TracingSettings | None, + tracing_config: TracingConfig, log_base_level: LogLevelInt, noisy_loggers: tuple[str, ...] | None, ) -> None: @@ -262,7 +259,7 @@ def setup_loggers( """ formatter = _configure_common_logging_settings( log_format_local_dev_enabled=log_format_local_dev_enabled, - tracing_settings=tracing_settings, + tracing_config=tracing_config, log_base_level=log_base_level, noisy_loggers=noisy_loggers, ) @@ -328,7 +325,7 @@ def async_loggers( *, log_format_local_dev_enabled: bool, logger_filter_mapping: dict[LoggerName, list[MessageSubstring]], - tracing_settings: TracingSettings | None, + tracing_config: TracingConfig, log_base_level: LogLevelInt, noisy_loggers: tuple[str, ...] | None, ) -> Iterator[None]: @@ -376,7 +373,7 @@ def async_loggers( """ formatter = _configure_common_logging_settings( log_format_local_dev_enabled=log_format_local_dev_enabled, - tracing_settings=tracing_settings, + tracing_config=tracing_config, log_base_level=log_base_level, noisy_loggers=noisy_loggers, ) diff --git a/packages/service-library/src/servicelib/tracing.py b/packages/service-library/src/servicelib/tracing.py index a95f386495e6..a7513391e992 100644 --- a/packages/service-library/src/servicelib/tracing.py +++ b/packages/service-library/src/servicelib/tracing.py @@ -1,14 +1,16 @@ -from collections.abc import Callable, Coroutine from contextlib import contextmanager from contextvars import Token -from functools import wraps -from typing import Any, Final, TypeAlias +from typing import Final, Self, TypeAlias import pyinstrument import pyinstrument.renderers from opentelemetry import context as otcontext from opentelemetry import trace from opentelemetry.instrumentation.logging import LoggingInstrumentor +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased +from pydantic import BaseModel, ConfigDict, model_validator from settings_library.tracing import TracingSettings TracingContext: TypeAlias = otcontext.Context | None @@ -40,9 +42,48 @@ def use_tracing_context(context: TracingContext): otcontext.detach(token) -def setup_log_tracing(tracing_settings: TracingSettings): - _ = tracing_settings - LoggingInstrumentor().instrument(set_logging_format=False) +class TracingConfig(BaseModel): + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + service_name: str + tracing_settings: TracingSettings | None + tracer_provider: TracerProvider | None + + @model_validator(mode="after") + def _check_tracing_fields(self): + if (self.tracing_settings is None) != (self.tracer_provider is None): + msg = "Both 'tracing_settings' and 'tracer_provider' must be None or both not None." + raise ValueError(msg) + return self + + @property + def tracing_enabled(self) -> bool: + return self.tracer_provider is not None and self.tracing_settings is not None + + @classmethod + def create( + cls, tracing_settings: TracingSettings | None, service_name: str + ) -> Self: + tracer_provider: TracerProvider | None = None + if tracing_settings: + resource = Resource(attributes={"service.name": service_name}) + sampler = ParentBased( + root=TraceIdRatioBased( + tracing_settings.TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY + ) + ) + tracer_provider = TracerProvider(resource=resource, sampler=sampler) + return cls( + service_name=service_name, + tracing_settings=tracing_settings, + tracer_provider=tracer_provider, + ) + + +def setup_log_tracing(tracing_config: TracingConfig): + if tracing_config.tracing_enabled: + LoggingInstrumentor().instrument( + set_logging_format=False, tracer_provider=tracing_config.tracer_provider + ) def get_trace_id_header() -> dict[str, str] | None: @@ -57,39 +98,31 @@ def get_trace_id_header() -> dict[str, str] | None: return None -def with_profiled_span( - func: Callable[..., Coroutine[Any, Any, Any]], -) -> Callable[..., Coroutine[Any, Any, Any]]: - """Decorator that wraps an async function in an OpenTelemetry span with pyinstrument profiling.""" - - @wraps(func) - async def wrapper(*args: Any, **kwargs: Any) -> Any: - if not _is_tracing(): - return await func(*args, **kwargs) - - tracer = trace.get_tracer(_TRACER_NAME) - span_name = f"{func.__module__}.{func.__qualname__}" - - with tracer.start_as_current_span(span_name) as span: - profiler = pyinstrument.Profiler(async_mode="enabled") - profiler.start() - - try: - return await func(*args, **kwargs) - - except Exception as e: - span.record_exception(e) - span.set_status(trace.Status(trace.StatusCode.ERROR, f"{e}")) - raise - - finally: - profiler.stop() - renderer = pyinstrument.renderers.ConsoleRenderer( - unicode=True, color=False, show_all=True - ) - span.set_attribute( - _PROFILE_ATTRIBUTE_NAME, - profiler.output(renderer=renderer), - ) - - return wrapper +@contextmanager +def profiled_span(*, tracing_config: TracingConfig, span_name: str): + if not _is_tracing(): + return + tracer = trace.get_tracer( + _TRACER_NAME, tracer_provider=tracing_config.tracer_provider + ) + with tracer.start_as_current_span(span_name) as span: + profiler = pyinstrument.Profiler(async_mode="enabled") + profiler.start() + + try: + yield + + except Exception as e: + span.record_exception(e) + span.set_status(trace.Status(trace.StatusCode.ERROR, f"{e}")) + raise + + finally: + profiler.stop() + renderer = pyinstrument.renderers.ConsoleRenderer( + unicode=True, color=False, show_all=True + ) + span.set_attribute( + _PROFILE_ATTRIBUTE_NAME, + profiler.output(renderer=renderer), + ) diff --git a/packages/service-library/tests/aiohttp/test_tracing.py b/packages/service-library/tests/aiohttp/test_tracing.py index c9fb30d7de85..199cb7e9fe57 100644 --- a/packages/service-library/tests/aiohttp/test_tracing.py +++ b/packages/service-library/tests/aiohttp/test_tracing.py @@ -2,6 +2,7 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable +import asyncio import importlib from collections.abc import Callable from functools import partial @@ -14,8 +15,8 @@ from opentelemetry import trace from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from pydantic import ValidationError -from servicelib.aiohttp.tracing import get_tracing_lifespan -from servicelib.tracing import _OSPARC_TRACE_ID_HEADER +from servicelib.aiohttp.tracing import TRACING_CONFIG_KEY, setup_tracing +from servicelib.tracing import _OSPARC_TRACE_ID_HEADER, TracingConfig from settings_library.tracing import TracingSettings @@ -40,17 +41,25 @@ def set_and_clean_settings_env_vars( monkeypatch.setenv( "TRACING_OPENTELEMETRY_COLLECTOR_PORT", f"{tracing_settings_in[1]}" ) + sampling_probability_mocked = False + if tracing_settings_in[2]: + sampling_probability_mocked = True + monkeypatch.setenv( + "TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY", tracing_settings_in[2] + ) yield if endpoint_mocked: monkeypatch.delenv("TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT") if port_mocked: monkeypatch.delenv("TRACING_OPENTELEMETRY_COLLECTOR_PORT") + if sampling_probability_mocked: + monkeypatch.delenv("TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY") @pytest.mark.parametrize( "tracing_settings_in", [ - ("http://opentelemetry-collector", 4318), + ("http://opentelemetry-collector", 4318, 1.0), ], indirect=True, ) @@ -63,18 +72,19 @@ async def test_valid_tracing_settings( app = web.Application() service_name = "simcore_service_webserver" tracing_settings = TracingSettings() - async for _ in get_tracing_lifespan( - app=app, service_name=service_name, tracing_settings=tracing_settings - )(app): + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name=service_name + ) + async for _ in setup_tracing(app=app, tracing_config=tracing_config)(app): pass @pytest.mark.parametrize( "tracing_settings_in", [ - ("http://opentelemetry-collector", 80), - ("opentelemetry-collector", 4318), - ("httsdasp://ot@##el-collector", 4318), + ("http://opentelemetry-collector", 80, 1.0), + ("opentelemetry-collector", 4318, 1.0), + ("httsdasp://ot@##el-collector", 4318, 1.0), ], indirect=True, ) @@ -111,14 +121,14 @@ def manage_package(request): "tracing_settings_in, manage_package", [ ( - ("http://opentelemetry-collector", 4318), + ("http://opentelemetry-collector", 4318, 1.0), ( "opentelemetry-instrumentation-botocore", "opentelemetry.instrumentation.botocore", ), ), ( - ("http://opentelemetry-collector", "4318"), + ("http://opentelemetry-collector", "4318", 1.0), ( "opentelemetry-instrumentation-aiopg", "opentelemetry.instrumentation.aiopg", @@ -140,24 +150,19 @@ async def test_tracing_setup_package_detection( app = web.Application() service_name = "simcore_service_webserver" tracing_settings = TracingSettings() - async for _ in get_tracing_lifespan( - app=app, - service_name=service_name, - tracing_settings=tracing_settings, - )(app): + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name=service_name + ) + async for _ in setup_tracing(app=app, tracing_config=tracing_config)(app): # idempotency - async for _ in get_tracing_lifespan( - app=app, - service_name=service_name, - tracing_settings=tracing_settings, - )(app): + async for _ in setup_tracing(app=app, tracing_config=tracing_config)(app): pass @pytest.mark.parametrize( "tracing_settings_in", [ - ("http://opentelemetry-collector", 4318), + ("http://opentelemetry-collector", 4318, 1.0), ], indirect=True, ) @@ -174,6 +179,10 @@ async def test_trace_id_in_response_header( app = web.Application() service_name = "simcore_service_webserver" tracing_settings = TracingSettings() + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name=service_name + ) + app[TRACING_CONFIG_KEY] = tracing_config async def handler(handler_data: dict, request: web.Request) -> web.Response: current_span = trace.get_current_span() @@ -187,10 +196,9 @@ async def handler(handler_data: dict, request: web.Request) -> web.Response: handler_data = dict() app.router.add_get("/", partial(handler, handler_data)) - async for _ in get_tracing_lifespan( + async for _ in setup_tracing( app=app, - service_name=service_name, - tracing_settings=tracing_settings, + tracing_config=tracing_config, add_response_trace_id_header=True, )(app): client = await aiohttp_client(app) @@ -201,3 +209,58 @@ async def handler(handler_data: dict, request: web.Request) -> web.Response: assert ( trace_id == handler_data[_OSPARC_TRACE_ID_HEADER] ) # Ensure trace IDs match + + +@pytest.mark.parametrize( + "tracing_settings_in", + [ + ("http://opentelemetry-collector", 4318, 0.05), + ], + indirect=True, +) +async def test_TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY_effective( + mock_otel_collector: InMemorySpanExporter, + aiohttp_client: Callable, + set_and_clean_settings_env_vars: Callable[[], None], + tracing_settings_in, +): + """ + This test checks that the TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY setting in TracingSettings + is effective by sending 1000 requests and verifying that the number of collected traces + is close to 0.05 * 1000 (with some tolerance). + """ + n_requests = 1000 + tolerance_probability = 0.5 + + app = web.Application() + service_name = "simcore_service_webserver" + tracing_settings = TracingSettings() + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name=service_name + ) + app[TRACING_CONFIG_KEY] = tracing_config + + async def handler(request: web.Request) -> web.Response: + return web.Response(text="ok") + + app.router.add_get("/", handler) + + async for _ in setup_tracing(app=app, tracing_config=tracing_config)(app): + client = await aiohttp_client(app) + + await asyncio.gather(*(client.get("/") for _ in range(n_requests))) + trace_ids = { + span.context.trace_id + for span in mock_otel_collector.get_finished_spans() + if span.context is not None + } + n_traces = len(trace_ids) + expected_num_traces = int( + tracing_settings.TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY * n_requests + ) + tolerance = int(tolerance_probability * expected_num_traces) + assert ( + expected_num_traces - tolerance + <= n_traces + <= expected_num_traces + tolerance + ), f"Expected roughly {expected_num_traces} distinct trace ids, got {n_traces}" diff --git a/packages/service-library/tests/fastapi/test_http_client_thin.py b/packages/service-library/tests/fastapi/test_http_client_thin.py index 0a1f800f5107..0fc72f354ceb 100644 --- a/packages/service-library/tests/fastapi/test_http_client_thin.py +++ b/packages/service-library/tests/fastapi/test_http_client_thin.py @@ -24,6 +24,7 @@ expect_status, retry_on_errors, ) +from servicelib.tracing import TracingConfig _TIMEOUT_OVERWRITE: Final[int] = 1 @@ -71,8 +72,11 @@ def request_timeout() -> int: @pytest.fixture async def thick_client(request_timeout: int) -> AsyncIterable[FakeThickClient]: + tracing_config = TracingConfig.create( + service_name="test-client", tracing_settings=None + ) async with FakeThickClient( - total_retry_interval=request_timeout, tracing_settings=None + total_retry_interval=request_timeout, tracing_config=tracing_config ) as client: yield client @@ -99,8 +103,11 @@ async def test_retry_on_errors( test_url: str, caplog_info_level: pytest.LogCaptureFixture, ) -> None: + tracing_config = TracingConfig.create( + service_name="test-client", tracing_settings=None + ) client = FakeThickClient( - total_retry_interval=request_timeout, tracing_settings=None + total_retry_interval=request_timeout, tracing_config=tracing_config ) with pytest.raises(ClientHttpError): @@ -125,7 +132,12 @@ async def raises_request_error(self) -> Response: request=Request(method="GET", url=test_url), ) - client = ATestClient(total_retry_interval=request_timeout, tracing_settings=None) + tracing_config = TracingConfig.create( + service_name="test-client", tracing_settings=None + ) + client = ATestClient( + total_retry_interval=request_timeout, tracing_config=tracing_config + ) with pytest.raises(ClientHttpError): await client.raises_request_error() @@ -151,7 +163,12 @@ async def raises_http_error(self) -> Response: msg = "mock_http_error" raise HTTPError(msg) - client = ATestClient(total_retry_interval=request_timeout, tracing_settings=None) + tracing_config = TracingConfig.create( + service_name="test-client", tracing_settings=None + ) + client = ATestClient( + total_retry_interval=request_timeout, tracing_config=tracing_config + ) with pytest.raises(ClientHttpError): await client.raises_http_error() @@ -165,7 +182,10 @@ async def public_method_ok(self) -> Response: # type: ignore """this method will be ok even if no code is used""" # OK - OKTestClient(total_retry_interval=request_timeout, tracing_settings=None) + tracing_config = TracingConfig.create( + service_name="test-client", tracing_settings=None + ) + OKTestClient(total_retry_interval=request_timeout, tracing_config=tracing_config) class FailWrongAnnotationTestClient(BaseThinClient): async def public_method_wrong_annotation(self) -> None: @@ -173,7 +193,8 @@ async def public_method_wrong_annotation(self) -> None: with pytest.raises(AssertionError, match="should return an instance"): FailWrongAnnotationTestClient( - total_retry_interval=request_timeout, tracing_settings=None + total_retry_interval=request_timeout, + tracing_config=tracing_config, ) class FailNoAnnotationTestClient(BaseThinClient): @@ -182,7 +203,8 @@ async def public_method_no_annotation(self): with pytest.raises(AssertionError, match="should return an instance"): FailNoAnnotationTestClient( - total_retry_interval=request_timeout, tracing_settings=None + total_retry_interval=request_timeout, + tracing_config=tracing_config, ) @@ -207,8 +229,11 @@ async def get_wrong_state(self) -> Response: respx_mock.get(url_get_200_ok).mock(return_value=Response(codes.OK)) respx_mock.get(get_wrong_state).mock(return_value=Response(codes.OK)) + tracing_config = TracingConfig.create( + service_name="test-client", tracing_settings=None + ) test_client = ATestClient( - total_retry_interval=request_timeout, tracing_settings=None + total_retry_interval=request_timeout, tracing_config=tracing_config ) # OK @@ -230,8 +255,11 @@ async def test_retry_timeout_overwrite( request_timeout: int, caplog_info_level: pytest.LogCaptureFixture, ) -> None: + tracing_config = TracingConfig.create( + service_name="test-client", tracing_settings=None + ) client = FakeThickClient( - total_retry_interval=request_timeout, tracing_settings=None + total_retry_interval=request_timeout, tracing_config=tracing_config ) caplog_info_level.clear() diff --git a/packages/service-library/tests/fastapi/test_tracing.py b/packages/service-library/tests/fastapi/test_tracing.py index 148f4e43776d..32b199cee263 100644 --- a/packages/service-library/tests/fastapi/test_tracing.py +++ b/packages/service-library/tests/fastapi/test_tracing.py @@ -24,7 +24,8 @@ from servicelib.tracing import ( _OSPARC_TRACE_ID_HEADER, _PROFILE_ATTRIBUTE_NAME, - with_profiled_span, + TracingConfig, + profiled_span, ) from settings_library.tracing import TracingSettings @@ -55,18 +56,26 @@ def set_and_clean_settings_env_vars( monkeypatch.setenv( "TRACING_OPENTELEMETRY_COLLECTOR_PORT", f"{tracing_settings_in[1]}" ) + sampling_probability_mocked = False + if tracing_settings_in[2]: + sampling_probability_mocked = True + monkeypatch.setenv( + "TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY", f"{tracing_settings_in[2]}" + ) yield if endpoint_mocked: monkeypatch.delenv("TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT") if port_mocked: monkeypatch.delenv("TRACING_OPENTELEMETRY_COLLECTOR_PORT") + if sampling_probability_mocked: + monkeypatch.delenv("TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY") @pytest.mark.parametrize( "tracing_settings_in", [ - ("http://opentelemetry-collector", 4318), - ("http://opentelemetry-collector", "4318"), + ("http://opentelemetry-collector", 4318, 1.0), + ("http://opentelemetry-collector", "4318", 1.0), ], indirect=True, ) @@ -77,13 +86,14 @@ async def test_valid_tracing_settings( tracing_settings_in: Callable[[], dict[str, Any]], ): tracing_settings = TracingSettings() + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name="Mock-Openetlemetry-Pytest" + ) async for _ in get_tracing_instrumentation_lifespan( - tracing_settings=tracing_settings, - service_name="Mock-Openetlemetry-Pytest", + tracing_config=tracing_config, )(app=mocked_app): async for _ in get_tracing_instrumentation_lifespan( - tracing_settings=tracing_settings, - service_name="Mock-Openetlemetry-Pytest", + tracing_config=tracing_config, )(app=mocked_app): pass @@ -91,15 +101,16 @@ async def test_valid_tracing_settings( @pytest.mark.parametrize( "tracing_settings_in", [ - ("http://opentelemetry-collector", 80), - ("http://opentelemetry-collector", 1238712936), - ("opentelemetry-collector", 4318), - ("httsdasp://ot@##el-collector", 4318), - (" !@#$%^&*()[]{};:,<>?\\|`~+=/'\"", 4318), + ("http://opentelemetry-collector", 80, 0.5), + ("http://opentelemetry-collector", 1238712936, 0.5), + ("opentelemetry-collector", 4318, 0.5), + ("httsdasp://ot@##el-collector", 4318, 0.5), + (" !@#$%^&*()[]{};:,<>?\\|`~+=/'\"", 4318, 0.5), # The following exceeds max DNS name length ( "".join(random.choice(string.ascii_letters) for _ in range(300)), "1238712936", + 0.5, ), # noqa: S311 ], indirect=True, @@ -113,9 +124,11 @@ async def test_invalid_tracing_settings( app = mocked_app with pytest.raises((BaseException, ValidationError, TypeError)): # noqa: PT012 tracing_settings = TracingSettings() + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name="Mock-Openetlemetry-Pytest" + ) async for _ in get_tracing_instrumentation_lifespan( - tracing_settings=tracing_settings, - service_name="Mock-Openetlemetry-Pytest", + tracing_config=tracing_config, )(app=app): pass @@ -143,14 +156,14 @@ def manage_package(request): "tracing_settings_in, manage_package", [ ( - ("http://opentelemetry-collector", 4318), + ("http://opentelemetry-collector", 4318, 1.0), ( "opentelemetry-instrumentation-botocore", "opentelemetry.instrumentation.botocore", ), ), ( - ("http://opentelemetry-collector", "4318"), + ("http://opentelemetry-collector", "4318", 1.0), ( "opentelemetry-instrumentation-aiopg", "opentelemetry.instrumentation.aiopg", @@ -169,14 +182,15 @@ async def test_tracing_setup_package_detection( package_name = manage_package importlib.import_module(package_name) tracing_settings = TracingSettings() + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name="Mock-Openetlemetry-Pytest" + ) async for _ in get_tracing_instrumentation_lifespan( - tracing_settings=tracing_settings, - service_name="Mock-Openetlemetry-Pytest", + tracing_config=tracing_config, )(app=mocked_app): # idempotency check async for _ in get_tracing_instrumentation_lifespan( - tracing_settings=tracing_settings, - service_name="Mock-Openetlemetry-Pytest", + tracing_config=tracing_config, )(app=mocked_app): pass @@ -184,7 +198,7 @@ async def test_tracing_setup_package_detection( @pytest.mark.parametrize( "tracing_settings_in", [ - ("http://opentelemetry-collector", 4318), + ("http://opentelemetry-collector", 4318, 1.0), ], indirect=True, ) @@ -203,6 +217,9 @@ async def test_trace_id_in_response_header( server_response: PlainTextResponse | HTTPException, ) -> None: tracing_settings = TracingSettings() + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name="Mock-Openetlemetry-Pytest" + ) handler_data = dict() @@ -218,10 +235,11 @@ async def handler(handler_data: dict): mocked_app.get("/")(partial(handler, handler_data)) async for _ in get_tracing_instrumentation_lifespan( - tracing_settings=tracing_settings, - service_name="Mock-OpenTelemetry-Pytest", + tracing_config=tracing_config, )(app=mocked_app): - initialize_fastapi_app_tracing(mocked_app, add_response_trace_id_header=True) + initialize_fastapi_app_tracing( + mocked_app, tracing_config=tracing_config, add_response_trace_id_header=True + ) client = TestClient(mocked_app) response = client.get("/") assert _OSPARC_TRACE_ID_HEADER in response.headers @@ -233,7 +251,7 @@ async def handler(handler_data: dict): @pytest.mark.parametrize( "tracing_settings_in", [ - ("http://opentelemetry-collector", 4318), + ("http://opentelemetry-collector", 4318, 1.0), ], indirect=True, ) @@ -252,26 +270,30 @@ async def test_with_profile_span( server_response: PlainTextResponse | HTTPException, ): tracing_settings = TracingSettings() + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name="Mock-Openetlemetry-Pytest" + ) handler_data = dict() - @with_profiled_span async def handler(handler_data: dict): - current_span = trace.get_current_span() - handler_data[_OSPARC_TRACE_ID_HEADER] = format( - current_span.get_span_context().trace_id, "032x" - ) - if isinstance(server_response, HTTPException): - raise server_response - return server_response + with profiled_span(tracing_config=tracing_config, span_name="my favorite span"): + current_span = trace.get_current_span() + handler_data[_OSPARC_TRACE_ID_HEADER] = format( + current_span.get_span_context().trace_id, "032x" + ) + if isinstance(server_response, HTTPException): + raise server_response + return server_response mocked_app.get("/")(partial(handler, handler_data)) async for _ in get_tracing_instrumentation_lifespan( - tracing_settings=tracing_settings, - service_name="Mock-OpenTelemetry-Pytest", + tracing_config=tracing_config, )(app=mocked_app): - initialize_fastapi_app_tracing(mocked_app, add_response_trace_id_header=True) + initialize_fastapi_app_tracing( + mocked_app, tracing_config=tracing_config, add_response_trace_id_header=True + ) client = TestClient(mocked_app) _ = client.get("/") trace_id = handler_data.get(_OSPARC_TRACE_ID_HEADER) @@ -284,3 +306,61 @@ async def handler(handler_data: dict): for span in spans if span.context is not None and span.attributes is not None ) + + +@pytest.mark.parametrize( + "tracing_settings_in", + [ + ("http://opentelemetry-collector", 4318, 0.05), + ], + indirect=True, +) +async def test_TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY_effective( + mock_otel_collector: InMemorySpanExporter, + mocked_app: FastAPI, + set_and_clean_settings_env_vars: Callable[[], None], + tracing_settings_in: Callable[[], dict[str, Any]], +): + """ + This test checks that the TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY setting in TracingSettings + is effective by sending 1000 requests and verifying that the number of collected traces + is close to 0.05 * 1000 (with some tolerance). + """ + n_requests = 1000 + tolerance_probability = 0.5 + + tracing_settings = TracingSettings() + tracing_config = TracingConfig.create( + tracing_settings=tracing_settings, service_name="Mock-OpenTelemetry-Pytest" + ) + + async def handler(): + return PlainTextResponse("ok") + + mocked_app.get("/")(handler) + + async for _ in get_tracing_instrumentation_lifespan( + tracing_config=tracing_config, + )(app=mocked_app): + initialize_fastapi_app_tracing( + mocked_app, tracing_config=tracing_config, add_response_trace_id_header=True + ) + client = TestClient(mocked_app) + for _ in range(n_requests): + client.get("/") + trace_ids = { + span.context.trace_id + for span in mock_otel_collector.get_finished_spans() + if span.context is not None + } + n_traces = len(trace_ids) + expected_num_traces = int( + tracing_settings.TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY * n_requests + ) + # Allow a 50% tolerance due to randomness + tolerance = int(tolerance_probability * expected_num_traces) + assert ( + expected_num_traces - tolerance + <= n_traces + <= expected_num_traces + tolerance + ), f"Expected roughly {expected_num_traces} distinct trace ids, got {n_traces}" diff --git a/packages/service-library/tests/test_logging_utils.py b/packages/service-library/tests/test_logging_utils.py index 63f0d69c7883..80d6d5d0249d 100644 --- a/packages/service-library/tests/test_logging_utils.py +++ b/packages/service-library/tests/test_logging_utils.py @@ -25,6 +25,7 @@ log_exceptions, set_parent_module_log_level, ) +from servicelib.tracing import TracingConfig from tenacity import ( retry, retry_if_exception_type, @@ -439,10 +440,19 @@ def test_set_parent_module_log_level_(caplog: pytest.LogCaptureFixture): assert "child warning" in caplog.text +@pytest.fixture +def tracing_config() -> TracingConfig: + return TracingConfig.create( + service_name="test-service", + tracing_settings=None, # disable tracing in tests + ) + + @pytest.mark.parametrize("log_format_local_dev_enabled", [True, False]) def test_setup_async_loggers_basic( caplog: pytest.LogCaptureFixture, log_format_local_dev_enabled: bool, + tracing_config: TracingConfig, ): """Test basic async logging setup without filters.""" caplog.clear() @@ -451,7 +461,7 @@ def test_setup_async_loggers_basic( with async_loggers( log_format_local_dev_enabled=log_format_local_dev_enabled, logger_filter_mapping={}, # No filters for this test - tracing_settings=None, # No tracing for this test + tracing_config=tracing_config, # No tracing for this test log_base_level=logging.INFO, # Set base log level noisy_loggers=(), # No noisy loggers for this test ): @@ -463,6 +473,7 @@ def test_setup_async_loggers_basic( def test_setup_async_loggers_with_filters( caplog: pytest.LogCaptureFixture, + tracing_config: TracingConfig, ): caplog.clear() caplog.set_level(logging.INFO) @@ -475,7 +486,7 @@ def test_setup_async_loggers_with_filters( with async_loggers( log_format_local_dev_enabled=True, logger_filter_mapping=filter_mapping, - tracing_settings=None, # No tracing for this test + tracing_config=tracing_config, # no tracing in this test log_base_level=logging.INFO, # Set base log level noisy_loggers=(), # No noisy loggers for this test ): @@ -502,6 +513,7 @@ def test_setup_async_loggers_with_filters( def test_setup_async_loggers_with_tracing_settings( caplog: pytest.LogCaptureFixture, + tracing_config: TracingConfig, ): """Test async logging setup with tracing settings.""" caplog.clear() @@ -512,7 +524,7 @@ def test_setup_async_loggers_with_tracing_settings( with async_loggers( log_format_local_dev_enabled=False, logger_filter_mapping={}, # No filters for this test - tracing_settings=None, + tracing_config=tracing_config, log_base_level=logging.INFO, # Set base log level noisy_loggers=(), # No noisy loggers for this test ): @@ -524,6 +536,7 @@ def test_setup_async_loggers_with_tracing_settings( def test_setup_async_loggers_context_manager_cleanup( caplog: pytest.LogCaptureFixture, + tracing_config: TracingConfig, ): """Test that async logging context manager properly cleans up.""" caplog.clear() @@ -534,7 +547,7 @@ def test_setup_async_loggers_context_manager_cleanup( with async_loggers( log_format_local_dev_enabled=True, logger_filter_mapping={}, - tracing_settings=None, + tracing_config=tracing_config, log_base_level=logging.INFO, # Set base log level noisy_loggers=(), # No noisy loggers for this test ): @@ -546,6 +559,7 @@ def test_setup_async_loggers_context_manager_cleanup( def test_setup_async_loggers_exception_handling( caplog: pytest.LogCaptureFixture, + tracing_config: TracingConfig, ): """Test that async logging handles exceptions gracefully.""" caplog.clear() @@ -560,7 +574,7 @@ def _raise_test_exception(): with async_loggers( log_format_local_dev_enabled=True, logger_filter_mapping={}, - tracing_settings=None, + tracing_config=tracing_config, log_base_level=logging.INFO, # Set base log level noisy_loggers=(), # No noisy loggers for this test ): diff --git a/packages/settings-library/src/settings_library/tracing.py b/packages/settings-library/src/settings_library/tracing.py index c8263dd8be24..adafd7883733 100644 --- a/packages/settings-library/src/settings_library/tracing.py +++ b/packages/settings-library/src/settings_library/tracing.py @@ -15,3 +15,7 @@ class TracingSettings(BaseCustomSettings): TRACING_OPENTELEMETRY_COLLECTOR_PORT: Annotated[ RegisteredPortInt, Field(description="Opentelemetry compatible collector port") ] + TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY: Annotated[ + float, + Field(description="Probability of sampling traces (0.0 - 1.0)", ge=0.0, le=1.0), + ] diff --git a/services/agent/src/simcore_service_agent/core/application.py b/services/agent/src/simcore_service_agent/core/application.py index e7972c1042a1..f6f06b6bf150 100644 --- a/services/agent/src/simcore_service_agent/core/application.py +++ b/services/agent/src/simcore_service_agent/core/application.py @@ -7,9 +7,11 @@ override_fastapi_openapi_method, ) from servicelib.fastapi.tracing import ( + get_tracing_config, initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import ( API_VTAG, @@ -30,13 +32,20 @@ _logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings | None = None) -> FastAPI: +def create_app( + settings: ApplicationSettings | None = None, + tracing_config: TracingConfig | None = None, +) -> FastAPI: if settings is None: settings = ApplicationSettings.create_from_envs() _logger.info( "Application settings: %s", json_dumps(settings, indent=2, sort_keys=True), ) + if tracing_config is None: + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=settings.AGENT_TRACING + ) assert settings.SC_BOOT_MODE # nosec app = FastAPI( @@ -49,9 +58,10 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: ) override_fastapi_openapi_method(app) app.state.settings = settings + app.state.tracing_config = tracing_config if settings.AGENT_TRACING: - setup_tracing(app, settings.AGENT_TRACING, APP_NAME) + setup_tracing(app, get_tracing_config(app)) setup_instrumentation(app) @@ -62,7 +72,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: setup_rpc_api_routes(app) if settings.AGENT_TRACING: - initialize_fastapi_app_tracing(app) + initialize_fastapi_app_tracing(app, tracing_config=get_tracing_config(app)) async def _on_startup() -> None: print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 diff --git a/services/agent/src/simcore_service_agent/main.py b/services/agent/src/simcore_service_agent/main.py index 1af4eb695c24..5d51293c64ba 100644 --- a/services/agent/src/simcore_service_agent/main.py +++ b/services/agent/src/simcore_service_agent/main.py @@ -4,9 +4,12 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event +from servicelib.tracing import TracingConfig from simcore_service_agent.core.application import create_app from simcore_service_agent.core.settings import ApplicationSettings +from ._meta import APP_NAME + _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -19,10 +22,13 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.AGENT_TRACING + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.AGENT_VOLUMES_LOG_FILTER_MAPPING, - tracing_settings=app_settings.AGENT_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -31,6 +37,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/api-server/src/simcore_service_api_server/celery_worker/worker_main.py b/services/api-server/src/simcore_service_api_server/celery_worker/worker_main.py index 82881b6af698..c9e99cda269f 100644 --- a/services/api-server/src/simcore_service_api_server/celery_worker/worker_main.py +++ b/services/api-server/src/simcore_service_api_server/celery_worker/worker_main.py @@ -8,6 +8,7 @@ ) from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.logging_utils import setup_loggers +from servicelib.tracing import TracingConfig from ..core.application import create_app from ..core.settings import ApplicationSettings @@ -16,11 +17,15 @@ def get_app(): _settings = ApplicationSettings.create_from_envs() - + _tracing_settings = _settings.API_SERVER_TRACING + _tracing_config = TracingConfig.create( + tracing_settings=_tracing_settings, + service_name="api-server-celery-worker", + ) setup_loggers( log_format_local_dev_enabled=_settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_settings.API_SERVER_LOG_FILTER_MAPPING, - tracing_settings=_settings.API_SERVER_TRACING, + tracing_config=_tracing_config, log_base_level=_settings.log_level, noisy_loggers=None, ) diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 32a5f6df6415..c38e03470360 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -10,6 +10,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .. import exceptions from .._meta import API_VERSION, API_VTAG, APP_NAME @@ -49,14 +50,23 @@ def _label_title_and_version(settings: ApplicationSettings, title: str, version: return title, version -def create_app(settings: ApplicationSettings | None = None) -> FastAPI: +def create_app( + settings: ApplicationSettings | None = None, + tracing_config: TracingConfig | None = None, +) -> FastAPI: if settings is None: settings = ApplicationSettings.create_from_envs() _logger.info( "Application settings: %s", json_dumps(settings, indent=2, sort_keys=True), ) + if tracing_config is None: + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=settings.API_SERVER_TRACING + ) + assert settings # nosec + assert tracing_config # nosec # Labeling title = "osparc.io public API" @@ -80,9 +90,10 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: add_pagination(app) app.state.settings = settings + app.state.tracing_config = tracing_config if settings.API_SERVER_TRACING: - setup_tracing(app, settings.API_SERVER_TRACING, APP_NAME) + setup_tracing(app, tracing_config) if settings.API_SERVER_POSTGRES: setup_postgres(app) @@ -96,7 +107,11 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: setup_prometheus_instrumentation(app) if settings.API_SERVER_TRACING: - initialize_fastapi_app_tracing(app, add_response_trace_id_header=True) + initialize_fastapi_app_tracing( + app, + tracing_config=tracing_config, + add_response_trace_id_header=True, + ) if settings.API_SERVER_WEBSERVER: webserver.setup( diff --git a/services/api-server/src/simcore_service_api_server/main.py b/services/api-server/src/simcore_service_api_server/main.py index 6b8ccc1783f1..ace337b472e1 100644 --- a/services/api-server/src/simcore_service_api_server/main.py +++ b/services/api-server/src/simcore_service_api_server/main.py @@ -6,9 +6,12 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event +from servicelib.tracing import TracingConfig from simcore_service_api_server.core.application import create_app from simcore_service_api_server.core.settings import ApplicationSettings +from ._meta import APP_NAME + _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -21,10 +24,13 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.API_SERVER_TRACING + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.API_SERVER_LOG_FILTER_MAPPING, - tracing_settings=app_settings.API_SERVER_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -33,6 +39,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/api-server/src/simcore_service_api_server/utils/client_base.py b/services/api-server/src/simcore_service_api_server/utils/client_base.py index 173090b8965d..6aa567fd43ec 100644 --- a/services/api-server/src/simcore_service_api_server/utils/client_base.py +++ b/services/api-server/src/simcore_service_api_server/utils/client_base.py @@ -5,7 +5,7 @@ import httpx from fastapi import FastAPI from httpx import AsyncClient, Timeout -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from settings_library.tracing import TracingSettings from .app_data import AppDataMixin @@ -60,7 +60,10 @@ def setup_client_instance( timeout=Timeout(_DEFAULT_BASE_SERVICE_CLIENT_API_TIMEOUT_SECONDS), ) if tracing_settings: - setup_httpx_client_tracing(client) + setup_httpx_client_tracing( + client, + tracing_config=get_tracing_config(app), + ) # events def _create_instance() -> None: diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/application.py b/services/autoscaling/src/simcore_service_autoscaling/core/application.py index ba833512565e..27ec57257d6f 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/application.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/application.py @@ -5,6 +5,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import ( API_VERSION, @@ -34,7 +35,7 @@ logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings) -> FastAPI: +def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) -> FastAPI: app = FastAPI( debug=settings.AUTOSCALING_DEBUG, title=APP_NAME, @@ -46,11 +47,12 @@ def create_app(settings: ApplicationSettings) -> FastAPI: ) # STATE app.state.settings = settings + app.state.tracing_config = tracing_config assert app.state.settings.API_VERSION == API_VERSION # nosec # PLUGINS SETUP - if app.state.settings.AUTOSCALING_TRACING: - setup_tracing(app, app.state.settings.AUTOSCALING_TRACING, APP_NAME) + if tracing_config.tracing_enabled: + setup_tracing(app, tracing_config=tracing_config) setup_instrumentation(app) setup_api_routes(app) @@ -60,8 +62,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_ssm(app) setup_redis(app) - if app.state.settings.AUTOSCALING_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) setup_auto_scaler_background_task(app) setup_warm_buffer_machines_pool_task(app) diff --git a/services/autoscaling/src/simcore_service_autoscaling/main.py b/services/autoscaling/src/simcore_service_autoscaling/main.py index b1f7055d75df..48b5fe82b450 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/main.py +++ b/services/autoscaling/src/simcore_service_autoscaling/main.py @@ -5,10 +5,13 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI +from servicelib import tracing from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event from simcore_service_autoscaling.core.application import create_app from simcore_service_autoscaling.core.settings import ApplicationSettings +from ._meta import APP_NAME + _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -22,10 +25,13 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = tracing.TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.AUTOSCALING_TRACING + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.AUTOSCALING_LOG_FILTER_MAPPING, - tracing_settings=app_settings.AUTOSCALING_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -34,6 +40,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index 8af62e808c8f..0b4482dd066b 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -63,6 +63,7 @@ delenvs_from_dict, setenvs_from_dict, ) +from servicelib.tracing import TracingConfig from settings_library.rabbit import RabbitSettings from settings_library.ssm import SSMSettings from simcore_service_autoscaling.constants import PRE_PULLED_IMAGES_EC2_TAG_KEY @@ -415,7 +416,11 @@ def enabled_rabbitmq( @pytest.fixture async def initialized_app(app_environment: EnvVarsDict) -> AsyncIterator[FastAPI]: settings = ApplicationSettings.create_from_envs() - app = create_app(settings) + tracing_config = TracingConfig.create( + service_name=settings.APP_NAME, + tracing_settings=None, # disable tracing in tests + ) + app = create_app(settings, tracing_config=tracing_config) # NOTE: the timeout is sometime too small for CI machines, and even larger machines async with LifespanManager( app, startup_timeout=_LIFESPAN_TIMEOUT, shutdown_timeout=_LIFESPAN_TIMEOUT diff --git a/services/catalog/src/simcore_service_catalog/clients/director.py b/services/catalog/src/simcore_service_catalog/clients/director.py index 367f9cb0e2e4..f3041ad2a5c0 100644 --- a/services/catalog/src/simcore_service_catalog/clients/director.py +++ b/services/catalog/src/simcore_service_catalog/clients/director.py @@ -15,7 +15,7 @@ from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from pydantic import NonNegativeInt, TypeAdapter -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from servicelib.logging_utils import log_catch, log_context from starlette import status from tenacity.asyncio import AsyncRetrying @@ -146,7 +146,10 @@ def __init__(self, base_url: str, app: FastAPI): timeout=settings.CATALOG_CLIENT_REQUEST.HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT, ) if settings.CATALOG_TRACING: - setup_httpx_client_tracing(self.client) + setup_httpx_client_tracing( + self.client, + tracing_config=get_tracing_config(app=app), + ) assert settings.CATALOG_DIRECTOR # nosec self.vtag = settings.CATALOG_DIRECTOR.DIRECTOR_VTAG diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index 1d5797418624..9ca99db23656 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -14,12 +14,12 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from starlette.middleware.base import BaseHTTPMiddleware from .._meta import ( API_VERSION, API_VTAG, - APP_NAME, PROJECT_NAME, SUMMARY, ) @@ -32,6 +32,7 @@ def create_app( *, + tracing_config: TracingConfig, settings: ApplicationSettings | None = None, logging_lifespan: Lifespan | None = None, ) -> FastAPI: @@ -57,15 +58,16 @@ def create_app( # STATE app.state.settings = settings + app.state.tracing_config = tracing_config # MIDDLEWARES - if settings.CATALOG_TRACING: - setup_tracing(app, settings.CATALOG_TRACING, APP_NAME) + if tracing_config.tracing_enabled: + setup_tracing(app, tracing_config=tracing_config) if settings.CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) - if settings.CATALOG_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) if settings.SC_BOOT_MODE != BootModeEnum.PRODUCTION: # middleware to time requests (ONLY for development) diff --git a/services/catalog/src/simcore_service_catalog/main.py b/services/catalog/src/simcore_service_catalog/main.py index 60f5da962d47..f899adda2bff 100644 --- a/services/catalog/src/simcore_service_catalog/main.py +++ b/services/catalog/src/simcore_service_catalog/main.py @@ -6,6 +6,7 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_lifespan +from servicelib.tracing import TracingConfig from simcore_service_catalog.core.application import create_app from simcore_service_catalog.core.settings import ApplicationSettings @@ -24,10 +25,14 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + tracing_settings=app_settings.CATALOG_TRACING, + service_name="catalog", + ) logging_lifespan = create_logging_lifespan( log_format_local_dev_enabled=app_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.CATALOG_LOG_FILTER_MAPPING, - tracing_settings=app_settings.CATALOG_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -37,4 +42,8 @@ def app_factory() -> FastAPI: json_dumps(app_settings, indent=2, sort_keys=True), ) - return create_app(settings=app_settings, logging_lifespan=logging_lifespan) + return create_app( + settings=app_settings, + tracing_config=tracing_config, + logging_lifespan=logging_lifespan, + ) diff --git a/services/catalog/tests/unit/conftest.py b/services/catalog/tests/unit/conftest.py index 0088fa436a0b..a431e1955f58 100644 --- a/services/catalog/tests/unit/conftest.py +++ b/services/catalog/tests/unit/conftest.py @@ -32,6 +32,8 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.tracing import TracingConfig +from simcore_service_catalog._meta import APP_NAME from simcore_service_catalog.core.application import create_app from simcore_service_catalog.core.settings import ApplicationSettings @@ -145,7 +147,10 @@ async def app( # create instance assert app_environment - app_under_test = create_app() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=None # disable tracing in tests + ) + app_under_test = create_app(tracing_config=tracing_config) assert spy_app.on_startup.call_count == 0 assert spy_app.on_shutdown.call_count == 0 @@ -172,7 +177,10 @@ def client( # create instance assert app_environment - app_under_test = create_app() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=None # disable tracing in tests + ) + app_under_test = create_app(tracing_config=tracing_config) assert ( spy_app.on_startup.call_count == 0 diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py index 1c8c7e448c0a..a4c0a98d9871 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py @@ -8,6 +8,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import ( API_VERSION, @@ -29,7 +30,7 @@ _logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings) -> FastAPI: +def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) -> FastAPI: _logger.info("app settings: %s", settings.model_dump_json(indent=1)) app = FastAPI( @@ -43,13 +44,13 @@ def create_app(settings: ApplicationSettings) -> FastAPI: ) # STATE app.state.settings = settings + app.state.tracing_config = tracing_config assert app.state.settings.API_VERSION == API_VERSION # nosec - if app.state.settings.CLUSTERS_KEEPER_TRACING: + if tracing_config.tracing_enabled: setup_tracing( app, - app.state.settings.CLUSTERS_KEEPER_TRACING, - APP_NAME, + tracing_config, ) if app.state.settings.CLUSTERS_KEEPER_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) @@ -63,8 +64,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_redis(app) setup_clusters_management(app) - if app.state.settings.CLUSTERS_KEEPER_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) # ERROR HANDLERS # EVENTS diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py index d4e4bdf99ee4..88f891c8f0d5 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py @@ -6,9 +6,12 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event +from servicelib.tracing import TracingConfig from simcore_service_clusters_keeper.core.application import create_app from simcore_service_clusters_keeper.core.settings import ApplicationSettings +from ._meta import APP_NAME + _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -22,10 +25,13 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.CLUSTERS_KEEPER_TRACING + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.CLUSTERS_KEEPER_LOG_FILTER_MAPPING, - tracing_settings=app_settings.CLUSTERS_KEEPER_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -34,6 +40,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index f6c4283f77f9..e5ec5afd891c 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -26,9 +26,11 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.tracing import TracingConfig from settings_library.ec2 import EC2Settings from settings_library.rabbit import RabbitSettings from settings_library.ssm import SSMSettings +from simcore_service_clusters_keeper._meta import APP_NAME from simcore_service_clusters_keeper.core.application import create_app from simcore_service_clusters_keeper.core.settings import ( CLUSTERS_KEEPER_ENV_PREFIX, @@ -253,7 +255,10 @@ async def initialized_app( app_environment: EnvVarsDict, is_pdb_enabled: bool ) -> AsyncIterator[FastAPI]: settings = ApplicationSettings.create_from_envs() - app = create_app(settings) + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=None # disable tracing in tests + ) + app = create_app(settings, tracing_config=tracing_config) async with LifespanManager(app, shutdown_timeout=None if is_pdb_enabled else 20): yield app diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py index 0813da017416..9a17363136bd 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/scheduler.py @@ -16,7 +16,6 @@ async def dask_setup(scheduler: distributed.Scheduler) -> None: """This is a special function recognized by dask when starting with flag --preload""" assert scheduler # nosec - settings = ApplicationSettings.create_from_envs() setup_app_logging(settings) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/logs.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/logs.py index 76eaef674313..e5da1b40c1ef 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/logs.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/utils/logs.py @@ -1,7 +1,9 @@ from typing import Final from servicelib.logging_utils import setup_loggers +from servicelib.tracing import TracingConfig +from .._meta import PROJECT_NAME from ..settings import ApplicationSettings _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -12,10 +14,13 @@ def setup_app_logging(settings: ApplicationSettings) -> None: + tracing_config = TracingConfig.create( + service_name=PROJECT_NAME, tracing_settings=None # no tracing for dask sidecar + ) setup_loggers( log_format_local_dev_enabled=settings.DASK_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=settings.DASK_LOG_FILTER_MAPPING, - tracing_settings=None, # no tracing for dask sidecar + tracing_config=tracing_config, log_base_level=settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py index 397a686e7463..e861ef0ee7a0 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py @@ -14,6 +14,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from starlette.middleware.base import BaseHTTPMiddleware from .._meta import API_VERSION, API_VTAG, APP_NAME @@ -30,7 +31,7 @@ _logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings) -> FastAPI: +def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) -> FastAPI: app = FastAPI( debug=settings.SC_BOOT_MODE in [BootModeEnum.DEBUG, BootModeEnum.DEVELOPMENT, BootModeEnum.LOCAL], @@ -45,12 +46,12 @@ def create_app(settings: ApplicationSettings) -> FastAPI: add_pagination(app) app.state.settings = settings + app.state.tracing_config = tracing_config - if app.state.settings.DATCORE_ADAPTER_TRACING: + if tracing_config.tracing_enabled: setup_tracing( app, - app.state.settings.DATCORE_ADAPTER_TRACING, - APP_NAME, + tracing_config, ) if app.state.settings.DATCORE_ADAPTER_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) @@ -62,8 +63,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI: ) app.add_middleware(GZipMiddleware) - if app.state.settings.DATCORE_ADAPTER_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) # events app.add_event_handler("startup", on_startup) diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/main.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/main.py index 0a46179524df..a3a1a44c2273 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/main.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/main.py @@ -5,10 +5,13 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI +from servicelib import tracing from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event from simcore_service_datcore_adapter.core.application import create_app from simcore_service_datcore_adapter.core.settings import ApplicationSettings +from ._meta import APP_NAME + _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -20,10 +23,13 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = tracing.TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.DATCORE_ADAPTER_TRACING + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.DATCORE_ADAPTER_LOG_FILTER_MAPPING, - tracing_settings=app_settings.DATCORE_ADAPTER_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -32,6 +38,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/director-v2/src/simcore_service_director_v2/cli/_client.py b/services/director-v2/src/simcore_service_director_v2/cli/_client.py index 872c08f3b5f6..78bdfd2caa0f 100644 --- a/services/director-v2/src/simcore_service_director_v2/cli/_client.py +++ b/services/director-v2/src/simcore_service_director_v2/cli/_client.py @@ -5,16 +5,22 @@ expect_status, retry_on_errors, ) +from servicelib.tracing import TracingConfig + +from .._meta import APP_NAME class ThinDV2LocalhostClient(BaseThinClient): BASE_ADDRESS: str = "http://localhost:8000" # NOSONAR def __init__(self): + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=None + ) super().__init__( total_retry_interval=10, default_http_client_timeout=Timeout(5), - tracing_settings=None, + tracing_config=tracing_config, ) def _get_url(self, postfix: str) -> str: diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 9b68f4b09a42..fb3cfe4d25c2 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -13,9 +13,11 @@ ) from servicelib.fastapi.profiler import initialize_profiler from servicelib.fastapi.tracing import ( + get_tracing_config, initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY from ..api.entrypoints import api_router @@ -119,10 +121,13 @@ def create_base_app( if app_settings is None: app_settings = AppSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.DIRECTOR_V2_TRACING + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.DIRECTOR_V2_LOG_FILTER_MAPPING, - tracing_settings=app_settings.DIRECTOR_V2_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -147,6 +152,7 @@ def create_base_app( ) override_fastapi_openapi_method(app) app.state.settings = app_settings + app.state.tracing_config = tracing_config app.include_router(api_router) @@ -169,8 +175,8 @@ def create_app( # noqa: C901, PLR0912 substitutions.setup(app) - if settings.DIRECTOR_V2_TRACING: - setup_tracing(app, settings.DIRECTOR_V2_TRACING, APP_NAME) + if get_tracing_config(app).tracing_enabled: + setup_tracing(app, get_tracing_config(app)) if settings.DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED: instrumentation.setup(app) @@ -198,11 +204,11 @@ def create_app( # noqa: C901, PLR0912 db.setup(app, settings.POSTGRES) - if settings.DIRECTOR_V2_TRACING: - initialize_fastapi_app_tracing(app) + if get_tracing_config(app).tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=get_tracing_config(app)) if settings.DYNAMIC_SERVICES.DIRECTOR_V2_DYNAMIC_SERVICES_ENABLED: - dynamic_services.setup(app, tracing_settings=settings.DIRECTOR_V2_TRACING) + dynamic_services.setup(app) dynamic_scheduler_enabled = settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR and ( settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER diff --git a/services/director-v2/src/simcore_service_director_v2/modules/catalog.py b/services/director-v2/src/simcore_service_director_v2/modules/catalog.py index d9d4c3e61440..9923f151a819 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/catalog.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/catalog.py @@ -11,7 +11,7 @@ from models_library.services_resources import ServiceResourcesDict from models_library.users import UserID from pydantic import TypeAdapter -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from settings_library.catalog import CatalogSettings from settings_library.tracing import TracingSettings @@ -35,7 +35,10 @@ async def on_startup() -> None: timeout=app.state.settings.CLIENT_REQUEST.HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT, ) if tracing_settings: - setup_httpx_client_tracing(client=client) + setup_httpx_client_tracing( + client=client, + tracing_config=get_tracing_config(app), + ) CatalogClient.create( app, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py b/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py index 8b8d10468125..1c000d1606df 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/director_v0.py @@ -13,7 +13,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from servicelib.logging_utils import log_decorator from settings_library.director_v0 import DirectorV0Settings from settings_library.tracing import TracingSettings @@ -40,7 +40,10 @@ def on_startup() -> None: timeout=app.state.settings.CLIENT_REQUEST.HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT, ) if tracing_settings: - setup_httpx_client_tracing(client=client) + setup_httpx_client_tracing( + client=client, + tracing_config=get_tracing_config(app), + ) DirectorV0Client.create( app, client=client, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_services.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_services.py index acbc08849a69..81e142f6ddd1 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_services.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_services.py @@ -1,28 +1,27 @@ -""" Module that takes care of communications with dynamic services v0 - - -""" +"""Module that takes care of communications with dynamic services v0""" import logging from dataclasses import dataclass import httpx from fastapi import FastAPI -from servicelib.fastapi.tracing import setup_httpx_client_tracing -from settings_library.tracing import TracingSettings +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from ..utils.client_decorators import handle_errors, handle_retry logger = logging.getLogger(__name__) -def setup(app: FastAPI, tracing_settings: TracingSettings | None) -> None: +def setup(app: FastAPI) -> None: def on_startup() -> None: client = httpx.AsyncClient( timeout=app.state.settings.CLIENT_REQUEST.HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT ) - if tracing_settings: - setup_httpx_client_tracing(client=client) + if get_tracing_config(app).tracing_enabled: + setup_httpx_client_tracing( + client=client, + tracing_config=get_tracing_config(app=app), + ) ServicesClient.create( app, client=client, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py index 3a3cc1c3118d..2abe3f23dd64 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_thin.py @@ -12,7 +12,7 @@ expect_status, retry_on_errors, ) -from settings_library.tracing import TracingSettings +from servicelib.fastapi.tracing import get_tracing_config from ....core.dynamic_services_settings.scheduler import ( DynamicServicesSchedulerSettings, @@ -32,9 +32,7 @@ def __init__(self, app: FastAPI): scheduler_settings: DynamicServicesSchedulerSettings = ( app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER ) - tracing_settings: TracingSettings | None = ( - app.state.settings.DIRECTOR_V2_TRACING - ) + tracing_config = get_tracing_config(app) # timeouts self._health_request_timeout = Timeout(1.0, connect=1.0) @@ -57,7 +55,7 @@ def __init__(self, app: FastAPI): scheduler_settings.DYNAMIC_SIDECAR_API_REQUEST_TIMEOUT, connect=scheduler_settings.DYNAMIC_SIDECAR_API_CONNECT_TIMEOUT, ), - tracing_settings=tracing_settings, + tracing_config=tracing_config, ) def _get_url( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py b/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py index 550b2eddfeff..51c68e990a3c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/resource_usage_tracker_client.py @@ -1,5 +1,4 @@ -""" Interface to communicate with the resource usage tracker -""" +"""Interface to communicate with the resource usage tracker""" import contextlib import logging @@ -23,7 +22,7 @@ ) from models_library.services import ServiceKey, ServiceVersion from models_library.wallets import WalletID -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from ..core.errors import PricingPlanUnitNotFoundError from ..core.settings import AppSettings @@ -37,12 +36,17 @@ class ResourceUsageTrackerClient: exit_stack: contextlib.AsyncExitStack @classmethod - def create(cls, settings: AppSettings) -> "ResourceUsageTrackerClient": + def create( + cls, app: FastAPI, settings: AppSettings + ) -> "ResourceUsageTrackerClient": client = httpx.AsyncClient( base_url=settings.DIRECTOR_V2_RESOURCE_USAGE_TRACKER.api_base_url, ) if settings.DIRECTOR_V2_TRACING: - setup_httpx_client_tracing(client=client) + setup_httpx_client_tracing( + client=client, + tracing_config=get_tracing_config(app), + ) exit_stack = contextlib.AsyncExitStack() return cls(client=client, exit_stack=exit_stack) @@ -166,7 +170,7 @@ def setup(cls, app: FastAPI): assert not hasattr(app.state, "resource_usage_api") # nosec app_settings: AppSettings = app.state.settings - app.state.resource_usage_api = api = cls.create(app_settings) + app.state.resource_usage_api = api = cls.create(app, app_settings) async def on_startup(): await api.start() diff --git a/services/director-v2/src/simcore_service_director_v2/modules/storage.py b/services/director-v2/src/simcore_service_director_v2/modules/storage.py index 08e18de0aeb6..1729c2f824b3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/storage.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/storage.py @@ -1,14 +1,12 @@ -""" Module that takes care of communications with director v0 service +"""Module that takes care of communications with director v0 service""" - -""" import logging from dataclasses import dataclass import httpx from fastapi import FastAPI, HTTPException from models_library.users import UserID -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from servicelib.logging_utils import log_decorator from settings_library.s3 import S3Settings from settings_library.storage import StorageSettings @@ -40,7 +38,10 @@ def on_startup() -> None: timeout=app.state.settings.CLIENT_REQUEST.HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT, ) if tracing_settings: - setup_httpx_client_tracing(client=client) + setup_httpx_client_tracing( + client=client, + tracing_config=get_tracing_config(app), + ) StorageClient.create( app, client=client, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py index c748fc1cd1b0..d0cbea08727e 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_public.py @@ -1,23 +1,24 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +from collections.abc import AsyncIterable, Callable, Iterator from contextlib import contextmanager -from typing import Any, AsyncIterable, Callable, Iterator +from typing import Any from unittest.mock import AsyncMock -from models_library.api_schemas_dynamic_sidecar.containers import ( - ActivityInfoOrNone -) import pytest from common_library.json_serialization import json_dumps from faker import Faker from fastapi import FastAPI, status from httpx import HTTPError, Response +from models_library.api_schemas_dynamic_sidecar.containers import ActivityInfoOrNone from models_library.sidecar_volumes import VolumeCategory, VolumeStatus from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.fastapi.http_client_thin import ClientHttpError, UnexpectedStatusError +from servicelib.tracing import TracingConfig +from simcore_service_director_v2._meta import APP_NAME from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.modules.dynamic_sidecar.api_client._public import ( SidecarsClient, @@ -65,6 +66,10 @@ async def sidecars_client( ) -> AsyncIterable[SidecarsClient]: app = FastAPI() app.state.settings = AppSettings.create_from_envs() + app.state.tracing_config = TracingConfig.create( + service_name=APP_NAME, + tracing_settings=None, # disable tracing in tests + ) # WARNING: pytest gets confused with 'setup', use instead alias 'api_client_setup' await api_client_setup(app) @@ -369,7 +374,9 @@ async def test_get_service_activity( status_code=status.HTTP_200_OK, text=json_dumps(mock_dict) ), ) as client: - assert await client.get_service_activity(dynamic_sidecar_endpoint) == TypeAdapter(ActivityInfoOrNone).validate_python(mock_dict) + assert await client.get_service_activity( + dynamic_sidecar_endpoint + ) == TypeAdapter(ActivityInfoOrNone).validate_python(mock_dict) async def test_free_reserved_disk_space( diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py index 7e9d4f429a49..77dd294ecbaa 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_client_api_thin.py @@ -16,6 +16,8 @@ from respx import MockRouter, Route from respx.types import SideEffectTypes from servicelib.docker_constants import SUFFIX_EGRESS_PROXY_NAME +from servicelib.tracing import TracingConfig +from simcore_service_director_v2._meta import APP_NAME from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.modules.dynamic_sidecar.api_client._thin import ( ThinSidecarsClient, @@ -52,6 +54,11 @@ def mocked_app( app = FastAPI() app.state.settings = AppSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=APP_NAME, + tracing_settings=None, # disable tracing in tests + ) + app.state.tracing_config = tracing_config return app diff --git a/services/director/src/simcore_service_director/core/application.py b/services/director/src/simcore_service_director/core/application.py index fb14ce4e86cf..f44e6dd9e09a 100644 --- a/services/director/src/simcore_service_director/core/application.py +++ b/services/director/src/simcore_service_director/core/application.py @@ -8,6 +8,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import ( API_VERSION, @@ -24,7 +25,7 @@ _logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings) -> FastAPI: +def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) -> FastAPI: app = FastAPI( debug=settings.DIRECTOR_DEBUG, title=APP_NAME, @@ -36,11 +37,12 @@ def create_app(settings: ApplicationSettings) -> FastAPI: ) # STATE app.state.settings = settings + app.state.tracing_config = tracing_config assert app.state.settings.API_VERSION == API_VERSION # nosec # PLUGINS SETUP - if app.state.settings.DIRECTOR_TRACING: - setup_tracing(app, app.state.settings.DIRECTOR_TRACING, APP_NAME) + if tracing_config.tracing_enabled: + setup_tracing(app, tracing_config) setup_api_routes(app) @@ -50,12 +52,12 @@ def create_app(settings: ApplicationSettings) -> FastAPI: app, max_keepalive_connections=settings.DIRECTOR_REGISTRY_CLIENT_MAX_KEEPALIVE_CONNECTIONS, default_timeout=settings.DIRECTOR_REGISTRY_CLIENT_TIMEOUT, - tracing_settings=settings.DIRECTOR_TRACING, + tracing_config=tracing_config, ) setup_registry(app) - if app.state.settings.DIRECTOR_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) # ERROR HANDLERS set_app_default_http_error_handlers(app) diff --git a/services/director/src/simcore_service_director/main.py b/services/director/src/simcore_service_director/main.py index 5ad1c4b03d9a..e087c2f13e5a 100644 --- a/services/director/src/simcore_service_director/main.py +++ b/services/director/src/simcore_service_director/main.py @@ -6,6 +6,7 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event +from servicelib.tracing import TracingConfig from simcore_service_director.core.application import create_app from simcore_service_director.core.settings import ApplicationSettings @@ -20,10 +21,14 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=app_settings.APP_NAME, + tracing_settings=app_settings.DIRECTOR_TRACING, + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.DIRECTOR_LOG_FILTER_MAPPING, - tracing_settings=app_settings.DIRECTOR_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -32,6 +37,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/director/tests/unit/conftest.py b/services/director/tests/unit/conftest.py index 68c20076f76f..ad60821585bd 100644 --- a/services/director/tests/unit/conftest.py +++ b/services/director/tests/unit/conftest.py @@ -13,7 +13,9 @@ from fastapi import FastAPI from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict +from servicelib.tracing import TracingConfig from settings_library.docker_registry import RegistrySettings +from simcore_service_director._meta import APP_NAME from simcore_service_director.core.application import create_app from simcore_service_director.core.settings import ApplicationSettings @@ -163,7 +165,10 @@ def app_settings(app_environment: EnvVarsDict) -> ApplicationSettings: async def app( app_settings: ApplicationSettings, is_pdb_enabled: bool ) -> AsyncIterator[FastAPI]: - the_test_app = create_app(settings=app_settings) + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=None # disable tracing in tests + ) + the_test_app = create_app(settings=app_settings, tracing_config=tracing_config) async with LifespanManager( the_test_app, startup_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_STARTUP, diff --git a/services/director/tests/unit/test_core_settings.py b/services/director/tests/unit/test_core_settings.py index 122d6baea9af..ec9c2f4460cf 100644 --- a/services/director/tests/unit/test_core_settings.py +++ b/services/director/tests/unit/test_core_settings.py @@ -100,7 +100,6 @@ def test_docker_container_env_sample(monkeypatch: pytest.MonkeyPatch): SWARM_STACK_NAME=master-simcore TERM=xterm TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318 - TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=50 TRAEFIK_SIMCORE_ZONE=master_internal_simcore_stack VIRTUAL_ENV=/home/scu/.venv LOG_FORMAT_LOCAL_DEV_ENABLED=1 @@ -150,7 +149,6 @@ def test_docker_compose_environment_sample( "STORAGE_ENDPOINT": "master_storage:8080", "SWARM_STACK_NAME": "master-simcore", "TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT": "http://jaeger:4318", - "TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE": "50", "TRAEFIK_SIMCORE_ZONE": "master_internal_simcore_stack", }, ) diff --git a/services/docker-compose-ops.yml b/services/docker-compose-ops.yml index d6a336c9dacc..8da9002dbf40 100644 --- a/services/docker-compose-ops.yml +++ b/services/docker-compose-ops.yml @@ -117,7 +117,6 @@ services: - interactive_services_subnet environment: TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE: ${TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE} - TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE: ${TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE} TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT} volumes: minio_data: diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 44ef1e437fd7..b89d5b62e9f5 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -10,7 +10,7 @@ x-tracing-open-telemetry: &tracing_open_telemetry_environs TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE: ${TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} - TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE: ${TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE} + TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY: ${TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY} x-webserver-diagnostics: &webserver_diagnostics_environs DIAGNOSTICS_HEALTHCHECK_ENABLED: ${DIAGNOSTICS_HEALTHCHECK_ENABLED} @@ -1507,6 +1507,7 @@ services: - "--tracing.addinternals" - "--tracing.otlp=true" - "--tracing.otlp.http=true" + - "--tracing.sampleRate=${TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY}" healthcheck: # NOTE: this healthcheck to check if traefik is up and running must be run on the ping entrypoint defined in command! test: traefik healthcheck --ping --ping.entryPoint=ping --entryPoints.ping.address=:9082 diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py index a2617cec567e..6685558c6258 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py @@ -6,8 +6,9 @@ from servicelib.fastapi.openapi import override_fastapi_openapi_method from servicelib.fastapi.profiler import initialize_profiler from servicelib.fastapi.tracing import initialize_fastapi_app_tracing +from servicelib.tracing import TracingConfig -from .._meta import API_VERSION, API_VTAG, PROJECT_NAME, SUMMARY +from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY from ..api.frontend import initialize_frontend from ..api.rest.routes import initialize_rest_api from . import events @@ -17,8 +18,13 @@ def create_app( settings: ApplicationSettings | None = None, logging_lifespan: Lifespan | None = None, + tracing_config: TracingConfig | None = None, ) -> FastAPI: app_settings = settings or ApplicationSettings.create_from_envs() + app_tracing_config = tracing_config or TracingConfig.create( + tracing_settings=app_settings.DYNAMIC_SCHEDULER_TRACING, + service_name=APP_NAME, + ) app = FastAPI( title=f"{PROJECT_NAME} web API", @@ -30,13 +36,15 @@ def create_app( ), redoc_url=None, lifespan=events.create_app_lifespan( - settings=app_settings, logging_lifespan=logging_lifespan + tracing_config=app_tracing_config, + logging_lifespan=logging_lifespan, ), ) override_fastapi_openapi_method(app) # STATE app.state.settings = app_settings + app.state.tracing_config = app_tracing_config assert app.state.settings.API_VERSION == API_VERSION # nosec initialize_rest_api(app) @@ -49,7 +57,7 @@ def create_app( if app_settings.DYNAMIC_SCHEDULER_PROFILING: initialize_profiler(app) - if app_settings.DYNAMIC_SCHEDULER_TRACING: - initialize_fastapi_app_tracing(app) + if app_tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=app_tracing_config) return app diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py index d2b557401a42..b7afd1092240 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py @@ -15,8 +15,9 @@ create_postgres_database_input_state, ) from servicelib.fastapi.tracing import get_tracing_instrumentation_lifespan +from servicelib.tracing import TracingConfig -from .._meta import APP_FINISHED_BANNER_MSG, APP_NAME, APP_STARTED_BANNER_MSG +from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.routes import rpc_api_routes_lifespan from ..repository.events import repository_lifespan_manager from ..services.catalog import catalog_lifespan @@ -54,18 +55,18 @@ async def _settings_lifespan(app: FastAPI) -> AsyncIterator[State]: def create_app_lifespan( - settings: ApplicationSettings, logging_lifespan: Lifespan | None + tracing_config: TracingConfig, + logging_lifespan: Lifespan | None, ) -> LifespanManager: app_lifespan = LifespanManager() if logging_lifespan: app_lifespan.add(logging_lifespan) app_lifespan.add(_settings_lifespan) - if settings.DYNAMIC_SCHEDULER_TRACING: + if tracing_config.tracing_enabled: app_lifespan.add( get_tracing_instrumentation_lifespan( - tracing_settings=settings.DYNAMIC_SCHEDULER_TRACING, - service_name=APP_NAME, + tracing_config=tracing_config, ) ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py index bf1f22e27e61..a08411cc2c5c 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py @@ -8,9 +8,12 @@ from servicelib.fastapi.logging_lifespan import ( create_logging_lifespan, ) +from servicelib.tracing import TracingConfig from simcore_service_dynamic_scheduler.core.application import create_app from simcore_service_dynamic_scheduler.core.settings import ApplicationSettings +from ._meta import APP_NAME + _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -24,10 +27,14 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + tracing_settings=app_settings.DYNAMIC_SCHEDULER_TRACING, + service_name=APP_NAME, + ) logging_lifespan = create_logging_lifespan( log_format_local_dev_enabled=app_settings.DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING, - tracing_settings=app_settings.DYNAMIC_SCHEDULER_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -36,4 +43,8 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - return create_app(settings=app_settings, logging_lifespan=logging_lifespan) + return create_app( + settings=app_settings, + logging_lifespan=logging_lifespan, + tracing_config=tracing_config, + ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/catalog/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/catalog/_thin_client.py index 98cf8b7e0aeb..3d845d26946c 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/catalog/_thin_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/catalog/_thin_client.py @@ -11,6 +11,7 @@ expect_status, retry_on_errors, ) +from servicelib.fastapi.tracing import get_tracing_config from yarl import URL from ...core.settings import ApplicationSettings @@ -34,7 +35,7 @@ def __init__(self, app: FastAPI) -> None: "set_to_app_state", }, base_url=settings.DYNAMIC_SCHEDULER_CATALOG_SETTINGS.api_base_url, - tracing_settings=settings.DYNAMIC_SCHEDULER_TRACING, + tracing_config=get_tracing_config(app), ) @retry_on_errors() diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v0/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v0/_thin_client.py index d6d2cd272212..442b0205d612 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v0/_thin_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v0/_thin_client.py @@ -11,6 +11,7 @@ expect_status, retry_on_errors, ) +from servicelib.fastapi.tracing import get_tracing_config from yarl import URL from ...core.settings import ApplicationSettings @@ -36,7 +37,7 @@ def __init__(self, app: FastAPI) -> None: "set_to_app_state", }, base_url=settings.DYNAMIC_SCHEDULER_DIRECTOR_V0_SETTINGS.endpoint, - tracing_settings=settings.DYNAMIC_SCHEDULER_TRACING, + tracing_config=get_tracing_config(app), ) @retry_on_errors() diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py index c3afea528188..884972c5b878 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_thin_client.py @@ -24,6 +24,7 @@ expect_status, retry_on_errors, ) +from servicelib.fastapi.tracing import get_tracing_config from servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services import ( DEFAULT_LEGACY_WB_TO_DV2_HTTP_REQUESTS_TIMEOUT_S, ) @@ -41,7 +42,7 @@ def __init__(self, app: FastAPI) -> None: DEFAULT_LEGACY_WB_TO_DV2_HTTP_REQUESTS_TIMEOUT_S ), extra_allowed_method_names={"attach_lifespan_to"}, - tracing_settings=settings.DYNAMIC_SCHEDULER_TRACING, + tracing_config=get_tracing_config(app), ) @retry_on_errors() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index 0de264932ac1..599c8c810e6a 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -11,12 +11,14 @@ override_fastapi_openapi_method, ) from servicelib.fastapi.tracing import ( + get_tracing_config, initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from simcore_sdk.node_ports_common.exceptions import NodeNotFound -from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY, __version__ +from .._meta import API_VERSION, API_VTAG, APP_NAME, SUMMARY, __version__ from ..api.rest import get_main_router from ..api.rpc.routes import setup_rpc_api_routes from ..models.schemas.application_health import ApplicationHealth @@ -118,10 +120,13 @@ def compose_spec(self) -> str | None: def create_base_app() -> FastAPI: # settings app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.DYNAMIC_SIDECAR_TRACING + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.DY_SIDECAR_LOG_FILTER_MAPPING, - tracing_settings=app_settings.DYNAMIC_SIDECAR_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -145,6 +150,7 @@ def create_base_app() -> FastAPI: ) override_fastapi_openapi_method(app) app.state.settings = app_settings + app.state.tracing_config = tracing_config app.include_router(get_main_router(app)) @@ -170,9 +176,10 @@ def create_app() -> FastAPI: setup_shared_store(app) app.state.application_health = ApplicationHealth() application_settings: ApplicationSettings = app.state.settings + tracing_config = get_tracing_config(app) - if application_settings.DYNAMIC_SIDECAR_TRACING: - setup_tracing(app, application_settings.DYNAMIC_SIDECAR_TRACING, PROJECT_NAME) + if tracing_config.tracing_enabled: + setup_tracing(app, tracing_config) setup_rabbitmq(app) setup_rpc_api_routes(app) @@ -194,8 +201,11 @@ def create_app() -> FastAPI: if application_settings.are_prometheus_metrics_enabled: setup_prometheus_metrics(app) - if application_settings.DYNAMIC_SIDECAR_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing( + app, + tracing_config=tracing_config, + ) # ERROR HANDLERS ------------ app.add_exception_handler( diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py index d44dea344147..4dfac730a20e 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py @@ -5,6 +5,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import ( API_VERSION, @@ -28,8 +29,15 @@ logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings | None = None) -> FastAPI: +def create_app( + settings: ApplicationSettings | None = None, + tracing_config: TracingConfig | None = None, +) -> FastAPI: app_settings = settings or ApplicationSettings.create_from_envs() + tracing_config = tracing_config or TracingConfig.create( + service_name=app_settings.APP_NAME, + tracing_settings=app_settings.EFS_GUARDIAN_TRACING, + ) app = FastAPI( debug=app_settings.EFS_GUARDIAN_DEBUG, @@ -42,9 +50,10 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: ) # STATE app.state.settings = app_settings + app.state.tracing_config = tracing_config assert app.state.settings.API_VERSION == API_VERSION # nosec - if app.state.settings.EFS_GUARDIAN_TRACING: - setup_tracing(app, app.state.settings.EFS_GUARDIAN_TRACING, APP_NAME) + if tracing_config.tracing_enabled: + setup_tracing(app, tracing_config) # PLUGINS SETUP setup_rabbitmq(app) @@ -60,8 +69,8 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: setup_fire_and_forget(app) - if app.state.settings.EFS_GUARDIAN_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) # EVENTS async def _on_startup() -> None: diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/main.py b/services/efs-guardian/src/simcore_service_efs_guardian/main.py index 230016b548c2..8120171503a3 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/main.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/main.py @@ -5,6 +5,7 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI +from servicelib import tracing from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event from simcore_service_efs_guardian.core.application import create_app from simcore_service_efs_guardian.core.settings import ApplicationSettings @@ -22,10 +23,14 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = tracing.TracingConfig.create( + service_name=app_settings.APP_NAME, + tracing_settings=app_settings.EFS_GUARDIAN_TRACING, + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.EFS_GUARDIAN_LOG_FILTER_MAPPING, - tracing_settings=app_settings.EFS_GUARDIAN_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -34,6 +39,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/invitations/src/simcore_service_invitations/core/application.py b/services/invitations/src/simcore_service_invitations/core/application.py index fcfea7234dd2..275a0596b433 100644 --- a/services/invitations/src/simcore_service_invitations/core/application.py +++ b/services/invitations/src/simcore_service_invitations/core/application.py @@ -7,6 +7,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import ( API_VERSION, @@ -22,7 +23,10 @@ from .settings import ApplicationSettings -def create_app(settings: ApplicationSettings | None = None) -> FastAPI: +def create_app( + settings: ApplicationSettings | None = None, + tracing_config: TracingConfig | None = None, +) -> FastAPI: app = FastAPI( title=f"{PROJECT_NAME} web API", @@ -36,10 +40,14 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: # STATE app.state.settings = settings or ApplicationSettings() # type: ignore[call-arg] + tracing_config = tracing_config or TracingConfig.create( + service_name=APP_NAME, tracing_settings=app.state.settings.INVITATIONS_TRACING + ) + app.state.tracing_config = tracing_config assert app.state.settings.API_VERSION == API_VERSION # nosec - if app.state.settings.INVITATIONS_TRACING: - setup_tracing(app, app.state.settings.INVITATIONS_TRACING, APP_NAME) + if tracing_config.tracing_enabled: + setup_tracing(app, tracing_config=tracing_config) # PLUGINS SETUP setup_api_routes(app) @@ -47,8 +55,8 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: if app.state.settings.INVITATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) - if app.state.settings.INVITATIONS_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) # ERROR HANDLERS exceptions_handlers.setup(app) diff --git a/services/invitations/src/simcore_service_invitations/main.py b/services/invitations/src/simcore_service_invitations/main.py index d59e54918a07..cd34c68daa39 100644 --- a/services/invitations/src/simcore_service_invitations/main.py +++ b/services/invitations/src/simcore_service_invitations/main.py @@ -6,6 +6,7 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event +from servicelib.tracing import TracingConfig from simcore_service_invitations.core.application import create_app from simcore_service_invitations.core.settings import ApplicationSettings @@ -19,10 +20,14 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=app_settings.APP_NAME, + tracing_settings=app_settings.INVITATIONS_TRACING, + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.INVITATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.INVITATIONS_LOG_FILTER_MAPPING, - tracing_settings=app_settings.INVITATIONS_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -31,6 +36,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index 63517b52d5b4..4d87d4e62970 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -13,6 +13,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import API_VTAG, APP_NAME, SUMMARY, VERSION from ..api.rest.routing import initialize_rest_api @@ -25,8 +26,12 @@ def create_app( settings: ApplicationSettings | None = None, logging_lifespan: Lifespan | None = None, + tracing_config: TracingConfig | None = None, ) -> FastAPI: settings = settings or ApplicationSettings.create_from_envs() + tracing_config = tracing_config or TracingConfig.create( + service_name=APP_NAME, tracing_settings=settings.NOTIFICATIONS_TRACING + ) assert settings.SC_BOOT_MODE # nosec app = FastAPI( @@ -40,16 +45,17 @@ def create_app( ) override_fastapi_openapi_method(app) app.state.settings = settings + app.state.tracing_config = tracing_config - if settings.NOTIFICATIONS_TRACING: - setup_tracing(app, settings.NOTIFICATIONS_TRACING, APP_NAME) # pragma: no cover + if tracing_config.tracing_enabled: + setup_tracing(app, tracing_config=tracing_config) initialize_rest_api(app) if settings.NOTIFICATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED: initialize_prometheus_instrumentation(app) - if settings.NOTIFICATIONS_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) return app diff --git a/services/notifications/src/simcore_service_notifications/main.py b/services/notifications/src/simcore_service_notifications/main.py index cda95f9dd9f5..84d32dc27052 100644 --- a/services/notifications/src/simcore_service_notifications/main.py +++ b/services/notifications/src/simcore_service_notifications/main.py @@ -4,11 +4,14 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_lifespan +from servicelib.tracing import TracingConfig from simcore_service_notifications.core.application import create_app from simcore_service_notifications.core.settings import ( ApplicationSettings, ) +from ._meta import APP_NAME + _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -19,10 +22,13 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.NOTIFICATIONS_TRACING + ) logging_lifespan = create_logging_lifespan( log_format_local_dev_enabled=app_settings.NOTIFICATIONS_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.NOTIFICATIONS_VOLUMES_LOG_FILTER_MAPPING, - tracing_settings=app_settings.NOTIFICATIONS_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -31,4 +37,8 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - return create_app(settings=app_settings, logging_lifespan=logging_lifespan) + return create_app( + settings=app_settings, + logging_lifespan=logging_lifespan, + tracing_config=tracing_config, + ) diff --git a/services/opentelemetry-collector-config.yaml b/services/opentelemetry-collector-config.yaml index 7386666956e4..4cd5d760dd97 100644 --- a/services/opentelemetry-collector-config.yaml +++ b/services/opentelemetry-collector-config.yaml @@ -19,7 +19,7 @@ processors: timeout: 5s send_batch_size: ${TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE} probabilistic_sampler: - sampling_percentage: ${TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE} + sampling_percentage: 100 filter/drop_healthcheck: error_mode: ignore traces: diff --git a/services/payments/src/simcore_service_payments/core/application.py b/services/payments/src/simcore_service_payments/core/application.py index 94c6d2ee7e76..3413bd9501c2 100644 --- a/services/payments/src/simcore_service_payments/core/application.py +++ b/services/payments/src/simcore_service_payments/core/application.py @@ -7,6 +7,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import ( API_VERSION, @@ -30,10 +31,15 @@ from .settings import ApplicationSettings -def create_app(settings: ApplicationSettings | None = None) -> FastAPI: +def create_app( + settings: ApplicationSettings | None = None, + tracing_config: TracingConfig | None = None, +) -> FastAPI: app_settings = settings or ApplicationSettings.create_from_envs() - + app_tracing_config = tracing_config or TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.PAYMENTS_TRACING + ) app = FastAPI( title=f"{PROJECT_NAME} web API", description=SUMMARY, @@ -46,11 +52,12 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: # STATE app.state.settings = app_settings + app.state.tracing_config = app_tracing_config assert app.state.settings.API_VERSION == API_VERSION # nosec # PLUGINS SETUP - if app.state.settings.PAYMENTS_TRACING: - setup_tracing(app, app.state.settings.PAYMENTS_TRACING, APP_NAME) + if app_tracing_config.tracing_enabled: + setup_tracing(app, app_tracing_config) # API w/ postgres db setup_postgres(app) @@ -77,8 +84,8 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: if app.state.settings.PAYMENTS_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) - if app.state.settings.PAYMENTS_TRACING: - initialize_fastapi_app_tracing(app) + if app_tracing_config: + initialize_fastapi_app_tracing(app, tracing_config=app_tracing_config) # ERROR HANDLERS # ... add here ... diff --git a/services/payments/src/simcore_service_payments/main.py b/services/payments/src/simcore_service_payments/main.py index 604d4adaa1bd..558df4781c27 100644 --- a/services/payments/src/simcore_service_payments/main.py +++ b/services/payments/src/simcore_service_payments/main.py @@ -6,6 +6,7 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event +from servicelib.tracing import TracingConfig from simcore_service_payments.core.application import create_app from simcore_service_payments.core.settings import ApplicationSettings @@ -22,10 +23,14 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=app_settings.APP_NAME, + tracing_settings=app_settings.PAYMENTS_TRACING, + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.PAYMENTS_LOG_FILTER_MAPPING, - tracing_settings=app_settings.PAYMENTS_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -34,6 +39,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index 812ab0870742..b07307a9a3fd 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -1,4 +1,4 @@ -""" Interface to communicate with the payment's gateway +"""Interface to communicate with the payment's gateway - httpx client with base_url to PAYMENTS_GATEWAY_URL - Fake gateway service in services/payments/scripts/example_payment_gateway.py @@ -25,7 +25,7 @@ HealthMixinMixin, ) from servicelib.fastapi.httpx_utils import to_curl_command -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from ..core.settings import ApplicationSettings from ..models.payments_gateway import ( @@ -216,6 +216,9 @@ def setup_payments_gateway(app: FastAPI): ), ) if settings.PAYMENTS_TRACING: - setup_httpx_client_tracing(api.client) + setup_httpx_client_tracing( + api.client, + tracing_config=get_tracing_config(app), + ) api.attach_lifespan_to(app) api.set_to_app_state(app) diff --git a/services/payments/src/simcore_service_payments/services/resource_usage_tracker.py b/services/payments/src/simcore_service_payments/services/resource_usage_tracker.py index 6ae5d424fdf8..d02146ca16e5 100644 --- a/services/payments/src/simcore_service_payments/services/resource_usage_tracker.py +++ b/services/payments/src/simcore_service_payments/services/resource_usage_tracker.py @@ -1,10 +1,9 @@ -""" Interface to communicate with the Resource Usage Tracker (RUT) +"""Interface to communicate with the Resource Usage Tracker (RUT) - httpx client with base_url to PAYMENTS_RESOURCE_USAGE_TRACKER """ - import logging from datetime import datetime from decimal import Decimal @@ -25,7 +24,7 @@ BaseHTTPApi, HealthMixinMixin, ) -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from ..core.settings import ApplicationSettings @@ -75,6 +74,9 @@ def setup_resource_usage_tracker(app: FastAPI): base_url=settings.PAYMENTS_RESOURCE_USAGE_TRACKER.base_url, ) if settings.PAYMENTS_TRACING: - setup_httpx_client_tracing(api.client) + setup_httpx_client_tracing( + api.client, + tracing_config=get_tracing_config(app), + ) api.set_to_app_state(app) api.attach_lifespan_to(app) diff --git a/services/payments/src/simcore_service_payments/services/stripe.py b/services/payments/src/simcore_service_payments/services/stripe.py index 349de908d7e5..92dc755d73c9 100644 --- a/services/payments/src/simcore_service_payments/services/stripe.py +++ b/services/payments/src/simcore_service_payments/services/stripe.py @@ -1,4 +1,4 @@ -""" Interface to communicate with the Resource Usage Tracker (RUT) +"""Interface to communicate with the Resource Usage Tracker (RUT) - httpx client with base_url to PAYMENTS_RESOURCE_USAGE_TRACKER @@ -19,7 +19,7 @@ BaseHTTPApi, HealthMixinMixin, ) -from servicelib.fastapi.tracing import setup_httpx_client_tracing +from servicelib.fastapi.tracing import get_tracing_config, setup_httpx_client_tracing from ..core.errors import StripeRuntimeError from ..core.settings import ApplicationSettings @@ -93,7 +93,10 @@ def setup_stripe(app: FastAPI): auth=_StripeBearerAuth(settings.PAYMENTS_STRIPE_API_SECRET.get_secret_value()), ) if settings.PAYMENTS_TRACING: - setup_httpx_client_tracing(api.client) + setup_httpx_client_tracing( + api.client, + tracing_config=get_tracing_config(app), + ) api.set_to_app_state(app) api.attach_lifespan_to(app) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py index fb3bdf2d1e84..58d8eb65b8d4 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py @@ -6,6 +6,7 @@ initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from .._meta import ( API_VERSION, @@ -34,7 +35,7 @@ _logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings) -> FastAPI: +def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) -> FastAPI: app = FastAPI( debug=settings.RESOURCE_USAGE_TRACKER_DEBUG, title=f"{PROJECT_NAME} web API", @@ -51,11 +52,10 @@ def create_app(settings: ApplicationSettings) -> FastAPI: assert app.state.settings.API_VERSION == API_VERSION # nosec # PLUGINS SETUP - if app.state.settings.RESOURCE_USAGE_TRACKER_TRACING: + if tracing_config.tracing_enabled: setup_tracing( app, - app.state.settings.RESOURCE_USAGE_TRACKER_TRACING, - app.state.settings.APP_NAME, + tracing_config, ) setup_api_routes(app) fire_and_forget_setup(app) @@ -73,8 +73,11 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_process_message_running_service(app) # Requires Rabbit - if app.state.settings.RESOURCE_USAGE_TRACKER_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing( + app, + tracing_config=tracing_config, + ) # ERROR HANDLERS setup_exception_handlers(app) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py index 42d3184f155f..212b5cac4548 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/main.py @@ -6,6 +6,7 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event +from servicelib.tracing import TracingConfig from simcore_service_resource_usage_tracker.core.application import create_app from simcore_service_resource_usage_tracker.core.settings import ApplicationSettings @@ -22,10 +23,14 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=app_settings.APP_NAME, + tracing_settings=app_settings.RESOURCE_USAGE_TRACKER_TRACING, + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.RESOURCE_USAGE_TRACKER_LOG_FILTER_MAPPING, - tracing_settings=app_settings.RESOURCE_USAGE_TRACKER_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -34,6 +39,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/resource-usage-tracker/tests/unit/conftest.py b/services/resource-usage-tracker/tests/unit/conftest.py index 7269ffae0096..eb5f37e2de6f 100644 --- a/services/resource-usage-tracker/tests/unit/conftest.py +++ b/services/resource-usage-tracker/tests/unit/conftest.py @@ -22,6 +22,7 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict +from servicelib.tracing import TracingConfig from settings_library.rabbit import RabbitSettings from simcore_service_resource_usage_tracker.core.application import create_app from simcore_service_resource_usage_tracker.core.settings import ApplicationSettings @@ -121,15 +122,27 @@ def app_settings( @pytest.fixture -async def initialized_app(app_settings: ApplicationSettings) -> AsyncIterator[FastAPI]: - app = create_app(app_settings) +def tracing_config(app_settings: ApplicationSettings) -> TracingConfig: + return TracingConfig.create( + service_name="resource-usage-tracker-tests", + tracing_settings=None, # disable tracing in tests + ) + + +@pytest.fixture +async def initialized_app( + app_settings: ApplicationSettings, tracing_config: TracingConfig +) -> AsyncIterator[FastAPI]: + app = create_app(app_settings, tracing_config=tracing_config) async with LifespanManager(app): yield app @pytest.fixture -def client(app_settings: ApplicationSettings) -> Iterator[TestClient]: - app = create_app(app_settings) +def client( + app_settings: ApplicationSettings, tracing_config: TracingConfig +) -> Iterator[TestClient]: + app = create_app(app_settings, tracing_config=tracing_config) with TestClient(app, base_url="http://testserver.test") as client: yield client diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py index 8bf5e5ce5ba7..2bc186e3c161 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py @@ -3,10 +3,10 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -from collections.abc import AsyncIterable, Callable +from collections.abc import AsyncIterable, Awaitable, Callable from datetime import datetime, timezone from random import choice -from typing import Any, Awaitable +from typing import Any import httpx import pytest @@ -24,6 +24,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.tracing import TracingConfig from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.resource_tracker_credit_transactions import ( CreditTransactionClassification, @@ -67,7 +68,11 @@ async def initialized_app( postgres_host_config: dict[str, str], ) -> AsyncIterable[FastAPI]: settings = ApplicationSettings.create_from_envs() - app = create_app(settings) + tracing_config = TracingConfig.create( + service_name="resource-usage-tracker", + tracing_settings=None, # disable tracing in tests + ) + app = create_app(settings, tracing_config=tracing_config) async with LifespanManager(app): yield app diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index cf3bb4546fcc..6e2ee80af1e8 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -18,9 +18,11 @@ from servicelib.fastapi.openapi import override_fastapi_openapi_method from servicelib.fastapi.profiler import ProfilerMiddleware from servicelib.fastapi.tracing import ( + get_tracing_config, initialize_fastapi_app_tracing, setup_tracing, ) +from servicelib.tracing import TracingConfig from starlette.middleware.base import BaseHTTPMiddleware from .._meta import ( @@ -46,7 +48,9 @@ _logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings) -> FastAPI: # noqa: C901 +def create_app( + settings: ApplicationSettings, tracing_config: TracingConfig +) -> FastAPI: # noqa: C901 app = FastAPI( debug=settings.SC_BOOT_MODE in [BootModeEnum.DEBUG, BootModeEnum.DEVELOPMENT, BootModeEnum.LOCAL], @@ -62,13 +66,17 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # noqa: C901 # STATE app.state.settings = settings + app.state.tracing_config = tracing_config - if settings.STORAGE_TRACING: - setup_tracing(app, settings.STORAGE_TRACING, APP_NAME) + if tracing_config.tracing_enabled: + setup_tracing(app, tracing_config) setup_db(app) setup_s3(app) - setup_client_session(app, tracing_settings=settings.STORAGE_TRACING) + setup_client_session( + app, + tracing_config=get_tracing_config(app), + ) if settings.STORAGE_CELERY: setup_task_manager(app, settings=settings.STORAGE_CELERY) @@ -102,8 +110,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # noqa: C901 if settings.STORAGE_MONITORING_ENABLED: setup_prometheus_instrumentation(app) - if settings.STORAGE_TRACING: - initialize_fastapi_app_tracing(app) + if tracing_config.tracing_enabled: + initialize_fastapi_app_tracing(app, tracing_config=tracing_config) async def _on_startup() -> None: if settings.STORAGE_WORKER_MODE: diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index f2282decaa59..4762791a5b8b 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -6,9 +6,12 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event +from servicelib.tracing import TracingConfig from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings +from ._meta import APP_NAME + _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -24,10 +27,13 @@ def app_factory() -> FastAPI: app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=app_settings.STORAGE_TRACING + ) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.STORAGE_LOG_FILTER_MAPPING, - tracing_settings=app_settings.STORAGE_TRACING, + tracing_config=tracing_config, log_base_level=app_settings.log_level, noisy_loggers=_NOISY_LOGGERS, ) @@ -36,6 +42,6 @@ def app_factory() -> FastAPI: "Application settings: %s", json_dumps(app_settings, indent=2, sort_keys=True), ) - app = create_app(settings=app_settings) + app = create_app(settings=app_settings, tracing_config=tracing_config) app.add_event_handler("shutdown", logging_shutdown_event) return app diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index f2e90e900244..b82fff90abde 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -8,17 +8,22 @@ ) from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.logging_utils import setup_loggers +from servicelib.tracing import TracingConfig from ...api._worker_tasks.tasks import setup_worker_tasks from ...core.application import create_app from ...core.settings import ApplicationSettings _settings = ApplicationSettings.create_from_envs() +_tracing_config = TracingConfig.create( + tracing_settings=_settings.STORAGE_TRACING, + service_name="storage-celery-worker", +) setup_loggers( log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, - tracing_settings=_settings.STORAGE_TRACING, + tracing_config=_tracing_config, log_base_level=_settings.log_level, noisy_loggers=None, ) @@ -27,7 +32,7 @@ assert _settings.STORAGE_CELERY # nosec app = create_celery_app(_settings.STORAGE_CELERY) -app_server = FastAPIAppServer(app=create_app(_settings)) +app_server = FastAPIAppServer(app=create_app(_settings, tracing_config=_tracing_config)) def worker_init_wrapper(sender, **kwargs): diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 802c3fab387c..1831fab085a9 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -65,6 +65,7 @@ from servicelib.aiohttp import status from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from servicelib.tracing import TracingConfig from servicelib.utils import limited_gather from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.tokens import tokens @@ -236,7 +237,11 @@ async def initialized_app( mock_celery_app: None, app_settings: ApplicationSettings, ) -> AsyncIterator[FastAPI]: - app = create_app(app_settings) + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-api", + ) + app = create_app(app_settings, tracing_config=tracing_config) # NOTE: the timeout is sometime too small for CI machines, and even larger machines async with LifespanManager( app, startup_timeout=_LIFESPAN_TIMEOUT, shutdown_timeout=_LIFESPAN_TIMEOUT @@ -1013,8 +1018,14 @@ async def with_storage_celery_worker( # Signals must be explicitily connected monkeypatch.setenv("STORAGE_WORKER_MODE", "true") app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-api", + ) - app_server = FastAPIAppServer(app=create_app(app_settings)) + app_server = FastAPIAppServer( + app=create_app(app_settings, tracing_config=tracing_config) + ) def _on_worker_init_wrapper(sender: WorkController, **_kwargs): return on_worker_init(sender, app_server, **_kwargs) diff --git a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py index 0531f02c6b10..9a411b6dbb83 100644 --- a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py +++ b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py @@ -24,6 +24,7 @@ from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID from pytest_simcore.helpers.logging_tools import log_context +from servicelib.tracing import TracingConfig from servicelib.utils import unused_port from simcore_service_storage._meta import API_VTAG from simcore_service_storage.core.application import create_app @@ -63,7 +64,11 @@ async def _wait_for_server_ready(server: URL) -> None: @pytest.fixture async def real_storage_server(app_settings: ApplicationSettings) -> AsyncIterator[URL]: settings = ApplicationSettings.create_from_envs() - app = create_app(settings) + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-api", + ) + app = create_app(settings, tracing_config=tracing_config) storage_port = unused_port() with log_context( logging.INFO, diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index 67cb5bf49f1c..43f0d655e1d2 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -8,8 +8,11 @@ from aiohttp import web from servicelib.aiohttp.application import create_safe_application +from servicelib.aiohttp.tracing import TRACING_CONFIG_KEY +from servicelib.tracing import TracingConfig from ._meta import ( + APP_NAME, WELCOME_AUTH_APP_MSG, WELCOME_DB_LISTENER_MSG, WELCOME_GC_MSG, @@ -99,18 +102,20 @@ async def _finished_banner(app: web.Application): return _finished_banner -def create_application() -> web.Application: +def create_application(tracing_config: TracingConfig) -> web.Application: """ Initializes service """ app = create_safe_application() setup_settings(app) + app[TRACING_CONFIG_KEY] = tracing_config # WARNING: setup order matters # NOTE: compute setup order https://github.com/ITISFoundation/osparc-simcore/issues/1142 # core modules - setup_app_tracing(app) # WARNING: must be UPPERMOST middleware + if tracing_config.tracing_enabled: + setup_app_tracing(app) # WARNING: must be UPPERMOST middleware setup_db(app) setup_redis(app) setup_session(app) @@ -205,6 +210,10 @@ def create_application_auth() -> web.Application: app = create_safe_application() settings = setup_settings(app) + tracing_config = TracingConfig.create( + settings.WEBSERVER_TRACING, service_name=APP_NAME + ) + app[TRACING_CONFIG_KEY] = tracing_config assert settings.WEBSERVER_APP_FACTORY_NAME == "WEBSERVER_AUTHZ_APP_FACTORY" # nosec # Monitoring and diagnostics diff --git a/services/web/server/src/simcore_service_webserver/cli.py b/services/web/server/src/simcore_service_webserver/cli.py index de59ad46ddfe..960b8c4166ce 100644 --- a/services/web/server/src/simcore_service_webserver/cli.py +++ b/services/web/server/src/simcore_service_webserver/cli.py @@ -20,8 +20,10 @@ import typer from aiohttp import web from common_library.json_serialization import json_dumps +from servicelib.tracing import TracingConfig from settings_library.utils_cli import create_settings_command +from ._meta import APP_NAME from .application_settings import ApplicationSettings from .login import cli as login_cli @@ -37,6 +39,7 @@ def _setup_app_from_settings( settings: ApplicationSettings, + tracing_config: TracingConfig, ) -> tuple[web.Application, dict]: # NOTE: keeping imports here to reduce CLI load time from .application import create_application @@ -49,7 +52,7 @@ def _setup_app_from_settings( # given configs and changing those would not have # a meaningful RoI. config = convert_to_app_config(settings) - app = create_application() + app = create_application(tracing_config=tracing_config) return (app, config) @@ -62,6 +65,9 @@ async def app_factory() -> web.Application: from .log import setup_logging app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + app_settings.WEBSERVER_TRACING, service_name=APP_NAME + ) _logger.info( "Application settings: %s", @@ -72,12 +78,14 @@ async def app_factory() -> web.Application: "Using application factory: %s", app_settings.WEBSERVER_APP_FACTORY_NAME ) - logging_lifespan_cleanup_event = setup_logging(app_settings) + logging_lifespan_cleanup_event = setup_logging( + app_settings, tracing_config=tracing_config + ) if app_settings.WEBSERVER_APP_FACTORY_NAME == "WEBSERVER_AUTHZ_APP_FACTORY": app = create_application_auth() else: - app, _ = _setup_app_from_settings(app_settings) + app, _ = _setup_app_from_settings(app_settings, tracing_config) app.on_cleanup.append(logging_lifespan_cleanup_event) return app @@ -120,5 +128,8 @@ def run(): from .application import run_service app_settings = ApplicationSettings.create_from_envs() - app, cfg = _setup_app_from_settings(app_settings) + app_tracing_config = TracingConfig.create( + app_settings.WEBSERVER_TRACING, service_name=APP_NAME + ) + app, cfg = _setup_app_from_settings(app_settings, app_tracing_config) run_service(app, cfg) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py index 1b204b21e4b1..299c2574b7ba 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py @@ -28,7 +28,6 @@ ) from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.rest_constants import RESPONSE_MODEL_POLICY -from servicelib.tracing import with_profiled_span from ..._meta import API_VTAG as VTAG from ...constants import RQ_PRODUCT_KEY @@ -203,7 +202,6 @@ async def list_computations_latest_iteration_tasks( name="list_computation_collection_runs", ) @login_required -@with_profiled_span @permission_required("services.pipeline.*") @permission_required("project.read") async def list_computation_collection_runs(request: web.Request) -> web.Response: diff --git a/services/web/server/src/simcore_service_webserver/log.py b/services/web/server/src/simcore_service_webserver/log.py index 271c28bd7b95..7e8ef62e23d1 100644 --- a/services/web/server/src/simcore_service_webserver/log.py +++ b/services/web/server/src/simcore_service_webserver/log.py @@ -9,6 +9,7 @@ from aiohttp import web from aiohttp.log import access_logger from servicelib.logging_utils import async_loggers +from servicelib.tracing import TracingConfig from simcore_service_webserver.application_settings import ApplicationSettings _NOISY_LOGGERS: Final[tuple[str, ...]] = ( @@ -30,7 +31,9 @@ CleanupEvent: TypeAlias = Callable[[web.Application], Awaitable[None]] -def setup_logging(app_settings: ApplicationSettings) -> CleanupEvent: +def setup_logging( + app_settings: ApplicationSettings, tracing_config: TracingConfig +) -> CleanupEvent: exit_stack = AsyncExitStack() exit_stack.enter_context( async_loggers( @@ -38,7 +41,7 @@ def setup_logging(app_settings: ApplicationSettings) -> CleanupEvent: noisy_loggers=_NOISY_LOGGERS, log_format_local_dev_enabled=app_settings.WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.WEBSERVER_LOG_FILTER_MAPPING, - tracing_settings=app_settings.WEBSERVER_TRACING, + tracing_config=tracing_config, ) ) diff --git a/services/web/server/src/simcore_service_webserver/tracing.py b/services/web/server/src/simcore_service_webserver/tracing.py index 552fb8393c64..cbaa5022840f 100644 --- a/services/web/server/src/simcore_service_webserver/tracing.py +++ b/services/web/server/src/simcore_service_webserver/tracing.py @@ -1,23 +1,13 @@ import logging from aiohttp import web -from servicelib.aiohttp.tracing import get_tracing_lifespan -from settings_library.tracing import TracingSettings +from servicelib.aiohttp.tracing import TRACING_CONFIG_KEY, setup_tracing -from .application_keys import APP_SETTINGS_APPKEY -from .application_settings import get_application_settings from .application_setup import ModuleCategory, app_setup_func log = logging.getLogger(__name__) -def get_plugin_settings(app: web.Application) -> TracingSettings: - settings = app[APP_SETTINGS_APPKEY].WEBSERVER_TRACING - assert settings, "setup_settings not called?" # nosec - assert isinstance(settings, TracingSettings) # nosec - return settings - - @app_setup_func( __name__, ModuleCategory.ADDON, settings_name="WEBSERVER_TRACING", logger=log ) @@ -32,14 +22,10 @@ def setup_app_tracing(app: web.Application): """ - app_settings = get_application_settings(app) - tracing_settings: TracingSettings = get_plugin_settings(app) - app.cleanup_ctx.append( - get_tracing_lifespan( + setup_tracing( app=app, - tracing_settings=tracing_settings, - service_name=app_settings.APP_NAME, + tracing_config=app[TRACING_CONFIG_KEY], add_response_trace_id_header=True, ) ) diff --git a/services/web/server/src/simcore_service_webserver/user_notifications/_controller/rest/user_notification_rest.py b/services/web/server/src/simcore_service_webserver/user_notifications/_controller/rest/user_notification_rest.py index 679179857025..010d15a325ff 100644 --- a/services/web/server/src/simcore_service_webserver/user_notifications/_controller/rest/user_notification_rest.py +++ b/services/web/server/src/simcore_service_webserver/user_notifications/_controller/rest/user_notification_rest.py @@ -9,7 +9,6 @@ parse_request_body_as, parse_request_path_parameters_as, ) -from servicelib.tracing import with_profiled_span from ...._meta import API_VTAG from ....login.decorators import login_required @@ -74,7 +73,6 @@ async def mark_notification_as_read(request: web.Request) -> web.Response: @routes.get(f"/{API_VTAG}/me/permissions", name="list_user_permissions") @login_required -@with_profiled_span @permission_required("user.permissions.read") async def list_user_permissions(request: web.Request) -> web.Response: req_ctx = UsersRequestContext.model_validate(request) diff --git a/services/web/server/tests/unit/isolated/test_tracing.py b/services/web/server/tests/unit/isolated/test_tracing.py index 88eec9626a0f..7006585fb7d2 100644 --- a/services/web/server/tests/unit/isolated/test_tracing.py +++ b/services/web/server/tests/unit/isolated/test_tracing.py @@ -4,11 +4,11 @@ import pytest -from opentelemetry.instrumentation.aiohttp_server import ( - middleware as aiohttp_opentelemetry_middleware, -) from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict +from servicelib.aiohttp.tracing import aiohttp_server_opentelemetry_middleware +from servicelib.tracing import TracingConfig +from simcore_service_webserver._meta import APP_NAME from simcore_service_webserver.application import create_application from simcore_service_webserver.application_settings import ApplicationSettings @@ -32,7 +32,10 @@ def test_middleware_restrictions_opentelemetry_is_second_middleware( ): settings = ApplicationSettings.create_from_envs() assert settings.WEBSERVER_TRACING + tracing_config = TracingConfig.create( + service_name=APP_NAME, tracing_settings=settings.WEBSERVER_TRACING + ) - app = create_application() + app = create_application(tracing_config=tracing_config) assert app.middlewares - assert app.middlewares[0] is aiohttp_opentelemetry_middleware + assert app.middlewares[0] is aiohttp_server_opentelemetry_middleware diff --git a/services/web/server/tests/unit/with_dbs/03/test__openapi_specs.py b/services/web/server/tests/unit/with_dbs/03/test__openapi_specs.py index 1d1b7303f6ed..400591447f6e 100644 --- a/services/web/server/tests/unit/with_dbs/03/test__openapi_specs.py +++ b/services/web/server/tests/unit/with_dbs/03/test__openapi_specs.py @@ -13,6 +13,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.openapi_specs import Entrypoint +from servicelib.tracing import TracingConfig from simcore_service_webserver.application import create_application from simcore_service_webserver.application_settings import get_application_settings from simcore_service_webserver.rest._utils import get_openapi_specs_path @@ -59,7 +60,10 @@ def app(app_environment: EnvVarsDict) -> web.Application: # - routings happen during setup! # - all plugins are setup but app is NOT started (i.e events are not triggered) # - app_ = create_application() + tracing_config = TracingConfig.create( + service_name="test-webserver", tracing_settings=None + ) + app_ = create_application(tracing_config=tracing_config) print(get_application_settings(app_).model_dump_json(indent=1)) return app_ diff --git a/services/web/server/tests/unit/with_dbs/03/test_session.py b/services/web/server/tests/unit/with_dbs/03/test_session.py index 2bf9a190e994..6efc53955761 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_session.py +++ b/services/web/server/tests/unit/with_dbs/03/test_session.py @@ -14,6 +14,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import NewUser +from servicelib.tracing import TracingConfig from simcore_service_webserver.application import create_application from simcore_service_webserver.session._cookie_storage import ( SharedCookieEncryptedCookieStorage, @@ -57,7 +58,10 @@ async def _get_user_session(request: web.Request): session = await get_session(request) return web.json_response(dict(session)) - app = create_application() + tracing_config = TracingConfig.create( + service_name="test-webserver", tracing_settings=None + ) + app = create_application(tracing_config=tracing_config) disable_static_webserver(app) app.add_routes(extra_routes) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 085f8ea02eb4..6aed77fc3ea3 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -54,6 +54,7 @@ from pytest_simcore.helpers.webserver_projects import NewProject from pytest_simcore.helpers.webserver_users import UserInfoDict from redis import Redis +from servicelib import tracing from servicelib.common_aiopg_utils import DSN from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.async_jobs.async_jobs import ( @@ -228,7 +229,10 @@ async def web_server( assert app_environment # original APP - app = create_application() + tracing_config = tracing.TracingConfig.create( + service_name="test-webserver", tracing_settings=None + ) + app = create_application(tracing_config=tracing_config) disable_static_webserver(app)