Use crawl4ai with Open Telemetry

[juny76]

It appears that Crawl4AI's internal logging mechanism does not fully support OpenTelemetry's logging instrumentation. When attempting to use opentelemetry.instrumentation.logging.LoggingInstrumentor().instrument(set_logging_format=True), the log output generated by Crawl4AI becomes malformed, with fields not aligning correctly as shown in the attached screenshot/example.

Given this observed incompatibility, we would like to inquire:

• Are there any recommended workarounds or official solutions available right now to properly integrate Crawl4AI's internal logging with OpenTelemetry for distributed tracing?
• Are there any plans for future development to refactor Crawl4AI's internal logging to use Python's standard logging module, or to provide native OpenTelemetry integration (e.g., specific instrumentations for Crawl4AI) that would resolve this issue?

Here is my code:

import logging
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from typing import Any

import uvicorn
from fastapi import FastAPI
from opentelemetry.instrumentation.logging import LoggingInstrumentor
from prometheus_client import REGISTRY
from prometheus_client.openmetrics.exposition import (
    CONTENT_TYPE_LATEST,
    generate_latest,
)
from starlette.responses import Response
from uvicorn.config import LOGGING_CONFIG

from api.middleware import PrometheusMiddleware
from api.rest_routes import router
from api.utils import EndpointFilter, setup_observability
from clients import db_client
from clients.http import HTTPRequestClient
from common.http_request import HTTPClient
from config.settings import Configs

# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("api")

app_config = Configs.get_app_config()
services_config = Configs.get_services_config()


@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, Any]:  # pyright: ignore[reportUnusedParameter]
    http_client: HTTPClient = HTTPRequestClient.get_http_client()
    _ = await http_client.startup()
    try:
        yield

    finally:
        db_client.close()
        logger.info(msg="Disconnected Redis database")


# FastAPI app
app = FastAPI(
    root_path=app_config.PROXY_ROOT_PATH,
    lifespan=lifespan,
)


app.add_middleware(PrometheusMiddleware)
# Set the OpenTelemetry observability up
setup_observability(
    app=app, service_name=app_config.NAME, trace_endpoint=services_config.TEMPO_URL
)

# Setup the instrumentator for stdlib, to synchronize the logging config
LoggingInstrumentor().instrument(set_logging_format=True)
logging.getLogger("uvicorn.access").addFilter(EndpointFilter())
logger = logging.getLogger(__name__)

import logging
import random
import re
import ssl

import aiohttp
from crawl4ai import (
    AsyncWebCrawler,
    CacheMode,
    CrawlerRunConfig,
)
from crawl4ai.content_filter_strategy import PruningContentFilter
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
from crawl4ai.models import CrawlResult
from lxml import html
from markdownify import markdownify as md
from playwright.async_api import async_playwright

from clients.http import HTTPRequestClient
from common.http_request import HTTPClient, RequestMethod


class WebCrawler:
    @staticmethod
    async def _crawl_with_crawl4ai(url: str):
        """
        Crawl a webpage using the Crawl4AI library.

        This method asynchronously fetches the content of the given URL using Crawl4AI.
        It applies specific configurations such as caching mode, excluded tags,
        and content filtering. The function returns the extracted markdown content
        if the crawl is successful; otherwise, it logs an error and returns an empty string.

        Args:
            url (str): The URL of the webpage to crawl.

        Returns:
            str: The extracted markdown content if successful; otherwise, an empty string.
        """

        config = CrawlerRunConfig(
            cache_mode=CacheMode.BYPASS,
            screenshot=False,
            magic=False,
            excluded_tags=["header", "footer", "nav"],
            remove_overlay_elements=True,
            exclude_external_links=True,
            exclude_social_media_links=True,
            markdown_generator=DefaultMarkdownGenerator(
                content_filter=PruningContentFilter(
                    threshold=0.48,
                    threshold_type="fixed",
                    min_word_threshold=0,
                ),
            ),
        )

        async with AsyncWebCrawler(verbose=True) as crawler:
            crawl_result: str = ""
            try:
                result: CrawlResult = await crawler.arun(  # Type hint added here
                    url=url,
                    config=config,
                )

                if result.success:  # Check attribute exists
                    logger.info(f"Successfully crawled: {result.url}")
                    # Check if markdown attribute exists and has fit_markdown
                    if result.markdown:
                        crawl_result = result.markdown.fit_markdown
                    return crawl_result
                else:
                    error_msg = result.error_message
                    logger.warning(f"Failed to crawl: {result.url}, Error: {error_msg}")
                    return crawl_result
            except Exception as e:
                logger.error(
                    f"Exception during crawl4ai processing for {url}: {str(e)}"
                )
                return crawl_result