From ab53ce658d150333c43e76565ccf1f17a88c0535 Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Mon, 18 Aug 2025 14:16:49 -0400 Subject: [PATCH 1/2] debug log projects: enable debug log through environment variable and have HTTP request/responses being logged as airbyte messages --- airbyte_cdk/entrypoint.py | 4 ++-- airbyte_cdk/logger.py | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/airbyte_cdk/entrypoint.py b/airbyte_cdk/entrypoint.py index 76a1be32e..54c207487 100644 --- a/airbyte_cdk/entrypoint.py +++ b/airbyte_cdk/entrypoint.py @@ -22,7 +22,7 @@ from airbyte_cdk.connector import TConfig from airbyte_cdk.exception_handler import init_uncaught_exception_handler -from airbyte_cdk.logger import PRINT_BUFFER, init_logger +from airbyte_cdk.logger import PRINT_BUFFER, init_logger, is_platform_debug_log_enabled from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteMessage, @@ -158,7 +158,7 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]: if not cmd: raise Exception("No command passed") - if hasattr(parsed_args, "debug") and parsed_args.debug: + if (hasattr(parsed_args, "debug") and parsed_args.debug) or is_platform_debug_log_enabled(): self.logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG) self.logger.debug("Debug logs enabled") diff --git a/airbyte_cdk/logger.py b/airbyte_cdk/logger.py index 13c3b4676..dcd3171fd 100644 --- a/airbyte_cdk/logger.py +++ b/airbyte_cdk/logger.py @@ -1,10 +1,10 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import json import logging import logging.config +import os from typing import Any, Callable, Mapping, Optional, Tuple import orjson @@ -40,6 +40,10 @@ } +def is_platform_debug_log_enabled(): + return os.environ.get("LOG_LEVEL", "info").lower() == "debug" + + def init_logger(name: Optional[str] = None) -> logging.Logger: """Initial set up of logger""" logger = logging.getLogger(name) @@ -73,8 +77,20 @@ def format(self, record: logging.LogRecord) -> str: airbyte_level = self.level_mapping.get(record.levelno, "INFO") if airbyte_level == Level.DEBUG: extras = self.extract_extra_args_from_record(record) - debug_dict = {"type": "DEBUG", "message": record.getMessage(), "data": extras} - return filter_secrets(json.dumps(debug_dict)) + if is_platform_debug_log_enabled(): + # We have a different behavior between debug logs enabled through `--debug` argument and debug logs + # enabled through environment variable. The reason is that for platform logs, we need to have these + # printed as AirbyteMessage which is not the case with the current previous implementation. + # Why not migrate both to AirbyteMessages then? AirbyteMessages do not support having structured logs. + # which means that the DX would be degraded compared to the current solution (devs will need to identify + # the `log.message` field and figure out where in this field is the response while the current solution + # have a specific field that is structured for extras. + message = f"{filter_secrets(record.getMessage())} ///\nExtra logs: {filter_secrets(json.dumps(extras))}" + log_message = AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=airbyte_level, message=message)) + return orjson.dumps(AirbyteMessageSerializer.dump(log_message)).decode() + else: + debug_dict = {"type": "DEBUG", "message": record.getMessage(), "data": extras} + return filter_secrets(json.dumps(debug_dict)) else: message = super().format(record) message = filter_secrets(message) From ec49c93afaa823bbb5971134fde1d8e5947af42c Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Mon, 18 Aug 2025 14:31:08 -0400 Subject: [PATCH 2/2] format and mypy --- airbyte_cdk/logger.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/logger.py b/airbyte_cdk/logger.py index dcd3171fd..4223bda55 100644 --- a/airbyte_cdk/logger.py +++ b/airbyte_cdk/logger.py @@ -40,7 +40,7 @@ } -def is_platform_debug_log_enabled(): +def is_platform_debug_log_enabled() -> bool: return os.environ.get("LOG_LEVEL", "info").lower() == "debug" @@ -86,7 +86,9 @@ def format(self, record: logging.LogRecord) -> str: # the `log.message` field and figure out where in this field is the response while the current solution # have a specific field that is structured for extras. message = f"{filter_secrets(record.getMessage())} ///\nExtra logs: {filter_secrets(json.dumps(extras))}" - log_message = AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=airbyte_level, message=message)) + log_message = AirbyteMessage( + type=Type.LOG, log=AirbyteLogMessage(level=airbyte_level, message=message) + ) return orjson.dumps(AirbyteMessageSerializer.dump(log_message)).decode() else: debug_dict = {"type": "DEBUG", "message": record.getMessage(), "data": extras}