Skip to content

Commit 2ed0411

Browse files
sduenassduenas
committed
[config] Add structured logs
Organizing log data in a structured format makes easier to read and analyze them. With this commit, log messages will always have a structured format. The default mode is to print the messages to the console in a plain format, but it can also be configured to print them in JSON format. We have added structured logs to GrimoireLab using the 'structlog' package. By default, tests will run silent. If developers want to get log messages printed to the console, activate the environment variable 'GRIMOIRELAB_TESTING_VERBOSE'. Signed-off-by: Santiago Dueñas <[email protected]>
1 parent 6cd6074 commit 2ed0411

File tree

7 files changed

+279
-48
lines changed

7 files changed

+279
-48
lines changed

‎README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ $ pip install -e .
8080
```
8181

8282
#### Poetry
83-
We use [poetry](https://python-poetry.org/) for dependency management and
83+
We use [poetry](https://python-poetry.org/) for dependency management and
8484
packaging. You can install it following its [documentation](https://python-poetry.org/docs/#installation).
8585
Once you have installed it, you can install grimoirelab-core and the dependencies
8686
in a project isolated environment using:
@@ -122,7 +122,7 @@ Commands:
122122

123123
## Configuration
124124

125-
The first step is to run a Redis server, a MySQL database and an OpenSearch
125+
The first step is to run a Redis server, a MySQL database and an OpenSearch
126126
container that will be used for communicating components and storing results.
127127
Please refer to their documentation to know how to install and run them.
128128

@@ -190,3 +190,6 @@ Valkey, and OpenSearch).
190190
```
191191
(.venv)$ pytest
192192
```
193+
194+
Set the environment variable `GRIMOIRELAB_TESTING_VERBOSE` to activate the
195+
verbose mode.

‎config/settings/testing.py‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,33 @@
1-
from grimoirelab.core.config.settings import * # noqa: F403,F401
2-
from grimoirelab.core.config.settings import INSTALLED_APPS, _RQ_DATABASE, RQ
3-
1+
import os
42
import warnings
53

4+
from grimoirelab.core.config.settings import * # noqa: F403,F401
5+
from grimoirelab.core.config.settings import INSTALLED_APPS, _RQ_DATABASE, RQ, LOGGING
6+
67
import django_rq.queues
78

89
from fakeredis import FakeRedis, FakeStrictRedis
910

11+
1012
INSTALLED_APPS.append("tests")
1113

12-
LOGGING = {
13-
"version": 1,
14-
"disable_existing_loggers": True,
15-
"loggers": {
16-
"grimoirelab.core": {"level": "CRITICAL"},
17-
},
14+
GRIMOIRELAB_TESTING_VERBOSE = os.environ.get("GRIMOIRELAB_TESTING_VERBOSE", "False").lower() in ("true", "1")
15+
16+
# Logging configuration for testing
17+
#
18+
# By default, logging is silent and doesn't print messages to the console.
19+
# Set 'GRIMOIRELAB_TESTING_VERBOSE' to 'True' to print messages to the console.
20+
#
21+
LOGGING["handlers"]["testing"] = {
22+
"class": "logging.NullHandler",
23+
}
24+
25+
LOGGING["loggers"] = {
26+
"": {
27+
"handlers": ["default"] if GRIMOIRELAB_TESTING_VERBOSE else ["testing"],
28+
"level": "DEBUG",
29+
"propagate": True,
30+
}
1831
}
1932

2033
SQL_MODE = [

‎poetry.lock‎

Lines changed: 66 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ opensearch-py = "^3.0.0"
6161
djangorestframework-simplejwt = "^5.4.0"
6262
django-storages = {version = "^1.14.6", extras = ["google"]}
6363
drf-spectacular = "^0.28.0"
64+
django-structlog = "^9.0.1"
65+
structlog = "^25.2.0"
6466
grimoirelab-toolkit = {version = "^1.1.0", allow-prereleases = true}
6567
perceval = {version = "^1.3.4", allow-prereleases = true}
6668
sortinghat = {version = "^1.11.0", allow-prereleases = true}

‎src/grimoirelab/core/config/logging.py‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# GrimoireLab logging setup.
4+
#
5+
# Logging is configured using 'structlog', 'django-structlog',
6+
# and the standard library 'logging' module.
7+
#
8+
# More info:
9+
#
10+
# https://django-structlog.readthedocs.io/en/latest/
11+
# https://docs.djangoproject.com/en/4.2/topics/logging/#configuring-logging
12+
#
13+
14+
import structlog
15+
16+
17+
# Default column styles for the console renderer.
18+
19+
styles = structlog.dev._ColorfulStyles
20+
21+
logger_name_formatter = structlog.dev.KeyValueColumnFormatter(
22+
key_style=None,
23+
value_style=styles.bright + styles.logger_name,
24+
reset_style=styles.reset,
25+
value_repr=str,
26+
prefix="[",
27+
postfix="]"
28+
)
29+
30+
console_columns = [
31+
structlog.dev.Column(
32+
"timestamp",
33+
structlog.dev.KeyValueColumnFormatter(
34+
key_style=None,
35+
value_style=styles.timestamp,
36+
reset_style=styles.reset,
37+
value_repr=str
38+
),
39+
),
40+
structlog.dev.Column(
41+
"level",
42+
structlog.dev.LogLevelColumnFormatter(
43+
structlog.dev.ConsoleRenderer.get_default_level_styles(),
44+
reset_style=styles.reset,
45+
),
46+
),
47+
structlog.dev.Column("logger", logger_name_formatter),
48+
structlog.dev.Column("logger_name", logger_name_formatter),
49+
structlog.dev.Column(
50+
"event",
51+
structlog.dev.KeyValueColumnFormatter(
52+
key_style=None,
53+
value_style=styles.bright,
54+
reset_style=styles.reset,
55+
value_repr=str
56+
),
57+
),
58+
# Default formatter for all keys not explicitly mentioned.
59+
structlog.dev.Column(
60+
"",
61+
structlog.dev.KeyValueColumnFormatter(
62+
key_style=styles.kv_key,
63+
value_style=styles.kv_value,
64+
reset_style=styles.reset,
65+
value_repr=str
66+
),
67+
)
68+
]
69+
70+
71+
# Configuration of chain processors for logs not generated by structlog.
72+
# This will add default fields such the log level, timestamp, etc.
73+
74+
pre_chain_processors = [
75+
structlog.processors.TimeStamper(fmt="iso", utc=True),
76+
structlog.stdlib.add_log_level,
77+
structlog.stdlib.add_logger_name,
78+
structlog.stdlib.ExtraAdder()
79+
]
80+
81+
82+
# Default logging configuration for GrimoireLab
83+
84+
_GRIMOIRELAB_LOGGING_CONFIG = {
85+
"version": 1,
86+
"disable_existing_loggers": False,
87+
"formatters": {
88+
"plain": {
89+
"()": structlog.stdlib.ProcessorFormatter,
90+
"processors": [
91+
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
92+
structlog.dev.ConsoleRenderer(columns=console_columns),
93+
],
94+
"foreign_pre_chain": pre_chain_processors,
95+
},
96+
"json": {
97+
"()": structlog.stdlib.ProcessorFormatter,
98+
"processors": [
99+
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
100+
structlog.processors.JSONRenderer(),
101+
],
102+
"foreign_pre_chain": pre_chain_processors,
103+
},
104+
"not_structured": {
105+
"format": "[%(asctime)s - %(name)s - %(levelname)s] - %(message)s"
106+
},
107+
},
108+
"handlers": {
109+
"default": {
110+
"class": "logging.StreamHandler",
111+
"formatter": "plain",
112+
},
113+
"json": {
114+
"class": "logging.StreamHandler",
115+
"formatter": "json",
116+
},
117+
"scheduler": {
118+
"class": "logging.StreamHandler",
119+
"formatter": "not_structured",
120+
},
121+
}
122+
}
123+
124+
125+
def configure_grimoirelab_logging(
126+
json_mode: bool = False,
127+
debug: bool = False
128+
) -> None:
129+
"""
130+
Set up the GrimoireLab logging settings.
131+
132+
:param json_mode: If True, use JSON format for logging.
133+
:param debug: If True, set logging level to DEBUG.
134+
"""
135+
structlog.configure(
136+
processors=[
137+
structlog.contextvars.merge_contextvars,
138+
structlog.stdlib.filter_by_level,
139+
structlog.stdlib.PositionalArgumentsFormatter(),
140+
structlog.processors.TimeStamper(fmt="iso", utc=True),
141+
structlog.stdlib.add_log_level,
142+
structlog.stdlib.add_logger_name,
143+
structlog.processors.StackInfoRenderer(),
144+
structlog.processors.format_exc_info,
145+
structlog.processors.UnicodeDecoder(),
146+
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
147+
],
148+
logger_factory=structlog.stdlib.LoggerFactory(),
149+
wrapper_class=structlog.stdlib.BoundLogger,
150+
cache_logger_on_first_use=True,
151+
)
152+
153+
logging_settings = dict(_GRIMOIRELAB_LOGGING_CONFIG)
154+
logging_settings["loggers"] = {
155+
"": {
156+
"handlers": ["json"] if json_mode else ["default"],
157+
"level": "DEBUG" if debug else "INFO",
158+
},
159+
"grimoirelab.core.scheduler": {
160+
"handlers": ["scheduler"],
161+
}
162+
}
163+
164+
return logging_settings

0 commit comments

Comments
 (0)