From b5e54e7e1c4dd852250c12291718418b914feecb Mon Sep 17 00:00:00 2001 From: antoshkka Date: Thu, 24 Jul 2025 23:06:44 +0300 Subject: [PATCH 001/151] feat core: remove deprecated Statement() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI commit_hash:72a2e6ea20943579b7f719592584584fd152f2aa --- core/include/userver/storages/query.hpp | 3 --- ydb/src/ydb/table.cpp | 12 ++++++------ ydb/src/ydb/transaction.cpp | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/core/include/userver/storages/query.hpp b/core/include/userver/storages/query.hpp index e80f182c2d1d..7f98db676e22 100644 --- a/core/include/userver/storages/query.hpp +++ b/core/include/userver/storages/query.hpp @@ -80,9 +80,6 @@ class Query { /// @returns zero terminated view to the statement that is alive as long as *this is alive utils::zstring_view GetStatementView() const noexcept; - /// @deprecated Use GetStatementView() - const std::string& Statement() const { return dynamic_.statement_; } - /// @returns logging mode LogMode GetLogMode() const noexcept { return log_mode_; } diff --git a/ydb/src/ydb/table.cpp b/ydb/src/ydb/table.cpp index 8c3e0e37bafa..013beaf6015f 100644 --- a/ydb/src/ydb/table.cpp +++ b/ydb/src/ydb/table.cpp @@ -179,13 +179,13 @@ ScanQueryResults TableClient::ExecuteScanQuery( auto future = impl::RetryOperation( context, - [query = query.Statement(), + [query, params = std::move(builder).Build(), scan_settings = std::move(scan_settings), settings = context.settings, deadline = context.deadline](NYdb::NTable::TTableClient& table_client) mutable { impl::ApplyToRequestSettings(scan_settings, settings, deadline); - return table_client.StreamExecuteScanQuery(impl::ToString(query), params, scan_settings); + return table_client.StreamExecuteScanQuery(impl::ToString(query.GetStatementView()), params, scan_settings); } ); @@ -355,7 +355,7 @@ ExecuteResponse TableClient::ExecuteDataQuery( auto future = impl::RetryOperation( context, - [query = query.Statement(), + [query, params = std::move(builder).Build(), exec_settings = ToExecQuerySettings(query_settings), settings = context.settings, @@ -363,7 +363,7 @@ ExecuteResponse TableClient::ExecuteDataQuery( impl::ApplyToRequestSettings(exec_settings, settings, deadline); const auto tx_settings = PrepareTxSettings(settings); const auto tx = NYdb::NTable::TTxControl::BeginTx(tx_settings).CommitTx(); - return session.ExecuteDataQuery(impl::ToString(query), tx, params, exec_settings); + return session.ExecuteDataQuery(impl::ToString(query.GetStatementView()), tx, params, exec_settings); } ); @@ -392,7 +392,7 @@ ExecuteResponse TableClient::ExecuteQuery( auto future = impl::RetryQuery( context, - [query = query.Statement(), + [query, params = std::move(builder).Build(), exec_settings = std::move(exec_settings), settings = context.settings, @@ -400,7 +400,7 @@ ExecuteResponse TableClient::ExecuteQuery( impl::ApplyToRequestSettings(exec_settings, settings, deadline); const auto tx_settings = PrepareQueryTxSettings(settings); const auto tx = NYdb::NQuery::TTxControl::BeginTx(tx_settings).CommitTx(); - return session.ExecuteQuery(impl::ToString(query), tx, params, exec_settings); + return session.ExecuteQuery(impl::ToString(query.GetStatementView()), tx, params, exec_settings); } ); diff --git a/ydb/src/ydb/transaction.cpp b/ydb/src/ydb/transaction.cpp index 63f4fe6fe758..18b9b08f11a4 100644 --- a/ydb/src/ydb/transaction.cpp +++ b/ydb/src/ydb/transaction.cpp @@ -149,7 +149,7 @@ ExecuteResponse Transaction::Execute( auto error_guard = ErrorGuard(); auto execute_fut = ydb_tx_.GetSession().ExecuteDataQuery( - impl::ToString(query.Statement()), + impl::ToString(query.GetStatementView()), NYdb::NTable::TTxControl::Tx(ydb_tx_), std::move(internal_params), exec_settings From bd9ee6451b9a2eb01a7cf54e095a5bf0d25dc7e5 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Fri, 25 Jul 2025 10:43:38 +0300 Subject: [PATCH 002/151] feat libraries: replace deprecated GetName() with GetOptionalName() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI commit_hash:9b3b3c6ecb561848b82b4e6e0e48482751f6796d --- postgresql/include/userver/cache/base_postgres_cache.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/include/userver/cache/base_postgres_cache.hpp b/postgresql/include/userver/cache/base_postgres_cache.hpp index f047d3c1d630..5e4a603308c5 100644 --- a/postgresql/include/userver/cache/base_postgres_cache.hpp +++ b/postgresql/include/userver/cache/base_postgres_cache.hpp @@ -550,7 +550,7 @@ storages::postgres::Query PostgreCache::GetDeltaQuery() { const storages::postgres::Query query = PolicyCheckerType::GetQuery(); return storages::postgres::Query{ fmt::format("{} {} {}", query.GetStatementView(), GetDeltaWhereClause(), GetOrderByClause()), - query.GetName(), + query.GetOptionalName(), }; } else { return GetAllQuery(); From f971e07f345836efa3910fe2d0ff9eb91e612621 Mon Sep 17 00:00:00 2001 From: antonaksenov Date: Fri, 25 Jul 2025 11:48:26 +0300 Subject: [PATCH 003/151] refactor dynamic_config: move public testsuite classes from pytest plugin commit_hash:9c994de87e27a55450a990914265d4cc577d8658 --- .mapping.json | 1 + .../dynamic_configs/tests/test_changelog.py | 8 +- .../dynamic_configs/tests/test_examples.py | 6 +- .../tests_kill_switches/constants.py | 4 +- .../pytest_plugins/pytest_userver/dynconf.py | 387 ++++++++++++++++ .../pytest_userver/plugins/dynamic_config.py | 412 +----------------- 6 files changed, 417 insertions(+), 401 deletions(-) create mode 100644 testsuite/pytest_plugins/pytest_userver/dynconf.py diff --git a/.mapping.json b/.mapping.json index df75d12bbcf2..705837474a3e 100644 --- a/.mapping.json +++ b/.mapping.json @@ -4234,6 +4234,7 @@ "testsuite/pytest_plugins/pytest_userver/__init__.py":"taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/__init__.py", "testsuite/pytest_plugins/pytest_userver/chaos.py":"taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/chaos.py", "testsuite/pytest_plugins/pytest_userver/client.py":"taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/client.py", + "testsuite/pytest_plugins/pytest_userver/dynconf.py":"taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/dynconf.py", "testsuite/pytest_plugins/pytest_userver/grpc/__init__.py":"taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/grpc/__init__.py", "testsuite/pytest_plugins/pytest_userver/grpc/_mocked_errors.py":"taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/grpc/_mocked_errors.py", "testsuite/pytest_plugins/pytest_userver/grpc/_reflection.py":"taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/grpc/_reflection.py", diff --git a/core/functional_tests/dynamic_configs/tests/test_changelog.py b/core/functional_tests/dynamic_configs/tests/test_changelog.py index e0b581b5f52b..8f9383f3ddb6 100644 --- a/core/functional_tests/dynamic_configs/tests/test_changelog.py +++ b/core/functional_tests/dynamic_configs/tests/test_changelog.py @@ -1,18 +1,18 @@ import contextlib import pytest -from pytest_userver.plugins import dynamic_config +from pytest_userver import dynconf class ConfigHelper: def __init__(self, *, cache_invalidation_state, defaults): self.cache_invalidation_state = cache_invalidation_state self.defaults = defaults - self.changelog = dynamic_config._Changelog() + self.changelog = dynconf._Changelog() @contextlib.contextmanager def new_config(self): - config = dynamic_config.DynamicConfig( + config = dynconf.DynamicConfig( initial_values=self.defaults, defaults=None, config_cache_components=[], @@ -24,7 +24,7 @@ def new_config(self): def get_updated_since(self, config, timestamp): updates = self.changelog.get_updated_since( - dynamic_config._create_config_dict( + dynconf._create_config_dict( config.get_values_unsafe(), config.get_kill_switches_disabled_unsafe(), ), diff --git a/core/functional_tests/dynamic_configs/tests/test_examples.py b/core/functional_tests/dynamic_configs/tests/test_examples.py index 1299d1a79f01..a46a10dc8481 100644 --- a/core/functional_tests/dynamic_configs/tests/test_examples.py +++ b/core/functional_tests/dynamic_configs/tests/test_examples.py @@ -1,12 +1,12 @@ import pytest # /// [kill switch disabling using a pytest marker] -from pytest_userver.plugins import dynamic_config as dynamic_config_lib +from pytest_userver import dynconf @pytest.mark.config( - USERVER_LOG_REQUEST=dynamic_config_lib.USE_STATIC_DEFAULT, - USERVER_LOG_REQUEST_HEADERS=dynamic_config_lib.USE_STATIC_DEFAULT, + USERVER_LOG_REQUEST=dynconf.USE_STATIC_DEFAULT, + USERVER_LOG_REQUEST_HEADERS=dynconf.USE_STATIC_DEFAULT, ) async def test_something(service_client): # Test implementation diff --git a/core/functional_tests/dynamic_configs/tests_kill_switches/constants.py b/core/functional_tests/dynamic_configs/tests_kill_switches/constants.py index c8dbd0ca3ed8..8b8bc84612d2 100644 --- a/core/functional_tests/dynamic_configs/tests_kill_switches/constants.py +++ b/core/functional_tests/dynamic_configs/tests_kill_switches/constants.py @@ -1,8 +1,8 @@ import pytest -from pytest_userver.plugins import dynamic_config as dynamic_config_lib +from pytest_userver import dynconf CONFIG_KEY = 'TEST_CONFIG_KEY' NEW_VALUE = 'new_value' -KILL_SWITCH_DISABLED_MARK = pytest.mark.config(**{CONFIG_KEY: dynamic_config_lib.USE_STATIC_DEFAULT}) +KILL_SWITCH_DISABLED_MARK = pytest.mark.config(**{CONFIG_KEY: dynconf.USE_STATIC_DEFAULT}) diff --git a/testsuite/pytest_plugins/pytest_userver/dynconf.py b/testsuite/pytest_plugins/pytest_userver/dynconf.py new file mode 100644 index 000000000000..91172046c678 --- /dev/null +++ b/testsuite/pytest_plugins/pytest_userver/dynconf.py @@ -0,0 +1,387 @@ +""" +Python module that provides classes and constants +that may be useful when working with the +@ref dynamic_config_testsuite "dynamic config in testsuite". + +@ingroup userver_testsuite +""" + +import contextlib +import copy +import dataclasses +import datetime +from typing import Any +from typing import Dict +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple + +from pytest_userver.plugins import caches + + +class BaseError(Exception): + """Base class for exceptions from this module""" + + +class DynamicConfigNotFoundError(BaseError): + """Config parameter was not found and no default was provided""" + + +class DynamicConfigUninitialized(BaseError): + """ + Calling `dynamic_config.get` before defaults are fetched from the service. + Try adding a dependency on `service_client` in your fixture. + """ + + +class InvalidDefaultsError(BaseError): + """Dynamic config defaults action returned invalid response""" + + +class UnknownConfigError(BaseError): + """Invalid dynamic config name in `@pytest.mark.config`""" + + +ConfigValuesDict = Dict[str, Any] + + +class _RemoveKey: + pass + + +_REMOVE_KEY = _RemoveKey() + + +class _Missing: + pass + + +_MISSING = _Missing() + + +@dataclasses.dataclass(frozen=True) +class _ConfigEntry: + value: Any + static_default_preferred: bool + + +_ConfigDict = Dict[str, _ConfigEntry | _RemoveKey] + + +def _create_config_dict(values: ConfigValuesDict, kill_switches_disabled: Optional[Set[str]] = None) -> _ConfigDict: + if kill_switches_disabled is None: + kill_switches_disabled = set() + + result = {} + for key, value in values.items(): + static_default_preferred = key in kill_switches_disabled + result[key] = _ConfigEntry(value, static_default_preferred) + return result + + +@dataclasses.dataclass +class _ChangelogEntry: + timestamp: str + dirty_keys: Set[str] + state: _ConfigDict + prev_state: _ConfigDict + + @classmethod + def new( + cls, + *, + previous: Optional['_ChangelogEntry'], + timestamp: str, + ) -> '_ChangelogEntry': + if previous: + prev_state = previous.state + else: + prev_state = {} + return cls( + timestamp=timestamp, + dirty_keys=set(), + state=prev_state.copy(), + prev_state=prev_state, + ) + + @property + def has_changes(self) -> bool: + return bool(self.dirty_keys) + + def update(self, values: _ConfigDict): + for key, value in values.items(): + if value == self.prev_state.get(key, _MISSING): + self.dirty_keys.discard(key) + else: + self.dirty_keys.add(key) + self.state.update(values) + + +@dataclasses.dataclass(frozen=True) +class _Updates: + timestamp: str + values: ConfigValuesDict + removed: List[str] + kill_switches_disabled: List[str] + + def is_empty(self) -> bool: + return not self.values and not self.removed + + +class _Changelog: + timestamp: datetime.datetime + committed_entries: List[_ChangelogEntry] + staged_entry: _ChangelogEntry + + def __init__(self): + self.timestamp = datetime.datetime.fromtimestamp( + 0, + datetime.timezone.utc, + ) + self.committed_entries = [] + self.staged_entry = _ChangelogEntry.new( + timestamp=self.service_timestamp(), + previous=None, + ) + + def service_timestamp(self) -> str: + return self.timestamp.strftime('%Y-%m-%dT%H:%M:%SZ') + + def next_timestamp(self) -> str: + self.timestamp += datetime.timedelta(seconds=1) + return self.service_timestamp() + + def commit(self) -> _ChangelogEntry: + """Commit staged changed if any and return last committed entry.""" + entry = self.staged_entry + if entry.has_changes or not self.committed_entries: + self.staged_entry = _ChangelogEntry.new( + timestamp=self.next_timestamp(), + previous=entry, + ) + self.committed_entries.append(entry) + return self.committed_entries[-1] + + def get_updated_since( + self, + config_dict: _ConfigDict, + updated_since: str, + ids: Optional[List[str]] = None, + ) -> _Updates: + entry = self.commit() + config_dict, removed = self._get_updated_since(config_dict, updated_since) + if ids: + config_dict = {name: config_dict[name] for name in ids if name in config_dict} + removed = [name for name in removed if name in ids] + + values = {} + kill_switches_disabled = [] + for name, config_entry in config_dict.items(): + values[name] = config_entry.value + if config_entry.static_default_preferred: + kill_switches_disabled.append(name) + + return _Updates( + timestamp=entry.timestamp, + values=values, + removed=removed, + kill_switches_disabled=kill_switches_disabled, + ) + + def _get_updated_since( + self, + config_dict: _ConfigDict, + updated_since: str, + ) -> Tuple[_ConfigDict, List[str]]: + if not updated_since: + return config_dict, [] + dirty_keys = set() + last_known_state = {} + for entry in reversed(self.committed_entries): + if entry.timestamp > updated_since: + dirty_keys.update(entry.dirty_keys) + else: + if entry.timestamp == updated_since: + last_known_state = entry.state + break + # We don't want to send them again + result = {} + removed = [] + for key in dirty_keys: + config_entry = config_dict.get(key, _REMOVE_KEY) + if last_known_state.get(key, _MISSING) != config_entry: + if config_entry is _REMOVE_KEY: + removed.append(key) + else: + result[key] = config_entry + return result, removed + + def add_entries(self, config_dict: _ConfigDict) -> None: + self.staged_entry.update(config_dict) + + @contextlib.contextmanager + def rollback(self, defaults: ConfigValuesDict) -> Iterator[None]: + try: + yield + finally: + self._do_rollback(defaults) + + def _do_rollback(self, defaults: ConfigValuesDict) -> None: + if not self.committed_entries: + return + + maybe_dirty = set() + for entry in self.committed_entries: + maybe_dirty.update(entry.dirty_keys) + + last = self.committed_entries[-1] + last_state = last.state + config_dict = _create_config_dict(defaults) + dirty_keys = set() + reverted = {} + for key in maybe_dirty: + original = config_dict.get(key, _REMOVE_KEY) + if last_state[key] != original: + dirty_keys.add(key) + reverted[key] = original + + entry = _ChangelogEntry( + timestamp=last.timestamp, + state=last.state, + dirty_keys=dirty_keys, + prev_state={}, + ) + self.committed_entries = [entry] + self.staged_entry = _ChangelogEntry( + timestamp=self.staged_entry.timestamp, + dirty_keys=dirty_keys.copy(), + state=reverted, + prev_state=entry.state, + ) + + +class DynamicConfig: + """ + @brief Simple dynamic config backend. + + @see @ref pytest_userver.plugins.dynamic_config.dynamic_config "dynamic_config" + """ + + def __init__( + self, + *, + initial_values: ConfigValuesDict, + defaults: Optional[ConfigValuesDict], + config_cache_components: Iterable[str], + cache_invalidation_state: caches.InvalidationState, + changelog: _Changelog, + ) -> None: + self._values = initial_values.copy() + self._kill_switches_disabled = set() + # Defaults are only there for convenience, to allow accessing them + # in tests using dynamic_config.get. They are not sent to the service. + self._defaults = defaults + self._cache_invalidation_state = cache_invalidation_state + self._config_cache_components = config_cache_components + self._changelog = changelog + + def set_values(self, values: ConfigValuesDict) -> None: + self.set_values_unsafe(copy.deepcopy(values)) + + def set_values_unsafe(self, values: ConfigValuesDict) -> None: + self._values.update(values) + for key in values: + self._kill_switches_disabled.discard(key) + + config_dict = _create_config_dict(values) + self._changelog.add_entries(config_dict) + self._sync_with_service() + + def set(self, **values) -> None: + self.set_values(values) + + def switch_to_static_default(self, *keys: str) -> None: + for key in keys: + self._kill_switches_disabled.add(key) + + config_dict = _create_config_dict( + values={key: self._values.get(key, None) for key in keys}, kill_switches_disabled=set(keys) + ) + self._changelog.add_entries(config_dict) + self._sync_with_service() + + def switch_to_dynamic_value(self, *keys: str) -> None: + for key in keys: + self._kill_switches_disabled.discard(key) + + config_dict = _create_config_dict(values={key: self._values[key] for key in keys if key in self._values}) + self._changelog.add_entries(config_dict) + self._sync_with_service() + + def get_values_unsafe(self) -> ConfigValuesDict: + return self._values + + def get_kill_switches_disabled_unsafe(self) -> Set[str]: + return self._kill_switches_disabled + + def get(self, key: str, default: Any = None) -> Any: + if key in self._values and key not in self._kill_switches_disabled: + return copy.deepcopy(self._values[key]) + if self._defaults is not None and key in self._defaults: + return copy.deepcopy(self._defaults[key]) + if default is not None: + return default + if self._defaults is None: + raise DynamicConfigUninitialized( + f'Defaults for config {key!r} have not yet been fetched ' + 'from the service. Options:\n' + '1. add a dependency on service_client in your fixture;\n' + '2. pass `default` parameter to `dynamic_config.get`', + ) + raise DynamicConfigNotFoundError(f'Config {key!r} is not found') + + def remove_values(self, keys: Iterable[str]) -> None: + extra_keys = set(keys).difference(self._values.keys()) + if extra_keys: + raise DynamicConfigNotFoundError( + f'Attempting to remove nonexistent configs: {extra_keys}', + ) + for key in keys: + self._values.pop(key) + self._kill_switches_disabled.discard(key) + + self._changelog.add_entries({key: _REMOVE_KEY for key in keys}) + self._sync_with_service() + + def remove(self, key: str) -> None: + return self.remove_values([key]) + + @contextlib.contextmanager + def modify(self, key: str) -> Any: + value = self.get(key) + yield value + self.set_values({key: value}) + + @contextlib.contextmanager + def modify_many( + self, + *keys: Tuple[str, ...], + ) -> Tuple[Any, ...]: + values = tuple(self.get(key) for key in keys) + yield values + self.set_values(dict(zip(keys, values))) + + def _sync_with_service(self) -> None: + self._cache_invalidation_state.invalidate( + self._config_cache_components, + ) + + +class UseStaticDefault: + pass + + +USE_STATIC_DEFAULT = UseStaticDefault() diff --git a/testsuite/pytest_plugins/pytest_userver/plugins/dynamic_config.py b/testsuite/pytest_plugins/pytest_userver/plugins/dynamic_config.py index 520e7c2cb98c..47244406e0ab 100644 --- a/testsuite/pytest_plugins/pytest_userver/plugins/dynamic_config.py +++ b/testsuite/pytest_plugins/pytest_userver/plugins/dynamic_config.py @@ -3,25 +3,16 @@ """ # pylint: disable=redefined-outer-name -import contextlib -import copy import dataclasses -import datetime import json import pathlib -from typing import Any from typing import Callable -from typing import Dict from typing import Iterable -from typing import Iterator -from typing import List from typing import Optional -from typing import Set -from typing import Tuple import pytest -from pytest_userver.plugins import caches +from pytest_userver import dynconf USERVER_CONFIG_HOOKS = [ 'userver_config_dynconf_cache', @@ -33,371 +24,6 @@ } -class BaseError(Exception): - """Base class for exceptions from this module""" - - -class DynamicConfigNotFoundError(BaseError): - """Config parameter was not found and no default was provided""" - - -class DynamicConfigUninitialized(BaseError): - """ - Calling `dynamic_config.get` before defaults are fetched from the service. - Try adding a dependency on `service_client` in your fixture. - """ - - -class InvalidDefaultsError(BaseError): - """Dynamic config defaults action returned invalid response""" - - -class UnknownConfigError(BaseError): - """Invalid dynamic config name in `@pytest.mark.config`""" - - -ConfigValuesDict = Dict[str, Any] - - -class _RemoveKey: - pass - - -_REMOVE_KEY = _RemoveKey() - - -class _Missing: - pass - - -_MISSING = _Missing() - - -@dataclasses.dataclass(frozen=True) -class _ConfigEntry: - value: Any - static_default_preferred: bool - - -_ConfigDict = Dict[str, _ConfigEntry | _RemoveKey] - - -def _create_config_dict(values: ConfigValuesDict, kill_switches_disabled: Optional[Set[str]] = None) -> _ConfigDict: - if kill_switches_disabled is None: - kill_switches_disabled = set() - - result = {} - for key, value in values.items(): - static_default_preferred = key in kill_switches_disabled - result[key] = _ConfigEntry(value, static_default_preferred) - return result - - -@dataclasses.dataclass -class _ChangelogEntry: - timestamp: str - dirty_keys: Set[str] - state: _ConfigDict - prev_state: _ConfigDict - - @classmethod - def new( - cls, - *, - previous: Optional['_ChangelogEntry'], - timestamp: str, - ) -> '_ChangelogEntry': - if previous: - prev_state = previous.state - else: - prev_state = {} - return cls( - timestamp=timestamp, - dirty_keys=set(), - state=prev_state.copy(), - prev_state=prev_state, - ) - - @property - def has_changes(self) -> bool: - return bool(self.dirty_keys) - - def update(self, values: _ConfigDict): - for key, value in values.items(): - if value == self.prev_state.get(key, _MISSING): - self.dirty_keys.discard(key) - else: - self.dirty_keys.add(key) - self.state.update(values) - - -@dataclasses.dataclass(frozen=True) -class _Updates: - timestamp: str - values: ConfigValuesDict - removed: List[str] - kill_switches_disabled: List[str] - - def is_empty(self) -> bool: - return not self.values and not self.removed - - -class _Changelog: - timestamp: datetime.datetime - committed_entries: List[_ChangelogEntry] - staged_entry: _ChangelogEntry - - def __init__(self): - self.timestamp = datetime.datetime.fromtimestamp( - 0, - datetime.timezone.utc, - ) - self.committed_entries = [] - self.staged_entry = _ChangelogEntry.new( - timestamp=self.service_timestamp(), - previous=None, - ) - - def service_timestamp(self) -> str: - return self.timestamp.strftime('%Y-%m-%dT%H:%M:%SZ') - - def next_timestamp(self) -> str: - self.timestamp += datetime.timedelta(seconds=1) - return self.service_timestamp() - - def commit(self) -> _ChangelogEntry: - """Commit staged changed if any and return last committed entry.""" - entry = self.staged_entry - if entry.has_changes or not self.committed_entries: - self.staged_entry = _ChangelogEntry.new( - timestamp=self.next_timestamp(), - previous=entry, - ) - self.committed_entries.append(entry) - return self.committed_entries[-1] - - def get_updated_since( - self, - config_dict: _ConfigDict, - updated_since: str, - ids: Optional[List[str]] = None, - ) -> _Updates: - entry = self.commit() - config_dict, removed = self._get_updated_since(config_dict, updated_since) - if ids: - config_dict = {name: config_dict[name] for name in ids if name in config_dict} - removed = [name for name in removed if name in ids] - - values = {} - kill_switches_disabled = [] - for name, config_entry in config_dict.items(): - values[name] = config_entry.value - if config_entry.static_default_preferred: - kill_switches_disabled.append(name) - - return _Updates( - timestamp=entry.timestamp, - values=values, - removed=removed, - kill_switches_disabled=kill_switches_disabled, - ) - - def _get_updated_since( - self, - config_dict: _ConfigDict, - updated_since: str, - ) -> Tuple[_ConfigDict, List[str]]: - if not updated_since: - return config_dict, [] - dirty_keys = set() - last_known_state = {} - for entry in reversed(self.committed_entries): - if entry.timestamp > updated_since: - dirty_keys.update(entry.dirty_keys) - else: - if entry.timestamp == updated_since: - last_known_state = entry.state - break - # We don't want to send them again - result = {} - removed = [] - for key in dirty_keys: - config_entry = config_dict.get(key, _REMOVE_KEY) - if last_known_state.get(key, _MISSING) != config_entry: - if config_entry is _REMOVE_KEY: - removed.append(key) - else: - result[key] = config_entry - return result, removed - - def add_entries(self, config_dict: _ConfigDict) -> None: - self.staged_entry.update(config_dict) - - @contextlib.contextmanager - def rollback(self, defaults: ConfigValuesDict) -> Iterator[None]: - try: - yield - finally: - self._do_rollback(defaults) - - def _do_rollback(self, defaults: ConfigValuesDict) -> None: - if not self.committed_entries: - return - - maybe_dirty = set() - for entry in self.committed_entries: - maybe_dirty.update(entry.dirty_keys) - - last = self.committed_entries[-1] - last_state = last.state - config_dict = _create_config_dict(defaults) - dirty_keys = set() - reverted = {} - for key in maybe_dirty: - original = config_dict.get(key, _REMOVE_KEY) - if last_state[key] != original: - dirty_keys.add(key) - reverted[key] = original - - entry = _ChangelogEntry( - timestamp=last.timestamp, - state=last.state, - dirty_keys=dirty_keys, - prev_state={}, - ) - self.committed_entries = [entry] - self.staged_entry = _ChangelogEntry( - timestamp=self.staged_entry.timestamp, - dirty_keys=dirty_keys.copy(), - state=reverted, - prev_state=entry.state, - ) - - -class DynamicConfig: - """ - @brief Simple dynamic config backend. - - @see @ref pytest_userver.plugins.dynamic_config.dynamic_config "dynamic_config" - """ - - def __init__( - self, - *, - initial_values: ConfigValuesDict, - defaults: Optional[ConfigValuesDict], - config_cache_components: Iterable[str], - cache_invalidation_state: caches.InvalidationState, - changelog: _Changelog, - ) -> None: - self._values = initial_values.copy() - self._kill_switches_disabled = set() - # Defaults are only there for convenience, to allow accessing them - # in tests using dynamic_config.get. They are not sent to the service. - self._defaults = defaults - self._cache_invalidation_state = cache_invalidation_state - self._config_cache_components = config_cache_components - self._changelog = changelog - - def set_values(self, values: ConfigValuesDict) -> None: - self.set_values_unsafe(copy.deepcopy(values)) - - def set_values_unsafe(self, values: ConfigValuesDict) -> None: - self._values.update(values) - for key in values: - self._kill_switches_disabled.discard(key) - - config_dict = _create_config_dict(values) - self._changelog.add_entries(config_dict) - self._sync_with_service() - - def set(self, **values) -> None: - self.set_values(values) - - def switch_to_static_default(self, *keys: str) -> None: - for key in keys: - self._kill_switches_disabled.add(key) - - config_dict = _create_config_dict( - values={key: self._values.get(key, None) for key in keys}, kill_switches_disabled=set(keys) - ) - self._changelog.add_entries(config_dict) - self._sync_with_service() - - def switch_to_dynamic_value(self, *keys: str) -> None: - for key in keys: - self._kill_switches_disabled.discard(key) - - config_dict = _create_config_dict(values={key: self._values[key] for key in keys if key in self._values}) - self._changelog.add_entries(config_dict) - self._sync_with_service() - - def get_values_unsafe(self) -> ConfigValuesDict: - return self._values - - def get_kill_switches_disabled_unsafe(self) -> Set[str]: - return self._kill_switches_disabled - - def get(self, key: str, default: Any = None) -> Any: - if key in self._values and key not in self._kill_switches_disabled: - return copy.deepcopy(self._values[key]) - if self._defaults is not None and key in self._defaults: - return copy.deepcopy(self._defaults[key]) - if default is not None: - return default - if self._defaults is None: - raise DynamicConfigUninitialized( - f'Defaults for config {key!r} have not yet been fetched ' - 'from the service. Options:\n' - '1. add a dependency on service_client in your fixture;\n' - '2. pass `default` parameter to `dynamic_config.get`', - ) - raise DynamicConfigNotFoundError(f'Config {key!r} is not found') - - def remove_values(self, keys: Iterable[str]) -> None: - extra_keys = set(keys).difference(self._values.keys()) - if extra_keys: - raise DynamicConfigNotFoundError( - f'Attempting to remove nonexistent configs: {extra_keys}', - ) - for key in keys: - self._values.pop(key) - self._kill_switches_disabled.discard(key) - - self._changelog.add_entries({key: _REMOVE_KEY for key in keys}) - self._sync_with_service() - - def remove(self, key: str) -> None: - return self.remove_values([key]) - - @contextlib.contextmanager - def modify(self, key: str) -> Any: - value = self.get(key) - yield value - self.set_values({key: value}) - - @contextlib.contextmanager - def modify_many( - self, - *keys: Tuple[str, ...], - ) -> Tuple[Any, ...]: - values = tuple(self.get(key) for key in keys) - yield values - self.set_values(dict(zip(keys, values))) - - def _sync_with_service(self) -> None: - self._cache_invalidation_state.invalidate( - self._config_cache_components, - ) - - -class UseStaticDefault: - pass - - -USE_STATIC_DEFAULT = UseStaticDefault() - - @pytest.fixture def dynamic_config( request, @@ -409,7 +35,7 @@ def dynamic_config( dynamic_config_changelog, _dynconf_load_json_cached, dynconf_cache_names, -) -> DynamicConfig: +) -> dynconf.DynamicConfig: """ Fixture that allows to control dynamic config values used by the service. @@ -434,7 +60,7 @@ def dynamic_config( @ingroup userver_testsuite_fixtures """ - config = DynamicConfig( + config = dynconf.DynamicConfig( initial_values=config_service_defaults, defaults=_dynamic_config_defaults_storage.snapshot, config_cache_components=dynconf_cache_names, @@ -449,7 +75,7 @@ def dynamic_config( updates.update(values) for marker in request.node.iter_markers('config'): value_update_kwargs = { - key: value for key, value in marker.kwargs.items() if value is not USE_STATIC_DEFAULT + key: value for key, value in marker.kwargs.items() if value is not dynconf.USE_STATIC_DEFAULT } value_updates_json = object_substitute(value_update_kwargs) updates.update(value_updates_json) @@ -457,7 +83,9 @@ def dynamic_config( kill_switches_disabled = [] for marker in request.node.iter_markers('config'): - kill_switches_disabled.extend(key for key, value in marker.kwargs.items() if value is USE_STATIC_DEFAULT) + kill_switches_disabled.extend( + key for key, value in marker.kwargs.items() if value is dynconf.USE_STATIC_DEFAULT + ) config.switch_to_static_default(*kill_switches_disabled) yield config @@ -501,7 +129,7 @@ def load(path: pathlib.Path): @pytest.fixture -def taxi_config(dynamic_config) -> DynamicConfig: +def taxi_config(dynamic_config) -> dynconf.DynamicConfig: """ Deprecated, use `dynamic_config` instead. """ @@ -509,7 +137,7 @@ def taxi_config(dynamic_config) -> DynamicConfig: @pytest.fixture(scope='session') -def dynamic_config_fallback_patch() -> ConfigValuesDict: +def dynamic_config_fallback_patch() -> dynconf.ConfigValuesDict: """ Override this fixture to replace some dynamic config values specifically for testsuite tests: @@ -529,7 +157,7 @@ def dynamic_config_fallback_patch(): def config_service_defaults( config_fallback_path, dynamic_config_fallback_patch, -) -> ConfigValuesDict: +) -> dynconf.ConfigValuesDict: """ Fixture that returns default values for dynamic config. You may override it in your local conftest.py or fixture: @@ -560,13 +188,13 @@ def config_service_defaults(): @dataclasses.dataclass(frozen=False) class _ConfigDefaults: - snapshot: Optional[ConfigValuesDict] + snapshot: Optional[dynconf.ConfigValuesDict] async def update(self, client, dynamic_config) -> None: if self.snapshot is None: defaults = await client.get_dynamic_config_defaults() if not isinstance(defaults, dict): - raise InvalidDefaultsError() + raise dynconf.InvalidDefaultsError() self.snapshot = defaults # pylint:disable=protected-access dynamic_config._defaults = defaults @@ -675,11 +303,11 @@ def _patch_config(config, _config_vars) -> None: # @cond -# TODO publish _Changelog and document how to use it in custom config service +# TODO publish dynconf._Changelog and document how to use it in custom config service # mocks. @pytest.fixture(scope='session') -def dynamic_config_changelog() -> _Changelog: - return _Changelog() +def dynamic_config_changelog() -> dynconf._Changelog: + return dynconf._Changelog() # @endcond @@ -688,8 +316,8 @@ def dynamic_config_changelog() -> _Changelog: @pytest.fixture def mock_configs_service( mockserver, - dynamic_config: DynamicConfig, - dynamic_config_changelog: _Changelog, + dynamic_config: dynconf.DynamicConfig, + dynamic_config_changelog: dynconf._Changelog, ) -> None: """ Adds a mockserver handler that forwards dynamic_config to service's @@ -701,7 +329,7 @@ def mock_configs_service( @mockserver.json_handler('/configs-service/configs/values') def _mock_configs(request): updates = dynamic_config_changelog.get_updated_since( - _create_config_dict( + dynconf._create_config_dict( dynamic_config.get_values_unsafe(), dynamic_config.get_kill_switches_disabled_unsafe(), ), @@ -725,7 +353,7 @@ def _mock_configs_status(_request): @pytest.fixture -def _userver_dynconfig_cache_control(dynamic_config_changelog: _Changelog): +def _userver_dynconfig_cache_control(dynamic_config_changelog: dynconf._Changelog): def cache_control(updater, timestamp): entry = dynamic_config_changelog.commit() if entry.timestamp == timestamp: @@ -768,6 +396,6 @@ def check(): message = _CHECK_CONFIG_ERROR.format( ', '.join(f'{key}=...' for key in sorted(unknown_configs)), ) - raise UnknownConfigError(message) + raise dynconf.UnknownConfigError(message) return check From 9c399437c148dcb399db6e5eb7aa86c1330996d1 Mon Sep 17 00:00:00 2001 From: miglitskii Date: Fri, 25 Jul 2025 11:49:01 +0300 Subject: [PATCH 004/151] feat userver: set span level=debug for set_config queries commit_hash:6d87140a7bb14147e27868001d98a04627ef41c3 --- .../postgres/detail/connection_impl.cpp | 17 +++++++++++++---- .../postgres/detail/connection_impl.hpp | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/postgresql/src/storages/postgres/detail/connection_impl.cpp b/postgresql/src/storages/postgres/detail/connection_impl.cpp index a44a8aed9bd2..c349467363da 100644 --- a/postgresql/src/storages/postgres/detail/connection_impl.cpp +++ b/postgresql/src/storages/postgres/detail/connection_impl.cpp @@ -857,7 +857,12 @@ ResultSet ConnectionImpl::ExecuteCommand(const Query& query, engine::Deadline de return ExecuteCommand(query, kNoParams, deadline); } -ResultSet ConnectionImpl::ExecuteCommand(const Query& query, const QueryParameters& params, engine::Deadline deadline) { +ResultSet ConnectionImpl::ExecuteCommand( + const Query& query, + const QueryParameters& params, + engine::Deadline deadline, + logging::Level span_log_level +) { if (settings_.prepared_statements == ConnectionSettings::kNoPreparedStatements) { return ExecuteCommandNoPrepare(query, params, deadline); } @@ -871,6 +876,8 @@ ResultSet ConnectionImpl::ExecuteCommand(const Query& query, const QueryParamete CheckDeadlineReached(deadline); const auto network_timeout = std::chrono::duration_cast(deadline.TimeLeft()); auto span = MakeQuerySpan(query, {network_timeout, GetStatementTimeout()}); + span.SetLogLevel(span_log_level); + if (testsuite::AreTestpointsAvailable() && query.GetOptionalNameView()) { ReportStatement(*query.GetOptionalNameView()); } @@ -978,13 +985,15 @@ void ConnectionImpl::SendCommandNoPrepare(const Query& query, engine::Deadline d void ConnectionImpl::SendCommandNoPrepare( const Query& query, const QueryParameters& params, - engine::Deadline deadline + engine::Deadline deadline, + logging::Level span_log_level ) { CheckBusy(); CheckDeadlineReached(deadline); auto span = MakeQuerySpan( query, {std::chrono::duration_cast(deadline.TimeLeft()), GetStatementTimeout()} ); + span.SetLogLevel(span_log_level); auto scope = span.CreateScopeTime(); ++stats_.execute_total; conn_wrapper_.SendQuery(query.GetStatementView(), params, scope); @@ -1002,9 +1011,9 @@ void ConnectionImpl::SetParameter( StaticQueryParameters<3> params; params.Write(db_types_, name, value, is_transaction_scope); if (IsPipelineActive()) { - SendCommandNoPrepare(kSetConfigQuery, detail::QueryParameters{params}, deadline); + SendCommandNoPrepare(kSetConfigQuery, detail::QueryParameters{params}, deadline, logging::Level::kDebug); } else { - ExecuteCommand(kSetConfigQuery, detail::QueryParameters{params}, deadline); + ExecuteCommand(kSetConfigQuery, detail::QueryParameters{params}, deadline, logging::Level::kDebug); } } diff --git a/postgresql/src/storages/postgres/detail/connection_impl.hpp b/postgresql/src/storages/postgres/detail/connection_impl.hpp index 601b2779e0e6..9e20ab3d1c6d 100644 --- a/postgresql/src/storages/postgres/detail/connection_impl.hpp +++ b/postgresql/src/storages/postgres/detail/connection_impl.hpp @@ -169,7 +169,12 @@ class ConnectionImpl { ResultSet ExecuteCommand(const Query& query, engine::Deadline deadline); - ResultSet ExecuteCommand(const Query& query, const detail::QueryParameters& params, engine::Deadline deadline); + ResultSet ExecuteCommand( + const Query& query, + const detail::QueryParameters& params, + engine::Deadline deadline, + logging::Level span_log_level = logging::Level::kInfo + ); ResultSet ExecuteCommandNoPrepare(const Query& query, engine::Deadline deadline); @@ -177,7 +182,12 @@ class ConnectionImpl { void SendCommandNoPrepare(const Query& query, engine::Deadline deadline); - void SendCommandNoPrepare(const Query& query, const QueryParameters& params, engine::Deadline deadline); + void SendCommandNoPrepare( + const Query& query, + const QueryParameters& params, + engine::Deadline deadline, + logging::Level span_log_level = logging::Level::kInfo + ); void SetParameter( std::string_view name, From 60d6f445cc07fe6a4806ced9326ea447b1d47c4c Mon Sep 17 00:00:00 2001 From: korsunandrei Date: Fri, 25 Jul 2025 16:04:11 +0300 Subject: [PATCH 005/151] feat userver/postgresql: add prepared statements override toggle in command control commit_hash:ce0f4f1d92e71f6d8c8337d6a372b1562a5e72fd --- .../POSTGRES_DEFAULT_COMMAND_CONTROL.yaml | 2 + .../POSTGRES_HANDLERS_COMMAND_CONTROL.yaml | 2 + .../POSTGRES_QUERIES_COMMAND_CONTROL.yaml | 2 + .../userver/storages/postgres/notify.hpp | 2 +- .../userver/storages/postgres/options.hpp | 17 ++++- .../userver/storages/postgres/portal.hpp | 2 +- .../userver/storages/postgres/query_queue.hpp | 2 +- .../postgres/detail/connection_impl.cpp | 26 ++++++- .../postgres/detail/connection_impl.hpp | 5 +- .../src/storages/postgres/postgres_config.cpp | 7 +- .../postgres/tests/connection_pgtest.cpp | 60 +++++++-------- .../storages/postgres/tests/pool_pgtest.cpp | 74 +++++++++++++++++++ 12 files changed, 160 insertions(+), 41 deletions(-) diff --git a/postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml b/postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml index f3f200f58917..88c22572c1cb 100644 --- a/postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml +++ b/postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml @@ -10,3 +10,5 @@ schema: statement_timeout_ms: type: integer minimum: 1 + prepared_statements_enabled: + type: boolean diff --git a/postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml b/postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml index af0d2de1d0f6..6fa2265f91b6 100644 --- a/postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml +++ b/postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml @@ -19,3 +19,5 @@ schema: statement_timeout_ms: type: integer minimum: 1 + prepared_statements_enabled: + type: boolean diff --git a/postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml b/postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml index 717e51d18c5d..a8ad09107f50 100644 --- a/postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml +++ b/postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml @@ -15,3 +15,5 @@ schema: statement_timeout_ms: type: integer minimum: 1 + prepared_statements_enabled: + type: boolean diff --git a/postgresql/include/userver/storages/postgres/notify.hpp b/postgresql/include/userver/storages/postgres/notify.hpp index 8913ee1852ea..74c2c8d627bb 100644 --- a/postgresql/include/userver/storages/postgres/notify.hpp +++ b/postgresql/include/userver/storages/postgres/notify.hpp @@ -51,7 +51,7 @@ class [[nodiscard]] NotifyScope final { private: struct Impl; - USERVER_NAMESPACE::utils::FastPimpl pimpl_; + USERVER_NAMESPACE::utils::FastPimpl pimpl_; }; } // namespace storages::postgres diff --git a/postgresql/include/userver/storages/postgres/options.hpp b/postgresql/include/userver/storages/postgres/options.hpp index 21f52841a21b..aec71b13d6a1 100644 --- a/postgresql/include/userver/storages/postgres/options.hpp +++ b/postgresql/include/userver/storages/postgres/options.hpp @@ -112,15 +112,26 @@ struct CommandControl { /// PostgreSQL server-side timeout TimeoutDuration statement_timeout_ms{}; - constexpr CommandControl(TimeoutDuration network_timeout_ms, TimeoutDuration statement_timeout_ms) - : network_timeout_ms(network_timeout_ms), statement_timeout_ms(statement_timeout_ms) {} + enum class PreparedStatementsOptionOverride { kNoOverride, kEnabled, kDisabled }; + + PreparedStatementsOptionOverride prepared_statements_enabled{PreparedStatementsOptionOverride::kNoOverride}; + + constexpr CommandControl( + TimeoutDuration network_timeout_ms, + TimeoutDuration statement_timeout_ms, + PreparedStatementsOptionOverride prepared_statements_enabled = PreparedStatementsOptionOverride::kNoOverride + ) + : network_timeout_ms(network_timeout_ms), + statement_timeout_ms(statement_timeout_ms), + prepared_statements_enabled(prepared_statements_enabled) {} constexpr CommandControl WithExecuteTimeout(TimeoutDuration n) const noexcept { return {n, statement_timeout_ms}; } constexpr CommandControl WithStatementTimeout(TimeoutDuration s) const noexcept { return {network_timeout_ms, s}; } bool operator==(const CommandControl& rhs) const { - return network_timeout_ms == rhs.network_timeout_ms && statement_timeout_ms == rhs.statement_timeout_ms; + return network_timeout_ms == rhs.network_timeout_ms && statement_timeout_ms == rhs.statement_timeout_ms && + prepared_statements_enabled == rhs.prepared_statements_enabled; } bool operator!=(const CommandControl& rhs) const { return !(*this == rhs); } diff --git a/postgresql/include/userver/storages/postgres/portal.hpp b/postgresql/include/userver/storages/postgres/portal.hpp index e314c0b4bcf9..a5ff72c810a6 100644 --- a/postgresql/include/userver/storages/postgres/portal.hpp +++ b/postgresql/include/userver/storages/postgres/portal.hpp @@ -57,7 +57,7 @@ class Portal { static bool IsSupportedByDriver() noexcept; private: - static constexpr std::size_t kImplSize = 88; + static constexpr std::size_t kImplSize = 96; static constexpr std::size_t kImplAlign = 8; struct Impl; diff --git a/postgresql/include/userver/storages/postgres/query_queue.hpp b/postgresql/include/userver/storages/postgres/query_queue.hpp index d42b7e0ee113..0bcf555bac0c 100644 --- a/postgresql/include/userver/storages/postgres/query_queue.hpp +++ b/postgresql/include/userver/storages/postgres/query_queue.hpp @@ -99,7 +99,7 @@ class QueryQueue final { detail::ConnectionPtr conn_; struct QueriesStorage; - USERVER_NAMESPACE::utils::FastPimpl queries_storage_; + USERVER_NAMESPACE::utils::FastPimpl queries_storage_; }; template diff --git a/postgresql/src/storages/postgres/detail/connection_impl.cpp b/postgresql/src/storages/postgres/detail/connection_impl.cpp index c349467363da..cb4649b81806 100644 --- a/postgresql/src/storages/postgres/detail/connection_impl.cpp +++ b/postgresql/src/storages/postgres/detail/connection_impl.cpp @@ -383,8 +383,11 @@ ResultSet ConnectionImpl::ExecuteCommand( } auto deadline = testsuite_pg_ctl_.MakeExecuteDeadline(NetworkTimeout(statement_cmd_ctl)); - SetStatementTimeout(std::move(statement_cmd_ctl)); - return ExecuteCommand(query, params, deadline); + SetStatementTimeout(statement_cmd_ctl); + + return PreparedStatementsEnabled(statement_cmd_ctl) + ? ExecuteCommand(query, params, deadline, logging::Level::kInfo, true) + : ExecuteCommandNoPrepare(query, params, deadline); } void ConnectionImpl::Begin( @@ -700,6 +703,19 @@ TimeoutDuration ConnectionImpl::CurrentNetworkTimeout() const { return GetDefaultCommandControl().network_timeout_ms; } +bool ConnectionImpl::PreparedStatementsEnabled(OptionalCommandControl cmd_ctl) const { + if (!!cmd_ctl && + cmd_ctl->prepared_statements_enabled != CommandControl::PreparedStatementsOptionOverride::kNoOverride) { + return cmd_ctl->prepared_statements_enabled == CommandControl::PreparedStatementsOptionOverride::kEnabled; + } + if (GetDefaultCommandControl().prepared_statements_enabled != + CommandControl::PreparedStatementsOptionOverride::kNoOverride) { + return GetDefaultCommandControl().prepared_statements_enabled == + CommandControl::PreparedStatementsOptionOverride::kEnabled; + } + return ArePreparedStatementsEnabled(); +} + void ConnectionImpl::SetConnectionStatementTimeout(TimeoutDuration timeout, engine::Deadline deadline) { timeout = testsuite_pg_ctl_.MakeStatementTimeout(timeout); if (IsPipelineActive() && settings_.deadline_propagation_enabled) { @@ -861,9 +877,11 @@ ResultSet ConnectionImpl::ExecuteCommand( const Query& query, const QueryParameters& params, engine::Deadline deadline, - logging::Level span_log_level + logging::Level span_log_level, + bool ignore_prepared_statements_setting ) { - if (settings_.prepared_statements == ConnectionSettings::kNoPreparedStatements) { + if (!ignore_prepared_statements_setting && + settings_.prepared_statements == ConnectionSettings::kNoPreparedStatements) { return ExecuteCommandNoPrepare(query, params, deadline); } diff --git a/postgresql/src/storages/postgres/detail/connection_impl.hpp b/postgresql/src/storages/postgres/detail/connection_impl.hpp index 9e20ab3d1c6d..f05e4d3d3907 100644 --- a/postgresql/src/storages/postgres/detail/connection_impl.hpp +++ b/postgresql/src/storages/postgres/detail/connection_impl.hpp @@ -151,6 +151,8 @@ class ConnectionImpl { TimeoutDuration NetworkTimeout(OptionalCommandControl) const; TimeoutDuration CurrentNetworkTimeout() const; + bool PreparedStatementsEnabled(OptionalCommandControl cmd_ctl) const; + void SetConnectionStatementTimeout(TimeoutDuration timeout, engine::Deadline deadline); void SetStatementTimeout(TimeoutDuration timeout, engine::Deadline deadline); @@ -173,7 +175,8 @@ class ConnectionImpl { const Query& query, const detail::QueryParameters& params, engine::Deadline deadline, - logging::Level span_log_level = logging::Level::kInfo + logging::Level span_log_level = logging::Level::kInfo, + bool ignore_prepared_statements_setting = false ); ResultSet ExecuteCommandNoPrepare(const Query& query, engine::Deadline deadline); diff --git a/postgresql/src/storages/postgres/postgres_config.cpp b/postgresql/src/storages/postgres/postgres_config.cpp index 274380e18058..b97baedb9fea 100644 --- a/postgresql/src/storages/postgres/postgres_config.cpp +++ b/postgresql/src/storages/postgres/postgres_config.cpp @@ -15,8 +15,8 @@ namespace storages::postgres { CommandControl Parse(const formats::json::Value& elem, formats::parse::To) { CommandControl result{components::Postgres::kDefaultCommandControl}; for (const auto& [name, val] : formats::common::Items(elem)) { - const auto ms = std::chrono::milliseconds{val.As()}; if (name == "network_timeout_ms") { + const auto ms = std::chrono::milliseconds{val.As()}; result.network_timeout_ms = ms; if (result.network_timeout_ms.count() <= 0) { throw InvalidConfig{ @@ -25,6 +25,7 @@ CommandControl Parse(const formats::json::Value& elem, formats::parse::To()}; result.statement_timeout_ms = ms; if (result.statement_timeout_ms.count() <= 0) { throw InvalidConfig{ @@ -32,6 +33,10 @@ CommandControl Parse(const formats::json::Value& elem, formats::parse::To() + ? CommandControl::PreparedStatementsOptionOverride::kEnabled + : CommandControl::PreparedStatementsOptionOverride::kDisabled; } else { LOG_WARNING() << "Unknown parameter " << name << " in PostgreSQL config"; } diff --git a/postgresql/src/storages/postgres/tests/connection_pgtest.cpp b/postgresql/src/storages/postgres/tests/connection_pgtest.cpp index 55d78d27155e..8eee0dc57c4a 100644 --- a/postgresql/src/storages/postgres/tests/connection_pgtest.cpp +++ b/postgresql/src/storages/postgres/tests/connection_pgtest.cpp @@ -411,38 +411,40 @@ UTEST_F(PostgreCustomConnection, Connect) { } UTEST_F(PostgreCustomConnection, NoPreparedStatements) { - UEXPECT_NO_THROW(pg::detail::Connection::Connect( - GetDsnFromEnv(), - nullptr, - GetTaskProcessor(), - GetTaskStorage(), - kConnectionId, - kNoPreparedStatements, - GetTestCmdCtls(), - {}, - {}, - {}, - std::make_shared() - )); + UEXPECT_NO_THROW(MakeConnection(GetDsnFromEnv(), GetTaskProcessor(), kNoPreparedStatements)); +} + +UTEST_F(PostgreCustomConnection, PreparedStatementsOverrideEnabled) { + pg::detail::ConnectionPtr conn{nullptr}; + UEXPECT_NO_THROW(conn = MakeConnection(GetDsnFromEnv(), GetTaskProcessor(), kNoPreparedStatements)); + ASSERT_TRUE(conn); + const auto old_stats = conn->GetStatsAndReset(); + auto cc = pg::CommandControl{ + std::chrono::milliseconds{100}, + std::chrono::milliseconds{10}, + pg::CommandControl::PreparedStatementsOptionOverride::kEnabled}; + UEXPECT_NO_THROW(conn->Execute("select 1", {}, cc)); + const auto stats = conn->GetStatsAndReset(); + EXPECT_GT(stats.prepared_statements_current, old_stats.prepared_statements_current); +} + +UTEST_F(PostgreCustomConnection, PreparedStatementsOverrideDisabled) { + pg::detail::ConnectionPtr conn{nullptr}; + UEXPECT_NO_THROW(conn = MakeConnection(GetDsnFromEnv(), GetTaskProcessor(), kCachePreparedStatements)); + ASSERT_TRUE(conn); + const auto old_stats = conn->GetStatsAndReset(); + auto cc = pg::CommandControl{ + std::chrono::milliseconds{100}, + std::chrono::milliseconds{10}, + pg::CommandControl::PreparedStatementsOptionOverride::kDisabled}; + UEXPECT_NO_THROW(conn->Execute("select 1", {}, cc)); + const auto stats = conn->GetStatsAndReset(); + EXPECT_EQ(stats.prepared_statements_current, old_stats.prepared_statements_current); } UTEST_F(PostgreCustomConnection, NoUserTypes) { - std::unique_ptr conn; - UASSERT_NO_THROW( - conn = pg::detail::Connection::Connect( - GetDsnFromEnv(), - nullptr, - GetTaskProcessor(), - GetTaskStorage(), - kConnectionId, - kNoUserTypes, - GetTestCmdCtls(), - {}, - {}, - {}, - std::make_shared() - ) - ); + pg::detail::ConnectionPtr conn{nullptr}; + UASSERT_NO_THROW(conn = MakeConnection(GetDsnFromEnv(), GetTaskProcessor(), kNoUserTypes)); ASSERT_TRUE(conn); UEXPECT_NO_THROW(conn->Execute("select 1")); diff --git a/postgresql/src/storages/postgres/tests/pool_pgtest.cpp b/postgresql/src/storages/postgres/tests/pool_pgtest.cpp index 9e1eb7765df0..d92c509723cd 100644 --- a/postgresql/src/storages/postgres/tests/pool_pgtest.cpp +++ b/postgresql/src/storages/postgres/tests/pool_pgtest.cpp @@ -518,6 +518,80 @@ UTEST_P(PostgrePool, DefaultCmdCtl) { EXPECT_EQ(kTestCmdCtl, pool->GetDefaultCommandControl()); } +UTEST_P(PostgrePool, PreparedStatementsDisabledOverrideCommandControl) { + auto pool = pg::detail::ConnectionPool::Create( + GetDsnFromEnv(), + nullptr, + GetTaskProcessor(), + "", + GetParam(), + {1, 1, 10}, + kCachePreparedStatements, + {}, + storages::postgres::DefaultCommandControls( + pg::CommandControl{ + std::chrono::milliseconds{100}, + std::chrono::milliseconds{10}, + pg::CommandControl::PreparedStatementsOptionOverride::kDisabled}, + {}, + {} + ), + testsuite::PostgresControl{}, + error_injection::Settings{}, + {}, + dynamic_config::GetDefaultSource(), + std::make_shared() + ); + { + pg::detail::ConnectionPtr conn(nullptr); + + UASSERT_NO_THROW(conn = pool->Acquire(MakeDeadline())); + + const auto old_stats = conn->GetStatsAndReset(); + UEXPECT_NO_THROW(conn->Execute("select 1")); + + const auto stats = conn->GetStatsAndReset(); + EXPECT_EQ(stats.prepared_statements_current, old_stats.prepared_statements_current); + } +} + +UTEST_P(PostgrePool, PreparedStatementsEnabledOverrideCommandControl) { + auto pool = pg::detail::ConnectionPool::Create( + GetDsnFromEnv(), + nullptr, + GetTaskProcessor(), + "", + GetParam(), + {1, 1, 10}, + kNoPreparedStatements, + {}, + storages::postgres::DefaultCommandControls( + pg::CommandControl{ + std::chrono::milliseconds{100}, + std::chrono::milliseconds{10}, + pg::CommandControl::PreparedStatementsOptionOverride::kEnabled}, + {}, + {} + ), + testsuite::PostgresControl{}, + error_injection::Settings{}, + {}, + dynamic_config::GetDefaultSource(), + std::make_shared() + ); + { + pg::detail::ConnectionPtr conn(nullptr); + + UASSERT_NO_THROW(conn = pool->Acquire(MakeDeadline())); + + const auto old_stats = conn->GetStatsAndReset(); + UEXPECT_NO_THROW(conn->Execute("select 1")); + + const auto stats = conn->GetStatsAndReset(); + EXPECT_GT(stats.prepared_statements_current, old_stats.prepared_statements_current); + } +} + UTEST_P(PostgrePool, CheckUserTypes) { std::shared_ptr pool; auto conn_settings = kCachePreparedStatements; From 122dd84aaa3580fec71c5d15a05e7fdf4ac7ae2e Mon Sep 17 00:00:00 2001 From: antoshkka Date: Fri, 25 Jul 2025 21:43:51 +0300 Subject: [PATCH 006/151] feat core: remove deprecated GetName() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI commit_hash:a6ddd19bfc3d93414dc886be751b8025fbfc04c0 --- core/include/userver/storages/query.hpp | 3 --- ydb/src/ydb/impl/stats_scope.cpp | 5 ++--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/core/include/userver/storages/query.hpp b/core/include/userver/storages/query.hpp index 7f98db676e22..a3e243a988a9 100644 --- a/core/include/userver/storages/query.hpp +++ b/core/include/userver/storages/query.hpp @@ -74,9 +74,6 @@ class Query { /// @returns optional queury name suitable for construction of a new Query std::optional GetOptionalName() const { return std::optional{GetOptionalNameView()}; } - /// @deprecated Use GetOptionalName() or GetOptionalNameView() - const std::optional& GetName() const { return dynamic_.name_; } - /// @returns zero terminated view to the statement that is alive as long as *this is alive utils::zstring_view GetStatementView() const noexcept; diff --git a/ydb/src/ydb/impl/stats_scope.cpp b/ydb/src/ydb/impl/stats_scope.cpp index b79ce111f1a5..fa1656756e44 100644 --- a/ydb/src/ydb/impl/stats_scope.cpp +++ b/ydb/src/ydb/impl/stats_scope.cpp @@ -11,14 +11,13 @@ namespace ydb::impl { namespace { StatsCounters& GetCountersForQuery(Stats& stats, const Query& query) { - const auto& query_name = query.GetName(); - + auto query_name = query.GetOptionalName(); if (!query_name) { return stats.unnamed_queries; } const auto insertion_result = - stats.by_query.TryEmplace(std::string{query_name.value()}, stats.by_database_histogram_bounds); + stats.by_query.TryEmplace(std::string{std::move(*query_name)}, stats.by_database_histogram_bounds); // No need to retain a shared_ptr. References to items are stable. // Items are never removed from the map. return *insertion_result.value; From aaebbbfc518fafda7a3aa9e46b83b4469af80d6f Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 28 Jul 2025 12:55:43 +0300 Subject: [PATCH 007/151] feat secdist: update-period defaults to 10s commit_hash:53958151cc6a6a0507b4120e9963efb746453d03 --- core/include/userver/storages/secdist/component.hpp | 2 +- core/src/storages/secdist/component.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/include/userver/storages/secdist/component.hpp b/core/include/userver/storages/secdist/component.hpp index 0b77a86c8219..6f60c3794e30 100644 --- a/core/include/userver/storages/secdist/component.hpp +++ b/core/include/userver/storages/secdist/component.hpp @@ -30,7 +30,7 @@ namespace components { /// Name | Description | Default value /// ---- | ----------- | ------------- /// provider | optional secdist provider component name | 'default-secdist-provider' -/// update-period | period between data updates in utils::StringToDuration() suitable format ('0s' for no updates) | 0s +/// update-period | period between data updates in utils::StringToDuration() suitable format ('0s' for no updates) | 10s // clang-format on diff --git a/core/src/storages/secdist/component.cpp b/core/src/storages/secdist/component.cpp index 6bf27a973f4a..5b1757514b97 100644 --- a/core/src/storages/secdist/component.cpp +++ b/core/src/storages/secdist/component.cpp @@ -19,7 +19,7 @@ ParseSettings(const ComponentConfig& config, const ComponentContext& context) { const auto provider_name = config["provider"].As("default-secdist-provider"); settings.provider = &context.FindComponent(provider_name); - settings.update_period = utils::StringToDuration(config["update-period"].As("0s")); + settings.update_period = utils::StringToDuration(config["update-period"].As("10s")); return settings; } @@ -43,7 +43,7 @@ additionalProperties: false update-period: type: string description: period between data updates in utils::StringToDuration() suitable format ('0s' for no updates) - defaultDescription: 0s + defaultDescription: 10s provider: type: string description: optional secdist provider component name From ed47ac4fc3494f9a49efb0a61d7f28f597ea0005 Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 28 Jul 2025 14:11:49 +0300 Subject: [PATCH 008/151] feat chaotic: "format: uri" commit_hash:01ba4bc120f0728406ad7d733ec9919e4775299a --- .mapping.json | 1 + chaotic/chaotic/back/cpp/translator.py | 6 +++++- chaotic/chaotic/front/types.py | 2 ++ chaotic/integration_tests/schemas/uri.yaml | 8 ++++++++ chaotic/integration_tests/tests/render/simple.cpp | 12 ++++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 chaotic/integration_tests/schemas/uri.yaml diff --git a/.mapping.json b/.mapping.json index 705837474a3e..f1b12c1c885b 100644 --- a/.mapping.json +++ b/.mapping.json @@ -292,6 +292,7 @@ "chaotic/integration_tests/schemas/pattern.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/pattern.yaml", "chaotic/integration_tests/schemas/recursion.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/recursion.yaml", "chaotic/integration_tests/schemas/string64.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/string64.yaml", + "chaotic/integration_tests/schemas/uri.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/uri.yaml", "chaotic/integration_tests/schemas/uuid.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/uuid.yaml", "chaotic/integration_tests/tests/lib/array.cpp":"taxi/uservices/userver/chaotic/integration_tests/tests/lib/array.cpp", "chaotic/integration_tests/tests/lib/multiple_ints.cpp":"taxi/uservices/userver/chaotic/integration_tests/tests/lib/multiple_ints.cpp", diff --git a/chaotic/chaotic/back/cpp/translator.py b/chaotic/chaotic/back/cpp/translator.py index 1dfddd056e44..fefa06cbcc3e 100644 --- a/chaotic/chaotic/back/cpp/translator.py +++ b/chaotic/chaotic/back/cpp/translator.py @@ -443,7 +443,11 @@ def _gen_string( ) user_cpp_type = f'userver::utils::StrongTypedef<{typedef_tag}, std::string>' - if schema.format and schema.format != types.StringFormat.BINARY: + if schema.format == types.StringFormat.URI: + assert not validators.pattern, '"format: uri" and "pattern" are not yet implemented' + validators.pattern = '^[a-z][-a-z0-9+.]*:.*' + + if schema.format and schema.format not in {types.StringFormat.BINARY, types.StringFormat.URI}: if schema.format == types.StringFormat.BYTE: format_cpp_type = 'crypto::base64::String64' elif schema.format == types.StringFormat.UUID: diff --git a/chaotic/chaotic/front/types.py b/chaotic/chaotic/front/types.py index f9654dd386b7..65d36ae5ca7e 100644 --- a/chaotic/chaotic/front/types.py +++ b/chaotic/chaotic/front/types.py @@ -257,6 +257,7 @@ class StringFormat(enum.Enum): DATE_TIME_ISO_BASIC = enum.auto() DATE_TIME_FRACTION = enum.auto() UUID = enum.auto() + URI = enum.auto() @classmethod def from_string(cls, data: str) -> 'StringFormat': @@ -274,6 +275,7 @@ def from_string(cls, data: str) -> 'StringFormat': 'date-time-iso-basic': StringFormat.DATE_TIME_ISO_BASIC, 'date-time-fraction': StringFormat.DATE_TIME_FRACTION, 'uuid': StringFormat.UUID, + 'uri': StringFormat.URI, } diff --git a/chaotic/integration_tests/schemas/uri.yaml b/chaotic/integration_tests/schemas/uri.yaml new file mode 100644 index 000000000000..0b26fe9913c5 --- /dev/null +++ b/chaotic/integration_tests/schemas/uri.yaml @@ -0,0 +1,8 @@ +definitions: + ObjectUri: + type: object + additionalProperties: false + properties: + uri: + type: string + format: uri diff --git a/chaotic/integration_tests/tests/render/simple.cpp b/chaotic/integration_tests/tests/render/simple.cpp index 3334aebd32ed..b2a481cbb743 100644 --- a/chaotic/integration_tests/tests/render/simple.cpp +++ b/chaotic/integration_tests/tests/render/simple.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -382,6 +383,17 @@ TEST(Simple, Uuid) { EXPECT_EQ(str, uuid); } +TEST(Simple, Uri) { + auto uri = "http://example.com"; + auto json = formats::json::MakeObject("uri", uri); + auto obj = json.As(); + + EXPECT_EQ(obj.uri, uri); + + auto str = Serialize(obj, formats::serialize::To())["uri"].As(); + EXPECT_EQ(str, uri); +} + TEST(SIMPLE, String64) { auto str64 = crypto::base64::String64{"hello, userver!"}; auto obj = ns::ObjectString64{str64}; From 43d48c5921646a097684c58feb1a8b4f4e7607aa Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Mon, 28 Jul 2025 14:32:11 +0300 Subject: [PATCH 009/151] refactor grpc: cleanup client CallState commit_hash:005f29f09e48e35ade73a85a33f68094fc8ca3ec --- .../userver/ugrpc/client/impl/call_state.hpp | 2 + grpc/src/ugrpc/client/impl/async_methods.cpp | 14 +----- grpc/src/ugrpc/client/impl/call_state.cpp | 44 +++++++++++++++++++ grpc/src/ugrpc/client/impl/tracing.cpp | 38 ---------------- grpc/src/ugrpc/client/impl/tracing.hpp | 7 --- 5 files changed, 48 insertions(+), 57 deletions(-) diff --git a/grpc/include/userver/ugrpc/client/impl/call_state.hpp b/grpc/include/userver/ugrpc/client/impl/call_state.hpp index bfa31f86888d..1b2f48a14fe9 100644 --- a/grpc/include/userver/ugrpc/client/impl/call_state.hpp +++ b/grpc/include/userver/ugrpc/client/impl/call_state.hpp @@ -192,6 +192,8 @@ bool IsWriteAvailable(const StreamingCallState&) noexcept; bool IsWriteAndCheckAvailable(const StreamingCallState&) noexcept; +void HandleCallStatistics(CallState& state, const grpc::Status& status) noexcept; + } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/async_methods.cpp b/grpc/src/ugrpc/client/impl/async_methods.cpp index 8cbe6bb10f96..8b50b94e84cc 100644 --- a/grpc/src/ugrpc/client/impl/async_methods.cpp +++ b/grpc/src/ugrpc/client/impl/async_methods.cpp @@ -12,16 +12,6 @@ namespace ugrpc::client::impl { namespace { -void ProcessCallStatistics(CallState& state) noexcept { - const auto& status = state.GetStatus(); - auto& stats = state.GetStatsScope(); - stats.OnExplicitFinish(status.error_code()); - if (status.error_code() == grpc::StatusCode::DEADLINE_EXCEEDED && state.IsDeadlinePropagated()) { - stats.OnCancelledByDeadlinePropagation(); - } - stats.Flush(); -} - void SetStatusAndResetSpan(CallState& state, const grpc::Status& status) noexcept { SetStatusForSpan(state.GetSpan(), status); state.ResetSpan(); @@ -52,10 +42,10 @@ WaitAndTryCancelIfNeeded(ugrpc::impl::AsyncMethodInvocation& invocation, grpc::C } void ProcessFinish(CallState& state, const google::protobuf::Message* final_response) { - ProcessCallStatistics(state); - const auto& status = state.GetStatus(); + HandleCallStatistics(state, status); + if (final_response && status.ok()) { MiddlewarePipeline::PostRecvMessage(state, *final_response); } diff --git a/grpc/src/ugrpc/client/impl/call_state.cpp b/grpc/src/ugrpc/client/impl/call_state.cpp index 1da8fa4cd8cf..b0eaec7e618c 100644 --- a/grpc/src/ugrpc/client/impl/call_state.cpp +++ b/grpc/src/ugrpc/client/impl/call_state.cpp @@ -2,17 +2,51 @@ #include +#include +#include #include +#include #include +#include #include #include +#include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { +namespace { + +void SetupSpan(std::optional& span_holder, std::string_view call_name) { + UASSERT(!span_holder); + span_holder.emplace(utils::StrCat("external_grpc/", call_name), utils::impl::SourceLocation::Current()); + auto& span = span_holder->Get(); + span.DetachFromCoroStack(); +} + +void AddTracingMetadata(grpc::ClientContext& client_context, const tracing::Span& span) { + if (const auto span_id = span.GetSpanIdForChildLogs()) { + client_context.AddMetadata(ugrpc::impl::kXYaTraceId, ugrpc::impl::ToGrpcString(span.GetTraceId())); + client_context.AddMetadata(ugrpc::impl::kXYaSpanId, ugrpc::impl::ToGrpcString(*span_id)); + client_context.AddMetadata(ugrpc::impl::kXYaRequestId, ugrpc::impl::ToGrpcString(span.GetLink())); + + constexpr std::string_view kDefaultOtelTraceFlags = "01"; + auto traceparent = + tracing::opentelemetry::BuildTraceParentHeader(span.GetTraceId(), *span_id, kDefaultOtelTraceFlags); + + if (!traceparent.has_value()) { + LOG_LIMITED_DEBUG("Cannot build opentelemetry traceparent header ({})", traceparent.error()); + return; + } + client_context.AddMetadata(ugrpc::impl::kTraceParent, ugrpc::impl::ToGrpcString(traceparent.value())); + } +} + +} // namespace + RpcConfigValues::RpcConfigValues(const dynamic_config::Snapshot& config) : enforce_task_deadline(config[::dynamic_config::USERVER_GRPC_CLIENT_ENABLE_DEADLINE_PROPAGATION]) {} @@ -26,6 +60,7 @@ CallState::CallState(CallParams&& params, CallKind call_kind) middlewares_(params.middlewares), call_kind_(call_kind) { UINVARIANT(!client_name_.empty(), "client name should not be empty"); + SetupSpan(span_, call_name_.Get()); client_context_ = CallOptionsAccessor::CreateClientContext(params.call_options); @@ -160,6 +195,15 @@ bool IsWriteAndCheckAvailable(const StreamingCallState& state) noexcept { return !state.AreWritesFinished() && !state.IsFinished(); } +void HandleCallStatistics(CallState& state, const grpc::Status& status) noexcept { + auto& stats = state.GetStatsScope(); + stats.OnExplicitFinish(status.error_code()); + if (grpc::StatusCode::DEADLINE_EXCEEDED == status.error_code() && state.IsDeadlinePropagated()) { + stats.OnCancelledByDeadlinePropagation(); + } + stats.Flush(); +} + } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/tracing.cpp b/grpc/src/ugrpc/client/impl/tracing.cpp index fc938de8519b..bf3bb4f72715 100644 --- a/grpc/src/ugrpc/client/impl/tracing.cpp +++ b/grpc/src/ugrpc/client/impl/tracing.cpp @@ -1,51 +1,13 @@ #include -#include -#include #include -#include -#include -#include -#include -#include #include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -namespace { - -constexpr std::string_view kDefaultOtelTraceFlags = "01"; - -} // namespace - -void SetupSpan(std::optional& span_holder, std::string_view call_name) { - UASSERT(!span_holder); - span_holder.emplace(utils::StrCat("external_grpc/", call_name), utils::impl::SourceLocation::Current()); - auto& span = span_holder->Get(); - span.DetachFromCoroStack(); -} - -void AddTracingMetadata(grpc::ClientContext& client_context, tracing::Span& span) { - if (const auto span_id = span.GetSpanIdForChildLogs()) { - client_context.AddMetadata(ugrpc::impl::kXYaTraceId, ugrpc::impl::ToGrpcString(span.GetTraceId())); - client_context.AddMetadata(ugrpc::impl::kXYaSpanId, ugrpc::impl::ToGrpcString(*span_id)); - client_context.AddMetadata(ugrpc::impl::kXYaRequestId, ugrpc::impl::ToGrpcString(span.GetLink())); - - auto traceparent = - tracing::opentelemetry::BuildTraceParentHeader(span.GetTraceId(), *span_id, kDefaultOtelTraceFlags); - - if (!traceparent.has_value()) { - LOG_LIMITED_DEBUG( - ) << fmt::format("Cannot build opentelemetry traceparent header ({})", traceparent.error()); - return; - } - client_context.AddMetadata(ugrpc::impl::kTraceParent, ugrpc::impl::ToGrpcString(traceparent.value())); - } -} - void SetErrorForSpan(tracing::Span& span, std::string_view error_message) noexcept { try { span.SetLogLevel(logging::Level::kWarning); diff --git a/grpc/src/ugrpc/client/impl/tracing.hpp b/grpc/src/ugrpc/client/impl/tracing.hpp index 93176b710849..f32376a1bac7 100644 --- a/grpc/src/ugrpc/client/impl/tracing.hpp +++ b/grpc/src/ugrpc/client/impl/tracing.hpp @@ -1,22 +1,15 @@ #pragma once -#include #include -#include #include -#include #include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -void SetupSpan(std::optional& span_holder, std::string_view call_name); - -void AddTracingMetadata(grpc::ClientContext& client_context, tracing::Span& span); - void SetStatusForSpan(tracing::Span& span, const grpc::Status& status) noexcept; void SetErrorForSpan(tracing::Span& span, std::string_view error_message) noexcept; From 6da1866eb037cbce6d888fa5d7af39d0fe852d71 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Mon, 28 Jul 2025 15:20:45 +0300 Subject: [PATCH 010/151] feat core: avoid dynamic initialization in storages::Query while keeping its size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the code safer by avoiding static initialization order fiasco Tests: протестировано CI Relates: https://nda.ya.ru/t/joCzhxD_7Gwpsb commit_hash:942916ae5c6ddbf046d0dda35c64254767fb05a5 --- .mapping.json | 1 + core/include/userver/storages/query.hpp | 25 +++-- core/src/storages/query.cpp | 19 +++- core/src/storages/query_test.cpp | 131 ++++++++++++++++++++++++ 4 files changed, 162 insertions(+), 14 deletions(-) create mode 100644 core/src/storages/query_test.cpp diff --git a/.mapping.json b/.mapping.json index f1b12c1c885b..d27a663b0381 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1799,6 +1799,7 @@ "core/src/server/websocket/server.cpp":"taxi/uservices/userver/core/src/server/websocket/server.cpp", "core/src/server/websocket/websocket_handler.cpp":"taxi/uservices/userver/core/src/server/websocket/websocket_handler.cpp", "core/src/storages/query.cpp":"taxi/uservices/userver/core/src/storages/query.cpp", + "core/src/storages/query_test.cpp":"taxi/uservices/userver/core/src/storages/query_test.cpp", "core/src/storages/secdist/component.cpp":"taxi/uservices/userver/core/src/storages/secdist/component.cpp", "core/src/storages/secdist/helpers.cpp":"taxi/uservices/userver/core/src/storages/secdist/helpers.cpp", "core/src/storages/secdist/provider_component.cpp":"taxi/uservices/userver/core/src/storages/secdist/provider_component.cpp", diff --git a/core/include/userver/storages/query.hpp b/core/include/userver/storages/query.hpp index a3e243a988a9..adac7e46db82 100644 --- a/core/include/userver/storages/query.hpp +++ b/core/include/userver/storages/query.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -17,9 +18,10 @@ class Span; namespace storages { -/// @brief Holds a query, its name and logging mode. +/// @brief Holds a query, its name and logging mode; used by all the SQL databases of userver. /// -/// Prefer using via @ref scripts/docs/en/userver/sql_files.md or use a const variable with `name` specified. +/// Prefer using via @ref scripts/docs/en/userver/sql_files.md or use a const variable with `name` specified +/// as NameLiteral. /// /// @note You may write a query in `.sql` file and generate a header file with Query from it. /// See @ref scripts/docs/en/userver/sql_files.md for more information. @@ -60,13 +62,14 @@ class Query { Query& operator=(const Query& other) = default; Query& operator=(Query&& other) = default; - /*TODO: constexpr*/ Query(utils::StringLiteral statement, NameLiteral name, LogMode log_mode) - : Query(std::string{statement}, Name{std::string{name}}, log_mode) {} + /// Constructor that omits dynamic initialization and relies on `statement` and `name` being string literals + constexpr Query(utils::StringLiteral statement, NameLiteral name, LogMode log_mode) + : data_{StaticStrings{statement, name}}, log_mode_{log_mode} {} Query(const char* statement, std::optional name = std::nullopt, LogMode log_mode = LogMode::kFull) : Query(std::string{statement}, std::move(name), log_mode) {} Query(std::string statement, std::optional name = std::nullopt, LogMode log_mode = LogMode::kFull) - : dynamic_{statement, std::move(name)}, log_mode_(log_mode) {} + : data_{DynamicStrings{std::move(statement), std::move(name)}}, log_mode_(log_mode) {} /// @returns view to the query name that is alive as long as *this is alive std::optional GetOptionalNameView() const noexcept; @@ -82,12 +85,16 @@ class Query { private: struct DynamicStrings { - std::string statement_{}; - std::optional name_{}; + std::string statement_; + std::optional name_; }; + struct StaticStrings { + utils::StringLiteral statement_; + std::optional name_; + }; + struct NameViewVisitor; - DynamicStrings dynamic_{}; - + std::variant data_ = StaticStrings{utils::StringLiteral{""}, std::nullopt}; LogMode log_mode_ = LogMode::kFull; }; diff --git a/core/src/storages/query.cpp b/core/src/storages/query.cpp index 062a0107ef3e..d4a77d4c54f0 100644 --- a/core/src/storages/query.cpp +++ b/core/src/storages/query.cpp @@ -1,17 +1,26 @@ #include -#include -#include - USERVER_NAMESPACE_BEGIN namespace storages { +struct Query::NameViewVisitor { + std::optional operator()(const DynamicStrings& x) const noexcept { + return (x.name_ ? std::optional{x.name_->GetUnderlying()} : std::nullopt); + } + + std::optional operator()(const StaticStrings& x) const noexcept { + return std::optional{x.name_}; + } +}; + std::optional Query::GetOptionalNameView() const noexcept { - return dynamic_.name_ ? std::optional{dynamic_.name_->GetUnderlying()} : std::nullopt; + return std::visit(NameViewVisitor{}, data_); } -utils::zstring_view Query::GetStatementView() const noexcept { return utils::zstring_view{dynamic_.statement_}; } +utils::zstring_view Query::GetStatementView() const noexcept { + return std::visit([](const auto& x) { return utils::zstring_view{x.statement_}; }, data_); +} } // namespace storages diff --git a/core/src/storages/query_test.cpp b/core/src/storages/query_test.cpp new file mode 100644 index 000000000000..3e7cbd62c426 --- /dev/null +++ b/core/src/storages/query_test.cpp @@ -0,0 +1,131 @@ +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +constexpr const char* kStatement = "The query that is too long to fit into SSO because it is too long"; +constexpr const char* kQueryName = "query_name_that_does_not_fir_into_SSO_because_it_is_too_big"; + +// Validate that variable initializes without dynamic initialization +USERVER_IMPL_CONSTINIT const storages::Query kStaticInit{ + kStatement, + storages::Query::NameLiteral{kQueryName}, + storages::Query::LogMode::kFull, +}; + +// Validate that variable initializes without dynamic initialization +USERVER_IMPL_CONSTINIT const storages::Query kEmptyStaticInit{}; + +} // namespace + +TEST(Query, StaticInit) { + EXPECT_EQ(kStaticInit.GetStatementView(), kStatement); + EXPECT_EQ(kStaticInit.GetStatementView().c_str(), kStatement); + EXPECT_EQ(kStaticInit.GetOptionalNameView(), kQueryName); + EXPECT_EQ(kStaticInit.GetOptionalNameView()->c_str(), kQueryName); + + auto other = kStaticInit; + EXPECT_EQ(other.GetStatementView(), kStatement); + EXPECT_EQ(other.GetStatementView().c_str(), kStatement); + EXPECT_EQ(other.GetOptionalNameView(), kQueryName); + EXPECT_EQ(other.GetOptionalNameView()->c_str(), kQueryName); + + auto other2 = std::move(other); + EXPECT_EQ(other2.GetStatementView(), kStatement); + EXPECT_EQ(other2.GetStatementView().c_str(), kStatement); + EXPECT_EQ(other2.GetOptionalNameView(), kQueryName); + EXPECT_EQ(other2.GetOptionalNameView()->c_str(), kQueryName); + + other2 = kStaticInit; + EXPECT_EQ(other2.GetStatementView(), kStatement); + EXPECT_EQ(other2.GetStatementView().c_str(), kStatement); + EXPECT_EQ(other2.GetOptionalNameView(), kQueryName); + EXPECT_EQ(other2.GetOptionalName()->GetUnderlying(), kQueryName); + EXPECT_EQ(other2.GetOptionalNameView()->c_str(), kQueryName); + + EXPECT_EQ(kStaticInit.GetLogMode(), storages::Query::LogMode::kFull); +} + +TEST(Query, StaticInitEmpty) { + EXPECT_EQ(kEmptyStaticInit.GetStatementView(), ""); + EXPECT_FALSE(kEmptyStaticInit.GetOptionalNameView()); +} + +TEST(Query, RunTime) { + { + storages::Query query{ + std::string{kStatement}, + storages::Query::Name{kQueryName}, + }; + + EXPECT_EQ(query.GetStatementView(), kStatement); + EXPECT_NE(query.GetStatementView().c_str(), kStatement); + EXPECT_EQ(query.GetOptionalNameView(), kQueryName); + EXPECT_NE(query.GetOptionalNameView()->c_str(), kQueryName); + + auto query_copy = query; + auto query_copy2 = std::move(query); + auto query_copy3 = query_copy2; + EXPECT_EQ(query_copy3.GetStatementView(), kStatement); + EXPECT_NE(query_copy3.GetStatementView().c_str(), kStatement); + EXPECT_EQ(query_copy3.GetOptionalNameView(), kQueryName); + EXPECT_NE(query_copy3.GetOptionalNameView()->c_str(), kQueryName); + + query = kStaticInit; + EXPECT_EQ(query.GetStatementView(), kStatement); + EXPECT_EQ(query.GetStatementView().c_str(), kStatement); + EXPECT_EQ(query.GetOptionalNameView(), kQueryName); + EXPECT_EQ(query.GetOptionalNameView()->c_str(), kQueryName); + + query = query_copy; + EXPECT_EQ(query.GetStatementView(), kStatement); + EXPECT_NE(query.GetStatementView().c_str(), kStatement); + EXPECT_EQ(query.GetOptionalNameView(), kQueryName); + EXPECT_NE(query.GetOptionalNameView()->c_str(), kQueryName); + + query = std::move(query_copy2); + EXPECT_EQ(query.GetStatementView(), kStatement); + EXPECT_EQ(query.GetOptionalNameView(), kQueryName); + } + { + storages::Query query{ + kStatement, + storages::Query::Name{kQueryName}, + }; + + EXPECT_EQ(query.GetStatementView(), kStatement); + EXPECT_EQ(query.GetOptionalNameView(), kQueryName); + } +} + +TEST(Query, SelfConstruct) { + storages::Query query{ + std::string{kStaticInit.GetStatementView()}, + kStaticInit.GetOptionalName(), + }; + EXPECT_EQ(query.GetStatementView(), kStatement); + EXPECT_EQ(query.GetOptionalNameView(), kQueryName); +} + +TEST(Query, OneLiteral) { + const storages::Query query{ + "statement", + storages::Query::Name{"name"}, + }; + EXPECT_EQ(query.GetStatementView(), "statement"); + EXPECT_EQ(query.GetOptionalNameView(), "name"); +} + +TEST(Query, FmtRuntime) { + const storages::Query query{ + "statement {}", + storages::Query::Name{"name"}, + }; + EXPECT_EQ(fmt::format(fmt::runtime(query.GetStatementView()), "OK"), "statement OK"); +} + +USERVER_NAMESPACE_END From 63f3634fe1fc35dec7c5f3aacb1e0cdc7a423fa0 Mon Sep 17 00:00:00 2001 From: mvkab Date: Mon, 28 Jul 2025 15:26:49 +0300 Subject: [PATCH 011/151] feat kafka: bump kafka recipe version to 4.0.0 commit_hash:51674f6bb5a0c63f838e34b1a4daa37bd0e04d07 --- scripts/kafka/install_kafka.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/kafka/install_kafka.sh b/scripts/kafka/install_kafka.sh index 4ede3ff9c43a..d2c6dc7b329a 100755 --- a/scripts/kafka/install_kafka.sh +++ b/scripts/kafka/install_kafka.sh @@ -14,9 +14,9 @@ if [ -z ${KAFKA_PATH+x} ]; then fi fi -curl https://dlcdn.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz -o kafka.tgz +curl https://dlcdn.apache.org/kafka/4.0.0/kafka_2.13-4.0.0.tgz -o kafka.tgz mkdir -p ${KAFKA_PATH} tar xf kafka.tgz --directory="${KAFKA_PATH}" -cp -r ${KAFKA_PATH}/kafka_2.13-3.8.0/* ${KAFKA_PATH} -rm -rf ${KAFKA_PATH}/kafka_2.13-3.8.0 +cp -r ${KAFKA_PATH}/kafka_2.13-4.0.0/* ${KAFKA_PATH} +rm -rf ${KAFKA_PATH}/kafka_2.13-4.0.0 rm kafka.tgz From 3bd59dea6a604770e906da04d2f10c528023a2da Mon Sep 17 00:00:00 2001 From: Reavolt Date: Mon, 28 Jul 2025 18:27:07 +0300 Subject: [PATCH 012/151] feat clickhouse: add parameter store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавлен новый объект userver::storages::clickhouse::ParameterStore store; Реализована возможность накапливать параметры для ClickHouse-запросов через store.PushBack(data); Добавлен Escape для типа boost::uuids::uuid Tests: протестировано CI Pull Request resolved: https://github.com/userver-framework/userver/pull/959 commit_hash:b527016249c5a4ddab905c0469c853faaa9055c4 --- .mapping.json | 2 + .../userver/storages/clickhouse/cluster.hpp | 7 + .../storages/clickhouse/io/impl/escape.hpp | 4 + .../storages/clickhouse/parameter_store.hpp | 54 ++++++ .../src/storages/clickhouse/cluster.cpp | 10 + .../storages/clickhouse/io/impl/escape.cpp | 6 + .../storages/tests/parameter_store_chtest.cpp | 174 ++++++++++++++++++ 7 files changed, 257 insertions(+) create mode 100644 clickhouse/include/userver/storages/clickhouse/parameter_store.hpp create mode 100644 clickhouse/src/storages/tests/parameter_store_chtest.cpp diff --git a/.mapping.json b/.mapping.json index d27a663b0381..62b9ac0b0f80 100644 --- a/.mapping.json +++ b/.mapping.json @@ -398,6 +398,7 @@ "clickhouse/include/userver/storages/clickhouse/io/type_traits.hpp":"taxi/uservices/userver/clickhouse/include/userver/storages/clickhouse/io/type_traits.hpp", "clickhouse/include/userver/storages/clickhouse/io/typedefs.hpp":"taxi/uservices/userver/clickhouse/include/userver/storages/clickhouse/io/typedefs.hpp", "clickhouse/include/userver/storages/clickhouse/options.hpp":"taxi/uservices/userver/clickhouse/include/userver/storages/clickhouse/options.hpp", + "clickhouse/include/userver/storages/clickhouse/parameter_store.hpp":"taxi/uservices/userver/clickhouse/include/userver/storages/clickhouse/parameter_store.hpp", "clickhouse/include/userver/storages/clickhouse/query.hpp":"taxi/uservices/userver/clickhouse/include/userver/storages/clickhouse/query.hpp", "clickhouse/library.yaml":"taxi/uservices/userver/clickhouse/library.yaml", "clickhouse/src/storages/clickhouse/cluster.cpp":"taxi/uservices/userver/clickhouse/src/storages/clickhouse/cluster.cpp", @@ -460,6 +461,7 @@ "clickhouse/src/storages/tests/metrics_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/metrics_chtest.cpp", "clickhouse/src/storages/tests/misc_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/misc_chtest.cpp", "clickhouse/src/storages/tests/nullable_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/nullable_chtest.cpp", + "clickhouse/src/storages/tests/parameter_store_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/parameter_store_chtest.cpp", "clickhouse/src/storages/tests/uint16_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/uint16_chtest.cpp", "clickhouse/src/storages/tests/uint32_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/uint32_chtest.cpp", "clickhouse/src/storages/tests/uint64_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/uint64_chtest.cpp", diff --git a/clickhouse/include/userver/storages/clickhouse/cluster.hpp b/clickhouse/include/userver/storages/clickhouse/cluster.hpp index 35ec4430bce2..9b617a071b06 100644 --- a/clickhouse/include/userver/storages/clickhouse/cluster.hpp +++ b/clickhouse/include/userver/storages/clickhouse/cluster.hpp @@ -23,6 +23,7 @@ USERVER_NAMESPACE_BEGIN namespace storages::clickhouse { class ExecutionResult; +class ParameterStore; namespace impl { struct ClickhouseSettings; @@ -67,6 +68,12 @@ class Cluster final { template ExecutionResult Execute(OptionalCommandControl, const Query& query, const Args&... args) const; + /// @overload + ExecutionResult Execute(const Query& query, const ParameterStore& params) const; + + /// @overload + ExecutionResult Execute(OptionalCommandControl, const Query& query, const ParameterStore& params) const; + /// @brief Insert data at some host of the cluster; /// `T` is expected to be a struct of vectors of same length. /// @param table_name table to insert into diff --git a/clickhouse/include/userver/storages/clickhouse/io/impl/escape.hpp b/clickhouse/include/userver/storages/clickhouse/io/impl/escape.hpp index c27a909e3abf..d3187b19d0b7 100644 --- a/clickhouse/include/userver/storages/clickhouse/io/impl/escape.hpp +++ b/clickhouse/include/userver/storages/clickhouse/io/impl/escape.hpp @@ -5,6 +5,8 @@ #include #include +#include + #include #include @@ -35,6 +37,8 @@ std::string Escape(const char* source); std::string Escape(const std::string& source); std::string Escape(std::string_view source); +std::string Escape(const boost::uuids::uuid& uuid); + std::string Escape(std::chrono::system_clock::time_point source); std::string Escape(DateTime64Milli source); std::string Escape(DateTime64Micro source); diff --git a/clickhouse/include/userver/storages/clickhouse/parameter_store.hpp b/clickhouse/include/userver/storages/clickhouse/parameter_store.hpp new file mode 100644 index 000000000000..9c264c00b63c --- /dev/null +++ b/clickhouse/include/userver/storages/clickhouse/parameter_store.hpp @@ -0,0 +1,54 @@ +#pragma once + +/// @file userver/storages/clickhouse/parameter_store.hpp +/// @brief @copybrief storages::clickhouse::ParameterStore + +#include + +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::clickhouse { + +/// @ingroup userver_containers +/// +/// @brief Class for dynamic ClickHouse parameter list construction. +/// +/// @snippet storages/tests/parameter_store_chtest.cpp basic usage +class ParameterStore { +public: + ParameterStore() = default; + ParameterStore(const ParameterStore&) = delete; + ParameterStore(ParameterStore&&) = default; + ParameterStore& operator=(const ParameterStore&) = delete; + ParameterStore& operator=(ParameterStore&&) = default; + + /// @brief Adds a parameter to the end of the parameter list. + /// @note Currently only built-in/system types are supported. + template + ParameterStore& PushBack(const T& param) { + parameters_.push_back(io::impl::Escape(param)); + return *this; + } + + /// @cond + // For internal use only + Query MakeQueryWithArgs(const Query& query) const { + // we should throw on params count mismatch + // TODO : https://st.yandex-team.ru/TAXICOMMON-5066 + return Query{fmt::vformat(std::string_view{query.GetStatementView()}, parameters_), query.GetOptionalName()}; + } + /// @endcond + +private: + fmt::dynamic_format_arg_store parameters_{}; +}; + +} // namespace storages::clickhouse + +USERVER_NAMESPACE_END diff --git a/clickhouse/src/storages/clickhouse/cluster.cpp b/clickhouse/src/storages/clickhouse/cluster.cpp index f1610185dc52..4ee2ba969bc9 100644 --- a/clickhouse/src/storages/clickhouse/cluster.cpp +++ b/clickhouse/src/storages/clickhouse/cluster.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -50,6 +51,15 @@ Cluster::Cluster( Cluster::~Cluster() = default; +ExecutionResult Cluster::Execute(const Query& query, const ParameterStore& params) const { + return DoExecute(OptionalCommandControl{}, params.MakeQueryWithArgs(query)); +} + +ExecutionResult Cluster::Execute(OptionalCommandControl optional_cc, const Query& query, const ParameterStore& params) + const { + return DoExecute(optional_cc, params.MakeQueryWithArgs(query)); +} + ExecutionResult Cluster::DoExecute(OptionalCommandControl optional_cc, const Query& query) const { return GetPool().Execute(optional_cc, query); } diff --git a/clickhouse/src/storages/clickhouse/io/impl/escape.cpp b/clickhouse/src/storages/clickhouse/io/impl/escape.cpp index 3aeac92ed9d2..edeb0fa996d8 100644 --- a/clickhouse/src/storages/clickhouse/io/impl/escape.cpp +++ b/clickhouse/src/storages/clickhouse/io/impl/escape.cpp @@ -2,6 +2,8 @@ #include +#include + #include USERVER_NAMESPACE_BEGIN @@ -73,6 +75,10 @@ std::string Escape(std::string_view source) { return result; } +std::string Escape(const boost::uuids::uuid& uuid) { + return fmt::format("toUUID('{}')", boost::uuids::to_string(uuid)); +} + std::string Escape(std::chrono::system_clock::time_point source) { return fmt::format( "toDateTime({})", std::chrono::duration_cast(source.time_since_epoch()).count() diff --git a/clickhouse/src/storages/tests/parameter_store_chtest.cpp b/clickhouse/src/storages/tests/parameter_store_chtest.cpp new file mode 100644 index 000000000000..20c3f0c61f3e --- /dev/null +++ b/clickhouse/src/storages/tests/parameter_store_chtest.cpp @@ -0,0 +1,174 @@ +#include +#include +#include + +#include "utils_test.hpp" + +USERVER_NAMESPACE_BEGIN + +UTEST(ClickhouseParameterStore, ParamsEqualPlaceholders) { + ClusterWrapper cluster; + cluster->Execute(storages::Query{"CREATE TABLE IF NOT EXISTS users (id UInt64, name String) ENGINE = Memory"}); + /// [basic usage] + storages::Query q{"SELECT {} FROM {} WHERE id = {}"}; + storages::clickhouse::ParameterStore params; + params.PushBack("name"); + params.PushBack("users"); + params.PushBack(42); + EXPECT_NO_THROW(cluster->Execute(q, params)); + /// [basic usage] +} + +UTEST(ClickhouseParameterStore, LessParamsThanPlaceholders) { + ClusterWrapper cluster; + cluster->Execute(storages::Query{"CREATE TABLE IF NOT EXISTS users (id UInt64, name String) ENGINE = Memory"}); + storages::Query q{"SELECT {} FROM {} WHERE id = {}"}; + storages::clickhouse::ParameterStore params1; + params1.PushBack("name"); + params1.PushBack("users"); + EXPECT_THROW(cluster->Execute(q, params1), std::runtime_error); + + storages::clickhouse::ParameterStore params2; + params2.PushBack("name"); + EXPECT_THROW(cluster->Execute(q, params2), std::runtime_error); + + storages::clickhouse::ParameterStore params3; + EXPECT_THROW(cluster->Execute(q, params3), std::runtime_error); +} + +UTEST(ClickhouseParameterStore, MoreParamsThanPlaceholders) { + ClusterWrapper cluster; + cluster->Execute(storages::Query{"CREATE TABLE IF NOT EXISTS users (id UInt64, name String) ENGINE = Memory"}); + storages::Query q{"SELECT {} FROM {} WHERE id = {}"}; + storages::clickhouse::ParameterStore params; + params.PushBack("name"); + params.PushBack("users"); + params.PushBack(42); + params.PushBack("extra"); + EXPECT_NO_THROW(cluster->Execute(q, params)); +} + +UTEST(ClickhouseParameterStore, NoPlaceholdersNoParams) { + ClusterWrapper cluster; + storages::Query q{"SELECT 1"}; + storages::clickhouse::ParameterStore params; + EXPECT_NO_THROW(cluster->Execute(q, params)); +} + +UTEST(ClickhouseParameterStore, NoPlaceholdersWithParams) { + ClusterWrapper cluster; + storages::Query q{"SELECT 1"}; + storages::clickhouse::ParameterStore params; + params.PushBack(1); + EXPECT_NO_THROW(cluster->Execute(q, params)); +} + +UTEST(ClickhouseParameterStore, OnlyBraces) { + ClusterWrapper cluster; + storages::Query q{"SELECT {}"}; + storages::clickhouse::ParameterStore params1; + params1.PushBack(1); + EXPECT_NO_THROW(cluster->Execute(q, params1)); + + storages::clickhouse::ParameterStore params2; + EXPECT_THROW(cluster->Execute(q, params2), std::runtime_error); + + storages::clickhouse::ParameterStore params3; + params3.PushBack(1); + params3.PushBack(2); + EXPECT_NO_THROW(cluster->Execute(q, params3)); +} + +UTEST(ClickhouseParameterStore, MultipleBracesCases) { + ClusterWrapper cluster; + storages::Query q{"SELECT {}, {}, {}"}; + storages::clickhouse::ParameterStore params1; + params1.PushBack(1); + params1.PushBack(2); + params1.PushBack(3); + EXPECT_NO_THROW(cluster->Execute(q, params1)); + + storages::clickhouse::ParameterStore params2; + params2.PushBack(1); + params2.PushBack(2); + EXPECT_THROW(cluster->Execute(q, params2), std::runtime_error); + + storages::clickhouse::ParameterStore params3; + params3.PushBack(1); + params3.PushBack(2); + params3.PushBack(3); + params3.PushBack(4); + EXPECT_NO_THROW(cluster->Execute(q, params3)); +} + +UTEST(ClickhouseParameterStore, IndexedPlaceholdersExactParams) { + ClusterWrapper cluster; + cluster->Execute(storages::Query{"CREATE TABLE IF NOT EXISTS users (id UInt64, name String) ENGINE = Memory"}); + storages::Query q{"SELECT {1} FROM {0} WHERE id = {2}"}; + storages::clickhouse::ParameterStore params; + params.PushBack("users"); + params.PushBack("name"); + params.PushBack(42); + EXPECT_NO_THROW(cluster->Execute(q, params)); +} + +UTEST(ClickhouseParameterStore, IndexedPlaceholdersLessParams) { + ClusterWrapper cluster; + storages::Query q{"SELECT {0}, {1}, {2} FROM users"}; + storages::clickhouse::ParameterStore params; + params.PushBack("id"); + params.PushBack("name"); + EXPECT_THROW(cluster->Execute(q, params), std::runtime_error); +} + +UTEST(ClickhouseParameterStore, IndexedPlaceholdersMoreParams) { + ClusterWrapper cluster; + storages::Query q{"SELECT {2}, {1}, {0} FROM users"}; + storages::clickhouse::ParameterStore params; + params.PushBack("name"); + params.PushBack("id"); + params.PushBack("age"); + params.PushBack("extra"); + EXPECT_NO_THROW(cluster->Execute(q, params)); +} + +UTEST(ClickhouseParameterStore, IndexedPlaceholdersRepeatedIndexes) { + ClusterWrapper cluster; + storages::Query q{"SELECT {0}, {0}, {1} FROM users"}; + storages::clickhouse::ParameterStore params; + params.PushBack("id"); + params.PushBack("name"); + EXPECT_NO_THROW(cluster->Execute(q, params)); +} + +UTEST(ClickhouseParameterStore, IndexedPlaceholdersNonSequential) { + ClusterWrapper cluster; + storages::Query q{"SELECT {2}, {0}, {5} FROM users"}; + storages::clickhouse::ParameterStore params; + params.PushBack("first"); + params.PushBack("second"); + params.PushBack("third"); + params.PushBack("fourth"); + params.PushBack("fifth"); + params.PushBack("sixth"); + EXPECT_NO_THROW(cluster->Execute(q, params)); +} + +UTEST(ClickhouseParameterStore, IndexedPlaceholdersTooFewParamsForMaxIndex) { + ClusterWrapper cluster; + storages::Query q{"SELECT {2}, {0}, {5} FROM users"}; + storages::clickhouse::ParameterStore params; + params.PushBack("first"); + params.PushBack("second"); + params.PushBack("third"); + EXPECT_THROW(cluster->Execute(q, params), std::runtime_error); +} + +UTEST(ClickhouseParameterStore, IndexedPlaceholdersZeroParams) { + ClusterWrapper cluster; + storages::Query q{"SELECT {0} FROM users"}; + storages::clickhouse::ParameterStore params; + EXPECT_THROW(cluster->Execute(q, params), std::runtime_error); +} + +USERVER_NAMESPACE_END From cbb60196f7c4387109c209ea0a1d528709f7998e Mon Sep 17 00:00:00 2001 From: "alexander.klyuchev" Date: Mon, 28 Jul 2025 19:27:58 +0300 Subject: [PATCH 013/151] feat grpc: added client certs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI Co-authored-by: Antony Polukhin Pull Request resolved: https://github.com/userver-framework/userver/pull/954 commit_hash:5b4acf7b0709a4b34d63c300e502a2539367a189 --- .../ugrpc/client/client_factory_component.hpp | 3 +- .../ugrpc/client/client_factory_component.cpp | 24 ++++++++++- .../client/impl/client_factory_config.cpp | 40 +++++++++++++++---- .../client/impl/client_factory_config.hpp | 2 + scripts/docs/en/userver/grpc/grpc.md | 33 ++++++++++----- 5 files changed, 80 insertions(+), 22 deletions(-) diff --git a/grpc/include/userver/ugrpc/client/client_factory_component.hpp b/grpc/include/userver/ugrpc/client/client_factory_component.hpp index 9c8d00a3edd6..5af0145db360 100644 --- a/grpc/include/userver/ugrpc/client/client_factory_component.hpp +++ b/grpc/include/userver/ugrpc/client/client_factory_component.hpp @@ -52,7 +52,8 @@ using MiddlewareRunnerComponentBase = USERVER_NAMESPACE::middlewares::RunnerComp /// Name | Description | Default value /// ---- | ----------- | ------------- /// channel-args | a map of channel arguments, see gRPC Core docs | {} -/// auth-type | authentication method, see above | - +/// auth-type | authentication method, see @ref grpc_ssl_authentication "Authentication" | - +/// ssl-credentials-options | TLS/SSL options, see @ref grpc_ssl_authentication "Authentication" | - /// default-service-config | default service config, see above | - /// channel-count | Number of underlying grpc::Channel objects | 1 /// middlewares | middlewares names to use | - diff --git a/grpc/src/ugrpc/client/client_factory_component.cpp b/grpc/src/ugrpc/client/client_factory_component.cpp index 4803e5740e2d..0e2a5ae1d9f6 100644 --- a/grpc/src/ugrpc/client/client_factory_component.cpp +++ b/grpc/src/ugrpc/client/client_factory_component.cpp @@ -36,9 +36,11 @@ ClientFactoryComponent::ClientFactoryComponent( const auto config_source = context.FindComponent().GetSource(); auto& testsuite_grpc = context.FindComponent().GetGrpcControl(); - auto factory_config = config.As(); - + if (!testsuite_grpc.IsTlsEnabled() && factory_config.auth_type == impl::AuthType::kSsl) { + LOG_INFO() << "Disabling TLS/SSL dues to testsuite config for gRPC"; + factory_config.auth_type = impl::AuthType::kInsecure; + } const auto* secdist = GetSecdist(context); factory_.emplace( @@ -75,6 +77,24 @@ additionalProperties: false enum: - insecure - ssl + ssl-credentials-options: + type: object + description: SSL options for cases when `auth-type` is `ssl` + defaultDescription: '{}' + additionalProperties: false + properties: + pem-root-certs: + type: string + description: The path to file containing the PEM encoding of the server root certificates + defaultDescription: absent + pem-private-key: + type: string + description: The path to file containing the PEM encoding of the client's private key + defaultDescription: absent + pem-cert-chain: + type: string + description: The path to file containing the PEM encoding of the client's certificate chain + defaultDescription: absent auth-token: type: string description: auth token name from secdist diff --git a/grpc/src/ugrpc/client/impl/client_factory_config.cpp b/grpc/src/ugrpc/client/impl/client_factory_config.cpp index 68c3861588fc..211d30fdc996 100644 --- a/grpc/src/ugrpc/client/impl/client_factory_config.cpp +++ b/grpc/src/ugrpc/client/impl/client_factory_config.cpp @@ -7,6 +7,8 @@ #include #include +#include +#include #include USERVER_NAMESPACE_BEGIN @@ -15,14 +17,33 @@ namespace ugrpc::client::impl { namespace { -std::shared_ptr MakeDefaultCredentials(impl::AuthType type) { - switch (type) { - case AuthType::kInsecure: - return grpc::InsecureChannelCredentials(); - case AuthType::kSsl: - return grpc::SslCredentials({}); +std::shared_ptr MakeCredentials(const ClientFactoryConfig& config) { + if (config.auth_type == AuthType::kSsl) { + LOG_INFO() << "GRPC client (SSL) initialized..."; + return grpc::SslCredentials(config.ssl_credentials_options); + } else { + LOG_INFO() << "GRPC client (non SSL) initialized..."; + return grpc::InsecureChannelCredentials(); + } +} + +grpc::SslCredentialsOptions MakeCredentialsOptions(const yaml_config::YamlConfig& config) { + grpc::SslCredentialsOptions result; + if (config.IsMissing()) { + return result; + } + + if (!config["pem-root-certs"].IsMissing()) { + result.pem_root_certs = fs::blocking::ReadFileContents(config["pem-root-certs"].As()); + } + if (!config["pem-private-key"].IsMissing()) { + result.pem_private_key = fs::blocking::ReadFileContents(config["pem-private-key"].As()); + } + if (!config["pem-cert-chain"].IsMissing()) { + result.pem_cert_chain = fs::blocking::ReadFileContents(config["pem-cert-chain"].As()); } - UINVARIANT(false, "Invalid AuthType"); + + return result; } grpc::ChannelArguments MakeChannelArgs(const yaml_config::YamlConfig& channel_args) { @@ -54,6 +75,9 @@ AuthType Parse(const yaml_config::YamlConfig& value, formats::parse::To) { ClientFactoryConfig config; config.auth_type = value["auth-type"].As(AuthType::kInsecure); + if (config.auth_type == AuthType::kSsl) { + config.ssl_credentials_options = MakeCredentialsOptions(value["ssl-credentials-options"]); + } config.channel_args = MakeChannelArgs(value["channel-args"]); config.default_service_config = value["default-service-config"].As>(); config.channel_count = value["channel-count"].As(config.channel_count); @@ -62,7 +86,7 @@ ClientFactoryConfig Parse(const yaml_config::YamlConfig& value, formats::parse:: ClientFactorySettings MakeFactorySettings(ClientFactoryConfig&& config, const storages::secdist::SecdistConfig* secdist) { - auto credentials = MakeDefaultCredentials(config.auth_type); + std::shared_ptr credentials = MakeCredentials(config); std::unordered_map> client_credentials; if (secdist) { diff --git a/grpc/src/ugrpc/client/impl/client_factory_config.hpp b/grpc/src/ugrpc/client/impl/client_factory_config.hpp index 52d86eb318c7..4537de1734b3 100644 --- a/grpc/src/ugrpc/client/impl/client_factory_config.hpp +++ b/grpc/src/ugrpc/client/impl/client_factory_config.hpp @@ -18,6 +18,8 @@ enum class AuthType { struct ClientFactoryConfig final { AuthType auth_type{AuthType::kInsecure}; + grpc::SslCredentialsOptions ssl_credentials_options{}; + /// Optional grpc-core channel args /// @see https://grpc.github.io/grpc/core/group__grpc__arg__keys.html grpc::ChannelArguments channel_args{}; diff --git a/scripts/docs/en/userver/grpc/grpc.md b/scripts/docs/en/userver/grpc/grpc.md index ef52229ce266..b5c8b307ed50 100644 --- a/scripts/docs/en/userver/grpc/grpc.md +++ b/scripts/docs/en/userver/grpc/grpc.md @@ -63,9 +63,10 @@ Read the documentation on gRPC streams: On errors, exceptions from userver/ugrpc/client/exceptions.hpp are thrown. It is recommended to catch them outside the entire stream interaction. You can catch exceptions for [specific gRPC error codes](https://grpc.github.io/grpc/core/md_doc_statuscodes.html) or all at once. +@anchor grpc_ssl_authentication ### TLS / SSL -May be enabled for gRPC client via: +May be enabled for gRPC client via @ref ugrpc::client::ClientFactoryComponent static config option: ``` # yaml @@ -73,15 +74,25 @@ components_manager: components: grpc-client-factory: auth-type: ssl + ssl-credentials-options: + pem-root-certs: /path/to/server/cert + pem-private-key: /path/to/private/key + pem-cert-chain: /path/to/cert/chain ``` -Available values are: +Available `auth-type` values are: - `insecure` (default) - `ssl` -SSL has to be disabled in tests, because it -requires the server to have a public domain name, which it does not in tests. +Default (system) authentication keys are used by default. To change that provide the `ssl-credentials-options` static +config options with the following options: + +* `pem-root-certs` - a path to file containing the PEM encoding of the server root certificates +* `pem-private-key` - a path to file containing the PEM encoding of the client's private key +* `pem-cert-chain` - The path to file containing the PEM encoding of the client's certificate chain + +SSL has to be disabled in tests, because it requires the server to have a public domain name, which it does not in tests. In testsuite, SSL in gRPC clients is disabled automatically. @@ -89,13 +100,13 @@ In testsuite, SSL in gRPC clients is disabled automatically. Main page: @ref scripts/docs/en/userver/grpc/client_middlewares.md. -Client behaviour can be modified with a middleware. Middleware code is executed before or after the client code. +Client behaviour can be modified with a middleware. Middleware code is executed before or after the client code. Use @ref ugrpc::client::MiddlewareBase to implement new middlewares. #### List of standard client middlewares: 1. `grpc-client-logging` with component ugrpc::client::middlewares::log::Component - logs requests and responses. - 2. `grpc-client-deadline-propagation` with component ugrpc::client::middlewares::deadline_propagation::Component - activates + 2. `grpc-client-deadline-propagation` with component ugrpc::client::middlewares::deadline_propagation::Component - activates @ref scripts/docs/en/userver/deadline_propagation.md. 3. `grpc-client-baggage` with component ugrpc::client::middlewares::baggage::Component - passes request baggage to subrequests. 3. `grpc-client-headers-propagator` with component ugrpc::client::middlewares::headers_propagator::Component - propagates headers. @@ -175,7 +186,7 @@ Use ugrpc::server::MiddlewareBase to implement new middlewares. #### List of standard server middlewares: 1. `grpc-server-logging` with component ugrpc::server::middlewares::log::Component - logs requests. - 2. `grpc-server-deadline-propagation` with component ugrpc::server::middlewares::deadline_propagation::Component - activates + 2. `grpc-server-deadline-propagation` with component ugrpc::server::middlewares::deadline_propagation::Component - activates @ref scripts/docs/en/userver/deadline_propagation.md. 3. `grpc-server-congestion-control` with component ugrpc::server::middlewares::congestion_control::Component - limits requests. See Congestion Control section of @ref scripts/docs/en/userver/tutorial/production_service.md. @@ -193,7 +204,7 @@ See native [docs about compression](https://github.com/grpc/grpc/blob/master/doc ### Key notes from the native gRPC documentation 1. When a compression level is not specified for either the channel or the message, the default channel level none is considered: data MUST NOT be compressed. - 2. You can set a compression **algorithm** and a compression **level** on the server side. + 2. You can set a compression **algorithm** and a compression **level** on the server side. 3. On client side, one can set a compression **algorithm**. 4. `GRPC_COMPRESS_LEVEL_LOW` mapping to "gzip -3" and `GRPC_COMPRESS_LEVEL_HIGH` mapping to "gzip -9". @@ -213,7 +224,7 @@ Config example: ## gRPC Logs -Each gRPC call generates a span ( `tracing::Span` ) containing tags which are inherited by all child logs. +Each gRPC call generates a span ( `tracing::Span` ) containing tags which are inherited by all child logs. Additionally, if logging is activated, a separate log is generated for every gRPC request-response in `grpc-server-logging` and `grpc-client-logging` middlewares. @@ -224,7 +235,7 @@ gRPC logs are listed below. Middleware component name | Key | Value ---------------------------------- |----------------------- | ------------------------ builtin | `error` | error flag, `true` -builtin | `grpc_code` | error code `grpc::StatusCode`, [list](https://grpc.github.io/grpc/core/md_doc_statuscodes.html) +builtin | `grpc_code` | error code `grpc::StatusCode`, [list](https://grpc.github.io/grpc/core/md_doc_statuscodes.html) builtin | `error_msg` | error message `grpc-client-logging` | `meta_type` | call name `grpc-client-logging` | `grpc_type` | `request` or `response` @@ -270,7 +281,7 @@ message Creds { grpc-core is a lower level library, its logs are forwarded to the userver default logger. In this process only error level logs get through from grpc-core to the userver default logger if the default settings are used. However, the -default settings can be overridden and more verbose logging can be achieved. +default settings can be overridden and more verbose logging can be achieved. To do this you need to change the value of `native-log-level` in the static config file in the components `grpc-client-common` and `grpc-server`: From 9fd73bbfc7541f3c26cc7402993168a22b704e12 Mon Sep 17 00:00:00 2001 From: Tikhon Date: Mon, 28 Jul 2025 20:17:21 +0300 Subject: [PATCH 014/151] feat redis: add sentinel_password option and make sure that it work for non-subscribe cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds support for password authentication when connecting to Redis Sentinel. Closes: https://github.com/userver-framework/userver/issues/924 Allows configuring Sentinel password via `sentinel_password` option in secdist config. Example: ```json { "redis_settings": { "taxi-tmp": { "password": "redispwd", "sentinel_password": "sentpwd", /* <---- `AUTH sentpwd` will be executed on connection to sentinel` */ "database_index": 5, "secure_connection": false, "shards": [ { "name": "mymaster" } ], "sentinels": [ { "host": "127.0.0.1", "port": 26379 } ] } } } ``` Tests: протестировано CI Pull Request resolved: https://github.com/userver-framework/userver/pull/925 commit_hash:84ec2914e9b505c1655600cf3ce6ccf2281ae75c --- .../userver/storages/redis/component.hpp | 1 + .../src/storages/redis/impl/secdist_redis.hpp | 1 + redis/src/storages/redis/impl/sentinel.cpp | 4 +- redis/src/storages/redis/impl/server_test.cpp | 171 ++++++++++++++++++ .../redis/impl/subscribe_sentinel.cpp | 8 +- redis/src/storages/redis/redis_secdist.cpp | 2 + 6 files changed, 183 insertions(+), 4 deletions(-) diff --git a/redis/include/userver/storages/redis/component.hpp b/redis/include/userver/storages/redis/component.hpp index 7470c47abe1e..59e23d20fd8b 100644 --- a/redis/include/userver/storages/redis/component.hpp +++ b/redis/include/userver/storages/redis/component.hpp @@ -99,6 +99,7 @@ namespace components { /// "redis_settings": { /// "some_name_of_your_database": { /// "password": "the_password_of_your_database", +/// "sentinel_password": "the_password_for_sentinels_if_any", /// "sentinels": [ /// {"host": "the_host1_of_your_database", "port": 11564} /// ], diff --git a/redis/src/storages/redis/impl/secdist_redis.hpp b/redis/src/storages/redis/impl/secdist_redis.hpp index 5b2765d27153..7e855e8239e8 100644 --- a/redis/src/storages/redis/impl/secdist_redis.hpp +++ b/redis/src/storages/redis/impl/secdist_redis.hpp @@ -21,6 +21,7 @@ struct RedisSettings { std::vector shards; std::vector sentinels; storages::redis::Password password{std::string()}; + storages::redis::Password sentinel_password{std::string()}; storages::redis::ConnectionSecurity secure_connection{storages::redis::ConnectionSecurity::kNone}; std::size_t database_index{0}; }; diff --git a/redis/src/storages/redis/impl/sentinel.cpp b/redis/src/storages/redis/impl/sentinel.cpp index 4d9b24c24754..736b8667b5e6 100644 --- a/redis/src/storages/redis/impl/sentinel.cpp +++ b/redis/src/storages/redis/impl/sentinel.cpp @@ -147,6 +147,7 @@ std::shared_ptr Sentinel::CreateSentinel( const testsuite::RedisControl& testsuite_redis_control ) { const auto& password = settings.password; + const auto& sentinel_password = settings.sentinel_password; const std::vector& shards = settings.shards; LOG_DEBUG() << "shards.size() = " << shards.size(); @@ -157,13 +158,12 @@ std::shared_ptr Sentinel::CreateSentinel( LOG_DEBUG() << "sentinels.size() = " << settings.sentinels.size(); for (const auto& sentinel : settings.sentinels) { LOG_DEBUG() << "sentinel: host = " << sentinel.host << " port = " << sentinel.port; - // SENTINEL MASTERS/SLAVES works without auth, sentinel has no AUTH command. // CLUSTER SLOTS works after auth only. Masters and slaves used instead of // sentinels in cluster mode. conns.emplace_back( sentinel.host, sentinel.port, - (key_shard_factory.IsClusterStrategy() ? password : Password("")), + (key_shard_factory.IsClusterStrategy() ? password : sentinel_password), false, settings.secure_connection ); diff --git a/redis/src/storages/redis/impl/server_test.cpp b/redis/src/storages/redis/impl/server_test.cpp index 268c137db991..13d9b6835e69 100644 --- a/redis/src/storages/redis/impl/server_test.cpp +++ b/redis/src/storages/redis/impl/server_test.cpp @@ -7,7 +7,14 @@ #include #include #include +#include #include +#include +#include +#include +#include + +#include USERVER_NAMESPACE_BEGIN @@ -44,6 +51,52 @@ bool IsConnected(const storages::redis::impl::Redis& redis) { return redis.GetState() == storages::redis::RedisState::kConnected; } +struct MockSentinelServers { + static constexpr size_t kRedisThreadCount = 1; + static constexpr std::string_view kRedisName = "redis_name"; + + void RegisterSentinelMastersSlaves() { + std::vector slave_infos; + std::string redis_name{kRedisName}; + for (const auto& slave : slaves) { + slave_infos.emplace_back(redis_name, kLocalhost, slave.GetPort()); + } + + for (auto& sentinel : sentinels) { + sentinel.RegisterSentinelMastersHandler({{redis_name, kLocalhost, masters[0].GetPort()}}); + sentinel.RegisterSentinelSlavesHandler(redis_name, slave_infos); + } + } + + template + void ForEachServer(const Function& visitor) { + for (auto& server : masters) { + visitor(server); + } + for (auto& server : slaves) { + visitor(server); + } + for (auto& server : sentinels) { + visitor(server); + } + } + + MockRedisServer masters[1] = { + MockRedisServer{"master0"}, + }; + MockRedisServer slaves[2] = { + MockRedisServer{"slave0"}, + MockRedisServer{"slave1"}, + }; + MockRedisServer sentinels[3] = { + MockRedisServer{"sentinel0"}, + MockRedisServer{"sentinel1"}, + MockRedisServer{"sentinel2"}, + }; + std::shared_ptr thread_pool = + std::make_shared(1, kRedisThreadCount); +}; + } // namespace TEST(Redis, NoPassword) { @@ -101,6 +154,124 @@ TEST(Redis, AuthTimeout) { PeriodicCheck([&] { return !IsConnected(*redis); }); } +UTEST(Redis, SentinelAuth) { + MockSentinelServers mock; + mock.RegisterSentinelMastersSlaves(); + mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); + auto& [masters, slaves, sentinels, thread_pool] = mock; + + secdist::RedisSettings settings; + settings.shards = {std::string{MockSentinelServers::kRedisName}}; + settings.sentinel_password = storages::redis::Password("pass"); + settings.sentinels.reserve(std::size(sentinels)); + for (const auto& sentinel : sentinels) { + settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); + } + + std::vector auth_handlers; + auth_handlers.reserve(std::size(sentinels)); + for (auto& sentinel : sentinels) { + auth_handlers.push_back(sentinel.RegisterStatusReplyHandler("AUTH", "OK")); + } + std::vector no_auth_handlers; + no_auth_handlers.reserve(std::size(masters) + std::size(slaves)); + for (auto& server : masters) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + for (auto& server : slaves) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + + auto sentinel_client = storages::redis::impl::Sentinel::CreateSentinel( + thread_pool, settings, "test_shard_group_name", dynamic_config::GetDefaultSource(), "test_client_name", {""} + ); + sentinel_client->WaitConnectedDebug(std::empty(slaves)); + + for (auto& handler : auth_handlers) { + EXPECT_TRUE(handler->WaitForFirstReply(kSmallPeriod)); + } + + for (auto& handler : no_auth_handlers) { + EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); + } + + for (const auto& sentinel : sentinels) { + EXPECT_TRUE(sentinel.WaitForFirstPingReply(kSmallPeriod)); + } +} + +// TODO: TAXICOMMON-10834. Looks like AUTH to sentinel is not sent in case of SUBSCRIBE +UTEST(Redis, DISABLED_SentinelAuthSubscribe) { + MockSentinelServers mock; + mock.RegisterSentinelMastersSlaves(); + mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); + auto& [masters, slaves, sentinels, thread_pool] = mock; + // Sentinels do NOT receive SUBSCRIBE + std::vector subscribe_handlers; + for (auto& server : masters) { + subscribe_handlers.push_back(server.RegisterHandlerWithConstReply("SUBSCRIBE", 1)); + } + for (auto& server : slaves) { + subscribe_handlers.push_back(server.RegisterHandlerWithConstReply("SUBSCRIBE", 1)); + } + + secdist::RedisSettings settings; + settings.shards = {std::string{MockSentinelServers::kRedisName}}; + settings.sentinel_password = storages::redis::Password("pass"); + settings.sentinels.reserve(std::size(sentinels)); + for (const auto& sentinel : sentinels) { + settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); + } + + std::vector auth_handlers; + auth_handlers.reserve(std::size(sentinels)); + for (auto& sentinel : sentinels) { + auth_handlers.push_back(sentinel.RegisterStatusReplyHandler("AUTH", "OK")); + } + std::vector no_auth_handlers; + no_auth_handlers.reserve(std::size(masters) + std::size(slaves)); + for (auto& server : masters) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + for (auto& server : slaves) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + + storages::redis::CommandControl cc{}; + testsuite::RedisControl redis_control{}; + auto subscribe_sentinel = storages::redis::impl::SubscribeSentinel::Create( + thread_pool, + settings, + "test_shard_group_name", + dynamic_config::GetDefaultSource(), + "test_client_name", + {""}, + cc, + redis_control + ); + subscribe_sentinel->WaitConnectedDebug(std::empty(slaves)); + std::shared_ptr client = + std::make_shared(std::move(subscribe_sentinel)); + + storages::redis::SubscriptionToken::OnMessageCb callback = [](const std::string& channel, + const std::string& message) { + EXPECT_TRUE(false) << "Should not be called. Channel = " << channel << ", message = " << message; + }; + auto subscription = client->Subscribe("channel_name", std::move(callback)); + + for (auto& handler : subscribe_handlers) { + EXPECT_TRUE(handler->WaitForFirstReply(utest::kMaxTestWaitTime)); + } + + for (auto& handler : auth_handlers) { + EXPECT_TRUE(handler->WaitForFirstReply(kSmallPeriod)); + } + + for (auto& handler : no_auth_handlers) { + EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); + } +} + TEST(Redis, Select) { MockRedisServer server{"redis_db"}; auto ping_handler = server.RegisterPingHandler(); diff --git a/redis/src/storages/redis/impl/subscribe_sentinel.cpp b/redis/src/storages/redis/impl/subscribe_sentinel.cpp index 5def26657735..0197ba10b467 100644 --- a/redis/src/storages/redis/impl/subscribe_sentinel.cpp +++ b/redis/src/storages/redis/impl/subscribe_sentinel.cpp @@ -84,6 +84,7 @@ std::shared_ptr SubscribeSentinel::Create( const testsuite::RedisControl& testsuite_redis_control ) { const auto& password = settings.password; + const auto& sentinel_password = settings.password; const std::vector& shards = settings.shards; LOG_DEBUG() << "shards.size() = " << shards.size(); @@ -96,11 +97,14 @@ std::shared_ptr SubscribeSentinel::Create( LOG_DEBUG() << "sentinels.size() = " << settings.sentinels.size(); for (const auto& sentinel : settings.sentinels) { LOG_DEBUG() << "sentinel: host = " << sentinel.host << " port = " << sentinel.port; - // SENTINEL MASTERS/SLAVES works without auth, sentinel has no AUTH command. // CLUSTER SLOTS works after auth only. Masters and slaves used instead of // sentinels in cluster mode. conns.emplace_back( - sentinel.host, sentinel.port, (is_cluster_mode ? password : Password("")), false, settings.secure_connection + sentinel.host, + sentinel.port, + (is_cluster_mode ? password : sentinel_password), + false, + settings.secure_connection ); } LOG_DEBUG() << "redis command_control: " << command_control.ToString(); diff --git a/redis/src/storages/redis/redis_secdist.cpp b/redis/src/storages/redis/redis_secdist.cpp index 7b6dd5bb9070..af5a3b1d0d53 100644 --- a/redis/src/storages/redis/redis_secdist.cpp +++ b/redis/src/storages/redis/redis_secdist.cpp @@ -35,6 +35,8 @@ RedisMapSettings::RedisMapSettings(const formats::json::Value& doc) { USERVER_NAMESPACE::secdist::RedisSettings settings; settings.password = storages::redis::Password(GetString(client_settings, "password")); + settings.sentinel_password = + storages::redis::Password(client_settings["sentinel_password"].As("")); settings.secure_connection = GetValue(client_settings, "secure_connection", false) ? USERVER_NAMESPACE::storages::redis::ConnectionSecurity::kTLS : USERVER_NAMESPACE::storages::redis::ConnectionSecurity::kNone; From cb3053fdcf7ca16d59ea960c148d58bbb2c82d0c Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Mon, 28 Jul 2025 21:34:55 +0300 Subject: [PATCH 015/151] fix grpc: fix test 'DeadlineToTimespec.Base' commit_hash:4c8a30fe5fe6f3e9d6f232c7d841b1d831b7f634 --- grpc/tests/time_utils_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grpc/tests/time_utils_test.cpp b/grpc/tests/time_utils_test.cpp index 4a4241b95ca0..062d62682bda 100644 --- a/grpc/tests/time_utils_test.cpp +++ b/grpc/tests/time_utils_test.cpp @@ -22,8 +22,8 @@ UTEST(DeadlineToTimespec, Base) { const auto deadline = ugrpc::TimespecToDeadline(t); const auto hi = engine::Deadline::FromDuration(duration); EXPECT_TRUE(deadline.IsReachable()); - EXPECT_LT(lo, deadline); - EXPECT_LT(deadline, hi); + EXPECT_THAT(lo, testing::AnyOf(testing::Lt(deadline), testing::Eq(deadline))); + EXPECT_THAT(deadline, testing::AnyOf(testing::Lt(hi), testing::Eq(hi))); } UTEST(DeadlineToTimespec, FromUnreachableDeadline) { From 583e0ddbed4c19d667acf674e325a1e7d2c1bee8 Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Mon, 28 Jul 2025 21:36:58 +0300 Subject: [PATCH 016/151] refactor grpc: introduce MiddlewarePipeline commit_hash:5e94d96016586df4aeb0a3b5c3e8d480dc963f7f --- .mapping.json | 2 + .../userver/ugrpc/client/impl/call_state.hpp | 8 ++- .../ugrpc/client/impl/middleware_hooks.hpp | 38 ++++++++++ .../ugrpc/client/impl/middleware_pipeline.hpp | 18 +++-- .../include/userver/ugrpc/client/impl/rpc.hpp | 24 +++---- .../ugrpc/client/stream_read_future.hpp | 2 +- grpc/src/ugrpc/client/common_component.cpp | 1 - grpc/src/ugrpc/client/impl/async_methods.cpp | 6 +- grpc/src/ugrpc/client/impl/call_state.cpp | 10 ++- .../ugrpc/client/impl/middleware_hooks.cpp | 71 +++++++++++++++++++ .../ugrpc/client/impl/middleware_pipeline.cpp | 37 +++------- grpc/src/ugrpc/client/impl/rpc.cpp | 1 - grpc/src/ugrpc/tests/standalone_client.cpp | 2 - 13 files changed, 153 insertions(+), 67 deletions(-) create mode 100644 grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp create mode 100644 grpc/src/ugrpc/client/impl/middleware_hooks.cpp diff --git a/.mapping.json b/.mapping.json index 62b9ac0b0f80..4c81e29e5bbe 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2120,6 +2120,7 @@ "grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp", "grpc/include/userver/ugrpc/client/impl/completion_queue_pool.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/completion_queue_pool.hpp", "grpc/include/userver/ugrpc/client/impl/graceful_stream_finish.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/graceful_stream_finish.hpp", + "grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp", "grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp", "grpc/include/userver/ugrpc/client/impl/prepare_call.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/prepare_call.hpp", "grpc/include/userver/ugrpc/client/impl/rpc.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/rpc.hpp", @@ -2242,6 +2243,7 @@ "grpc/src/ugrpc/client/impl/client_factory_config.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_factory_config.hpp", "grpc/src/ugrpc/client/impl/client_qos.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_qos.cpp", "grpc/src/ugrpc/client/impl/completion_queue_pool.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/completion_queue_pool.cpp", + "grpc/src/ugrpc/client/impl/middleware_hooks.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/middleware_hooks.cpp", "grpc/src/ugrpc/client/impl/middleware_pipeline.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/middleware_pipeline.cpp", "grpc/src/ugrpc/client/impl/retry_policy.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/retry_policy.cpp", "grpc/src/ugrpc/client/impl/retry_policy.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/retry_policy.hpp", diff --git a/grpc/include/userver/ugrpc/client/impl/call_state.hpp b/grpc/include/userver/ugrpc/client/impl/call_state.hpp index 1b2f48a14fe9..190a2d3dc274 100644 --- a/grpc/include/userver/ugrpc/client/impl/call_state.hpp +++ b/grpc/include/userver/ugrpc/client/impl/call_state.hpp @@ -13,8 +13,8 @@ #include #include +#include #include -#include #include #include #include @@ -56,7 +56,7 @@ class CallState { const RpcConfigValues& GetConfigValues() const noexcept; - const Middlewares& GetMiddlewares() const noexcept; + const MiddlewarePipeline& GetMiddlewarePipeline() const noexcept; CallKind GetCallKind() const noexcept; @@ -88,7 +88,7 @@ class CallState { RpcConfigValues config_values_; - const Middlewares& middlewares_; + MiddlewarePipeline middleware_pipeline_; CallKind call_kind_{}; @@ -194,6 +194,8 @@ bool IsWriteAndCheckAvailable(const StreamingCallState&) noexcept; void HandleCallStatistics(CallState& state, const grpc::Status& status) noexcept; +void RunMiddlewarePipeline(CallState& state, const MiddlewareHooks& hooks); + } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp b/grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp new file mode 100644 index 000000000000..9649715f82fe --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { +class MiddlewareBase; +class MiddlewareCallContext; +} // namespace ugrpc::client + +namespace ugrpc::client::impl { + +class MiddlewareHooks { +public: + void SetStartCall() noexcept; + void SetSendMessage(const google::protobuf::Message& send_message) noexcept; + void SetRecvMessage(const google::protobuf::Message& recv_message) noexcept; + void SetStatus(const grpc::Status& status) noexcept; + + void Run(const MiddlewareBase& middleware, MiddlewareCallContext& context) const; + +private: + bool start_call_{false}; + const google::protobuf::Message* send_message_{}; + const google::protobuf::Message* recv_message_{}; + const grpc::Status* status_{}; +}; + +MiddlewareHooks StartCallHooks(const google::protobuf::Message* request = nullptr) noexcept; +MiddlewareHooks SendMessageHooks(const google::protobuf::Message& send_message) noexcept; +MiddlewareHooks RecvMessageHooks(const google::protobuf::Message& recv_message) noexcept; +MiddlewareHooks FinishHooks(const grpc::Status& status, const google::protobuf::Message* response = nullptr) noexcept; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp b/grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp index 2edf785cc6dd..347ad8e1bf1b 100644 --- a/grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp +++ b/grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp @@ -1,22 +1,20 @@ #pragma once -#include -#include - -#include +#include +#include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -struct MiddlewarePipeline { - static void PreStartCall(CallState& state); - - static void PreSendMessage(CallState& state, const google::protobuf::Message& message); +class MiddlewarePipeline { +public: + explicit MiddlewarePipeline(const Middlewares& middlewares) : middlewares_{middlewares} {} - static void PostRecvMessage(CallState& state, const google::protobuf::Message& message); + void Run(const MiddlewareHooks& hooks, MiddlewareCallContext& context) const; - static void PostFinish(CallState& state); +private: + const Middlewares& middlewares_; }; } // namespace ugrpc::client::impl diff --git a/grpc/include/userver/ugrpc/client/impl/rpc.hpp b/grpc/include/userver/ugrpc/client/impl/rpc.hpp index 97c4fb2597e5..a7b9ab5c83ec 100644 --- a/grpc/include/userver/ugrpc/client/impl/rpc.hpp +++ b/grpc/include/userver/ugrpc/client/impl/rpc.hpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include @@ -335,11 +334,9 @@ UnaryCall::UnaryCall( const Request& request ) : state_{std::move(params)}, context_{utils::impl::InternalTag{}, state_} { - MiddlewarePipeline::PreStartCall(state_); - if constexpr (std::is_base_of_v) { - MiddlewarePipeline::PreSendMessage(state_, request); - } + RunMiddlewarePipeline(state_, StartCallHooks(ToBaseMessage(&request))); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) reader_ = prepare_unary_call(state_.GetStub(), &state_.GetClientContext(), request, &state_.GetQueue()); reader_->StartCall(); @@ -382,8 +379,7 @@ InputStream::InputStream( const Request& request ) : state_{std::move(params), CallKind::kInputStream}, context_{utils::impl::InternalTag{}, state_} { - MiddlewarePipeline::PreStartCall(state_); - MiddlewarePipeline::PreSendMessage(state_, request); + RunMiddlewarePipeline(state_, StartCallHooks(ToBaseMessage(&request))); // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) stream_ = impl::PrepareCall( @@ -408,7 +404,7 @@ bool InputStream::Read(Response& response) { } if (impl::Read(*stream_, response, state_)) { - MiddlewarePipeline::PostRecvMessage(state_, response); + RunMiddlewarePipeline(state_, RecvMessageHooks(response)); return true; } else { // Finish can only be called once all the data is read, otherwise the @@ -425,7 +421,7 @@ OutputStream::OutputStream( PrepareClientStreamingCall prepare_async_method ) : state_{std::move(params), CallKind::kOutputStream}, context_{utils::impl::InternalTag{}, state_} { - MiddlewarePipeline::PreStartCall(state_); + RunMiddlewarePipeline(state_, StartCallHooks()); // 'response_' will be filled upon successful 'Finish' async call // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) @@ -448,7 +444,7 @@ bool OutputStream::Write(const Request& request) { return false; } - MiddlewarePipeline::PreSendMessage(state_, request); + RunMiddlewarePipeline(state_, SendMessageHooks(request)); // Don't buffer writes, otherwise in an event subscription scenario, events // may never actually be delivered @@ -464,7 +460,7 @@ void OutputStream::WriteAndCheck(const Request& request) { throw RpcError(state_.GetCallName(), "'WriteAndCheck' called on a finished or closed stream"); } - MiddlewarePipeline::PreSendMessage(state_, request); + RunMiddlewarePipeline(state_, SendMessageHooks(request)); // Don't buffer writes, otherwise in an event subscription scenario, events // may never actually be delivered @@ -495,7 +491,7 @@ BidirectionalStream::BidirectionalStream( PrepareBidiStreamingCall prepare_async_method ) : state_{std::move(params), CallKind::kBidirectionalStream}, context_{utils::impl::InternalTag{}, state_} { - MiddlewarePipeline::PreStartCall(state_); + RunMiddlewarePipeline(state_, StartCallHooks()); // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) stream_ = impl::PrepareCall(prepare_async_method, state_.GetStub(), &state_.GetClientContext(), &state_.GetQueue()); @@ -541,7 +537,7 @@ bool BidirectionalStream::Write(const Request& request) { { const auto lock = state_.TakeMutexIfBidirectional(); - MiddlewarePipeline::PreSendMessage(state_, request); + RunMiddlewarePipeline(state_, SendMessageHooks(request)); } // Don't buffer writes, optimize for ping-pong-style interaction @@ -559,7 +555,7 @@ void BidirectionalStream::WriteAndCheck(const Request& reques { const auto lock = state_.TakeMutexIfBidirectional(); - MiddlewarePipeline::PreSendMessage(state_, request); + RunMiddlewarePipeline(state_, SendMessageHooks(request)); } // Don't buffer writes, optimize for ping-pong-style interaction diff --git a/grpc/include/userver/ugrpc/client/stream_read_future.hpp b/grpc/include/userver/ugrpc/client/stream_read_future.hpp index 6232711b8d8c..3dbfb22376ac 100644 --- a/grpc/include/userver/ugrpc/client/stream_read_future.hpp +++ b/grpc/include/userver/ugrpc/client/stream_read_future.hpp @@ -105,7 +105,7 @@ bool StreamReadFuture::Get() { impl::Finish(*stream_, *state, /*final_response=*/nullptr, /*throw_on_error=*/true); } else { if (recv_message_) { - impl::MiddlewarePipeline::PostRecvMessage(*state, *recv_message_); + RunMiddlewarePipeline(*state, impl::RecvMessageHooks(*recv_message_)); } } return result == ugrpc::impl::AsyncMethodInvocation::WaitStatus::kOk; diff --git a/grpc/src/ugrpc/client/common_component.cpp b/grpc/src/ugrpc/client/common_component.cpp index 3c9e9eaceb1e..08d6639fef0c 100644 --- a/grpc/src/ugrpc/client/common_component.cpp +++ b/grpc/src/ugrpc/client/common_component.cpp @@ -6,7 +6,6 @@ #include #include -#include #include USERVER_NAMESPACE_BEGIN diff --git a/grpc/src/ugrpc/client/impl/async_methods.cpp b/grpc/src/ugrpc/client/impl/async_methods.cpp index 8b50b94e84cc..52dfb743b37b 100644 --- a/grpc/src/ugrpc/client/impl/async_methods.cpp +++ b/grpc/src/ugrpc/client/impl/async_methods.cpp @@ -46,11 +46,7 @@ void ProcessFinish(CallState& state, const google::protobuf::Message* final_resp HandleCallStatistics(state, status); - if (final_response && status.ok()) { - MiddlewarePipeline::PostRecvMessage(state, *final_response); - } - - MiddlewarePipeline::PostFinish(state); + RunMiddlewarePipeline(state, FinishHooks(status, final_response)); SetStatusAndResetSpan(state, status); } diff --git a/grpc/src/ugrpc/client/impl/call_state.cpp b/grpc/src/ugrpc/client/impl/call_state.cpp index b0eaec7e618c..fda710a6ea69 100644 --- a/grpc/src/ugrpc/client/impl/call_state.cpp +++ b/grpc/src/ugrpc/client/impl/call_state.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -57,7 +58,7 @@ CallState::CallState(CallParams&& params, CallKind call_kind) stats_scope_(params.statistics), queue_(params.queue), config_values_(params.config), - middlewares_(params.middlewares), + middleware_pipeline_(params.middlewares), call_kind_(call_kind) { UINVARIANT(!client_name_.empty(), "client name should not be empty"); @@ -77,7 +78,7 @@ grpc::CompletionQueue& CallState::GetQueue() const noexcept { return queue_; } const RpcConfigValues& CallState::GetConfigValues() const noexcept { return config_values_; } -const Middlewares& CallState::GetMiddlewares() const noexcept { return middlewares_; } +const MiddlewarePipeline& CallState::GetMiddlewarePipeline() const noexcept { return middleware_pipeline_; } std::string_view CallState::GetCallName() const noexcept { return call_name_.Get(); } @@ -204,6 +205,11 @@ void HandleCallStatistics(CallState& state, const grpc::Status& status) noexcept stats.Flush(); } +void RunMiddlewarePipeline(CallState& state, const MiddlewareHooks& hooks) { + MiddlewareCallContext middleware_call_context{state}; + state.GetMiddlewarePipeline().Run(hooks, middleware_call_context); +} + } // namespace ugrpc::client::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/middleware_hooks.cpp b/grpc/src/ugrpc/client/impl/middleware_hooks.cpp new file mode 100644 index 000000000000..df13ef1c6eb9 --- /dev/null +++ b/grpc/src/ugrpc/client/impl/middleware_hooks.cpp @@ -0,0 +1,71 @@ +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +void MiddlewareHooks::SetStartCall() noexcept { start_call_ = true; } + +void MiddlewareHooks::SetSendMessage(const google::protobuf::Message& send_message) noexcept { + send_message_ = &send_message; +} + +void MiddlewareHooks::SetRecvMessage(const google::protobuf::Message& recv_message) noexcept { + recv_message_ = &recv_message; +} + +void MiddlewareHooks::SetStatus(const grpc::Status& status) noexcept { status_ = &status; } + +void MiddlewareHooks::Run(const MiddlewareBase& middleware, MiddlewareCallContext& context) const { + if (start_call_) { + middleware.PreStartCall(context); + } + + if (send_message_) { + middleware.PreSendMessage(context, *send_message_); + } + + if (recv_message_) { + middleware.PostRecvMessage(context, *recv_message_); + } + + if (status_) { + middleware.PostFinish(context, *status_); + } +} + +MiddlewareHooks StartCallHooks(const google::protobuf::Message* request) noexcept { + MiddlewareHooks hooks; + hooks.SetStartCall(); + if (request) { + hooks.SetSendMessage(*request); + } + return hooks; +} + +MiddlewareHooks SendMessageHooks(const google::protobuf::Message& send_message) noexcept { + MiddlewareHooks hooks; + hooks.SetSendMessage(send_message); + return hooks; +} + +MiddlewareHooks RecvMessageHooks(const google::protobuf::Message& recv_message) noexcept { + MiddlewareHooks hooks; + hooks.SetRecvMessage(recv_message); + return hooks; +} + +MiddlewareHooks FinishHooks(const grpc::Status& status, const google::protobuf::Message* response) noexcept { + MiddlewareHooks hooks; + if (status.ok() && response) { + hooks.SetRecvMessage(*response); + } + hooks.SetStatus(status); + return hooks; +} + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/middleware_pipeline.cpp b/grpc/src/ugrpc/client/impl/middleware_pipeline.cpp index c9c9128e74b8..ef47e2ee16c2 100644 --- a/grpc/src/ugrpc/client/impl/middleware_pipeline.cpp +++ b/grpc/src/ugrpc/client/impl/middleware_pipeline.cpp @@ -1,38 +1,19 @@ #include -#include -#include +#include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -void MiddlewarePipeline::PreStartCall(CallState& state) { - MiddlewareCallContext context{state}; - for (const auto& mw : state.GetMiddlewares()) { - mw->PreStartCall(context); - } -} - -void MiddlewarePipeline::PreSendMessage(CallState& state, const google::protobuf::Message& message) { - MiddlewareCallContext context{state}; - for (const auto& mw : state.GetMiddlewares()) { - mw->PreSendMessage(context, message); - } -} - -void MiddlewarePipeline::PostRecvMessage(CallState& state, const google::protobuf::Message& message) { - MiddlewareCallContext context{state}; - for (const auto& mw : state.GetMiddlewares()) { - mw->PostRecvMessage(context, message); - } -} - -void MiddlewarePipeline::PostFinish(CallState& state) { - const auto& status = state.GetStatus(); - MiddlewareCallContext context{state}; - for (const auto& mw : state.GetMiddlewares()) { - mw->PostFinish(context, status); +void MiddlewarePipeline::Run(const MiddlewareHooks& hooks, MiddlewareCallContext& context) const { + try { + for (const auto& m : middlewares_) { + hooks.Run(*m, context); + } + } catch (const std::exception& ex) { + LOG_WARNING() << "Run middlewares failed: " << ex; + throw; } } diff --git a/grpc/src/ugrpc/client/impl/rpc.cpp b/grpc/src/ugrpc/client/impl/rpc.cpp index ebfaf75f3e70..adc895149c1d 100644 --- a/grpc/src/ugrpc/client/impl/rpc.cpp +++ b/grpc/src/ugrpc/client/impl/rpc.cpp @@ -3,7 +3,6 @@ #include #include -#include USERVER_NAMESPACE_BEGIN diff --git a/grpc/src/ugrpc/tests/standalone_client.cpp b/grpc/src/ugrpc/tests/standalone_client.cpp index a432bc8cf784..ace1ec42930c 100644 --- a/grpc/src/ugrpc/tests/standalone_client.cpp +++ b/grpc/src/ugrpc/tests/standalone_client.cpp @@ -5,8 +5,6 @@ #include #include -#include - USERVER_NAMESPACE_BEGIN namespace ugrpc::tests { From a48d540009ff3e34eaa7077ce0d3b275a4fb3e67 Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 28 Jul 2025 22:42:25 +0300 Subject: [PATCH 017/151] docs userver: translate dynconfigs desc to English commit_hash:e3cd63fb33593f04b125cd2cbf4aa0c04a6d3380 --- .../HTTP_CLIENT_CONNECTION_POOL_SIZE.yaml | 3 +- .../HTTP_CLIENT_CONNECT_THROTTLE.yaml | 8 +- core/dynamic_configs/USERVER_CACHES.yaml | 2 +- ...VER_CANCEL_HANDLE_REQUEST_BY_DEADLINE.yaml | 3 +- core/dynamic_configs/USERVER_DUMPS.yaml | 4 +- .../USERVER_FILES_CONTENT_TYPE_MAP.yaml | 2 +- core/dynamic_configs/USERVER_HTTP_PROXY.yaml | 6 +- .../USERVER_LOG_DYNAMIC_DEBUG.yaml | 5 +- core/dynamic_configs/USERVER_LOG_REQUEST.yaml | 2 +- .../USERVER_LOG_REQUEST_HEADERS.yaml | 4 +- ...USERVER_LOG_REQUEST_HEADERS_WHITELIST.yaml | 7 +- core/dynamic_configs/USERVER_LRU_CACHES.yaml | 3 +- .../dynamic_configs/USERVER_NO_LOG_SPANS.yaml | 6 +- .../dynamic_configs/USERVER_RPS_CCONTROL.yaml | 3 +- .../USERVER_RPS_CCONTROL_ENABLED.yaml | 4 +- ...USERVER_TASK_PROCESSOR_PROFILER_DEBUG.yaml | 4 +- .../USERVER_TASK_PROCESSOR_QOS.yaml | 2 +- ...CONGESTION_CONTROL_DATABASES_SETTINGS.yaml | 4 + .../MONGO_CONGESTION_CONTROL_SETTINGS.yaml | 2 +- .../MONGO_CONNECTION_POOL_SETTINGS.yaml | 4 + .../MONGO_DEFAULT_MAX_TIME_MS.yaml | 4 +- ...STGRES_CONNECTION_PIPELINE_EXPERIMENT.yaml | 4 +- .../POSTGRES_CONNECTION_POOL_SETTINGS.yaml | 26 +- .../POSTGRES_CONNECTION_SETTINGS.yaml | 21 +- .../POSTGRES_CONNLIMIT_MODE_AUTO_ENABLED.yaml | 6 +- .../POSTGRES_DEFAULT_COMMAND_CONTROL.yaml | 8 +- .../POSTGRES_HANDLERS_COMMAND_CONTROL.yaml | 9 +- .../POSTGRES_OMIT_DESCRIBE_IN_EXECUTE.yaml | 4 +- .../POSTGRES_QUERIES_COMMAND_CONTROL.yaml | 21 +- .../POSTGRES_STATEMENT_METRICS_SETTINGS.yaml | 17 +- .../POSTGRES_TOPOLOGY_SETTINGS.yaml | 22 +- .../REDIS_COMMANDS_BUFFERING_SETTINGS.yaml | 13 +- .../REDIS_DEFAULT_COMMAND_CONTROL.yaml | 13 +- .../REDIS_METRICS_SETTINGS.yaml | 7 +- .../REDIS_PUBSUB_METRICS_SETTINGS.yaml | 7 +- .../REDIS_REPLICA_MONITORING_SETTINGS.yaml | 7 +- .../REDIS_RETRY_BUDGET_SETTINGS.yaml | 8 +- ...IS_SUBSCRIBER_DEFAULT_COMMAND_CONTROL.yaml | 3 +- ...PTIONS_REBALANCE_MIN_INTERVAL_SECONDS.yaml | 4 +- .../dynamic_configs/REDIS_WAIT_CONNECTED.yaml | 4 +- scripts/docs/en/schemas/dynamic_configs.md | 1182 ----------------- 41 files changed, 242 insertions(+), 1226 deletions(-) diff --git a/core/dynamic_configs/HTTP_CLIENT_CONNECTION_POOL_SIZE.yaml b/core/dynamic_configs/HTTP_CLIENT_CONNECTION_POOL_SIZE.yaml index 16049a3f7cea..1a566ad63216 100644 --- a/core/dynamic_configs/HTTP_CLIENT_CONNECTION_POOL_SIZE.yaml +++ b/core/dynamic_configs/HTTP_CLIENT_CONNECTION_POOL_SIZE.yaml @@ -1,5 +1,6 @@ default: 1000 -description: '' +description: Open connections pool size for curl (CURLMOPT_MAXCONNECTS). `-1` means + "detect by multiplying easy handles count on 4". schema: x-usrv-cpp-type: std::size_t type: integer diff --git a/core/dynamic_configs/HTTP_CLIENT_CONNECT_THROTTLE.yaml b/core/dynamic_configs/HTTP_CLIENT_CONNECT_THROTTLE.yaml index c7ce51c4273d..b6b35d76659b 100644 --- a/core/dynamic_configs/HTTP_CLIENT_CONNECT_THROTTLE.yaml +++ b/core/dynamic_configs/HTTP_CLIENT_CONNECT_THROTTLE.yaml @@ -1,7 +1,13 @@ default: max-size: 100 token-update-interval-ms: 0 -description: '' +description: | + Token bucket throttling options for new connections (socket(3)). + * `*-limit` - token bucket size, set to `0` to disable the limit + * `*-per-second` - token bucket size refill speed + 0 as any value disables the throttling. + First, the http-*/https-* limit is processed, after that per-host-* limit is processed. + All three token buckets work independently on all cluster hosts. schema: type: object properties: diff --git a/core/dynamic_configs/USERVER_CACHES.yaml b/core/dynamic_configs/USERVER_CACHES.yaml index 1426338ba4fe..7dae90b0473d 100644 --- a/core/dynamic_configs/USERVER_CACHES.yaml +++ b/core/dynamic_configs/USERVER_CACHES.yaml @@ -1,5 +1,5 @@ default: {} -description: '' +description: Cache update dynamic parameters schema: type: object additionalProperties: diff --git a/core/dynamic_configs/USERVER_CANCEL_HANDLE_REQUEST_BY_DEADLINE.yaml b/core/dynamic_configs/USERVER_CANCEL_HANDLE_REQUEST_BY_DEADLINE.yaml index 35ff566e5c41..c4bc776398d5 100644 --- a/core/dynamic_configs/USERVER_CANCEL_HANDLE_REQUEST_BY_DEADLINE.yaml +++ b/core/dynamic_configs/USERVER_CANCEL_HANDLE_REQUEST_BY_DEADLINE.yaml @@ -1,4 +1,5 @@ default: false -description: '' +description: Controls whether the http request task should be cancelled when the deadline + received from the client is reached. schema: type: boolean diff --git a/core/dynamic_configs/USERVER_DUMPS.yaml b/core/dynamic_configs/USERVER_DUMPS.yaml index ab2da0531dea..be0527d848bb 100644 --- a/core/dynamic_configs/USERVER_DUMPS.yaml +++ b/core/dynamic_configs/USERVER_DUMPS.yaml @@ -1,5 +1,7 @@ default: {} -description: '' +description: | + Dynamic dump configuration. If the options are set for some dump then those + options override the static configuration. schema: type: object additionalProperties: diff --git a/core/dynamic_configs/USERVER_FILES_CONTENT_TYPE_MAP.yaml b/core/dynamic_configs/USERVER_FILES_CONTENT_TYPE_MAP.yaml index f0668ce39334..dccfdd85cffc 100644 --- a/core/dynamic_configs/USERVER_FILES_CONTENT_TYPE_MAP.yaml +++ b/core/dynamic_configs/USERVER_FILES_CONTENT_TYPE_MAP.yaml @@ -10,7 +10,7 @@ default: .png: "image/png" .svg: "image/svg+xml" __default__: "text/plain" -description: '' +description: Dynamic config for mapping extension files with HTTP header content type. schema: type: object properties: diff --git a/core/dynamic_configs/USERVER_HTTP_PROXY.yaml b/core/dynamic_configs/USERVER_HTTP_PROXY.yaml index 75bb218c708a..ba0c06e3349c 100644 --- a/core/dynamic_configs/USERVER_HTTP_PROXY.yaml +++ b/core/dynamic_configs/USERVER_HTTP_PROXY.yaml @@ -1,4 +1,8 @@ default: '' -description: '' +description: | + Proxy for all the HTTP and HTTPS clients. Empty string disables proxy usage. + + Proxy string may be prefixed with `[scheme]://` to specify which kind of proxy is used. Schemes match the [libcurl supported ones](https://curl.se/libcurl/c/CURLOPT_PROXY.html). + A proxy host string can also embed user and password. schema: type: string diff --git a/core/dynamic_configs/USERVER_LOG_DYNAMIC_DEBUG.yaml b/core/dynamic_configs/USERVER_LOG_DYNAMIC_DEBUG.yaml index 473c8b9d7bf3..24a128c707b3 100644 --- a/core/dynamic_configs/USERVER_LOG_DYNAMIC_DEBUG.yaml +++ b/core/dynamic_configs/USERVER_LOG_DYNAMIC_DEBUG.yaml @@ -1,7 +1,10 @@ default: force-enabled: [] force-disabled: [] -description: '' +description: | + Logging per line and file overrides. + Log locations are defined as path prefix from the Arcadia root (`taxi/uservices/services/`). + Location of file may be followed by `:[line index]` to specify 1 exact log in that file. schema: type: object additionalProperties: false diff --git a/core/dynamic_configs/USERVER_LOG_REQUEST.yaml b/core/dynamic_configs/USERVER_LOG_REQUEST.yaml index a39ebb899183..84a4cdcb5f45 100644 --- a/core/dynamic_configs/USERVER_LOG_REQUEST.yaml +++ b/core/dynamic_configs/USERVER_LOG_REQUEST.yaml @@ -1,4 +1,4 @@ default: true -description: '' +description: Controls HTTP requests and responses logging. schema: type: boolean diff --git a/core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS.yaml b/core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS.yaml index 35ff566e5c41..8536334d06ac 100644 --- a/core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS.yaml +++ b/core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS.yaml @@ -1,4 +1,6 @@ default: false -description: '' +description: | + Controls whether the logging of HTTP headers in handlers is performed. + To ensure safety, all header values will be output as `***` unless specified in @ref USERVER_LOG_REQUEST_HEADERS_WHITELIST. schema: type: boolean diff --git a/core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS_WHITELIST.yaml b/core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS_WHITELIST.yaml index eda08fd8e3f0..53e14bfd00a4 100644 --- a/core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS_WHITELIST.yaml +++ b/core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS_WHITELIST.yaml @@ -1,5 +1,10 @@ default: [] -description: '' +description: | + If the @ref USERVER_LOG_REQUEST_HEADERS option is enabled, + you can control which HTTP headers are logged, including their values. + Header is suitable if it exactly matches one of the values in the whitelist. + Any headers that are not on the whitelist will have their values replaced with *** in the logs. + E.g. ["User-Agent", "Accept-Encoding"]. schema: type: array x-taxi-cpp-type: std::unordered_set diff --git a/core/dynamic_configs/USERVER_LRU_CACHES.yaml b/core/dynamic_configs/USERVER_LRU_CACHES.yaml index 1c5d78e2e823..43a628d3ebfe 100644 --- a/core/dynamic_configs/USERVER_LRU_CACHES.yaml +++ b/core/dynamic_configs/USERVER_LRU_CACHES.yaml @@ -1,5 +1,6 @@ default: {} -description: '' +description: Dynamic config for controlling size and cache entry lifetime of the LRU + based caches. schema: type: object additionalProperties: diff --git a/core/dynamic_configs/USERVER_NO_LOG_SPANS.yaml b/core/dynamic_configs/USERVER_NO_LOG_SPANS.yaml index 46078000bd38..f17f4f954273 100644 --- a/core/dynamic_configs/USERVER_NO_LOG_SPANS.yaml +++ b/core/dynamic_configs/USERVER_NO_LOG_SPANS.yaml @@ -1,7 +1,11 @@ default: prefixes: [] names: [] -description: '' +description: | + Prefixes or full names of tracing::Span instances to not log. + + In order for `USERVER_NO_LOG_SPANS` to work, you should `Append` components::LoggingConfigurator + component or use components::CommonComponentList. schema: type: object additionalProperties: false diff --git a/core/dynamic_configs/USERVER_RPS_CCONTROL.yaml b/core/dynamic_configs/USERVER_RPS_CCONTROL.yaml index a2e3e0bfd0ce..011d72fb1ffa 100644 --- a/core/dynamic_configs/USERVER_RPS_CCONTROL.yaml +++ b/core/dynamic_configs/USERVER_RPS_CCONTROL.yaml @@ -7,7 +7,8 @@ default: up-level: 2 down-level: 1 no-limit-seconds: 1000 -description: '' +description: | + Dynamic config for components::Server congestion control. schema: type: object additionalProperties: false diff --git a/core/dynamic_configs/USERVER_RPS_CCONTROL_ENABLED.yaml b/core/dynamic_configs/USERVER_RPS_CCONTROL_ENABLED.yaml index 3478eab09950..a260baa8cc0a 100644 --- a/core/dynamic_configs/USERVER_RPS_CCONTROL_ENABLED.yaml +++ b/core/dynamic_configs/USERVER_RPS_CCONTROL_ENABLED.yaml @@ -1,5 +1,7 @@ default: false -description: '' +description: | + Controls whether congestion control limiting of RPS is performed (if main task processor is overloaded, + then the server starts rejecting some requests). schema: type: boolean diff --git a/core/dynamic_configs/USERVER_TASK_PROCESSOR_PROFILER_DEBUG.yaml b/core/dynamic_configs/USERVER_TASK_PROCESSOR_PROFILER_DEBUG.yaml index 258e32a21ee1..158934b84afb 100644 --- a/core/dynamic_configs/USERVER_TASK_PROCESSOR_PROFILER_DEBUG.yaml +++ b/core/dynamic_configs/USERVER_TASK_PROCESSOR_PROFILER_DEBUG.yaml @@ -7,7 +7,9 @@ default: enabled: false # 1sec execution-slice-threshold-us: 1000000 -description: '' +description: | + Dynamic config for profiling the coroutine based engine of userver. + Dictionary key names are the names of engine::TaskProcessor. schema: type: object properties: {} diff --git a/core/dynamic_configs/USERVER_TASK_PROCESSOR_QOS.yaml b/core/dynamic_configs/USERVER_TASK_PROCESSOR_QOS.yaml index 690004701be8..75a116cebf92 100644 --- a/core/dynamic_configs/USERVER_TASK_PROCESSOR_QOS.yaml +++ b/core/dynamic_configs/USERVER_TASK_PROCESSOR_QOS.yaml @@ -5,7 +5,7 @@ default: action: ignore length_limit: 5000 time_limit_us: 3000 -description: '' +description: Controls engine::TaskProcessor quality of service dynamic config. schema: type: object required: diff --git a/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_DATABASES_SETTINGS.yaml b/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_DATABASES_SETTINGS.yaml index 2115d3eb3620..67bf1fb0e05c 100644 --- a/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_DATABASES_SETTINGS.yaml +++ b/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_DATABASES_SETTINGS.yaml @@ -2,6 +2,10 @@ default: {} description: | Whether Congestion Control is enabled for specified MongoDB databases. Overrides settings from MONGO_CONGESTION_CONTROL_ENABLED. + + Dictionary keys can be either the service **component name** (not database name!) + or `__default__`. The latter configuration is applied for every non-matching + Mongo component of the service. schema: type: object example: | diff --git a/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_SETTINGS.yaml b/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_SETTINGS.yaml index 3d5a1ec2c0c5..741093610e57 100644 --- a/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_SETTINGS.yaml +++ b/mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_SETTINGS.yaml @@ -1,5 +1,5 @@ default: {} -description: '' +description: Congestion Control settings for MongoDB schema: type: object additionalProperties: false diff --git a/mongo/dynamic_configs/MONGO_CONNECTION_POOL_SETTINGS.yaml b/mongo/dynamic_configs/MONGO_CONNECTION_POOL_SETTINGS.yaml index d2b1c37411ac..20856612851d 100644 --- a/mongo/dynamic_configs/MONGO_CONNECTION_POOL_SETTINGS.yaml +++ b/mongo/dynamic_configs/MONGO_CONNECTION_POOL_SETTINGS.yaml @@ -3,6 +3,10 @@ description: | Options for MongoDB connections pool (uservices) for service. This options replace static config. When using MultiMongo all pools are updated. + + Dictionary keys can be either the service **component name** (not database name!) + or `__default__`. The latter configuration is applied for every non-matching + Mongo component of the service. schema: type: object example: | diff --git a/mongo/dynamic_configs/MONGO_DEFAULT_MAX_TIME_MS.yaml b/mongo/dynamic_configs/MONGO_DEFAULT_MAX_TIME_MS.yaml index 510d4a72c6cb..cbb842647679 100644 --- a/mongo/dynamic_configs/MONGO_DEFAULT_MAX_TIME_MS.yaml +++ b/mongo/dynamic_configs/MONGO_DEFAULT_MAX_TIME_MS.yaml @@ -1,5 +1,7 @@ default: 0 -description: '' +description: | + Dynamic config that controls default $maxTimeMS for mongo requests + (0 - disables default timeout). schema: x-usrv-cpp-type: std::chrono::milliseconds type: integer diff --git a/postgresql/dynamic_configs/POSTGRES_CONNECTION_PIPELINE_EXPERIMENT.yaml b/postgresql/dynamic_configs/POSTGRES_CONNECTION_PIPELINE_EXPERIMENT.yaml index da574df790d7..51faa418ef6a 100644 --- a/postgresql/dynamic_configs/POSTGRES_CONNECTION_PIPELINE_EXPERIMENT.yaml +++ b/postgresql/dynamic_configs/POSTGRES_CONNECTION_PIPELINE_EXPERIMENT.yaml @@ -1,5 +1,7 @@ default: 0 -description: '' +description: | + Dynamic config that turns on pipeline-mode experiment for pg connections. The value + specifies the current version of the experiment. Specify 0 to disable. schema: type: integer minimum: 0 diff --git a/postgresql/dynamic_configs/POSTGRES_CONNECTION_POOL_SETTINGS.yaml b/postgresql/dynamic_configs/POSTGRES_CONNECTION_POOL_SETTINGS.yaml index d9b475b9fd8d..c2e98cd85ff1 100644 --- a/postgresql/dynamic_configs/POSTGRES_CONNECTION_POOL_SETTINGS.yaml +++ b/postgresql/dynamic_configs/POSTGRES_CONNECTION_POOL_SETTINGS.yaml @@ -1,7 +1,31 @@ default: {} -description: '' +description: | + Dynamic config that controls connection pool settings of PostgreSQL driver. + + Dictionary keys can be either the service **component name** (not database name!) + or `__default__`. The latter configuration is applied for every non-matching + PostgreSQL component of the service. + + Take note that it overrides the static configuration values of the service! schema: type: object + example: | + { + // Settings for a specific component. + "postgresql-orders": { + "min_pool_size": 8, + "max_pool_size": 50, + "max_queue_size": 200, + "connecting_limit": 10 + }, + // All other components. + "__default__": { + "min_pool_size": 4, + "max_pool_size": 15, + "max_queue_size": 200, + "connecting_limit": 8 + } + } properties: __default__: $ref: "#/definitions/PoolSettings" diff --git a/postgresql/dynamic_configs/POSTGRES_CONNECTION_SETTINGS.yaml b/postgresql/dynamic_configs/POSTGRES_CONNECTION_SETTINGS.yaml index 57115f817e30..e86f925ca333 100644 --- a/postgresql/dynamic_configs/POSTGRES_CONNECTION_SETTINGS.yaml +++ b/postgresql/dynamic_configs/POSTGRES_CONNECTION_SETTINGS.yaml @@ -1,7 +1,26 @@ default: {} -description: '' +description: | + Dynamic config that controls settings for newly created connections of + PostgreSQL driver. + + Dictionary keys can be either the service **component name** (not database name!) + or `__default__`. The latter configuration is applied for every non-matching + PostgreSQL component of the service. + + Take note that it overrides the static configuration values of the service! schema: type: object + example: | + { + "__default__": { + "persistent-prepared-statements": true, + "user-types-enabled": true, + "check-user-types": false, + "max-prepared-cache-size": 5000, + "ignore-unused-query-params": false, + "recent-errors-threshold": 2 + } + } properties: __default__: $ref: "#/definitions/ConnectionSettings" diff --git a/postgresql/dynamic_configs/POSTGRES_CONNLIMIT_MODE_AUTO_ENABLED.yaml b/postgresql/dynamic_configs/POSTGRES_CONNLIMIT_MODE_AUTO_ENABLED.yaml index 97231207637d..293e5a43fb23 100644 --- a/postgresql/dynamic_configs/POSTGRES_CONNLIMIT_MODE_AUTO_ENABLED.yaml +++ b/postgresql/dynamic_configs/POSTGRES_CONNLIMIT_MODE_AUTO_ENABLED.yaml @@ -1,4 +1,8 @@ default: false -description: 'Option used to only disable a connlimit-auto mode.' +description: | + Dynamic config that enables connlimit_mode: auto for PostgreSQL connections. + Auto mode ignores static and dynamic max_connections configs and verifies + that the cluster services use max_connections equals to PostgreSQL server's + max_connections divided by service instance count. schema: type: boolean diff --git a/postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml b/postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml index 88c22572c1cb..e9b6b4d3f82c 100644 --- a/postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml +++ b/postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml @@ -1,5 +1,9 @@ default: {} -description: '' +description: | + Dynamic config that controls default network and statement timeouts. + Overrides the built-in timeouts from components::Postgres::kDefaultCommandControl, + but could be overridden by POSTGRES_HANDLERS_COMMAND_CONTROL, + POSTGRES_QUERIES_COMMAND_CONTROL and storages::postgres::CommandControl. schema: type: object additionalProperties: false @@ -10,5 +14,3 @@ schema: statement_timeout_ms: type: integer minimum: 1 - prepared_statements_enabled: - type: boolean diff --git a/postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml b/postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml index 6fa2265f91b6..9141daef934e 100644 --- a/postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml +++ b/postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml @@ -1,5 +1,10 @@ default: {} -description: '' +description: | + Dynamic config that controls per-handle statement and network timeouts. + Overrides POSTGRES_DEFAULT_COMMAND_CONTROL and + built-in timeouts from components::Postgres::kDefaultCommandControl, + but may be overridden by POSTGRES_QUERIES_COMMAND_CONTROL and + storages::postgres::CommandControl. schema: type: object additionalProperties: @@ -19,5 +24,3 @@ schema: statement_timeout_ms: type: integer minimum: 1 - prepared_statements_enabled: - type: boolean diff --git a/postgresql/dynamic_configs/POSTGRES_OMIT_DESCRIBE_IN_EXECUTE.yaml b/postgresql/dynamic_configs/POSTGRES_OMIT_DESCRIBE_IN_EXECUTE.yaml index da574df790d7..30eab8c3be05 100644 --- a/postgresql/dynamic_configs/POSTGRES_OMIT_DESCRIBE_IN_EXECUTE.yaml +++ b/postgresql/dynamic_configs/POSTGRES_OMIT_DESCRIBE_IN_EXECUTE.yaml @@ -1,5 +1,7 @@ default: 0 -description: '' +description: | + Dynamic config that turns on omit-describe-in-execute experiment for pg queries. The value + specifies the current version of the experiment. Specify 0 to disable. schema: type: integer minimum: 0 diff --git a/postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml b/postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml index a8ad09107f50..87ac79413fd1 100644 --- a/postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml +++ b/postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml @@ -1,7 +1,24 @@ default: {} -description: '' +description: | + Dynamic config that controls per-query/per-transaction statement and network timeouts, if those were not explicitly set via + storages::postgres::CommandControl. Overrides the @ref POSTGRES_HANDLERS_COMMAND_CONTROL, @ref POSTGRES_QUERIES_COMMAND_CONTROL, + @ref POSTGRES_DEFAULT_COMMAND_CONTROL, and built-in timeouts from components::Postgres::kDefaultCommandControl. + + Transaction timeouts in POSTGRES_QUERIES_COMMAND_CONTROL override the per-query timeouts in POSTGRES_QUERIES_COMMAND_CONTROL, + so the latter are ignored if transaction timeouts are set. schema: type: object + example: | + { + "cleanup_processed_data": { + "network_timeout_ms": 92000, + "statement_timeout_ms": 90000 + }, + "select_recent_users": { + "network_timeout_ms": 70, + "statement_timeout_ms": 30 + } + } additionalProperties: $ref: "#/definitions/CommandControl" definitions: @@ -15,5 +32,3 @@ schema: statement_timeout_ms: type: integer minimum: 1 - prepared_statements_enabled: - type: boolean diff --git a/postgresql/dynamic_configs/POSTGRES_STATEMENT_METRICS_SETTINGS.yaml b/postgresql/dynamic_configs/POSTGRES_STATEMENT_METRICS_SETTINGS.yaml index 11e92c76ab41..2b2f13a67361 100644 --- a/postgresql/dynamic_configs/POSTGRES_STATEMENT_METRICS_SETTINGS.yaml +++ b/postgresql/dynamic_configs/POSTGRES_STATEMENT_METRICS_SETTINGS.yaml @@ -1,7 +1,22 @@ default: {} -description: '' +description: | + Dynamic config that controls statement metrics settings for specific service. + + Dictionary keys can be either the service **component name** (not database name!) + or `__default__`. The latter configuration is applied for every non-matching + PostgreSQL component of the service. + + The value of `max_statement_metrics` controls the maximum size of LRU-cache + for named statement metrics. When set to 0 (default) no metrics are being + exported. schema: type: object + example: | + { + "postgresql-grocery_orders": { + "max_statement_metrics": 50 + } + } additionalProperties: $ref: "#/definitions/StatementMetricsSettings" definitions: diff --git a/postgresql/dynamic_configs/POSTGRES_TOPOLOGY_SETTINGS.yaml b/postgresql/dynamic_configs/POSTGRES_TOPOLOGY_SETTINGS.yaml index 18740863c7b5..e9aba2d9c0b7 100644 --- a/postgresql/dynamic_configs/POSTGRES_TOPOLOGY_SETTINGS.yaml +++ b/postgresql/dynamic_configs/POSTGRES_TOPOLOGY_SETTINGS.yaml @@ -1,7 +1,27 @@ default: {} -description: '' +description: | + Dynamic config that controls topology settings of service's PostgreSQL + components. + + Dictionary keys can be either the service **component name** (not database name!) + or `__default__`. The latter configuration is applied for every non-matching + PostgreSQL component of the service. + + Take note that it overrides the static configuration values of the service! schema: type: object + example: | + { + // settings for a specific component + "postgresql-orders": { + "max_replication_lag_ms": 0, + "disabled_replicas": ["sas-01.db.yandex.net", "vla-02.db.yandex.net"] + }, + // settings for all other components + "__default__": { + "max_replication_lag_ms": 60000 + } + } additionalProperties: $ref: "#/definitions/TopologySettings" definitions: diff --git a/redis/dynamic_configs/REDIS_COMMANDS_BUFFERING_SETTINGS.yaml b/redis/dynamic_configs/REDIS_COMMANDS_BUFFERING_SETTINGS.yaml index d647fb9d582d..3bd804daa118 100644 --- a/redis/dynamic_configs/REDIS_COMMANDS_BUFFERING_SETTINGS.yaml +++ b/redis/dynamic_configs/REDIS_COMMANDS_BUFFERING_SETTINGS.yaml @@ -1,8 +1,19 @@ default: buffering_enabled: false watch_command_timer_interval_us: 0 -description: '' +description: | + Dynamic config that controls command buffering for specific service. + Enabling of this config activates a delay in sending commands. When commands are sent, they are combined into a single tcp packet and sent together. + First command arms timer and then during `watch_command_timer_interval_us` commands are accumulated in the buffer + + Command buffering is disabled by default. schema: + example: | + { + "buffering_enabled": true, + "commands_buffering_threshold": 10, + "watch_command_timer_interval_us": 1000 + } type: object additionalProperties: false properties: diff --git a/redis/dynamic_configs/REDIS_DEFAULT_COMMAND_CONTROL.yaml b/redis/dynamic_configs/REDIS_DEFAULT_COMMAND_CONTROL.yaml index 1646a7a1d13c..a5161c06a862 100644 --- a/redis/dynamic_configs/REDIS_DEFAULT_COMMAND_CONTROL.yaml +++ b/redis/dynamic_configs/REDIS_DEFAULT_COMMAND_CONTROL.yaml @@ -1,6 +1,17 @@ default: {} -description: '' +description: | + Dynamic config that overrides the default timeouts, number of retries and + server selection strategy for redis commands. schema: + example: | + { + "best_dc_count": 0, + "max_ping_latency_ms": 0, + "max_retries": 4, + "strategy": "default", + "timeout_all_ms": 2000, + "timeout_single_ms": 500 + } additionalProperties: false properties: best_dc_count: diff --git a/redis/dynamic_configs/REDIS_METRICS_SETTINGS.yaml b/redis/dynamic_configs/REDIS_METRICS_SETTINGS.yaml index 6eb3c2e3a8e3..b9ee4e94e90d 100644 --- a/redis/dynamic_configs/REDIS_METRICS_SETTINGS.yaml +++ b/redis/dynamic_configs/REDIS_METRICS_SETTINGS.yaml @@ -1,5 +1,10 @@ default: {} -description: '' +description: | + Dynamic config that controls the metric settings for specific service. + + Dictionary keys can be either the **database name** (not the component name!) + or `__default__`. The latter configuration is applied for every non-matching + Redis database/sentinel of the service. schema: type: object additionalProperties: false diff --git a/redis/dynamic_configs/REDIS_PUBSUB_METRICS_SETTINGS.yaml b/redis/dynamic_configs/REDIS_PUBSUB_METRICS_SETTINGS.yaml index b80d451b389a..1bbbbb5d052b 100644 --- a/redis/dynamic_configs/REDIS_PUBSUB_METRICS_SETTINGS.yaml +++ b/redis/dynamic_configs/REDIS_PUBSUB_METRICS_SETTINGS.yaml @@ -1,5 +1,10 @@ default: {} -description: '' +description: | + Dynamic config that controls the redis pubsub metric settings for specific service. + + Dictionary keys can be either the **database name** (not the component name!) + or `__default__`. The latter configuration is applied for every non-matching + Redis database/sentinel of the service. schema: type: object additionalProperties: false diff --git a/redis/dynamic_configs/REDIS_REPLICA_MONITORING_SETTINGS.yaml b/redis/dynamic_configs/REDIS_REPLICA_MONITORING_SETTINGS.yaml index ab1fe3cc4bcd..3579f1a543e7 100644 --- a/redis/dynamic_configs/REDIS_REPLICA_MONITORING_SETTINGS.yaml +++ b/redis/dynamic_configs/REDIS_REPLICA_MONITORING_SETTINGS.yaml @@ -1,5 +1,10 @@ default: {"__default__": {"enable-monitoring": false, "forbid-requests-to-syncing-replicas": false}} -description: '' +description: | + Dynamic config that controls the monitoring settings for synchronizing replicas. + + Dictionary keys can be either the **database name** (not the component name!) + or `__default__`. The latter configuration is applied for every non-matching + Redis database/sentinel of the service. schema: type: object description: settings for each database in service diff --git a/redis/dynamic_configs/REDIS_RETRY_BUDGET_SETTINGS.yaml b/redis/dynamic_configs/REDIS_RETRY_BUDGET_SETTINGS.yaml index 76d777e73bdd..2afaaf94eefc 100644 --- a/redis/dynamic_configs/REDIS_RETRY_BUDGET_SETTINGS.yaml +++ b/redis/dynamic_configs/REDIS_RETRY_BUDGET_SETTINGS.yaml @@ -1,5 +1,11 @@ default: {"__default__": {"max-tokens": 100, "token-ratio": 0.1, "enabled": true}} -description: '' +description: | + Dynamic config that controls the retry budget (throttling) settings for + components::Redis. + + Dictionary keys can be either the **database name** (not the component name!) + or `__default__`. The latter configuration is applied for every non-matching + Redis database/sentinel of the service. schema: type: object description: retry budget settings for each database in service diff --git a/redis/dynamic_configs/REDIS_SUBSCRIBER_DEFAULT_COMMAND_CONTROL.yaml b/redis/dynamic_configs/REDIS_SUBSCRIBER_DEFAULT_COMMAND_CONTROL.yaml index a0c17efaa90e..131cdfb67f8d 100644 --- a/redis/dynamic_configs/REDIS_SUBSCRIBER_DEFAULT_COMMAND_CONTROL.yaml +++ b/redis/dynamic_configs/REDIS_SUBSCRIBER_DEFAULT_COMMAND_CONTROL.yaml @@ -1,5 +1,6 @@ default: {} -description: '' +description: | + The same as REDIS_DEFAULT_COMMAND_CONTROL but for subscription clients. schema: type: object additionalProperties: false diff --git a/redis/dynamic_configs/REDIS_SUBSCRIPTIONS_REBALANCE_MIN_INTERVAL_SECONDS.yaml b/redis/dynamic_configs/REDIS_SUBSCRIPTIONS_REBALANCE_MIN_INTERVAL_SECONDS.yaml index 85898967eb5d..eade8a32d20d 100644 --- a/redis/dynamic_configs/REDIS_SUBSCRIPTIONS_REBALANCE_MIN_INTERVAL_SECONDS.yaml +++ b/redis/dynamic_configs/REDIS_SUBSCRIPTIONS_REBALANCE_MIN_INTERVAL_SECONDS.yaml @@ -1,5 +1,7 @@ default: 30 -description: '' +description: | + Dynamic config that controls the minimal interval between redis subscription + clients rebalancing. schema: minimum: 0 type: integer diff --git a/redis/dynamic_configs/REDIS_WAIT_CONNECTED.yaml b/redis/dynamic_configs/REDIS_WAIT_CONNECTED.yaml index 0b5856c0969d..78b2bffc1be2 100644 --- a/redis/dynamic_configs/REDIS_WAIT_CONNECTED.yaml +++ b/redis/dynamic_configs/REDIS_WAIT_CONNECTED.yaml @@ -2,7 +2,9 @@ default: mode: master_or_slave throw_on_fail: false timeout-ms: 11000 -description: '' +description: | + Dynamic config that controls if services will wait for connections with redis + instances. schema: type: object additionalProperties: false diff --git a/scripts/docs/en/schemas/dynamic_configs.md b/scripts/docs/en/schemas/dynamic_configs.md index 7a8d01d123fc..501f5cb76b4b 100644 --- a/scripts/docs/en/schemas/dynamic_configs.md +++ b/scripts/docs/en/schemas/dynamic_configs.md @@ -1,644 +1,3 @@ -## Dynamic config schemas - -Here you can find schemas of dynamic configs used by userver itself. -For general information on dynamic configs, see -@ref scripts/docs/en/userver/dynamic_config.md - - -@anchor HTTP_CLIENT_CONNECT_THROTTLE -## HTTP_CLIENT_CONNECT_THROTTLE - -Token bucket throttling options for new connections (socket(3)). -* `*-limit` - token bucket size, set to `0` to disable the limit -* `*-per-second` - token bucket size refill speed - -@include core/dynamic_configs/HTTP_CLIENT_CONNECT_THROTTLE.yaml - -**Example:** -```json -{ - "http-limit": 6000, - "http-per-second": 1500, - "https-limit": 100, - "https-per-second": 25, - "per-host-limit": 3000, - "per-host-per-second": 500 -} -``` - -Used by components::HttpClient, affects the behavior of clients::http::Client and all the clients that use it. - - -@anchor HTTP_CLIENT_CONNECTION_POOL_SIZE -## HTTP_CLIENT_CONNECTION_POOL_SIZE -Open connections pool size for curl (CURLMOPT_MAXCONNECTS). `-1` means -"detect by multiplying easy handles count on 4". - -@include core/dynamic_configs/HTTP_CLIENT_CONNECTION_POOL_SIZE.yaml - -**Example:** -``` -5000 -``` - -Used by components::HttpClient, affects the behavior of clients::http::Client and all the clients that use it. - -@anchor MONGO_CONGESTION_CONTROL_DATABASES_SETTINGS -## MONGO_CONGESTION_CONTROL_DATABASES_SETTINGS - -Whether Congestion Control is enabled for specified MongoDB databases. -Overrides settings from @ref MONGO_CONGESTION_CONTROL_ENABLED. - -@include mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_DATABASES_SETTINGS.yaml - -Dictionary keys can be either the service **component name** (not database name!) -or `__default__`. The latter configuration is applied for every non-matching -Mongo component of the service. - -Used by components::Mongo, components::MultiMongo. - - -@anchor MONGO_CONGESTION_CONTROL_ENABLED -## MONGO_CONGESTION_CONTROL_ENABLED - -Whether Congestion Control is enabled for MongoDB - -@include mongo/dynamic_configs/MONGO_CONGESTION_CONTROL_ENABLED.yaml - -**Example:** -``` -true -``` - -Used by components::Mongo, components::MultiMongo. - - -@anchor MONGO_CONGESTION_CONTROL_SETTINGS -## MONGO_CONGESTION_CONTROL_SETTINGS - -Congestion Control settings for MongoDB - -``` -yaml -schema: - type: object - additionalProperties: false - properties: - errors-threshold-percent: - description: Percent of errors to enable СС - type: number - deactivate-delta: - description: СС turned off if this amount of free connections is reached - type: integer - timings-burst-times-threshold: - description: CC is turned on if request times grow to this value - type: number - min-timings-ms: - description: minimal value of timeings after which the CC heuristics turn on - type: integer - min-limit: - description: minimal value of connections after which the CC heuristics turn on - type: integer - min-qps: - description: minimal value of queries per second after which the CC heuristics turn on - type: integer -``` - -Used by components::Mongo, components::MultiMongo. - - -@anchor MONGO_CONNECTION_POOL_SETTINGS -## MONGO_CONNECTION_POOL_SETTINGS - -Options for MongoDB connections pool. Overrides the static config values. -For components::MultiMongo all pools are updated. - -@include mongo/dynamic_configs/MONGO_CONNECTION_POOL_SETTINGS.yaml - -Dictionary keys can be either the service **component name** (not database name!) -or `__default__`. The latter configuration is applied for every non-matching -Mongo component of the service. - -Used by components::Mongo, components::MultiMongo. - - -@anchor MONGO_DEFAULT_MAX_TIME_MS -## MONGO_DEFAULT_MAX_TIME_MS - -Dynamic config that controls default $maxTimeMS for mongo requests (0 - disables default timeout). - -@include mongo/dynamic_configs/MONGO_DEFAULT_MAX_TIME_MS.yaml - -**Example:** -``` -200 -``` - -Used by components::Mongo, components::MultiMongo. - - -@anchor MONGO_DEADLINE_PROPAGATION_ENABLED_V2 -## MONGO_DEADLINE_PROPAGATION_ENABLED_V2 - -Dynamic config that controls whether task-inherited deadline is accounted for -while executing mongodb queries. - -@include mongo/dynamic_configs/MONGO_DEADLINE_PROPAGATION_ENABLED_V2.yaml - -**Example:** -```json -false -``` - -Used by components::Mongo, components::MultiMongo. - - -@anchor POSTGRES_DEFAULT_COMMAND_CONTROL -## POSTGRES_DEFAULT_COMMAND_CONTROL - -Dynamic config that controls default network and statement timeouts. Overrides the built-in timeouts from components::Postgres::kDefaultCommandControl, -but could be overridden by @ref POSTGRES_HANDLERS_COMMAND_CONTROL, @ref POSTGRES_QUERIES_COMMAND_CONTROL and storages::postgres::CommandControl. - -@include postgresql/dynamic_configs/POSTGRES_DEFAULT_COMMAND_CONTROL.yaml - -**Example:** -```json -{ - "network_timeout_ms": 750, - "statement_timeout_ms": 500 -} -``` - -Used by components::Postgres. - - -@anchor POSTGRES_HANDLERS_COMMAND_CONTROL -## POSTGRES_HANDLERS_COMMAND_CONTROL - -Dynamic config that controls per-handle statement and network timeouts. Overrides @ref POSTGRES_DEFAULT_COMMAND_CONTROL and -built-in timeouts from components::Postgres::kDefaultCommandControl, but may be overridden by @ref POSTGRES_QUERIES_COMMAND_CONTROL and -storages::postgres::CommandControl. - -@include postgresql/dynamic_configs/POSTGRES_HANDLERS_COMMAND_CONTROL.yaml - -**Example:** -```json -{ - "/v2/rules/create": { - "POST": { - "network_timeout_ms": 500, - "statement_timeout_ms": 250 - } - }, - "/v2/rules/select": { - "POST": { - "network_timeout_ms": 70, - "statement_timeout_ms": 30 - } - } -} -``` - -Used by components::Postgres. - - -@anchor POSTGRES_QUERIES_COMMAND_CONTROL -## POSTGRES_QUERIES_COMMAND_CONTROL - -Dynamic config that controls per-query/per-transaction statement and network timeouts, if those were not explicitly set via -storages::postgres::CommandControl. Overrides the @ref POSTGRES_HANDLERS_COMMAND_CONTROL, @ref POSTGRES_QUERIES_COMMAND_CONTROL, -@ref POSTGRES_DEFAULT_COMMAND_CONTROL, and built-in timeouts from components::Postgres::kDefaultCommandControl. - -Transaction timeouts in POSTGRES_QUERIES_COMMAND_CONTROL override the per-query timeouts in POSTGRES_QUERIES_COMMAND_CONTROL, -so the latter are ignored if transaction timeouts are set. - -@include postgresql/dynamic_configs/POSTGRES_QUERIES_COMMAND_CONTROL.yaml - -**Example:** -```json -{ - "cleanup_processed_data": { - "network_timeout_ms": 92000, - "statement_timeout_ms": 90000 - }, - "select_recent_users": { - "network_timeout_ms": 70, - "statement_timeout_ms": 30 - } -} -``` - -Used by components::Postgres. - - -@anchor POSTGRES_CONNECTION_POOL_SETTINGS -## POSTGRES_CONNECTION_POOL_SETTINGS - -Dynamic config that controls connection pool settings of PostgreSQL driver. - -Dictionary keys can be either the service **component name** (not database name!) -or `__default__`. The latter configuration is applied for every non-matching -PostgreSQL component of the service. - -Take note that it overrides the static configuration values of the service! - -@include postgresql/dynamic_configs/POSTGRES_CONNECTION_POOL_SETTINGS.yaml - -**Example:** -```json -{ - "__default__": { - "min_pool_size": 4, - "max_pool_size": 15, - "max_queue_size": 200, - "connecting_limit": 10 - }, - "postgresql-orders": { - "min_pool_size": 8, - "max_pool_size": 50, - "max_queue_size": 200, - "connecting_limit": 8 - } -} -``` - -Used by components::Postgres. - - -@anchor POSTGRES_TOPOLOGY_SETTINGS -## POSTGRES_TOPOLOGY_SETTINGS - -Dynamic config that controls topology settings of service's PostgreSQL -components. - -Dictionary keys can be either the service **component name** (not database name!) -or `__default__`. The latter configuration is applied for every non-matching -PostgreSQL component of the service. - -Take note that it overrides the static configuration values of the service! - -``` -yaml -type: object -additionalProperties: false -properties: - max_replication_lag_ms: - type: integer - minimum: 0 - description: maximum allowed replication lag. If equals 0 no replication - lag checks are performed - disabled_replicas: - type: array - description: List of manually disabled replicas (FQDNs). - items: - type: string -required: - - max_replication_lag_ms -``` - -**Example** -```json -{ - "__default__": { - "max_replication_lag_ms": 60000, - "disabled_replicas": ["replica-01.example.com", "replica-02.example.com"] - } -} -``` - -Used by components::Postgres. - - -@anchor POSTGRES_CONNECTION_SETTINGS -## POSTGRES_CONNECTION_SETTINGS - -Dynamic config that controls settings for newly created connections of -PostgreSQL driver. - -Dictionary keys can be either the service **component name** (not database name!) -or `__default__`. The latter configuration is applied for every non-matching -PostgreSQL component of the service. - -Take note that it overrides the static configuration values of the service! - -@include postgresql/dynamic_configs/POSTGRES_CONNECTION_SETTINGS.yaml - -**Example:** -```json -{ - "__default__": { - "persistent-prepared-statements": true, - "user-types-enabled": true, - "max-prepared-cache-size": 5000, - "ignore-unused-query-params": false, - "recent-errors-threshold": 2, - "max-ttl-sec": 3600 - } -} -``` - -Used by components::Postgres. - - -@anchor POSTGRES_CONNLIMIT_MODE_AUTO_ENABLED -## POSTGRES_CONNLIMIT_MODE_AUTO_ENABLED - -Dynamic config that enables connlimit_mode: auto for PostgreSQL connections. -Auto mode ignores static and dynamic max_connections configs and verifies -that the cluster services use max_connections equals to PostgreSQL server's -max_connections divided by service instance count. - -@include postgresql/dynamic_configs/POSTGRES_CONNLIMIT_MODE_AUTO_ENABLED.yaml - -Used by components::Postgres. - - -@anchor POSTGRES_STATEMENT_METRICS_SETTINGS -## POSTGRES_STATEMENT_METRICS_SETTINGS - -Dynamic config that controls statement metrics settings for specific service. - -Dictionary keys can be either the service **component name** (not database name!) -or `__default__`. The latter configuration is applied for every non-matching -PostgreSQL component of the service. - -The value of `max_statement_metrics` controls the maximum size of LRU-cache -for named statement metrics. When set to 0 (default) no metrics are being -exported. - -The exported data can be found as `postgresql.statement_timings`. - -@include postgresql/dynamic_configs/POSTGRES_STATEMENT_METRICS_SETTINGS.yaml - -```json -{ - "postgresql-database_name": { - "max_statement_metrics": 50 - }, - "__default__": { - "max_statement_metrics": 150 - } -} -``` - -Used by components::Postgres. - - -@anchor POSTGRES_OMIT_DESCRIBE_IN_EXECUTE -## POSTGRES_OMIT_DESCRIBE_IN_EXECUTE - -Dynamic config that turns on omit-describe-in-execute experiment for pg queries. The value -specifies the current version of the experiment. Specify 0 to disable. - -@include postgresql/dynamic_configs/POSTGRES_OMIT_DESCRIBE_IN_EXECUTE.yaml - - -@anchor POSTGRES_CONNECTION_PIPELINE_EXPERIMENT -## POSTGRES_CONNECTION_PIPELINE_EXPERIMENT - -Dynamic config that turns on pipeline-mode experiment for pg connections. The value -specifies the current version of the experiment. Specify 0 to disable. - -@include postgresql/dynamic_configs/POSTGRES_CONNECTION_PIPELINE_EXPERIMENT.yaml - - -@anchor REDIS_COMMANDS_BUFFERING_SETTINGS -## REDIS_COMMANDS_BUFFERING_SETTINGS - -Dynamic config that controls command buffering for specific service. -Enabling of this config activates a delay in sending commands. When commands are sent, they are combined into a single tcp packet and sent together. -First command arms timer and then during `watch_command_timer_interval_us` commands are accumulated in the buffer - - -Command buffering is disabled by default. - -@include redis/dynamic_configs/REDIS_COMMANDS_BUFFERING_SETTINGS.yaml - -**Example:** -```json -{ - "buffering_enabled": true, - "commands_buffering_threshold": 10, - "watch_command_timer_interval_us": 1000 -} -``` - -Used by components::Redis. - - -@anchor REDIS_DEFAULT_COMMAND_CONTROL -## REDIS_DEFAULT_COMMAND_CONTROL - -Dynamic config that overrides the default timeouts, number of retries and -server selection strategy for redis commands. - -@include redis/dynamic_configs/REDIS_DEFAULT_COMMAND_CONTROL.yaml - -**Example:** -```json -{ - "best_dc_count": 0, - "max_ping_latency_ms": 0, - "max_retries": 4, - "strategy": "default", - "timeout_all_ms": 2000, - "timeout_single_ms": 500 -} -``` - -Used by components::Redis. - - -@anchor REDIS_METRICS_SETTINGS -## REDIS_METRICS_SETTINGS - -Dynamic config that controls the metric settings for specific service. - -Dictionary keys can be either the **database name** (not the component name!) -or `__default__`. The latter configuration is applied for every non-matching -Redis database/sentinel of the service. - -``` -yaml -type: object -additionalProperties: - $ref: "#/definitions/MetricsSettings" -definitions: - MetricsSettings: - type: object - additionalProperties: false - properties: - timings-enabled: - type: boolean - default: true - description: enable timings statistics - command-timings-enabled: - type: boolean - default: false - description: enable statistics for individual commands - request-sizes-enabled: - type: boolean - default: false - description: enable request sizes statistics - reply-sizes-enabled: - type: boolean - default: false - description: enable response sizes statistics -``` - -**Example:** -```json -{ - "redis-database_name": { - "timings-enabled": true, - "command-timings-enabled": false, - "request-sizes-enabled": false, - "reply-sizes-enabled": false - } -} -``` - -Used by components::Redis. - - -@anchor REDIS_PUBSUB_METRICS_SETTINGS -## REDIS_PUBSUB_METRICS_SETTINGS - -Dynamic config that controls the redis pubsub metric settings for specific service. - -Dictionary keys can be either the **database name** (not the component name!) -or `__default__`. The latter configuration is applied for every non-matching -Redis database/sentinel of the service. - -``` -yaml -type: object -additionalProperties: - $ref: "#/definitions/PubsubMetricsSettings" -definitions: - PubsubMetricsSettings: - type: object - additionalProperties: false - properties: - per-shard-stats-enabled: - type: boolean - default: true - description: enable collecting statistics by shard -``` - -**Example:** -```json -{ - "redis-database_name": { - "per-shard-stats-enabled": true - } -} -``` - -Used by components::Redis. - - -@anchor REDIS_RETRY_BUDGET_SETTINGS -## REDIS_RETRY_BUDGET_SETTINGS - -Dynamic config that controls the retry budget (throttling) settings for -components::Redis. - -Dictionary keys can be either the **database name** (not the component name!) -or `__default__`. The latter configuration is applied for every non-matching -Redis database/sentinel of the service. - -@include redis/dynamic_configs/REDIS_RETRY_BUDGET_SETTINGS.yaml - -**Example:** -```json -{ - "__default__": { - "max-tokens": 100, - "token-ratio": 0.1, - "enabled": true - } -} -``` - -Used by components::Redis. - -@anchor REDIS_REPLICA_MONITORING_SETTINGS -## REDIS_REPLICA_MONITORING_SETTINGS - -Настройки отслеживания синхронизации реплик redis - -Dynamic config that controls the monitoring settings for synchronizing replicas. - -Dictionary keys can be either the **database name** (not the component name!) -or `__default__`. The latter configuration is applied for every non-matching -Redis database/sentinel of the service. - -``` -yaml -type: object -additionalProperties: - $ref: '#/definitions/BaseSettings' -definitions: - BaseSettings: - type: object - additionalProperties: false - properties: - enable-monitoring: - description: set to `true` to turn on monitoring - type: boolean - forbid-requests-to-syncing-replicas: - description: set to true to forbid requests to syncing replicas - type: boolean - required: - - enable-monitoring - - forbid-requests-to-syncing-replicas -``` - -Used by components::Redis. - -**Example:** -```json -{ - "__default__": { - "enable-monitoring": false, - "forbid-requests-to-syncing-replicas": false - } -} -``` - -@anchor REDIS_SUBSCRIBER_DEFAULT_COMMAND_CONTROL -## REDIS_SUBSCRIBER_DEFAULT_COMMAND_CONTROL - -The same as @ref REDIS_DEFAULT_COMMAND_CONTROL but for subscription clients. - -@include redis/dynamic_configs/REDIS_SUBSCRIBER_DEFAULT_COMMAND_CONTROL.yaml - -**Example:** -```json -{ - "best_dc_count": 0, - "max_ping_latency_ms": 0, - "strategy": "default", - "timeout_all_ms": 2000, - "timeout_single_ms": 500 -} -``` - -Used by components::Redis. - - -@anchor REDIS_SUBSCRIPTIONS_REBALANCE_MIN_INTERVAL_SECONDS -## REDIS_SUBSCRIPTIONS_REBALANCE_MIN_INTERVAL_SECONDS - -Dynamic config that controls the minimal interval between redis subscription -clients rebalancing. - -@include redis/dynamic_configs/REDIS_SUBSCRIPTIONS_REBALANCE_MIN_INTERVAL_SECONDS.yaml - -Used by components::Redis. - - @anchor REDIS_WAIT_CONNECTED ## REDIS_WAIT_CONNECTED @@ -657,544 +16,3 @@ instances. ``` Used by components::Redis. - - -@anchor USERVER_CACHES -## USERVER_CACHES - -Cache update dynamic parameters. - -@include core/dynamic_configs/USERVER_CACHES.yaml - -**Example:** -```json -{ - "some-cache-name": { - "full-update-interval-ms": 86400000, - "update-interval-ms": 30000, - "update-jitter-ms": 1000, - "alert-on-failing-to-update-times": 2 - } -} -``` - -Used by all the caches derived from components::CachingComponentBase. - - -@anchor USERVER_CANCEL_HANDLE_REQUEST_BY_DEADLINE -## USERVER_CANCEL_HANDLE_REQUEST_BY_DEADLINE - -Controls whether the http request task should be cancelled when the deadline received from the client is reached. - -@include core/dynamic_configs/USERVER_CANCEL_HANDLE_REQUEST_BY_DEADLINE.yaml - -**Example:** -``` -true -``` - -Used by components::Server. - -@anchor USERVER_DEADLINE_PROPAGATION_ENABLED -## USERVER_DEADLINE_PROPAGATION_ENABLED - -When `false`, disables deadline propagation in the service. This includes: - -- reading the task-inherited deadline from HTTP headers and gRPC metadata; -- interrupting operations when deadline expires; -- propagating the deadline to downstream services and databases. - -@include core/dynamic_configs/USERVER_DEADLINE_PROPAGATION_ENABLED.yaml - -**Example:** -``` -true -``` - -Used by components::Server and ugrpc::server::ServerComponent. - -@anchor USERVER_DUMPS -## USERVER_DUMPS - -Dynamic dump configuration. If the options are set for some dump then those -options override the static configuration. - -``` -yaml -schema: - type: object - additionalProperties: - type: object - additionalProperties: false - properties: - dumps-enabled: - type: boolean - min-dump-interval: - description: dump interval in milliseconds - type: integer - minimum: 0 -``` - -**Example:** -```json -{ - "some-cache-name": { - "dumps-enabled": true, - "min-dump-interval": 100000 - } -} -``` - -Used by dump::Dumper, especially by all the caches derived from components::CachingComponentBase. - -@anchor USERVER_HTTP_PROXY -## USERVER_HTTP_PROXY - -Proxy for all the HTTP and HTTPS clients. Empty string disables proxy usage. - -Proxy string may be prefixed with `[scheme]://` to specify which kind of proxy is used. Schemes match the [libcurl supported ones](https://curl.se/libcurl/c/CURLOPT_PROXY.html). -A proxy host string can also embed user and password. - -@include core/dynamic_configs/USERVER_HTTP_PROXY.yaml - -**Example:** -``` -localhost:8090 -``` - -Used by components::HttpClient, affects the behavior of clients::http::Client and all the clients that use it. - -@anchor USERVER_LOG_DYNAMIC_DEBUG -## USERVER_LOG_DYNAMIC_DEBUG - -Logging per line and file overrides. -Log locations are defined as path prefix from the Arcadia root (`taxi/uservices/services/`). -Location of file may be followed by `:[line index]` to specify 1 exact log in that file. - -See @ref USERVER_LOG_DYNAMIC_DEBUG_details for detailed semantics. - -@note In order for `USERVER_LOG_DYNAMIC_DEBUG` to work, you should `Append` @ref components::LoggingConfigurator -component or use @ref components::CommonComponentList. - -``` -yaml -default: - force-enabled: [] - force-disabled: [] - -schema: - type: object - additionalProperties: false - required: - - force-enabled - - force-disabled - properties: - force-enabled: - type: array - description: Locations of logs to turn on. This option is deprecated, consider using "force-enabled-level" instead. - items: - type: string - - force-disabled: - type: array - description: Locations of logs to turn off, logs with level WARNING and higher will not be affected. - This option is deprecated, consider using "force-disabled-level" instead. - items: - type: string - - force-enabled-level: - type: object - description: | - Locations of logs to turn on with level equal or higher to given. - For example, to turn on all logs with level WARNING or higher in "userver/grpc", - all logs with level DEBUG or higher in file "userver/core/src/server/server.cpp" - and 1 log (since exact line is specified) with level TRACE in file "userver/core/src/server/http/http_request_parser.cpp": - "force-enabled-level": { - "taxi/uservices/userver/grpc": "WARNING", - "taxi/uservices/userver/core/src/server/server.cpp": "DEBUG", - "taxi/uservices/userver/core/src/server/http/http_request_parser.cpp:128": "TRACE" - } - additionalProperties: - type: string - - force-disabled-level: - type: object - description: | - Locations of logs to turn off with level equal or lower to given. - For example, to turn off all logs with level ERROR or lower in "userver/grpc", - all logs with level INFO or lower in file "userver/core/src/server/server.cpp" - and 1 log (since exact line is specified) with level TRACE in file "userver/core/src/server/http/http_request_parser.cpp": - "force-disabled-level": { - "taxi/uservices/userver/grpc": "ERROR", - "taxi/uservices/userver/core/src/server/server.cpp": "INFO", - "taxi/uservices/userver/core/src/server/http/http_request_parser.cpp:128": "TRACE" - } - additionalProperties: - type: string -``` - -@warning Use @ref USERVER_NO_LOG_SPANS to disable Span logs. - -Used by components::LoggingConfigurator. - - -@anchor USERVER_LOG_REQUEST -## USERVER_LOG_REQUEST - -Controls HTTP requests and responses logging. - -@include core/dynamic_configs/USERVER_LOG_REQUEST.yaml - -**Example:** -``` -false -``` - -Used by components::Server. - -@anchor USERVER_LOG_REQUEST_HEADERS -## USERVER_LOG_REQUEST_HEADERS - -Controls whether the logging of HTTP headers in handlers is performed. - -@note To ensure safety, all header values will be output as `***` unless specified in @ref USERVER_LOG_REQUEST_HEADERS_WHITELIST. - -@include core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS.yaml - -**Example:** -``` -false -``` - -Used by components::Server. - -@anchor USERVER_LOG_REQUEST_HEADERS_WHITELIST -## USERVER_LOG_REQUEST_HEADERS_WHITELIST - -If the @ref USERVER_LOG_REQUEST_HEADERS option is enabled, you can control which HTTP headers are logged, including their values. Header is suitable if it exactly matches one of the values in the whitelist. Any headers that are not on the whitelist will have their values replaced with *** in the logs. - -@include core/dynamic_configs/USERVER_LOG_REQUEST_HEADERS_WHITELIST.yaml - -**Example:** -``` -["User-Agent", "Accept-Encoding"] -``` - -Used by server::handlers::HttpHandlerBase. - -@anchor USERVER_LRU_CACHES -## USERVER_LRU_CACHES - -Dynamic config for controlling size and cache entry lifetime of the LRU based caches. - -@include core/dynamic_configs/USERVER_LRU_CACHES.yaml - -**Example:** -```json -{ - "some-cache-name": { - "lifetime-ms": 5000, - "size": 100000 - }, - "some-other-cache-name": { - "lifetime-ms": 5000, - "size": 400000 - } -} -``` - -Used by all the caches derived from cache::LruCacheComponent. - - -@anchor USERVER_NO_LOG_SPANS -## USERVER_NO_LOG_SPANS - -Prefixes or full names of tracing::Span instances to not log. - -@note In order for `USERVER_NO_LOG_SPANS` to work, you should `Append` @ref components::LoggingConfigurator -component or use @ref components::CommonComponentList. - -@include core/dynamic_configs/USERVER_NO_LOG_SPANS.yaml - -**Example:** -```json -{ - "names": [ - "mongo_find" - ], - "prefixes": [ - "http", - "handler" - ] -} -``` - -To disable specific log lines not related to spans use @ref USERVER_LOG_DYNAMIC_DEBUG. - -Used by components::LoggingConfigurator and all the logging facilities. - -@anchor USERVER_RPS_CCONTROL -## USERVER_RPS_CCONTROL - -Dynamic config for components::Server congestion control. - -``` -yaml -schema: - type: object - additionalProperties: false - properties: - min-limit: - type: integer - minimum: 1 - description: | - Minimal RPS limit value. - - up-rate-percent: - type: number - minimum: 0 - exclusiveMinimum: true - description: | - On how many percents increase the RPS limit every second if not in overloaded state. - - down-rate-percent: - type: number - minimum: 0 - maximum: 100 - exclusiveMinimum: true - description: | - On how many percents decrease the RPS limit every second if in overloaded state. - - overload-on-seconds: - type: integer - minimum: 1 - description: | - How many seconds overload should be higher than `up-level`, - to switch to overload state and start limiting RPS. - - overload-off-seconds: - type: integer - minimum: 1 - description: | - How many seconds overload should be higher than `down-level`, - to switch from overload state and the RPS limit start increasing. - - up-level: - type: integer - minimum: 1 - description: | - Overload level that should least for `overload-on-seconds` to switch to - overloaded state. - - down-level: - type: integer - minimum: 1 - description: | - Overload level that should least for `overload-on-seconds` to - increase RPS limits. - - no-limit-seconds: - type: integer - minimum: 1 - description: | - Seconds without overload to switch from overloaded state and - remove all the RPS limits. - - load-limit-crit-percent: - type: integer - description: | - On reaching this load percent immediately switch to overloaded state. -``` - -**Example:** -```json -{ - "down-level": 8, - "down-rate-percent": 1, - "load-limit-crit-percent": 70, - "min-limit": 2, - "no-limit-seconds": 120, - "overload-off-seconds": 8, - "overload-on-seconds": 8, - "up-level": 2, - "up-rate-percent": 1 -} -``` - -Used by congestion_control::Component. - - -@anchor USERVER_RPS_CCONTROL_ENABLED -## USERVER_RPS_CCONTROL_ENABLED - -Controls whether congestion control limiting of RPS is performed (if main task processor is overloaded, -then the server starts rejecting some requests). - -@include core/dynamic_configs/USERVER_RPS_CCONTROL_ENABLED.yaml - -**Example:** -``` -true -``` - -Used by congestion_control::Component. - -@anchor USERVER_TASK_PROCESSOR_PROFILER_DEBUG -## USERVER_TASK_PROCESSOR_PROFILER_DEBUG - -Dynamic config for profiling the coroutine based engine of userver. -Dictionary key names are the names of engine::TaskProcessor. - -``` -yaml -schema: - type: object - properties: {} - additionalProperties: - $ref: '#/definitions/TaskProcessorSettings' - definitions: - TaskProcessorSettings: - type: object - required: - - enabled - - execution-slice-threshold-us - additionalProperties: false - properties: - enabled: - type: boolean - description: | - Set to `true` to turn on the profiling - execution-slice-threshold-us: - type: integer - description: | - Execution threshold of a coroutine on a thread without context switch. - If the threshold is reached then the coroutine is logged, otherwise - does nothing. - minimum: 1 -``` - -**Example:** -```json -{ - "fs-task-processor": { - "enabled": false, - "execution-slice-threshold-us": 1000000 - }, - "main-task-processor": { - "enabled": false, - "execution-slice-threshold-us": 2000 - } -} -``` - -Used by components::ManagerControllerComponent. - -@anchor USERVER_TASK_PROCESSOR_QOS -## USERVER_TASK_PROCESSOR_QOS - -Controls engine::TaskProcessor quality of service dynamic config. - -``` -yaml -schema: - type: object - required: - - default-service - additionalProperties: false - properties: - default-service: - type: object - additionalProperties: false - required: - - default-task-processor - properties: - default-task-processor: - type: object - additionalProperties: false - required: - - wait_queue_overload - properties: - wait_queue_overload: - type: object - additionalProperties: false - required: - - time_limit_us - - length_limit - - action - properties: - time_limit_us: - type: integer - minimum: 0 - description: | - Wait in queue time after which the `action is applied`. - length_limit: - type: integer - minimum: 0 - description: | - Queue size after which the `action` is applied. - action - action: - type: string - enum: - - ignore - - cancel - description: | - Action to perform on tasks on queue overload. - `cancel` - cancels the tasks - sensor_time_limit_us: - type: integer - minimum: 0 - description: | - Wait in queue time after which the overload events for - RPS congestion control are generated. -``` - -**Example:** -```json -{ - "default-service": { - "default-task-processor": { - "wait_queue_overload": { - "action": "cancel", - "length_limit": 5000, - "sensor_time_limit_us": 12000, - "time_limit_us": 0 - } - } - } -} -``` - -Used by components::ManagerControllerComponent. - -@anchor USERVER_FILES_CONTENT_TYPE_MAP -## USERVER_FILES_CONTENT_TYPE_MAP - -Dynamic config for mapping extension files with HTTP header content type. - -@include core/dynamic_configs/USERVER_FILES_CONTENT_TYPE_MAP.yaml - -**Example:** -```json -{ - ".css": "text/css", - ".gif": "image/gif", - ".htm": "text/html", - ".html": "text/html", - ".jpeg": "image/jpeg", - ".js": "application/javascript", - ".json": "application/json", - ".md": "text/markdown", - ".png": "image/png", - ".svg": "image/svg+xml", - "__default__": "text/plain" -} -``` - -Used by server::handlers::HttpHandlerStatic - ----------- - -@htmlonly
@endhtmlonly -⇦ @ref scripts/docs/en/userver/dynamic_config.md | @ref scripts/docs/en/userver/log_level_running_service.md ⇨ -@htmlonly
@endhtmlonly From 4278aee490eb2bdca8cd0513885e694286ea5e47 Mon Sep 17 00:00:00 2001 From: fdr400 Date: Tue, 29 Jul 2025 10:19:51 +0300 Subject: [PATCH 018/151] refactor kafka: use utils::zstring_view for topic name use zstring_view for topic commit_hash:1d043ff2e0f02596f28d7266859a0427d1b0bf49 --- kafka/include/userver/kafka/producer.hpp | 4 ++-- kafka/src/kafka/impl/producer_impl.cpp | 4 ++-- kafka/src/kafka/impl/producer_impl.hpp | 4 ++-- kafka/src/kafka/producer.cpp | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kafka/include/userver/kafka/producer.hpp b/kafka/include/userver/kafka/producer.hpp index 2a07220a3cc0..ec58c9568145 100644 --- a/kafka/include/userver/kafka/producer.hpp +++ b/kafka/include/userver/kafka/producer.hpp @@ -103,7 +103,7 @@ class Producer final { /// a sense to retry the message sending. /// @snippet kafka/tests/producer_kafkatest.cpp Producer retryable error void Send( - const std::string& topic_name, + utils::zstring_view topic_name, std::string_view key, std::string_view message, std::optional partition = kUnassignedPartition, @@ -134,7 +134,7 @@ class Producer final { private: void SendImpl( - const std::string& topic_name, + utils::zstring_view topic_name, std::string_view key, std::string_view message, std::optional partition, diff --git a/kafka/src/kafka/impl/producer_impl.cpp b/kafka/src/kafka/impl/producer_impl.cpp index 70299246392f..bda8fe940c73 100644 --- a/kafka/src/kafka/impl/producer_impl.cpp +++ b/kafka/src/kafka/impl/producer_impl.cpp @@ -128,7 +128,7 @@ ProducerImpl::ProducerImpl( const Stats& ProducerImpl::GetStats() const { return stats_; } DeliveryResult ProducerImpl::Send( - const std::string& topic_name, + utils::zstring_view topic_name, std::string_view key, std::string_view message, std::optional partition, @@ -144,7 +144,7 @@ DeliveryResult ProducerImpl::Send( } engine::Future ProducerImpl::ScheduleMessageDelivery( - const std::string& topic_name, + utils::zstring_view topic_name, std::string_view key, std::string_view message, std::optional partition, diff --git a/kafka/src/kafka/impl/producer_impl.hpp b/kafka/src/kafka/impl/producer_impl.hpp index 2c8e20a6527a..893ff1b8dad8 100644 --- a/kafka/src/kafka/impl/producer_impl.hpp +++ b/kafka/src/kafka/impl/producer_impl.hpp @@ -33,7 +33,7 @@ class ProducerImpl final { /// @brief Send the message and waits for its delivery. /// While waiting handles other messages delivery reports, errors and logs. [[nodiscard]] DeliveryResult Send( - const std::string& topic_name, + utils::zstring_view topic_name, std::string_view key, std::string_view message, std::optional partition, @@ -54,7 +54,7 @@ class ProducerImpl final { /// @brief Schedules the message delivery. /// @returns the future for delivery result, which must be awaited. [[nodiscard]] engine::Future ScheduleMessageDelivery( - const std::string& topic_name, + utils::zstring_view topic_name, std::string_view key, std::string_view message, std::optional partition, diff --git a/kafka/src/kafka/producer.cpp b/kafka/src/kafka/producer.cpp index d9d71be6fbb8..4e95c4ba729c 100644 --- a/kafka/src/kafka/producer.cpp +++ b/kafka/src/kafka/producer.cpp @@ -99,13 +99,13 @@ Producer::~Producer() { } void Producer::Send( - const std::string& topic_name, + utils::zstring_view topic_name, std::string_view key, std::string_view message, std::optional partition, HeaderViews headers ) const { - utils::Async(producer_task_processor_, "producer_send", [this, &topic_name, key, message, partition, &headers] { + utils::Async(producer_task_processor_, "producer_send", [this, topic_name, key, message, partition, &headers] { SendImpl(topic_name, key, message, partition, impl::HeadersHolder{headers}); }).Get(); } @@ -136,7 +136,7 @@ void Producer::DumpMetric(utils::statistics::Writer& writer) const { } void Producer::SendImpl( - const std::string& topic_name, + utils::zstring_view topic_name, std::string_view key, std::string_view message, std::optional partition, From 570c59f38c89c318222bfa37b5198eb1b35ca3d6 Mon Sep 17 00:00:00 2001 From: segoon Date: Tue, 29 Jul 2025 13:35:41 +0300 Subject: [PATCH 019/151] feat userver: generate docs for dynamic configs commit_hash:edd57b7f01cffecc1136361e7c11bb3ca5ad3ca0 --- .gitignore | 1 + .mapping.json | 2 + CMakeLists.txt | 3 + cmake/UserverGenerateDynamicConfigsDocs.cmake | 12 ++++ scripts/docs/dynamic_config_yaml_to_md.py | 59 +++++++++++++++++++ scripts/docs/make_docs.sh | 5 ++ 6 files changed, 82 insertions(+) create mode 100644 cmake/UserverGenerateDynamicConfigsDocs.cmake create mode 100755 scripts/docs/dynamic_config_yaml_to_md.py diff --git a/.gitignore b/.gitignore index ba946e20b25b..ac32cfefe46c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ static-analyzer-report .settings* .clangd .vscode +scripts/docs/en/dynamic_configs diff --git a/.mapping.json b/.mapping.json index 4c81e29e5bbe..e4867720abf6 100644 --- a/.mapping.json +++ b/.mapping.json @@ -515,6 +515,7 @@ "cmake/UserverCodegenTarget.cmake":"taxi/uservices/userver/cmake/UserverCodegenTarget.cmake", "cmake/UserverCxxCompileOptionsIfSupported.cmake":"taxi/uservices/userver/cmake/UserverCxxCompileOptionsIfSupported.cmake", "cmake/UserverEmbedFile.cmake":"taxi/uservices/userver/cmake/UserverEmbedFile.cmake", + "cmake/UserverGenerateDynamicConfigsDocs.cmake":"taxi/uservices/userver/cmake/UserverGenerateDynamicConfigsDocs.cmake", "cmake/UserverGrpcTargets.cmake":"taxi/uservices/userver/cmake/UserverGrpcTargets.cmake", "cmake/UserverModule.cmake":"taxi/uservices/userver/cmake/UserverModule.cmake", "cmake/UserverPack.cmake":"taxi/uservices/userver/cmake/UserverPack.cmake", @@ -3845,6 +3846,7 @@ "scripts/docs/doxygen-awesome-css/doxygen-awesome-tabs.js":"taxi/uservices/userver/scripts/docs/doxygen-awesome-css/doxygen-awesome-tabs.js", "scripts/docs/doxygen-awesome-css/doxygen-awesome.css":"taxi/uservices/userver/scripts/docs/doxygen-awesome-css/doxygen-awesome.css", "scripts/docs/doxygen.conf":"taxi/uservices/userver/scripts/docs/doxygen.conf", + "scripts/docs/dynamic_config_yaml_to_md.py":"taxi/uservices/userver/scripts/docs/dynamic_config_yaml_to_md.py", "scripts/docs/en/cpp/def_groups.hpp":"taxi/uservices/userver/scripts/docs/en/cpp/def_groups.hpp", "scripts/docs/en/deps/alpine.md":"taxi/uservices/userver/scripts/docs/en/deps/alpine.md", "scripts/docs/en/deps/arch.md":"taxi/uservices/userver/scripts/docs/en/deps/arch.md", diff --git a/CMakeLists.txt b/CMakeLists.txt index 125c984423ad..221aec71706a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -188,6 +188,7 @@ include(FindPackageRequired) include(IncludeWhatYouUse) include(UserverTestsuite) include(UserverCodegenTarget) +include(UserverGenerateDynamicConfigsDocs) include(CheckCompileFlags) include(CMakePackageConfigHelpers) @@ -363,4 +364,6 @@ if(USERVER_BUILD_TESTS) endif() endif() +_userver_add_target_gen_dynamic_configs_docs() + _userver_print_features_list() diff --git a/cmake/UserverGenerateDynamicConfigsDocs.cmake b/cmake/UserverGenerateDynamicConfigsDocs.cmake new file mode 100644 index 000000000000..c80a0303c308 --- /dev/null +++ b/cmake/UserverGenerateDynamicConfigsDocs.cmake @@ -0,0 +1,12 @@ +function(_userver_add_target_gen_dynamic_configs_docs) + file(GLOB YAML_FILENAMES */dynamic_configs/*.yaml) + + add_custom_target( + userver-gen-dynamic-configs-docs + COMMENT "Generate dynamic_configs .md docs" + COMMAND + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/docs/dynamic_config_yaml_to_md.py + -o ${CMAKE_CURRENT_BINARY_DIR}/docs-dynamic-configs + ${YAML_FILENAMES} + ) +endfunction() diff --git a/scripts/docs/dynamic_config_yaml_to_md.py b/scripts/docs/dynamic_config_yaml_to_md.py new file mode 100755 index 000000000000..0c42e74da70e --- /dev/null +++ b/scripts/docs/dynamic_config_yaml_to_md.py @@ -0,0 +1,59 @@ +#!/usr/bin/python + +import argparse +import os +import pathlib + +import yaml + + +def format_md(name, content, content_raw): + description = content.get('description', '') + example_text = content.get('schema', {}).get('example', '') + if example_text: + example = f'**Example:**\n```json\n{example_text}\n```' + else: + example = '' + text = f"""@anchor {name} +## {name} Dynamic Config +{description} + +**Schema:** +``` +# yaml +{content_raw} +``` + +{example} +""" + # TODO: used by component XX + return text + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-o') + parser.add_argument('yaml', nargs='+') + return parser.parse_args() + + +def handle_file(ifname, ofname): + with open(ifname) as ifile: + content_raw = ifile.read() + content = yaml.load(content_raw, yaml.Loader) + + md = format_md(pathlib.Path(ifname).stem, content, content_raw) + + with open(ofname, 'w') as ofile: + ofile.write(md) + + +def main(): + args = parse_args() + os.makedirs(args.o, exist_ok=True) + for fname in args.yaml: + handle_file(fname, pathlib.Path(args.o) / (pathlib.Path(fname).stem + '.md')) + + +if __name__ == '__main__': + main() diff --git a/scripts/docs/make_docs.sh b/scripts/docs/make_docs.sh index 6bcb0212687c..453bf0d87c4c 100755 --- a/scripts/docs/make_docs.sh +++ b/scripts/docs/make_docs.sh @@ -48,6 +48,11 @@ CMAKE_VERSION=$("$CMAKE_COMMAND" --version | grep -oP '\d+\.\d+') echo "Building target userver-codegen." "$CMAKE_COMMAND" --build "$BUILD_DIR" --target userver-codegen +echo "Building target userver-gen-dynamic-configs-docs." +"$CMAKE_COMMAND" --build "$BUILD_DIR" --target userver-gen-dynamic-configs-docs +rm -f scripts/docs/en/dynamic_configs +ln -s ../../../$BUILD_DIR/docs-dynamic-configs scripts/docs/en/dynamic_configs + # Run doxygen. rm -rf "$BUILD_DIR/docs" || : From 3f57030c4cb05e490f79b966af8d0c65cd213646 Mon Sep 17 00:00:00 2001 From: mvkab Date: Tue, 29 Jul 2025 14:24:49 +0300 Subject: [PATCH 020/151] test kafka: switch a real kafka in to mock commit_hash:43b046c70975da323b21b416138b6e5a2033260c --- kafka/src/kafka/impl/holders.cpp | 3 + kafka/tests/producer_kafkatest.cpp | 2 +- .../userver/kafka/utest/kafka_fixture.hpp | 11 +-- kafka/utest/src/kafka/utest/kafka_fixture.cpp | 82 ++++++++++++++++--- 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/kafka/src/kafka/impl/holders.cpp b/kafka/src/kafka/impl/holders.cpp index aa1d2a60fa6b..08dd30ee9cb6 100644 --- a/kafka/src/kafka/impl/holders.cpp +++ b/kafka/src/kafka/impl/holders.cpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -47,6 +48,8 @@ template class HolderBase; template class HolderBase; template class HolderBase; template class HolderBase; +template class HolderBase; +template class HolderBase; struct ConfHolder::Impl { explicit Impl(rd_kafka_conf_t* conf) : conf(conf) {} diff --git a/kafka/tests/producer_kafkatest.cpp b/kafka/tests/producer_kafkatest.cpp index 9bbfef3134e3..a14d8e0845ea 100644 --- a/kafka/tests/producer_kafkatest.cpp +++ b/kafka/tests/producer_kafkatest.cpp @@ -297,7 +297,7 @@ UTEST_F(ProducerTest, ManyProducersManySendAsyncSingleThread) { UTEST_F_MT(ProducerTest, ManyProducersManySendAsync, 4 + 4) { constexpr std::size_t kProducerCount{4}; - constexpr std::size_t kSendCount{300}; + constexpr std::size_t kSendCount{200}; constexpr std::size_t kTopicCount{kSendCount / 10}; const std::deque producers = diff --git a/kafka/utest/include/userver/kafka/utest/kafka_fixture.hpp b/kafka/utest/include/userver/kafka/utest/kafka_fixture.hpp index b8c6c2880c5b..86d0262ae59d 100644 --- a/kafka/utest/include/userver/kafka/utest/kafka_fixture.hpp +++ b/kafka/utest/include/userver/kafka/utest/kafka_fixture.hpp @@ -11,8 +11,8 @@ #include #include #include +#include #include - USERVER_NAMESPACE_BEGIN namespace kafka::utest { @@ -66,9 +66,9 @@ class KafkaCluster : public ::testing::Test { KafkaCluster(); - ~KafkaCluster() override = default; + ~KafkaCluster() override; - std::string GenerateTopic(); + std::string GenerateTopic(std::uint32_t partition_cnt = 1); std::vector GenerateTopics(std::size_t count); @@ -121,13 +121,14 @@ class KafkaCluster : public ::testing::Test { private: impl::Secret AddBootstrapServers(impl::Secret secrets) const; + std::string InitBootstrapServers(); private: static std::atomic kTopicsCount; - + class MockCluster; + utils::Box mock_; const std::string bootstrap_servers_; }; - } // namespace kafka::utest USERVER_NAMESPACE_END diff --git a/kafka/utest/src/kafka/utest/kafka_fixture.cpp b/kafka/utest/src/kafka/utest/kafka_fixture.cpp index eba447375a5f..d417b830e7ef 100644 --- a/kafka/utest/src/kafka/utest/kafka_fixture.cpp +++ b/kafka/utest/src/kafka/utest/kafka_fixture.cpp @@ -4,31 +4,27 @@ #include #include +#include +#include #include #include #include +#include #include +#include USERVER_NAMESPACE_BEGIN namespace kafka::utest { - -namespace { - constexpr const char* kTestsuiteKafkaServerHost{"TESTSUITE_KAFKA_SERVER_HOST"}; constexpr const char* kDefaultKafkaServerHost{"localhost"}; constexpr const char* kTestsuiteKafkaServerPort{"TESTSUITE_KAFKA_SERVER_PORT"}; constexpr const char* kDefaultKafkaServerPort{"9099"}; -constexpr const char* kRecipeKafkaBrokersList{"KAFKA_RECIPE_BROKER_LIST"}; std::string FetchBrokerList() { const auto env = engine::subprocess::GetCurrentEnvironmentVariablesPtr(); - if (const auto* brokers_list = env->GetValueOptional(kRecipeKafkaBrokersList)) { - return *brokers_list; - } - std::string server_host{kDefaultKafkaServerHost}; std::string server_port{kDefaultKafkaServerPort}; const auto* host = env->GetValueOptional(kTestsuiteKafkaServerHost); @@ -42,10 +38,10 @@ std::string FetchBrokerList() { return fmt::format("{}:{}", server_host, server_port); } +namespace { impl::Secret MakeSecrets(std::string_view bootstrap_servers) { impl::Secret secrets{}; secrets.brokers = bootstrap_servers; - return secrets; } @@ -58,19 +54,83 @@ impl::ProducerConfiguration PatchDeliveryTimeout(impl::ProducerConfiguration con return configuration; } +bool IsKafkaClusterConfigured() { + static std::optional is_cluster_configured; + if (!is_cluster_configured.has_value()) { + const auto env = engine::subprocess::GetCurrentEnvironmentVariablesPtr(); + is_cluster_configured = + env->GetValueOptional(kTestsuiteKafkaServerHost) || env->GetValueOptional(kTestsuiteKafkaServerPort); + } + return *is_cluster_configured; +} } // namespace +class KafkaCluster::MockCluster { + using ProducerHolder = impl::HolderBase; + using MockClusterHolder = impl::HolderBase; + +public: + MockCluster() + : conf_(CreateConfiguredConf()), + handle_(rd_kafka_new(RD_KAFKA_PRODUCER, conf_.GetHandle(), nullptr, sizeof(nullptr))), + mock_cluster_(rd_kafka_mock_cluster_new(handle_.GetHandle(), /*broker_cnt=*/1)) { + UINVARIANT(handle_.GetHandle(), "Failed to create fake producer"); + UINVARIANT(mock_cluster_.GetHandle(), "Failed to get mock cluster handle"); + conf_.ForgetUnderlyingConf(); + } + static impl::ConfHolder CreateConfiguredConf() { + rd_kafka_conf_t* conf = rd_kafka_conf_new(); + UINVARIANT(conf, "Failed to create Kafka config"); + rd_kafka_conf_set(conf, "log_level", "2", nullptr, sizeof(nullptr)); + return impl::ConfHolder(conf); + } + ~MockCluster() = default; + rd_kafka_mock_cluster_t* GetMockCluster() const { return mock_cluster_.GetHandle(); } + +private: + impl::ConfHolder conf_; + ProducerHolder handle_; + MockClusterHolder mock_cluster_; +}; + bool operator==(const Message& lhs, const Message& rhs) { return std::tie(lhs.topic, lhs.key, lhs.payload, lhs.partition) == std::tie(rhs.topic, rhs.key, rhs.payload, rhs.partition); } -KafkaCluster::KafkaCluster() : bootstrap_servers_(FetchBrokerList()) {} +std::string KafkaCluster::InitBootstrapServers() { + if (IsKafkaClusterConfigured()) { + LOG_DEBUG("Using real Kafka"); + return FetchBrokerList(); + } + LOG_DEBUG("Using mock Kafka"); + mock_ = utils::Box(); + return rd_kafka_mock_cluster_bootstraps(mock_->GetMockCluster()); +} + +KafkaCluster::KafkaCluster() : bootstrap_servers_(InitBootstrapServers()) { + UINVARIANT(!bootstrap_servers_.empty(), "Empty bootstrap_servers"); +} std::atomic KafkaCluster::kTopicsCount{0}; -std::string KafkaCluster::GenerateTopic() { return fmt::format("tt-{}", kTopicsCount.fetch_add(1)); } +KafkaCluster::~KafkaCluster() = default; + +std::string KafkaCluster::GenerateTopic(std::uint32_t partition_cnt) { + const std::string topic = fmt::format("tt-{}", kTopicsCount.fetch_add(1)); + if (!IsKafkaClusterConfigured()) { + const auto err = rd_kafka_mock_topic_create( + mock_->GetMockCluster(), topic.c_str(), utils::numeric_cast(partition_cnt), /*replication_factor=*/1 + ); + if (err != RD_KAFKA_RESP_ERR_NO_ERROR) { + LOG_ERROR("Failed to create topic {}: {}", topic, rd_kafka_err2str(err)); + } else { + LOG_INFO("Successful topic creation {}", topic); + } + } + return topic; +} std::vector KafkaCluster::GenerateTopics(std::size_t count) { std::vector topics{count}; From 7a2586b31e4b4bfa5f9949dc974b00984eed33d8 Mon Sep 17 00:00:00 2001 From: charzik Date: Tue, 29 Jul 2025 15:43:55 +0300 Subject: [PATCH 021/151] feat websocket: add ping poing logic commit_hash:056b428561623a5e50da910e01314b79f2418122 --- core/functional_tests/websocket/service.cpp | 24 +++++++++++++++++++ .../websocket/static_config.yaml | 6 +++++ .../websocket/tests/test_websocket.py | 22 +++++++++++++++++ .../userver/server/websocket/server.hpp | 10 ++++++++ core/src/server/websocket/server.cpp | 14 +++++++++++ 5 files changed, 76 insertions(+) diff --git a/core/functional_tests/websocket/service.cpp b/core/functional_tests/websocket/service.cpp index ce897150ed5d..373eed8a0eb2 100644 --- a/core/functional_tests/websocket/service.cpp +++ b/core/functional_tests/websocket/service.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -109,11 +110,34 @@ class WebsocketsFullDuplexHandler final : public server::websocket::WebsocketHan } }; +class WebsocketsPingPongHandler final : public server::websocket::WebsocketHandlerBase { +public: + static constexpr std::string_view kName = "websocket-ping-pong-handler"; + + using WebsocketHandlerBase::WebsocketHandlerBase; + + void Handle(server::websocket::WebSocketConnection& chat, server::request::RequestContext&) const override { + std::chrono::milliseconds time_without_sends{0}; + while (!engine::current_task::ShouldCancel()) { + if (chat.NotAnsweredSequentialPingsCount() > 3) { + LOG_WARNING() << "Ping not answered, closing connection"; + chat.Close(server::websocket::CloseStatus::kGoingAway); + break; + } + + chat.SendPing(); + time_without_sends += std::chrono::milliseconds(200); + engine::InterruptibleSleepFor(std::chrono::milliseconds(200)); + } + } +}; + int main(int argc, char* argv[]) { const auto component_list = components::MinimalServerComponentList() .Append() .Append() .Append() + .Append() .Append() .Append() .Append() diff --git a/core/functional_tests/websocket/static_config.yaml b/core/functional_tests/websocket/static_config.yaml index 0cef6e562308..bdfdfebed637 100644 --- a/core/functional_tests/websocket/static_config.yaml +++ b/core/functional_tests/websocket/static_config.yaml @@ -39,6 +39,12 @@ components_manager: task_processor: main-task-processor # Run it on CPU bound task processor max-remote-payload: 100000 fragment-size: 10 + websocket-ping-pong-handler: + path: /ping-pong + method: GET + task_processor: main-task-processor + max-remote-payload: 100000 + fragment-size: 10 testsuite-support: diff --git a/core/functional_tests/websocket/tests/test_websocket.py b/core/functional_tests/websocket/tests/test_websocket.py index 24899c2eecd8..004a95af7c76 100644 --- a/core/functional_tests/websocket/tests/test_websocket.py +++ b/core/functional_tests/websocket/tests/test_websocket.py @@ -1,3 +1,4 @@ +import asyncio import logging import pytest @@ -120,3 +121,24 @@ async def test_two_but_handler_alt(websocket_client): for _ in range(10): msg = await chat1.recv() assert msg == 'A' + + +async def test_ping_pong(websocket_client): + async with websocket_client.get('ping-pong'): + await asyncio.sleep(1) + + +async def test_ping_pong_close(websocket_client): + async with websocket_client.get('ping-pong') as chat: + websocket_client.ping_interval = None + websocket_client.ping_timeout = None + + for _ in range(20): + try: + await chat.recv() + await asyncio.sleep(1) + except websockets.exceptions.ConnectionClosed: + connection_closed_by_ping = True + break + + assert connection_closed_by_ping diff --git a/core/include/userver/server/websocket/server.hpp b/core/include/userver/server/websocket/server.hpp index 3fc8332ccd9a..0caaf03eb015 100644 --- a/core/include/userver/server/websocket/server.hpp +++ b/core/include/userver/server/websocket/server.hpp @@ -95,6 +95,16 @@ class WebSocketConnection { virtual void Send(const Message& message) = 0; virtual void SendText(std::string_view message) = 0; + /// @brief Send a ping message to websocket. + /// @throws engine::io::IoException in case of socket errors + virtual void SendPing() = 0; + + /// @brief Get the number of not answered sequential pings; + /// calls to SendPing() increment this value, Recv and TryRecv + /// reset this value if some 'pong' is received. + /// @returns the number of not answered sequential pings + virtual std::size_t NotAnsweredSequentialPingsCount() = 0; + template void SendBinary(const ContiguousContainer& message) { static_assert( diff --git a/core/src/server/websocket/server.cpp b/core/src/server/websocket/server.cpp index 610ece9941b2..e99f2cfa315e 100644 --- a/core/src/server/websocket/server.cpp +++ b/core/src/server/websocket/server.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -59,6 +61,8 @@ class WebSocketConnectionImpl final : public WebSocketConnection { Config config; + std::atomic ping_pending_count_{0}; + public: WebSocketConnectionImpl( std::unique_ptr io_, @@ -118,6 +122,15 @@ class WebSocketConnectionImpl final : public WebSocketConnection { SendExtended(mext); } + void SendPing() override { + MessageExtended ping_msg{{}, impl::WSOpcodes::kPing, {}}; + SendExtended(ping_msg); + LOG_TRACE() << "Sent keep-alive ping"; + ping_pending_count_.fetch_add(1); + } + + std::size_t NotAnsweredSequentialPingsCount() override { return ping_pending_count_.load(); } + void DoSendBinary(utils::span message) override { MessageExtended mext{message, impl::WSOpcodes::kBinary, {}}; SendExtended(mext); @@ -173,6 +186,7 @@ class WebSocketConnectionImpl final : public WebSocketConnection { } if (frame_.pong_received) { frame_.pong_received = false; + ping_pending_count_ = 0; continue; } if (frame_.waiting_continuation) continue; From 31ae52eda465d84293791120a22bf10196833159 Mon Sep 17 00:00:00 2001 From: Konstantin Goncharik Date: Tue, 29 Jul 2025 17:29:21 +0300 Subject: [PATCH 022/151] feat core: add support for Google Benchmark ASLR-disabling feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://github.com/userver-framework/userver/issues/960 I didn't find any compile-time way to check version of google-benchmark in c++ I had to made some cmake stuff... Tests: протестировано CI Pull Request resolved: https://github.com/userver-framework/userver/pull/961 commit_hash:e5a7dc4f00fc07bdc30ce64f22f0612e56c3597a --- .mapping.json | 1 + core/benchmarks/main.cpp | 5 +++++ universal/CMakeLists.txt | 10 ++++++++++ universal/benchmarks/main_check_aslr_disable.cpp | 5 +++++ 4 files changed, 21 insertions(+) create mode 100644 universal/benchmarks/main_check_aslr_disable.cpp diff --git a/.mapping.json b/.mapping.json index e4867720abf6..f85353ae42d8 100644 --- a/.mapping.json +++ b/.mapping.json @@ -4559,6 +4559,7 @@ "universal/CMakeLists.txt":"taxi/uservices/userver/universal/CMakeLists.txt", "universal/README.md":"taxi/uservices/userver/universal/README.md", "universal/benchmarks/main.cpp":"taxi/uservices/userver/universal/benchmarks/main.cpp", + "universal/benchmarks/main_check_aslr_disable.cpp":"taxi/uservices/userver/universal/benchmarks/main_check_aslr_disable.cpp", "universal/include/userver/cache/impl/lru.hpp":"taxi/uservices/userver/universal/include/userver/cache/impl/lru.hpp", "universal/include/userver/cache/impl/slru.hpp":"taxi/uservices/userver/universal/include/userver/cache/impl/slru.hpp", "universal/include/userver/cache/lru_map.hpp":"taxi/uservices/userver/universal/include/userver/cache/lru_map.hpp", diff --git a/core/benchmarks/main.cpp b/core/benchmarks/main.cpp index 75484ba06a47..ea29bbb4ec54 100644 --- a/core/benchmarks/main.cpp +++ b/core/benchmarks/main.cpp @@ -4,6 +4,10 @@ #include int main(int argc, char** argv) { +#ifndef USERVER_IMPL_NO_MAYBE_REENTER_WITHOUT_ASLR_EXISTS + ::benchmark::MaybeReenterWithoutASLR(argc, argv); +#endif + USERVER_NAMESPACE::utils::impl::FinishStaticRegistration(); const USERVER_NAMESPACE::logging::DefaultLoggerLevelScope level_scope{USERVER_NAMESPACE::logging::Level::kError}; @@ -11,4 +15,5 @@ int main(int argc, char** argv) { ::benchmark::Initialize(&argc, argv); if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); } diff --git a/universal/CMakeLists.txt b/universal/CMakeLists.txt index c93ca019c125..480374ccf261 100644 --- a/universal/CMakeLists.txt +++ b/universal/CMakeLists.txt @@ -315,6 +315,16 @@ if(USERVER_FEATURE_UTEST) include(SetupGBench) set(_benchmark_target benchmark::benchmark) endif() + + try_compile( + HAVE_MAYBE_REENTER_WITHOUT_ASLR ${CMAKE_CURRENT_BINARY_DIR}/benchmarks/main_check_aslr_disable + SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/main_check_aslr_disable.cpp + LINK_LIBRARIES benchmark::benchmark + ) + if(NOT HAVE_MAYBE_REENTER_WITHOUT_ASLR) + target_compile_definitions(${PROJECT_NAME}-internal-ubench INTERFACE USERVER_IMPL_NO_MAYBE_REENTER_WITHOUT_ASLR_EXISTS) + endif() + target_link_libraries(${PROJECT_NAME}-internal-ubench INTERFACE ${_benchmark_target} ${PROJECT_NAME}) endif() diff --git a/universal/benchmarks/main_check_aslr_disable.cpp b/universal/benchmarks/main_check_aslr_disable.cpp new file mode 100644 index 000000000000..43aacddca5c4 --- /dev/null +++ b/universal/benchmarks/main_check_aslr_disable.cpp @@ -0,0 +1,5 @@ +#include +int main() { + void* p = (void*)&benchmark::MaybeReenterWithoutASLR; + return 0; +} From 1b84485be1bd1686a2144a48429447315825d8d0 Mon Sep 17 00:00:00 2001 From: akrivoschekov Date: Tue, 29 Jul 2025 19:37:22 +0300 Subject: [PATCH 023/151] Revert commit rXXXXXX, feat redis: add sentinel_password option and make sure that it work for non-subscribe cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: дождались успешной сборки taxi/uservices/userver/redis/testing commit_hash:515c3ec82b388d38db03e011597bd5aa95859241 --- .../userver/storages/redis/component.hpp | 1 - .../src/storages/redis/impl/secdist_redis.hpp | 1 - redis/src/storages/redis/impl/sentinel.cpp | 4 +- redis/src/storages/redis/impl/server_test.cpp | 171 ------------------ .../redis/impl/subscribe_sentinel.cpp | 8 +- redis/src/storages/redis/redis_secdist.cpp | 2 - 6 files changed, 4 insertions(+), 183 deletions(-) diff --git a/redis/include/userver/storages/redis/component.hpp b/redis/include/userver/storages/redis/component.hpp index 59e23d20fd8b..7470c47abe1e 100644 --- a/redis/include/userver/storages/redis/component.hpp +++ b/redis/include/userver/storages/redis/component.hpp @@ -99,7 +99,6 @@ namespace components { /// "redis_settings": { /// "some_name_of_your_database": { /// "password": "the_password_of_your_database", -/// "sentinel_password": "the_password_for_sentinels_if_any", /// "sentinels": [ /// {"host": "the_host1_of_your_database", "port": 11564} /// ], diff --git a/redis/src/storages/redis/impl/secdist_redis.hpp b/redis/src/storages/redis/impl/secdist_redis.hpp index 7e855e8239e8..5b2765d27153 100644 --- a/redis/src/storages/redis/impl/secdist_redis.hpp +++ b/redis/src/storages/redis/impl/secdist_redis.hpp @@ -21,7 +21,6 @@ struct RedisSettings { std::vector shards; std::vector sentinels; storages::redis::Password password{std::string()}; - storages::redis::Password sentinel_password{std::string()}; storages::redis::ConnectionSecurity secure_connection{storages::redis::ConnectionSecurity::kNone}; std::size_t database_index{0}; }; diff --git a/redis/src/storages/redis/impl/sentinel.cpp b/redis/src/storages/redis/impl/sentinel.cpp index 736b8667b5e6..4d9b24c24754 100644 --- a/redis/src/storages/redis/impl/sentinel.cpp +++ b/redis/src/storages/redis/impl/sentinel.cpp @@ -147,7 +147,6 @@ std::shared_ptr Sentinel::CreateSentinel( const testsuite::RedisControl& testsuite_redis_control ) { const auto& password = settings.password; - const auto& sentinel_password = settings.sentinel_password; const std::vector& shards = settings.shards; LOG_DEBUG() << "shards.size() = " << shards.size(); @@ -158,12 +157,13 @@ std::shared_ptr Sentinel::CreateSentinel( LOG_DEBUG() << "sentinels.size() = " << settings.sentinels.size(); for (const auto& sentinel : settings.sentinels) { LOG_DEBUG() << "sentinel: host = " << sentinel.host << " port = " << sentinel.port; + // SENTINEL MASTERS/SLAVES works without auth, sentinel has no AUTH command. // CLUSTER SLOTS works after auth only. Masters and slaves used instead of // sentinels in cluster mode. conns.emplace_back( sentinel.host, sentinel.port, - (key_shard_factory.IsClusterStrategy() ? password : sentinel_password), + (key_shard_factory.IsClusterStrategy() ? password : Password("")), false, settings.secure_connection ); diff --git a/redis/src/storages/redis/impl/server_test.cpp b/redis/src/storages/redis/impl/server_test.cpp index 13d9b6835e69..268c137db991 100644 --- a/redis/src/storages/redis/impl/server_test.cpp +++ b/redis/src/storages/redis/impl/server_test.cpp @@ -7,14 +7,7 @@ #include #include #include -#include #include -#include -#include -#include -#include - -#include USERVER_NAMESPACE_BEGIN @@ -51,52 +44,6 @@ bool IsConnected(const storages::redis::impl::Redis& redis) { return redis.GetState() == storages::redis::RedisState::kConnected; } -struct MockSentinelServers { - static constexpr size_t kRedisThreadCount = 1; - static constexpr std::string_view kRedisName = "redis_name"; - - void RegisterSentinelMastersSlaves() { - std::vector slave_infos; - std::string redis_name{kRedisName}; - for (const auto& slave : slaves) { - slave_infos.emplace_back(redis_name, kLocalhost, slave.GetPort()); - } - - for (auto& sentinel : sentinels) { - sentinel.RegisterSentinelMastersHandler({{redis_name, kLocalhost, masters[0].GetPort()}}); - sentinel.RegisterSentinelSlavesHandler(redis_name, slave_infos); - } - } - - template - void ForEachServer(const Function& visitor) { - for (auto& server : masters) { - visitor(server); - } - for (auto& server : slaves) { - visitor(server); - } - for (auto& server : sentinels) { - visitor(server); - } - } - - MockRedisServer masters[1] = { - MockRedisServer{"master0"}, - }; - MockRedisServer slaves[2] = { - MockRedisServer{"slave0"}, - MockRedisServer{"slave1"}, - }; - MockRedisServer sentinels[3] = { - MockRedisServer{"sentinel0"}, - MockRedisServer{"sentinel1"}, - MockRedisServer{"sentinel2"}, - }; - std::shared_ptr thread_pool = - std::make_shared(1, kRedisThreadCount); -}; - } // namespace TEST(Redis, NoPassword) { @@ -154,124 +101,6 @@ TEST(Redis, AuthTimeout) { PeriodicCheck([&] { return !IsConnected(*redis); }); } -UTEST(Redis, SentinelAuth) { - MockSentinelServers mock; - mock.RegisterSentinelMastersSlaves(); - mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); - auto& [masters, slaves, sentinels, thread_pool] = mock; - - secdist::RedisSettings settings; - settings.shards = {std::string{MockSentinelServers::kRedisName}}; - settings.sentinel_password = storages::redis::Password("pass"); - settings.sentinels.reserve(std::size(sentinels)); - for (const auto& sentinel : sentinels) { - settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); - } - - std::vector auth_handlers; - auth_handlers.reserve(std::size(sentinels)); - for (auto& sentinel : sentinels) { - auth_handlers.push_back(sentinel.RegisterStatusReplyHandler("AUTH", "OK")); - } - std::vector no_auth_handlers; - no_auth_handlers.reserve(std::size(masters) + std::size(slaves)); - for (auto& server : masters) { - no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); - } - for (auto& server : slaves) { - no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); - } - - auto sentinel_client = storages::redis::impl::Sentinel::CreateSentinel( - thread_pool, settings, "test_shard_group_name", dynamic_config::GetDefaultSource(), "test_client_name", {""} - ); - sentinel_client->WaitConnectedDebug(std::empty(slaves)); - - for (auto& handler : auth_handlers) { - EXPECT_TRUE(handler->WaitForFirstReply(kSmallPeriod)); - } - - for (auto& handler : no_auth_handlers) { - EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); - } - - for (const auto& sentinel : sentinels) { - EXPECT_TRUE(sentinel.WaitForFirstPingReply(kSmallPeriod)); - } -} - -// TODO: TAXICOMMON-10834. Looks like AUTH to sentinel is not sent in case of SUBSCRIBE -UTEST(Redis, DISABLED_SentinelAuthSubscribe) { - MockSentinelServers mock; - mock.RegisterSentinelMastersSlaves(); - mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); - auto& [masters, slaves, sentinels, thread_pool] = mock; - // Sentinels do NOT receive SUBSCRIBE - std::vector subscribe_handlers; - for (auto& server : masters) { - subscribe_handlers.push_back(server.RegisterHandlerWithConstReply("SUBSCRIBE", 1)); - } - for (auto& server : slaves) { - subscribe_handlers.push_back(server.RegisterHandlerWithConstReply("SUBSCRIBE", 1)); - } - - secdist::RedisSettings settings; - settings.shards = {std::string{MockSentinelServers::kRedisName}}; - settings.sentinel_password = storages::redis::Password("pass"); - settings.sentinels.reserve(std::size(sentinels)); - for (const auto& sentinel : sentinels) { - settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); - } - - std::vector auth_handlers; - auth_handlers.reserve(std::size(sentinels)); - for (auto& sentinel : sentinels) { - auth_handlers.push_back(sentinel.RegisterStatusReplyHandler("AUTH", "OK")); - } - std::vector no_auth_handlers; - no_auth_handlers.reserve(std::size(masters) + std::size(slaves)); - for (auto& server : masters) { - no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); - } - for (auto& server : slaves) { - no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); - } - - storages::redis::CommandControl cc{}; - testsuite::RedisControl redis_control{}; - auto subscribe_sentinel = storages::redis::impl::SubscribeSentinel::Create( - thread_pool, - settings, - "test_shard_group_name", - dynamic_config::GetDefaultSource(), - "test_client_name", - {""}, - cc, - redis_control - ); - subscribe_sentinel->WaitConnectedDebug(std::empty(slaves)); - std::shared_ptr client = - std::make_shared(std::move(subscribe_sentinel)); - - storages::redis::SubscriptionToken::OnMessageCb callback = [](const std::string& channel, - const std::string& message) { - EXPECT_TRUE(false) << "Should not be called. Channel = " << channel << ", message = " << message; - }; - auto subscription = client->Subscribe("channel_name", std::move(callback)); - - for (auto& handler : subscribe_handlers) { - EXPECT_TRUE(handler->WaitForFirstReply(utest::kMaxTestWaitTime)); - } - - for (auto& handler : auth_handlers) { - EXPECT_TRUE(handler->WaitForFirstReply(kSmallPeriod)); - } - - for (auto& handler : no_auth_handlers) { - EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); - } -} - TEST(Redis, Select) { MockRedisServer server{"redis_db"}; auto ping_handler = server.RegisterPingHandler(); diff --git a/redis/src/storages/redis/impl/subscribe_sentinel.cpp b/redis/src/storages/redis/impl/subscribe_sentinel.cpp index 0197ba10b467..5def26657735 100644 --- a/redis/src/storages/redis/impl/subscribe_sentinel.cpp +++ b/redis/src/storages/redis/impl/subscribe_sentinel.cpp @@ -84,7 +84,6 @@ std::shared_ptr SubscribeSentinel::Create( const testsuite::RedisControl& testsuite_redis_control ) { const auto& password = settings.password; - const auto& sentinel_password = settings.password; const std::vector& shards = settings.shards; LOG_DEBUG() << "shards.size() = " << shards.size(); @@ -97,14 +96,11 @@ std::shared_ptr SubscribeSentinel::Create( LOG_DEBUG() << "sentinels.size() = " << settings.sentinels.size(); for (const auto& sentinel : settings.sentinels) { LOG_DEBUG() << "sentinel: host = " << sentinel.host << " port = " << sentinel.port; + // SENTINEL MASTERS/SLAVES works without auth, sentinel has no AUTH command. // CLUSTER SLOTS works after auth only. Masters and slaves used instead of // sentinels in cluster mode. conns.emplace_back( - sentinel.host, - sentinel.port, - (is_cluster_mode ? password : sentinel_password), - false, - settings.secure_connection + sentinel.host, sentinel.port, (is_cluster_mode ? password : Password("")), false, settings.secure_connection ); } LOG_DEBUG() << "redis command_control: " << command_control.ToString(); diff --git a/redis/src/storages/redis/redis_secdist.cpp b/redis/src/storages/redis/redis_secdist.cpp index af5a3b1d0d53..7b6dd5bb9070 100644 --- a/redis/src/storages/redis/redis_secdist.cpp +++ b/redis/src/storages/redis/redis_secdist.cpp @@ -35,8 +35,6 @@ RedisMapSettings::RedisMapSettings(const formats::json::Value& doc) { USERVER_NAMESPACE::secdist::RedisSettings settings; settings.password = storages::redis::Password(GetString(client_settings, "password")); - settings.sentinel_password = - storages::redis::Password(client_settings["sentinel_password"].As("")); settings.secure_connection = GetValue(client_settings, "secure_connection", false) ? USERVER_NAMESPACE::storages::redis::ConnectionSecurity::kTLS : USERVER_NAMESPACE::storages::redis::ConnectionSecurity::kNone; From a8899022c183095141a627b48290928591383d0b Mon Sep 17 00:00:00 2001 From: fdr400 Date: Wed, 30 Jul 2025 12:14:47 +0300 Subject: [PATCH 024/151] test kafka: remove unittests non-mock Kafka support disable tests in sandbox remove support for real cluster commit_hash:250a2bb55953b56f628e9253357a973e081d9da1 --- kafka/CMakeLists.txt | 5 -- kafka/utest/src/kafka/utest/kafka_fixture.cpp | 67 ++++--------------- samples/kafka_service/CMakeLists.txt | 11 +-- 3 files changed, 15 insertions(+), 68 deletions(-) diff --git a/kafka/CMakeLists.txt b/kafka/CMakeLists.txt index 30826613a4d6..dd7c202d58f5 100644 --- a/kafka/CMakeLists.txt +++ b/kafka/CMakeLists.txt @@ -12,11 +12,6 @@ userver_module( LINK_LIBRARIES_PRIVATE RdKafka::rdkafka DBTEST_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/tests" DBTEST_LINK_LIBRARIES userver::kafka-utest - DBTEST_DATABASES kafka - DBTEST_ENV - "TESTSUITE_KAFKA_SERVER_START_TIMEOUT=120.0" "TESTSUITE_KAFKA_SERVER_HOST=[::1]" - "TESTSUITE_KAFKA_SERVER_PORT=8099" "TESTSUITE_KAFKA_CONTROLLER_PORT=8100" - "TESTSUITE_KAFKA_CUSTOM_TOPICS=bt:4,lt-1:4,lt-2:4,tt-1:1,tt-2:1,tt-3:1,tt-4:1,tt-5:1,tt-6:1,tt-7:1,tt-8:1" ) target_compile_options(${PROJECT_NAME} PRIVATE "-Wno-ignored-qualifiers") diff --git a/kafka/utest/src/kafka/utest/kafka_fixture.cpp b/kafka/utest/src/kafka/utest/kafka_fixture.cpp index d417b830e7ef..21942e363d48 100644 --- a/kafka/utest/src/kafka/utest/kafka_fixture.cpp +++ b/kafka/utest/src/kafka/utest/kafka_fixture.cpp @@ -17,26 +17,6 @@ USERVER_NAMESPACE_BEGIN namespace kafka::utest { -constexpr const char* kTestsuiteKafkaServerHost{"TESTSUITE_KAFKA_SERVER_HOST"}; -constexpr const char* kDefaultKafkaServerHost{"localhost"}; -constexpr const char* kTestsuiteKafkaServerPort{"TESTSUITE_KAFKA_SERVER_PORT"}; -constexpr const char* kDefaultKafkaServerPort{"9099"}; - -std::string FetchBrokerList() { - const auto env = engine::subprocess::GetCurrentEnvironmentVariablesPtr(); - - std::string server_host{kDefaultKafkaServerHost}; - std::string server_port{kDefaultKafkaServerPort}; - const auto* host = env->GetValueOptional(kTestsuiteKafkaServerHost); - if (host) { - server_host = *host; - } - const auto* port = env->GetValueOptional(kTestsuiteKafkaServerPort); - if (port) { - server_port = *port; - } - return fmt::format("{}:{}", server_host, server_port); -} namespace { impl::Secret MakeSecrets(std::string_view bootstrap_servers) { @@ -54,16 +34,6 @@ impl::ProducerConfiguration PatchDeliveryTimeout(impl::ProducerConfiguration con return configuration; } -bool IsKafkaClusterConfigured() { - static std::optional is_cluster_configured; - - if (!is_cluster_configured.has_value()) { - const auto env = engine::subprocess::GetCurrentEnvironmentVariablesPtr(); - is_cluster_configured = - env->GetValueOptional(kTestsuiteKafkaServerHost) || env->GetValueOptional(kTestsuiteKafkaServerPort); - } - return *is_cluster_configured; -} } // namespace class KafkaCluster::MockCluster { @@ -79,13 +49,17 @@ class KafkaCluster::MockCluster { UINVARIANT(mock_cluster_.GetHandle(), "Failed to get mock cluster handle"); conf_.ForgetUnderlyingConf(); } + + ~MockCluster() = default; + static impl::ConfHolder CreateConfiguredConf() { rd_kafka_conf_t* conf = rd_kafka_conf_new(); UINVARIANT(conf, "Failed to create Kafka config"); rd_kafka_conf_set(conf, "log_level", "2", nullptr, sizeof(nullptr)); + return impl::ConfHolder(conf); } - ~MockCluster() = default; + rd_kafka_mock_cluster_t* GetMockCluster() const { return mock_cluster_.GetHandle(); } private: @@ -99,17 +73,7 @@ bool operator==(const Message& lhs, const Message& rhs) { std::tie(rhs.topic, rhs.key, rhs.payload, rhs.partition); } -std::string KafkaCluster::InitBootstrapServers() { - if (IsKafkaClusterConfigured()) { - LOG_DEBUG("Using real Kafka"); - return FetchBrokerList(); - } - LOG_DEBUG("Using mock Kafka"); - mock_ = utils::Box(); - return rd_kafka_mock_cluster_bootstraps(mock_->GetMockCluster()); -} - -KafkaCluster::KafkaCluster() : bootstrap_servers_(InitBootstrapServers()) { +KafkaCluster::KafkaCluster() : bootstrap_servers_{rd_kafka_mock_cluster_bootstraps(mock_->GetMockCluster())} { UINVARIANT(!bootstrap_servers_.empty(), "Empty bootstrap_servers"); } @@ -118,17 +82,14 @@ std::atomic KafkaCluster::kTopicsCount{0}; KafkaCluster::~KafkaCluster() = default; std::string KafkaCluster::GenerateTopic(std::uint32_t partition_cnt) { - const std::string topic = fmt::format("tt-{}", kTopicsCount.fetch_add(1)); - if (!IsKafkaClusterConfigured()) { - const auto err = rd_kafka_mock_topic_create( - mock_->GetMockCluster(), topic.c_str(), utils::numeric_cast(partition_cnt), /*replication_factor=*/1 - ); - if (err != RD_KAFKA_RESP_ERR_NO_ERROR) { - LOG_ERROR("Failed to create topic {}: {}", topic, rd_kafka_err2str(err)); - } else { - LOG_INFO("Successful topic creation {}", topic); - } - } + std::string topic = fmt::format("tt-{}", kTopicsCount.fetch_add(1)); + const auto err = rd_kafka_mock_topic_create( + mock_->GetMockCluster(), topic.c_str(), utils::numeric_cast(partition_cnt), /*replication_factor=*/1 + ); + UINVARIANT( + err == RD_KAFKA_RESP_ERR_NO_ERROR, fmt::format("Failed to create topic '{}': {}", topic, rd_kafka_err2str(err)) + ); + return topic; } diff --git a/samples/kafka_service/CMakeLists.txt b/samples/kafka_service/CMakeLists.txt index 90aa70c2e6ec..8d8ed198621d 100644 --- a/samples/kafka_service/CMakeLists.txt +++ b/samples/kafka_service/CMakeLists.txt @@ -27,16 +27,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_objs) # /// [Kafka service sample - kafka unit test cmake] add_executable(${PROJECT_NAME}-unittest "unittest/kafka_test.cpp") target_link_libraries(${PROJECT_NAME}-unittest PRIVATE ${PROJECT_NAME}_objs userver::kafka-utest) -userver_add_utest( - NAME - ${PROJECT_NAME}-unittest - DATABASES - kafka - TEST_ENV - "TESTSUITE_KAFKA_SERVER_START_TIMEOUT=120.0" - "TESTSUITE_KAFKA_SERVER_HOST=[::1]" - "TESTSUITE_KAFKA_CUSTOM_TOPICS=test-topic-1:1,test-topic-2:1" -) +userver_add_utest(NAME ${PROJECT_NAME}-unittest) # /// [Kafka service sample - kafka unit test cmake] From 922af0d9f596d2c55d251c2252c56c0dc53410ac Mon Sep 17 00:00:00 2001 From: fdr400 Date: Wed, 30 Jul 2025 12:46:50 +0300 Subject: [PATCH 025/151] feat grpc: add lowlevel client tests commit_hash:1d635a19d04e86860958632298556026db0c80b5 --- .mapping.json | 10 ++ grpc/functional_tests/CMakeLists.txt | 3 + grpc/functional_tests/lowlevel/CMakeLists.txt | 13 +++ grpc/functional_tests/lowlevel/main.cpp | 25 +++++ .../lowlevel/proto/samples/greeter.proto | 15 +++ .../lowlevel/src/client_runner.cpp | 47 ++++++++++ .../lowlevel/src/client_runner.hpp | 26 +++++ .../lowlevel/static_config.yaml | 86 +++++++++++++++++ .../tests-client-protocol/conftest.py | 74 +++++++++++++++ .../lowlevel/tests-client-protocol/http2.py | 79 ++++++++++++++++ .../test_http2_raw_responses.py | 94 +++++++++++++++++++ .../lowlevel/tests-client-protocol/utils.py | 9 ++ 12 files changed, 481 insertions(+) create mode 100644 grpc/functional_tests/lowlevel/CMakeLists.txt create mode 100644 grpc/functional_tests/lowlevel/main.cpp create mode 100644 grpc/functional_tests/lowlevel/proto/samples/greeter.proto create mode 100644 grpc/functional_tests/lowlevel/src/client_runner.cpp create mode 100644 grpc/functional_tests/lowlevel/src/client_runner.hpp create mode 100644 grpc/functional_tests/lowlevel/static_config.yaml create mode 100644 grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py create mode 100644 grpc/functional_tests/lowlevel/tests-client-protocol/http2.py create mode 100644 grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py create mode 100644 grpc/functional_tests/lowlevel/tests-client-protocol/utils.py diff --git a/.mapping.json b/.mapping.json index f85353ae42d8..849bca703dac 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2041,6 +2041,16 @@ "grpc/functional_tests/basic_server/tests-unix-socket/conftest.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py", "grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py", "grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py", + "grpc/functional_tests/lowlevel/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/lowlevel/CMakeLists.txt", + "grpc/functional_tests/lowlevel/main.cpp":"taxi/uservices/userver/grpc/functional_tests/lowlevel/main.cpp", + "grpc/functional_tests/lowlevel/proto/samples/greeter.proto":"taxi/uservices/userver/grpc/functional_tests/lowlevel/proto/samples/greeter.proto", + "grpc/functional_tests/lowlevel/src/client_runner.cpp":"taxi/uservices/userver/grpc/functional_tests/lowlevel/src/client_runner.cpp", + "grpc/functional_tests/lowlevel/src/client_runner.hpp":"taxi/uservices/userver/grpc/functional_tests/lowlevel/src/client_runner.hpp", + "grpc/functional_tests/lowlevel/static_config.yaml":"taxi/uservices/userver/grpc/functional_tests/lowlevel/static_config.yaml", + "grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py":"taxi/uservices/userver/grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py", + "grpc/functional_tests/lowlevel/tests-client-protocol/http2.py":"taxi/uservices/userver/grpc/functional_tests/lowlevel/tests-client-protocol/http2.py", + "grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py":"taxi/uservices/userver/grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py", + "grpc/functional_tests/lowlevel/tests-client-protocol/utils.py":"taxi/uservices/userver/grpc/functional_tests/lowlevel/tests-client-protocol/utils.py", "grpc/functional_tests/metrics/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/metrics/CMakeLists.txt", "grpc/functional_tests/metrics/config_vars.yaml":"taxi/uservices/userver/grpc/functional_tests/metrics/config_vars.yaml", "grpc/functional_tests/metrics/proto/samples/greeter.proto":"taxi/uservices/userver/grpc/functional_tests/metrics/proto/samples/greeter.proto", diff --git a/grpc/functional_tests/CMakeLists.txt b/grpc/functional_tests/CMakeLists.txt index a9d1370c2e5d..64defe33d3f9 100644 --- a/grpc/functional_tests/CMakeLists.txt +++ b/grpc/functional_tests/CMakeLists.txt @@ -16,3 +16,6 @@ add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-middleware-server) add_subdirectory(middleware_client) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-middleware-client) + +add_subdirectory(lowlevel) +add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-lowlevel) diff --git a/grpc/functional_tests/lowlevel/CMakeLists.txt b/grpc/functional_tests/lowlevel/CMakeLists.txt new file mode 100644 index 000000000000..9b64acfadae7 --- /dev/null +++ b/grpc/functional_tests/lowlevel/CMakeLists.txt @@ -0,0 +1,13 @@ +project(userver-grpc-tests-lowlevel CXX) + +add_executable(${PROJECT_NAME} + main.cpp + src/client_runner.cpp) + +target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + +include(UserverGrpcTargets) +userver_add_grpc_library(${PROJECT_NAME}-proto PROTOS samples/greeter.proto) +target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}-proto) + +userver_chaos_testsuite_add(TESTS_DIRECTORY tests-client-protocol) diff --git a/grpc/functional_tests/lowlevel/main.cpp b/grpc/functional_tests/lowlevel/main.cpp new file mode 100644 index 000000000000..b31d32919c58 --- /dev/null +++ b/grpc/functional_tests/lowlevel/main.cpp @@ -0,0 +1,25 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +int main(int argc, const char* const argv[]) { + const auto component_list = components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append() + .Append() + .AppendComponentList(ugrpc::client::MinimalComponentList()) + .Append(); + + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/grpc/functional_tests/lowlevel/proto/samples/greeter.proto b/grpc/functional_tests/lowlevel/proto/samples/greeter.proto new file mode 100644 index 000000000000..b239fa823b85 --- /dev/null +++ b/grpc/functional_tests/lowlevel/proto/samples/greeter.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package samples.api; + +service GreeterService { + rpc SayHello(GreetingRequest) returns(GreetingResponse) {} +} + +message GreetingRequest { + string name = 1; +} + +message GreetingResponse { + string greeting = 1; +} diff --git a/grpc/functional_tests/lowlevel/src/client_runner.cpp b/grpc/functional_tests/lowlevel/src/client_runner.cpp new file mode 100644 index 000000000000..5152130ab973 --- /dev/null +++ b/grpc/functional_tests/lowlevel/src/client_runner.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +ClientRunner::ClientRunner(const components::ComponentConfig& config, const components::ComponentContext& context) + : server::handlers::HttpHandlerJsonBase{config, context}, + client_{context.FindComponent() + .GetFactory() + .MakeClient<::samples::api::GreeterServiceClient>( + "client", + config["server-endpoint"].As() + )} {} + +auto ClientRunner::HandleRequestJsonThrow( + const HttpRequest& /*request*/, + const Value& /*request_json*/, + RequestContext& /*context*/ +) const -> Value { + std::optional<::grpc::Status> grpc_status; + try { + [[maybe_unused]] auto response = client_.SayHello({}); + grpc_status.emplace(::grpc::Status::OK); + } catch (const ugrpc::client::ErrorWithStatus& ex) { + grpc_status.emplace(ex.GetStatus()); + } catch (const std::exception& ex) { + return {}; + } + + return formats::json::MakeObject("grpc-status", ugrpc::ToString(grpc_status->error_code())); +} + +yaml_config::Schema ClientRunner::GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( +type: object +description: gRPC client runner +additionalProperties: false +properties: + server-endpoint: + description: endpoint http2 server is listening + type: string +)"); +} + +USERVER_NAMESPACE_END diff --git a/grpc/functional_tests/lowlevel/src/client_runner.hpp b/grpc/functional_tests/lowlevel/src/client_runner.hpp new file mode 100644 index 000000000000..8f4be181e0d2 --- /dev/null +++ b/grpc/functional_tests/lowlevel/src/client_runner.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +class ClientRunner final : public server::handlers::HttpHandlerJsonBase { +public: + static constexpr std::string_view kName{"client-runner"}; + + ClientRunner(const components::ComponentConfig& config, const components::ComponentContext& context); + + auto HandleRequestJsonThrow(const HttpRequest& request, const Value& /*request_json*/, RequestContext& /*context*/) + const -> Value override; + + static yaml_config::Schema GetStaticConfigSchema(); + +private: + const ::samples::api::GreeterServiceClient client_; +}; + +USERVER_NAMESPACE_END diff --git a/grpc/functional_tests/lowlevel/static_config.yaml b/grpc/functional_tests/lowlevel/static_config.yaml new file mode 100644 index 000000000000..b4f1af732fe8 --- /dev/null +++ b/grpc/functional_tests/lowlevel/static_config.yaml @@ -0,0 +1,86 @@ +# yaml +components_manager: + components: + # The required common components + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: debug + overflow_behavior: discard + + # Contains machinery common to all gRPC clients + grpc-client-common: + # The TaskProcessor for blocking connection initiation + blocking-task-processor: grpc-blocking-task-processor + + # Creates gRPC clients + grpc-client-factory: + disable-all-pipeline-middlewares: true + + middlewares: + grpc-client-logging: + enabled: true + + # Optional channel parameters for gRPC Core + # https://grpc.github.io/grpc/core/group__grpc__arg__keys.html + channel-args: {} + + default-service-config: > + { + "methodConfig": [{ + "name": [{}], + "retryPolicy": { + "maxAttempts": 3, + "initialBackoff": "0.01s", + "maxBackoff": "0.3s", + "backoffMultiplier": 2, + "retryableStatusCodes": [ + "UNAVAILABLE", + "ABORTED", + "CANCELLED", + "DEADLINE_EXCEEDED" + ] + } + }] + } + + grpc-client-middlewares-pipeline: {} + + grpc-client-logging: {} + + server: + listener: + port: 8092 + task_processor: main-task-processor + + # Testsuite components + testsuite-support: {} + tests-control: + method: POST + path: /tests/{action} + skip-unregistered-testpoints: true + task_processor: main-task-processor + testpoint-timeout: 10s + throttling_enabled: false + http-client: # for testsuite + fs-task-processor: main-task-processor + dns-client: # for http-client + fs-task-processor: fs-task-processor + + # User-defined component + client-runner: + path: /client + method: GET,POST + + default_task_processor: main-task-processor # Task processor in which components start. + + task_processors: + grpc-blocking-task-processor: # For blocking gRPC channel creation + worker_threads: 2 + thread_name: grpc-worker + main-task-processor: # For non-blocking operations + worker_threads: 8 + fs-task-processor: # For blocking filesystem operations + worker_threads: 2 diff --git a/grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py b/grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py new file mode 100644 index 000000000000..2962ee146943 --- /dev/null +++ b/grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py @@ -0,0 +1,74 @@ +import asyncio +import asyncio.streams as streams +import asyncio.trsock as trsock +import json +from typing import Any +from typing import AsyncGenerator +from typing import Callable +from typing import Dict + +import grpc +import http2 +import pytest + +import testsuite.utils.net as net_utils + +import utils + +pytest_plugins = [ + 'pytest_userver.plugins.core', +] + +USERVER_CONFIG_HOOKS = ['userver_config_http2_server_endpoint'] + + +@pytest.fixture(scope='session') +def retryable_status_codes(_retry_policy) -> list[grpc.StatusCode]: + return [utils.status_from_str(status) for status in _retry_policy['retryableStatusCodes']] + + +@pytest.fixture(scope='session') +def max_attempts(_retry_policy) -> int: + return _retry_policy['maxAttempts'] + + +@pytest.fixture(scope='session') +def _retry_policy(_default_service_config) -> Dict[str, Any]: + return _default_service_config['methodConfig'][0]['retryPolicy'] + + +@pytest.fixture(scope='session') +def _default_service_config(service_config) -> Dict[str, Any]: + return json.loads( + service_config['components_manager']['components']['grpc-client-factory']['default-service-config'] + ) + + +@pytest.fixture(scope='session', autouse=True) +async def grpc_server() -> AsyncGenerator[http2.GrpcServer, None]: + server = http2.GrpcServer() + + def _protocol_factory(): + _loop = asyncio.get_running_loop() + _reader = streams.StreamReader(loop=_loop) + return streams.StreamReaderProtocol(_reader, server, loop=_loop) + + async with net_utils.create_tcp_server(_protocol_factory) as s: + socket: trsock.TransportSocket = s.sockets[0] + endpoint = ':'.join(map(str, socket.getsockname())) + + server.endpoint = endpoint + + yield server + + +@pytest.fixture(scope='session') +def userver_config_http2_server_endpoint( + grpc_server: http2.GrpcServer, +) -> Callable[[Any, Any], None]: + def _patch_config(config_yaml, _config_vars): + components = config_yaml['components_manager']['components'] + + components['client-runner']['server-endpoint'] = grpc_server.endpoint + + return _patch_config diff --git a/grpc/functional_tests/lowlevel/tests-client-protocol/http2.py b/grpc/functional_tests/lowlevel/tests-client-protocol/http2.py new file mode 100644 index 000000000000..1605fbd22d17 --- /dev/null +++ b/grpc/functional_tests/lowlevel/tests-client-protocol/http2.py @@ -0,0 +1,79 @@ +import asyncio.streams as streams +import dataclasses +from typing import Callable +from typing import List +from typing import Optional +from typing import Tuple + +import h2.config as config +import h2.connection as connection +import h2.errors as errors +import h2.events as events + + +class Frame: + "Base class for all frames" + + +@dataclasses.dataclass(frozen=True) +class HeadersFrame(Frame): + headers: List[Tuple[str, str]] + end_stream: bool = False + + +@dataclasses.dataclass(frozen=True) +class DataFrame(Frame): + data: bytes + + +@dataclasses.dataclass(frozen=True) +class RstStreamFrame(Frame): + error_code: errors.ErrorCodes + + +@dataclasses.dataclass +class GrpcServer: + response_factory: Optional[Callable[[], List[Frame]]] = None + endpoint: Optional[str] = None + + async def __call__(self, reader: streams.StreamReader, writer: streams.StreamWriter) -> None: + conn = connection.H2Connection(config=config.H2Configuration(client_side=False)) + conn.initiate_connection() + writer.write(conn.data_to_send()) + + try: + while True: + data = await reader.read(65536) + if not data: + break + + for event in conn.receive_data(data): + if isinstance(event, events.RequestReceived): + self._handle_request(conn, event) + elif isinstance(event, events.DataReceived): + conn.acknowledge_received_data(event.flow_controlled_length, event.stream_id) + + writer.write(conn.data_to_send()) + await writer.drain() + except ConnectionResetError: + pass + finally: + writer.close() + await writer.wait_closed() + + def _handle_request(self, conn: connection.H2Connection, event: events.RequestReceived): + stream_id = event.stream_id + assert stream_id is not None + + if not self.response_factory: + return + + for frame in self.response_factory(): + if isinstance(frame, HeadersFrame): + conn.send_headers(stream_id, headers=frame.headers, end_stream=frame.end_stream) + elif isinstance(frame, DataFrame): + conn.send_data(stream_id, data=frame.data) + elif isinstance(frame, RstStreamFrame): + conn.reset_stream(stream_id, error_code=frame.error_code) + else: + assert False diff --git a/grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py b/grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py new file mode 100644 index 000000000000..ddd8d129deb0 --- /dev/null +++ b/grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py @@ -0,0 +1,94 @@ +from typing import List +from typing import Tuple + +import grpc +import http2 +import pytest + +import utils + +# mapped statuses according to https://grpc.github.io/grpc/cpp/md_doc_http-grpc-status-mapping.html +HTTP2_TO_GRPC = { + 400: grpc.StatusCode.INTERNAL, + 401: grpc.StatusCode.UNAUTHENTICATED, + 403: grpc.StatusCode.PERMISSION_DENIED, + 404: grpc.StatusCode.UNIMPLEMENTED, + 429: grpc.StatusCode.UNAVAILABLE, + 502: grpc.StatusCode.UNAVAILABLE, + 503: grpc.StatusCode.UNAVAILABLE, + 504: grpc.StatusCode.UNAVAILABLE, +} + +HTTP_STATUSES = list(HTTP2_TO_GRPC) + [ + # not mapped statuses + # TODO: think about 1xx, because their handling differs from other codes + 201, + 303, + 405, + 501, +] + + +@pytest.mark.parametrize('http_status', HTTP_STATUSES) +@pytest.mark.parametrize( + 'grpc_status_code', + [grpc.StatusCode.OK, grpc.StatusCode.ABORTED, grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE], +) +async def test_grpc_client_ignores_any_http_status_when_grpc_status_exists( + service_client, + http_status: int, + grpc_status_code: grpc.StatusCode, + grpc_server: http2.GrpcServer, +) -> None: + def _response_factory() -> List[http2.Frame]: + # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html + # Trailers-Only is only HEADERS http2 frame (with END_STREAM flag) + + headers: List[Tuple[str, str]] = [ + (':status', str(http_status)), + ('content-type', 'application/grpc'), + ('grpc-status', utils.status_to_str(grpc_status_code)), + ] + + return [http2.HeadersFrame(headers=headers, end_stream=True)] + + grpc_server.response_factory = _response_factory + + expected_grpc_status_code = grpc_status_code + actual_grpc_status_code = await run_client(service_client) + + assert actual_grpc_status_code == expected_grpc_status_code + + +@pytest.mark.parametrize('http_status', HTTP_STATUSES) +async def test_grpc_client_synthesizes_grpc_status_from_http_status( + service_client, http_status: int, grpc_server: http2.GrpcServer +) -> None: + def _response_factory() -> List[http2.Frame]: + # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html + # Trailers-Only is only HEADERS http2 frame (with END_STREAM flag) + + headers: List[Tuple[str, str]] = [ + (':status', str(http_status)), + ('content-type', 'application/grpc'), + ] + + return [http2.HeadersFrame(headers=headers, end_stream=True)] + + grpc_server.response_factory = _response_factory + + expected_grpc_status_code = http2_status_to_grpc(http_status) + actual_grpc_status_code = await run_client(service_client) + + assert actual_grpc_status_code == expected_grpc_status_code + + +async def run_client(service_client) -> grpc.StatusCode: + resp = await service_client.post('/client') + assert resp.status == 200 + + return utils.status_from_str(resp.json()['grpc-status']) + + +def http2_status_to_grpc(http_status: int) -> grpc.StatusCode: + return HTTP2_TO_GRPC.get(http_status, grpc.StatusCode.UNKNOWN) diff --git a/grpc/functional_tests/lowlevel/tests-client-protocol/utils.py b/grpc/functional_tests/lowlevel/tests-client-protocol/utils.py new file mode 100644 index 000000000000..d3a68d870cec --- /dev/null +++ b/grpc/functional_tests/lowlevel/tests-client-protocol/utils.py @@ -0,0 +1,9 @@ +import grpc + + +def status_from_str(grpc_status_code_str: str) -> grpc.StatusCode: + return getattr(grpc.StatusCode, grpc_status_code_str) + + +def status_to_str(grpc_status_code: grpc.StatusCode) -> str: + return str(grpc_status_code.value[0]) From e05f8861e996327d5c09c57fae7db1fe760d2dfd Mon Sep 17 00:00:00 2001 From: abramov-alex Date: Wed, 30 Jul 2025 15:33:29 +0300 Subject: [PATCH 026/151] fix grpc-protovalidate: wrap ValidatorFactory with ThreadLocal ValidatorFactory is not safe to use with userver coroutines. This PR introduces wrapper functions that handle factory state correctly as a ThreadLocal variable. Tests: CI commit_hash:849ce4a897d715191b8c7e12693569cad43a75d2 --- .mapping.json | 3 + .../grpc-protovalidate/buf_validate.hpp | 2 +- .../userver/grpc-protovalidate/validate.hpp | 107 ++++++++++++++ .../grpc-protovalidate/client/middleware.cpp | 41 +++--- .../grpc-protovalidate/client/middleware.hpp | 9 -- .../src/grpc-protovalidate/impl/utils.cpp | 7 +- .../grpc-protovalidate/server/middleware.cpp | 65 ++++---- .../grpc-protovalidate/server/middleware.hpp | 9 -- .../src/grpc-protovalidate/validate.cpp | 139 ++++++++++++++++++ .../tests/validator_test.cpp | 98 ++++++++++++ 10 files changed, 398 insertions(+), 82 deletions(-) create mode 100644 libraries/grpc-protovalidate/include/userver/grpc-protovalidate/validate.hpp create mode 100644 libraries/grpc-protovalidate/src/grpc-protovalidate/validate.cpp create mode 100644 libraries/grpc-protovalidate/tests/validator_test.cpp diff --git a/.mapping.json b/.mapping.json index 849bca703dac..94d29b6beca2 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2510,6 +2510,7 @@ "libraries/grpc-protovalidate/include/userver/grpc-protovalidate/client/component.hpp":"taxi/uservices/userver/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/client/component.hpp", "libraries/grpc-protovalidate/include/userver/grpc-protovalidate/client/exceptions.hpp":"taxi/uservices/userver/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/client/exceptions.hpp", "libraries/grpc-protovalidate/include/userver/grpc-protovalidate/server/component.hpp":"taxi/uservices/userver/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/server/component.hpp", + "libraries/grpc-protovalidate/include/userver/grpc-protovalidate/validate.hpp":"taxi/uservices/userver/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/validate.hpp", "libraries/grpc-protovalidate/library.yaml":"taxi/uservices/userver/libraries/grpc-protovalidate/library.yaml", "libraries/grpc-protovalidate/src/grpc-protovalidate/client/component.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/src/grpc-protovalidate/client/component.cpp", "libraries/grpc-protovalidate/src/grpc-protovalidate/client/exceptions.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/src/grpc-protovalidate/client/exceptions.cpp", @@ -2520,12 +2521,14 @@ "libraries/grpc-protovalidate/src/grpc-protovalidate/server/component.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/src/grpc-protovalidate/server/component.cpp", "libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.cpp", "libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.hpp":"taxi/uservices/userver/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.hpp", + "libraries/grpc-protovalidate/src/grpc-protovalidate/validate.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/src/grpc-protovalidate/validate.cpp", "libraries/grpc-protovalidate/tests/proto/types/constrained.proto":"taxi/uservices/userver/libraries/grpc-protovalidate/tests/proto/types/constrained.proto", "libraries/grpc-protovalidate/tests/proto/types/unit_test.proto":"taxi/uservices/userver/libraries/grpc-protovalidate/tests/proto/types/unit_test.proto", "libraries/grpc-protovalidate/tests/utils.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/tests/utils.cpp", "libraries/grpc-protovalidate/tests/utils.hpp":"taxi/uservices/userver/libraries/grpc-protovalidate/tests/utils.hpp", "libraries/grpc-protovalidate/tests/validator_client_test.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/tests/validator_client_test.cpp", "libraries/grpc-protovalidate/tests/validator_service_test.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/tests/validator_service_test.cpp", + "libraries/grpc-protovalidate/tests/validator_test.cpp":"taxi/uservices/userver/libraries/grpc-protovalidate/tests/validator_test.cpp", "libraries/grpc-reflection/CMakeLists.txt":"taxi/uservices/userver/libraries/grpc-reflection/CMakeLists.txt", "libraries/grpc-reflection/__module_deps__.yaml":"taxi/uservices/userver/libraries/grpc-reflection/__module_deps__.yaml", "libraries/grpc-reflection/functional_tests/CMakeLists.txt":"taxi/uservices/userver/libraries/grpc-reflection/functional_tests/CMakeLists.txt", diff --git a/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/buf_validate.hpp b/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/buf_validate.hpp index 20ff8d980090..a695c7232ec4 100644 --- a/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/buf_validate.hpp +++ b/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/buf_validate.hpp @@ -10,4 +10,4 @@ #pragma clang system_header #endif -#include +#include // IWYU pragma: export diff --git a/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/validate.hpp b/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/validate.hpp new file mode 100644 index 000000000000..d5a3d3b29971 --- /dev/null +++ b/libraries/grpc-protovalidate/include/userver/grpc-protovalidate/validate.hpp @@ -0,0 +1,107 @@ +#pragma once + +/// @file +/// @brief Coroutine-safe wrappers around protovalidate-cc. + +#include +#include +#include + +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace grpc_protovalidate { + +class ValidationError { +public: + /// @brief Type of validation error. + enum class Type { + + /// @brief Validation failed due to validator internal errors. + /// In most cases this indicates [protovalidate](https://github.com/bufbuild/protovalidate) + /// CEL expression errors in the *proto* file. + kInternal = 1, + + /// @brief Validation failed due to violations of + /// [protovalidate](https://github.com/bufbuild/protovalidate) constraints by the input message. + kRule = 2 + }; + + ValidationError(Type type, std::string description, std::string message_name); + + /// @brief Creates an error of type @c Type::kRule. + ValidationError(buf::validate::ValidationResult result, std::string message_name); + + /// @brief Get error type. + Type GetType() const; + + /// @brief A fully-qualified name of the message type this error originates from. + const std::string& GetMessageName() const; + + /// @brief A human-readable description of the error. + const std::string& GetDescription() const; + + /// @brief A list of found contraint violations. + /// + /// The list is empty if this is a @c Type::kInternal error. + const std::vector& GetViolations() const; + + /// @brief Constructs a @c grpc::Status of this error. + /// + /// Message contains a short human-readable representation of the error. + /// If include_violations is true, details contain the list of violations. Otherwise, details are empty. + grpc::Status GetGrpcStatus(bool include_violations = true) const; + +private: + buf::validate::Violations MakeViolationsProto() const; + + Type type_; + std::string description_; + std::optional result_; + std::string message_name_; +}; + +logging::LogHelper& operator<<(logging::LogHelper& lh, const ValidationError& error); + +class ValidationResult { +public: + ValidationResult() = default; + ValidationResult(ValidationError error); + + /// @returns `true` iff the validation found no violations. + bool IsSuccess() const; + + explicit operator bool() const { return IsSuccess(); } + + /// @returns ValidationError with the description of the violations. + /// @pre `IsSuccess() == true`. + const ValidationError& GetError() const&; + + /// @returns ValidationError with the description of the violations. + /// @pre `IsSuccess() == true`. + ValidationError&& GetError() &&; + +private: + std::optional error_; +}; + +struct ValidationParams { + /// @brief If true, does not check remaining constraints after the first error is encountered. + bool fail_fast = false; +}; + +/// @brief Coroutine-safe wrapper around Validate from protovalidate-cc. +/// +/// @returns std::nullopt if no violations have been found. +/// Using @c buf::validate::ValidatorFactory is not safe in a coroutine context and may cause crashes. +/// This method uses ThreadLocal to ensure no unexpected coroutine-context switches occur during validation. +ValidationResult ValidateMessage(const google::protobuf::Message& message, const ValidationParams& params = {}); + +} // namespace grpc_protovalidate + +USERVER_NAMESPACE_END diff --git a/libraries/grpc-protovalidate/src/grpc-protovalidate/client/middleware.cpp b/libraries/grpc-protovalidate/src/grpc-protovalidate/client/middleware.cpp index 6f19ffcc332a..21da8fb6f147 100644 --- a/libraries/grpc-protovalidate/src/grpc-protovalidate/client/middleware.cpp +++ b/libraries/grpc-protovalidate/src/grpc-protovalidate/client/middleware.cpp @@ -1,11 +1,13 @@ #include +#include + #include #include +#include #include - -#include +#include USERVER_NAMESPACE_BEGIN @@ -16,8 +18,7 @@ const ValidationSettings& Settings::Get(std::string_view method_name) const { return it != per_method.end() ? it->second : global; } -Middleware::Middleware(const Settings& settings) - : settings_(settings), validator_factory_(grpc_protovalidate::impl::CreateProtoValidatorFactory()) {} +Middleware::Middleware(const Settings& settings) : settings_(settings) {} Middleware::~Middleware() = default; @@ -25,26 +26,20 @@ void Middleware::PostRecvMessage( ugrpc::client::MiddlewareCallContext& context, const google::protobuf::Message& message ) const { - const auto& settings = settings_.Get(context.GetCallName()); - google::protobuf::Arena arena; - - auto validator = validator_factory_->NewValidator(&arena, settings.fail_fast); - auto result = validator.Validate(message); - - if (!result.ok()) { - // Usually that means some CEL expression in a proto file can not be - // parsed or evaluated. - LOG_ERROR() << "Validator internal error (check response constraints in the proto file): " << result.status(); - throw ValidatorError(context.GetCallName()); - } else if (result.value().violations_size() > 0) { - for (const auto& violation : result.value().violations()) { - LOG_ERROR() << "Response constraint violation: " << violation.proto(); - } - - throw ResponseError(context.GetCallName(), std::move(result).value()); - } else { - LOG_DEBUG() << "Response is valid"; + const ValidationSettings& settings = settings_.Get(context.GetCallName()); + const ValidationResult result = ValidateMessage(message, {.fail_fast = settings.fail_fast}); + if (result.IsSuccess()) { + return; + } + const ValidationError& error = result.GetError(); + switch (error.GetType()) { + case ValidationError::Type::kInternal: + throw ValidatorError(context.GetCallName()); + case ValidationError::Type::kRule: + LOG_WARNING() << error; + throw ResponseError(context.GetCallName(), error.GetViolations()); } + UINVARIANT(false, "Unexpected error type"); } } // namespace grpc_protovalidate::client diff --git a/libraries/grpc-protovalidate/src/grpc-protovalidate/client/middleware.hpp b/libraries/grpc-protovalidate/src/grpc-protovalidate/client/middleware.hpp index 40359b39dca0..8d70bf6effc1 100644 --- a/libraries/grpc-protovalidate/src/grpc-protovalidate/client/middleware.hpp +++ b/libraries/grpc-protovalidate/src/grpc-protovalidate/client/middleware.hpp @@ -1,18 +1,10 @@ #pragma once -#include -#include #include #include #include -namespace buf::validate { - -class ValidatorFactory; - -} // namespace buf::validate - USERVER_NAMESPACE_BEGIN namespace grpc_protovalidate::client { @@ -38,7 +30,6 @@ class Middleware final : public ugrpc::client::MiddlewareBase { private: Settings settings_; - std::unique_ptr validator_factory_; }; } // namespace grpc_protovalidate::client diff --git a/libraries/grpc-protovalidate/src/grpc-protovalidate/impl/utils.cpp b/libraries/grpc-protovalidate/src/grpc-protovalidate/impl/utils.cpp index bd10dc768b39..ca0b49bc6314 100644 --- a/libraries/grpc-protovalidate/src/grpc-protovalidate/impl/utils.cpp +++ b/libraries/grpc-protovalidate/src/grpc-protovalidate/impl/utils.cpp @@ -5,6 +5,7 @@ #include +#include #include namespace { @@ -91,7 +92,11 @@ namespace grpc_protovalidate::impl { std::unique_ptr CreateProtoValidatorFactory() { auto result = buf::validate::ValidatorFactory::New(); UINVARIANT(result.ok(), "Failed to create validator factory"); - return std::move(result).value(); + std::unique_ptr factory = std::move(result).value(); + for (const google::protobuf::Descriptor* descriptor : ugrpc::impl::GetGeneratedMessages()) { + factory->Add(descriptor); + } + return factory; } } // namespace grpc_protovalidate::impl diff --git a/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.cpp b/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.cpp index a3b12f8e7354..02ce7b5046b5 100644 --- a/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.cpp +++ b/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.cpp @@ -1,62 +1,49 @@ #include -#include -#include +#include +#include #include -#include - -#include +#include USERVER_NAMESPACE_BEGIN namespace grpc_protovalidate::server { +namespace { + +void LogError(const ValidationError& error) { + switch (error.GetType()) { + case ValidationError::Type::kInternal: + LOG_ERROR() << error; + return; + case ValidationError::Type::kRule: + LOG_WARNING() << error; + return; + } + UINVARIANT(false, "Unexpected error type"); +} + +} // namespace + const ValidationSettings& Settings::Get(std::string_view method_name) const { auto it = per_method.find(method_name); return it != per_method.end() ? it->second : global; } -Middleware::Middleware(const Settings& settings) - : settings_(settings), validator_factory_(grpc_protovalidate::impl::CreateProtoValidatorFactory()) {} +Middleware::Middleware(const Settings& settings) : settings_(settings) {} Middleware::~Middleware() = default; void Middleware::PostRecvMessage(ugrpc::server::MiddlewareCallContext& context, google::protobuf::Message& request) const { - const auto& settings = settings_.Get(context.GetCallName()); - google::protobuf::Arena arena; - - auto validator = validator_factory_->NewValidator(&arena, settings.fail_fast); - auto result = validator.Validate(request); - - if (!result.ok()) { - // Usually that means some CEL expression in a proto file can not be parsed or evaluated. - context.SetError(grpc::Status{ - grpc::StatusCode::INTERNAL, - fmt::format( - "Validator internal error (check request constraints in the proto file): {}", result.status().ToString() - )}); - } else if (result.value().violations_size() > 0) { - for (const auto& violation : result.value().violations()) { - LOG_ERROR() << "Request constraint violation: " << violation.proto(); - } - - if (settings.send_violations) { - google::rpc::Status gstatus; - gstatus.set_code(grpc::StatusCode::INVALID_ARGUMENT); - gstatus.set_message(fmt::format("Request violates constraints (count={})", result.value().violations_size()) - ); - gstatus.add_details()->PackFrom(result.value().proto()); - context.SetError(ugrpc::ToGrpcStatus(gstatus)); - } else { - context.SetError(grpc::Status{ - grpc::StatusCode::INVALID_ARGUMENT, - fmt::format("Request violates constraints (count={})", result.value().violations_size())}); - } - } else { - LOG_DEBUG() << "Request is valid"; + const ValidationSettings& settings = settings_.Get(context.GetCallName()); + const ValidationResult result = ValidateMessage(request, {.fail_fast = settings.fail_fast}); + if (result.IsSuccess()) { + return; } + LogError(result.GetError()); + context.SetError(result.GetError().GetGrpcStatus(settings.send_violations)); } } // namespace grpc_protovalidate::server diff --git a/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.hpp b/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.hpp index 2eb0a0182563..7af6f249c09e 100644 --- a/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.hpp +++ b/libraries/grpc-protovalidate/src/grpc-protovalidate/server/middleware.hpp @@ -1,18 +1,10 @@ #pragma once -#include -#include #include #include #include -namespace buf::validate { - -class ValidatorFactory; - -} // namespace buf::validate - USERVER_NAMESPACE_BEGIN namespace grpc_protovalidate::server { @@ -39,7 +31,6 @@ class Middleware final : public ugrpc::server::MiddlewareBase { private: Settings settings_; - std::unique_ptr validator_factory_; }; } // namespace grpc_protovalidate::server diff --git a/libraries/grpc-protovalidate/src/grpc-protovalidate/validate.cpp b/libraries/grpc-protovalidate/src/grpc-protovalidate/validate.cpp new file mode 100644 index 000000000000..ef64de85afbd --- /dev/null +++ b/libraries/grpc-protovalidate/src/grpc-protovalidate/validate.cpp @@ -0,0 +1,139 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace grpc_protovalidate { + +namespace { + +compiler::ThreadLocal kValidatorFactory = [] { return impl::CreateProtoValidatorFactory(); }; + +} // namespace + +ValidationError::ValidationError(Type type, std::string description, std::string message_name) + : type_(type), + description_(fmt::format("Message '{}' validation error: {}", message_name, std::move(description))), + result_(std::nullopt), + message_name_(std::move(message_name)) {} + +ValidationError::ValidationError(buf::validate::ValidationResult result, std::string message_name) + : type_(Type::kRule), + description_(fmt::format( + "Message '{}' validation error: {} constraint(s) violated", + message_name, + result.violations_size() + )), + result_(std::move(result)), + message_name_(std::move(message_name)) {} + +ValidationError::Type ValidationError::GetType() const { return type_; } + +const std::string& ValidationError::GetMessageName() const { return message_name_; } + +const std::string& ValidationError::GetDescription() const { return description_; } + +const std::vector& ValidationError::GetViolations() const { + if (GetType() == Type::kInternal || !result_.has_value()) { + static constexpr std::vector kNoViolations; + return kNoViolations; + } + return result_->violations(); +} + +grpc::Status ValidationError::GetGrpcStatus(bool include_violations) const { + google::rpc::Status gstatus; + gstatus.set_message(GetDescription()); + switch (GetType()) { + case Type::kInternal: + gstatus.set_code(grpc::StatusCode::INTERNAL); + break; + case Type::kRule: { + gstatus.set_code(grpc::StatusCode::INVALID_ARGUMENT); + break; + } + } + if (include_violations) { + gstatus.add_details()->PackFrom(MakeViolationsProto()); + } + return ugrpc::ToGrpcStatus(gstatus); +} + +buf::validate::Violations ValidationError::MakeViolationsProto() const { + buf::validate::Violations proto; + std::transform( + GetViolations().begin(), + GetViolations().end(), + RepeatedPtrFieldBackInserter(proto.mutable_violations()), + [](const buf::validate::RuleViolation& violation) { return violation.proto(); } + ); + return proto; +} + +logging::LogHelper& operator<<(logging::LogHelper& lh, const ValidationError& error) { + lh << error.GetDescription(); + for (const buf::validate::RuleViolation& violation : error.GetViolations()) { + lh << violation.proto(); + } + return lh; +} + +ValidationResult::ValidationResult(ValidationError error) : error_(std::move(error)) {} + +bool ValidationResult::IsSuccess() const { return !error_.has_value(); } + +const ValidationError& ValidationResult::GetError() const& { + if (IsSuccess()) { + throw std::logic_error("Requested error for success validation result"); + } + return error_.value(); +} + +ValidationError&& ValidationResult::GetError() && { + if (IsSuccess()) { + throw std::logic_error("Requested error for success validation result"); + } + return std::move(error_).value(); +} + +ValidationResult ValidateMessage(const google::protobuf::Message& message, const ValidationParams& params) { + auto validator_factory = kValidatorFactory.Use(); + google::protobuf::Arena arena; + auto validator = (*validator_factory)->NewValidator(&arena, params.fail_fast); + auto result = validator.Validate(message); + if (!result.ok()) { + return ValidationError( + ValidationError::Type::kInternal, + fmt::format( + "internal protovalidate error (check constraints syntax in the proto file) - {}", + result.status().ToString() + ), + message.GetTypeName() + ); + } + if (result->violations_size() != 0) { + return ValidationError(result.value(), message.GetTypeName()); + } + return ValidationResult(); +} + +} // namespace grpc_protovalidate + +USERVER_NAMESPACE_END diff --git a/libraries/grpc-protovalidate/tests/validator_test.cpp b/libraries/grpc-protovalidate/tests/validator_test.cpp new file mode 100644 index 000000000000..727bc0eeb298 --- /dev/null +++ b/libraries/grpc-protovalidate/tests/validator_test.cpp @@ -0,0 +1,98 @@ +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include "utils.hpp" + +USERVER_NAMESPACE_BEGIN + +UTEST(ValidateTest, Valid) { + auto result = grpc_protovalidate::ValidateMessage(tests::CreateValidMessage(2)); + EXPECT_TRUE(result.IsSuccess()); + UEXPECT_THROW_MSG(result.GetError(), std::logic_error, "Requested error for success validation result"); +} + +UTEST_MT(ValidateTest, ValidMultithreaded, 4) { + std::vector tasks; + for (std::size_t i = 1; i < 250; ++i) { + tasks.emplace_back(utils::Async("ValidateTestMultithreaded", [i]() { + auto _ = grpc_protovalidate::ValidateMessage(tests::CreateValidMessage(i)); + })); + } + UEXPECT_NO_THROW(engine::WaitAllChecked(tasks)); +} + +UTEST(EnsureValidTest, InvalidDefault) { + constexpr auto kExpectedMessage = "Message 'types.ConstrainedMessage' validation error: 18 constraint(s) violated"; + auto result = grpc_protovalidate::ValidateMessage(tests::CreateInvalidMessage()); + + ASSERT_FALSE(result.IsSuccess()); + const auto error = std::move(result).GetError(); + + EXPECT_EQ(error.GetType(), grpc_protovalidate::ValidationError::Type::kRule); + EXPECT_EQ(error.GetMessageName(), "types.ConstrainedMessage"); + EXPECT_EQ(error.GetDescription(), kExpectedMessage); + EXPECT_THAT(error.GetViolations(), testing::SizeIs(18)); + + const grpc::Status status_no_details = error.GetGrpcStatus(false); + EXPECT_EQ(status_no_details.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_EQ(status_no_details.error_message(), kExpectedMessage); + + const grpc::Status status_with_details = error.GetGrpcStatus(true); + EXPECT_EQ(status_with_details.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_EQ(status_with_details.error_message(), kExpectedMessage); + EXPECT_GT(status_with_details.error_details().size(), 0); +} + +UTEST(EnsureValidTest, InvalidFailFast) { + constexpr auto kExpectedMessage = "Message 'types.ConstrainedMessage' validation error: 1 constraint(s) violated"; + const auto result = grpc_protovalidate::ValidateMessage(tests::CreateInvalidMessage(), {.fail_fast = true}); + + ASSERT_FALSE(result.IsSuccess()); + const auto error = std::move(result).GetError(); + + EXPECT_EQ(error.GetType(), grpc_protovalidate::ValidationError::Type::kRule); + EXPECT_EQ(error.GetMessageName(), "types.ConstrainedMessage"); + EXPECT_EQ(error.GetDescription(), kExpectedMessage); + EXPECT_THAT(error.GetViolations(), testing::SizeIs(1)); + + const grpc::Status status_no_details = error.GetGrpcStatus(false); + EXPECT_EQ(status_no_details.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_EQ(status_no_details.error_message(), kExpectedMessage); + + const grpc::Status status_with_details = error.GetGrpcStatus(true); + EXPECT_EQ(status_with_details.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_EQ(status_with_details.error_message(), kExpectedMessage); + EXPECT_GT(status_with_details.error_details().size(), 0); +} + +UTEST(EnsureValidTest, InvalidConstraintsDefault) { + constexpr auto kExpectedMessage = + "Message 'types.InvalidConstraints' validation error: internal protovalidate error (check constraints " + "syntax in the proto file) - INVALID_ARGUMENT: no_such_field : non_existent_field"; + const auto result = grpc_protovalidate::ValidateMessage(types::InvalidConstraints(), {.fail_fast = true}); + + ASSERT_FALSE(result.IsSuccess()); + const auto error = std::move(result).GetError(); + + EXPECT_EQ(error.GetType(), grpc_protovalidate::ValidationError::Type::kInternal); + EXPECT_EQ(error.GetMessageName(), "types.InvalidConstraints"); + EXPECT_EQ(error.GetDescription(), kExpectedMessage); + EXPECT_THAT(error.GetViolations(), testing::SizeIs(0)); + + const grpc::Status status = error.GetGrpcStatus(); + EXPECT_EQ(status.error_code(), grpc::StatusCode::INTERNAL); + EXPECT_EQ(status.error_message(), kExpectedMessage); +} + +USERVER_NAMESPACE_END From c6e5d8ca27effb39e71e0054285be28963fb5c64 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Wed, 30 Jul 2025 19:41:52 +0300 Subject: [PATCH 027/151] feat redis: add sentinel_password option and make sure that it work for subscribe and non-subscribe cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 515c3ec82b388d38db03e011597bd5aa95859241, reversing changes made to 2cea4d26a612c87c4068bca0fda87a28aace7f86. Added more tests, that revealed issues of initial commit. Issues were fixed. Tests: протестировано CI и в тетсинге commit_hash:21045d5fa4b4ddcb65277982e9bfa3aa5b73854d --- .../userver/storages/redis/component.hpp | 1 + .../src/storages/redis/impl/secdist_redis.hpp | 1 + redis/src/storages/redis/impl/sentinel.cpp | 11 +- redis/src/storages/redis/impl/server_test.cpp | 313 ++++++++++++++++++ .../redis/impl/subscribe_sentinel.cpp | 8 +- .../redis/impl/subscription_storage.cpp | 4 +- redis/src/storages/redis/redis_secdist.cpp | 2 + 7 files changed, 334 insertions(+), 6 deletions(-) diff --git a/redis/include/userver/storages/redis/component.hpp b/redis/include/userver/storages/redis/component.hpp index 7470c47abe1e..59e23d20fd8b 100644 --- a/redis/include/userver/storages/redis/component.hpp +++ b/redis/include/userver/storages/redis/component.hpp @@ -99,6 +99,7 @@ namespace components { /// "redis_settings": { /// "some_name_of_your_database": { /// "password": "the_password_of_your_database", +/// "sentinel_password": "the_password_for_sentinels_if_any", /// "sentinels": [ /// {"host": "the_host1_of_your_database", "port": 11564} /// ], diff --git a/redis/src/storages/redis/impl/secdist_redis.hpp b/redis/src/storages/redis/impl/secdist_redis.hpp index 5b2765d27153..7e855e8239e8 100644 --- a/redis/src/storages/redis/impl/secdist_redis.hpp +++ b/redis/src/storages/redis/impl/secdist_redis.hpp @@ -21,6 +21,7 @@ struct RedisSettings { std::vector shards; std::vector sentinels; storages::redis::Password password{std::string()}; + storages::redis::Password sentinel_password{std::string()}; storages::redis::ConnectionSecurity secure_connection{storages::redis::ConnectionSecurity::kNone}; std::size_t database_index{0}; }; diff --git a/redis/src/storages/redis/impl/sentinel.cpp b/redis/src/storages/redis/impl/sentinel.cpp index 4d9b24c24754..1e48fcce7f51 100644 --- a/redis/src/storages/redis/impl/sentinel.cpp +++ b/redis/src/storages/redis/impl/sentinel.cpp @@ -147,6 +147,7 @@ std::shared_ptr Sentinel::CreateSentinel( const testsuite::RedisControl& testsuite_redis_control ) { const auto& password = settings.password; + const auto& sentinel_password = settings.sentinel_password; const std::vector& shards = settings.shards; LOG_DEBUG() << "shards.size() = " << shards.size(); @@ -157,13 +158,12 @@ std::shared_ptr Sentinel::CreateSentinel( LOG_DEBUG() << "sentinels.size() = " << settings.sentinels.size(); for (const auto& sentinel : settings.sentinels) { LOG_DEBUG() << "sentinel: host = " << sentinel.host << " port = " << sentinel.port; - // SENTINEL MASTERS/SLAVES works without auth, sentinel has no AUTH command. // CLUSTER SLOTS works after auth only. Masters and slaves used instead of // sentinels in cluster mode. conns.emplace_back( sentinel.host, sentinel.port, - (key_shard_factory.IsClusterStrategy() ? password : Password("")), + (key_shard_factory.IsClusterStrategy() ? password : sentinel_password), false, settings.secure_connection ); @@ -193,7 +193,12 @@ std::shared_ptr Sentinel::CreateSentinel( } void Sentinel::Stop() noexcept { - impl_.reset(); + if (impl_) { + // ev watchers could be performing right now, so we have to call Stop() before impl_.reset() to make sure + // that watchers stop and do not get impl_ == nullptr during their run. + impl_->Stop(); + impl_.reset(); + } UASSERT(!impl_); } diff --git a/redis/src/storages/redis/impl/server_test.cpp b/redis/src/storages/redis/impl/server_test.cpp index 268c137db991..5391728dd56b 100644 --- a/redis/src/storages/redis/impl/server_test.cpp +++ b/redis/src/storages/redis/impl/server_test.cpp @@ -7,7 +7,14 @@ #include #include #include +#include #include +#include +#include +#include +#include + +#include USERVER_NAMESPACE_BEGIN @@ -44,6 +51,92 @@ bool IsConnected(const storages::redis::impl::Redis& redis) { return redis.GetState() == storages::redis::RedisState::kConnected; } +struct MockSentinelServers { + static constexpr size_t kRedisThreadCount = 1; + static constexpr std::string_view kRedisName = "redis_name"; + + void RegisterSentinelMastersSlaves() { + std::vector slave_infos; + std::string redis_name{kRedisName}; + for (const auto& slave : slaves) { + slave_infos.emplace_back(redis_name, kLocalhost, slave.GetPort()); + } + + for (auto& sentinel : sentinels) { + sentinel.RegisterSentinelMastersHandler({{redis_name, kLocalhost, masters[0].GetPort()}}); + sentinel.RegisterSentinelSlavesHandler(redis_name, slave_infos); + } + } + + template + void ForEachServer(const Function& visitor) { + for (auto& server : masters) { + visitor(server); + } + for (auto& server : slaves) { + visitor(server); + } + for (auto& server : sentinels) { + visitor(server); + } + } + + void CreateSentinelClientAndWait(const secdist::RedisSettings& settings) { + auto sentinel_client = storages::redis::impl::Sentinel::CreateSentinel( + thread_pool, settings, "test_shard_group_name", dynamic_config::GetDefaultSource(), "test_client_name", {""} + ); + sentinel_client->WaitConnectedDebug(std::empty(slaves)); + } + + void CreateSubscribeSentinelClientAndWait(const secdist::RedisSettings& settings) { + // Sentinels do NOT receive SUBSCRIBE + std::vector subscribe_handlers; + for (auto& server : masters) { + subscribe_handlers.push_back(server.RegisterHandlerWithConstReply("SUBSCRIBE", 1)); + } + for (auto& server : slaves) { + subscribe_handlers.push_back(server.RegisterHandlerWithConstReply("SUBSCRIBE", 1)); + } + + storages::redis::CommandControl cc{}; + testsuite::RedisControl redis_control{}; + auto dynconf = dynamic_config::GetDefaultSource(); + using storages::redis::impl::SubscribeSentinel; + auto subscribe_sentinel = SubscribeSentinel::Create( + thread_pool, settings, "test_shard_group_name", dynconf, "test_client_name", {""}, cc, redis_control + ); + subscribe_sentinel->WaitConnectedDebug(std::empty(slaves)); + + std::shared_ptr client = + std::make_shared(std::move(subscribe_sentinel)); + + storages::redis::SubscriptionToken::OnMessageCb callback = [](const std::string& channel, + const std::string& message) { + EXPECT_TRUE(false) << "Should not be called. Channel = " << channel << ", message = " << message; + }; + auto subscription = client->Subscribe("channel_name", std::move(callback)); + + for (auto& handler : subscribe_handlers) { + EXPECT_TRUE(handler->WaitForFirstReply(utest::kMaxTestWaitTime)); + } + } + + MockRedisServer masters[1] = { + MockRedisServer{"master0"}, + }; + MockRedisServer slaves[2] = { + MockRedisServer{"slave0"}, + MockRedisServer{"slave1"}, + }; + MockRedisServer sentinels[3] = { + MockRedisServer{"sentinel0"}, + MockRedisServer{"sentinel1"}, + MockRedisServer{"sentinel2"}, + }; + std::shared_ptr thread_pool = + std::make_shared(1, kRedisThreadCount); +}; + } // namespace TEST(Redis, NoPassword) { @@ -101,6 +194,226 @@ TEST(Redis, AuthTimeout) { PeriodicCheck([&] { return !IsConnected(*redis); }); } +UTEST(Redis, SentinelAuth) { + MockSentinelServers mock; + mock.RegisterSentinelMastersSlaves(); + mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); + auto& [masters, slaves, sentinels, thread_pool] = mock; + + secdist::RedisSettings settings; + settings.shards = {std::string{MockSentinelServers::kRedisName}}; + settings.sentinel_password = storages::redis::Password("pass"); + settings.sentinels.reserve(std::size(sentinels)); + for (const auto& sentinel : sentinels) { + settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); + } + + std::vector auth_handlers; + auth_handlers.reserve(std::size(sentinels)); + for (auto& sentinel : sentinels) { + auth_handlers.push_back(sentinel.RegisterStatusReplyHandler("AUTH", "OK")); + } + std::vector no_auth_handlers; + no_auth_handlers.reserve(std::size(masters) + std::size(slaves)); + for (auto& server : masters) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + for (auto& server : slaves) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + + mock.CreateSentinelClientAndWait(settings); + + for (auto& handler : auth_handlers) { + EXPECT_TRUE(handler->WaitForFirstReply(kSmallPeriod)); + } + + for (auto& handler : no_auth_handlers) { + EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); + } + + for (const auto& sentinel : sentinels) { + EXPECT_TRUE(sentinel.WaitForFirstPingReply(kSmallPeriod)); + } +} + +UTEST(Redis, SentinelNoAuthButPassword) { + MockSentinelServers mock; + mock.RegisterSentinelMastersSlaves(); + mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); + auto& [masters, slaves, sentinels, thread_pool] = mock; + + secdist::RedisSettings settings; + settings.shards = {std::string{MockSentinelServers::kRedisName}}; + settings.password = storages::redis::Password("pass"); + settings.sentinels.reserve(std::size(sentinels)); + for (const auto& sentinel : sentinels) { + settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); + } + + std::vector no_auth_handlers; + no_auth_handlers.reserve(std::size(sentinels)); + for (auto& sentinel : sentinels) { + no_auth_handlers.push_back(sentinel.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + std::vector auth_handlers; + auth_handlers.reserve(std::size(masters) + std::size(slaves)); + for (auto& server : masters) { + auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "OK")); + } + for (auto& server : slaves) { + auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "OK")); + } + + mock.CreateSentinelClientAndWait(settings); + + for (auto& handler : auth_handlers) { + EXPECT_TRUE(handler->WaitForFirstReply(kSmallPeriod)); + } + + for (auto& handler : no_auth_handlers) { + EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); + } + + for (const auto& sentinel : sentinels) { + EXPECT_TRUE(sentinel.WaitForFirstPingReply(kSmallPeriod)); + } +} + +UTEST(Redis, SentinelNoAuth) { + MockSentinelServers mock; + mock.RegisterSentinelMastersSlaves(); + mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); + auto& [masters, slaves, sentinels, thread_pool] = mock; + + secdist::RedisSettings settings; + settings.shards = {std::string{MockSentinelServers::kRedisName}}; + settings.sentinels.reserve(std::size(sentinels)); + for (const auto& sentinel : sentinels) { + settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); + } + + std::vector no_auth_handlers; + no_auth_handlers.reserve(std::size(sentinels) + std::size(masters) + std::size(slaves)); + mock.ForEachServer([&](auto& server) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + }); + + mock.CreateSentinelClientAndWait(settings); + + for (auto& handler : no_auth_handlers) { + EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); + } + + for (const auto& sentinel : sentinels) { + EXPECT_TRUE(sentinel.WaitForFirstPingReply(kSmallPeriod)); + } +} + +UTEST(Redis, SentinelAuthSubscribe) { + MockSentinelServers mock; + mock.RegisterSentinelMastersSlaves(); + mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); + auto& [masters, slaves, sentinels, thread_pool] = mock; + + secdist::RedisSettings settings; + settings.shards = {std::string{MockSentinelServers::kRedisName}}; + settings.sentinel_password = storages::redis::Password("pass"); + settings.sentinels.reserve(std::size(sentinels)); + for (const auto& sentinel : sentinels) { + settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); + } + + std::vector auth_handlers; + auth_handlers.reserve(std::size(sentinels)); + for (auto& sentinel : sentinels) { + auth_handlers.push_back(sentinel.RegisterStatusReplyHandler("AUTH", "OK")); + } + std::vector no_auth_handlers; + no_auth_handlers.reserve(std::size(masters) + std::size(slaves)); + for (auto& server : masters) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + for (auto& server : slaves) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + + mock.CreateSubscribeSentinelClientAndWait(settings); + + for (auto& handler : auth_handlers) { + EXPECT_TRUE(handler->WaitForFirstReply(kSmallPeriod)); + } + + for (auto& handler : no_auth_handlers) { + EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); + } +} + +UTEST(Redis, SentinelNoAuthSubscribeButPassword) { + MockSentinelServers mock; + mock.RegisterSentinelMastersSlaves(); + mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); + auto& [masters, slaves, sentinels, thread_pool] = mock; + + secdist::RedisSettings settings; + settings.shards = {std::string{MockSentinelServers::kRedisName}}; + settings.password = storages::redis::Password("pass"); + settings.sentinels.reserve(std::size(sentinels)); + for (const auto& sentinel : sentinels) { + settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); + } + + std::vector no_auth_handlers; + no_auth_handlers.reserve(std::size(sentinels)); + for (auto& sentinel : sentinels) { + no_auth_handlers.push_back(sentinel.RegisterStatusReplyHandler("AUTH", "FAIL")); + } + std::vector auth_handlers; + auth_handlers.reserve(std::size(masters) + std::size(slaves)); + for (auto& server : masters) { + auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "OK")); + } + for (auto& server : slaves) { + auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "OK")); + } + + mock.CreateSubscribeSentinelClientAndWait(settings); + + for (auto& handler : auth_handlers) { + EXPECT_TRUE(handler->WaitForFirstReply(kSmallPeriod)); + } + + for (auto& handler : no_auth_handlers) { + EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); + } +} + +UTEST(Redis, SentinelNoAuthSubscribe) { + MockSentinelServers mock; + mock.RegisterSentinelMastersSlaves(); + mock.ForEachServer([](auto& server) { server.RegisterPingHandler(); }); + auto& [masters, slaves, sentinels, thread_pool] = mock; + + secdist::RedisSettings settings; + settings.shards = {std::string{MockSentinelServers::kRedisName}}; + settings.sentinels.reserve(std::size(sentinels)); + for (const auto& sentinel : sentinels) { + settings.sentinels.emplace_back(kLocalhost, sentinel.GetPort()); + } + + std::vector no_auth_handlers; + no_auth_handlers.reserve(std::size(sentinels) + std::size(masters) + std::size(slaves)); + mock.ForEachServer([&](auto& server) { + no_auth_handlers.push_back(server.RegisterStatusReplyHandler("AUTH", "FAIL")); + }); + + mock.CreateSubscribeSentinelClientAndWait(settings); + + for (auto& handler : no_auth_handlers) { + EXPECT_FALSE(handler->WaitForFirstReply(kWaitPeriod)); + } +} + TEST(Redis, Select) { MockRedisServer server{"redis_db"}; auto ping_handler = server.RegisterPingHandler(); diff --git a/redis/src/storages/redis/impl/subscribe_sentinel.cpp b/redis/src/storages/redis/impl/subscribe_sentinel.cpp index 5def26657735..dd65cacdd2ac 100644 --- a/redis/src/storages/redis/impl/subscribe_sentinel.cpp +++ b/redis/src/storages/redis/impl/subscribe_sentinel.cpp @@ -84,6 +84,7 @@ std::shared_ptr SubscribeSentinel::Create( const testsuite::RedisControl& testsuite_redis_control ) { const auto& password = settings.password; + const auto& sentinel_password = settings.sentinel_password; const std::vector& shards = settings.shards; LOG_DEBUG() << "shards.size() = " << shards.size(); @@ -96,11 +97,14 @@ std::shared_ptr SubscribeSentinel::Create( LOG_DEBUG() << "sentinels.size() = " << settings.sentinels.size(); for (const auto& sentinel : settings.sentinels) { LOG_DEBUG() << "sentinel: host = " << sentinel.host << " port = " << sentinel.port; - // SENTINEL MASTERS/SLAVES works without auth, sentinel has no AUTH command. // CLUSTER SLOTS works after auth only. Masters and slaves used instead of // sentinels in cluster mode. conns.emplace_back( - sentinel.host, sentinel.port, (is_cluster_mode ? password : Password("")), false, settings.secure_connection + sentinel.host, + sentinel.port, + (is_cluster_mode ? password : sentinel_password), + false, + settings.secure_connection ); } LOG_DEBUG() << "redis command_control: " << command_control.ToString(); diff --git a/redis/src/storages/redis/impl/subscription_storage.cpp b/redis/src/storages/redis/impl/subscription_storage.cpp index c92bb30711fc..490526e76a17 100644 --- a/redis/src/storages/redis/impl/subscription_storage.cpp +++ b/redis/src/storages/redis/impl/subscription_storage.cpp @@ -774,7 +774,9 @@ void SubscriptionStorage::Unsubscribe(SubscriptionId subscription_id) { storage_ void SubscriptionStorage::Stop() { storage_impl_.ClearCallbackMaps(); - rebalance_schedulers_.clear(); + + // rebalance_schedulers_ are accessed concurrently, do not clear() them here. + for (auto& scheduler : rebalance_schedulers_) scheduler->Stop(); } void SubscriptionStorage::SetCommandControl(const CommandControl& control) { storage_impl_.SetCommandControl(control); } diff --git a/redis/src/storages/redis/redis_secdist.cpp b/redis/src/storages/redis/redis_secdist.cpp index 7b6dd5bb9070..af5a3b1d0d53 100644 --- a/redis/src/storages/redis/redis_secdist.cpp +++ b/redis/src/storages/redis/redis_secdist.cpp @@ -35,6 +35,8 @@ RedisMapSettings::RedisMapSettings(const formats::json::Value& doc) { USERVER_NAMESPACE::secdist::RedisSettings settings; settings.password = storages::redis::Password(GetString(client_settings, "password")); + settings.sentinel_password = + storages::redis::Password(client_settings["sentinel_password"].As("")); settings.secure_connection = GetValue(client_settings, "secure_connection", false) ? USERVER_NAMESPACE::storages::redis::ConnectionSecurity::kTLS : USERVER_NAMESPACE::storages::redis::ConnectionSecurity::kNone; From 5669f5fec2e6c0bdde5d56b0484701f245b0e0ca Mon Sep 17 00:00:00 2001 From: Fedor Osetrov <33493672+fdr400@users.noreply.github.com> Date: Wed, 30 Jul 2025 19:57:15 +0300 Subject: [PATCH 028/151] fix grpc: do not test unix-socket + TLS for MacOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI Pull Request resolved: https://github.com/userver-framework/userver/pull/978 commit_hash:848613645273aee5b706d7ad0a51299769525b3f --- .../basic_server/tests-tls/conftest.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/grpc/functional_tests/basic_server/tests-tls/conftest.py b/grpc/functional_tests/basic_server/tests-tls/conftest.py index b1b35f271edc..f0d4129735d4 100644 --- a/grpc/functional_tests/basic_server/tests-tls/conftest.py +++ b/grpc/functional_tests/basic_server/tests-tls/conftest.py @@ -1,4 +1,5 @@ import pathlib +import platform import grpc import pytest @@ -14,14 +15,21 @@ @pytest.fixture(scope='session') -def prepare_service_config(): +def prepare_service_config(get_free_port): def _do_patch(config_yaml, config_vars): components = config_yaml['components_manager']['components'] - components['grpc-server']['tls'] = { + grpc_server = components['grpc-server'] + grpc_server['tls'] = { 'key': str(TESTDIR / 'private_key.key'), 'cert': str(TESTDIR / 'cert.crt'), } + # MacOS does not support unix-socket + TLS + if platform.system() == 'Darwin': + grpc_server.pop('unix-socket-path', None) + if 'port' not in grpc_server: + grpc_server['port'] = get_free_port() + return _do_patch From 092c4aa0cecfd983fb89ce23414ce4064e0aae60 Mon Sep 17 00:00:00 2001 From: MichaelKab Date: Wed, 30 Jul 2025 20:19:34 +0300 Subject: [PATCH 029/151] feat CI: bump Kafka version to 4.0 in CI checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------------------ Note: by creating a PR or an issue you automatically agree to the CLA. See [CONTRIBUTING.md](https://github.com/userver-framework/userver/blob/develop/CONTRIBUTING.md). Feel free to remove this note, the agreement holds. Tests: протестировано CI Pull Request resolved: https://github.com/userver-framework/userver/pull/979 commit_hash:fb3a76a40f4d63d889d5db4eeb66babf8db4dbe3 --- .github/workflows/ci-conan.yml | 14 +++++------- .github/workflows/ci.yml | 3 ++- .github/workflows/macos.yml | 4 +--- .mapping.json | 2 +- scripts/docker/setup-base-ubuntu-22.04-env.sh | 8 +++---- scripts/kafka/install_kafka.sh | 22 ------------------- scripts/kafka/ubuntu_install_kafka.sh | 10 +++++++++ 7 files changed, 24 insertions(+), 39 deletions(-) delete mode 100755 scripts/kafka/install_kafka.sh create mode 100755 scripts/kafka/ubuntu_install_kafka.sh diff --git a/.github/workflows/ci-conan.yml b/.github/workflows/ci-conan.yml index be66f9c24bd1..121cb20bc950 100644 --- a/.github/workflows/ci-conan.yml +++ b/.github/workflows/ci-conan.yml @@ -21,8 +21,10 @@ jobs: include: - os: ubuntu-22.04 conanflags: '' + tests-env: 'JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64' - os: macos-latest conanflags: '-o python_path=python3.11' + tests-env: '' steps: - name: Checkout @@ -45,8 +47,7 @@ jobs: sudo apt update sudo apt install -y postgresql redis mongodb-org mongodb-mongosh - - sudo ./scripts/kafka/install_kafka.sh + sudo ./scripts/kafka/ubuntu_install_kafka.sh ./scripts/rabbitmq/ubuntu_install_rabbitmq_server.sh @@ -55,10 +56,7 @@ jobs: run: | brew update brew tap mongodb/brew - brew install clang-format postgresql redis rabbitmq mongodb-community - - sudo ./scripts/kafka/install_kafka.sh - + brew install clang-format postgresql redis kafka rabbitmq mongodb-community brew install python@3.11 - name: Install common packages @@ -73,7 +71,7 @@ jobs: conan create . --build=missing -s:a compiler.cppstd=17 -pr:b=default ${{matrix.conanflags}} - name: Test userver conan package - run: | + run: |- mv libraries/easy/samples/3_json samples/ mv scripts/tests/conanfile.py samples/ rm -rf userver/cmake/ @@ -96,6 +94,6 @@ jobs: s3api \ ; do mv conanfile.py $SAMPLE/ - conan test $SAMPLE/ --build=never -s:a compiler.cppstd=17 -pr:b=default ${{matrix.conanflags}} ${USERVER_VERSION} + ${{matrix.tests-env}} conan test $SAMPLE/ --build=never -s:a compiler.cppstd=17 -pr:b=default ${{matrix.conanflags}} ${USERVER_VERSION} mv $SAMPLE/conanfile.py ./ done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 801d152da39e..b107f2e6e457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ name: Ubuntu - feature/** env: + JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64 UBSAN_OPTIONS: print_stacktrace=1 ASAN_OPTIONS: detect_odr_violation=2 CCACHE_DIR: /home/runner/.cache/ccache @@ -184,7 +185,7 @@ jobs: - name: Install Kafka run: | - sudo ./scripts/kafka/install_kafka.sh + sudo ./scripts/kafka/ubuntu_install_kafka.sh - name: Install RabbitMQ packages run: | diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index a92b4dfbe651..1e7bab22e807 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -73,9 +73,7 @@ jobs: - name: Install test dependencies run: | brew tap mongodb/brew - brew install clickhouse redis mongodb-community rabbitmq - - ./scripts/kafka/install_kafka.sh + brew install clickhouse redis kafka mongodb-community rabbitmq - name: Setup ccache run: | diff --git a/.mapping.json b/.mapping.json index 94d29b6beca2..636ebc6b605a 100644 --- a/.mapping.json +++ b/.mapping.json @@ -4079,7 +4079,7 @@ "scripts/grpc/templates/service.usrv.hpp.jinja":"taxi/uservices/userver/scripts/grpc/templates/service.usrv.hpp.jinja", "scripts/grpc/templates/utils.inc.jinja":"taxi/uservices/userver/scripts/grpc/templates/utils.inc.jinja", "scripts/human_logs.py":"taxi/uservices/userver/scripts/human_logs.py", - "scripts/kafka/install_kafka.sh":"taxi/uservices/userver/scripts/kafka/install_kafka.sh", + "scripts/kafka/ubuntu_install_kafka.sh":"taxi/uservices/userver/scripts/kafka/ubuntu_install_kafka.sh", "scripts/migrate_from_legacy_redis_ns.sh":"taxi/uservices/userver/scripts/migrate_from_legacy_redis_ns.sh", "scripts/mongo/ubuntu-install-mongodb.sh":"taxi/uservices/userver/scripts/mongo/ubuntu-install-mongodb.sh", "scripts/perf-blocking-syscall":"taxi/uservices/userver/scripts/perf-blocking-syscall", diff --git a/scripts/docker/setup-base-ubuntu-22.04-env.sh b/scripts/docker/setup-base-ubuntu-22.04-env.sh index 38ccff1c33a9..8edb7272d96c 100755 --- a/scripts/docker/setup-base-ubuntu-22.04-env.sh +++ b/scripts/docker/setup-base-ubuntu-22.04-env.sh @@ -149,13 +149,13 @@ git clone --depth 1 -b ${ROCKSDB_VERSION} https://github.com/facebook/rocksdb cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Debug -DROCKSDB_BUILD_SHARED=OFF -DWITH_TESTS=OFF -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DUSE_RTTI=ON .. && make -j $(nproc) && make install) # Installing Kafka -DEBIAN_FRONTEND=noninteractive apt install -y default-jre +DEBIAN_FRONTEND=noninteractive apt install -y openjdk-17-jdk -curl https://dlcdn.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz -o kafka.tgz +curl https://dlcdn.apache.org/kafka/4.0.0/kafka_2.13-4.0.0.tgz -o kafka.tgz mkdir -p /etc/kafka tar xf kafka.tgz --directory=/etc/kafka -cp -r /etc/kafka/kafka_2.13-3.8.0/* /etc/kafka/ -rm -rf /etc/kafka/kafka_2.13-3.8.0 +cp -r /etc/kafka/kafka_2.13-4.0.0/* /etc/kafka/ +rm -rf /etc/kafka/kafka_2.13-4.0.0 # Set UTC timezone TZ=Etc/UTC diff --git a/scripts/kafka/install_kafka.sh b/scripts/kafka/install_kafka.sh deleted file mode 100755 index d2c6dc7b329a..000000000000 --- a/scripts/kafka/install_kafka.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -if [ "$(uname)" = "Darwin" ]; then - brew install openjdk -else - apt install -y default-jre -fi - -if [ -z ${KAFKA_PATH+x} ]; then - if [ "$(uname)" = "Darwin" ]; then - export KAFKA_PATH="/opt/homebrew/opt/kafka/libexec" - else - export KAFKA_PATH="/etc/kafka" - fi -fi - -curl https://dlcdn.apache.org/kafka/4.0.0/kafka_2.13-4.0.0.tgz -o kafka.tgz -mkdir -p ${KAFKA_PATH} -tar xf kafka.tgz --directory="${KAFKA_PATH}" -cp -r ${KAFKA_PATH}/kafka_2.13-4.0.0/* ${KAFKA_PATH} -rm -rf ${KAFKA_PATH}/kafka_2.13-4.0.0 -rm kafka.tgz diff --git a/scripts/kafka/ubuntu_install_kafka.sh b/scripts/kafka/ubuntu_install_kafka.sh new file mode 100755 index 000000000000..16067bcf9ce4 --- /dev/null +++ b/scripts/kafka/ubuntu_install_kafka.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +sudo apt install -y openjdk-17-jdk + +curl https://dlcdn.apache.org/kafka/4.0.0/kafka_2.13-4.0.0.tgz -o kafka.tgz +mkdir -p /etc/kafka +tar xf kafka.tgz --directory=/etc/kafka +cp -r /etc/kafka/kafka_2.13-4.0.0/* /etc/kafka/ +rm -rf /etc/kafka/kafka_2.13-4.0.0 +rm kafka.tgz From cbd203a02dc689925527807bd9f18357dc4e8895 Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Wed, 30 Jul 2025 20:58:06 +0300 Subject: [PATCH 030/151] fix grpc: remove unused setting from config schema commit_hash:e0744f8c902f0df655008f34aef363e48068ece0 --- grpc/src/ugrpc/client/client_factory_component.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/grpc/src/ugrpc/client/client_factory_component.cpp b/grpc/src/ugrpc/client/client_factory_component.cpp index 0e2a5ae1d9f6..6521eecbde7f 100644 --- a/grpc/src/ugrpc/client/client_factory_component.cpp +++ b/grpc/src/ugrpc/client/client_factory_component.cpp @@ -95,9 +95,6 @@ additionalProperties: false type: string description: The path to file containing the PEM encoding of the client's certificate chain defaultDescription: absent - auth-token: - type: string - description: auth token name from secdist default-service-config: type: string description: | From 5b37af67f3e120e4406483ebf6e261228a8e7247 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Thu, 31 Jul 2025 12:24:39 +0300 Subject: [PATCH 031/151] feat docs: update docs to answer recent user questions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/userver-framework/userver/issues/975 Tests: протестировано CI commit_hash:9f6566f20868c572c023efff147bbf7e35f09d50 --- README.md | 10 ++++++---- .../metrics/tests/static/metrics_values.txt | 13 +++++++++++++ .../userver/storages/mongo/component.hpp | 6 ++++++ .../userver/storages/postgres/component.hpp | 4 ++++ postgresql/src/storages/postgres/component.cpp | 3 ++- .../userver/storages/redis/component.hpp | 3 +++ .../service-template/userver-create-service.sh | 18 ++++++++---------- 7 files changed, 42 insertions(+), 15 deletions(-) mode change 100644 => 100755 scripts/docs/examples/service-template/userver-create-service.sh diff --git a/README.md b/README.md index 4ec58b702f8e..f24126ff9686 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,18 @@ [![uservice-dynconf Docker build](https://github.com/userver-framework/uservice-dynconf/actions/workflows/docker.yaml/badge.svg?branch=develop)](https://github.com/userver-framework/uservice-dynconf/actions/workflows/docker.yaml) [![urealmedium CI](https://github.com/userver-framework/realmedium_sample/actions/workflows/ci.yml/badge.svg?branch=develop)](https://github.com/userver-framework/realmedium_sample/actions/workflows/ci.yml) [![urealmedium Docker build](https://github.com/userver-framework/realmedium_sample/actions/workflows/docker.yaml/badge.svg?branch=develop)](https://github.com/userver-framework/realmedium_sample/actions/workflows/docker.yaml) +[![upastebin CI](https://github.com/userver-framework/upastebin/actions/workflows/ci.yml/badge.svg?branch=develop)](https://github.com/userver-framework/upastebin/actions/workflows/ci.yml) +[![upastebin Docker](https://github.com/userver-framework/upastebin/actions/workflows/docker.yaml/badge.svg?branch=develop)](https://github.com/userver-framework/upastebin/actions/workflows/docker.ymal) **userver** is an open source asynchronous framework with a rich set of abstractions for fast and comfortable creation of C++ microservices, services and utilities. The framework solves the problem of efficient I/O interactions transparently for -the developers. Operations that would typically suspend the thread of +the developers. Operations that would typically suspend the thread of execution do not suspend it. Instead of that, the thread processes other requests and tasks and returns to the handling of the operation only when it is -guaranteed to execute immediately: +guaranteed to execute immediately: ```cpp #include @@ -59,7 +61,7 @@ avoid CPU-consuming context switches from OS, efficiently utilize the CPU with a small amount of execution threads. -You can learn more about history and key features of userver from our +You can learn more about history and key features of userver from our [publications and videos](https://userver.tech/dc/d30/md_en_2userver_2publications.html). ## Other Features @@ -74,7 +76,7 @@ You can learn more about history and key features of userver from our * On-the-fly configurable drivers, options of the deadline propagation, timeouts, congestion-control. * Comprehensive set of asynchronous low-level synchronization primitives and - OS abstractions. + OS abstractions. [See the docs for more info](https://userver.tech/de/d6a/md_en_2index.html). diff --git a/core/functional_tests/metrics/tests/static/metrics_values.txt b/core/functional_tests/metrics/tests/static/metrics_values.txt index d8915ee2c35f..a981f2f2c402 100644 --- a/core/functional_tests/metrics/tests/static/metrics_values.txt +++ b/core/functional_tests/metrics/tests/static/metrics_values.txt @@ -1,7 +1,10 @@ +# Engine related errors/alerts alerts.cache_update_error: GAUGE 0 alerts.config_parse_error: GAUGE 0 alerts.dynamic_debug_invalid_location: GAUGE 0 alerts.log_reopening_error: GAUGE 0 + +# Info on cahces cache.any.documents.parse_failures.v2: cache_name=dynamic-config-client-updater RATE 0 cache.any.documents.parse_failures.v2: cache_name=sample-cache RATE 0 cache.any.documents.parse_failures: cache_name=dynamic-config-client-updater GAUGE 0 @@ -94,6 +97,7 @@ cache.misses.v2: cache_name=sample-lru-cache RATE 0 cache.misses: cache_name=sample-lru-cache GAUGE 0 cache.stale.v2: cache_name=sample-lru-cache RATE 0 cache.stale: cache_name=sample-lru-cache GAUGE 0 + congestion-control.rps.is-custom-status-activated: GAUGE 0 cpu_time_sec: GAUGE 0 dns-client.replies: dns_reply_source=cached RATE 0 @@ -183,7 +187,11 @@ engine.task-processors.tasks.running: task_processor=monitor-task-processor GAUG engine.task-processors.worker-threads: task_processor=fs-task-processor GAUGE 0 engine.task-processors.worker-threads: task_processor=main-task-processor GAUGE 0 engine.task-processors.worker-threads: task_processor=monitor-task-processor GAUGE 0 + +# How long the engine is running. It does not account components state (i.e. `components::State::IsAnyComponentInFatalState()`), +# the engine could be fine, the components could be in components::ComponentHealth::kFatal state. engine.uptime-seconds: GAUGE 0 + http.by-fallback.implicit-http-options.handler.cancelled-by-deadline: http_handler=handler-implicit-http-options, version=2 RATE 0 http.by-fallback.implicit-http-options.handler.deadline-received: http_handler=handler-implicit-http-options, version=2 RATE 0 http.by-fallback.implicit-http-options.handler.in-flight: http_handler=handler-implicit-http-options, version=2 GAUGE 0 @@ -367,6 +375,8 @@ http.handler.too-many-requests-in-flight: http_handler=handler-on-log-rotate, ht http.handler.too-many-requests-in-flight: http_handler=handler-ping, http_path=/ping, version=2 RATE 0 http.handler.too-many-requests-in-flight: http_handler=handler-server-monitor, http_path=/service/monitor, version=2 RATE 0 http.handler.too-many-requests-in-flight: http_handler=tests-control, http_path=/tests/_action_, version=2 RATE 0 + +# HTTP server metrics aggregated by all the handlers http.handler.total.cancelled-by-deadline: version=2 RATE 0 http.handler.total.deadline-received: version=2 RATE 0 http.handler.total.in-flight: version=2 GAUGE 0 @@ -386,6 +396,7 @@ http.handler.total.timings: percentile=p99, version=2 GAUGE 0 http.handler.total.timings: percentile=p99_6, version=2 GAUGE 0 http.handler.total.timings: percentile=p99_9, version=2 GAUGE 0 http.handler.total.too-many-requests-in-flight: version=2 RATE 0 + httpclient.cancelled-by-deadline: http_destination=http://localhost:00000/configs-service/configs/values, version=2 RATE 0 httpclient.cancelled-by-deadline: version=2 RATE 0 httpclient.errors: http_destination=http://localhost:00000/configs-service/configs/values, http_error=cancelled, version=2 RATE 0 @@ -478,6 +489,8 @@ logger.total: logger=default RATE 0 major_pagefaults: GAUGE 0 open_files: GAUGE 0 rss_kb: GAUGE 0 + +# HTTP server related metrics (no gRPC metrics!) server.connections.active: GAUGE 0 server.connections.closed.v2: RATE 0 server.connections.closed: GAUGE 0 diff --git a/mongo/include/userver/storages/mongo/component.hpp b/mongo/include/userver/storages/mongo/component.hpp index d4572168180a..2ae30f82bdbe 100644 --- a/mongo/include/userver/storages/mongo/component.hpp +++ b/mongo/include/userver/storages/mongo/component.hpp @@ -77,6 +77,9 @@ namespace components { /// `dbconnection#env: THE_ENV_VARIABLE_WITH_CONNECTION_STRING` as described /// in yaml_config::YamlConfig. /// +/// Note that if the `dbalias` option is provided and the components::Secdist component has `update-period` other +/// than 0, then new connections are created or gracefully closed as the secdist configuration change to new value. +/// /// ## Secdist format /// /// If a `dbalias` option is provided, for example @@ -173,6 +176,9 @@ inline constexpr bool kHasValidate = true; /// ----- | ----------- /// terse | Default value, report only cumulative stats and read/write totals /// full | Separate metrics for each operation, divided by read preference or write concern +/// +/// Note that if the components::Secdist component has `update-period` other +/// than 0, then new connections are created or gracefully closed as the secdist configuration change to new value. // clang-format on diff --git a/postgresql/include/userver/storages/postgres/component.hpp b/postgresql/include/userver/storages/postgres/component.hpp index 772b67f7e004..458b355ac2ec 100644 --- a/postgresql/include/userver/storages/postgres/component.hpp +++ b/postgresql/include/userver/storages/postgres/component.hpp @@ -73,6 +73,9 @@ namespace components { /// is just a list of DSNs and the Postgres component takes care of discovering /// the cluster's topology itself. /// +/// Note that if the components::Secdist component has `update-period` other +/// than 0, then new connections are created or gracefully closed as the secdist configuration change to new value. +/// /// ### Predefined roles /// /// In predefined roles format the component requires single-host connection @@ -118,6 +121,7 @@ namespace components { /// Name | Description | Default value /// ----------------------- | ----------------------------------------------------------------------------- | ------------- /// dbalias | name of the database in secdist config (if available) | -- +/// name_alias | name alias to use in dynamic configs | name of the component /// dbconnection | connection DSN string (used if no dbalias specified) | -- /// blocking_task_processor | name of task processor for background blocking operations | engine::current_task::GetBlockingTaskProcessor() /// max_replication_lag | replication lag limit for usable slaves | 60s diff --git a/postgresql/src/storages/postgres/component.cpp b/postgresql/src/storages/postgres/component.cpp index 67bf61744580..66339f64e39c 100644 --- a/postgresql/src/storages/postgres/component.cpp +++ b/postgresql/src/storages/postgres/component.cpp @@ -279,7 +279,8 @@ additionalProperties: false description: name of the database in secdist config (if available) name_alias: type: string - description: name alias to use in configs (by default - component name) + description: name alias to use in dynamic configs + defaultDescription: name of the component dbconnection: type: string description: connection DSN string (used if no dbalias specified) diff --git a/redis/include/userver/storages/redis/component.hpp b/redis/include/userver/storages/redis/component.hpp index 59e23d20fd8b..c50e4a40ec74 100644 --- a/redis/include/userver/storages/redis/component.hpp +++ b/redis/include/userver/storages/redis/component.hpp @@ -115,6 +115,9 @@ namespace components { /// /// @snippet redis/functional_tests/integration_tests/tests/conftest.py Sample pytest redis configuration /// +/// Note that if the components::Secdist component has `update-period` other +/// than 0, then new connections are created or gracefully closed as the secdist configuration change to new value. +/// /// ## Cluster Valkey or Cluster Redis setup /// /// Valkey/Redis cluster is the new recommended way of setting up key-value datastores with improved stability. diff --git a/scripts/docs/examples/service-template/userver-create-service.sh b/scripts/docs/examples/service-template/userver-create-service.sh old mode 100644 new mode 100755 index 532cef7b4874..552f21639eca --- a/scripts/docs/examples/service-template/userver-create-service.sh +++ b/scripts/docs/examples/service-template/userver-create-service.sh @@ -1,10 +1,8 @@ -userver-create-service() { - REPO_URL="https://github.com/userver-framework/userver.git" - BRANCH="develop" - WORKDIR="/tmp/userver-create-service" - if [ ! -d "$WORKDIR" ]; then - mkdir -p "$WORKDIR" - git clone -q --depth 1 --branch "$REPO_URL" "$BRANCH" "$WORKDIR" - fi - "$WORKDIR/scripts/userver-create-service" "$@" -} +REPO_URL="https://github.com/userver-framework/userver.git" +BRANCH="develop" +WORKDIR="/tmp/userver-create-service" +if [ ! -d "$WORKDIR" ]; then + mkdir -p "$WORKDIR" + git clone -q --depth 1 --branch "$BRANCH" "$REPO_URL" "$WORKDIR" +fi +"$WORKDIR/scripts/userver-create-service" "$@" From b889acb1bea04ef9faa2a7dca3a3f33bdfb7f38f Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Thu, 31 Jul 2025 13:01:41 +0300 Subject: [PATCH 032/151] cc grpc: reorder ClientFactoryComponent config commit_hash:8758ed285614ed0b61732e65fabcdb4c1107dd3c --- .../ugrpc/client/client_factory_component.hpp | 2 +- .../ugrpc/client/client_factory_component.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/grpc/include/userver/ugrpc/client/client_factory_component.hpp b/grpc/include/userver/ugrpc/client/client_factory_component.hpp index 5af0145db360..3329c19bbeaa 100644 --- a/grpc/include/userver/ugrpc/client/client_factory_component.hpp +++ b/grpc/include/userver/ugrpc/client/client_factory_component.hpp @@ -51,9 +51,9 @@ using MiddlewareRunnerComponentBase = USERVER_NAMESPACE::middlewares::RunnerComp /// /// Name | Description | Default value /// ---- | ----------- | ------------- -/// channel-args | a map of channel arguments, see gRPC Core docs | {} /// auth-type | authentication method, see @ref grpc_ssl_authentication "Authentication" | - /// ssl-credentials-options | TLS/SSL options, see @ref grpc_ssl_authentication "Authentication" | - +/// channel-args | a map of channel arguments, see gRPC Core docs | {} /// default-service-config | default service config, see above | - /// channel-count | Number of underlying grpc::Channel objects | 1 /// middlewares | middlewares names to use | - diff --git a/grpc/src/ugrpc/client/client_factory_component.cpp b/grpc/src/ugrpc/client/client_factory_component.cpp index 6521eecbde7f..ce863439f058 100644 --- a/grpc/src/ugrpc/client/client_factory_component.cpp +++ b/grpc/src/ugrpc/client/client_factory_component.cpp @@ -62,14 +62,6 @@ type: object description: Provides a ClientFactory in the component system additionalProperties: false properties: - channel-args: - type: object - description: a map of channel arguments, see gRPC Core docs - defaultDescription: '{}' - additionalProperties: - type: string - description: value of channel argument, must be string or integer - properties: {} auth-type: type: string description: an optional authentication method @@ -95,6 +87,14 @@ additionalProperties: false type: string description: The path to file containing the PEM encoding of the client's certificate chain defaultDescription: absent + channel-args: + type: object + description: a map of channel arguments, see gRPC Core docs + defaultDescription: '{}' + additionalProperties: + type: string + description: value of channel argument, must be string or integer + properties: {} default-service-config: type: string description: | From 561fbbebd10ebff223413d672c433d9afae99264 Mon Sep 17 00:00:00 2001 From: aselutin Date: Thu, 31 Jul 2025 14:56:10 +0300 Subject: [PATCH 033/151] feat integration-tests: remove integration tests launch commit_hash:614b1a3e1ac265aa548e5dc3ed85c1985c5c61ab --- .mapping.json | 11 -- .../integration_tests/config_vars.yaml | 18 --- .../schemas/postgresql/key_value.sql | 5 - .../integration_tests/secure_data.json | 1 - .../integration_tests/service.cpp | 138 ------------------ .../integration_tests/static_config.yaml | 81 ---------- .../integration_tests/tests/conftest.py | 61 -------- .../integration_tests/tests/test_failover.py | 30 ---- .../tests/test_ntrx_failure.py | 17 --- .../tests/test_statement_log_mode.py | 22 --- .../tests/test_trx_failure.py | 23 --- .../integration_tests/tests/test_ttl.py | 20 --- 12 files changed, 427 deletions(-) delete mode 100644 postgresql/functional_tests/integration_tests/config_vars.yaml delete mode 100644 postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql delete mode 100644 postgresql/functional_tests/integration_tests/secure_data.json delete mode 100644 postgresql/functional_tests/integration_tests/service.cpp delete mode 100644 postgresql/functional_tests/integration_tests/static_config.yaml delete mode 100644 postgresql/functional_tests/integration_tests/tests/conftest.py delete mode 100644 postgresql/functional_tests/integration_tests/tests/test_failover.py delete mode 100644 postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py delete mode 100644 postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py delete mode 100644 postgresql/functional_tests/integration_tests/tests/test_trx_failure.py delete mode 100644 postgresql/functional_tests/integration_tests/tests/test_ttl.py diff --git a/.mapping.json b/.mapping.json index 636ebc6b605a..704c7830ce5d 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2908,17 +2908,6 @@ "postgresql/functional_tests/connlimit_max/schemas/postgresql/key_value.sql":"taxi/uservices/userver/postgresql/functional_tests/connlimit_max/schemas/postgresql/key_value.sql", "postgresql/functional_tests/connlimit_max/static_config.yaml":"taxi/uservices/userver/postgresql/functional_tests/connlimit_max/static_config.yaml", "postgresql/functional_tests/connlimit_max/tests/test_connlimit.py":"taxi/uservices/userver/postgresql/functional_tests/connlimit_max/tests/test_connlimit.py", - "postgresql/functional_tests/integration_tests/config_vars.yaml":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/config_vars.yaml", - "postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql", - "postgresql/functional_tests/integration_tests/secure_data.json":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/secure_data.json", - "postgresql/functional_tests/integration_tests/service.cpp":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/service.cpp", - "postgresql/functional_tests/integration_tests/static_config.yaml":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/static_config.yaml", - "postgresql/functional_tests/integration_tests/tests/conftest.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/conftest.py", - "postgresql/functional_tests/integration_tests/tests/test_failover.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_failover.py", - "postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py", - "postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py", - "postgresql/functional_tests/integration_tests/tests/test_trx_failure.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_trx_failure.py", - "postgresql/functional_tests/integration_tests/tests/test_ttl.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_ttl.py", "postgresql/functional_tests/metrics/CMakeLists.txt":"taxi/uservices/userver/postgresql/functional_tests/metrics/CMakeLists.txt", "postgresql/functional_tests/metrics/config_vars.yaml":"taxi/uservices/userver/postgresql/functional_tests/metrics/config_vars.yaml", "postgresql/functional_tests/metrics/schemas/postgresql/key_value.sql":"taxi/uservices/userver/postgresql/functional_tests/metrics/schemas/postgresql/key_value.sql", diff --git a/postgresql/functional_tests/integration_tests/config_vars.yaml b/postgresql/functional_tests/integration_tests/config_vars.yaml deleted file mode 100644 index 57a86a6d3210..000000000000 --- a/postgresql/functional_tests/integration_tests/config_vars.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# yaml -server-name: test-core-metrics 1.0 -service-name: test_core_metrics -logger-level: info - -config-server-url: http://localhost:8083/ -server-port: 8185 -monitor-server-port: 8186 - -testsuite-enabled: false - -userver-dumps-root: /var/cache/test_core_metrics/userver-dumps/ -access-log-path: /var/log/test_core_metrics/access.log -access-tskv-log-path: /var/log/test_core_metrics/access_tskv.log -default-log-path: /var/log/test_core_metrics/server.log -secdist-path: /etc/test_core_metrics/secure_data.json - -config-cache: /var/cache/test_core_metrics/config_cache.json diff --git a/postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql b/postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql deleted file mode 100644 index 42812ff0b9bc..000000000000 --- a/postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE IF NOT EXISTS key_value_table ( - key VARCHAR PRIMARY KEY, - value VARCHAR, - updated TIMESTAMPTZ NOT NULL DEFAULT NOW() -) diff --git a/postgresql/functional_tests/integration_tests/secure_data.json b/postgresql/functional_tests/integration_tests/secure_data.json deleted file mode 100644 index 0967ef424bce..000000000000 --- a/postgresql/functional_tests/integration_tests/secure_data.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/postgresql/functional_tests/integration_tests/service.cpp b/postgresql/functional_tests/integration_tests/service.cpp deleted file mode 100644 index ba28b6491b21..000000000000 --- a/postgresql/functional_tests/integration_tests/service.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace chaos { - -class KeyValue final : public server::handlers::HttpHandlerBase { -public: - static constexpr std::string_view kName = "handler-key-value"; - - KeyValue(const components::ComponentConfig& config, const components::ComponentContext& context); - - std::string HandleRequestThrow(const server::http::HttpRequest& request, server::request::RequestContext&) - const override; - -private: - std::string GetValue(std::string_view key, const server::http::HttpRequest& request) const; - std::string PostValue(std::string_view key, const server::http::HttpRequest& request) const; - std::string DeleteValue(std::string_view key) const; - - storages::postgres::ClusterPtr pg_cluster_; -}; - -KeyValue::KeyValue(const components::ComponentConfig& config, const components::ComponentContext& context) - : HttpHandlerBase(config, context), - pg_cluster_(context.FindComponent("key-value-database").GetCluster()) { - constexpr auto kCreateTable = R"~( - CREATE TABLE IF NOT EXISTS key_value_table ( - key VARCHAR PRIMARY KEY, - value VARCHAR - ) - )~"; - - using storages::postgres::ClusterHostType; - pg_cluster_->Execute(ClusterHostType::kMaster, kCreateTable); -} - -std::string KeyValue::HandleRequestThrow(const server::http::HttpRequest& request, server::request::RequestContext&) - const { - const auto& key = request.GetArg("key"); - if (key.empty()) { - throw server::handlers::ClientError(server::handlers::ExternalBody{"No 'key' query argument"}); - } - - switch (request.GetMethod()) { - case server::http::HttpMethod::kGet: - return GetValue(key, request); - case server::http::HttpMethod::kPost: - return PostValue(key, request); - case server::http::HttpMethod::kDelete: - return DeleteValue(key); - default: - throw server::handlers::ClientError(server::handlers::ExternalBody{ - fmt::format("Unsupported method {}", request.GetMethod())}); - } -} - -const storages::postgres::Query kSelectValue{ - "SELECT value FROM key_value_table WHERE key=$1", - storages::postgres::Query::Name{"sample_select_value"}, -}; - -std::string KeyValue::GetValue(std::string_view key, const server::http::HttpRequest& request) const { - const storages::postgres::ResultSet res = - pg_cluster_->Execute(storages::postgres::ClusterHostType::kSlave, kSelectValue, key); - if (res.IsEmpty()) { - request.SetResponseStatus(server::http::HttpStatus::kNotFound); - return {}; - } - - return res.AsSingleRow(); -} - -const storages::postgres::Query kInsertValue{ - "INSERT INTO key_value_table (key, value) " - "VALUES ($1, $2) " - "ON CONFLICT DO NOTHING", - storages::postgres::Query::Name{"sample_insert_value"}, -}; - -std::string KeyValue::PostValue(std::string_view key, const server::http::HttpRequest& request) const { - const auto& value = request.GetArg("value"); - - storages::postgres::Transaction transaction = - pg_cluster_->Begin("sample_transaction_insert_key_value", storages::postgres::ClusterHostType::kMaster, {}); - - auto res = transaction.Execute(kInsertValue, key, value); - if (res.RowsAffected()) { - transaction.Commit(); - request.SetResponseStatus(server::http::HttpStatus::kCreated); - return std::string{value}; - } - - res = transaction.Execute(kSelectValue, key); - transaction.Rollback(); - - auto result = res.AsSingleRow(); - if (result != value) { - request.SetResponseStatus(server::http::HttpStatus::kConflict); - } - - return res.AsSingleRow(); -} - -std::string KeyValue::DeleteValue(std::string_view key) const { - auto res = pg_cluster_->Execute( - storages::postgres::ClusterHostType::kMaster, "DELETE FROM key_value_table WHERE key=$1", key - ); - return std::to_string(res.RowsAffected()); -} - -} // namespace chaos - -int main(int argc, char* argv[]) { - const auto component_list = components::MinimalServerComponentList() - .Append() - .Append() - .Append() - .Append("key-value-database") - .Append() - .Append() - .Append(); - return utils::DaemonMain(argc, argv, component_list); -} diff --git a/postgresql/functional_tests/integration_tests/static_config.yaml b/postgresql/functional_tests/integration_tests/static_config.yaml deleted file mode 100644 index 3313b3291164..000000000000 --- a/postgresql/functional_tests/integration_tests/static_config.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# yaml -components_manager: - components: - handler-key-value: - path: /v1/key-value - task_processor: main-task-processor - method: GET,DELETE,POST - - key-value-database: - dbconnection: 'postgresql://testsuite@localhost:15433/pg_key_value' - blocking_task_processor: fs-task-processor - dns_resolver: async - - testsuite-support: - - http-client: - fs-task-processor: main-task-processor - - tests-control: - method: POST - path: /tests/{action} - skip-unregistered-testpoints: true - task_processor: main-task-processor - testpoint-timeout: 10s - testpoint-url: $mockserver/testpoint - throttling_enabled: false - - server: - listener: - port: 8187 - task_processor: main-task-processor - listener-monitor: - port: $monitor-server-port - port#fallback: 8086 - connection: - in_buffer_size: 32768 - requests_queue_size_threshold: 100 - task_processor: main-task-processor - logging: - fs-task-processor: fs-task-processor - loggers: - default: - file_path: '@stderr' - level: debug - overflow_behavior: discard - - handler-server-monitor: - path: /service/monitor - method: GET - task_processor: main-task-processor - - # /// [dynamic config] - # yaml - dynamic-config: - defaults: - POSTGRES_CONNECTION_SETTINGS: - key-value-database: - max-ttl-sec: 60 - recent-errors-threshold: 100000 - POSTGRES_DEFAULT_COMMAND_CONTROL: - network_timeout_ms: 750 - statement_timeout_ms: 500 - POSTGRES_QUERIES_COMMAND_CONTROL: - sample_select_value: - network_timeout_ms: 70 - statement_timeout_ms: 40 - sample_transaction_insert_key_value: - network_timeout_ms: 200 - statement_timeout_ms: 150 - # /// [dynamic config] - - dns-client: - fs-task-processor: fs-task-processor - - task_processors: - main-task-processor: - worker_threads: 4 - fs-task-processor: - worker_threads: 4 - - default_task_processor: main-task-processor diff --git a/postgresql/functional_tests/integration_tests/tests/conftest.py b/postgresql/functional_tests/integration_tests/tests/conftest.py deleted file mode 100644 index 614a3cd7a8b9..000000000000 --- a/postgresql/functional_tests/integration_tests/tests/conftest.py +++ /dev/null @@ -1,61 +0,0 @@ -import pathlib - -import pytest -import pytest_asyncio - -import taxi.integration_testing as it - -pytest_plugins = [ - 'testsuite.pytest_plugin', - 'pytest_userver.plugins.caches', - 'pytest_userver.plugins.config', - 'pytest_userver.plugins.dumps', - 'pytest_userver.plugins.dynamic_config', - 'pytest_userver.plugins.log_capture', - 'pytest_userver.plugins.service', - 'pytest_userver.plugins.service_client', - 'pytest_userver.plugins.testpoint', - 'pytest_userver.plugins.postgresql', -] - - -USERVER_CONFIG_HOOKS = ['userver_pg_config'] - - -@pytest_asyncio.fixture(name='testenv', scope='session') -async def _testenv( - service_source_dir: pathlib.Path, - request: pytest.FixtureRequest, -) -> it.Environment: - env = it.Environment({ - **it.CORE, - **it.DOCKER, - 'database_common': it.databases.DatabaseCommon(), - **it.mockserver.create_mockserver(), - **it.databases.pgsql.create_pgsql(replicas=1), - }) - env.configure(it.PytestConfig(request.config)) - env.pgsql.discover_schemas( - service_source_dir.joinpath('schemas/postgresql'), - service_name='pg', - ) - async with env.run(): - import time - - time.sleep(60) - yield env - - -@pytest_asyncio.fixture(scope='session') -async def userver_pg_config(testenv: it.Environment): - settings = await testenv.pgsql_primary_container.get_conn_info() - settings = settings.replace(host='127.0.0.1') - - def _hook_db_config(config_yaml, config_vars): - components = config_yaml['components_manager']['components'] - db = components['key-value-database'] - db['dbconnection'] = ( - f'postgresql://{settings.user}:{settings.password}@{settings.host}:{settings.port}/pg_key_value' - ) - - return _hook_db_config diff --git a/postgresql/functional_tests/integration_tests/tests/test_failover.py b/postgresql/functional_tests/integration_tests/tests/test_failover.py deleted file mode 100644 index b2123383ec61..000000000000 --- a/postgresql/functional_tests/integration_tests/tests/test_failover.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio - -import pytest - -import taxi.integration_testing as it - -FAILOVER_DEADLINE_SEC = 30 # maximum time allowed to finish failover - - -@pytest.mark.skip(reason='Flacky fix in TAXICOMMON-6756') -async def test_hard_failover(service_client, testenv: it.Environment): - response = await service_client.post('/v1/key-value?key=foo&value=bar') - assert response.status == 201 - assert response.content == b'bar' - - response = await service_client.get('/v1/key-value?key=foo') - assert response.status == 200 - assert response.content == b'bar' - - await asyncio.sleep(30) - testenv.pgsql_primary_container.kill() - - for _ in range(FAILOVER_DEADLINE_SEC): - response = await service_client.get('/v1/key-value?key=foo') - if response.status == 500: - await asyncio.sleep(1) - continue - break - - assert response.status == 200 diff --git a/postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py b/postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py deleted file mode 100644 index a312f6a125e5..000000000000 --- a/postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py +++ /dev/null @@ -1,17 +0,0 @@ -# /// [fault injection] -async def test_ntrx_fail(service_client, pgsql, userver_pg_ntrx): - response = await service_client.delete('/v1/key-value?key=foo') - assert response.status == 200 - - response = await service_client.post('/v1/key-value?key=foo&value=bar') - assert response.status == 201 - - with userver_pg_ntrx.mock_failure('sample_select_value'): - response = await service_client.get('/v1/key-value?key=foo') - assert response.status == 500 - - response = await service_client.get('/v1/key-value?key=foo') - assert response.status == 200 - assert response.content == b'bar' - - # /// [fault injection] diff --git a/postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py b/postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py deleted file mode 100644 index 82c764778bc4..000000000000 --- a/postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest - - -@pytest.fixture(scope='session') -def userver_pg_log_config(): - def patch_config(config_yaml, _config_vars) -> None: - components = config_yaml['components_manager']['components'] - db = components['key-value-database'] - db['statement-log-mode'] = 'hide' - - return patch_config - - -@pytest.mark.uservice_oneshot(config_hooks=['userver_pg_log_config']) -async def test_make_request(service_client): - async with service_client.capture_logs() as capture: - response = await service_client.post('/v1/key-value?key=foo&value=bar') - assert response.status in (200, 201) - - records = capture.select(stopwatch_name='pg_query') - assert len(records) == 1, capture.select() - assert 'db_statement' not in records[0] diff --git a/postgresql/functional_tests/integration_tests/tests/test_trx_failure.py b/postgresql/functional_tests/integration_tests/tests/test_trx_failure.py deleted file mode 100644 index 2ea2930bdb47..000000000000 --- a/postgresql/functional_tests/integration_tests/tests/test_trx_failure.py +++ /dev/null @@ -1,23 +0,0 @@ -async def test_trx_ok(service_client, pgsql): - response = await service_client.post('/v1/key-value?key=foo&value=bar') - assert response.status == 201 - assert response.content == b'bar' - - response = await service_client.get('/v1/key-value?key=foo') - assert response.status == 200 - assert response.content == b'bar' - - -# /// [fault injection] -async def test_trx_fail(service_client, pgsql, userver_pg_trx): - response = await service_client.delete('/v1/key-value?key=foo') - assert response.status == 200 - - userver_pg_trx.enable_failure('sample_transaction_insert_key_value') - - response = await service_client.post('/v1/key-value?key=foo&value=bar') - assert response.status == 500 - - response = await service_client.get('/v1/key-value?key=foo') - assert response.status == 404 - # /// [fault injection] diff --git a/postgresql/functional_tests/integration_tests/tests/test_ttl.py b/postgresql/functional_tests/integration_tests/tests/test_ttl.py deleted file mode 100644 index 091bb87a8fba..000000000000 --- a/postgresql/functional_tests/integration_tests/tests/test_ttl.py +++ /dev/null @@ -1,20 +0,0 @@ -async def _make_request(service_client): - response = await service_client.post('/v1/key-value?key=foo&value=bar') - assert response.status in (200, 201) - assert response.content == b'bar' - - -async def test_connection_max_ttl(service_client, monitor_client, mocked_time): - await _make_request(service_client) - - async with monitor_client.metrics_diff( - prefix='postgresql.connections', - diff_gauge=True, - ) as diff: - # with max-ttl-sec set to 60 we expect the db connection to be reopened - # on the next request - mocked_time.sleep(61) - await _make_request(service_client) - - assert diff.value_at('closed') == 1 - assert diff.value_at('opened') == 1 From 23f7b74b7e47a5f67f7044fa460ee8be2054e42d Mon Sep 17 00:00:00 2001 From: Fedor Osetrov <33493672+fdr400@users.noreply.github.com> Date: Thu, 31 Jul 2025 15:16:47 +0300 Subject: [PATCH 034/151] fix kafka: link kafka-utest with rdkafka target (for includes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------------------ Note: by creating a PR or an issue you automatically agree to the CLA. See [CONTRIBUTING.md](https://github.com/userver-framework/userver/blob/develop/CONTRIBUTING.md). Feel free to remove this note, the agreement holds. Tests: протестировано CI Pull Request resolved: https://github.com/userver-framework/userver/pull/980 commit_hash:0598904ca821f2a4f4564c1a47115ad2445fac05 --- kafka/utest/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kafka/utest/CMakeLists.txt b/kafka/utest/CMakeLists.txt index d873439ade48..8e9097294687 100644 --- a/kafka/utest/CMakeLists.txt +++ b/kafka/utest/CMakeLists.txt @@ -3,7 +3,10 @@ project(userver-kafka-utest) file(GLOB_RECURSE KAFKA_LIB_UTEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) add_library(${PROJECT_NAME} STATIC ${KAFKA_LIB_UTEST_SOURCES}) -target_link_libraries(${PROJECT_NAME} PUBLIC userver::kafka userver::utest) +target_link_libraries(${PROJECT_NAME} + PUBLIC userver::kafka userver::utest + PRIVATE RdKafka::rdkafka +) target_include_directories(${PROJECT_NAME} PUBLIC $) if(KAFKA_CPM) From 97d3c1108b0fb6449443ae2f1b939962ef21ea80 Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Thu, 31 Jul 2025 18:05:39 +0300 Subject: [PATCH 035/151] refactor grpc: mv tracing.hpp to includes commit_hash:630c4e659f047a4c6173861302936d8d239b7ef4 --- .mapping.json | 2 +- .../userver}/ugrpc/client/impl/tracing.hpp | 0 grpc/src/ugrpc/client/impl/async_methods.cpp | 2 +- grpc/src/ugrpc/client/impl/call_state.cpp | 2 +- grpc/src/ugrpc/client/impl/tracing.cpp | 22 +++++++++---------- 5 files changed, 14 insertions(+), 14 deletions(-) rename grpc/{src => include/userver}/ugrpc/client/impl/tracing.hpp (100%) diff --git a/.mapping.json b/.mapping.json index 704c7830ce5d..c8f2e1530646 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2138,6 +2138,7 @@ "grpc/include/userver/ugrpc/client/impl/stub_any.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/stub_any.hpp", "grpc/include/userver/ugrpc/client/impl/stub_handle.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/stub_handle.hpp", "grpc/include/userver/ugrpc/client/impl/stub_pool.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/stub_pool.hpp", + "grpc/include/userver/ugrpc/client/impl/tracing.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/tracing.hpp", "grpc/include/userver/ugrpc/client/middlewares/baggage/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/middlewares/baggage/component.hpp", "grpc/include/userver/ugrpc/client/middlewares/baggage/middleware.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/middlewares/baggage/middleware.hpp", "grpc/include/userver/ugrpc/client/middlewares/base.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/middlewares/base.hpp", @@ -2262,7 +2263,6 @@ "grpc/src/ugrpc/client/impl/stub_handle.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/stub_handle.cpp", "grpc/src/ugrpc/client/impl/stub_pool.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/stub_pool.cpp", "grpc/src/ugrpc/client/impl/tracing.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/tracing.cpp", - "grpc/src/ugrpc/client/impl/tracing.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/tracing.hpp", "grpc/src/ugrpc/client/middlewares/baggage/middleware.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/middlewares/baggage/middleware.cpp", "grpc/src/ugrpc/client/middlewares/base.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/middlewares/base.cpp", "grpc/src/ugrpc/client/middlewares/deadline_propagation/middleware.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/middlewares/deadline_propagation/middleware.cpp", diff --git a/grpc/src/ugrpc/client/impl/tracing.hpp b/grpc/include/userver/ugrpc/client/impl/tracing.hpp similarity index 100% rename from grpc/src/ugrpc/client/impl/tracing.hpp rename to grpc/include/userver/ugrpc/client/impl/tracing.hpp diff --git a/grpc/src/ugrpc/client/impl/async_methods.cpp b/grpc/src/ugrpc/client/impl/async_methods.cpp index 52dfb743b37b..d4fc019c47bf 100644 --- a/grpc/src/ugrpc/client/impl/async_methods.cpp +++ b/grpc/src/ugrpc/client/impl/async_methods.cpp @@ -2,9 +2,9 @@ #include -#include #include #include +#include USERVER_NAMESPACE_BEGIN diff --git a/grpc/src/ugrpc/client/impl/call_state.cpp b/grpc/src/ugrpc/client/impl/call_state.cpp index fda710a6ea69..ae33153e5fe4 100644 --- a/grpc/src/ugrpc/client/impl/call_state.cpp +++ b/grpc/src/ugrpc/client/impl/call_state.cpp @@ -8,11 +8,11 @@ #include #include +#include #include #include #include -#include #include USERVER_NAMESPACE_BEGIN diff --git a/grpc/src/ugrpc/client/impl/tracing.cpp b/grpc/src/ugrpc/client/impl/tracing.cpp index bf3bb4f72715..4ab9cf8c5738 100644 --- a/grpc/src/ugrpc/client/impl/tracing.cpp +++ b/grpc/src/ugrpc/client/impl/tracing.cpp @@ -1,4 +1,4 @@ -#include +#include #include @@ -8,16 +8,6 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -void SetErrorForSpan(tracing::Span& span, std::string_view error_message) noexcept { - try { - span.SetLogLevel(logging::Level::kWarning); - span.AddTag(tracing::kErrorFlag, true); - span.AddTag(tracing::kErrorMessage, std::string{error_message}); - } catch (const std::exception& ex) { - LOG_LIMITED_ERROR() << "Can not set error for span: " << ex; - } -} - void SetStatusForSpan(tracing::Span& span, const grpc::Status& status) noexcept { try { span.AddTag("grpc_code", ugrpc::ToString(status.error_code())); @@ -29,6 +19,16 @@ void SetStatusForSpan(tracing::Span& span, const grpc::Status& status) noexcept } } +void SetErrorForSpan(tracing::Span& span, std::string_view error_message) noexcept { + try { + span.SetLogLevel(logging::Level::kWarning); + span.AddTag(tracing::kErrorFlag, true); + span.AddTag(tracing::kErrorMessage, std::string{error_message}); + } catch (const std::exception& ex) { + LOG_LIMITED_ERROR() << "Can not set error for span: " << ex; + } +} + } // namespace ugrpc::client::impl USERVER_NAMESPACE_END From 0c4130add05166ba3f47f9fcaaf48b1a8edbf52f Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Fri, 1 Aug 2025 11:16:08 +0300 Subject: [PATCH 036/151] feat grpc: add PerformUnaryCall commit_hash:5b60e26a88fba194479652aa11e7c8f9053f74b5 --- .mapping.json | 2 ++ .../userver/ugrpc/client/generic_client.hpp | 14 +--------- .../userver/ugrpc/client/generic_options.hpp | 28 +++++++++++++++++++ .../userver/ugrpc/client/impl/call_params.hpp | 2 +- .../ugrpc/client/impl/codegen_definitions.hpp | 1 + .../ugrpc/client/impl/perform_unary_call.hpp | 23 +++++++++++++++ grpc/src/ugrpc/client/generic_client.cpp | 8 +++++- scripts/grpc/templates/client.usrv.cpp.jinja | 10 ++++++- 8 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 grpc/include/userver/ugrpc/client/generic_options.hpp create mode 100644 grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp diff --git a/.mapping.json b/.mapping.json index c8f2e1530646..61056344ec83 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2116,6 +2116,7 @@ "grpc/include/userver/ugrpc/client/exceptions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/exceptions.hpp", "grpc/include/userver/ugrpc/client/fwd.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/fwd.hpp", "grpc/include/userver/ugrpc/client/generic_client.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/generic_client.hpp", + "grpc/include/userver/ugrpc/client/generic_options.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/generic_options.hpp", "grpc/include/userver/ugrpc/client/graceful_stream_finish.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/graceful_stream_finish.hpp", "grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp", "grpc/include/userver/ugrpc/client/impl/async_methods.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_methods.hpp", @@ -2133,6 +2134,7 @@ "grpc/include/userver/ugrpc/client/impl/graceful_stream_finish.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/graceful_stream_finish.hpp", "grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/middleware_hooks.hpp", "grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/middleware_pipeline.hpp", + "grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp", "grpc/include/userver/ugrpc/client/impl/prepare_call.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/prepare_call.hpp", "grpc/include/userver/ugrpc/client/impl/rpc.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/rpc.hpp", "grpc/include/userver/ugrpc/client/impl/stub_any.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/stub_any.hpp", diff --git a/grpc/include/userver/ugrpc/client/generic_client.hpp b/grpc/include/userver/ugrpc/client/generic_client.hpp index 4936ae87f913..7db93ed8dfe2 100644 --- a/grpc/include/userver/ugrpc/client/generic_client.hpp +++ b/grpc/include/userver/ugrpc/client/generic_client.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -17,19 +18,6 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::client { -struct GenericOptions { - /// If non-`nullopt`, metrics are accounted for specified fake call name. - /// If `nullopt`, writes a set of metrics per real call name. - /// If the microservice serves as a proxy and has untrusted clients, it is - /// a good idea to have this option set to non-`nullopt` to avoid - /// the situations where an upstream client can spam various RPCs with - /// non-existent names, which leads to this microservice spamming RPCs - /// with non-existent names, which leads to creating storage for infinite - /// metrics and causes OOM. - /// The default is to specify `"Generic/Generic"` fake call name. - std::optional metrics_call_name{"Generic/Generic"}; -}; - /// @ingroup userver_clients /// /// @brief Allows to talk to gRPC services (generic and normal) using dynamic diff --git a/grpc/include/userver/ugrpc/client/generic_options.hpp b/grpc/include/userver/ugrpc/client/generic_options.hpp new file mode 100644 index 000000000000..8a81419725cc --- /dev/null +++ b/grpc/include/userver/ugrpc/client/generic_options.hpp @@ -0,0 +1,28 @@ +#pragma once + +/// @file userver/ugrpc/client/generic_options.hpp +/// @brief @copybrief ugrpc::client::GenericOptions + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { + +struct GenericOptions { + /// If non-`nullopt`, metrics are accounted for specified fake call name. + /// If `nullopt`, writes a set of metrics per real call name. + /// If the microservice serves as a proxy and has untrusted clients, it is + /// a good idea to have this option set to non-`nullopt` to avoid + /// the situations where an upstream client can spam various RPCs with + /// non-existent names, which leads to this microservice spamming RPCs + /// with non-existent names, which leads to creating storage for infinite + /// metrics and causes OOM. + /// The default is to specify `"Generic/Generic"` fake call name. + std::optional metrics_call_name{"Generic/Generic"}; +}; + +} // namespace ugrpc::client + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/call_params.hpp b/grpc/include/userver/ugrpc/client/impl/call_params.hpp index 7259bf1de9d9..f4755aa19586 100644 --- a/grpc/include/userver/ugrpc/client/impl/call_params.hpp +++ b/grpc/include/userver/ugrpc/client/impl/call_params.hpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include #include diff --git a/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp b/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp index 800978a3d0ab..69ccfd65f2bd 100644 --- a/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp +++ b/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp @@ -10,4 +10,5 @@ #include #include +#include #include diff --git a/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp b/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp new file mode 100644 index 000000000000..a223a7054429 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/perform_unary_call.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +template +Response PerformUnaryCall( + CallParams&& params, + PrepareUnaryCallProxy&& prepare_unary_call, + const Request& request +) { + UnaryCall unary_call{std::move(params), std::move(prepare_unary_call), request}; + return unary_call.Finish(); +} + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/generic_client.cpp b/grpc/src/ugrpc/client/generic_client.cpp index 4822e7d9b227..28c29ce5f4ea 100644 --- a/grpc/src/ugrpc/client/generic_client.cpp +++ b/grpc/src/ugrpc/client/generic_client.cpp @@ -5,6 +5,7 @@ #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -46,7 +47,12 @@ grpc::ByteBuffer GenericClient::UnaryCall( CallOptions call_options, GenericOptions generic_options ) const { - return AsyncUnaryCall(call_name, request, std::move(call_options), std::move(generic_options)).Get(); + auto method_name = utils::StrCat("/", call_name); + return impl::PerformUnaryCall( + impl::CreateGenericCallParams(impl_, call_name, std::move(call_options), std::move(generic_options)), + impl::PrepareUnaryCallProxy(&grpc::GenericStub::PrepareUnaryCall, std::move(method_name)), + request + ); } } // namespace ugrpc::client diff --git a/scripts/grpc/templates/client.usrv.cpp.jinja b/scripts/grpc/templates/client.usrv.cpp.jinja index 8b4bb69441e4..5211cc6ba856 100644 --- a/scripts/grpc/templates/client.usrv.cpp.jinja +++ b/scripts/grpc/templates/client.usrv.cpp.jinja @@ -77,7 +77,15 @@ inline const bool k{{service.name}}TypesRegistration = const {{ method.input_type | grpc_to_cpp_name }}& request, USERVER_NAMESPACE::ugrpc::client::CallOptions call_options ) const { - return Async{{method.name}}(request, std::move(call_options)).Get(); + return USERVER_NAMESPACE::ugrpc::client::impl::PerformUnaryCall( + USERVER_NAMESPACE::ugrpc::client::impl::CreateCallParams( + impl_, {{method_id}}, std::move(call_options) + ), + USERVER_NAMESPACE::ugrpc::client::impl::PrepareUnaryCallProxy( + &{{service.name}}::Stub::PrepareAsync{{method.name}} + ), + request + ); } {{service.name}}Client::{{method.name}}ResponseFuture {{service.name}}Client::Async{{method.name}}( From 1acf3a853bc590219fc95c65066be98cda3daa5c Mon Sep 17 00:00:00 2001 From: fdr400 Date: Fri, 1 Aug 2025 13:30:00 +0300 Subject: [PATCH 037/151] fix universal: link with corrent benchmark target commit_hash:b205f58423318ebd27b560694a39eaa982a8e3e3 --- universal/CMakeLists.txt | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/universal/CMakeLists.txt b/universal/CMakeLists.txt index 480374ccf261..7a6ce58d0a0c 100644 --- a/universal/CMakeLists.txt +++ b/universal/CMakeLists.txt @@ -316,13 +316,38 @@ if(USERVER_FEATURE_UTEST) set(_benchmark_target benchmark::benchmark) endif() - try_compile( - HAVE_MAYBE_REENTER_WITHOUT_ASLR ${CMAKE_CURRENT_BINARY_DIR}/benchmarks/main_check_aslr_disable - SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/main_check_aslr_disable.cpp - LINK_LIBRARIES benchmark::benchmark - ) + if(CMAKE_VERSION GREATER_EQUAL 3.29) + # alias targets are supported in try_compile from cmake 3.29 + try_compile( + HAVE_MAYBE_REENTER_WITHOUT_ASLR ${CMAKE_CURRENT_BINARY_DIR}/benchmarks/main_check_aslr_disable + SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/main_check_aslr_disable.cpp + LINK_LIBRARIES ${_benchmark_target} + ) + else() + if(USERVER_CONAN) + set(_benchmark_original_target benchmark::benchmark_main) + else() + if(TARGET UserverGBench) # userver module + set(_benchmark_original_target UserverGBench) + elseif(TARGET benchmark) # CPM module + set(_benchmark_original_target benchmark) + else() + set(_benchmark_original_target) # to fail compilation + endif() + endif() + + try_compile( + HAVE_MAYBE_REENTER_WITHOUT_ASLR ${CMAKE_CURRENT_BINARY_DIR}/benchmarks/main_check_aslr_disable + SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/main_check_aslr_disable.cpp + LINK_LIBRARIES ${_benchmark_original_target} + ) + endif() + if(NOT HAVE_MAYBE_REENTER_WITHOUT_ASLR) target_compile_definitions(${PROJECT_NAME}-internal-ubench INTERFACE USERVER_IMPL_NO_MAYBE_REENTER_WITHOUT_ASLR_EXISTS) + message(STATUS "Benchmark ASLR turned off") + else() + message(STATUS "Benchmark ASLR turned on") endif() target_link_libraries(${PROJECT_NAME}-internal-ubench INTERFACE ${_benchmark_target} ${PROJECT_NAME}) From 2882de8e82b924d3006c48ed580d4f37854afe71 Mon Sep 17 00:00:00 2001 From: aselutin Date: Fri, 1 Aug 2025 14:00:14 +0300 Subject: [PATCH 038/151] feat userver: revert integration tests revert integration tests commit_hash:095e6569deec6cab5285929b5f65d8654b5fcf20 --- .mapping.json | 10 ++ .../integration_tests/config_vars.yaml | 18 +++ .../schemas/postgresql/key_value.sql | 5 + .../integration_tests/secure_data.json | 1 + .../integration_tests/service.cpp | 138 ++++++++++++++++++ .../integration_tests/static_config.yaml | 81 ++++++++++ .../integration_tests/tests/conftest.py | 12 ++ .../tests/test_ntrx_failure.py | 21 +++ .../tests/test_statement_log_mode.py | 23 +++ .../tests/test_trx_failure.py | 28 ++++ .../integration_tests/tests/test_ttl.py | 24 +++ 11 files changed, 361 insertions(+) create mode 100644 postgresql/functional_tests/integration_tests/config_vars.yaml create mode 100644 postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql create mode 100644 postgresql/functional_tests/integration_tests/secure_data.json create mode 100644 postgresql/functional_tests/integration_tests/service.cpp create mode 100644 postgresql/functional_tests/integration_tests/static_config.yaml create mode 100644 postgresql/functional_tests/integration_tests/tests/conftest.py create mode 100644 postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py create mode 100644 postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py create mode 100644 postgresql/functional_tests/integration_tests/tests/test_trx_failure.py create mode 100644 postgresql/functional_tests/integration_tests/tests/test_ttl.py diff --git a/.mapping.json b/.mapping.json index 61056344ec83..b504ce87651e 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2910,6 +2910,16 @@ "postgresql/functional_tests/connlimit_max/schemas/postgresql/key_value.sql":"taxi/uservices/userver/postgresql/functional_tests/connlimit_max/schemas/postgresql/key_value.sql", "postgresql/functional_tests/connlimit_max/static_config.yaml":"taxi/uservices/userver/postgresql/functional_tests/connlimit_max/static_config.yaml", "postgresql/functional_tests/connlimit_max/tests/test_connlimit.py":"taxi/uservices/userver/postgresql/functional_tests/connlimit_max/tests/test_connlimit.py", + "postgresql/functional_tests/integration_tests/config_vars.yaml":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/config_vars.yaml", + "postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql", + "postgresql/functional_tests/integration_tests/secure_data.json":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/secure_data.json", + "postgresql/functional_tests/integration_tests/service.cpp":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/service.cpp", + "postgresql/functional_tests/integration_tests/static_config.yaml":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/static_config.yaml", + "postgresql/functional_tests/integration_tests/tests/conftest.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/conftest.py", + "postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py", + "postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py", + "postgresql/functional_tests/integration_tests/tests/test_trx_failure.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_trx_failure.py", + "postgresql/functional_tests/integration_tests/tests/test_ttl.py":"taxi/uservices/userver/postgresql/functional_tests/integration_tests/tests/test_ttl.py", "postgresql/functional_tests/metrics/CMakeLists.txt":"taxi/uservices/userver/postgresql/functional_tests/metrics/CMakeLists.txt", "postgresql/functional_tests/metrics/config_vars.yaml":"taxi/uservices/userver/postgresql/functional_tests/metrics/config_vars.yaml", "postgresql/functional_tests/metrics/schemas/postgresql/key_value.sql":"taxi/uservices/userver/postgresql/functional_tests/metrics/schemas/postgresql/key_value.sql", diff --git a/postgresql/functional_tests/integration_tests/config_vars.yaml b/postgresql/functional_tests/integration_tests/config_vars.yaml new file mode 100644 index 000000000000..57a86a6d3210 --- /dev/null +++ b/postgresql/functional_tests/integration_tests/config_vars.yaml @@ -0,0 +1,18 @@ +# yaml +server-name: test-core-metrics 1.0 +service-name: test_core_metrics +logger-level: info + +config-server-url: http://localhost:8083/ +server-port: 8185 +monitor-server-port: 8186 + +testsuite-enabled: false + +userver-dumps-root: /var/cache/test_core_metrics/userver-dumps/ +access-log-path: /var/log/test_core_metrics/access.log +access-tskv-log-path: /var/log/test_core_metrics/access_tskv.log +default-log-path: /var/log/test_core_metrics/server.log +secdist-path: /etc/test_core_metrics/secure_data.json + +config-cache: /var/cache/test_core_metrics/config_cache.json diff --git a/postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql b/postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql new file mode 100644 index 000000000000..42812ff0b9bc --- /dev/null +++ b/postgresql/functional_tests/integration_tests/schemas/postgresql/key_value.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS key_value_table ( + key VARCHAR PRIMARY KEY, + value VARCHAR, + updated TIMESTAMPTZ NOT NULL DEFAULT NOW() +) diff --git a/postgresql/functional_tests/integration_tests/secure_data.json b/postgresql/functional_tests/integration_tests/secure_data.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/postgresql/functional_tests/integration_tests/secure_data.json @@ -0,0 +1 @@ +{} diff --git a/postgresql/functional_tests/integration_tests/service.cpp b/postgresql/functional_tests/integration_tests/service.cpp new file mode 100644 index 000000000000..ba28b6491b21 --- /dev/null +++ b/postgresql/functional_tests/integration_tests/service.cpp @@ -0,0 +1,138 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace chaos { + +class KeyValue final : public server::handlers::HttpHandlerBase { +public: + static constexpr std::string_view kName = "handler-key-value"; + + KeyValue(const components::ComponentConfig& config, const components::ComponentContext& context); + + std::string HandleRequestThrow(const server::http::HttpRequest& request, server::request::RequestContext&) + const override; + +private: + std::string GetValue(std::string_view key, const server::http::HttpRequest& request) const; + std::string PostValue(std::string_view key, const server::http::HttpRequest& request) const; + std::string DeleteValue(std::string_view key) const; + + storages::postgres::ClusterPtr pg_cluster_; +}; + +KeyValue::KeyValue(const components::ComponentConfig& config, const components::ComponentContext& context) + : HttpHandlerBase(config, context), + pg_cluster_(context.FindComponent("key-value-database").GetCluster()) { + constexpr auto kCreateTable = R"~( + CREATE TABLE IF NOT EXISTS key_value_table ( + key VARCHAR PRIMARY KEY, + value VARCHAR + ) + )~"; + + using storages::postgres::ClusterHostType; + pg_cluster_->Execute(ClusterHostType::kMaster, kCreateTable); +} + +std::string KeyValue::HandleRequestThrow(const server::http::HttpRequest& request, server::request::RequestContext&) + const { + const auto& key = request.GetArg("key"); + if (key.empty()) { + throw server::handlers::ClientError(server::handlers::ExternalBody{"No 'key' query argument"}); + } + + switch (request.GetMethod()) { + case server::http::HttpMethod::kGet: + return GetValue(key, request); + case server::http::HttpMethod::kPost: + return PostValue(key, request); + case server::http::HttpMethod::kDelete: + return DeleteValue(key); + default: + throw server::handlers::ClientError(server::handlers::ExternalBody{ + fmt::format("Unsupported method {}", request.GetMethod())}); + } +} + +const storages::postgres::Query kSelectValue{ + "SELECT value FROM key_value_table WHERE key=$1", + storages::postgres::Query::Name{"sample_select_value"}, +}; + +std::string KeyValue::GetValue(std::string_view key, const server::http::HttpRequest& request) const { + const storages::postgres::ResultSet res = + pg_cluster_->Execute(storages::postgres::ClusterHostType::kSlave, kSelectValue, key); + if (res.IsEmpty()) { + request.SetResponseStatus(server::http::HttpStatus::kNotFound); + return {}; + } + + return res.AsSingleRow(); +} + +const storages::postgres::Query kInsertValue{ + "INSERT INTO key_value_table (key, value) " + "VALUES ($1, $2) " + "ON CONFLICT DO NOTHING", + storages::postgres::Query::Name{"sample_insert_value"}, +}; + +std::string KeyValue::PostValue(std::string_view key, const server::http::HttpRequest& request) const { + const auto& value = request.GetArg("value"); + + storages::postgres::Transaction transaction = + pg_cluster_->Begin("sample_transaction_insert_key_value", storages::postgres::ClusterHostType::kMaster, {}); + + auto res = transaction.Execute(kInsertValue, key, value); + if (res.RowsAffected()) { + transaction.Commit(); + request.SetResponseStatus(server::http::HttpStatus::kCreated); + return std::string{value}; + } + + res = transaction.Execute(kSelectValue, key); + transaction.Rollback(); + + auto result = res.AsSingleRow(); + if (result != value) { + request.SetResponseStatus(server::http::HttpStatus::kConflict); + } + + return res.AsSingleRow(); +} + +std::string KeyValue::DeleteValue(std::string_view key) const { + auto res = pg_cluster_->Execute( + storages::postgres::ClusterHostType::kMaster, "DELETE FROM key_value_table WHERE key=$1", key + ); + return std::to_string(res.RowsAffected()); +} + +} // namespace chaos + +int main(int argc, char* argv[]) { + const auto component_list = components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append("key-value-database") + .Append() + .Append() + .Append(); + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/postgresql/functional_tests/integration_tests/static_config.yaml b/postgresql/functional_tests/integration_tests/static_config.yaml new file mode 100644 index 000000000000..3313b3291164 --- /dev/null +++ b/postgresql/functional_tests/integration_tests/static_config.yaml @@ -0,0 +1,81 @@ +# yaml +components_manager: + components: + handler-key-value: + path: /v1/key-value + task_processor: main-task-processor + method: GET,DELETE,POST + + key-value-database: + dbconnection: 'postgresql://testsuite@localhost:15433/pg_key_value' + blocking_task_processor: fs-task-processor + dns_resolver: async + + testsuite-support: + + http-client: + fs-task-processor: main-task-processor + + tests-control: + method: POST + path: /tests/{action} + skip-unregistered-testpoints: true + task_processor: main-task-processor + testpoint-timeout: 10s + testpoint-url: $mockserver/testpoint + throttling_enabled: false + + server: + listener: + port: 8187 + task_processor: main-task-processor + listener-monitor: + port: $monitor-server-port + port#fallback: 8086 + connection: + in_buffer_size: 32768 + requests_queue_size_threshold: 100 + task_processor: main-task-processor + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: debug + overflow_behavior: discard + + handler-server-monitor: + path: /service/monitor + method: GET + task_processor: main-task-processor + + # /// [dynamic config] + # yaml + dynamic-config: + defaults: + POSTGRES_CONNECTION_SETTINGS: + key-value-database: + max-ttl-sec: 60 + recent-errors-threshold: 100000 + POSTGRES_DEFAULT_COMMAND_CONTROL: + network_timeout_ms: 750 + statement_timeout_ms: 500 + POSTGRES_QUERIES_COMMAND_CONTROL: + sample_select_value: + network_timeout_ms: 70 + statement_timeout_ms: 40 + sample_transaction_insert_key_value: + network_timeout_ms: 200 + statement_timeout_ms: 150 + # /// [dynamic config] + + dns-client: + fs-task-processor: fs-task-processor + + task_processors: + main-task-processor: + worker_threads: 4 + fs-task-processor: + worker_threads: 4 + + default_task_processor: main-task-processor diff --git a/postgresql/functional_tests/integration_tests/tests/conftest.py b/postgresql/functional_tests/integration_tests/tests/conftest.py new file mode 100644 index 000000000000..aa753caf5a2b --- /dev/null +++ b/postgresql/functional_tests/integration_tests/tests/conftest.py @@ -0,0 +1,12 @@ +pytest_plugins = [ + 'testsuite.pytest_plugin', + 'pytest_userver.plugins.caches', + 'pytest_userver.plugins.config', + 'pytest_userver.plugins.dumps', + 'pytest_userver.plugins.dynamic_config', + 'pytest_userver.plugins.log_capture', + 'pytest_userver.plugins.service', + 'pytest_userver.plugins.service_client', + 'pytest_userver.plugins.testpoint', + 'pytest_userver.plugins.postgresql', +] diff --git a/postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py b/postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py new file mode 100644 index 000000000000..8ea0c52a8d8f --- /dev/null +++ b/postgresql/functional_tests/integration_tests/tests/test_ntrx_failure.py @@ -0,0 +1,21 @@ +import pytest + + +# /// [fault injection] +@pytest.mark.skip('This test is broken. Fix it, please') +async def test_ntrx_fail(service_client, pgsql, userver_pg_ntrx): + response = await service_client.delete('/v1/key-value?key=foo') + assert response.status == 200 + + response = await service_client.post('/v1/key-value?key=foo&value=bar') + assert response.status == 201 + + with userver_pg_ntrx.mock_failure('sample_select_value'): + response = await service_client.get('/v1/key-value?key=foo') + assert response.status == 500 + + response = await service_client.get('/v1/key-value?key=foo') + assert response.status == 200 + assert response.content == b'bar' + + # /// [fault injection] diff --git a/postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py b/postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py new file mode 100644 index 000000000000..015c670febd9 --- /dev/null +++ b/postgresql/functional_tests/integration_tests/tests/test_statement_log_mode.py @@ -0,0 +1,23 @@ +import pytest + + +@pytest.fixture(scope='session') +def userver_pg_log_config(): + def patch_config(config_yaml, _config_vars) -> None: + components = config_yaml['components_manager']['components'] + db = components['key-value-database'] + db['statement-log-mode'] = 'hide' + + return patch_config + + +@pytest.mark.skip('This test is broken. Fix it, please') +@pytest.mark.uservice_oneshot(config_hooks=['userver_pg_log_config']) +async def test_make_request(service_client): + async with service_client.capture_logs() as capture: + response = await service_client.post('/v1/key-value?key=foo&value=bar') + assert response.status in (200, 201) + + records = capture.select(stopwatch_name='pg_query') + assert len(records) == 1, capture.select() + assert 'db_statement' not in records[0] diff --git a/postgresql/functional_tests/integration_tests/tests/test_trx_failure.py b/postgresql/functional_tests/integration_tests/tests/test_trx_failure.py new file mode 100644 index 000000000000..4d6ba83dff54 --- /dev/null +++ b/postgresql/functional_tests/integration_tests/tests/test_trx_failure.py @@ -0,0 +1,28 @@ +import pytest + + +@pytest.mark.skip('This test is broken. Fix it, please') +async def test_trx_ok(service_client, pgsql): + response = await service_client.post('/v1/key-value?key=foo&value=bar') + assert response.status == 201 + assert response.content == b'bar' + + response = await service_client.get('/v1/key-value?key=foo') + assert response.status == 200 + assert response.content == b'bar' + + +# /// [fault injection] +@pytest.mark.skip('This test is broken. Fix it, please') +async def test_trx_fail(service_client, pgsql, userver_pg_trx): + response = await service_client.delete('/v1/key-value?key=foo') + assert response.status == 200 + + userver_pg_trx.enable_failure('sample_transaction_insert_key_value') + + response = await service_client.post('/v1/key-value?key=foo&value=bar') + assert response.status == 500 + + response = await service_client.get('/v1/key-value?key=foo') + assert response.status == 404 + # /// [fault injection] diff --git a/postgresql/functional_tests/integration_tests/tests/test_ttl.py b/postgresql/functional_tests/integration_tests/tests/test_ttl.py new file mode 100644 index 000000000000..86fba009e2a2 --- /dev/null +++ b/postgresql/functional_tests/integration_tests/tests/test_ttl.py @@ -0,0 +1,24 @@ +import pytest + + +async def _make_request(service_client): + response = await service_client.post('/v1/key-value?key=foo&value=bar') + assert response.status in (200, 201) + assert response.content == b'bar' + + +@pytest.mark.skip('This test is broken. Fix it, please') +async def test_connection_max_ttl(service_client, monitor_client, mocked_time): + await _make_request(service_client) + + async with monitor_client.metrics_diff( + prefix='postgresql.connections', + diff_gauge=True, + ) as diff: + # with max-ttl-sec set to 60 we expect the db connection to be reopened + # on the next request + mocked_time.sleep(61) + await _make_request(service_client) + + assert diff.value_at('closed') == 1 + assert diff.value_at('opened') == 1 From 059c9fe9ed1999502fb6731a56b17c6fb1a0251f Mon Sep 17 00:00:00 2001 From: fdr400 Date: Fri, 1 Aug 2025 15:04:50 +0300 Subject: [PATCH 039/151] feat kafka: add GetTopics for ConsumerScope add get topics for ConsumerScope commit_hash:c9506c0e0cdbd7aa16346577df0bfd99b0c08be0 --- kafka/include/userver/kafka/consumer_scope.hpp | 3 +++ kafka/src/kafka/consumer_scope.cpp | 2 ++ kafka/tests/consumer_kafkatest.cpp | 13 +++++++++++++ 3 files changed, 18 insertions(+) diff --git a/kafka/include/userver/kafka/consumer_scope.hpp b/kafka/include/userver/kafka/consumer_scope.hpp index 0699338aca2a..c3004d41b309 100644 --- a/kafka/include/userver/kafka/consumer_scope.hpp +++ b/kafka/include/userver/kafka/consumer_scope.hpp @@ -81,6 +81,9 @@ class ConsumerScope final { ConsumerScope(ConsumerScope&&) noexcept = delete; ConsumerScope& operator=(ConsumerScope&&) noexcept = delete; + /// @brief Topics list consumer configured to subscribe. + const std::vector& GetTopics() const; + /// @brief Subscribes for configured topics and starts the consumer polling /// process. /// @note If `callback` throws an exception, entire message batch (also diff --git a/kafka/src/kafka/consumer_scope.cpp b/kafka/src/kafka/consumer_scope.cpp index 6b3749ec0855..1b6e36f53826 100644 --- a/kafka/src/kafka/consumer_scope.cpp +++ b/kafka/src/kafka/consumer_scope.cpp @@ -12,6 +12,8 @@ ConsumerScope::~ConsumerScope() { Stop(); } void ConsumerScope::Start(Callback callback) { consumer_.StartMessageProcessing(std::move(callback)); } +const std::vector& ConsumerScope::GetTopics() const { return consumer_.topics_; } + void ConsumerScope::Stop() noexcept { consumer_.Stop(); } void ConsumerScope::AsyncCommit() { consumer_.AsyncCommit(); } diff --git a/kafka/tests/consumer_kafkatest.cpp b/kafka/tests/consumer_kafkatest.cpp index b636b6792ad4..808d4fabd10a 100644 --- a/kafka/tests/consumer_kafkatest.cpp +++ b/kafka/tests/consumer_kafkatest.cpp @@ -36,6 +36,19 @@ UTEST_F(ConsumerTest, BrokenConfiguration) { UEXPECT_THROW(MakeConsumer("kafka-consumer", {}, consumer_configuration), std::runtime_error); } +UTEST_F(ConsumerTest, GetTopics) { + { + auto consumer = MakeConsumer("kafka-consumer", {}); + auto consumer_scope = consumer.MakeConsumerScope(); + ASSERT_TRUE(consumer_scope.GetTopics().empty()); + } + { + auto consumer = MakeConsumer("kafka-consumer", {"topic-1", "topic-2"}); + auto consumer_scope = consumer.MakeConsumerScope(); + EXPECT_THAT(consumer_scope.GetTopics(), ::testing::ElementsAre("topic-1", "topic-2")); + } +} + UTEST_F(ConsumerTest, OneConsumerSmallTopics) { const auto topic1 = GenerateTopic(); const auto topic2 = GenerateTopic(); From 9f57b6795d3fa7af42c8b9f38c52032219257f07 Mon Sep 17 00:00:00 2001 From: fdr400 Date: Fri, 1 Aug 2025 20:28:23 +0300 Subject: [PATCH 040/151] test grpc-client: add retries and RST_STREAM lowlevel tests add empty body test, some retries tests and RST\_STREAM tests commit_hash:8f09323c917f99dd5aa522714a0cb55955fa1df0 --- .../lowlevel/src/client_runner.cpp | 48 ++++- .../lowlevel/static_config.yaml | 3 +- .../tests-client-protocol/conftest.py | 8 - .../test_http2_raw_responses.py | 204 +++++++++++++++++- 4 files changed, 240 insertions(+), 23 deletions(-) diff --git a/grpc/functional_tests/lowlevel/src/client_runner.cpp b/grpc/functional_tests/lowlevel/src/client_runner.cpp index 5152130ab973..9b95459a057c 100644 --- a/grpc/functional_tests/lowlevel/src/client_runner.cpp +++ b/grpc/functional_tests/lowlevel/src/client_runner.cpp @@ -1,10 +1,39 @@ #include +#include + #include #include +#include USERVER_NAMESPACE_BEGIN +namespace { + +formats::json::Value HandleGet() { + static const auto grpc_version = utils::text::Split(grpc::Version(), "."); + + return formats::json::MakeObject( + "grpc-version", formats::json::MakeObject("major", grpc_version.at(0), "minor", grpc_version.at(1)) + ); +} + +formats::json::Value HandlePost(const ::samples::api::GreeterServiceClient& client) { + std::optional<::grpc::Status> grpc_status; + try { + [[maybe_unused]] auto response = client.SayHello({}); + grpc_status.emplace(::grpc::Status::OK); + } catch (const ugrpc::client::ErrorWithStatus& ex) { + grpc_status.emplace(ex.GetStatus()); + } catch (const std::exception& ex) { + return {}; + } + + return formats::json::MakeObject("grpc-status", ugrpc::ToString(grpc_status->error_code())); +} + +} // namespace + ClientRunner::ClientRunner(const components::ComponentConfig& config, const components::ComponentContext& context) : server::handlers::HttpHandlerJsonBase{config, context}, client_{context.FindComponent() @@ -15,21 +44,18 @@ ClientRunner::ClientRunner(const components::ComponentConfig& config, const comp )} {} auto ClientRunner::HandleRequestJsonThrow( - const HttpRequest& /*request*/, + const HttpRequest& request, const Value& /*request_json*/, RequestContext& /*context*/ ) const -> Value { - std::optional<::grpc::Status> grpc_status; - try { - [[maybe_unused]] auto response = client_.SayHello({}); - grpc_status.emplace(::grpc::Status::OK); - } catch (const ugrpc::client::ErrorWithStatus& ex) { - grpc_status.emplace(ex.GetStatus()); - } catch (const std::exception& ex) { - return {}; + switch (request.GetMethod()) { + case server::http::HttpMethod::kGet: + return HandleGet(); + case server::http::HttpMethod::kPost: + return HandlePost(client_); + default: + return {}; } - - return formats::json::MakeObject("grpc-status", ugrpc::ToString(grpc_status->error_code())); } yaml_config::Schema ClientRunner::GetStaticConfigSchema() { diff --git a/grpc/functional_tests/lowlevel/static_config.yaml b/grpc/functional_tests/lowlevel/static_config.yaml index b4f1af732fe8..11d157282917 100644 --- a/grpc/functional_tests/lowlevel/static_config.yaml +++ b/grpc/functional_tests/lowlevel/static_config.yaml @@ -40,7 +40,8 @@ components_manager: "UNAVAILABLE", "ABORTED", "CANCELLED", - "DEADLINE_EXCEEDED" + "DEADLINE_EXCEEDED", + "INTERNAL" ] } }] diff --git a/grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py b/grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py index 2962ee146943..d1957c171e43 100644 --- a/grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py +++ b/grpc/functional_tests/lowlevel/tests-client-protocol/conftest.py @@ -7,14 +7,11 @@ from typing import Callable from typing import Dict -import grpc import http2 import pytest import testsuite.utils.net as net_utils -import utils - pytest_plugins = [ 'pytest_userver.plugins.core', ] @@ -22,11 +19,6 @@ USERVER_CONFIG_HOOKS = ['userver_config_http2_server_endpoint'] -@pytest.fixture(scope='session') -def retryable_status_codes(_retry_policy) -> list[grpc.StatusCode]: - return [utils.status_from_str(status) for status in _retry_policy['retryableStatusCodes']] - - @pytest.fixture(scope='session') def max_attempts(_retry_policy) -> int: return _retry_policy['maxAttempts'] diff --git a/grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py b/grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py index ddd8d129deb0..66d51eb62cd7 100644 --- a/grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py +++ b/grpc/functional_tests/lowlevel/tests-client-protocol/test_http2_raw_responses.py @@ -19,7 +19,26 @@ 504: grpc.StatusCode.UNAVAILABLE, } -HTTP_STATUSES = list(HTTP2_TO_GRPC) + [ +# RST_STREAM error code mapping https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html (HTTP2 Transport Mapping) +RST_ERROR_CODE_TO_STATUS = { + http2.errors.ErrorCodes.NO_ERROR: grpc.StatusCode.INTERNAL, + http2.errors.ErrorCodes.PROTOCOL_ERROR: grpc.StatusCode.INTERNAL, + http2.errors.ErrorCodes.INTERNAL_ERROR: grpc.StatusCode.INTERNAL, + http2.errors.ErrorCodes.FLOW_CONTROL_ERROR: grpc.StatusCode.INTERNAL, + http2.errors.ErrorCodes.SETTINGS_TIMEOUT: grpc.StatusCode.INTERNAL, + http2.errors.ErrorCodes.FRAME_SIZE_ERROR: grpc.StatusCode.INTERNAL, + http2.errors.ErrorCodes.REFUSED_STREAM: grpc.StatusCode.UNAVAILABLE, # Indicates that no processing occurred and the request can be retried, possibly elsewhere. + # Mapped to call cancellation when sent by a client.Mapped to CANCELLED when sent by a server. + # Note that servers should only use this mechanism when they need to cancel a call but the payload byte sequence is incomplete. + http2.errors.ErrorCodes.CANCEL: grpc.StatusCode.CANCELLED, + http2.errors.ErrorCodes.COMPRESSION_ERROR: grpc.StatusCode.INTERNAL, + http2.errors.ErrorCodes.CONNECT_ERROR: grpc.StatusCode.INTERNAL, + http2.errors.ErrorCodes.ENHANCE_YOUR_CALM: grpc.StatusCode.RESOURCE_EXHAUSTED, # with additional error detail provided by runtime to indicate that the exhausted resource is bandwidth. + http2.errors.ErrorCodes.INADEQUATE_SECURITY: grpc.StatusCode.PERMISSION_DENIED, +} + +HTTP_STATUSES = [ + *list(HTTP2_TO_GRPC), # not mapped statuses # TODO: think about 1xx, because their handling differs from other codes 201, @@ -28,6 +47,47 @@ 501, ] +# Codes from client settings in static config +RETRYABLE_STATUS_CODES = [ + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.ABORTED, + grpc.StatusCode.CANCELLED, + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.INTERNAL, +] + +RETRYABLE_HTTP_STATUSES = sorted( + set(status for status, code in HTTP2_TO_GRPC.items() if code in RETRYABLE_STATUS_CODES) +) + +# Length-Prefixed-Message: Compressed-Flag (1 byte) Message-Length (4 bytes) Message +EMPTY_PROTO_MESSAGE = b'\x00' + b'\x00\x00\x00\x00' + + +async def test_success_response(service_client, grpc_server: http2.GrpcServer) -> None: + def _response_factory() -> List[http2.Frame]: + # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html + # Basic response is: + # - Response-Headers (HEADERS frame): HTTP-status Content-Type + # - Length-Prefixed-Message (DATA frame) + # - Trailers (HEADERS frame, END_STREAM): Status + + response_headers = http2.HeadersFrame([ + (':status', '200'), + ('content-type', 'application/grpc'), + ]) + message = http2.DataFrame(EMPTY_PROTO_MESSAGE) + trailers = http2.HeadersFrame([('grpc-status', utils.status_to_str(grpc.StatusCode.OK))], end_stream=True) + + return [response_headers, message, trailers] + + grpc_server.response_factory = _response_factory + + expected_grpc_status_code = grpc.StatusCode.OK + actual_grpc_status_code = await run_client(service_client) + + assert actual_grpc_status_code == expected_grpc_status_code + @pytest.mark.parametrize('http_status', HTTP_STATUSES) @pytest.mark.parametrize( @@ -36,9 +96,9 @@ ) async def test_grpc_client_ignores_any_http_status_when_grpc_status_exists( service_client, + grpc_server: http2.GrpcServer, http_status: int, grpc_status_code: grpc.StatusCode, - grpc_server: http2.GrpcServer, ) -> None: def _response_factory() -> List[http2.Frame]: # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html @@ -62,7 +122,9 @@ def _response_factory() -> List[http2.Frame]: @pytest.mark.parametrize('http_status', HTTP_STATUSES) async def test_grpc_client_synthesizes_grpc_status_from_http_status( - service_client, http_status: int, grpc_server: http2.GrpcServer + service_client, + grpc_server: http2.GrpcServer, + http_status: int, ) -> None: def _response_factory() -> List[http2.Frame]: # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html @@ -83,6 +145,133 @@ def _response_factory() -> List[http2.Frame]: assert actual_grpc_status_code == expected_grpc_status_code +@pytest.mark.parametrize('grpc_status_code', RETRYABLE_STATUS_CODES) +async def test_grpc_client_retries_trailers_only_with_grpc_status( + service_client, + grpc_server: http2.GrpcServer, + grpc_status_code: grpc.StatusCode, + max_attempts: int, # max attempts from service config +) -> None: + attempts = 0 + + def _response_factory() -> List[http2.Frame]: + # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html + # Trailers-Only is only HEADERS http2 frame (with END_STREAM flag) + + nonlocal attempts + attempts += 1 + + headers: List[Tuple[str, str]] = [ + (':status', '200'), + ('content-type', 'application/grpc'), + ('grpc-status', utils.status_to_str(grpc_status_code)), + ] + + return [http2.HeadersFrame(headers=headers, end_stream=True)] + + grpc_server.response_factory = _response_factory + + expected_grpc_status_code = grpc_status_code + actual_grpc_status_code = await run_client(service_client) + + assert actual_grpc_status_code == expected_grpc_status_code + assert attempts == max_attempts + + +@pytest.mark.parametrize('http_status', RETRYABLE_HTTP_STATUSES) +async def test_grpc_client_retries_trailers_only_with_retryable_http_status( + service_client, + grpc_server: http2.GrpcServer, + http_status: int, + max_attempts: int, # max attempts from service config +) -> None: + attempts = 0 + + def _response_factory() -> List[http2.Frame]: + # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html + # Trailers-Only is only HEADERS http2 frame (with END_STREAM flag) + + nonlocal attempts + attempts += 1 + + headers: List[Tuple[str, str]] = [ + (':status', str(http_status)), + ('content-type', 'application/grpc'), + ] + + return [http2.HeadersFrame(headers=headers, end_stream=True)] + + grpc_server.response_factory = _response_factory + + expected_grpc_status_code = http2_status_to_grpc(http_status) + actual_grpc_status_code = await run_client(service_client) + + assert actual_grpc_status_code == expected_grpc_status_code + assert attempts == max_attempts + + +@pytest.mark.parametrize('rst_stream_error_code', RST_ERROR_CODE_TO_STATUS) +@pytest.mark.parametrize('send_response_headers', [False, True]) +async def test_grpc_client_converts_rst_stream_to_grpc_status( + service_client, + grpc_server: http2.GrpcServer, + rst_stream_error_code: http2.errors.ErrorCodes, + send_response_headers: bool, +) -> None: + def _response_factory() -> List[http2.Frame]: + # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html + # Trailers-Only is only HEADERS http2 frame (with END_STREAM flag) + + response: List[http2.Frame] = [] + if send_response_headers: + response.append( + http2.HeadersFrame([ + (':status', '200'), + ('content-type', 'application/grpc'), + ]) + ) + response.append(http2.RstStreamFrame(rst_stream_error_code)) + + return response + + grpc_server.response_factory = _response_factory + + expected_grpc_status_code = RST_ERROR_CODE_TO_STATUS[rst_stream_error_code] + actual_grpc_status_code = await run_client(service_client) + + assert actual_grpc_status_code == expected_grpc_status_code + + +async def test_grpc_client_retries_rst_stream_refused_stream( + service_client, + grpc_server: http2.GrpcServer, + max_attempts: int, # max attempts from service config +) -> None: + # old gRPC version does not retry RST_STREAM(REFUSED_STREAM) + if await client_grpc_version(service_client) < 1.54: + pytest.skip('Does not work on old gRPC versions') + + attempts = 0 + + def _response_factory() -> List[http2.Frame]: + # https://grpc.github.io/grpc/core/md_doc__p_r_o_t_o_c_o_l-_h_t_t_p2.html + # Trailers-Only is only HEADERS http2 frame (with END_STREAM flag) + + nonlocal attempts + attempts += 1 + + return [http2.RstStreamFrame(http2.errors.ErrorCodes.REFUSED_STREAM)] + + grpc_server.response_factory = _response_factory + + expected_grpc_status_code = RST_ERROR_CODE_TO_STATUS[http2.errors.ErrorCodes.REFUSED_STREAM] + actual_grpc_status_code = await run_client(service_client) + + assert actual_grpc_status_code == expected_grpc_status_code + + assert attempts == max_attempts + + async def run_client(service_client) -> grpc.StatusCode: resp = await service_client.post('/client') assert resp.status == 200 @@ -90,5 +279,14 @@ async def run_client(service_client) -> grpc.StatusCode: return utils.status_from_str(resp.json()['grpc-status']) +async def client_grpc_version(service_client) -> float: + resp = await service_client.get('/client') + assert resp.status == 200 + + version = resp.json()['grpc-version'] + + return float(version['major']) + 0.01 * float(version['minor']) + + def http2_status_to_grpc(http_status: int) -> grpc.StatusCode: return HTTP2_TO_GRPC.get(http_status, grpc.StatusCode.UNKNOWN) From 0c4f0d60412271aefc1eb6b3a9cff218edd59c08 Mon Sep 17 00:00:00 2001 From: ksanvat Date: Mon, 4 Aug 2025 03:03:44 +0300 Subject: [PATCH 041/151] feat proto35: learn clients to include structs commit_hash:dc346e857a1d75323e698b50df8e164aff2de936 --- scripts/grpc/generator.py | 29 +++++++++++++++++++- scripts/grpc/templates/client.usrv.hpp.jinja | 3 ++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/scripts/grpc/generator.py b/scripts/grpc/generator.py index 8724cefb343b..28837c74de5a 100755 --- a/scripts/grpc/generator.py +++ b/scripts/grpc/generator.py @@ -11,8 +11,10 @@ `{client,handler}.usrv.pb.{hpp,cpp}.jinja` templates. """ +import dataclasses import enum import itertools +import json import os import sys from typing import Any @@ -48,6 +50,26 @@ def is_both(self) -> bool: return self == self.Both +@dataclasses.dataclass(frozen=True) +class Params: + structs: bool + + @classmethod + def parse(cls, data: str) -> 'Params': + json_data: Dict[Any, Any] = {} + if data: + json_data.update(json.loads(data)) + + result = cls( + structs=json_data.pop('structs', False), + ) + + if json_data: + raise Exception(f'Unknown params keys {sorted(json_data)}') + + return result + + def _grpc_to_cpp_name(in_str: str) -> str: return in_str.replace('.', '::') @@ -59,12 +81,14 @@ def _to_package_prefix(package: str): class _CodeGenerator: def __init__( self, + params: Params, proto_file: descriptor.FileDescriptorProto, response: plugin.CodeGeneratorResponse, jinja_env: jinja2.Environment, mode: Mode, skip_files_wo_service: bool, ) -> None: + self.params = params self.proto_file = proto_file self.response = response self.jinja_env = jinja_env @@ -95,7 +119,7 @@ def _generate_code_with_service(self) -> None: file_type=file_type, file_ext=file_ext, ) - file.content = template.render(proto=data) + file.content = template.render(params=self.params, proto=data) def _generate_code_empty(self) -> None: for file_type, file_ext in self._iter_src_files(): @@ -135,6 +159,8 @@ def generate( request = plugin.CodeGeneratorRequest() request.ParseFromString(data) + params = Params.parse(request.parameter) + response = plugin.CodeGeneratorResponse() if hasattr(response, 'FEATURE_PROTO3_OPTIONAL'): setattr( @@ -157,6 +183,7 @@ def generate( # pylint: disable=no-member for proto_file in request.proto_file: _CodeGenerator( + params=params, jinja_env=jinja_env, proto_file=proto_file, response=response, diff --git a/scripts/grpc/templates/client.usrv.hpp.jinja b/scripts/grpc/templates/client.usrv.hpp.jinja index 770399ed3025..37ea8241b4a2 100644 --- a/scripts/grpc/templates/client.usrv.hpp.jinja +++ b/scripts/grpc/templates/client.usrv.hpp.jinja @@ -7,6 +7,9 @@ #include #include "{{ proto.source_file_without_ext }}.pb.h" +{% if params.structs %} +#include <{{ proto.source_file_without_ext }}.structs.usrv.pb.hpp> +{% endif %} {% call utils.optional_namespace(proto.namespace) %} {% for service in proto.services %} From e8ae24aad9744b4bdb68630a90000c1a66f3879d Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 4 Aug 2025 14:57:04 +0300 Subject: [PATCH 042/151] feat chaotic: simplify third-party types usage Closes: https://github.com/userver-framework/userver/pull/899 Co-authored-by: Alexey Medvedev \ commit_hash:aebed1ae96c273975939a0d33842330b07f3d8c6 --- .mapping.json | 2 ++ chaotic/bin-dynamic-configs/main.py | 2 +- cmake/ChaoticGen.cmake | 19 +++++++++++++++---- .../storages/postgres/time_point_tz.hpp | 16 ++++++++++++++++ .../storages/postgres/time_point_tz.cpp | 18 ++++++++++++++++++ scripts/docs/en/userver/chaotic.md | 2 ++ 6 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 postgresql/include/userver/chaotic/io/userver/storages/postgres/time_point_tz.hpp create mode 100644 postgresql/src/chaotic/io/userver/storages/postgres/time_point_tz.cpp diff --git a/.mapping.json b/.mapping.json index b504ce87651e..6ad8348b6e82 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2938,6 +2938,7 @@ "postgresql/functional_tests/secdist-update/tests/test_secdist.py":"taxi/uservices/userver/postgresql/functional_tests/secdist-update/tests/test_secdist.py", "postgresql/include/userver/cache/base_postgres_cache.hpp":"taxi/uservices/userver/postgresql/include/userver/cache/base_postgres_cache.hpp", "postgresql/include/userver/cache/base_postgres_cache_fwd.hpp":"taxi/uservices/userver/postgresql/include/userver/cache/base_postgres_cache_fwd.hpp", + "postgresql/include/userver/chaotic/io/userver/storages/postgres/time_point_tz.hpp":"taxi/uservices/userver/postgresql/include/userver/chaotic/io/userver/storages/postgres/time_point_tz.hpp", "postgresql/include/userver/storages/postgres/cluster.hpp":"taxi/uservices/userver/postgresql/include/userver/storages/postgres/cluster.hpp", "postgresql/include/userver/storages/postgres/cluster_types.hpp":"taxi/uservices/userver/postgresql/include/userver/storages/postgres/cluster_types.hpp", "postgresql/include/userver/storages/postgres/component.hpp":"taxi/uservices/userver/postgresql/include/userver/storages/postgres/component.hpp", @@ -3024,6 +3025,7 @@ "postgresql/src/cache/base_postgres_cache.cpp":"taxi/uservices/userver/postgresql/src/cache/base_postgres_cache.cpp", "postgresql/src/cache/postgres_cache_test.cpp":"taxi/uservices/userver/postgresql/src/cache/postgres_cache_test.cpp", "postgresql/src/cache/postgres_cache_test_fwd.hpp":"taxi/uservices/userver/postgresql/src/cache/postgres_cache_test_fwd.hpp", + "postgresql/src/chaotic/io/userver/storages/postgres/time_point_tz.cpp":"taxi/uservices/userver/postgresql/src/chaotic/io/userver/storages/postgres/time_point_tz.cpp", "postgresql/src/storages/postgres/cluster.cpp":"taxi/uservices/userver/postgresql/src/storages/postgres/cluster.cpp", "postgresql/src/storages/postgres/cluster_types.cpp":"taxi/uservices/userver/postgresql/src/storages/postgres/cluster_types.cpp", "postgresql/src/storages/postgres/component.cpp":"taxi/uservices/userver/postgresql/src/storages/postgres/component.cpp", diff --git a/chaotic/bin-dynamic-configs/main.py b/chaotic/bin-dynamic-configs/main.py index 745e80f34c56..3bd031dbf8ba 100755 --- a/chaotic/bin-dynamic-configs/main.py +++ b/chaotic/bin-dynamic-configs/main.py @@ -54,7 +54,7 @@ def main(): compiler.parse_variable( file, name, - include_dirs=args.include_dir, + include_dirs=(args.include_dir or []), namespace='dynamic_config', ) compiler.generate_variable( diff --git a/cmake/ChaoticGen.cmake b/cmake/ChaoticGen.cmake index 9db3c1493a92..1f7b2088efe9 100644 --- a/cmake/ChaoticGen.cmake +++ b/cmake/ChaoticGen.cmake @@ -57,10 +57,11 @@ _userver_prepare_chaotic() # RELATIVE_TO - --relative-to option to chaotic-gen - FORMAT - can be ON/OFF, enable to format generated files, defaults # to USERVER_CHAOTIC_FORMAT - SCHEMAS - JSONSchema source files - ARGS - extra args to chaotic-gen - # INSTALL_INCLUDES_COMPONENT - component to install generated includes +# LINK_TARGETS - targets to link (used by x-usrv-cpp-type) function(userver_target_generate_chaotic TARGET) set(OPTIONS GENERATE_SERIALIZERS PARSE_EXTRA_FORMATS) set(ONE_VALUE_ARGS OUTPUT_DIR RELATIVE_TO FORMAT INSTALL_INCLUDES_COMPONENT OUTPUT_PREFIX ERASE_PATH_PREFIX) - set(MULTI_VALUE_ARGS SCHEMAS LAYOUT INCLUDE_DIRS) + set(MULTI_VALUE_ARGS SCHEMAS LAYOUT INCLUDE_DIRS LINK_TARGETS) cmake_parse_arguments(PARSE "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) get_property(CHAOTIC_BIN GLOBAL PROPERTY userver_chaotic_bin) @@ -133,6 +134,18 @@ function(userver_target_generate_chaotic TARGET) list(APPEND CHAOTIC_ARGS "--clang-format" "${CLANG_FORMAT}") _userver_initialize_codegen_flag() + + add_library("${TARGET}" ${SCHEMAS}) + target_link_libraries("${TARGET}" PUBLIC userver::chaotic ${PARSE_LINK_TARGETS}) + + get_target_property(TARGET_LIBRARIES "${TARGET}" LINK_LIBRARIES) + foreach(LIBRARY ${TARGET_LIBRARIES}) + get_target_property(DIRS "${LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES) + foreach(DIRECTORY ${DIRS}) + set(CHAOTIC_ARGS ${CHAOTIC_ARGS} -I ${DIRECTORY}) + endforeach() + endforeach() + add_custom_command( OUTPUT ${SCHEMAS} COMMAND ${CMAKE_COMMAND} -E env "USERVER_PYTHON=${USERVER_CHAOTIC_PYTHON_BINARY}" "${CHAOTIC_BIN}" @@ -142,8 +155,6 @@ function(userver_target_generate_chaotic TARGET) VERBATIM ${CODEGEN} ) _userver_codegen_register_files("${SCHEMAS}") - add_library("${TARGET}" ${SCHEMAS}) - target_link_libraries("${TARGET}" userver::chaotic) target_include_directories("${TARGET}" PUBLIC "$") target_include_directories("${TARGET}" PUBLIC "$") @@ -257,7 +268,7 @@ function(userver_target_generate_chaotic_dynamic_configs TARGET SCHEMAS_REGEX) OUTPUT ${OUTPUT_FILENAMES} COMMAND env "USERVER_PYTHON=${USERVER_CHAOTIC_PYTHON_BINARY}" "${CHAOTIC_DYNAMIC_CONFIGS_BIN}" ${CHAOTIC_EXTRA_ARGS} - -I ${CMAKE_CURRENT_LIST_DIR}/../chaotic/include -o "${OUTPUT_DIR}" ${CHGEN_FILENAMES} + -o "${OUTPUT_DIR}" ${CHGEN_FILENAMES} COMMENT "Generating dynamic configs${CONFIG_NAMES}" DEPENDS ${CHGEN_FILENAMES} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" diff --git a/postgresql/include/userver/chaotic/io/userver/storages/postgres/time_point_tz.hpp b/postgresql/include/userver/chaotic/io/userver/storages/postgres/time_point_tz.hpp new file mode 100644 index 000000000000..578be73c1cb5 --- /dev/null +++ b/postgresql/include/userver/chaotic/io/userver/storages/postgres/time_point_tz.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::postgres { + +TimePointTz Convert(const std::string& str, chaotic::convert::To); + +std::string Convert(const TimePointTz& tp, chaotic::convert::To); + +} // namespace storages::postgres + +USERVER_NAMESPACE_END diff --git a/postgresql/src/chaotic/io/userver/storages/postgres/time_point_tz.cpp b/postgresql/src/chaotic/io/userver/storages/postgres/time_point_tz.cpp new file mode 100644 index 000000000000..1e7fc29b61b0 --- /dev/null +++ b/postgresql/src/chaotic/io/userver/storages/postgres/time_point_tz.cpp @@ -0,0 +1,18 @@ +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::postgres { + +TimePointTz Convert(const std::string& str, chaotic::convert::To) { + return USERVER_NAMESPACE::storages::postgres::TimePointTz{USERVER_NAMESPACE::utils::datetime::Stringtime(str)}; +} + +std::string Convert(const TimePointTz& tp, chaotic::convert::To) { + return USERVER_NAMESPACE::utils::datetime::Timestring(tp.GetUnderlying()); +} + +} // namespace storages::postgres + +USERVER_NAMESPACE_END diff --git a/scripts/docs/en/userver/chaotic.md b/scripts/docs/en/userver/chaotic.md index 8eafa17ddd0a..f6efe8c0d9e2 100644 --- a/scripts/docs/en/userver/chaotic.md +++ b/scripts/docs/en/userver/chaotic.md @@ -362,6 +362,8 @@ in case of `x-usrv-cpp-type: X::Y`. The header must contain: 2) `Convert` functions (see below). `Convert` function is used to transform user type into JSONSchema type and vice versa. +You have to pass `LINK_TARGETS` parameter to `userver_target_generate_chaotic` to link with a target that provides the required header. + @include chaotic/integration_tests/include/userver/chaotic/io/my/custom_string.hpp From 1ff442e7dffd3df3b34f18fbfea345c5c5e5946f Mon Sep 17 00:00:00 2001 From: antoshkka Date: Mon, 4 Aug 2025 15:17:50 +0300 Subject: [PATCH 043/151] feat docs: update docs to answer more user questions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI commit_hash:0a5f357cc79069a84523100aa0b8186f950e63ce --- .../userver/storages/clickhouse/cluster.hpp | 3 ++ .../src/storages/tests/escape_chtest.cpp | 2 ++ .../metrics/tests/static/metrics_values.txt | 2 +- core/include/userver/alerts/source.hpp | 19 +++++++--- .../components/minimal_component_list.hpp | 1 - .../userver/utils/statistics/metric_tag.hpp | 17 +++++++-- core/src/alerts/source.cpp | 8 +++-- core/src/dynamic_config/storage/component.cpp | 2 +- core/src/logging/component.cpp | 35 ++++++++++++------- .../userver/storages/postgres/component.hpp | 2 +- scripts/docs/en/userver/dynamic_config.md | 3 +- scripts/docs/en/userver/functional_testing.md | 1 + scripts/docs/en/userver/sql_files.md | 13 +++++-- .../en/userver/tutorial/production_service.md | 2 +- .../pytest_userver/plugins/sql_coverage.py | 35 ++++++++++++++++--- 15 files changed, 109 insertions(+), 36 deletions(-) diff --git a/clickhouse/include/userver/storages/clickhouse/cluster.hpp b/clickhouse/include/userver/storages/clickhouse/cluster.hpp index 9b617a071b06..b895785f0590 100644 --- a/clickhouse/include/userver/storages/clickhouse/cluster.hpp +++ b/clickhouse/include/userver/storages/clickhouse/cluster.hpp @@ -65,6 +65,9 @@ class Cluster final { /// /// It is convinient to keep SQL queries in separate files, see @ref scripts/docs/en/userver/sql_files.md /// for more info. + /// + /// # Example usage: + /// @snippet clickhouse/src/storages/tests/escape_chtest.cpp basic_usage template ExecutionResult Execute(OptionalCommandControl, const Query& query, const Args&... args) const; diff --git a/clickhouse/src/storages/tests/escape_chtest.cpp b/clickhouse/src/storages/tests/escape_chtest.cpp index 6fbde4bbc377..79493d98c0a4 100644 --- a/clickhouse/src/storages/tests/escape_chtest.cpp +++ b/clickhouse/src/storages/tests/escape_chtest.cpp @@ -59,6 +59,7 @@ struct CppToClickhouse { UTEST(ExecuteWithArgs, Basic) { ClusterWrapper cluster{}; + /// [basic_usage] const storages::clickhouse::Query q{"SELECT {}, * FROM system.numbers limit {}"}; const auto result = cluster->Execute(q, "we", 5).As(); @@ -66,6 +67,7 @@ UTEST(ExecuteWithArgs, Basic) { EXPECT_EQ(result.strings.size(), 5); EXPECT_EQ(result.strings.front(), "we"); + /// [basic_usage] } UTEST(ExecuteWithArgs, DatesArgs) { diff --git a/core/functional_tests/metrics/tests/static/metrics_values.txt b/core/functional_tests/metrics/tests/static/metrics_values.txt index a981f2f2c402..694908a4c240 100644 --- a/core/functional_tests/metrics/tests/static/metrics_values.txt +++ b/core/functional_tests/metrics/tests/static/metrics_values.txt @@ -1,4 +1,4 @@ -# Engine related errors/alerts +# Alerts reporteed by instances of alerts::Source alerts.cache_update_error: GAUGE 0 alerts.config_parse_error: GAUGE 0 alerts.dynamic_debug_invalid_location: GAUGE 0 diff --git a/core/include/userver/alerts/source.hpp b/core/include/userver/alerts/source.hpp index 75b4b4b61da2..e65f53ddb8ac 100644 --- a/core/include/userver/alerts/source.hpp +++ b/core/include/userver/alerts/source.hpp @@ -4,6 +4,7 @@ /// @brief @copybrief alerts::Source #include +#include #include #include @@ -23,19 +24,29 @@ struct SourceData { void DumpMetric(utils::statistics::Writer& writer, const SourceData& m); } // namespace impl -/// @brief Alert source instance which is used to fire alerts via metrics. +/// @brief Alert source instance which is used to fire alerts via metrics for a specified amount of time. +/// +/// To declare an alert: +/// @snippet core/src/logging/component.cpp alert_declaration +/// +/// Tu fire or to stop an alert: +/// @snippet core/src/logging/component.cpp alert_usage +/// +/// For non alert-metrics consider using utils::statistics::MetricTag. class Source final { public: static constexpr std::chrono::seconds kDefaultDuration{120}; static constexpr std::chrono::hours kInfiniteDuration{24 * 365 * 10}; // In 10 years, someone should notice. - explicit Source(const std::string& name); + /// Constructs an alert source instance that will be reported as non-zero "alerts." + std::string{name} metric + /// in case of error + explicit Source(std::string_view name); /// Fire alert for duration seconds. - void FireAlert(utils::statistics::MetricsStorage& storage, std::chrono::seconds duration = kDefaultDuration); + void FireAlert(utils::statistics::MetricsStorage& storage, std::chrono::seconds duration = kDefaultDuration) const; /// Stop fired alert - void StopAlertNow(utils::statistics::MetricsStorage& storage); + void StopAlertNow(utils::statistics::MetricsStorage& storage) const; private: utils::statistics::MetricTag tag_; diff --git a/core/include/userver/components/minimal_component_list.hpp b/core/include/userver/components/minimal_component_list.hpp index d670b92fdb03..2f2b6e3d88c2 100644 --- a/core/include/userver/components/minimal_component_list.hpp +++ b/core/include/userver/components/minimal_component_list.hpp @@ -20,7 +20,6 @@ namespace components { /// * components::Tracer /// * components::ManagerControllerComponent /// * components::StatisticsStorage -/// * alerts::StorageComponent /// * components::DynamicConfig /// * tracing::DefaultTracingManagerLocator ComponentList MinimalComponentList(); diff --git a/core/include/userver/utils/statistics/metric_tag.hpp b/core/include/userver/utils/statistics/metric_tag.hpp index 89c338fe8276..fbae8517bab4 100644 --- a/core/include/userver/utils/statistics/metric_tag.hpp +++ b/core/include/userver/utils/statistics/metric_tag.hpp @@ -1,5 +1,8 @@ #pragma once +/// @file userver/utils/statistics/metric_tag.hpp +/// @brief @copybrief utils::statistics::MetricTag + #include #include #include @@ -10,7 +13,7 @@ USERVER_NAMESPACE_BEGIN namespace utils::statistics { -/// @brief Metric description +/// @brief Metric description and registration in a declarative way /// /// Use `MetricTag` for declarative style of metric registration and /// call `MetricStorage::GetMetric` for accessing metric data. Please @@ -23,12 +26,22 @@ namespace utils::statistics { /// @code /// void DumpMetric(utils::statistics::Writer&, const Metric&) /// @endcode +/// +/// For alerts consider using alerts::Source. +/// +/// ## Example usage: +/// +/// @snippet samples/tcp_full_duplex_service/main.cpp TCP sample - Stats tag +/// where `Stats` are defined in a following way: +/// @snippet samples/tcp_full_duplex_service/main.cpp TCP sample - Stats definition +/// +/// For a full usage example see @ref samples/tcp_full_duplex_service/main.cpp template class MetricTag final { public: /// Register metric, passing a copy of `args` to the constructor of `Metric` template - explicit MetricTag(const std::string& path, Args&&... args) : key_{typeid(Metric), path} { + explicit MetricTag(std::string path, Args&&... args) : key_{typeid(Metric), std::move(path)} { impl::RegisterMetricInfo(key_, impl::MakeMetricFactory(std::forward(args)...)); } diff --git a/core/src/alerts/source.cpp b/core/src/alerts/source.cpp index e340a67775c3..0c680e4e28e2 100644 --- a/core/src/alerts/source.cpp +++ b/core/src/alerts/source.cpp @@ -1,5 +1,7 @@ #include +#include + USERVER_NAMESPACE_BEGIN namespace alerts { @@ -13,15 +15,15 @@ void DumpMetric(utils::statistics::Writer& writer, const SourceData& m) { } } // namespace impl -Source::Source(const std::string& name) : tag_("alerts." + name) {} +Source::Source(std::string_view name) : tag_(fmt::format("alerts.{}", name)) {} -void Source::FireAlert(utils::statistics::MetricsStorage& storage, std::chrono::seconds duration) { +void Source::FireAlert(utils::statistics::MetricsStorage& storage, std::chrono::seconds duration) const { auto& metric = storage.GetMetric(tag_); metric.stop_timepoint = std::chrono::steady_clock::now() + duration; metric.fired = true; } -void Source::StopAlertNow(utils::statistics::MetricsStorage& storage) { +void Source::StopAlertNow(utils::statistics::MetricsStorage& storage) const { auto& metric = storage.GetMetric(tag_); metric.fired = false; } diff --git a/core/src/dynamic_config/storage/component.cpp b/core/src/dynamic_config/storage/component.cpp index 09606294c7db..351724d2b164 100644 --- a/core/src/dynamic_config/storage/component.cpp +++ b/core/src/dynamic_config/storage/component.cpp @@ -37,7 +37,7 @@ namespace { constexpr std::chrono::seconds kWaitInterval(5); -alerts::Source kConfigParseErrorAlert("config_parse_error"); +const alerts::Source kConfigParseErrorAlert("config_parse_error"); struct DynamicConfigStatistics final { std::atomic was_last_parse_successful{true}; diff --git a/core/src/logging/component.cpp b/core/src/logging/component.cpp index e3fb5353267b..c265f972a8cd 100644 --- a/core/src/logging/component.cpp +++ b/core/src/logging/component.cpp @@ -39,7 +39,22 @@ void ReopenLoggerFile(const std::shared_ptr& logger) { logger->Reopen(logging::impl::ReopenMode::kAppend); } -alerts::Source kLogReopeningAlert("log_reopening_error"); +/// [alert_declaration] +const alerts::Source kLogReopeningAlert{"log_reopening_error"}; +/// [alert_declaration] + +void ReportReopeningErrorAndThrow( + const std::vector& failed_loggers, + const std::string& result_messages +) { + std::cerr << fmt::format( + "[{:%Y-%m-%d %H:%M:%S %Z}] loggers [{}] failed to reopen the log file: logs are getting lost now", + std::chrono::system_clock::now(), + fmt::join(failed_loggers, ", ") + ); + + throw std::runtime_error("ReopenAll errors: " + result_messages); +} } // namespace @@ -223,19 +238,15 @@ void Logging::TryReopenFiles() { } LOG_INFO() << "Log rotated"; - if (!result_messages.empty()) { + const bool error_happened = !result_messages.empty(); + /// [alert_usage] + if (error_happened) { kLogReopeningAlert.FireAlert(*metrics_storage_); - const auto now = std::chrono::system_clock::now(); - std::cerr << fmt::format( - "[{:%Y-%m-%d %H:%M:%S %Z}] loggers [{}] failed to reopen the log " - "file: logs are getting lost now", - now, - fmt::join(failed_loggers, ", ") - ); - - throw std::runtime_error("ReopenAll errors: " + result_messages); + ReportReopeningErrorAndThrow(failed_loggers, result_messages); + } else { + kLogReopeningAlert.StopAlertNow(*metrics_storage_); } - kLogReopeningAlert.StopAlertNow(*metrics_storage_); + /// [alert_usage] } void Logging::WriteStatistics(utils::statistics::Writer& writer) const { diff --git a/postgresql/include/userver/storages/postgres/component.hpp b/postgresql/include/userver/storages/postgres/component.hpp index 458b355ac2ec..54d7cd7c689f 100644 --- a/postgresql/include/userver/storages/postgres/component.hpp +++ b/postgresql/include/userver/storages/postgres/component.hpp @@ -73,7 +73,7 @@ namespace components { /// is just a list of DSNs and the Postgres component takes care of discovering /// the cluster's topology itself. /// -/// Note that if the components::Secdist component has `update-period` other +/// Note that if the `dbalias` option is provided and components::Secdist component has `update-period` other /// than 0, then new connections are created or gracefully closed as the secdist configuration change to new value. /// /// ### Predefined roles diff --git a/scripts/docs/en/userver/dynamic_config.md b/scripts/docs/en/userver/dynamic_config.md index f9c3d6d028e4..9955077afc14 100644 --- a/scripts/docs/en/userver/dynamic_config.md +++ b/scripts/docs/en/userver/dynamic_config.md @@ -465,8 +465,7 @@ then the periodic config update will fail, and alerts will fire: if ((1) != previous (1) || (2) == 0) show alert ``` -2. An alert will be registered in alerts::StorageComponent, which can be - accessed from outside using the optional alerts::Handler component +2. An alert will be fired in via alerts::Source with metric name name `alerts.config_parse_error`. If the config service is not accessible at this point (down or overloaded), then the periodic config update will also obviously fail. diff --git a/scripts/docs/en/userver/functional_testing.md b/scripts/docs/en/userver/functional_testing.md index 23ff8e8eb19e..dfd7634e3983 100644 --- a/scripts/docs/en/userver/functional_testing.md +++ b/scripts/docs/en/userver/functional_testing.md @@ -51,6 +51,7 @@ Then create testsuite target: * DUMP_CONFIG, set to `TRUE` to tell the testsuite that there is no static config file in the file system and force the testsuite to retrieve config from a service itself, by running it with `--dump-config` option first. See @ref scripts/docs/en/userver/libraries/easy.md for usage example. +* SQL_LIBRARY, target to enable @ref sql_coverage_test_info "SQL coverage tests". Some of the most useful arguments for PYTEST_ARGS: diff --git a/scripts/docs/en/userver/sql_files.md b/scripts/docs/en/userver/sql_files.md index e77777765aa2..50d877187f08 100644 --- a/scripts/docs/en/userver/sql_files.md +++ b/scripts/docs/en/userver/sql_files.md @@ -26,11 +26,14 @@ const USERVER_NAMESPACE::storages::Query kSelectValue = { R"-( SELECT value FROM key_value_table WHERE key=$1 )-", - USERVER_NAMESPACE::storages::Query::Name("select_value"), + USERVER_NAMESPACE::storages::Query::NameLiteral("select_value"), USERVER_NAMESPACE::storages::Query::LogMode::kFull, }; @endcode +Each variable is statically initialized (has no dynamic|runtime initializtion), giving a protection against static +intialization order fiasco when those variables are used. + You may use it as usual by passing to @ref storages::postgres::Cluster::Execute() or @ref storages::clickhouse::Cluster for SQL files or @ref ydb::TableClient::ExecuteDataQuery() for YQL files: @@ -44,10 +47,14 @@ namespace samples_postgres_service { } @endcode -While writing tests, you can check the coverage of your SQL/YQL queries using the `sql_coverage` plugin. +@anchor sql_coverage_test_info +## SQL coverage test + +While writing tests, you can check the coverage of your SQL/YQL queries using the +@ref pytest_userver.plugins.sql_coverage "sql_coverage" plugin. To use it, you need to pass the target with generated queries to the `userver_testsuite_add_simple` (or `userver_testsuite_add`) function -in your CMakeLists.txt: +in your CMakeLists.txt as `SQL_LIBRARY` parameter: @snippet samples/postgres_service/CMakeLists.txt Postgres sql coverage - CMakeLists.txt diff --git a/scripts/docs/en/userver/tutorial/production_service.md b/scripts/docs/en/userver/tutorial/production_service.md index f5b0b6d9d2f7..7e7dd9d32bca 100644 --- a/scripts/docs/en/userver/tutorial/production_service.md +++ b/scripts/docs/en/userver/tutorial/production_service.md @@ -192,7 +192,7 @@ List of userver built-in metrics could be found at ### Alerts -Alerts is a way to propagate critical errors from your service to a monitoring system. +Alerts is a way to propagate critical errors from your service to a monitoring system via @ref alerts::Source. When the code identifies that something bad happened and a user should be notified about that, `alert_storage.FireAlert()` is called with the appropriate arguments. Then the alert subsystem diff --git a/testsuite/pytest_plugins/pytest_userver/plugins/sql_coverage.py b/testsuite/pytest_plugins/pytest_userver/plugins/sql_coverage.py index 6e95712a2548..ae684feee36a 100644 --- a/testsuite/pytest_plugins/pytest_userver/plugins/sql_coverage.py +++ b/testsuite/pytest_plugins/pytest_userver/plugins/sql_coverage.py @@ -1,5 +1,8 @@ """ -Plugin that imports the required fixtures for checking SQL/YQL coverage. +Plugin that imports the required fixtures for checking SQL/YQL coverage. See +@ref sql_coverage_test_info "SQL coverage tests" for more info. + +@ingroup userver_testsuite_fixtures """ from typing import Set @@ -14,7 +17,14 @@ @pytest.fixture def on_uncovered(): """ - Will be called when the coverage is incomplete. + Called when the coverage is incomplete. + + Override this fixture to change the way uncovered statements are reported or to ignore some of the statements + from coverage report. + + See @ref sql_coverage_test_info "SQL coverage tests" for more info. + + @ingroup userver_testsuite_fixtures """ def _on_uncovered(uncovered_statements: Set[str]): @@ -47,14 +57,25 @@ def validate(self, uncovered_callback: callable) -> None: @pytest.fixture(scope='session') -def sql_coverage(sql_files): +def sql_coverage(sql_files) -> Coverage: + """ + Returns data about the current coverage of statements. + + See @ref sql_coverage_test_info "SQL coverage tests" for more info. + + @ingroup userver_testsuite_fixtures + """ return Coverage(set(sql_files)) @pytest.fixture(scope='function', autouse=True) async def sql_statement_hook(testpoint, sql_coverage): """ - Hook that accepts requests from the testpoint. + Hook that accepts requests from the testpoint with information on PostgreSQL statements coverage. + + See @ref sql_coverage_test_info "SQL coverage tests" for more info. + + @ingroup userver_testsuite_fixtures """ @testpoint('sql_statement') @@ -67,7 +88,11 @@ def _hook(request): @pytest.fixture(scope='function', autouse=True) async def yql_statement_hook(testpoint, sql_coverage): """ - Hook that accepts requests from the testpoint. + Hook that accepts requests from the testpoint with information on YDB statements coverage. + + See @ref sql_coverage_test_info "SQL coverage tests" for more info. + + @ingroup userver_testsuite_fixtures """ @testpoint('yql_statement') From 57bc470025f86858c15e07ad6802baaa2106085e Mon Sep 17 00:00:00 2001 From: abramov-alex Date: Tue, 5 Aug 2025 10:39:26 +0300 Subject: [PATCH 044/151] cc userver: rename kIsDetected to IsDetected Tests: CI
commit_hash:eaee212c80d5c5c6960147e6250fc4477cfde805 --- core/include/userver/dump/aggregates.hpp | 2 +- core/include/userver/dump/meta_containers.hpp | 2 +- .../userver/server/handlers/exceptions.hpp | 8 ++--- .../utils/statistics/metric_tag_impl.hpp | 4 +-- .../statistics/percentile_format_json.hpp | 2 +- .../userver/cache/mongo_cache_type_traits.hpp | 22 +++++++------- .../userver/storages/mysql/convert.hpp | 2 +- .../userver/cache/base_postgres_cache.hpp | 30 +++++++++---------- .../storages/postgres/io/type_traits.hpp | 4 +-- .../include/userver/formats/common/meta.hpp | 8 ++--- universal/include/userver/utils/get_if.hpp | 2 +- universal/include/userver/utils/meta.hpp | 16 +++++----- .../include/userver/utils/meta_light.hpp | 14 ++++----- .../include/userver/utils/projected_set.hpp | 2 +- universal/src/utils/meta_test.cpp | 4 +-- .../include/userver/utest/parameter_names.hpp | 4 +-- 16 files changed, 63 insertions(+), 63 deletions(-) diff --git a/core/include/userver/dump/aggregates.hpp b/core/include/userver/dump/aggregates.hpp index c22c4b86a89e..bfe2652be03f 100644 --- a/core/include/userver/dump/aggregates.hpp +++ b/core/include/userver/dump/aggregates.hpp @@ -33,7 +33,7 @@ constexpr bool AreAllDumpable(std::index_sequence) { template constexpr bool IsDumpableAggregate() { - if constexpr (std::is_aggregate_v && !meta::kIsDetected) { + if constexpr (std::is_aggregate_v && !meta::IsDetected) { constexpr auto kSize = boost::pfr::tuple_size_v; static_assert( AreAllDumpable(std::make_index_sequence{}), diff --git a/core/include/userver/dump/meta_containers.hpp b/core/include/userver/dump/meta_containers.hpp index 0c4b64dc6231..67280c0b7fd4 100644 --- a/core/include/userver/dump/meta_containers.hpp +++ b/core/include/userver/dump/meta_containers.hpp @@ -53,7 +53,7 @@ using InsertResult = decltype(dump::Insert(std::declval(), std::declval inline constexpr bool kIsContainer = meta::kIsRange && std::is_default_constructible_v && meta::kIsSizable && - meta::kIsDetected; + meta::IsDetected; } // namespace dump diff --git a/core/include/userver/server/handlers/exceptions.hpp b/core/include/userver/server/handlers/exceptions.hpp index 0abfde4700ed..4f4c0057102c 100644 --- a/core/include/userver/server/handlers/exceptions.hpp +++ b/core/include/userver/server/handlers/exceptions.hpp @@ -107,13 +107,13 @@ template using HasInternalMessage = decltype(std::declval().GetInternalMessage()); template -inline constexpr bool kHasInternalMessage = meta::kIsDetected; +inline constexpr bool kHasInternalMessage = meta::IsDetected; template using HasExternalBody = decltype(std::declval().GetExternalBody()); template -inline constexpr bool kHasExternalBody = meta::kIsDetected; +inline constexpr bool kHasExternalBody = meta::IsDetected; template inline constexpr bool kIsMessageBuilder = kHasExternalBody; @@ -121,7 +121,7 @@ inline constexpr bool kIsMessageBuilder = kHasExternalBody; template struct MessageExtractor { static_assert( - meta::kIsDetected, + meta::IsDetected, "Please use your message builder to build external body for " "your error. See server::handlers::CustomHandlerException " "for more info" @@ -134,7 +134,7 @@ struct MessageExtractor { } std::string GetServiceCode() const { - if constexpr (meta::kIsDetected) { + if constexpr (meta::IsDetected) { return builder.GetServiceCode(); } else { return std::string{}; diff --git a/core/include/userver/utils/statistics/metric_tag_impl.hpp b/core/include/userver/utils/statistics/metric_tag_impl.hpp index fc0cc4d0279e..48dbedd0ab03 100644 --- a/core/include/userver/utils/statistics/metric_tag_impl.hpp +++ b/core/include/userver/utils/statistics/metric_tag_impl.hpp @@ -63,7 +63,7 @@ class MetricWrapperBase { template class MetricWrapper final : public MetricWrapperBase { static_assert( - meta::kIsDetected || kHasWriterSupport, + meta::IsDetected || kHasWriterSupport, "Provide a `void DumpMetric(utils::statistics::Writer&, const Metric&)`" "function in the namespace of `Metric`." ); @@ -94,7 +94,7 @@ class MetricWrapper final : public MetricWrapperBase { bool HasWriterSupport() const noexcept override { return kHasWriterSupport; } void Reset() override { - if constexpr (meta::kIsDetected) { + if constexpr (meta::IsDetected) { ResetMetric(data_); } } diff --git a/core/include/userver/utils/statistics/percentile_format_json.hpp b/core/include/userver/utils/statistics/percentile_format_json.hpp index 1a2183891a0c..1c95566948dd 100644 --- a/core/include/userver/utils/statistics/percentile_format_json.hpp +++ b/core/include/userver/utils/statistics/percentile_format_json.hpp @@ -24,7 +24,7 @@ std::string GetPercentileFieldName(double perc); template formats::json::ValueBuilder PercentileToJson(const T& perc, std::initializer_list percents) { static_assert( - meta::kIsDetected, + meta::IsDetected, "T must specify T::GetPercentile(double) returning " "json-serializable value" ); diff --git a/mongo/include/userver/cache/mongo_cache_type_traits.hpp b/mongo/include/userver/cache/mongo_cache_type_traits.hpp index eaca914a7c02..86039e973e81 100644 --- a/mongo/include/userver/cache/mongo_cache_type_traits.hpp +++ b/mongo/include/userver/cache/mongo_cache_type_traits.hpp @@ -22,17 +22,17 @@ namespace mongo_cache::impl { template using CollectionsField = decltype(T::kMongoCollectionsField); template -inline constexpr bool kHasCollectionsField = meta::kIsDetected; +inline constexpr bool kHasCollectionsField = meta::IsDetected; template using UpdateFieldName = decltype(T::kMongoUpdateFieldName); template -inline constexpr bool kHasUpdateFieldName = meta::kIsDetected; +inline constexpr bool kHasUpdateFieldName = meta::IsDetected; template using KeyField = decltype(T::kKeyField); template -inline constexpr bool kHasKeyField = meta::kIsDetected; +inline constexpr bool kHasKeyField = meta::IsDetected; template using DataType = typename T::DataType; @@ -42,29 +42,29 @@ inline constexpr bool kHasValidDataType = meta::kIsMap using HasSecondaryPreferred = decltype(T::kIsSecondaryPreferred); template -inline constexpr bool kHasSecondaryPreferred = meta::kIsDetected; +inline constexpr bool kHasSecondaryPreferred = meta::IsDetected; template using HasDeserializeObject = decltype(T::DeserializeObject); template -inline constexpr bool kHasDeserializeObject = meta::kIsDetected; +inline constexpr bool kHasDeserializeObject = meta::IsDetected; template using HasCorrectDeserializeObject = meta::ExpectSame< typename T::ObjectType, decltype(std::declval().DeserializeObject(std::declval()))>; template -inline constexpr bool kHasCorrectDeserializeObject = meta::kIsDetected; +inline constexpr bool kHasCorrectDeserializeObject = meta::IsDetected; template using HasDefaultDeserializeObject = decltype(T::kUseDefaultDeserializeObject); template -inline constexpr bool kHasDefaultDeserializeObject = meta::kIsDetected; +inline constexpr bool kHasDefaultDeserializeObject = meta::IsDetected; template using HasFindOperation = decltype(T::GetFindOperation); template -inline constexpr bool kHasFindOperation = meta::kIsDetected; +inline constexpr bool kHasFindOperation = meta::IsDetected; template using HasCorrectFindOperation = meta::ExpectSame< @@ -77,17 +77,17 @@ using HasCorrectFindOperation = meta::ExpectSame< ))>; template -inline constexpr bool kHasCorrectFindOperation = meta::kIsDetected; +inline constexpr bool kHasCorrectFindOperation = meta::IsDetected; template using HasDefaultFindOperation = decltype(T::kUseDefaultFindOperation); template -inline constexpr bool kHasDefaultFindOperation = meta::kIsDetected; +inline constexpr bool kHasDefaultFindOperation = meta::IsDetected; template using HasInvalidDocumentsSkipped = decltype(T::kAreInvalidDocumentsSkipped); template -inline constexpr bool kHasInvalidDocumentsSkipped = meta::kIsDetected; +inline constexpr bool kHasInvalidDocumentsSkipped = meta::IsDetected; template struct ClassByMemberPointer {}; diff --git a/mysql/include/userver/storages/mysql/convert.hpp b/mysql/include/userver/storages/mysql/convert.hpp index 19caee7baf60..d8fb52f199e7 100644 --- a/mysql/include/userver/storages/mysql/convert.hpp +++ b/mysql/include/userver/storages/mysql/convert.hpp @@ -18,7 +18,7 @@ template using HasConvert = decltype(Convert(std::declval(), convert::To{})); template -inline constexpr bool kHasConvert = meta::kIsDetected; +inline constexpr bool kHasConvert = meta::IsDetected; } // namespace impl diff --git a/postgresql/include/userver/cache/base_postgres_cache.hpp b/postgresql/include/userver/cache/base_postgres_cache.hpp index 5e4a603308c5..d4c90b72be56 100644 --- a/postgresql/include/userver/cache/base_postgres_cache.hpp +++ b/postgresql/include/userver/cache/base_postgres_cache.hpp @@ -126,12 +126,12 @@ namespace pg_cache::detail { template using ValueType = typename T::ValueType; template -inline constexpr bool kHasValueType = meta::kIsDetected; +inline constexpr bool kHasValueType = meta::IsDetected; template using RawValueTypeImpl = typename T::RawValueType; template -inline constexpr bool kHasRawValueType = meta::kIsDetected; +inline constexpr bool kHasRawValueType = meta::IsDetected; template using RawValueType = meta::DetectedOr, RawValueTypeImpl, T>; @@ -148,48 +148,48 @@ auto ExtractValue(RawValueType&& raw) { template using HasNameImpl = std::enable_if_t; template -inline constexpr bool kHasName = meta::kIsDetected; +inline constexpr bool kHasName = meta::IsDetected; // Component query in policy template using HasQueryImpl = decltype(T::kQuery); template -inline constexpr bool kHasQuery = meta::kIsDetected; +inline constexpr bool kHasQuery = meta::IsDetected; // Component GetQuery in policy template using HasGetQueryImpl = decltype(T::GetQuery()); template -inline constexpr bool kHasGetQuery = meta::kIsDetected; +inline constexpr bool kHasGetQuery = meta::IsDetected; // Component kWhere in policy template using HasWhere = decltype(T::kWhere); template -inline constexpr bool kHasWhere = meta::kIsDetected; +inline constexpr bool kHasWhere = meta::IsDetected; // Component kOrderBy in policy template using HasOrderBy = decltype(T::kOrderBy); template -inline constexpr bool kHasOrderBy = meta::kIsDetected; +inline constexpr bool kHasOrderBy = meta::IsDetected; // Update field template using HasUpdatedField = decltype(T::kUpdatedField); template -inline constexpr bool kHasUpdatedField = meta::kIsDetected; +inline constexpr bool kHasUpdatedField = meta::IsDetected; template using WantIncrementalUpdates = std::enable_if_t; template -inline constexpr bool kWantIncrementalUpdates = meta::kIsDetected; +inline constexpr bool kWantIncrementalUpdates = meta::IsDetected; // Key member in policy template using KeyMemberTypeImpl = std::decay_t>>; template -inline constexpr bool kHasKeyMember = meta::kIsDetected; +inline constexpr bool kHasKeyMember = meta::IsDetected; template using KeyMemberType = meta::DetectedType; @@ -252,7 +252,7 @@ using HasOnWritesDoneImpl = decltype(std::declval().OnWritesDone()); template void OnWritesDone(T& container) { - if constexpr (meta::kIsDetected) { + if constexpr (meta::IsDetected) { container.OnWritesDone(); } } @@ -261,12 +261,12 @@ template using HasCustomUpdatedImpl = decltype(T::GetLastKnownUpdated(std::declval>())); template -inline constexpr bool kHasCustomUpdated = meta::kIsDetected; +inline constexpr bool kHasCustomUpdated = meta::IsDetected; template using UpdatedFieldTypeImpl = typename T::UpdatedFieldType; template -inline constexpr bool kHasUpdatedFieldType = meta::kIsDetected; +inline constexpr bool kHasUpdatedFieldType = meta::IsDetected; template using UpdatedFieldType = meta::DetectedOr; @@ -307,7 +307,7 @@ using HasClusterHostTypeImpl = decltype(T::kClusterHostType); template constexpr storages::postgres::ClusterHostTypeFlags ClusterHostType() { - if constexpr (meta::kIsDetected) { + if constexpr (meta::IsDetected) { return T::kClusterHostType; } else { return storages::postgres::ClusterHostType::kSlave; @@ -320,7 +320,7 @@ using HasMayReturnNull = decltype(T::kMayReturnNull); template constexpr bool MayReturnNull() { - if constexpr (meta::kIsDetected) { + if constexpr (meta::IsDetected) { return T::kMayReturnNull; } else { return false; diff --git a/postgresql/include/userver/storages/postgres/io/type_traits.hpp b/postgresql/include/userver/storages/postgres/io/type_traits.hpp index 6b965f34138e..2d43f053d2be 100644 --- a/postgresql/include/userver/storages/postgres/io/type_traits.hpp +++ b/postgresql/include/userver/storages/postgres/io/type_traits.hpp @@ -58,13 +58,13 @@ template using HasOutputOperator = decltype(std::declval() << std::declval()); template -inline constexpr bool kHasOutputOperator = meta::kIsDetected; +inline constexpr bool kHasOutputOperator = meta::IsDetected; template using HasInputOperator = decltype(std::declval() >> std::declval()); template -inline constexpr bool kHasInputOperator = meta::kIsDetected; +inline constexpr bool kHasInputOperator = meta::IsDetected; ///@} ///@{ diff --git a/universal/include/userver/formats/common/meta.hpp b/universal/include/userver/formats/common/meta.hpp index e0388a632c37..36ef82b861f8 100644 --- a/universal/include/userver/formats/common/meta.hpp +++ b/universal/include/userver/formats/common/meta.hpp @@ -34,20 +34,20 @@ template using IsFormatValue = typename Value::ParseException; template -constexpr inline bool kHasParse = meta::kIsDetected; +constexpr inline bool kHasParse = meta::IsDetected; template -constexpr inline bool kHasSerialize = meta::kIsDetected; +constexpr inline bool kHasSerialize = meta::IsDetected; template -constexpr inline bool kHasConvert = meta::kIsDetected; +constexpr inline bool kHasConvert = meta::IsDetected; } // namespace impl /// Used in `Parse` overloads that are templated on `Value`, avoids clashing /// with `Parse` from string template -constexpr inline bool kIsFormatValue = meta::kIsDetected; +constexpr inline bool kIsFormatValue = meta::IsDetected; // Unwraps a transient type - tag types, for which ADL-found `Parse` returns // another type, not the type specified in `formats::parse::To`. For example, diff --git a/universal/include/userver/utils/get_if.hpp b/universal/include/userver/utils/get_if.hpp index 8c4aa128f4c1..5577ebec5dc5 100644 --- a/universal/include/userver/utils/get_if.hpp +++ b/universal/include/userver/utils/get_if.hpp @@ -20,7 +20,7 @@ template using IsPointerLike = decltype(std::declval() ? std::addressof(*std::declval()) : nullptr); template -inline constexpr bool kIsPointerLike = meta::kIsDetected; +inline constexpr bool kIsPointerLike = meta::IsDetected; } // namespace impl diff --git a/universal/include/userver/utils/meta.hpp b/universal/include/userver/utils/meta.hpp index 84bc7fd3403c..cefac2746522 100644 --- a/universal/include/userver/utils/meta.hpp +++ b/universal/include/userver/utils/meta.hpp @@ -32,7 +32,7 @@ using IsRange = ExpectSame()))>, std::decay_t()))>>; template -using IteratorType = std::enable_if_t, decltype(begin(std::declval()))>; +using IteratorType = std::enable_if_t, decltype(begin(std::declval()))>; template struct IsIterator : std::false_type {}; @@ -77,7 +77,7 @@ struct IsFixedSizeContainer> : std::bool_constant constexpr bool IsSingleRange() { if constexpr (sizeof...(Args) == 1) { - return kIsDetected; + return IsDetected; } else { return false; } @@ -89,16 +89,16 @@ template inline constexpr bool kIsVector = kIsInstantiationOf; template -inline constexpr bool kIsRange = kIsDetected; +inline constexpr bool kIsRange = IsDetected; /// Returns true if T is an ordered or unordered map or multimap template inline constexpr bool kIsMap = - kIsDetected && kIsDetected && kIsDetected; + IsDetected && IsDetected && IsDetected; /// Returns true if T is a map (but not a multimap!) template -inline constexpr bool kIsUniqueMap = kIsMap && kIsDetected< +inline constexpr bool kIsUniqueMap = kIsMap && IsDetected< impl::SubscriptOperatorResult, T>; // no operator[] in multimaps @@ -132,15 +132,15 @@ inline constexpr bool kIsStdHashable = /// @brief Check if std::size is applicable to container template -inline constexpr bool kIsSizable = kIsDetected; +inline constexpr bool kIsSizable = IsDetected; /// @brief Check if a container has `reserve` template -inline constexpr bool kIsReservable = kIsDetected; +inline constexpr bool kIsReservable = IsDetected; /// @brief Check if a container has 'push_back' template -inline constexpr bool kIsPushBackable = kIsDetected; +inline constexpr bool kIsPushBackable = IsDetected; /// @brief Check if a container has fixed size (e.g. std::array) template diff --git a/universal/include/userver/utils/meta_light.hpp b/universal/include/userver/utils/meta_light.hpp index 1c3fc86de0a2..25febbf73fcb 100644 --- a/universal/include/userver/utils/meta_light.hpp +++ b/universal/include/userver/utils/meta_light.hpp @@ -36,7 +36,7 @@ struct IsInstantiationOf> : std::true_type {}; } // namespace impl -/// @see utils::meta::kIsDetected +/// @see utils::meta::IsDetected struct NotDetected {}; #if defined(__cpp_concepts) || defined(DOXYGEN) @@ -53,31 +53,31 @@ struct NotDetected {}; /// template /// using HasValueType = typename T::ValueType; /// ... -/// if constexpr (utils::meta::kIsDetected) { ... } +/// if constexpr (utils::meta::IsDetected) { ... } /// @endcode template