Fixes aiopg hanging or raising psycopg2.OperationalError suddenly in gh-actions CI (#2138)

* Replaces localhost by 127.0.0.1 in all settings for pg hostname

* Patch in service-library

* Patches posgres-database

* patches webserver

Author: pcrespov

packages/postgres-database/src/simcore_postgres_database/utils.py

@@ -14,12 +14,12 @@ def build_url(
     database: str = "",
     user: str = "",
     password: str = "",
-    host: str = "localhost",
+    host: str = "127.0.0.1",
     port: int = 5432,
     **_kwargs,
 ) -> URL:
     """
-       Safe build pg url as 'postgresql+psycopg2://{user}:{password}@{host}:{port}/{database}'
+    Safe build pg url as 'postgresql+psycopg2://{user}:{password}@{host}:{port}/{database}'
     """
     dsn = URL.build(
         scheme="postgresql+psycopg2",

packages/postgres-database/tests/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       POSTGRES_USER: test
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
-      POSTGRES_HOST: localhost
+      POSTGRES_HOST: 127.0.0.1
       POSTGRES_PORT: 5432
     # NOTE: DO NOT ADD PERSISTENCE!
     ports:

packages/postgres-database/tests/test_utils.py

@@ -1,13 +1,12 @@
-from yarl import URL
-
 from simcore_postgres_database.utils import hide_dict_pass, hide_url_pass
+from yarl import URL
 
 
 def test_hide_url_pass():
 
     assert (
-        hide_url_pass(URL("postgres://username:password@localhost/myrailsdb"))
-        == "postgres://username:********@localhost/myrailsdb"
+        hide_url_pass(URL("postgres://username:password@127.0.0.1/myrailsdb"))
+        == "postgres://username:********@127.0.0.1/myrailsdb"
     )
 
 

packages/service-library/src/servicelib/aiopg_utils.py

@@ -43,7 +43,7 @@ class DataSourceName:
     user: str
     password: str = attr.ib(repr=False)
     database: str
-    host: str = "localhost"
+    host: str = "127.0.0.1"
     port: int = 5432
 
     # Attributes about the caller
@@ -60,14 +60,14 @@ def to_uri(self, with_query=False) -> str:
 
 
 def create_pg_engine(dsn: DataSourceName, minsize: int = 1, maxsize: int = 4):
-    """ Adapts the arguments of aiopg.sa.create_engine
+    """Adapts the arguments of aiopg.sa.create_engine
 
-        Returns a coroutine that is awaitable, i.e.
+    Returns a coroutine that is awaitable, i.e.
 
-        async with create_pg_engine as engine:
-            assert not engine.closed
+    async with create_pg_engine as engine:
+        assert not engine.closed
 
-        assert engine.closed
+    assert engine.closed
     """
     awaitable_engine_coro = create_engine(
         dsn.to_uri(),
@@ -154,8 +154,7 @@ def raise_http_unavailable_error(retry_state: RetryCallState):
 
 
 class PostgresRetryPolicyUponInitialization:
-    """ Retry policy upon service initialization
-    """
+    """Retry policy upon service initialization"""
 
     WAIT_SECS = 5
     ATTEMPTS_COUNT = 20
@@ -172,8 +171,7 @@ def __init__(self, logger: Optional[logging.Logger] = None):
 
 
 class PostgresRetryPolicyUponOperation:
-    """ Retry policy upon service operation
-    """
+    """Retry policy upon service operation"""
 
     WAIT_SECS = 2
     ATTEMPTS_COUNT = 3
@@ -195,8 +193,8 @@ def __init__(self, logger: Optional[logging.Logger] = None):
 
 
 def retry_pg_api(func):
-    """ Decorator to implement postgres service retry policy and
-        keep global  statistics on service attempt fails
+    """Decorator to implement postgres service retry policy and
+    keep global  statistics on service attempt fails
     """
     # TODO: temporary. For the time being, use instead postgres_service_retry_policy_kwargs
     _deco_func = retry(**postgres_service_retry_policy_kwargs)(func)

packages/service-library/tests/with_postgres/test_aiopg_utils.py

@@ -14,7 +14,6 @@
 import pytest
 import sqlalchemy as sa
 from aiohttp import web
-
 from servicelib.aiopg_utils import (
     DatabaseError,
     DataSourceName,
@@ -141,7 +140,7 @@ async def test_engine_when_idle_for_some_time():
 
 async def test_engine_when_pg_not_reachable(loop):
     dsn = DataSourceName(
-        database="db", user="foo", password="foo", host="localhost", port=123
+        database="db", user="foo", password="foo", host="127.0.0.1", port=123
     )
 
     with pytest.raises(psycopg2.OperationalError):

services/api-server/.env-devel

@@ -13,7 +13,7 @@ LOG_LEVEL=DEBUG
 POSTGRES_USER=test
 POSTGRES_PASSWORD=test
 POSTGRES_DB=test
-POSTGRES_HOST=localhost
+POSTGRES_HOST=127.0.0.1
 
 # Enables debug
 SC_BOOT_MODE=debug-ptvsd

services/api-server/tests/unit/conftest.py

@@ -37,7 +37,7 @@ def environment() -> Dict:
     env = {
         "WEBSERVER_HOST": "webserver",
         "WEBSERVER_SESSION_SECRET_KEY": "REPLACE ME with a key of at least length 32.",
-        "POSTGRES_HOST": "localhost",
+        "POSTGRES_HOST": "127.0.0.1",
         "POSTGRES_USER": "test",
         "POSTGRES_PASSWORD": "test",
         "POSTGRES_DB": "test",

services/api-server/tests/unit/test_settings.py

@@ -19,4 +19,4 @@ def test_min_environ_for_settings(project_env_devel_environment, monkeypatch):
     assert settings.boot_mode == BootModeEnum.PRODUCTION
     assert settings.loglevel == logging.DEBUG
 
-    assert settings.postgres.dsn == URL("postgresql://test:test@localhost:5432/test")
+    assert settings.postgres.dsn == URL("postgresql://test:test@127.0.0.1:5432/test")

services/api-server/tests/utils/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       - POSTGRES_USER=${POSTGRES_USER:-test}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test}
       - POSTGRES_DB=${POSTGRES_DB:-test}
-      - POSTGRES_HOST=${POSTGRES_HOST:-localhost}
+      - POSTGRES_HOST=${POSTGRES_HOST:-127.0.0.1}
       - POSTGRES_PORT=${POSTGRES_PORT:-5432}
     ports:
       - "5432:5432"

services/api-server/tests/utils/init-pg.py

@@ -8,18 +8,17 @@
 
 import aiopg.sa
 import faker
-import sqlalchemy as sa
-import yaml
-
 import simcore_postgres_database.cli as pg_cli
 import simcore_service_api_server.db.tables as pg
+import sqlalchemy as sa
+import yaml
 
 DSN_FORMAT = "postgresql://{user}:{password}@{host}:{port}/{database}"
 
 default_db_settings = dict(
     user=os.environ.get("POSTGRES_USER", "test"),
     password=os.environ.get("POSTGRES_PASSWORD", "test"),
-    host=os.environ.get("POSTGRES_HOST", "localhost"),
+    host=os.environ.get("POSTGRES_HOST", "127.0.0.1"),
     port=os.environ.get("POSTGRES_PORT", 5432),
     database=os.environ.get("POSTGRES_DB", 5432),
 )
@@ -37,7 +36,7 @@ def load_db_config() -> Dict:
     return dict(
         user=environ["POSTGRES_USER"],
         password=environ["POSTGRES_PASSWORD"],
-        host="localhost",
+        host="127.0.0.1",
         port=5432,
         database=environ["POSTGRES_DB"],
     )
@@ -64,7 +63,10 @@ def random_user(**overrides):
 
 def random_api_key(**overrides):
     data = dict(
-        user_id=1, display_name=fake.word(), api_key=uuid4(), api_secret=uuid4(),
+        user_id=1,
+        display_name=fake.word(),
+        api_key=uuid4(),
+        api_secret=uuid4(),
     )
     data.update(overrides)
     return data