diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8d83c31..012a480 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.12] fail-fast: false steps: diff --git a/.github/workflows/docs_publish.yml b/.github/workflows/docs_publish.yml index 4d25df9..41ef192 100644 --- a/.github/workflows/docs_publish.yml +++ b/.github/workflows/docs_publish.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.12] fail-fast: false steps: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index eeacb09..27a504e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -8,15 +8,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - pydantic-version: ["<2.0.0", ">=2.0.0"] - exclude: - - python-version: "3.9" - pydantic-version: "<2.0.0" - - python-version: "3.11" - pydantic-version: "<2.0.0" - - python-version: "3.12" - pydantic-version: "<2.0.0" + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + # pydantic-version: ["<2.0.0", ">=2.0.0"] + # exclude: + # - python-version: "3.9" + # pydantic-version: "<2.0.0" + # - python-version: "3.11" + # pydantic-version: "<2.0.0" + # - python-version: "3.12" + # pydantic-version: "<2.0.0" fail-fast: false @@ -56,8 +56,8 @@ jobs: pip install --upgrade pip pip install . pip install -r requirements-dev.txt - pip install "pydantic${{ matrix.pydantic-version }}" - pip install bluesky==1.11.0 + # pip install "pydantic${{ matrix.pydantic-version }}" + # pip install bluesky==1.11.0 pip list - name: Test with pytest diff --git a/bluesky_httpserver/app.py b/bluesky_httpserver/app.py index a659784..4779f4c 100644 --- a/bluesky_httpserver/app.py +++ b/bluesky_httpserver/app.py @@ -307,9 +307,12 @@ async def purge_expired_sessions_and_api_keys(): "QSERVER_ZMQ_INFO_ADDRESS to pass address of 0MQ information socket to HTTP Server." ) + zmq_encoding = os.getenv("QSERVER_ZMQ_ENCODING", None) + # Check if ZMQ setting were specified in config file. Overrid the parameters from EVs. zmq_control_addr = server_settings["qserver_zmq_configuration"].get("control_address", zmq_control_addr) zmq_info_addr = server_settings["qserver_zmq_configuration"].get("info_address", zmq_info_addr) + zmq_encoding = server_settings["qserver_zmq_configuration"].get("encoding", zmq_encoding) # Read public key from the environment variable or config file. zmq_public_key = os.environ.get("QSERVER_ZMQ_PUBLIC_KEY", None) @@ -323,12 +326,13 @@ async def purge_expired_sessions_and_api_keys(): logger.info( f"Connecting to RE Manager: \nControl 0MQ socket address: {zmq_control_addr}\n" - f"Information 0MQ socket address: {zmq_info_addr}" + f"Information 0MQ socket address: {zmq_info_addr}. Encoding: {zmq_encoding!r}\n" ) RM = REManagerAPI( zmq_control_addr=zmq_control_addr, zmq_info_addr=zmq_info_addr, + zmq_encoding=zmq_encoding, zmq_public_key=zmq_public_key, request_fail_exceptions=False, status_expiration_period=0.4, # Make it smaller than default diff --git a/bluesky_httpserver/config_schemas/service_configuration.yml b/bluesky_httpserver/config_schemas/service_configuration.yml index c2502c8..57343f7 100644 --- a/bluesky_httpserver/config_schemas/service_configuration.yml +++ b/bluesky_httpserver/config_schemas/service_configuration.yml @@ -29,6 +29,10 @@ properties: The address of 0MQ info (PUB-SUB) socket of RE Manager, used to publish console output e.g. tcp://localhost:60625. Overrides the address set using QSERVER_ZMQ_INFO_ADDRESS environment variable + encoding: + type: string + description: | + The encoding for 0MQ messages. Supported values: 'json' (default) and 'msgpack'. public_key: type: string description: | diff --git a/bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py b/bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py index 09ff6b0..6e1fdeb 100644 --- a/bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py +++ b/bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py @@ -1,4 +1,4 @@ -""""Add write scopes to default Roles." +""" "Add write scopes to default Roles." Revision ID: 722ff4e4fcc7 Revises: 481830dd6c11 diff --git a/bluesky_httpserver/schemas.py b/bluesky_httpserver/schemas.py index 5da0619..c52d8f2 100644 --- a/bluesky_httpserver/schemas.py +++ b/bluesky_httpserver/schemas.py @@ -144,6 +144,7 @@ class NodeMeta(pydantic.BaseModel): class Resource(generic_model, Generic[AttributesT, ResourceLinksT, ResourceMetaT]): "A JSON API Resource" + id: Union[str, uuid.UUID] attributes: AttributesT links: Optional[ResourceLinksT] = None @@ -257,6 +258,7 @@ class Session(pydantic.BaseModel, **orm): class Principal(pydantic.BaseModel, **orm): "Represents a User or Service" + # The id field (primary key) is intentionally not exposed to the application. # It is left as an internal database concern. uuid: uuid.UUID @@ -280,6 +282,7 @@ def from_orm(cls, orm, latest_activity=None): class AllowedScopes(pydantic.BaseModel, **orm): "Returns roles and current allowed scopes for a user authenticated with API key or token" + roles: List[str] = [] scopes: List[str] = [] diff --git a/bluesky_httpserver/tests/conftest.py b/bluesky_httpserver/tests/conftest.py index d8d75e2..ec69415 100644 --- a/bluesky_httpserver/tests/conftest.py +++ b/bluesky_httpserver/tests/conftest.py @@ -4,6 +4,7 @@ import pytest import requests from bluesky_queueserver.manager.comms import zmq_single_request +from bluesky_queueserver.manager.tests.common import set_qserver_zmq_encoding # noqa: F401 from xprocess import ProcessStarter import bluesky_httpserver.server as bqss @@ -44,6 +45,8 @@ def fastapi_server_fs(xprocess): def start(http_server_host=SERVER_ADDRESS, http_server_port=SERVER_PORT, api_key=API_KEY_FOR_TESTS): class Starter(ProcessStarter): + max_read_lines = 53 + env = dict(os.environ) if api_key: env["QSERVER_HTTP_SERVER_SINGLE_USER_API_KEY"] = api_key diff --git a/bluesky_httpserver/tests/test_console_output.py b/bluesky_httpserver/tests/test_console_output.py index 55c613d..df3e877 100644 --- a/bluesky_httpserver/tests/test_console_output.py +++ b/bluesky_httpserver/tests/test_console_output.py @@ -14,6 +14,7 @@ SERVER_PORT, fastapi_server_fs, request_to_json, + set_qserver_zmq_encoding, wait_for_environment_to_be_closed, wait_for_environment_to_be_created, wait_for_manager_state_idle, @@ -155,20 +156,28 @@ def test_http_server_stream_console_output_1( """ +@pytest.mark.parametrize("zmq_encoding", (None, "json", "msgpack")) @pytest.mark.parametrize("zmq_port", (None, 60619)) -def test_http_server_console_output_1(monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port): # noqa F811 +def test_http_server_console_output_1( + monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port, zmq_encoding # noqa F811 +): """ Test for ``console_output`` API (not a streaming version). """ # Start HTTP Server if zmq_port is not None: monkeypatch.setenv("QSERVER_ZMQ_INFO_ADDRESS", f"tcp://localhost:{zmq_port}") + if zmq_encoding is not None: + monkeypatch.setenv("QSERVER_ZMQ_ENCODING", zmq_encoding) fastapi_server_fs() # Start RE Manager - params = ["--zmq-publish-console", "ON"] + params = ["--zmq-publish-console=ON"] + if zmq_encoding: + params.append(f"--zmq-encoding={zmq_encoding}") + set_qserver_zmq_encoding(monkeypatch, encoding=zmq_encoding) if zmq_port is not None: - params.extend(["--zmq-info-addr", f"tcp://*:{zmq_port}"]) + params.extend([f"--zmq-info-addr=tcp://*:{zmq_port}"]) re_manager_cmd(params) script = _script1 diff --git a/docs/source/conf.py b/docs/source/conf.py index 29c7c93..0f17058 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -40,7 +40,7 @@ 'sphinx.ext.viewcode', 'IPython.sphinxext.ipython_directive', 'IPython.sphinxext.ipython_console_highlighting', - 'matplotlib.sphinxext.plot_directive', + # 'matplotlib.sphinxext.plot_directive', 'numpydoc', ] @@ -108,7 +108,7 @@ html_theme = 'sphinx_rtd_theme' import sphinx_rtd_theme -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e6c8cd8..0ed8796 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -100,6 +100,9 @@ The following environment variables are used to configure 0MQ communication sett pass the private key to RE Manager. Pass the public key to HTTP Server using ``QSERVER_ZMQ_PUBLIC_KEY`` environment variable. +- ``QSERVER_ZMQ_ENCODING`` - sets the encoding for the messages set to RE Manager over + 0MQ. Supported values: ``"json"`` (default) or ``"msgpack"``. + The same parameters can be specified by including ``qserver_zmq_configuration`` into the config file:: @@ -107,6 +110,7 @@ the config file:: control_address: tcp://localhost:60615 info_address: tcp://localhost:60625 public_key: ${PUBLIC_KEY} + encoding: json All parameters in the config file are optional and override the values passed using environment variables and the default values. The public key is typically passed using environment diff --git a/requirements-dev.txt b/requirements-dev.txt index 9c02702..dd7212a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,6 +13,8 @@ py sphinx ipython numpydoc +matplotlib +pandas sphinx sphinx_rtd_theme requests diff --git a/start_LDAP.sh b/start_LDAP.sh index eca0e38..d71c662 100644 --- a/start_LDAP.sh +++ b/start_LDAP.sh @@ -4,5 +4,5 @@ set -e # Start LDAP server in docker container sudo docker pull bitnami/openldap:latest -sudo docker-compose -f .github/workflows/docker-configs/ldap-docker-compose.yml up -d +sudo docker compose -f .github/workflows/docker-configs/ldap-docker-compose.yml up -d sudo docker ps