Add support for 0MQ encoding

.github/workflows/docs.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
       fail-fast: false
 
     steps:

.github/workflows/docs_publish.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
       fail-fast: false
 
     steps:

.github/workflows/testing.yml

@@ -8,15 +8,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-        pydantic-version: ["<2.0.0", ">=2.0.0"]
-        exclude:
-          - python-version: "3.9"
-            pydantic-version: "<2.0.0"
-          - python-version: "3.11"
-            pydantic-version: "<2.0.0"
-          - python-version: "3.12"
-            pydantic-version: "<2.0.0"
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        # pydantic-version: ["<2.0.0", ">=2.0.0"]
+        # exclude:
+        #   - python-version: "3.9"
+        #     pydantic-version: "<2.0.0"
+        #   - python-version: "3.11"
+        #     pydantic-version: "<2.0.0"
+        #   - python-version: "3.12"
+        #     pydantic-version: "<2.0.0"
 
       fail-fast: false
 
@@ -56,8 +56,8 @@ jobs:
         pip install --upgrade pip
         pip install .
         pip install -r requirements-dev.txt
-        pip install "pydantic${{ matrix.pydantic-version }}"
-        pip install bluesky==1.11.0
+        # pip install "pydantic${{ matrix.pydantic-version }}"
+        # pip install bluesky==1.11.0
 
         pip list
     - name: Test with pytest

bluesky_httpserver/app.py

@@ -307,9 +307,12 @@ async def purge_expired_sessions_and_api_keys():
                     "QSERVER_ZMQ_INFO_ADDRESS to pass address of 0MQ information socket to HTTP Server."
                 )
 
+        zmq_encoding = os.getenv("QSERVER_ZMQ_ENCODING", None)
+
         # Check if ZMQ setting were specified in config file. Overrid the parameters from EVs.
         zmq_control_addr = server_settings["qserver_zmq_configuration"].get("control_address", zmq_control_addr)
         zmq_info_addr = server_settings["qserver_zmq_configuration"].get("info_address", zmq_info_addr)
+        zmq_encoding = server_settings["qserver_zmq_configuration"].get("encoding", zmq_encoding)
 
         # Read public key from the environment variable or config file.
         zmq_public_key = os.environ.get("QSERVER_ZMQ_PUBLIC_KEY", None)
@@ -323,12 +326,13 @@ async def purge_expired_sessions_and_api_keys():
 
         logger.info(
             f"Connecting to RE Manager: \nControl 0MQ socket address: {zmq_control_addr}\n"
-            f"Information 0MQ socket address: {zmq_info_addr}"
+            f"Information 0MQ socket address: {zmq_info_addr}. Encoding: {zmq_encoding!r}\n"
         )
 
         RM = REManagerAPI(
             zmq_control_addr=zmq_control_addr,
             zmq_info_addr=zmq_info_addr,
+            zmq_encoding=zmq_encoding,
             zmq_public_key=zmq_public_key,
             request_fail_exceptions=False,
             status_expiration_period=0.4,  # Make it smaller than default

bluesky_httpserver/config_schemas/service_configuration.yml

@@ -29,6 +29,10 @@ properties:
           The address of 0MQ info (PUB-SUB) socket of RE Manager, used to publish console output
           e.g. tcp://localhost:60625. Overrides the address set using QSERVER_ZMQ_INFO_ADDRESS
           environment variable
+      encoding:
+        type: string
+        description: |
+          The encoding for 0MQ messages. Supported values: 'json' (default) and 'msgpack'.
       public_key:
         type: string
         description: |

bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py

@@ -1,4 +1,4 @@
-""""Add write scopes to default Roles."
+""" "Add write scopes to default Roles."
 
 Revision ID: 722ff4e4fcc7
 Revises: 481830dd6c11

bluesky_httpserver/schemas.py

@@ -144,6 +144,7 @@ class NodeMeta(pydantic.BaseModel):
 
 class Resource(generic_model, Generic[AttributesT, ResourceLinksT, ResourceMetaT]):
     "A JSON API Resource"
+
     id: Union[str, uuid.UUID]
     attributes: AttributesT
     links: Optional[ResourceLinksT] = None
@@ -257,6 +258,7 @@ class Session(pydantic.BaseModel, **orm):
 
 class Principal(pydantic.BaseModel, **orm):
     "Represents a User or Service"
+
     # The id field (primary key) is intentionally not exposed to the application.
     # It is left as an internal database concern.
     uuid: uuid.UUID
@@ -280,6 +282,7 @@ def from_orm(cls, orm, latest_activity=None):
 
 class AllowedScopes(pydantic.BaseModel, **orm):
     "Returns roles and current allowed scopes for a user authenticated with API key or token"
+
     roles: List[str] = []
     scopes: List[str] = []
 

bluesky_httpserver/tests/conftest.py

@@ -4,6 +4,7 @@
 import pytest
 import requests
 from bluesky_queueserver.manager.comms import zmq_single_request
+from bluesky_queueserver.manager.tests.common import set_qserver_zmq_encoding  # noqa: F401
 from xprocess import ProcessStarter
 
 import bluesky_httpserver.server as bqss
@@ -44,6 +45,8 @@ def fastapi_server_fs(xprocess):
 
     def start(http_server_host=SERVER_ADDRESS, http_server_port=SERVER_PORT, api_key=API_KEY_FOR_TESTS):
         class Starter(ProcessStarter):
+            max_read_lines = 53
+
             env = dict(os.environ)
             if api_key:
                 env["QSERVER_HTTP_SERVER_SINGLE_USER_API_KEY"] = api_key

bluesky_httpserver/tests/test_console_output.py

@@ -14,6 +14,7 @@
     SERVER_PORT,
     fastapi_server_fs,
     request_to_json,
+    set_qserver_zmq_encoding,
     wait_for_environment_to_be_closed,
     wait_for_environment_to_be_created,
     wait_for_manager_state_idle,
@@ -155,20 +156,28 @@ def test_http_server_stream_console_output_1(
 """
 
 
+@pytest.mark.parametrize("zmq_encoding", (None, "json", "msgpack"))
 @pytest.mark.parametrize("zmq_port", (None, 60619))
-def test_http_server_console_output_1(monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port):  # noqa F811
+def test_http_server_console_output_1(
+    monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port, zmq_encoding  # noqa F811
+):
     """
     Test for ``console_output`` API (not a streaming version).
     """
     # Start HTTP Server
     if zmq_port is not None:
         monkeypatch.setenv("QSERVER_ZMQ_INFO_ADDRESS", f"tcp://localhost:{zmq_port}")
+    if zmq_encoding is not None:
+        monkeypatch.setenv("QSERVER_ZMQ_ENCODING", zmq_encoding)
     fastapi_server_fs()
 
     # Start RE Manager
-    params = ["--zmq-publish-console", "ON"]
+    params = ["--zmq-publish-console=ON"]
+    if zmq_encoding:
+        params.append(f"--zmq-encoding={zmq_encoding}")
+        set_qserver_zmq_encoding(monkeypatch, encoding=zmq_encoding)
     if zmq_port is not None:
-        params.extend(["--zmq-info-addr", f"tcp://*:{zmq_port}"])
+        params.extend([f"--zmq-info-addr=tcp://*:{zmq_port}"])
     re_manager_cmd(params)
 
     script = _script1

docs/source/conf.py

@@ -40,7 +40,7 @@
     'sphinx.ext.viewcode',
     'IPython.sphinxext.ipython_directive',
     'IPython.sphinxext.ipython_console_highlighting',
-    'matplotlib.sphinxext.plot_directive',
+    # 'matplotlib.sphinxext.plot_directive',
     'numpydoc',
 ]
 
@@ -108,7 +108,7 @@
 html_theme = 'sphinx_rtd_theme'
 import sphinx_rtd_theme
 
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/source/configuration.rst

@@ -100,13 +100,17 @@ The following environment variables are used to configure 0MQ communication sett
   pass the private key to RE Manager. Pass the public key to HTTP Server using
   ``QSERVER_ZMQ_PUBLIC_KEY`` environment variable.
 
+- ``QSERVER_ZMQ_ENCODING`` - sets the encoding for the messages set to RE Manager over
+  0MQ. Supported values: ``"json"`` (default) or ``"msgpack"``.
+
 The same parameters can be specified by including ``qserver_zmq_configuration`` into
 the config file::
 
   qserver_zmq_configuration:
     control_address: tcp://localhost:60615
     info_address: tcp://localhost:60625
     public_key: ${PUBLIC_KEY}
+    encoding: json
 
 All parameters in the config file are optional and override the values passed using
 environment variables and the default values. The public key is typically passed using environment

requirements-dev.txt

@@ -13,6 +13,8 @@ py
 sphinx
 ipython
 numpydoc
+matplotlib
+pandas
 sphinx
 sphinx_rtd_theme
 requests

start_LDAP.sh

@@ -4,5 +4,5 @@ set -e
 
 # Start LDAP server in docker container
 sudo docker pull bitnami/openldap:latest
-sudo docker-compose -f .github/workflows/docker-configs/ldap-docker-compose.yml up -d
+sudo docker compose -f .github/workflows/docker-configs/ldap-docker-compose.yml up -d
 sudo docker ps