fix: over-saturation settings handling

Signed-off-by: Alon Kellner <[email protected]>

Author: AlonKellner-RedHat

src/guidellm/scheduler/constraints/saturation.py

@@ -62,6 +62,7 @@
     SchedulerUpdateAction,
 )
 from guidellm.schemas import RequestInfo
+from guidellm.settings import settings
 
 from .constraint import Constraint, PydanticConstraintInitializer
 from .factory import ConstraintsInitializerFactory
@@ -630,7 +631,8 @@ def validated_kwargs(
         :param over_saturation: Boolean to enable/disable with defaults, or dictionary
             with configuration parameters (min_seconds, max_window_seconds, etc.)
         :param kwargs: Additional keyword arguments supporting aliases like
-            "detect_saturation" for compatibility
+            "detect_saturation" for compatibility, or unpacked dict values when
+            dict is passed to factory
         :return: Validated dictionary with constraint configuration ready for
             initializer creation
         """
@@ -644,17 +646,49 @@ def validated_kwargs(
                 result = alias_value
                 break
 
+        # If over_saturation is None but kwargs contain constraint parameters,
+        # treat kwargs as an unpacked dict (happens when dict is passed to factory)
+        if result is None and kwargs:
+            constraint_keys = {
+                "enabled",
+                "min_seconds",
+                "max_window_seconds",
+                "moe_threshold",
+                "minimum_ttft",
+                "maximum_window_ratio",
+                "minimum_window_size",
+                "confidence",
+            }
+            if any(key in kwargs for key in constraint_keys):
+                # Reconstruct dict from kwargs
+                result = {key: kwargs[key] for key in constraint_keys if key in kwargs}
+
         if result is None:
             return {}
 
         if isinstance(result, bool):
-            return {"enabled": result}
+            # When a boolean is passed, read defaults from settings
+            return {
+                "enabled": result,
+                "min_seconds": kwargs.get(
+                    "min_seconds", settings.constraint_over_saturation_min_seconds
+                ),
+                "max_window_seconds": kwargs.get(
+                    "max_window_seconds",
+                    settings.constraint_over_saturation_max_window_seconds,
+                ),
+            }
         elif isinstance(result, dict):
-            # Extract configuration from dict
+            # Extract configuration from dict, reading from settings for missing values
             return {
                 "enabled": result.get("enabled", True),
-                "min_seconds": result.get("min_seconds", 30.0),
-                "max_window_seconds": result.get("max_window_seconds", 120.0),
+                "min_seconds": result.get(
+                    "min_seconds", settings.constraint_over_saturation_min_seconds
+                ),
+                "max_window_seconds": result.get(
+                    "max_window_seconds",
+                    settings.constraint_over_saturation_max_window_seconds,
+                ),
                 "moe_threshold": result.get("moe_threshold", 2.0),
                 "minimum_ttft": result.get("minimum_ttft", 2.5),
                 "maximum_window_ratio": result.get("maximum_window_ratio", 0.75),

tests/e2e/test_over_saturated_benchmark.py

@@ -22,7 +22,7 @@ def server():
         port=8000,
         model="databricks/dolly-v2-12b",
         mode="random",
-        time_to_first_token=10000,
+        time_to_first_token=60000,
         inter_token_latency=100,
         max_num_seqs=1,
     )
@@ -39,7 +39,7 @@ def test_over_saturated_benchmark(server: VllmSimServer):
     Another example test interacting with the server.
     """
     report_path = Path("tests/e2e/over_saturated_benchmarks.json")
-    rate = 1
+    rate = 10
 
     # Create and configure the guidellm client
     client = GuidellmClient(target=server.get_url(), output_path=report_path)
@@ -80,7 +80,7 @@ def test_over_saturated_benchmark_with_dict_config(server: VllmSimServer):
     Test over-saturation detection with dictionary configuration instead of boolean.
     """
     report_path = Path("tests/e2e/over_saturated_benchmarks_dict.json")
-    rate = 1
+    rate = 10
 
     # Create and configure the guidellm client
     client = GuidellmClient(target=server.get_url(), output_path=report_path)

tests/e2e/utils.py

@@ -93,8 +93,6 @@ def start_benchmark(
                 if over_saturation:
                     cmd_parts.append("--over-saturation=True")
             elif isinstance(over_saturation, dict):
-                import json
-
                 cmd_parts.append(f"--over-saturation '{json.dumps(over_saturation)}'")
 
         cmd_parts.extend(

tests/e2e/vllm-sim-macos.Dockerfile

@@ -2,15 +2,14 @@ FROM golang AS base
 
 WORKDIR /app
 
-ARG BUILDOS
 ARG BUILDARCH
 
 RUN apt-get update && \
     apt-get install -y libzmq3-dev pkg-config && \
     git clone https://github.com/llm-d/llm-d-inference-sim.git && \
     cd llm-d-inference-sim && \
     git checkout v0.3.0 && \
-    GOOS=${BUILDOS} GOARCH=${BUILDARCH} make build
+    GOOS=darwin GOARCH=${BUILDARCH} make build
 
 WORKDIR /app/llm-d-inference-sim
 

tests/unit/scheduler/test_over_saturation.py

@@ -18,6 +18,7 @@
     SerializableConstraintInitializer,
 )
 from guidellm.schemas import RequestInfo, RequestTimings
+from guidellm.settings import settings
 
 
 class TestOverSaturationConstraintInternal:
@@ -382,28 +383,46 @@ def test_create_constraint(self, valid_instances):
     @pytest.mark.smoke
     def test_validated_kwargs(self):
         """Test validated_kwargs method with various inputs."""
+
         result = OverSaturationConstraintInitializer.validated_kwargs(
             over_saturation=True
         )
-        assert result == {"enabled": True}
+        # When a boolean is passed, settings values should be included
+        assert result["enabled"] is True
+        assert result["min_seconds"] == settings.constraint_over_saturation_min_seconds
+        assert (
+            result["max_window_seconds"]
+            == settings.constraint_over_saturation_max_window_seconds
+        )
 
         result = OverSaturationConstraintInitializer.validated_kwargs(
             over_saturation=False
         )
-        assert result == {"enabled": False}
+        assert result["enabled"] is False
+        assert result["min_seconds"] == settings.constraint_over_saturation_min_seconds
+        assert (
+            result["max_window_seconds"]
+            == settings.constraint_over_saturation_max_window_seconds
+        )
 
         # Test with dict input
         result = OverSaturationConstraintInitializer.validated_kwargs(
             over_saturation={"enabled": True, "min_seconds": 20.0}
         )
         assert result["enabled"] is True
-        assert "min_seconds" in result
+        assert result["min_seconds"] == 20.0
+        # Other values should come from settings if not provided
+        assert (
+            result["max_window_seconds"]
+            == settings.constraint_over_saturation_max_window_seconds
+        )
 
         # Test with aliases
         result = OverSaturationConstraintInitializer.validated_kwargs(
             detect_saturation=True
         )
-        assert result == {"enabled": True}
+        assert result["enabled"] is True
+        assert result["min_seconds"] == settings.constraint_over_saturation_min_seconds
 
     @pytest.mark.smoke
     def test_marshalling(self, valid_instances):

tox.ini

@@ -29,6 +29,12 @@ commands =
     python -m pytest tests/e2e {posargs}
 
 
+[testenv:test-path]
+description = Run tests at a given path
+commands =
+    python -m pytest {posargs}
+
+
 [testenv:quality]
 description = Run all quality checks
 commands =