Skip to content

perf(router): optimize deployment filtering in pre-call checks #15611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Parent: [OCT] Alexsander PERF improvements
merged 2 commits into from
Oct 18, 2025
+9 −7
Merged

perf(router): optimize deployment filtering in pre-call checks #15611

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7f8912e
perf(router): optimize deployment filtering in pre-call checks
AlexsanderHamir Oct 16, 2025
c8e5094
Merge branch 'litellm_october_alexsander_stanging' of https://github.…
AlexsanderHamir Oct 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions litellm/router.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6657,7 +6657,7 @@ def _pre_call_checks( # noqa: PLR0915

_returned_deployments = copy.deepcopy(healthy_deployments)

invalid_model_indices = []
invalid_model_indices = set() # Use set for O(1) membership checks

try:
input_tokens = litellm.token_counter(messages=messages)
Expand Down Expand Up @@ -6707,7 +6707,7 @@ def _pre_call_checks( # noqa: PLR0915
isinstance(model_info["max_input_tokens"], int)
and input_tokens > model_info["max_input_tokens"]
):
invalid_model_indices.append(idx)
invalid_model_indices.add(idx)
_context_window_error = True
_potential_error_str += (
"Model={}, Max Input Tokens={}, Got={}".format(
Expand Down Expand Up @@ -6746,7 +6746,7 @@ def _pre_call_checks( # noqa: PLR0915
isinstance(_litellm_params["rpm"], int)
and _litellm_params["rpm"] <= current_request
):
invalid_model_indices.append(idx)
invalid_model_indices.add(idx)
_rate_limit_error = True
continue

Expand All @@ -6762,7 +6762,7 @@ def _pre_call_checks( # noqa: PLR0915
litellm_params=LiteLLM_Params(**_litellm_params),
allowed_model_region=allowed_model_region,
):
invalid_model_indices.append(idx)
invalid_model_indices.add(idx)
continue

## INVALID PARAMS ## -> catch 'gpt-3.5-turbo-16k' not supporting 'response_format' param
Expand Down Expand Up @@ -6791,7 +6791,7 @@ def _pre_call_checks( # noqa: PLR0915
verbose_router_logger.debug(
f"INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k={k}"
)
invalid_model_indices.append(idx)
invalid_model_indices.add(idx)

if len(invalid_model_indices) == len(_returned_deployments):
"""
Expand All @@ -6814,8 +6814,10 @@ def _pre_call_checks( # noqa: PLR0915
llm_provider="",
)
if len(invalid_model_indices) > 0:
for idx in reversed(invalid_model_indices):
_returned_deployments.pop(idx)
# Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
_returned_deployments = [
d for i, d in enumerate(_returned_deployments) if i not in invalid_model_indices
]

## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
if len(_returned_deployments) > 0:
Expand Down
Loading