Merge pull request #117 from GetStream/openrouter

[AI-194] Openrouter

Author: tschellenbach

dev.py

@@ -95,15 +95,14 @@ def mypy_plugins():
         "uv run mypy --install-types --non-interactive --exclude 'plugins/.*/tests/.*' plugins"
     )
 
-
 @cli.command()
 def check():
     """Run full check: ruff, mypy, and unit tests."""
     click.echo("Running full development check...")
 
     # Run ruff
     click.echo("\n=== 1. Ruff Linting ===")
-    run("uv run ruff check .")
+    run("uv run ruff check . --fix")
 
     # Run mypy on main package
     click.echo("\n=== 2. MyPy Type Checking ===")

plugins/openai/vision_agents/plugins/openai/openai_llm.py

@@ -107,6 +107,14 @@ async def simple_response(
             instructions=instructions,
         )
 
+    async def create_conversation(self):
+        if not self.openai_conversation:
+            self.openai_conversation = await self.client.conversations.create()
+
+    def add_conversation_history(self, kwargs):
+        if self.openai_conversation:
+            kwargs["conversation"] = self.openai_conversation.id
+
     async def create_response(
         self, *args: Any, **kwargs: Any
     ) -> LLMResponseEvent[OpenAIResponse]:
@@ -119,9 +127,9 @@ async def create_response(
         if "stream" not in kwargs:
             kwargs["stream"] = True
 
-        #if not self.openai_conversation:
-        #    self.openai_conversation = await self.client.conversations.create()
-        #kwargs["conversation"] = self.openai_conversation.id
+        # create the conversation if needed and add the required args
+        await self.create_conversation()
+        self.add_conversation_history(kwargs)
 
         # Add tools if available - convert to OpenAI format
         tools_spec = self._get_tools_for_provider()

plugins/openrouter/README.md

@@ -0,0 +1,34 @@
+# OpenRouter Plugin
+
+OpenRouter plugin for vision agents. This plugin provides LLM capabilities using OpenRouter's API, which is compatible with the OpenAI API format.
+
+## Note/ Issues
+
+Instruction following doesn't always work with openrouter atm.
+
+## Installation
+
+```bash
+uv pip install vision-agents-plugins-openrouter
+```
+
+## Usage
+
+```python
+from vision_agents.plugins import openrouter, getstream, elevenlabs, cartesia, deepgram, smart_turn
+
+
+agent = Agent(
+    edge=getstream.Edge(),
+    agent_user=User(name="OpenRouter AI"),
+    instructions="Be helpful and friendly to the user",
+    llm=openrouter.LLM(
+        model="anthropic/claude-haiku-4.5",  # Can also use other models like anthropic/claude-3-opus
+    ),
+    tts=elevenlabs.TTS(),
+    stt=deepgram.STT(),
+    turn_detection=smart_turn.TurnDetection(
+        buffer_duration=2.0, confidence_threshold=0.5
+    )
+)
+```

plugins/openrouter/example/__init__.py

@@ -0,0 +1,51 @@
+import asyncio
+import logging
+from uuid import uuid4
+
+from dotenv import load_dotenv
+
+from vision_agents.core import User
+from vision_agents.core.agents import Agent
+from vision_agents.plugins import openrouter, getstream, elevenlabs, deepgram, smart_turn
+
+load_dotenv()
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s [call_id=%(call_id)s] %(name)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+
+async def start_agent() -> None:
+    """Example agent using OpenRouter LLM.
+    
+    This example demonstrates how to use the OpenRouter plugin with a Vision Agent.
+    OpenRouter provides access to multiple LLM providers through a unified API.
+    
+    Set OPENROUTER_API_KEY environment variable before running.
+    """
+    agent = Agent(
+        edge=getstream.Edge(),
+        agent_user=User(name="OpenRouter AI"),
+        instructions="Be helpful and friendly to the user",
+        llm=openrouter.LLM(
+            model="openai/gpt-4o",  # Can also use other models like anthropic/claude-3-opus
+        ),
+        tts=elevenlabs.TTS(),
+        stt=deepgram.STT(),
+        turn_detection=smart_turn.TurnDetection(
+            buffer_duration=2.0, confidence_threshold=0.5
+        )
+    )
+    await agent.create_user()
+
+    call = agent.edge.client.video.call("default", str(uuid4()))
+    await agent.edge.open_demo(call)
+
+    with await agent.join(call):
+        await asyncio.sleep(5)
+        await agent.llm.simple_response(text="Hello! I'm powered by OpenRouter.")
+        await agent.finish()
+
+
+if __name__ == "__main__":
+    asyncio.run(start_agent())
+

plugins/openrouter/example/openrouter_example.py

@@ -0,0 +1,26 @@
+[project]
+name = "openrouter-example"
+version = "0.0.0"
+requires-python = ">=3.10"
+
+dependencies = [
+  "python-dotenv>=1.0",
+  "vision-agents-plugins-openrouter",
+  "vision-agents-plugins-getstream",
+  "vision-agents-plugins-elevenlabs",
+  "vision-agents-plugins-deepgram",
+  "vision-agents-plugins-cartesia",
+  "vision-agents-plugins-smart-turn",
+  "vision-agents",
+]
+
+[tool.uv.sources]
+"vision-agents-plugins-openrouter" = {path = "..", editable=true}
+"vision-agents-plugins-openai" = {path = "../../openai", editable=true}
+"vision-agents-plugins-getstream" = {path = "../../getstream", editable=true}
+"vision-agents-plugins-elevenlabs" = {path = "../../elevenlabs", editable=true}
+"vision-agents-plugins-deepgram" = {path = "../../deepgram", editable=true}
+"vision-agents-plugins-cartesia" = {path = "../../cartesia", editable=true}
+"vision-agents-plugins-smart-turn" = {path = "../../smart_turn", editable=true}
+"vision-agents" = {path = "../../../agents-core", editable=true}
+

plugins/openrouter/example/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vision-agents-plugins-openrouter"
+dynamic = ["version"]
+description = "OpenRouter plugin for vision agents"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+    "vision-agents",
+    "vision-agents-plugins-openai",
+]
+
+[project.urls]
+Documentation = "https://visionagents.ai/"
+Website = "https://visionagents.ai/"
+Source = "https://github.com/GetStream/Vision-Agents"
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "..", search_parent_directories = true, fallback_version = "0.0.0" }
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.uv.sources]
+vision-agents = { workspace = true }
+vision-agents-plugins-openai = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "pytest-asyncio>=1.0.0",
+]
+

plugins/openrouter/py.typed

@@ -0,0 +1,156 @@
+"""Tests for OpenRouter LLM plugin."""
+
+import os
+
+import pytest
+from dotenv import load_dotenv
+
+from vision_agents.core.agents.conversation import Message, InMemoryConversation
+from vision_agents.core.llm.events import (
+    LLMResponseChunkEvent,
+)
+from vision_agents.plugins.openrouter import LLM
+
+load_dotenv()
+
+
+class TestOpenRouterLLM:
+    """Test suite for OpenRouter LLM class."""
+
+    def assert_response_successful(self, response):
+        """Utility method to verify a response is successful.
+
+        A successful response has:
+        - response.text is set (not None and not empty)
+        - response.exception is None
+
+        Args:
+            response: LLMResponseEvent to check
+        """
+        assert response.text is not None, "Response text should not be None"
+        assert len(response.text) > 0, "Response text should not be empty"
+        assert not hasattr(response, "exception") or response.exception is None, (
+            f"Response should not have an exception, got: {getattr(response, 'exception', None)}"
+        )
+
+    def test_message(self):
+        """Test basic message normalization."""
+        messages = LLM._normalize_message("say hi")
+        assert isinstance(messages[0], Message)
+        message = messages[0]
+        assert message.original is not None
+        assert message.content == "say hi"
+
+    def test_advanced_message(self):
+        """Test advanced message format with image."""
+        img_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/d5/2023_06_08_Raccoon1.jpg/1599px-2023_06_08_Raccoon1.jpg"
+
+        advanced = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_text", "text": "what do you see in this image?"},
+                    {"type": "input_image", "image_url": f"{img_url}"},
+                ],
+            }
+        ]
+        messages = LLM._normalize_message(advanced)
+        assert messages[0].original is not None
+
+    @pytest.fixture
+    async def llm(self) -> LLM:
+        """Fixture for OpenRouter LLM with z-ai/glm-4.6 model."""
+        if not os.environ.get("OPENROUTER_API_KEY"):
+            pytest.skip("OPENROUTER_API_KEY environment variable not set")
+        
+        llm = LLM(model="anthropic/claude-haiku-4.5")
+        llm._conversation = InMemoryConversation("be friendly", [])
+        return llm
+
+    @pytest.mark.integration
+    async def test_simple(self, llm: LLM):
+        """Test simple response generation."""
+        response = await llm.simple_response(
+            "Explain quantum computing in 1 paragraph",
+        )
+
+        self.assert_response_successful(response)
+
+    @pytest.mark.integration
+    async def test_native_api(self, llm: LLM):
+        """Test native OpenAI-compatible API."""
+        response = await llm.create_response(
+            input="say hi", instructions="You are a helpful assistant."
+        )
+
+        self.assert_response_successful(response)
+        assert hasattr(response.original, "id")  # OpenAI-compatible response has id
+
+    @pytest.mark.integration
+    async def test_streaming(self, llm: LLM):
+        """Test streaming response."""
+        streamingWorks = False
+
+        @llm.events.subscribe
+        async def passed(event: LLMResponseChunkEvent):
+            nonlocal streamingWorks
+            streamingWorks = True
+
+        response = await llm.simple_response(
+            "Explain quantum computing in 1 paragraph",
+        )
+
+        await llm.events.wait()
+
+        self.assert_response_successful(response)
+        assert streamingWorks, "Streaming should have generated chunk events"
+
+    @pytest.mark.integration
+    async def test_memory(self, llm: LLM):
+        """Test conversation memory using simple_response."""
+        await llm.simple_response(
+            text="There are 2 dogs in the room",
+        )
+        response = await llm.simple_response(
+            text="How many paws are there in the room?",
+        )
+        
+        self.assert_response_successful(response)
+        assert "8" in response.text or "eight" in response.text.lower(), (
+            f"Expected '8' or 'eight' in response, got: {response.text}"
+        )
+
+    @pytest.mark.integration
+    async def test_native_memory(self, llm: LLM):
+        """Test conversation memory using native API."""
+        await llm.create_response(
+            input="There are 2 dogs in the room",
+        )
+        response = await llm.create_response(
+            input="How many paws are there in the room?",
+        )
+        
+        self.assert_response_successful(response)
+        assert "8" in response.text or "eight" in response.text.lower(), (
+            f"Expected '8' or 'eight' in response, got: {response.text}"
+        )
+
+    @pytest.mark.integration
+    async def test_instruction_following(self):
+        """Test that the LLM follows system instructions."""
+        if not os.environ.get("OPENROUTER_API_KEY"):
+            pytest.skip("OPENROUTER_API_KEY environment variable not set")
+
+        pytest.skip("instruction following doesnt always work")
+        llm = LLM(model="anthropic/claude-haiku-4.5")
+        llm._set_instructions("Only reply in 2 letter country shortcuts")
+
+        response = await llm.simple_response(
+            text="Which country is rainy, protected from water with dikes and below sea level?",
+        )
+
+        self.assert_response_successful(response)
+        assert "nl" in response.text.lower(), (
+            f"Expected 'NL' in response, got: {response.text}"
+        )
+