Merge branch 'main' into add-github-action-for-nix

Author: jessfraz

codex-rs/Cargo.lock

@@ -1,8 +1,13 @@
 [workspace]
 members = [
+    "backend-client",
     "ansi-escape",
+    "app-server",
     "apply-patch",
     "arg0",
+    "codex-backend-openapi-models",
+    "cloud-tasks",
+    "cloud-tasks-client",
     "cli",
     "common",
     "core",
@@ -23,6 +28,8 @@ members = [
     "responses-api-proxy",
     "otel",
     "tui",
+    "git-apply",
+    "utils/json-to-toml",
     "utils/readiness",
 ]
 resolver = "2"
@@ -37,7 +44,9 @@ edition = "2024"
 
 [workspace.dependencies]
 # Internal
+app_test_support = { path = "app-server/tests/common" }
 codex-ansi-escape = { path = "ansi-escape" }
+codex-app-server = { path = "app-server" }
 codex-apply-patch = { path = "apply-patch" }
 codex-arg0 = { path = "arg0" }
 codex-chatgpt = { path = "chatgpt" }
@@ -51,12 +60,14 @@ codex-login = { path = "login" }
 codex-mcp-client = { path = "mcp-client" }
 codex-mcp-server = { path = "mcp-server" }
 codex-ollama = { path = "ollama" }
+codex-otel = { path = "otel" }
 codex-process-hardening = { path = "process-hardening" }
 codex-protocol = { path = "protocol" }
 codex-protocol-ts = { path = "protocol-ts" }
+codex-responses-api-proxy = { path = "responses-api-proxy" }
 codex-rmcp-client = { path = "rmcp-client" }
 codex-tui = { path = "tui" }
-codex-otel = { path = "otel" }
+codex-utils-json-to-toml = { path = "utils/json-to-toml" }
 codex-utils-readiness = { path = "utils/readiness" }
 core_test_support = { path = "core/tests/common" }
 mcp-types = { path = "mcp-types" }
@@ -106,10 +117,10 @@ multimap = "0.10.0"
 nucleo-matcher = "0.3.1"
 openssl-sys = "*"
 opentelemetry = "0.30.0"
-opentelemetry_sdk = "0.30.0"
-opentelemetry-otlp = "0.30.0"
 opentelemetry-appender-tracing = "0.30.0"
+opentelemetry-otlp = "0.30.0"
 opentelemetry-semantic-conventions = "0.30.0"
+opentelemetry_sdk = "0.30.0"
 os_info = "3.12.0"
 owo-colors = "4.2.0"
 path-absolutize = "3.1.1"
@@ -148,14 +159,14 @@ tokio-test = "0.4"
 tokio-util = "0.7.16"
 toml = "0.9.5"
 toml_edit = "0.23.4"
+tonic = "0.13.1"
 tracing = "0.1.41"
 tracing-appender = "0.2.3"
 tracing-subscriber = "0.3.20"
 tracing-test = "0.2.5"
 tree-sitter = "0.25.9"
 tree-sitter-bash = "0.25.0"
 ts-rs = "11"
-tonic = "0.13.1"
 unicode-segmentation = "1.12.0"
 unicode-width = "0.2"
 url = "2"

codex-rs/Cargo.toml

@@ -25,12 +25,14 @@ Codex supports a rich set of configuration options. Note that the Rust CLI uses
 
 Codex CLI functions as an MCP client that can connect to MCP servers on startup. See the [`mcp_servers`](../docs/config.md#mcp_servers) section in the configuration documentation for details.
 
-It is still experimental, but you can also launch Codex as an MCP _server_ by running `codex mcp`. Use the [`@modelcontextprotocol/inspector`](https://github.com/modelcontextprotocol/inspector) to try it out:
+It is still experimental, but you can also launch Codex as an MCP _server_ by running `codex mcp-server`. Use the [`@modelcontextprotocol/inspector`](https://github.com/modelcontextprotocol/inspector) to try it out:
 
 ```shell
-npx @modelcontextprotocol/inspector codex mcp
+npx @modelcontextprotocol/inspector codex mcp-server
 ```
 
+Use `codex mcp` to add/list/get/remove MCP server launchers defined in `config.toml`, and `codex mcp-server` to run the MCP server directly.
+
 ### Notifications
 
 You can enable notifications by configuring a script that is run whenever the agent finishes a turn. The [notify documentation](../docs/config.md#notify) includes a detailed example that explains how to get desktop notifications via [terminal-notifier](https://github.com/julienXX/terminal-notifier) on macOS.

codex-rs/README.md

@@ -0,0 +1,51 @@
+[package]
+edition = "2024"
+name = "codex-app-server"
+version = { workspace = true }
+
+[[bin]]
+name = "codex-app-server"
+path = "src/main.rs"
+
+[lib]
+name = "codex_app_server"
+path = "src/lib.rs"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+codex-arg0 = { workspace = true }
+codex-common = { workspace = true, features = ["cli"] }
+codex-core = { workspace = true }
+codex-file-search = { workspace = true }
+codex-login = { workspace = true }
+codex-protocol = { workspace = true }
+codex-utils-json-to-toml = { workspace = true }
+# We should only be using mcp-types for JSON-RPC types: it would be nice to
+# split this out into a separate crate at some point.
+mcp-types = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+tokio = { workspace = true, features = [
+    "io-std",
+    "macros",
+    "process",
+    "rt-multi-thread",
+    "signal",
+] }
+tracing = { workspace = true, features = ["log"] }
+tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] }
+uuid = { workspace = true, features = ["serde", "v7"] }
+
+[dev-dependencies]
+app_test_support = { workspace = true }
+assert_cmd = { workspace = true }
+base64 = { workspace = true }
+core_test_support = { workspace = true }
+os_info = { workspace = true }
+pretty_assertions = { workspace = true }
+tempfile = { workspace = true }
+toml = { workspace = true }
+wiremock = { workspace = true }

codex-rs/app-server/Cargo.toml

@@ -1,7 +1,6 @@
 use crate::error_code::INTERNAL_ERROR_CODE;
 use crate::error_code::INVALID_REQUEST_ERROR_CODE;
 use crate::fuzzy_file_search::run_fuzzy_file_search;
-use crate::json_to_toml::json_to_toml;
 use crate::outgoing_message::OutgoingMessageSender;
 use crate::outgoing_message::OutgoingNotification;
 use codex_core::AuthManager;
@@ -77,6 +76,7 @@ use codex_protocol::mcp_protocol::SendUserMessageResponse;
 use codex_protocol::mcp_protocol::SendUserTurnParams;
 use codex_protocol::mcp_protocol::SendUserTurnResponse;
 use codex_protocol::mcp_protocol::ServerNotification;
+use codex_protocol::mcp_protocol::SessionConfiguredNotification;
 use codex_protocol::mcp_protocol::SetDefaultModelParams;
 use codex_protocol::mcp_protocol::SetDefaultModelResponse;
 use codex_protocol::mcp_protocol::UserInfoResponse;
@@ -85,6 +85,7 @@ use codex_protocol::models::ContentItem;
 use codex_protocol::models::ResponseItem;
 use codex_protocol::protocol::InputMessageKind;
 use codex_protocol::protocol::USER_MESSAGE_BEGIN;
+use codex_utils_json_to_toml::json_to_toml;
 use mcp_types::JSONRPCErrorError;
 use mcp_types::RequestId;
 use std::collections::HashMap;
@@ -153,6 +154,9 @@ impl CodexMessageProcessor {
 
     pub async fn process_request(&mut self, request: ClientRequest) {
         match request {
+            ClientRequest::Initialize { .. } => {
+                panic!("Initialize should be handled in MessageProcessor");
+            }
             ClientRequest::NewConversation { request_id, params } => {
                 // Do not tokio::spawn() to process new_conversation()
                 // asynchronously because we need to ensure the conversation is
@@ -762,11 +766,19 @@ impl CodexMessageProcessor {
                 session_configured,
                 ..
             }) => {
-                let event = Event {
-                    id: "".to_string(),
-                    msg: EventMsg::SessionConfigured(session_configured.clone()),
-                };
-                self.outgoing.send_event_as_notification(&event, None).await;
+                self.outgoing
+                    .send_server_notification(ServerNotification::SessionConfigured(
+                        SessionConfiguredNotification {
+                            session_id: session_configured.session_id,
+                            model: session_configured.model.clone(),
+                            reasoning_effort: session_configured.reasoning_effort,
+                            history_log_id: session_configured.history_log_id,
+                            history_entry_count: session_configured.history_entry_count,
+                            initial_messages: session_configured.initial_messages.clone(),
+                            rollout_path: session_configured.rollout_path.clone(),
+                        },
+                    ))
+                    .await;
                 let initial_messages = session_configured.initial_messages.map(|msgs| {
                     msgs.into_iter()
                         .filter(|event| {

codex-rs/mcp-server/src/codex_message_processor.rs renamed to codex-rs/app-server/src/codex_message_processor.rs

@@ -0,0 +1,2 @@
+pub(crate) const INVALID_REQUEST_ERROR_CODE: i64 = -32600;
+pub(crate) const INTERNAL_ERROR_CODE: i64 = -32603;

codex-rs/app-server/src/error_code.rs

@@ -0,0 +1,139 @@
+#![deny(clippy::print_stdout, clippy::print_stderr)]
+
+use std::io::ErrorKind;
+use std::io::Result as IoResult;
+use std::path::PathBuf;
+
+use codex_common::CliConfigOverrides;
+use codex_core::config::Config;
+use codex_core::config::ConfigOverrides;
+
+use mcp_types::JSONRPCMessage;
+use tokio::io::AsyncBufReadExt;
+use tokio::io::AsyncWriteExt;
+use tokio::io::BufReader;
+use tokio::io::{self};
+use tokio::sync::mpsc;
+use tracing::debug;
+use tracing::error;
+use tracing::info;
+use tracing_subscriber::EnvFilter;
+
+use crate::message_processor::MessageProcessor;
+use crate::outgoing_message::OutgoingMessage;
+use crate::outgoing_message::OutgoingMessageSender;
+
+mod codex_message_processor;
+mod error_code;
+mod fuzzy_file_search;
+mod message_processor;
+mod outgoing_message;
+
+/// Size of the bounded channels used to communicate between tasks. The value
+/// is a balance between throughput and memory usage – 128 messages should be
+/// plenty for an interactive CLI.
+const CHANNEL_CAPACITY: usize = 128;
+
+pub async fn run_main(
+    codex_linux_sandbox_exe: Option<PathBuf>,
+    cli_config_overrides: CliConfigOverrides,
+) -> IoResult<()> {
+    // Install a simple subscriber so `tracing` output is visible.  Users can
+    // control the log level with `RUST_LOG`.
+    tracing_subscriber::fmt()
+        .with_writer(std::io::stderr)
+        .with_env_filter(EnvFilter::from_default_env())
+        .init();
+
+    // Set up channels.
+    let (incoming_tx, mut incoming_rx) = mpsc::channel::<JSONRPCMessage>(CHANNEL_CAPACITY);
+    let (outgoing_tx, mut outgoing_rx) = mpsc::unbounded_channel::<OutgoingMessage>();
+
+    // Task: read from stdin, push to `incoming_tx`.
+    let stdin_reader_handle = tokio::spawn({
+        async move {
+            let stdin = io::stdin();
+            let reader = BufReader::new(stdin);
+            let mut lines = reader.lines();
+
+            while let Some(line) = lines.next_line().await.unwrap_or_default() {
+                match serde_json::from_str::<JSONRPCMessage>(&line) {
+                    Ok(msg) => {
+                        if incoming_tx.send(msg).await.is_err() {
+                            // Receiver gone – nothing left to do.
+                            break;
+                        }
+                    }
+                    Err(e) => error!("Failed to deserialize JSONRPCMessage: {e}"),
+                }
+            }
+
+            debug!("stdin reader finished (EOF)");
+        }
+    });
+
+    // Parse CLI overrides once and derive the base Config eagerly so later
+    // components do not need to work with raw TOML values.
+    let cli_kv_overrides = cli_config_overrides.parse_overrides().map_err(|e| {
+        std::io::Error::new(
+            ErrorKind::InvalidInput,
+            format!("error parsing -c overrides: {e}"),
+        )
+    })?;
+    let config = Config::load_with_cli_overrides(cli_kv_overrides, ConfigOverrides::default())
+        .map_err(|e| {
+            std::io::Error::new(ErrorKind::InvalidData, format!("error loading config: {e}"))
+        })?;
+
+    // Task: process incoming messages.
+    let processor_handle = tokio::spawn({
+        let outgoing_message_sender = OutgoingMessageSender::new(outgoing_tx);
+        let mut processor = MessageProcessor::new(
+            outgoing_message_sender,
+            codex_linux_sandbox_exe,
+            std::sync::Arc::new(config),
+        );
+        async move {
+            while let Some(msg) = incoming_rx.recv().await {
+                match msg {
+                    JSONRPCMessage::Request(r) => processor.process_request(r).await,
+                    JSONRPCMessage::Response(r) => processor.process_response(r).await,
+                    JSONRPCMessage::Notification(n) => processor.process_notification(n).await,
+                    JSONRPCMessage::Error(e) => processor.process_error(e),
+                }
+            }
+
+            info!("processor task exited (channel closed)");
+        }
+    });
+
+    // Task: write outgoing messages to stdout.
+    let stdout_writer_handle = tokio::spawn(async move {
+        let mut stdout = io::stdout();
+        while let Some(outgoing_message) = outgoing_rx.recv().await {
+            let msg: JSONRPCMessage = outgoing_message.into();
+            match serde_json::to_string(&msg) {
+                Ok(json) => {
+                    if let Err(e) = stdout.write_all(json.as_bytes()).await {
+                        error!("Failed to write to stdout: {e}");
+                        break;
+                    }
+                    if let Err(e) = stdout.write_all(b"\n").await {
+                        error!("Failed to write newline to stdout: {e}");
+                        break;
+                    }
+                }
+                Err(e) => error!("Failed to serialize JSONRPCMessage: {e}"),
+            }
+        }
+
+        info!("stdout writer exited (channel closed)");
+    });
+
+    // Wait for all tasks to finish.  The typical exit path is the stdin reader
+    // hitting EOF which, once it drops `incoming_tx`, propagates shutdown to
+    // the processor and then to the stdout task.
+    let _ = tokio::join!(stdin_reader_handle, processor_handle, stdout_writer_handle);
+
+    Ok(())
+}

codex-rs/mcp-server/src/fuzzy_file_search.rs renamed to codex-rs/app-server/src/fuzzy_file_search.rs

@@ -0,0 +1,10 @@
+use codex_app_server::run_main;
+use codex_arg0::arg0_dispatch_or_else;
+use codex_common::CliConfigOverrides;
+
+fn main() -> anyhow::Result<()> {
+    arg0_dispatch_or_else(|codex_linux_sandbox_exe| async move {
+        run_main(codex_linux_sandbox_exe, CliConfigOverrides::default()).await?;
+        Ok(())
+    })
+}