1/4 Replace realtime websocket transport with WebRTC

codex-rs/Cargo.toml

@@ -239,6 +239,7 @@ image = { version = "^0.25.9", default-features = false }
 include_dir = "0.7.4"
 indexmap = "2.12.0"
 insta = "1.46.3"
+interceptor = "0.17.1"
 inventory = "0.3.19"
 itertools = "0.14.0"
 jsonwebtoken = "9.3.1"
@@ -255,6 +256,7 @@ notify = "8.2.0"
 nucleo = { git = "https://github.com/helix-editor/nucleo.git", rev = "4253de9faabb4e5c6d81d946a5e35a90f87347ee" }
 once_cell = "1.20.2"
 openssl-sys = "*"
+opus-rs = "0.1.11"
 opentelemetry = "0.31.0"
 opentelemetry-appender-tracing = "0.31.0"
 opentelemetry-otlp = "0.31.0"
@@ -349,6 +351,7 @@ v8 = "=146.4.0"
 vt100 = "0.16.2"
 walkdir = "2.5.0"
 webbrowser = "1.0"
+webrtc = "0.17.1"
 which = "8"
 wildmatch = "2.6.1"
 zip = "2.4.2"

codex-rs/app-server/tests/suite/v2/realtime_conversation.rs

@@ -3,6 +3,8 @@ use anyhow::Result;
 use app_test_support::McpProcess;
 use app_test_support::create_mock_responses_server_sequence_unchecked;
 use app_test_support::to_response;
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
 use codex_app_server_protocol::JSONRPCError;
 use codex_app_server_protocol::JSONRPCResponse;
 use codex_app_server_protocol::LoginAccountResponse;
@@ -40,6 +42,16 @@ use tokio::time::timeout;
 const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
 const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex.";
 
+fn realtime_pcm_test_tone_20ms_base64() -> String {
+    let pcm_bytes: Vec<u8> = (0..480)
+        .flat_map(|index| {
+            let sample = if index % 2 == 0 { 1024_i16 } else { -1024_i16 };
+            sample.to_le_bytes()
+        })
+        .collect();
+    BASE64_STANDARD.encode(pcm_bytes)
+}
+
 #[tokio::test]
 async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
     skip_if_no_network!(Ok(()));
@@ -155,7 +167,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
         .send_thread_realtime_append_audio_request(ThreadRealtimeAppendAudioParams {
             thread_id: started.thread_id.clone(),
             audio: ThreadRealtimeAudioChunk {
-                data: "BQYH".to_string(),
+                data: realtime_pcm_test_tone_20ms_base64(),
                 sample_rate: 24_000,
                 num_channels: 1,
                 samples_per_channel: Some(480),

codex-rs/codex-api/Cargo.toml

@@ -14,25 +14,28 @@ codex-protocol = { workspace = true }
 codex-utils-rustls-provider = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
+interceptor = { workspace = true }
+opus-rs = { workspace = true }
+reqwest = { workspace = true, features = ["multipart"] }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["macros", "net", "rt", "sync", "time"] }
 tokio-tungstenite = { workspace = true }
-tungstenite = { workspace = true }
 tracing = { workspace = true }
 eventsource-stream = { workspace = true }
 regex-lite = { workspace = true }
 tokio-util = { workspace = true, features = ["codec"] }
+tungstenite = { workspace = true }
 url = { workspace = true }
+webrtc = { workspace = true }
 
 [dev-dependencies]
 anyhow = { workspace = true }
 assert_matches = { workspace = true }
 pretty_assertions = { workspace = true }
 tokio-test = { workspace = true }
 wiremock = { workspace = true }
-reqwest = { workspace = true }
 
 [lints]
 workspace = true