| use serde_json::Value as JsonValue; |
|
|
| use codex_code_mode_protocol::DEFAULT_IMAGE_DETAIL; |
| use codex_code_mode_protocol::FunctionCallOutputContentItem; |
| use codex_code_mode_protocol::ImageDetail; |
|
|
| use super::audio::wav_duration_seconds; |
|
|
| const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block"; |
| const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block"; |
| const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead"; |
| const INVALID_IMAGE_URL_ERROR: &str = |
| "Tool call failed: invalid image output. Pass a base64 data URI instead"; |
| const INVALID_AUDIO_URL_ERROR: &str = |
| "Tool call failed: invalid audio output. Pass a base64 data URI instead"; |
| const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail"; |
|
|
| pub(super) fn serialize_output_text( |
| scope: &mut v8::PinScope<'_, '_>, |
| value: v8::Local<'_, v8::Value>, |
| ) -> Result<String, String> { |
| if value.is_undefined() |
| || value.is_null() |
| || value.is_boolean() |
| || value.is_number() |
| || value.is_big_int() |
| || value.is_string() |
| { |
| return Ok(value.to_rust_string_lossy(scope)); |
| } |
|
|
| let tc = std::pin::pin!(v8::TryCatch::new(scope)); |
| let mut tc = tc.init(); |
| if let Some(stringified) = v8::json::stringify(&tc, value) { |
| return Ok(stringified.to_rust_string_lossy(&tc)); |
| } |
| if tc.has_caught() { |
| return Err(tc |
| .exception() |
| .map(|exception| value_to_error_text(&mut tc, exception)) |
| .unwrap_or_else(|| "unknown code mode exception".to_string())); |
| } |
| Ok(value.to_rust_string_lossy(&tc)) |
| } |
|
|
| pub(super) fn normalize_output_image( |
| scope: &mut v8::PinScope<'_, '_>, |
| value: v8::Local<'_, v8::Value>, |
| detail_override: Option<String>, |
| ) -> Result<FunctionCallOutputContentItem, ()> { |
| let result = (|| -> Result<FunctionCallOutputContentItem, String> { |
| let (image_url, detail) = if value.is_string() { |
| (value.to_rust_string_lossy(scope), None) |
| } else if value.is_object() && !value.is_array() { |
| let object = v8::Local::<v8::Object>::try_from(value) |
| .map_err(|_| IMAGE_HELPER_EXPECTS_MESSAGE.to_string())?; |
| if let Some(image) = parse_non_mcp_output_image(scope, object)? { |
| image |
| } else { |
| parse_mcp_output_image(scope, value)? |
| } |
| } else { |
| return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); |
| }; |
|
|
| if image_url.is_empty() { |
| return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); |
| } |
| let Some((scheme, _)) = image_url.split_once(':') else { |
| return Err(INVALID_IMAGE_URL_ERROR.to_string()); |
| }; |
| if scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https") { |
| return Err(REMOTE_IMAGE_URL_ERROR.to_string()); |
| } |
| if !scheme.eq_ignore_ascii_case("data") { |
| return Err(INVALID_IMAGE_URL_ERROR.to_string()); |
| } |
|
|
| let detail = detail_override.or(detail); |
| let detail = match detail { |
| Some(detail) => { |
| let normalized = detail.to_ascii_lowercase(); |
| Some(match normalized.as_str() { |
| "auto" => ImageDetail::Auto, |
| "low" => ImageDetail::Low, |
| "high" => ImageDetail::High, |
| "original" => ImageDetail::Original, |
| _ => { |
| return Err( |
| "image detail must be one of: auto, low, high, original".to_string() |
| ); |
| } |
| }) |
| } |
| None => Some(DEFAULT_IMAGE_DETAIL), |
| }; |
|
|
| Ok(FunctionCallOutputContentItem::InputImage { image_url, detail }) |
| })(); |
|
|
| match result { |
| Ok(item) => Ok(item), |
| Err(error_text) => { |
| throw_type_error(scope, &error_text); |
| Err(()) |
| } |
| } |
| } |
|
|
| fn parse_non_mcp_output_image( |
| scope: &mut v8::PinScope<'_, '_>, |
| object: v8::Local<'_, v8::Object>, |
| ) -> Result<Option<(String, Option<String>)>, String> { |
| let image_url_key = v8::String::new(scope, "image_url") |
| .ok_or_else(|| "failed to allocate image helper keys".to_string())?; |
| let Some(image_url) = object.get(scope, image_url_key.into()) else { |
| return Ok(None); |
| }; |
| if image_url.is_undefined() { |
| return Ok(None); |
| } |
| if !image_url.is_string() { |
| return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); |
| } |
| let detail_key = v8::String::new(scope, "detail") |
| .ok_or_else(|| "failed to allocate image helper keys".to_string())?; |
| let detail = parse_image_detail_value(scope, object.get(scope, detail_key.into()))?; |
| Ok(Some((image_url.to_rust_string_lossy(scope), detail))) |
| } |
|
|
| fn parse_mcp_output_image( |
| scope: &mut v8::PinScope<'_, '_>, |
| value: v8::Local<'_, v8::Value>, |
| ) -> Result<(String, Option<String>), String> { |
| let Some(result) = v8_value_to_json(scope, value)? else { |
| return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); |
| }; |
| let JsonValue::Object(result) = result else { |
| return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); |
| }; |
| let Some(item_type) = result.get("type").and_then(JsonValue::as_str) else { |
| return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); |
| }; |
| if item_type != "image" { |
| return Err(format!( |
| "image only accepts MCP image blocks, got \"{item_type}\"" |
| )); |
| } |
| let data = result |
| .get("data") |
| .and_then(JsonValue::as_str) |
| .ok_or_else(|| "image expected MCP image data".to_string())?; |
| if data.is_empty() { |
| return Err("image expected MCP image data".to_string()); |
| } |
|
|
| let image_url = if data.to_ascii_lowercase().starts_with("data:") { |
| data.to_string() |
| } else { |
| let mime_type = result |
| .get("mimeType") |
| .or_else(|| result.get("mime_type")) |
| .and_then(JsonValue::as_str) |
| .filter(|mime_type| !mime_type.is_empty()) |
| .unwrap_or("application/octet-stream"); |
| format!("data:{mime_type};base64,{data}") |
| }; |
| let detail = result |
| .get("_meta") |
| .and_then(JsonValue::as_object) |
| .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) |
| .and_then(JsonValue::as_str) |
| .filter(|detail| matches!(*detail, "auto" | "low" | "high" | "original")) |
| .map(str::to_string); |
| Ok((image_url, detail)) |
| } |
|
|
| fn parse_image_detail_value<'s>( |
| scope: &mut v8::PinScope<'s, '_>, |
| value: Option<v8::Local<'s, v8::Value>>, |
| ) -> Result<Option<String>, String> { |
| match value { |
| Some(value) if value.is_string() => Ok(Some(value.to_rust_string_lossy(scope))), |
| Some(value) if value.is_null() || value.is_undefined() => Ok(None), |
| Some(_) => Err("image detail must be a string when provided".to_string()), |
| None => Ok(None), |
| } |
| } |
|
|
| pub(super) fn normalize_output_audio( |
| scope: &mut v8::PinScope<'_, '_>, |
| value: v8::Local<'_, v8::Value>, |
| ) -> Result<FunctionCallOutputContentItem, ()> { |
| let result = (|| -> Result<FunctionCallOutputContentItem, String> { |
| let audio_url = if value.is_string() { |
| value.to_rust_string_lossy(scope) |
| } else if value.is_object() && !value.is_array() { |
| let object = v8::Local::<v8::Object>::try_from(value) |
| .map_err(|_| AUDIO_HELPER_EXPECTS_MESSAGE.to_string())?; |
| if let Some(audio_url) = parse_non_mcp_output_audio(scope, object)? { |
| audio_url |
| } else { |
| parse_mcp_output_audio(scope, value)? |
| } |
| } else { |
| return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); |
| }; |
|
|
| if audio_url.is_empty() { |
| return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); |
| } |
| let Some((scheme, _)) = audio_url.split_once(':') else { |
| return Err(INVALID_AUDIO_URL_ERROR.to_string()); |
| }; |
| if !scheme.eq_ignore_ascii_case("data") { |
| return Err(INVALID_AUDIO_URL_ERROR.to_string()); |
| } |
|
|
| |
| if wav_duration_seconds(&audio_url).is_some_and(|duration| duration < 0.025) { |
| return Ok(FunctionCallOutputContentItem::InputText { |
| text: "Audio output omitted because the clip is shorter than 25 ms; use a longer clip." |
| .to_string(), |
| }); |
| } |
|
|
| Ok(FunctionCallOutputContentItem::InputAudio { audio_url }) |
| })(); |
|
|
| match result { |
| Ok(item) => Ok(item), |
| Err(error_text) => { |
| throw_type_error(scope, &error_text); |
| Err(()) |
| } |
| } |
| } |
|
|
| fn parse_non_mcp_output_audio( |
| scope: &mut v8::PinScope<'_, '_>, |
| object: v8::Local<'_, v8::Object>, |
| ) -> Result<Option<String>, String> { |
| let audio_url_key = v8::String::new(scope, "audio_url") |
| .ok_or_else(|| "failed to allocate audio helper keys".to_string())?; |
| let Some(audio_url) = object.get(scope, audio_url_key.into()) else { |
| return Ok(None); |
| }; |
| if audio_url.is_undefined() { |
| return Ok(None); |
| } |
| if !audio_url.is_string() { |
| return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); |
| } |
| Ok(Some(audio_url.to_rust_string_lossy(scope))) |
| } |
|
|
| fn parse_mcp_output_audio( |
| scope: &mut v8::PinScope<'_, '_>, |
| value: v8::Local<'_, v8::Value>, |
| ) -> Result<String, String> { |
| let Some(result) = v8_value_to_json(scope, value)? else { |
| return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); |
| }; |
| let JsonValue::Object(result) = result else { |
| return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); |
| }; |
| let Some(item_type) = result.get("type").and_then(JsonValue::as_str) else { |
| return Err(AUDIO_HELPER_EXPECTS_MESSAGE.to_string()); |
| }; |
| if item_type != "audio" { |
| return Err(format!( |
| "audio only accepts MCP audio blocks, got \"{item_type}\"" |
| )); |
| } |
| let data = result |
| .get("data") |
| .and_then(JsonValue::as_str) |
| .ok_or_else(|| "audio expected MCP audio data".to_string())?; |
| if data.is_empty() { |
| return Err("audio expected MCP audio data".to_string()); |
| } |
|
|
| if data.to_ascii_lowercase().starts_with("data:") { |
| Ok(data.to_string()) |
| } else { |
| let mime_type = result |
| .get("mimeType") |
| .or_else(|| result.get("mime_type")) |
| .and_then(JsonValue::as_str) |
| .filter(|mime_type| !mime_type.is_empty()) |
| .unwrap_or("application/octet-stream"); |
| Ok(format!("data:{mime_type};base64,{data}")) |
| } |
| } |
|
|
| pub(super) fn v8_value_to_json( |
| scope: &mut v8::PinScope<'_, '_>, |
| value: v8::Local<'_, v8::Value>, |
| ) -> Result<Option<JsonValue>, String> { |
| |
| if value.is_undefined() { |
| return Ok(None); |
| } |
|
|
| let tc = std::pin::pin!(v8::TryCatch::new(scope)); |
| let mut tc = tc.init(); |
| let Some(stringified) = v8::json::stringify(&tc, value) else { |
| if tc.has_caught() { |
| return Err(tc |
| .exception() |
| .map(|exception| value_to_error_text(&mut tc, exception)) |
| .unwrap_or_else(|| "unknown code mode exception".to_string())); |
| } |
| return Ok(None); |
| }; |
| serde_json::from_str(&stringified.to_rust_string_lossy(&tc)) |
| .map(Some) |
| .map_err(|err| format!("failed to serialize JavaScript value: {err}")) |
| } |
|
|
| pub(super) fn json_to_v8<'s>( |
| scope: &mut v8::PinScope<'s, '_>, |
| value: &JsonValue, |
| ) -> Option<v8::Local<'s, v8::Value>> { |
| let json = serde_json::to_string(value).ok()?; |
| let json = v8::String::new(scope, &json)?; |
| v8::json::parse(scope, json) |
| } |
|
|
| pub(super) fn value_to_error_text( |
| scope: &mut v8::PinScope<'_, '_>, |
| value: v8::Local<'_, v8::Value>, |
| ) -> String { |
| if value.is_object() |
| && let Ok(object) = v8::Local::<v8::Object>::try_from(value) |
| && let Some(key) = v8::String::new(scope, "stack") |
| && let Some(stack) = object.get(scope, key.into()) |
| && stack.is_string() |
| { |
| return stack.to_rust_string_lossy(scope); |
| } |
| value.to_rust_string_lossy(scope) |
| } |
|
|
| pub(super) fn throw_type_error(scope: &mut v8::PinScope<'_, '_>, message: &str) { |
| if let Some(message) = v8::String::new(scope, message) { |
| scope.throw_exception(message.into()); |
| } |
| } |
|
|
