feat(core): improve byok editing (#15427)

fix #14287
fix #15359
fix #15424

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Redesigned workspace AI provider settings with connection testing,
storage options, model selection, capability management, ordering, and
custom endpoints.
* AI chat model choices now adapt to the selected workspace and
conversation route.
  * Added support for image-based AI requests.
* **Bug Fixes**
  * Improved handling of unavailable or outdated model selections.
* App configuration updates now reject overlapping paths and load
deterministically.
* **Tests**
* Expanded coverage for provider models, AI chat scoping, image
requests, and configuration validation.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/native/src/llm/core/prompt/session.rs

@@ -135,9 +135,25 @@ fn estimated_message_bytes(message: &PromptMessageContract) -> usize {
   let mut size = MESSAGE_FRAMING_BYTES
     .saturating_add(message.role.len())
     .saturating_add(message.content.len());
+  if let Some(attachments) = &message.attachments {
+    size = attachments.iter().fold(size, |size, attachment| {
+      size.saturating_add(
+        serde_json::to_vec(&attachment_metadata(attachment))
+          .map(|bytes| bytes.len())
+          .unwrap_or(usize::MAX),
+      )
+    });
+  }
   for value in [
-    message.attachments.as_ref().map(serde_json::to_vec),
-    message.params.as_ref().map(serde_json::to_vec),
+    message.params.as_ref().map(|params| {
+      let mut metadata = params.clone();
+      if let Some(Value::Array(attachments)) = metadata.get_mut("attachments") {
+        for attachment in attachments {
+          *attachment = attachment_metadata(attachment);
+        }
+      }
+      serde_json::to_vec(&metadata)
+    }),
     message.response_format.as_ref().map(serde_json::to_vec),
   ]
   .into_iter()
@@ -148,6 +164,32 @@ fn estimated_message_bytes(message: &PromptMessageContract) -> usize {
   size
 }
 
+fn attachment_metadata(attachment: &Value) -> Value {
+  if attachment.as_str().is_some_and(|value| value.starts_with("data:")) {
+    return Value::String("data:".to_string());
+  }
+  let Some(object) = attachment.as_object() else {
+    return attachment.clone();
+  };
+  let inline = matches!(object.get("kind").and_then(Value::as_str), Some("data" | "bytes"));
+  let metadata = object
+    .iter()
+    .map(|(key, value)| {
+      let value = if inline && key == "data" {
+        Value::Null
+      } else if matches!(key.as_str(), "url" | "attachment")
+        && value.as_str().is_some_and(|value| value.starts_with("data:"))
+      {
+        Value::String("data:".to_string())
+      } else {
+        value.clone()
+      };
+      (key.clone(), value)
+    })
+    .collect::<Map<_, _>>();
+  Value::Object(metadata)
+}
+
 fn select_history_turns(
   fixed_messages: &[PromptMessageContract],
   history: &[PromptMessageContract],
@@ -266,6 +308,64 @@ mod tests {
     let emoji = estimated_message_bytes(&message("user", "😀😀😀"));
     assert!(ascii < cjk);
     assert!(cjk < emoji);
+
+    let small = serde_json::from_value(json!({
+      "role": "user",
+      "content": "describe",
+      "attachments": [{ "kind": "bytes", "data": "aW1n", "mimeType": "image/png" }]
+    }))
+    .unwrap();
+    let large = serde_json::from_value(json!({
+      "role": "user",
+      "content": "describe",
+      "attachments": [{ "kind": "bytes", "data": "aW1n".repeat(100_000), "mimeType": "image/png" }]
+    }))
+    .unwrap();
+    assert_eq!(estimated_message_bytes(&small), estimated_message_bytes(&large));
+
+    let legacy_small = serde_json::from_value(json!({
+      "role": "user",
+      "content": "describe",
+      "attachments": ["data:image/png;base64,aW1n"]
+    }))
+    .unwrap();
+    let legacy_large = serde_json::from_value(json!({
+      "role": "user",
+      "content": "describe",
+      "attachments": [format!("data:image/png;base64,{}", "aW1n".repeat(100_000))]
+    }))
+    .unwrap();
+    assert_eq!(
+      estimated_message_bytes(&legacy_small),
+      estimated_message_bytes(&legacy_large)
+    );
+
+    let params_small = serde_json::from_value(json!({
+      "role": "user",
+      "content": "describe",
+      "params": {
+        "attachments": [{
+          "attachment": "data:image/png;base64,aW1n",
+          "mimeType": "image/png"
+        }]
+      }
+    }))
+    .unwrap();
+    let params_large = serde_json::from_value(json!({
+      "role": "user",
+      "content": "describe",
+      "params": {
+        "attachments": [{
+          "attachment": format!("data:image/png;base64,{}", "aW1n".repeat(100_000)),
+          "mimeType": "image/png"
+        }]
+      }
+    }))
+    .unwrap();
+    assert_eq!(
+      estimated_message_bytes(&params_small),
+      estimated_message_bytes(&params_large)
+    );
   }
 
   #[test]

packages/backend/native/src/llm/mod.rs

@@ -6,6 +6,16 @@ mod ffi;
 mod prompt_catalog;
 pub(crate) mod route;
 
+pub use action::copilot_action_recipe;
+pub use byok::{
+  ByokCapabilityInput, ByokCatalogModelOutput, ByokCatalogOutput, ByokCatalogProviderOutput, ByokEndpointInput,
+  ByokLocalLeaseOutput, ByokModelDeclarationInput, ByokModelProbeCheckOutput, ByokModelProbeOutput,
+  ByokProbeCheckInput, ByokProbeResultOutput, ByokProbeStatusOutput, ByokProfileDefinitionInput, ByokProfileOutput,
+  ByokValidationOutput, CreateByokLocalLeaseInput, CreateByokLocalLeaseProviderInput, CreateByokProfileInput,
+  ProbeByokDraftInput, ProbeByokProfileInput, ReorderByokProfilesInput, ReplaceByokProfileInput,
+  RotateByokCredentialInput, byok_catalog,
+};
+
 pub use self::core::{
   capability::llm_match_model_capabilities,
   model_registry::{llm_match_model_registry, llm_resolve_model_registry_variant},
@@ -20,16 +30,6 @@ pub use self::core::{
   structured_output::{llm_canonical_json_schema_hash, llm_validate_json_schema},
 };
 
-pub use action::copilot_action_recipe;
-pub use byok::{
-  ByokCapabilityInput, ByokCatalogModelOutput, ByokCatalogOutput, ByokCatalogProviderOutput, ByokEndpointInput,
-  ByokLocalLeaseOutput, ByokModelDeclarationInput, ByokModelProbeCheckOutput, ByokModelProbeOutput,
-  ByokProbeCheckInput, ByokProbeResultOutput, ByokProbeStatusOutput, ByokProfileDefinitionInput, ByokProfileOutput,
-  ByokValidationOutput, CreateByokLocalLeaseInput, CreateByokLocalLeaseProviderInput, CreateByokProfileInput,
-  ProbeByokDraftInput, ProbeByokProfileInput, ReorderByokProfilesInput, ReplaceByokProfileInput,
-  RotateByokCredentialInput, byok_catalog,
-};
-
 #[napi_derive::napi(catch_unwind)]
 pub fn llm_get_byok_catalog() -> ByokCatalogOutput {
   byok_catalog()
@@ -44,6 +44,8 @@ pub use route::{
   CopilotAccessProjection, CopilotExecuteInput, CopilotManagedTier, CopilotRouteCheckInput, CopilotTargetOverrideInput,
 };
 
+pub(crate) use self::core::contracts::LlmImageRequestContract;
+
 pub(crate) fn invalid_arg(message: impl Into<String>) -> napi::Error {
   napi::Error::new(napi::Status::InvalidArg, message.into())
 }