feat(server): migrate copilot to native (#14620)

#### PR Dependency Tree


* **PR #14620** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Native LLM workflows: structured outputs, embeddings, and reranking
plus richer multimodal attachments (images, audio, files) and improved
remote-attachment inlining.

* **Refactor**
* Tooling API unified behind a local tool-definition helper;
provider/adapters reorganized to route through native dispatch paths.

* **Chores**
* Dependency updates, removed legacy Google SDK integrations, and
increased front memory allocation.

* **Tests**
* Expanded end-to-end and streaming tests exercising native provider
flows, attachments, and rerank/structured scenarios.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/native/src/file_type.rs

@@ -1,3 +1,4 @@
+use matroska::Matroska;
 use mp4parse::{TrackType, read_mp4};
 use napi_derive::napi;
 
@@ -8,7 +9,13 @@ pub fn get_mime(input: &[u8]) -> String {
   } else {
     file_format::FileFormat::from_bytes(input).media_type().to_string()
   };
-  if mimetype == "video/mp4" {
+  if let Some(container) = matroska_container_kind(input).or(match mimetype.as_str() {
+    "video/webm" | "application/webm" => Some(ContainerKind::WebM),
+    "video/x-matroska" | "application/x-matroska" => Some(ContainerKind::Matroska),
+    _ => None,
+  }) {
+    detect_matroska_flavor(input, container, &mimetype)
+  } else if mimetype == "video/mp4" {
     detect_mp4_flavor(input)
   } else {
     mimetype
@@ -37,3 +44,68 @@ fn detect_mp4_flavor(input: &[u8]) -> String {
     Err(_) => "video/mp4".to_string(),
   }
 }
+
+#[derive(Clone, Copy)]
+enum ContainerKind {
+  WebM,
+  Matroska,
+}
+
+impl ContainerKind {
+  fn audio_mime(&self) -> &'static str {
+    match self {
+      ContainerKind::WebM => "audio/webm",
+      ContainerKind::Matroska => "audio/x-matroska",
+    }
+  }
+}
+
+fn detect_matroska_flavor(input: &[u8], container: ContainerKind, fallback: &str) -> String {
+  match Matroska::open(std::io::Cursor::new(input)) {
+    Ok(file) => {
+      let has_video = file.video_tracks().next().is_some();
+      let has_audio = file.audio_tracks().next().is_some();
+      if !has_video && has_audio {
+        container.audio_mime().to_string()
+      } else {
+        fallback.to_string()
+      }
+    }
+    Err(_) => fallback.to_string(),
+  }
+}
+
+fn matroska_container_kind(input: &[u8]) -> Option<ContainerKind> {
+  let header = &input[..1024.min(input.len())];
+  if header.windows(4).any(|window| window.eq_ignore_ascii_case(b"webm")) {
+    Some(ContainerKind::WebM)
+  } else if header.windows(8).any(|window| window.eq_ignore_ascii_case(b"matroska")) {
+    Some(ContainerKind::Matroska)
+  } else {
+    None
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  const AUDIO_ONLY_WEBM: &[u8] = include_bytes!("../fixtures/audio-only.webm");
+  const AUDIO_VIDEO_WEBM: &[u8] = include_bytes!("../fixtures/audio-video.webm");
+  const AUDIO_ONLY_MATROSKA: &[u8] = include_bytes!("../fixtures/audio-only.mka");
+
+  #[test]
+  fn detects_audio_only_webm_as_audio() {
+    assert_eq!(get_mime(AUDIO_ONLY_WEBM), "audio/webm");
+  }
+
+  #[test]
+  fn preserves_video_webm() {
+    assert_eq!(get_mime(AUDIO_VIDEO_WEBM), "video/webm");
+  }
+
+  #[test]
+  fn detects_audio_only_matroska_as_audio() {
+    assert_eq!(get_mime(AUDIO_ONLY_MATROSKA), "audio/x-matroska");
+  }
+}

packages/backend/native/src/llm.rs

@@ -5,9 +5,10 @@ use std::sync::{
 
 use llm_adapter::{
   backend::{
-    BackendConfig, BackendError, BackendProtocol, ReqwestHttpClient, dispatch_request, dispatch_stream_events_with,
+    BackendConfig, BackendError, BackendProtocol, DefaultHttpClient, dispatch_embedding_request, dispatch_request,
+    dispatch_rerank_request, dispatch_stream_events_with, dispatch_structured_request,
   },
-  core::{CoreRequest, StreamEvent},
+  core::{CoreRequest, EmbeddingRequest, RerankRequest, StreamEvent, StructuredRequest},
   middleware::{
     MiddlewareConfig, PipelineContext, RequestMiddleware, StreamMiddleware, citation_indexing, clamp_max_tokens,
     normalize_messages, run_request_middleware_chain, run_stream_middleware_chain, stream_event_normalize,
@@ -40,6 +41,20 @@ struct LlmDispatchPayload {
   middleware: LlmMiddlewarePayload,
 }
 
+#[derive(Debug, Clone, Deserialize)]
+struct LlmStructuredDispatchPayload {
+  #[serde(flatten)]
+  request: StructuredRequest,
+  #[serde(default)]
+  middleware: LlmMiddlewarePayload,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct LlmRerankDispatchPayload {
+  #[serde(flatten)]
+  request: RerankRequest,
+}
+
 #[napi]
 pub struct LlmStreamHandle {
   aborted: Arc<AtomicBool>,
@@ -61,7 +76,44 @@ pub fn llm_dispatch(protocol: String, backend_config_json: String, request_json:
   let request = apply_request_middlewares(payload.request, &payload.middleware)?;
 
   let response =
-    dispatch_request(&ReqwestHttpClient::default(), &config, protocol, &request).map_err(map_backend_error)?;
+    dispatch_request(&DefaultHttpClient::default(), &config, protocol, &request).map_err(map_backend_error)?;
+
+  serde_json::to_string(&response).map_err(map_json_error)
+}
+
+#[napi(catch_unwind)]
+pub fn llm_structured_dispatch(protocol: String, backend_config_json: String, request_json: String) -> Result<String> {
+  let protocol = parse_protocol(&protocol)?;
+  let config: BackendConfig = serde_json::from_str(&backend_config_json).map_err(map_json_error)?;
+  let payload: LlmStructuredDispatchPayload = serde_json::from_str(&request_json).map_err(map_json_error)?;
+  let request = apply_structured_request_middlewares(payload.request, &payload.middleware)?;
+
+  let response = dispatch_structured_request(&DefaultHttpClient::default(), &config, protocol, &request)
+    .map_err(map_backend_error)?;
+
+  serde_json::to_string(&response).map_err(map_json_error)
+}
+
+#[napi(catch_unwind)]
+pub fn llm_embedding_dispatch(protocol: String, backend_config_json: String, request_json: String) -> Result<String> {
+  let protocol = parse_protocol(&protocol)?;
+  let config: BackendConfig = serde_json::from_str(&backend_config_json).map_err(map_json_error)?;
+  let request: EmbeddingRequest = serde_json::from_str(&request_json).map_err(map_json_error)?;
+
+  let response = dispatch_embedding_request(&DefaultHttpClient::default(), &config, protocol, &request)
+    .map_err(map_backend_error)?;
+
+  serde_json::to_string(&response).map_err(map_json_error)
+}
+
+#[napi(catch_unwind)]
+pub fn llm_rerank_dispatch(protocol: String, backend_config_json: String, request_json: String) -> Result<String> {
+  let protocol = parse_protocol(&protocol)?;
+  let config: BackendConfig = serde_json::from_str(&backend_config_json).map_err(map_json_error)?;
+  let payload: LlmRerankDispatchPayload = serde_json::from_str(&request_json).map_err(map_json_error)?;
+
+  let response = dispatch_rerank_request(&DefaultHttpClient::default(), &config, protocol, &payload.request)
+    .map_err(map_backend_error)?;
 
   serde_json::to_string(&response).map_err(map_json_error)
 }
@@ -98,7 +150,7 @@ pub fn llm_dispatch_stream(
     let mut aborted_by_user = false;
     let mut callback_dispatch_failed = false;
 
-    let result = dispatch_stream_events_with(&ReqwestHttpClient::default(), &config, protocol, &request, |event| {
+    let result = dispatch_stream_events_with(&DefaultHttpClient::default(), &config, protocol, &request, |event| {
       if aborted_in_worker.load(Ordering::Relaxed) {
         aborted_by_user = true;
         return Err(BackendError::Http(STREAM_ABORTED_REASON.to_string()));
@@ -155,6 +207,27 @@ fn apply_request_middlewares(request: CoreRequest, middleware: &LlmMiddlewarePay
   Ok(run_request_middleware_chain(request, &middleware.config, &chain))
 }
 
+fn apply_structured_request_middlewares(
+  request: StructuredRequest,
+  middleware: &LlmMiddlewarePayload,
+) -> Result<StructuredRequest> {
+  let mut core = request.as_core_request();
+  core = apply_request_middlewares(core, middleware)?;
+
+  Ok(StructuredRequest {
+    model: core.model,
+    messages: core.messages,
+    schema: core
+      .response_schema
+      .ok_or_else(|| Error::new(Status::InvalidArg, "Structured request schema is required"))?,
+    max_tokens: core.max_tokens,
+    temperature: core.temperature,
+    reasoning: core.reasoning,
+    strict: request.strict,
+    response_mime_type: request.response_mime_type,
+  })
+}
+
 #[derive(Clone)]
 struct StreamPipeline {
   chain: Vec<StreamMiddleware>,
@@ -268,6 +341,7 @@ fn parse_protocol(protocol: &str) -> Result<BackendProtocol> {
     }
     "openai_responses" | "openai-responses" | "responses" => Ok(BackendProtocol::OpenaiResponses),
     "anthropic" | "anthropic_messages" | "anthropic-messages" => Ok(BackendProtocol::AnthropicMessages),
+    "gemini" | "gemini_generate_content" | "gemini-generate-content" => Ok(BackendProtocol::GeminiGenerateContent),
     other => Err(Error::new(
       Status::InvalidArg,
       format!("Unsupported llm backend protocol: {other}"),
@@ -293,6 +367,7 @@ mod tests {
     assert!(parse_protocol("chat-completions").is_ok());
     assert!(parse_protocol("responses").is_ok());
     assert!(parse_protocol("anthropic").is_ok());
+    assert!(parse_protocol("gemini").is_ok());
   }
 
   #[test]