feat(server): adapt gpt5 (#13478)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- New Features
- Added GPT-5 family and made GPT-5/-mini the new defaults for Copilot
scenarios and prompts.

- Bug Fixes
- Improved streaming chunk formats and reasoning/text semantics,
consistent attachment mediaType handling, and more reliable reranking
via log-prob handling.

- Refactor
- Unified maxOutputTokens usage; removed per-call step caps and migrated
several tools to a unified inputSchema shape.

- Chores
- Upgraded AI SDK dependencies and bumped an internal dependency
version.

- Tests
- Updated mocks and tests to reference GPT-5 variants and new stream
formats.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2025-08-13 10:32:15 +08:00
committed by GitHub
parent fda7e9008d
commit 072557eba1
29 changed files with 410 additions and 374 deletions
+7 -1
View File
@@ -1,5 +1,7 @@
use std::collections::HashSet;
use tiktoken_rs::{get_bpe_from_tokenizer, tokenizer::Tokenizer as TiktokenTokenizer};
#[napi]
pub struct Tokenizer {
inner: tiktoken_rs::CoreBPE,
@@ -7,6 +9,10 @@ pub struct Tokenizer {
#[napi]
pub fn from_model_name(model_name: String) -> Option<Tokenizer> {
if model_name.starts_with("gpt-5") {
let bpe = get_bpe_from_tokenizer(TiktokenTokenizer::O200kBase).ok()?;
return Some(Tokenizer { inner: bpe });
}
let bpe = tiktoken_rs::get_bpe_from_model(&model_name).ok()?;
Some(Tokenizer { inner: bpe })
}
@@ -31,7 +37,7 @@ mod tests {
#[test]
fn test_tokenizer() {
let tokenizer = from_model_name("gpt-4.1".to_string()).unwrap();
let tokenizer = from_model_name("gpt-5".to_string()).unwrap();
let content = "Hello, world!";
let count = tokenizer.count(content.to_string(), None);
assert!(count > 0);