mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-30 00:29:46 +08:00
feat(server): adapt gpt5 (#13478)
<!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - New Features - Added GPT-5 family and made GPT-5/-mini the new defaults for Copilot scenarios and prompts. - Bug Fixes - Improved streaming chunk formats and reasoning/text semantics, consistent attachment mediaType handling, and more reliable reranking via log-prob handling. - Refactor - Unified maxOutputTokens usage; removed per-call step caps and migrated several tools to a unified inputSchema shape. - Chores - Upgraded AI SDK dependencies and bumped an internal dependency version. - Tests - Updated mocks and tests to reference GPT-5 variants and new stream formats. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use tiktoken_rs::{get_bpe_from_tokenizer, tokenizer::Tokenizer as TiktokenTokenizer};
|
||||
|
||||
#[napi]
|
||||
pub struct Tokenizer {
|
||||
inner: tiktoken_rs::CoreBPE,
|
||||
@@ -7,6 +9,10 @@ pub struct Tokenizer {
|
||||
|
||||
#[napi]
|
||||
pub fn from_model_name(model_name: String) -> Option<Tokenizer> {
|
||||
if model_name.starts_with("gpt-5") {
|
||||
let bpe = get_bpe_from_tokenizer(TiktokenTokenizer::O200kBase).ok()?;
|
||||
return Some(Tokenizer { inner: bpe });
|
||||
}
|
||||
let bpe = tiktoken_rs::get_bpe_from_model(&model_name).ok()?;
|
||||
Some(Tokenizer { inner: bpe })
|
||||
}
|
||||
@@ -31,7 +37,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_tokenizer() {
|
||||
let tokenizer = from_model_name("gpt-4.1".to_string()).unwrap();
|
||||
let tokenizer = from_model_name("gpt-5".to_string()).unwrap();
|
||||
let content = "Hello, world!";
|
||||
let count = tokenizer.count(content.to_string(), None);
|
||||
assert!(count > 0);
|
||||
|
||||
Reference in New Issue
Block a user