Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-21 03:56:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): context awareness for copilot (#9611)

Browse Source 
fix PD-2167
fix PD-2169
fix PD-2190
This commit is contained in:
darkskygit
2025-03-13 11:44:55 +00:00
parent 05f3069efd
commit d8373f66e7
51 changed files with 2101 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/common/native/src/doc_loader/splitter/options.rs
+4 -3
View File
@@ -25,11 +25,11 @@ impl Default for SplitterOptions {
impl SplitterOptions {
pub fn new() -> Self {
SplitterOptions {
chunk_size: 512,
chunk_overlap: 0,
chunk_size: 7168,
chunk_overlap: 128,
model_name: String::from("gpt-3.5-turbo"),
encoding_name: String::from("cl100k_base"),
trim_chunks: false,
trim_chunks: true,
}
}
}
@@ -63,6 +63,7 @@ impl SplitterOptions {
pub fn get_tokenizer_from_str(s: &str) -> Option<Tokenizer> {
match s.to_lowercase().as_str() {
"o200k_base" => Some(Tokenizer::O200kBase),
"cl100k_base" => Some(Tokenizer::Cl100kBase),
"p50k_base" => Some(Tokenizer::P50kBase),
"r50k_base" => Some(Tokenizer::R50kBase),