Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-12 04:18:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): use native tokenizer impl (#6960)

Browse Source 
### Benchmark

`yarn workspace @affine/server-native bench`

```
┌─────────┬────────────┬─────────┬────────────────────┬──────────┬─────────┐
│ (index) │ Task Name  │ ops/sec │ Average Time (ns)  │ Margin   │ Samples │
├─────────┼────────────┼─────────┼────────────────────┼──────────┼─────────┤
│ 0       │ 'tiktoken' │ '5'     │ 176932518.76000002 │ '±4.71%' │ 100     │
│ 1       │ 'native'   │ '16'    │ 61041597.51000003  │ '±0.60%' │ 100     │
└─────────┴────────────┴─────────┴────────────────────┴──────────┴─────────┘
```
This commit is contained in:
Brooooooklyn
2024-05-16 07:55:10 +00:00
parent 46140039d9
commit c7ddd679fd
17 changed files with 206 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/backend/native/package.json
View File

@@ -28,6 +28,7 @@
},
"scripts": {
"test": "node --test ./__tests__/**/*.spec.js",
"bench": "node ./benchmark/index.js",
"build": "napi build --release --strip --no-const-enum",
"build:debug": "napi build"
},
@@ -36,6 +37,8 @@
"lib0": "^0.2.93",
"nx": "^19.0.0",
"nx-cloud": "^19.0.0",
"tiktoken": "^1.0.15",
"tinybench": "^2.8.0",
"yjs": "^13.6.14"
}
}