feat(server): use native tokenizer impl (#6960)

mirror of https://github.com/toeverything/AFFiNE.git synced 2026-09-07 01:09:54 +08:00

### Benchmark

`yarn workspace @affine/server-native bench`

```
┌─────────┬────────────┬─────────┬────────────────────┬──────────┬─────────┐
│ (index) │ Task Name  │ ops/sec │ Average Time (ns)  │ Margin   │ Samples │
├─────────┼────────────┼─────────┼────────────────────┼──────────┼─────────┤
│ 0       │ 'tiktoken' │ '5'     │ 176932518.76000002 │ '±4.71%' │ 100     │
│ 1       │ 'native'   │ '16'    │ 61041597.51000003  │ '±0.60%' │ 100     │
└─────────┴────────────┴─────────┴────────────────────┴──────────┴─────────┘
```

This commit is contained in:

Brooooooklyn

2024-05-16 07:55:10 +00:00

parent 46140039d9

commit c7ddd679fd

17 changed files with 206 additions and 39 deletions

									
										Cargo.toml
									
		+3
		
												View File
												
				@@ -6,6 +6,9 @@ members = [

				  "./packages/backend/native",

				]

				[workspace.dependencies]

				mimalloc = "0.1"

				[profile.dev.package.sqlx-macros]

				opt-level = 3