Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-13 21:05:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add cache for tokenizer (#13333)

Browse Source 
<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **Performance Improvements**
* Improved the efficiency of token encoder retrieval, resulting in
faster response times when working with supported models.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2025-07-28 11:50:39 +08:00
committed by GitHub
parent 8fd0d5c1e8
commit 0e3691e54e
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
packages/backend/server/src/native.ts
View File

@@ -16,16 +16,24 @@ export const mintChallengeResponse = async (resource: string, bits: number) => {
return serverNativeModule.mintChallengeResponse(resource, bits);
};
const ENCODER_CACHE = new Map<string, Tokenizer>();
export function getTokenEncoder(model?: string | null): Tokenizer | null {
if (!model) return null;
const cached = ENCODER_CACHE.get(model);
if (cached) return cached;
if (model.startsWith('gpt')) {
return serverNativeModule.fromModelName(model);
const encoder = serverNativeModule.fromModelName(model);
if (encoder) ENCODER_CACHE.set(model, encoder);
return encoder;
} else if (model.startsWith('dall')) {
// dalle don't need to calc the token
return null;
} else {
// c100k based model
return serverNativeModule.fromModelName('gpt-4');
const encoder = serverNativeModule.fromModelName('gpt-4');
if (encoder) ENCODER_CACHE.set('gpt-4', encoder);
return encoder;
}
}