From 0e3691e54e0cecc8449f37b5655a0e1e8d29e7d2 Mon Sep 17 00:00:00 2001
From: DarkSky <25152247+darkskygit@users.noreply.github.com>
Date: Mon, 28 Jul 2025 11:50:39 +0800
Subject: [PATCH] feat: add cache for tokenizer (#13333)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **Performance Improvements**
* Improved the efficiency of token encoder retrieval, resulting in
faster response times when working with supported models.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
---
 packages/backend/server/src/native.ts | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/packages/backend/server/src/native.ts b/packages/backend/server/src/native.ts
index 135d47424a..9506cc7e09 100644
--- a/packages/backend/server/src/native.ts
+++ b/packages/backend/server/src/native.ts
@@ -16,16 +16,24 @@ export const mintChallengeResponse = async (resource: string, bits: number) => {
   return serverNativeModule.mintChallengeResponse(resource, bits);
 };
 
+const ENCODER_CACHE = new Map<string, Tokenizer>();
+
 export function getTokenEncoder(model?: string | null): Tokenizer | null {
   if (!model) return null;
+  const cached = ENCODER_CACHE.get(model);
+  if (cached) return cached;
   if (model.startsWith('gpt')) {
-    return serverNativeModule.fromModelName(model);
+    const encoder = serverNativeModule.fromModelName(model);
+    if (encoder) ENCODER_CACHE.set(model, encoder);
+    return encoder;
   } else if (model.startsWith('dall')) {
     // dalle don't need to calc the token
     return null;
   } else {
     // c100k based model
-    return serverNativeModule.fromModelName('gpt-4');
+    const encoder = serverNativeModule.fromModelName('gpt-4');
+    if (encoder) ENCODER_CACHE.set('gpt-4', encoder);
+    return encoder;
   }
 }