feat(server): use native tokenizer impl (#6960)

### Benchmark

`yarn workspace @affine/server-native bench`

```
┌─────────┬────────────┬─────────┬────────────────────┬──────────┬─────────┐
│ (index) │ Task Name  │ ops/sec │ Average Time (ns)  │ Margin   │ Samples │
├─────────┼────────────┼─────────┼────────────────────┼──────────┼─────────┤
│ 0       │ 'tiktoken' │ '5'     │ 176932518.76000002 │ '±4.71%' │ 100     │
│ 1       │ 'native'   │ '16'    │ 61041597.51000003  │ '±0.60%' │ 100     │
└─────────┴────────────┴─────────┴────────────────────┴──────────┴─────────┘
```
This commit is contained in:
Brooooooklyn
2024-05-16 07:55:10 +00:00
parent 46140039d9
commit c7ddd679fd
17 changed files with 206 additions and 39 deletions
@@ -18,7 +18,7 @@ registerStorageProvider('fs', (config, bucket) => {
})
export class StorageProviderModule {}
export * from './native';
export * from '../../native';
export type {
BlobInputType,
BlobOutputType,
@@ -1,32 +0,0 @@
import { createRequire } from 'node:module';
let serverNativeModule: typeof import('@affine/server-native');
try {
serverNativeModule = await import('@affine/server-native');
} catch {
const require = createRequire(import.meta.url);
serverNativeModule =
process.arch === 'arm64'
? require('../../../server-native.arm64.node')
: process.arch === 'arm'
? require('../../../server-native.armv7.node')
: require('../../../server-native.node');
}
export const mergeUpdatesInApplyWay = serverNativeModule.mergeUpdatesInApplyWay;
export const verifyChallengeResponse = async (
response: any,
bits: number,
resource: string
) => {
if (typeof response !== 'string' || !response || !resource) return false;
return serverNativeModule.verifyChallengeResponse(response, bits, resource);
};
export const mintChallengeResponse = async (resource: string, bits: number) => {
if (!resource) return null;
return serverNativeModule.mintChallengeResponse(resource, bits);
};
export const getMime = serverNativeModule.getMime;
@@ -3,7 +3,7 @@ import { Readable } from 'node:stream';
import { crc32 } from '@node-rs/crc32';
import { getStreamAsBuffer } from 'get-stream';
import { getMime } from '../native';
import { getMime } from '../../../native';
import { BlobInputType, PutObjectMetadata } from './provider';
export async function toBuffer(input: BlobInputType): Promise<Buffer> {