mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-08-06 11:59:51 +08:00
feat(server): use native tokenizer impl (#6960)
### Benchmark `yarn workspace @affine/server-native bench` ``` ┌─────────┬────────────┬─────────┬────────────────────┬──────────┬─────────┐ │ (index) │ Task Name │ ops/sec │ Average Time (ns) │ Margin │ Samples │ ├─────────┼────────────┼─────────┼────────────────────┼──────────┼─────────┤ │ 0 │ 'tiktoken' │ '5' │ 176932518.76000002 │ '±4.71%' │ 100 │ │ 1 │ 'native' │ '16' │ 61041597.51000003 │ '±0.60%' │ 100 │ └─────────┴────────────┴─────────┴────────────────────┴──────────┴─────────┘ ```
This commit is contained in:
@@ -18,7 +18,7 @@ registerStorageProvider('fs', (config, bucket) => {
|
||||
})
|
||||
export class StorageProviderModule {}
|
||||
|
||||
export * from './native';
|
||||
export * from '../../native';
|
||||
export type {
|
||||
BlobInputType,
|
||||
BlobOutputType,
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
let serverNativeModule: typeof import('@affine/server-native');
|
||||
try {
|
||||
serverNativeModule = await import('@affine/server-native');
|
||||
} catch {
|
||||
const require = createRequire(import.meta.url);
|
||||
serverNativeModule =
|
||||
process.arch === 'arm64'
|
||||
? require('../../../server-native.arm64.node')
|
||||
: process.arch === 'arm'
|
||||
? require('../../../server-native.armv7.node')
|
||||
: require('../../../server-native.node');
|
||||
}
|
||||
|
||||
export const mergeUpdatesInApplyWay = serverNativeModule.mergeUpdatesInApplyWay;
|
||||
|
||||
export const verifyChallengeResponse = async (
|
||||
response: any,
|
||||
bits: number,
|
||||
resource: string
|
||||
) => {
|
||||
if (typeof response !== 'string' || !response || !resource) return false;
|
||||
return serverNativeModule.verifyChallengeResponse(response, bits, resource);
|
||||
};
|
||||
|
||||
export const mintChallengeResponse = async (resource: string, bits: number) => {
|
||||
if (!resource) return null;
|
||||
return serverNativeModule.mintChallengeResponse(resource, bits);
|
||||
};
|
||||
|
||||
export const getMime = serverNativeModule.getMime;
|
||||
@@ -3,7 +3,7 @@ import { Readable } from 'node:stream';
|
||||
import { crc32 } from '@node-rs/crc32';
|
||||
import { getStreamAsBuffer } from 'get-stream';
|
||||
|
||||
import { getMime } from '../native';
|
||||
import { getMime } from '../../../native';
|
||||
import { BlobInputType, PutObjectMetadata } from './provider';
|
||||
|
||||
export async function toBuffer(input: BlobInputType): Promise<Buffer> {
|
||||
|
||||
Reference in New Issue
Block a user