mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-23 13:29:02 +08:00
feat(nbstore): add indexer storage (#10953)
This commit is contained in:
EYHN
@@ -3,15 +3,26 @@ import { IndexedDBBlobStorage } from './blob';
|
||||
import { IndexedDBBlobSyncStorage } from './blob-sync';
|
||||
import { IndexedDBDocStorage } from './doc';
|
||||
import { IndexedDBDocSyncStorage } from './doc-sync';
|
||||
import { IndexedDBIndexerStorage } from './indexer';
|
||||
import { IndexedDBIndexerSyncStorage } from './indexer-sync';
|
||||
|
||||
export * from './blob';
|
||||
export * from './blob-sync';
|
||||
export * from './doc';
|
||||
export * from './doc-sync';
|
||||
export * from './indexer';
|
||||
export * from './indexer-sync';
|
||||
|
||||
export const idbStorages = [
|
||||
IndexedDBDocStorage,
|
||||
IndexedDBBlobStorage,
|
||||
IndexedDBDocSyncStorage,
|
||||
IndexedDBBlobSyncStorage,
|
||||
IndexedDBIndexerStorage,
|
||||
IndexedDBIndexerSyncStorage,
|
||||
] satisfies StorageConstructor[];
|
||||
|
||||
export const idbStoragesIndexerOnly = [
|
||||
IndexedDBIndexerStorage,
|
||||
IndexedDBIndexerSyncStorage,
|
||||
] satisfies StorageConstructor[];
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { share } from '../../connection';
|
||||
import type { DocClock } from '../../storage/doc';
|
||||
import { IndexerSyncStorageBase } from '../../storage/indexer-sync';
|
||||
import { IDBConnection, type IDBConnectionOptions } from './db';
|
||||
|
||||
export class IndexedDBIndexerSyncStorage extends IndexerSyncStorageBase {
|
||||
static readonly identifier = 'IndexedDBIndexerSyncStorage';
|
||||
|
||||
readonly connection = share(new IDBConnection(this.options));
|
||||
|
||||
constructor(private readonly options: IDBConnectionOptions) {
|
||||
super();
|
||||
}
|
||||
|
||||
async getDocIndexedClock(docId: string): Promise<DocClock | null> {
|
||||
const tx = this.connection.inner.db.transaction('indexerSync', 'readonly');
|
||||
const store = tx.store;
|
||||
const result = await store.get(docId);
|
||||
return result
|
||||
? { docId: result.docId, timestamp: result.indexedClock }
|
||||
: null;
|
||||
}
|
||||
|
||||
async setDocIndexedClock(docClock: DocClock): Promise<void> {
|
||||
const tx = this.connection.inner.db.transaction('indexerSync', 'readwrite');
|
||||
const store = tx.store;
|
||||
await store.put({
|
||||
docId: docClock.docId,
|
||||
indexedClock: docClock.timestamp,
|
||||
});
|
||||
}
|
||||
|
||||
async clearDocIndexedClock(docId: string): Promise<void> {
|
||||
const tx = this.connection.inner.db.transaction('indexerSync', 'readwrite');
|
||||
const store = tx.store;
|
||||
await store.delete(docId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { expect, test } from 'vitest';
|
||||
|
||||
import { bm25 } from '../bm25';
|
||||
|
||||
test('bm25', () => {
|
||||
expect(bm25(1, 1, 10, 10, 15)).toEqual(3.2792079793859643);
|
||||
expect(bm25(2, 1, 10, 10, 15) > bm25(1, 1, 10, 10, 15)).toBeTruthy();
|
||||
expect(bm25(1, 1, 10, 10, 15) > bm25(2, 1, 10, 100, 15)).toBeTruthy();
|
||||
expect(bm25(1, 1, 10, 10, 15) > bm25(1, 1, 10, 100, 15)).toBeTruthy();
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
import { expect, test } from 'vitest';
|
||||
|
||||
import { highlighter } from '../highlighter';
|
||||
|
||||
test('highlighter', () => {
|
||||
expect(highlighter('0123456789', '<b>', '</b>', [[3, 5]])).toEqual(
|
||||
'012<b>34</b>56789'
|
||||
);
|
||||
|
||||
expect(
|
||||
highlighter(
|
||||
'012345678901234567890123456789012345678901234567890123456789',
|
||||
'<b>',
|
||||
'</b>',
|
||||
[[59, 60]]
|
||||
)
|
||||
).toEqual('...0123456789012345678901234567890123456789012345678<b>9</b>');
|
||||
|
||||
expect(
|
||||
highlighter(
|
||||
'012345678901234567890123456789012345678901234567890123456789',
|
||||
'<b>',
|
||||
'</b>',
|
||||
[
|
||||
[10, 11],
|
||||
[49, 51],
|
||||
]
|
||||
)
|
||||
).toEqual(
|
||||
'0123456789<b>0</b>12345678901234567890123456789012345678<b>9</b>...'
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,128 @@
|
||||
import { expect, test } from 'vitest';
|
||||
|
||||
import { GeneralTokenizer } from '../tokenizer';
|
||||
|
||||
test('tokenizer', () => {
|
||||
{
|
||||
const tokens = new GeneralTokenizer().tokenize('hello world,\n AFFiNE');
|
||||
|
||||
expect(tokens).toEqual([
|
||||
{ term: 'hello', start: 0, end: 5 },
|
||||
{ term: 'world', start: 7, end: 12 },
|
||||
{ term: 'affine', start: 15, end: 21 },
|
||||
]);
|
||||
}
|
||||
|
||||
{
|
||||
const tokens = new GeneralTokenizer().tokenize('你好世界，阿芬');
|
||||
|
||||
expect(tokens).toEqual([
|
||||
{
|
||||
end: 2,
|
||||
start: 0,
|
||||
term: '你好',
|
||||
},
|
||||
{
|
||||
end: 3,
|
||||
start: 1,
|
||||
term: '好世',
|
||||
},
|
||||
{
|
||||
end: 4,
|
||||
start: 2,
|
||||
term: '世界',
|
||||
},
|
||||
{
|
||||
end: 7,
|
||||
start: 5,
|
||||
term: '阿芬',
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
{
|
||||
const tokens = new GeneralTokenizer().tokenize('1阿2芬');
|
||||
|
||||
expect(tokens).toEqual([
|
||||
{ term: '1', start: 0, end: 1 },
|
||||
{ term: '阿', start: 1, end: 2 },
|
||||
{ term: '2', start: 2, end: 3 },
|
||||
{ term: '芬', start: 3, end: 4 },
|
||||
]);
|
||||
}
|
||||
|
||||
{
|
||||
const tokens = new GeneralTokenizer().tokenize('안녕하세요 세계');
|
||||
|
||||
expect(tokens).toEqual([
|
||||
{
|
||||
end: 2,
|
||||
start: 0,
|
||||
term: '안녕',
|
||||
},
|
||||
{
|
||||
end: 3,
|
||||
start: 1,
|
||||
term: '녕하',
|
||||
},
|
||||
{
|
||||
end: 4,
|
||||
start: 2,
|
||||
term: '하세',
|
||||
},
|
||||
{
|
||||
end: 5,
|
||||
start: 3,
|
||||
term: '세요',
|
||||
},
|
||||
{
|
||||
end: 8,
|
||||
start: 6,
|
||||
term: '세계',
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
{
|
||||
const tokens = new GeneralTokenizer().tokenize('ハローワールド');
|
||||
|
||||
expect(tokens).toEqual([
|
||||
{ term: 'ハロ', start: 0, end: 2 },
|
||||
{ term: 'ロー', start: 1, end: 3 },
|
||||
{ term: 'ーワ', start: 2, end: 4 },
|
||||
{ term: 'ワー', start: 3, end: 5 },
|
||||
{ term: 'ール', start: 4, end: 6 },
|
||||
{ term: 'ルド', start: 5, end: 7 },
|
||||
]);
|
||||
}
|
||||
|
||||
{
|
||||
const tokens = new GeneralTokenizer().tokenize('はろーわーるど');
|
||||
|
||||
expect(tokens).toEqual([
|
||||
{ term: 'はろ', start: 0, end: 2 },
|
||||
{ term: 'ろー', start: 1, end: 3 },
|
||||
{ term: 'ーわ', start: 2, end: 4 },
|
||||
{ term: 'わー', start: 3, end: 5 },
|
||||
{ term: 'ーる', start: 4, end: 6 },
|
||||
{ term: 'るど', start: 5, end: 7 },
|
||||
]);
|
||||
}
|
||||
|
||||
{
|
||||
const tokens = new GeneralTokenizer().tokenize('👋1️⃣🚪👋🏿');
|
||||
|
||||
expect(tokens).toEqual([
|
||||
{ term: '👋', start: 0, end: 2 },
|
||||
{ term: '1️⃣', start: 2, end: 5 },
|
||||
{ term: '🚪', start: 5, end: 7 },
|
||||
{ term: '👋🏿', start: 7, end: 11 },
|
||||
]);
|
||||
}
|
||||
|
||||
{
|
||||
const tokens = new GeneralTokenizer().tokenize('1️');
|
||||
|
||||
expect(tokens).toEqual([{ term: '1️', start: 0, end: 2 }]);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Parameters of the BM25+ scoring algorithm. Customizing these is almost never
|
||||
* necessary, and finetuning them requires an understanding of the BM25 scoring
|
||||
* model.
|
||||
*
|
||||
* Some information about BM25 (and BM25+) can be found at these links:
|
||||
*
|
||||
* - https://en.wikipedia.org/wiki/Okapi_BM25
|
||||
* - https://opensourceconnections.com/blog/2015/10/16/bm25-the-next-generation-of-lucene-relevation/
|
||||
*/
|
||||
export type BM25Params = {
|
||||
/** Term frequency saturation point.
|
||||
*
|
||||
* Recommended values are between `1.2` and `2`. Higher values increase the
|
||||
* difference in score between documents with higher and lower term
|
||||
* frequencies. Setting this to `0` or a negative value is invalid. Defaults
|
||||
* to `1.2`
|
||||
*/
|
||||
k: number;
|
||||
|
||||
/**
|
||||
* Length normalization impact.
|
||||
*
|
||||
* Recommended values are around `0.75`. Higher values increase the weight
|
||||
* that field length has on scoring. Setting this to `0` (not recommended)
|
||||
* means that the field length has no effect on scoring. Negative values are
|
||||
* invalid. Defaults to `0.7`.
|
||||
*/
|
||||
b: number;
|
||||
|
||||
/**
|
||||
* BM25+ frequency normalization lower bound (usually called δ).
|
||||
*
|
||||
* Recommended values are between `0.5` and `1`. Increasing this parameter
|
||||
* increases the minimum relevance of one occurrence of a search term
|
||||
* regardless of its (possibly very long) field length. Negative values are
|
||||
* invalid. Defaults to `0.5`.
|
||||
*/
|
||||
d: number;
|
||||
};
|
||||
|
||||
const defaultBM25params: BM25Params = { k: 1.2, b: 0.7, d: 0.5 };
|
||||
|
||||
export const bm25 = (
|
||||
termFreq: number,
|
||||
matchingCount: number,
|
||||
totalCount: number,
|
||||
fieldLength: number,
|
||||
avgFieldLength: number,
|
||||
bm25params: BM25Params = defaultBM25params
|
||||
): number => {
|
||||
const { k, b, d } = bm25params;
|
||||
const invDocFreq = Math.log(
|
||||
1 + (totalCount - matchingCount + 0.5) / (matchingCount + 0.5)
|
||||
);
|
||||
return (
|
||||
invDocFreq *
|
||||
(d +
|
||||
(termFreq * (k + 1)) /
|
||||
(termFreq + k * (1 - b + b * (fieldLength / avgFieldLength))))
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,493 @@
|
||||
import { type IDBPDatabase, type IDBPTransaction, type StoreNames } from 'idb';
|
||||
|
||||
import {
|
||||
type AggregateOptions,
|
||||
type AggregateResult,
|
||||
type IndexerDocument,
|
||||
type IndexerFieldSchema,
|
||||
IndexerSchema,
|
||||
type Query,
|
||||
type SearchOptions,
|
||||
type SearchResult,
|
||||
} from '../../../storage';
|
||||
import type { DocStorageSchema } from '../schema';
|
||||
import { highlighter } from './highlighter';
|
||||
import {
|
||||
BooleanInvertedIndex,
|
||||
FullTextInvertedIndex,
|
||||
IntegerInvertedIndex,
|
||||
type InvertedIndex,
|
||||
StringInvertedIndex,
|
||||
} from './inverted-index';
|
||||
import { Match } from './match';
|
||||
|
||||
export type DataStructRWTransaction = IDBPTransaction<
|
||||
DocStorageSchema,
|
||||
ArrayLike<StoreNames<DocStorageSchema>>,
|
||||
'readwrite'
|
||||
>;
|
||||
|
||||
export type DataStructROTransaction = IDBPTransaction<
|
||||
DocStorageSchema,
|
||||
ArrayLike<StoreNames<DocStorageSchema>>,
|
||||
'readonly' | 'readwrite'
|
||||
>;
|
||||
|
||||
export class DataStruct {
|
||||
database: IDBPDatabase<DocStorageSchema> = null as any;
|
||||
invertedIndex = new Map<string, Map<string, InvertedIndex>>();
|
||||
|
||||
constructor() {
|
||||
for (const [tableName, table] of Object.entries(IndexerSchema)) {
|
||||
const tableInvertedIndex = new Map<string, InvertedIndex>();
|
||||
|
||||
for (const [fieldName, type] of Object.entries(table)) {
|
||||
const typeInfo: IndexerFieldSchema =
|
||||
typeof type === 'string' ? { type } : type;
|
||||
if ('index' in typeInfo && typeInfo.index === false) {
|
||||
// If index is false, we don't need to create an inverted index for this field.
|
||||
continue;
|
||||
}
|
||||
if (typeInfo.type === 'String') {
|
||||
tableInvertedIndex.set(
|
||||
fieldName,
|
||||
new StringInvertedIndex(tableName, fieldName)
|
||||
);
|
||||
} else if (typeInfo.type === 'Integer') {
|
||||
tableInvertedIndex.set(
|
||||
fieldName,
|
||||
new IntegerInvertedIndex(tableName, fieldName)
|
||||
);
|
||||
} else if (typeInfo.type === 'FullText') {
|
||||
tableInvertedIndex.set(
|
||||
fieldName,
|
||||
new FullTextInvertedIndex(tableName, fieldName)
|
||||
);
|
||||
} else if (typeInfo.type === 'Boolean') {
|
||||
tableInvertedIndex.set(
|
||||
fieldName,
|
||||
new BooleanInvertedIndex(tableName, fieldName)
|
||||
);
|
||||
} else {
|
||||
throw new Error(`Field type '${typeInfo.type}' not supported`);
|
||||
}
|
||||
}
|
||||
|
||||
this.invertedIndex.set(tableName, tableInvertedIndex);
|
||||
}
|
||||
}
|
||||
|
||||
private async update(
|
||||
trx: DataStructRWTransaction,
|
||||
table: keyof IndexerSchema,
|
||||
document: IndexerDocument
|
||||
) {
|
||||
const existsNid = await trx
|
||||
.objectStore('indexerRecords')
|
||||
.index('id')
|
||||
.getKey([table, document.id]);
|
||||
|
||||
const exists = existsNid
|
||||
? await trx.objectStore('indexerRecords').get(existsNid)
|
||||
: null;
|
||||
|
||||
if (!existsNid || !exists) {
|
||||
// if not exists, return
|
||||
return;
|
||||
}
|
||||
|
||||
// delete exists one
|
||||
await this.deleteByNid(trx, existsNid);
|
||||
|
||||
const dataMap = new Map([...exists.data, ...document.fields]); // merge exists data with new data
|
||||
const nid = await trx
|
||||
.objectStore('indexerRecords')
|
||||
.put({ table, id: document.id, data: dataMap });
|
||||
|
||||
for (const [key, values] of dataMap) {
|
||||
const type = IndexerSchema[table][
|
||||
key as keyof IndexerSchema[typeof table]
|
||||
] as IndexerFieldSchema;
|
||||
if (!type) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const typeInfo = typeof type === 'string' ? { type } : type;
|
||||
if (typeInfo.index !== false) {
|
||||
// If index is false, the field will not be indexed
|
||||
const iidx = this.invertedIndex.get(table)?.get(key);
|
||||
if (!iidx) {
|
||||
continue;
|
||||
}
|
||||
await iidx.insert(trx, nid, values);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async insert(
|
||||
trx: DataStructRWTransaction,
|
||||
table: keyof IndexerSchema,
|
||||
document: IndexerDocument
|
||||
) {
|
||||
const existsNid = await trx
|
||||
.objectStore('indexerRecords')
|
||||
.index('id')
|
||||
.getKey([table, document.id]);
|
||||
|
||||
if (existsNid) {
|
||||
// delete exists one
|
||||
await this.deleteByNid(trx, existsNid);
|
||||
}
|
||||
|
||||
const dataMap = document.fields;
|
||||
|
||||
const nid = await trx
|
||||
.objectStore('indexerRecords')
|
||||
.put({ table, id: document.id, data: dataMap });
|
||||
|
||||
for (const [key, values] of dataMap) {
|
||||
const type = IndexerSchema[table][
|
||||
key as keyof IndexerSchema[typeof table]
|
||||
] as IndexerFieldSchema;
|
||||
if (!type) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const typeInfo = typeof type === 'string' ? { type } : type;
|
||||
if (typeInfo.index !== false) {
|
||||
// If index is false, the field will not be indexed
|
||||
const iidx = this.invertedIndex.get(table)?.get(key);
|
||||
if (!iidx) {
|
||||
continue;
|
||||
}
|
||||
await iidx.insert(trx, nid, values);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async deleteByNid(trx: DataStructRWTransaction, nid: number) {
|
||||
await trx.objectStore('indexerRecords').delete(nid);
|
||||
|
||||
const indexIds = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('nid')
|
||||
.getAllKeys(nid);
|
||||
|
||||
for (const indexId of indexIds) {
|
||||
await trx.objectStore('invertedIndex').delete(indexId);
|
||||
}
|
||||
}
|
||||
|
||||
private async delete(
|
||||
trx: DataStructRWTransaction,
|
||||
table: keyof IndexerSchema,
|
||||
id: string
|
||||
) {
|
||||
const nid = await trx
|
||||
.objectStore('indexerRecords')
|
||||
.index('id')
|
||||
.getKey([table, id]);
|
||||
|
||||
if (nid) {
|
||||
await this.deleteByNid(trx, nid);
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async deleteByQuery(
|
||||
trx: DataStructRWTransaction,
|
||||
table: keyof IndexerSchema,
|
||||
query: Query<any>
|
||||
) {
|
||||
const match = await this.queryRaw(trx, table, query);
|
||||
|
||||
for (const nid of match.scores.keys()) {
|
||||
await this.deleteByNid(trx, nid);
|
||||
}
|
||||
}
|
||||
|
||||
async batchWrite(
|
||||
trx: DataStructRWTransaction,
|
||||
table: keyof IndexerSchema,
|
||||
deleteByQueries: Query<any>[],
|
||||
deletes: string[],
|
||||
inserts: IndexerDocument<any>[],
|
||||
updates: IndexerDocument<any>[]
|
||||
) {
|
||||
for (const query of deleteByQueries) {
|
||||
await this.deleteByQuery(trx, table, query);
|
||||
}
|
||||
for (const del of deletes) {
|
||||
await this.delete(trx, table, del);
|
||||
}
|
||||
for (const inst of inserts) {
|
||||
await this.insert(trx, table, inst);
|
||||
}
|
||||
for (const update of updates) {
|
||||
await this.update(trx, table, update);
|
||||
}
|
||||
}
|
||||
|
||||
async matchAll(
|
||||
trx: DataStructROTransaction,
|
||||
table: keyof IndexerSchema
|
||||
): Promise<Match> {
|
||||
const allNids = await trx
|
||||
.objectStore('indexerRecords')
|
||||
.index('table')
|
||||
.getAllKeys(table);
|
||||
const match = new Match();
|
||||
|
||||
for (const nid of allNids) {
|
||||
match.addScore(nid, 1);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
async queryRaw(
|
||||
trx: DataStructROTransaction,
|
||||
table: keyof IndexerSchema,
|
||||
query: Query<any>
|
||||
): Promise<Match> {
|
||||
if (query.type === 'match') {
|
||||
const iidx = this.invertedIndex.get(table)?.get(query.field as string);
|
||||
if (!iidx) {
|
||||
return new Match();
|
||||
}
|
||||
return await iidx.match(trx, query.match);
|
||||
} else if (query.type === 'boolean') {
|
||||
const weights = [];
|
||||
for (const q of query.queries) {
|
||||
weights.push(await this.queryRaw(trx, table, q));
|
||||
}
|
||||
if (query.occur === 'must') {
|
||||
return weights.reduce((acc, w) => acc.and(w));
|
||||
} else if (query.occur === 'must_not') {
|
||||
const total = weights.reduce((acc, w) => acc.and(w));
|
||||
return (await this.matchAll(trx, table)).exclude(total);
|
||||
} else if (query.occur === 'should') {
|
||||
return weights.reduce((acc, w) => acc.or(w));
|
||||
}
|
||||
} else if (query.type === 'all') {
|
||||
return await this.matchAll(trx, table);
|
||||
} else if (query.type === 'boost') {
|
||||
return (await this.queryRaw(trx, table, query.query)).boost(query.boost);
|
||||
} else if (query.type === 'exists') {
|
||||
const iidx = this.invertedIndex.get(table)?.get(query.field as string);
|
||||
if (!iidx) {
|
||||
return new Match();
|
||||
}
|
||||
return await iidx.all(trx);
|
||||
}
|
||||
throw new Error(`Query type '${query.type}' not supported`);
|
||||
}
|
||||
|
||||
async clear(trx: DataStructRWTransaction) {
|
||||
await trx.objectStore('indexerRecords').clear();
|
||||
await trx.objectStore('invertedIndex').clear();
|
||||
await trx.objectStore('indexerMetadata').clear();
|
||||
}
|
||||
|
||||
async search(
|
||||
trx: DataStructROTransaction,
|
||||
table: keyof IndexerSchema,
|
||||
query: Query<any>,
|
||||
options: SearchOptions<any> = {}
|
||||
): Promise<SearchResult<any, any>> {
|
||||
const pagination = {
|
||||
skip: options.pagination?.skip ?? 0,
|
||||
limit: options.pagination?.limit ?? 100,
|
||||
};
|
||||
|
||||
const match = await this.queryRaw(trx, table, query);
|
||||
|
||||
const nids = match
|
||||
.toArray()
|
||||
.slice(pagination.skip, pagination.skip + pagination.limit);
|
||||
|
||||
const nodes = [];
|
||||
for (const nid of nids) {
|
||||
const record = await trx.objectStore('indexerRecords').get(nid);
|
||||
if (!record) {
|
||||
continue;
|
||||
}
|
||||
nodes.push(this.resultNode(record, options, match, nid));
|
||||
}
|
||||
|
||||
return {
|
||||
pagination: {
|
||||
count: match.size(),
|
||||
hasMore: match.size() > pagination.limit + pagination.skip,
|
||||
limit: pagination.limit,
|
||||
skip: pagination.skip,
|
||||
},
|
||||
nodes: nodes,
|
||||
};
|
||||
}
|
||||
|
||||
async aggregate(
|
||||
trx: DataStructROTransaction,
|
||||
table: keyof IndexerSchema,
|
||||
query: Query<any>,
|
||||
field: string,
|
||||
options: AggregateOptions<any> = {}
|
||||
): Promise<AggregateResult<any, any>> {
|
||||
const pagination = {
|
||||
skip: options.pagination?.skip ?? 0,
|
||||
limit: options.pagination?.limit ?? 100,
|
||||
};
|
||||
|
||||
const hitPagination = options.hits
|
||||
? {
|
||||
skip: options.hits.pagination?.skip ?? 0,
|
||||
limit: options.hits.pagination?.limit ?? 3,
|
||||
}
|
||||
: { skip: 0, limit: 0 };
|
||||
|
||||
const match = await this.queryRaw(trx, table, query);
|
||||
|
||||
const nids = match.toArray();
|
||||
|
||||
const buckets: {
|
||||
key: string;
|
||||
nids: number[];
|
||||
hits: SearchResult<any, any>['nodes'];
|
||||
}[] = [];
|
||||
|
||||
for (const nid of nids) {
|
||||
const record = await trx.objectStore('indexerRecords').get(nid);
|
||||
if (!record) {
|
||||
continue;
|
||||
}
|
||||
const values = record.data.get(field);
|
||||
for (const value of values ?? []) {
|
||||
let bucket;
|
||||
let bucketIndex = buckets.findIndex(b => b.key === value);
|
||||
if (bucketIndex === -1) {
|
||||
bucket = { key: value, nids: [], hits: [] };
|
||||
buckets.push(bucket);
|
||||
bucketIndex = buckets.length - 1;
|
||||
} else {
|
||||
bucket = buckets[bucketIndex];
|
||||
}
|
||||
|
||||
if (
|
||||
bucketIndex >= pagination.skip &&
|
||||
bucketIndex < pagination.skip + pagination.limit
|
||||
) {
|
||||
bucket.nids.push(nid);
|
||||
if (
|
||||
bucket.nids.length - 1 >= hitPagination.skip &&
|
||||
bucket.nids.length - 1 < hitPagination.skip + hitPagination.limit
|
||||
) {
|
||||
bucket.hits.push(
|
||||
this.resultNode(record, options.hits ?? {}, match, nid)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
buckets: buckets
|
||||
.slice(pagination.skip, pagination.skip + pagination.limit)
|
||||
.map(bucket => {
|
||||
const result = {
|
||||
key: bucket.key,
|
||||
score: match.getScore(bucket.nids[0]),
|
||||
count: bucket.nids.length,
|
||||
} as AggregateResult<any, any>['buckets'][number];
|
||||
|
||||
if (options.hits) {
|
||||
(result as any).hits = {
|
||||
pagination: {
|
||||
count: bucket.nids.length,
|
||||
hasMore:
|
||||
bucket.nids.length > hitPagination.limit + hitPagination.skip,
|
||||
limit: hitPagination.limit,
|
||||
skip: hitPagination.skip,
|
||||
},
|
||||
nodes: bucket.hits,
|
||||
} as SearchResult<any, any>;
|
||||
}
|
||||
|
||||
return result;
|
||||
}),
|
||||
pagination: {
|
||||
count: buckets.length,
|
||||
hasMore: buckets.length > pagination.limit + pagination.skip,
|
||||
limit: pagination.limit,
|
||||
skip: pagination.skip,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async readonly(database: IDBPDatabase<DocStorageSchema>) {
|
||||
return database.transaction(
|
||||
['indexerRecords', 'invertedIndex', 'indexerMetadata'],
|
||||
'readonly',
|
||||
{ durability: 'relaxed' }
|
||||
);
|
||||
}
|
||||
|
||||
async readwrite(database: IDBPDatabase<DocStorageSchema>) {
|
||||
return database.transaction(
|
||||
['indexerRecords', 'invertedIndex', 'indexerMetadata'],
|
||||
'readwrite',
|
||||
{ durability: 'relaxed' }
|
||||
);
|
||||
}
|
||||
|
||||
private resultNode(
|
||||
record: { id: string; data: Map<string, string[]> },
|
||||
options: SearchOptions<any>,
|
||||
match?: Match,
|
||||
nid?: number
|
||||
): SearchResult<any, any>['nodes'][number] {
|
||||
const node = {
|
||||
id: record.id,
|
||||
score: match && nid ? match.getScore(nid) : 1,
|
||||
} as any;
|
||||
|
||||
if (options.fields) {
|
||||
const fields = {} as Record<string, string | string[]>;
|
||||
for (const field of options.fields as string[]) {
|
||||
fields[field] = record.data.get(field) ?? [''];
|
||||
if (fields[field].length === 1) {
|
||||
fields[field] = fields[field][0];
|
||||
}
|
||||
}
|
||||
node.fields = fields;
|
||||
}
|
||||
|
||||
if (match && nid && options.highlights) {
|
||||
const highlights = {} as Record<string, string[]>;
|
||||
for (const { field, before, end } of options.highlights) {
|
||||
const highlightValues = match.getHighlighters(nid, field);
|
||||
if (highlightValues) {
|
||||
const rawValues = record.data.get(field) ?? [];
|
||||
highlights[field] = Array.from(highlightValues)
|
||||
.map(([index, ranges]) => {
|
||||
const raw = rawValues[index];
|
||||
|
||||
if (raw) {
|
||||
return (
|
||||
highlighter(raw, before, end, ranges, {
|
||||
maxPrefix: 20,
|
||||
maxLength: 50,
|
||||
}) ?? ''
|
||||
);
|
||||
}
|
||||
|
||||
return '';
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
}
|
||||
node.highlights = highlights;
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
export function highlighter(
|
||||
originText: string,
|
||||
before: string,
|
||||
after: string,
|
||||
matches: [number, number][],
|
||||
{
|
||||
maxLength = 50,
|
||||
maxPrefix = 20,
|
||||
}: { maxLength?: number; maxPrefix?: number } = {}
|
||||
) {
|
||||
const merged = mergeRanges(matches);
|
||||
|
||||
if (merged.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const firstMatch = merged[0][0];
|
||||
const start = Math.max(
|
||||
0,
|
||||
Math.min(firstMatch - maxPrefix, originText.length - maxLength)
|
||||
);
|
||||
const end = Math.min(start + maxLength, originText.length);
|
||||
const text = originText.substring(start, end);
|
||||
|
||||
let result = '';
|
||||
|
||||
let pointer = 0;
|
||||
for (const match of merged) {
|
||||
const matchStart = match[0] - start;
|
||||
const matchEnd = match[1] - start;
|
||||
if (matchStart >= text.length) {
|
||||
break;
|
||||
}
|
||||
result += text.substring(pointer, matchStart);
|
||||
pointer = matchStart;
|
||||
const highlighted = text.substring(matchStart, matchEnd);
|
||||
|
||||
if (highlighted.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
result += `${before}${highlighted}${after}`;
|
||||
pointer = matchEnd;
|
||||
}
|
||||
result += text.substring(pointer);
|
||||
|
||||
if (start > 0) {
|
||||
result = `...${result}`;
|
||||
}
|
||||
|
||||
if (end < originText.length) {
|
||||
result = `${result}...`;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function mergeRanges(intervals: [number, number][]) {
|
||||
if (intervals.length === 0) return [];
|
||||
|
||||
intervals.sort((a, b) => a[0] - b[0]);
|
||||
|
||||
const merged = [intervals[0]];
|
||||
|
||||
for (let i = 1; i < intervals.length; i++) {
|
||||
const last = merged[merged.length - 1];
|
||||
const current = intervals[i];
|
||||
|
||||
if (current[0] <= last[1]) {
|
||||
last[1] = Math.max(last[1], current[1]);
|
||||
} else {
|
||||
merged.push(current);
|
||||
}
|
||||
}
|
||||
|
||||
return merged;
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
import { merge, Observable, of, Subject, throttleTime } from 'rxjs';
|
||||
|
||||
import type {
|
||||
AggregateOptions,
|
||||
AggregateResult,
|
||||
IndexerDocument,
|
||||
IndexerSchema,
|
||||
Query,
|
||||
SearchOptions,
|
||||
SearchResult,
|
||||
} from '../../../storage';
|
||||
import { IndexerStorageBase } from '../../../storage';
|
||||
import { IDBConnection, type IDBConnectionOptions } from '../db';
|
||||
import { DataStruct } from './data-struct';
|
||||
import { backoffRetry, exhaustMapWithTrailing, fromPromise } from './utils';
|
||||
|
||||
export class IndexedDBIndexerStorage extends IndexerStorageBase {
|
||||
static readonly identifier = 'IndexedDBIndexerStorage';
|
||||
readonly connection = new IDBConnection(this.options);
|
||||
override isReadonly = false;
|
||||
private readonly data = new DataStruct();
|
||||
private readonly tableUpdate$ = new Subject<string>();
|
||||
|
||||
/**
|
||||
* The write operations of IndexedDBIndexerStorage are first cached in pendingUpdates,
|
||||
* and then committed to IndexedDB in a batch through the refresh method.
|
||||
*/
|
||||
private readonly pendingUpdates: Record<
|
||||
keyof IndexerSchema,
|
||||
{
|
||||
deleteByQueries: Query<any>[];
|
||||
deletes: string[];
|
||||
inserts: IndexerDocument[];
|
||||
updates: IndexerDocument[];
|
||||
}
|
||||
> = {
|
||||
doc: { deleteByQueries: [], deletes: [], inserts: [], updates: [] },
|
||||
block: { deleteByQueries: [], deletes: [], inserts: [], updates: [] },
|
||||
};
|
||||
|
||||
get channel() {
|
||||
return this.connection.inner.channel;
|
||||
}
|
||||
|
||||
get database() {
|
||||
return this.connection.inner.db;
|
||||
}
|
||||
|
||||
constructor(private readonly options: IDBConnectionOptions) {
|
||||
super();
|
||||
}
|
||||
|
||||
override async search<
|
||||
T extends keyof IndexerSchema,
|
||||
const O extends SearchOptions<T>,
|
||||
>(table: T, query: Query<T>, options?: O): Promise<SearchResult<T, O>> {
|
||||
const trx = await this.data.readonly(this.database);
|
||||
return this.data.search(trx, table, query, options);
|
||||
}
|
||||
override async aggregate<
|
||||
T extends keyof IndexerSchema,
|
||||
const O extends AggregateOptions<T>,
|
||||
>(
|
||||
table: T,
|
||||
query: Query<T>,
|
||||
field: keyof IndexerSchema[T],
|
||||
options?: O
|
||||
): Promise<AggregateResult<T, O>> {
|
||||
const trx = await this.data.readonly(this.database);
|
||||
return this.data.aggregate(trx, table, query, field as string, options);
|
||||
}
|
||||
override search$<
|
||||
T extends keyof IndexerSchema,
|
||||
const O extends SearchOptions<T>,
|
||||
>(table: T, query: Query<T>, options?: O): Observable<SearchResult<T, O>> {
|
||||
return merge(of(1), this.watchTableUpdated(table)).pipe(
|
||||
throttleTime(3000, undefined, { leading: true, trailing: true }),
|
||||
exhaustMapWithTrailing(() => {
|
||||
return fromPromise(async () => {
|
||||
try {
|
||||
const trx = await this.data.readonly(this.database);
|
||||
return await this.data.search(trx, table, query, options);
|
||||
} catch (error) {
|
||||
console.error('search error', error);
|
||||
throw error;
|
||||
}
|
||||
}).pipe(backoffRetry());
|
||||
})
|
||||
);
|
||||
}
|
||||
override aggregate$<
|
||||
T extends keyof IndexerSchema,
|
||||
const O extends AggregateOptions<T>,
|
||||
>(
|
||||
table: T,
|
||||
query: Query<T>,
|
||||
field: keyof IndexerSchema[T],
|
||||
options?: O
|
||||
): Observable<AggregateResult<T, O>> {
|
||||
return merge(of(1), this.watchTableUpdated(table)).pipe(
|
||||
throttleTime(3000, undefined, { leading: true, trailing: true }),
|
||||
exhaustMapWithTrailing(() => {
|
||||
return fromPromise(async () => {
|
||||
try {
|
||||
const trx = await this.data.readonly(this.database);
|
||||
return await this.data.aggregate(
|
||||
trx,
|
||||
table,
|
||||
query,
|
||||
field as string,
|
||||
options
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('aggregate error', error);
|
||||
throw error;
|
||||
}
|
||||
}).pipe(backoffRetry());
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
override async deleteByQuery<T extends keyof IndexerSchema>(
|
||||
table: T,
|
||||
query: Query<T>
|
||||
): Promise<void> {
|
||||
this.pendingUpdates[table].deleteByQueries.push(query);
|
||||
}
|
||||
|
||||
override insert<T extends keyof IndexerSchema>(
|
||||
table: T,
|
||||
document: IndexerDocument
|
||||
): Promise<void> {
|
||||
this.pendingUpdates[table].inserts.push(document);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
override delete<T extends keyof IndexerSchema>(
|
||||
table: T,
|
||||
id: string
|
||||
): Promise<void> {
|
||||
this.pendingUpdates[table].deletes.push(id);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
override update<T extends keyof IndexerSchema>(
|
||||
table: T,
|
||||
document: IndexerDocument
|
||||
): Promise<void> {
|
||||
this.pendingUpdates[table].updates.push(document);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
override async refresh<T extends keyof IndexerSchema>(
|
||||
table: T
|
||||
): Promise<void> {
|
||||
const trx = await this.data.readwrite(this.database);
|
||||
const tables = table ? [table] : (['doc', 'block'] as const);
|
||||
for (const table of tables) {
|
||||
await this.data.batchWrite(
|
||||
trx,
|
||||
table,
|
||||
this.pendingUpdates[table].deleteByQueries,
|
||||
this.pendingUpdates[table].deletes,
|
||||
this.pendingUpdates[table].inserts,
|
||||
this.pendingUpdates[table].updates
|
||||
);
|
||||
this.pendingUpdates[table] = {
|
||||
deleteByQueries: [],
|
||||
deletes: [],
|
||||
inserts: [],
|
||||
updates: [],
|
||||
};
|
||||
}
|
||||
this.emitTableUpdated(table);
|
||||
}
|
||||
|
||||
private watchTableUpdated(table: keyof IndexerSchema) {
|
||||
return new Observable(subscriber => {
|
||||
const listener = (ev: MessageEvent) => {
|
||||
if (ev.data.type === 'indexer-updated' && ev.data.table === table) {
|
||||
subscriber.next(1);
|
||||
}
|
||||
};
|
||||
|
||||
const subscription = this.tableUpdate$.subscribe(updatedTable => {
|
||||
if (updatedTable === table) {
|
||||
subscriber.next(1);
|
||||
}
|
||||
});
|
||||
|
||||
this.channel.addEventListener('message', listener);
|
||||
return () => {
|
||||
this.channel.removeEventListener('message', listener);
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
emitTableUpdated(table: keyof IndexerSchema) {
|
||||
this.tableUpdate$.next(table);
|
||||
this.channel.postMessage({ type: 'indexer-updated', table });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
import { bm25 } from './bm25';
|
||||
import type {
|
||||
DataStructROTransaction,
|
||||
DataStructRWTransaction,
|
||||
} from './data-struct';
|
||||
import { Match } from './match';
|
||||
import { GeneralTokenizer, type Token } from './tokenizer';
|
||||
|
||||
export interface InvertedIndex {
|
||||
fieldKey: string;
|
||||
|
||||
match(trx: DataStructROTransaction, term: string): Promise<Match>;
|
||||
|
||||
all(trx: DataStructROTransaction): Promise<Match>;
|
||||
|
||||
insert(
|
||||
trx: DataStructRWTransaction,
|
||||
id: number,
|
||||
terms: string[]
|
||||
): Promise<void>;
|
||||
}
|
||||
|
||||
export class StringInvertedIndex implements InvertedIndex {
|
||||
constructor(
|
||||
readonly table: string,
|
||||
readonly fieldKey: string
|
||||
) {}
|
||||
|
||||
async match(trx: DataStructROTransaction, term: string): Promise<Match> {
|
||||
const objs = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('key')
|
||||
.getAll([
|
||||
this.table,
|
||||
InvertedIndexKey.forString(this.fieldKey, term).buffer(),
|
||||
]);
|
||||
const match = new Match();
|
||||
for (const obj of objs) {
|
||||
match.addScore(obj.nid, 1);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
async all(trx: DataStructROTransaction): Promise<Match> {
|
||||
const objs = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('key')
|
||||
.getAll(
|
||||
IDBKeyRange.bound(
|
||||
[this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
|
||||
[
|
||||
this.table,
|
||||
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
|
||||
]
|
||||
)
|
||||
);
|
||||
|
||||
const set = new Set<number>();
|
||||
for (const obj of objs) {
|
||||
set.add(obj.nid);
|
||||
}
|
||||
|
||||
const match = new Match();
|
||||
for (const nid of set) {
|
||||
match.addScore(nid, 1);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
|
||||
for (const term of terms) {
|
||||
await trx.objectStore('invertedIndex').put({
|
||||
table: this.table,
|
||||
key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
|
||||
nid: id,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class IntegerInvertedIndex implements InvertedIndex {
|
||||
constructor(
|
||||
readonly table: string,
|
||||
readonly fieldKey: string
|
||||
) {}
|
||||
|
||||
async match(trx: DataStructROTransaction, term: string): Promise<Match> {
|
||||
const objs = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('key')
|
||||
.getAll([
|
||||
this.table,
|
||||
InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(),
|
||||
]);
|
||||
const match = new Match();
|
||||
for (const obj of objs) {
|
||||
match.addScore(obj.nid, 1);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line sonarjs/no-identical-functions
|
||||
async all(trx: DataStructROTransaction): Promise<Match> {
|
||||
const objs = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('key')
|
||||
.getAll(
|
||||
IDBKeyRange.bound(
|
||||
[this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
|
||||
[
|
||||
this.table,
|
||||
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
|
||||
]
|
||||
)
|
||||
);
|
||||
|
||||
const set = new Set<number>();
|
||||
for (const obj of objs) {
|
||||
set.add(obj.nid);
|
||||
}
|
||||
|
||||
const match = new Match();
|
||||
for (const nid of set) {
|
||||
match.addScore(nid, 1);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
|
||||
for (const term of terms) {
|
||||
await trx.objectStore('invertedIndex').put({
|
||||
table: this.table,
|
||||
key: InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(),
|
||||
nid: id,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class BooleanInvertedIndex implements InvertedIndex {
|
||||
constructor(
|
||||
readonly table: string,
|
||||
readonly fieldKey: string
|
||||
) {}
|
||||
|
||||
// eslint-disable-next-line sonarjs/no-identical-functions
|
||||
async all(trx: DataStructROTransaction): Promise<Match> {
|
||||
const objs = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('key')
|
||||
.getAll(
|
||||
IDBKeyRange.bound(
|
||||
[this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
|
||||
[
|
||||
this.table,
|
||||
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
|
||||
]
|
||||
)
|
||||
);
|
||||
|
||||
const set = new Set<number>();
|
||||
for (const obj of objs) {
|
||||
set.add(obj.nid);
|
||||
}
|
||||
|
||||
const match = new Match();
|
||||
for (const nid of set) {
|
||||
match.addScore(nid, 1);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
async match(trx: DataStructROTransaction, term: string): Promise<Match> {
|
||||
const objs = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('key')
|
||||
.getAll([
|
||||
this.table,
|
||||
InvertedIndexKey.forBoolean(this.fieldKey, term === 'true').buffer(),
|
||||
]);
|
||||
const match = new Match();
|
||||
for (const obj of objs) {
|
||||
match.addScore(obj.nid, 1);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
|
||||
for (const term of terms) {
|
||||
await trx.objectStore('invertedIndex').put({
|
||||
table: this.table,
|
||||
key: InvertedIndexKey.forBoolean(
|
||||
this.fieldKey,
|
||||
term === 'true'
|
||||
).buffer(),
|
||||
nid: id,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class FullTextInvertedIndex implements InvertedIndex {
|
||||
constructor(
|
||||
readonly table: string,
|
||||
readonly fieldKey: string
|
||||
) {}
|
||||
|
||||
async match(trx: DataStructROTransaction, term: string): Promise<Match> {
|
||||
const queryTokens = new GeneralTokenizer().tokenize(term);
|
||||
const matched = new Map<
|
||||
number,
|
||||
Map<
|
||||
number, // index
|
||||
{
|
||||
score: number;
|
||||
ranges: [number, number][];
|
||||
}
|
||||
>
|
||||
>();
|
||||
const avgFieldLength =
|
||||
(
|
||||
await trx
|
||||
.objectStore('indexerMetadata')
|
||||
.get(`full-text:avg-field-length:${this.table}:${this.fieldKey}`)
|
||||
)?.value ?? 0;
|
||||
for (const token of queryTokens) {
|
||||
const key = InvertedIndexKey.forString(this.fieldKey, token.term);
|
||||
const objs = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('key')
|
||||
.getAll(
|
||||
IDBKeyRange.bound(
|
||||
[this.table, key.buffer()],
|
||||
[this.table, key.add1().buffer()],
|
||||
false,
|
||||
true
|
||||
)
|
||||
);
|
||||
const submatched: {
|
||||
nid: number;
|
||||
score: number;
|
||||
position: {
|
||||
index: number;
|
||||
ranges: [number, number][];
|
||||
};
|
||||
}[] = [];
|
||||
for (const obj of objs) {
|
||||
const key = InvertedIndexKey.fromBuffer(obj.key);
|
||||
const originTokenTerm = key.asString();
|
||||
const matchLength = token.term.length;
|
||||
const position = obj.pos ?? {
|
||||
i: 0,
|
||||
l: 0,
|
||||
rs: [],
|
||||
};
|
||||
const termFreq = position.rs.length;
|
||||
const totalCount = objs.length;
|
||||
const fieldLength = position.l;
|
||||
const score =
|
||||
bm25(termFreq, 1, totalCount, fieldLength, avgFieldLength) *
|
||||
(matchLength / originTokenTerm.length);
|
||||
const match = {
|
||||
score,
|
||||
positions: new Map(),
|
||||
};
|
||||
const ranges = match.positions.get(position.i) || [];
|
||||
ranges.push(
|
||||
...position.rs.map(([start, _end]) => [start, start + matchLength])
|
||||
);
|
||||
match.positions.set(position.i, ranges);
|
||||
submatched.push({
|
||||
nid: obj.nid,
|
||||
score,
|
||||
position: {
|
||||
index: position.i,
|
||||
ranges: position.rs.map(([start, _end]) => [
|
||||
start,
|
||||
start + matchLength,
|
||||
]),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// normalize score
|
||||
const maxScore = submatched.reduce((acc, s) => Math.max(acc, s.score), 0);
|
||||
const minScore = submatched.reduce((acc, s) => Math.min(acc, s.score), 0);
|
||||
for (const { nid, score, position } of submatched) {
|
||||
const normalizedScore =
|
||||
maxScore === minScore
|
||||
? score
|
||||
: (score - minScore) / (maxScore - minScore);
|
||||
const match =
|
||||
matched.get(nid) ??
|
||||
new Map<
|
||||
number, // index
|
||||
{
|
||||
score: number;
|
||||
ranges: [number, number][];
|
||||
}
|
||||
>();
|
||||
const item = match.get(position.index) || {
|
||||
score: 0,
|
||||
ranges: [],
|
||||
};
|
||||
item.score += normalizedScore;
|
||||
item.ranges.push(...position.ranges);
|
||||
match.set(position.index, item);
|
||||
matched.set(nid, match);
|
||||
}
|
||||
}
|
||||
const match = new Match();
|
||||
for (const [nid, items] of matched) {
|
||||
if (items.size === 0) {
|
||||
break;
|
||||
}
|
||||
let highestScore = -1;
|
||||
let highestIndex = -1;
|
||||
let highestRanges: [number, number][] = [];
|
||||
for (const [index, { score, ranges }] of items) {
|
||||
if (score > highestScore) {
|
||||
highestScore = score;
|
||||
highestIndex = index;
|
||||
highestRanges = ranges;
|
||||
}
|
||||
}
|
||||
match.addScore(nid, highestScore);
|
||||
match.addHighlighter(nid, this.fieldKey, highestIndex, highestRanges);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line sonarjs/no-identical-functions
|
||||
async all(trx: DataStructROTransaction): Promise<Match> {
|
||||
const objs = await trx
|
||||
.objectStore('invertedIndex')
|
||||
.index('key')
|
||||
.getAll(
|
||||
IDBKeyRange.bound(
|
||||
[this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
|
||||
[
|
||||
this.table,
|
||||
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
|
||||
]
|
||||
)
|
||||
);
|
||||
|
||||
const set = new Set<number>();
|
||||
for (const obj of objs) {
|
||||
set.add(obj.nid);
|
||||
}
|
||||
|
||||
const match = new Match();
|
||||
for (const nid of set) {
|
||||
match.addScore(nid, 1);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
|
||||
for (let i = 0; i < terms.length; i++) {
|
||||
const tokenMap = new Map<string, Token[]>();
|
||||
const originString = terms[i];
|
||||
|
||||
const tokens = new GeneralTokenizer().tokenize(originString);
|
||||
|
||||
for (const token of tokens) {
|
||||
const tokens = tokenMap.get(token.term) || [];
|
||||
tokens.push(token);
|
||||
tokenMap.set(token.term, tokens);
|
||||
}
|
||||
|
||||
for (const [term, tokens] of tokenMap) {
|
||||
await trx.objectStore('invertedIndex').put({
|
||||
table: this.table,
|
||||
key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
|
||||
nid: id,
|
||||
pos: {
|
||||
l: originString.length,
|
||||
i: i,
|
||||
rs: tokens.map(token => [token.start, token.end]),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const indexerMetadataStore = trx.objectStore('indexerMetadata');
|
||||
// update avg-field-length
|
||||
const totalCount =
|
||||
(
|
||||
await indexerMetadataStore.get(
|
||||
`full-text:field-count:${this.table}:${this.fieldKey}`
|
||||
)
|
||||
)?.value ?? 0;
|
||||
const avgFieldLength =
|
||||
(
|
||||
await indexerMetadataStore.get(
|
||||
`full-text:avg-field-length:${this.table}:${this.fieldKey}`
|
||||
)
|
||||
)?.value ?? 0;
|
||||
await indexerMetadataStore.put({
|
||||
key: `full-text:field-count:${this.table}:${this.fieldKey}`,
|
||||
value: totalCount + 1,
|
||||
});
|
||||
await indexerMetadataStore.put({
|
||||
key: `full-text:avg-field-length:${this.table}:${this.fieldKey}`,
|
||||
value:
|
||||
avgFieldLength +
|
||||
(terms.reduce((acc, term) => acc + term.length, 0) - avgFieldLength) /
|
||||
(totalCount + 1),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class InvertedIndexKey {
|
||||
constructor(
|
||||
readonly field: Uint8Array,
|
||||
readonly value: Uint8Array,
|
||||
readonly gap: Uint8Array = new Uint8Array([58])
|
||||
) {}
|
||||
|
||||
asString() {
|
||||
return new TextDecoder().decode(this.value);
|
||||
}
|
||||
|
||||
asInt64() {
|
||||
return new DataView(this.value.buffer).getBigInt64(
|
||||
0,
|
||||
false
|
||||
); /* big-endian */
|
||||
}
|
||||
|
||||
add1() {
|
||||
if (this.value.byteLength > 0) {
|
||||
const bytes = new Uint8Array(this.value.slice(0));
|
||||
let carry = 1;
|
||||
for (let i = bytes.length - 1; i >= 0 && carry > 0; i--) {
|
||||
const sum = bytes[i] + carry;
|
||||
bytes[i] = sum % 256;
|
||||
carry = sum >> 8;
|
||||
}
|
||||
return new InvertedIndexKey(this.field, bytes);
|
||||
} else {
|
||||
return new InvertedIndexKey(
|
||||
this.field,
|
||||
new Uint8Array(0),
|
||||
new Uint8Array([59])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
static forPrefix(field: string) {
|
||||
return new InvertedIndexKey(
|
||||
new TextEncoder().encode(field),
|
||||
new Uint8Array(0)
|
||||
);
|
||||
}
|
||||
|
||||
static forString(field: string, value: string) {
|
||||
return new InvertedIndexKey(
|
||||
new TextEncoder().encode(field),
|
||||
new TextEncoder().encode(value)
|
||||
);
|
||||
}
|
||||
|
||||
static forBoolean(field: string, value: boolean) {
|
||||
const bytes = new Uint8Array(1);
|
||||
bytes.set([value ? 1 : 0]);
|
||||
return new InvertedIndexKey(new TextEncoder().encode(field), bytes);
|
||||
}
|
||||
|
||||
static forInt64(field: string, value: bigint) {
|
||||
const bytes = new Uint8Array(8);
|
||||
new DataView(bytes.buffer).setBigInt64(0, value, false); /* big-endian */
|
||||
return new InvertedIndexKey(new TextEncoder().encode(field), bytes);
|
||||
}
|
||||
|
||||
buffer() {
|
||||
const tmp = new Uint8Array(
|
||||
this.field.byteLength + (this.value?.byteLength ?? 0) + 1
|
||||
);
|
||||
tmp.set(new Uint8Array(this.field), 0);
|
||||
tmp.set(new Uint8Array(this.gap), this.field.byteLength);
|
||||
if (this.value.byteLength > 0) {
|
||||
tmp.set(new Uint8Array(this.value), this.field.byteLength + 1);
|
||||
}
|
||||
return tmp.buffer;
|
||||
}
|
||||
|
||||
static fromBuffer(buffer: ArrayBuffer) {
|
||||
const array = new Uint8Array(buffer);
|
||||
const fieldLength = array.indexOf(58);
|
||||
const field = array.slice(0, fieldLength);
|
||||
const value = array.slice(fieldLength + 1);
|
||||
return new InvertedIndexKey(field, value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
export class Match {
|
||||
scores = new Map<number, number>();
|
||||
/**
|
||||
* nid -> field -> index(multi value field) -> [start, end][]
|
||||
*/
|
||||
highlighters = new Map<
|
||||
number,
|
||||
Map<string, Map<number, [number, number][]>>
|
||||
>();
|
||||
|
||||
constructor() {}
|
||||
|
||||
size() {
|
||||
return this.scores.size;
|
||||
}
|
||||
|
||||
getScore(id: number) {
|
||||
return this.scores.get(id) ?? 0;
|
||||
}
|
||||
|
||||
addScore(id: number, score: number) {
|
||||
const currentScore = this.scores.get(id) || 0;
|
||||
this.scores.set(id, currentScore + score);
|
||||
}
|
||||
|
||||
getHighlighters(id: number, field: string) {
|
||||
return this.highlighters.get(id)?.get(field);
|
||||
}
|
||||
|
||||
addHighlighter(
|
||||
id: number,
|
||||
field: string,
|
||||
index: number,
|
||||
newRanges: [number, number][]
|
||||
) {
|
||||
const fields =
|
||||
this.highlighters.get(id) ||
|
||||
new Map<string, Map<number, [number, number][]>>();
|
||||
const values = fields.get(field) || new Map<number, [number, number][]>();
|
||||
const ranges = values.get(index) || [];
|
||||
ranges.push(...newRanges);
|
||||
values.set(index, ranges);
|
||||
fields.set(field, values);
|
||||
this.highlighters.set(id, fields);
|
||||
}
|
||||
|
||||
and(other: Match) {
|
||||
const newWeight = new Match();
|
||||
for (const [id, score] of this.scores) {
|
||||
if (other.scores.has(id)) {
|
||||
newWeight.addScore(id, score + (other.scores.get(id) ?? 0));
|
||||
newWeight.copyExtData(this, id);
|
||||
newWeight.copyExtData(other, id);
|
||||
}
|
||||
}
|
||||
return newWeight;
|
||||
}
|
||||
|
||||
or(other: Match) {
|
||||
const newWeight = new Match();
|
||||
for (const [id, score] of this.scores) {
|
||||
newWeight.addScore(id, score);
|
||||
newWeight.copyExtData(this, id);
|
||||
}
|
||||
for (const [id, score] of other.scores) {
|
||||
newWeight.addScore(id, score);
|
||||
newWeight.copyExtData(other, id);
|
||||
}
|
||||
return newWeight;
|
||||
}
|
||||
|
||||
exclude(other: Match) {
|
||||
const newWeight = new Match();
|
||||
for (const [id, score] of this.scores) {
|
||||
if (!other.scores.has(id)) {
|
||||
newWeight.addScore(id, score);
|
||||
newWeight.copyExtData(this, id);
|
||||
}
|
||||
}
|
||||
return newWeight;
|
||||
}
|
||||
|
||||
boost(boost: number) {
|
||||
const newWeight = new Match();
|
||||
for (const [id, score] of this.scores) {
|
||||
newWeight.addScore(id, score * boost);
|
||||
newWeight.copyExtData(this, id);
|
||||
}
|
||||
return newWeight;
|
||||
}
|
||||
|
||||
toArray() {
|
||||
return Array.from(this.scores.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.map(e => e[0]);
|
||||
}
|
||||
|
||||
private copyExtData(from: Match, id: number) {
|
||||
for (const [field, values] of from.highlighters.get(id) ?? []) {
|
||||
for (const [index, ranges] of values) {
|
||||
this.addHighlighter(id, field, index, ranges);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import Graphemer from 'graphemer';
|
||||
|
||||
export interface Tokenizer {
|
||||
tokenize(text: string): Token[];
|
||||
}
|
||||
|
||||
export interface Token {
|
||||
term: string;
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
|
||||
export class SimpleTokenizer implements Tokenizer {
|
||||
tokenize(text: string): Token[] {
|
||||
const tokens: Token[] = [];
|
||||
let start = 0;
|
||||
let end = 0;
|
||||
let inWord = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
if (c.match(/[\n\r\p{Z}\p{P}]/u)) {
|
||||
if (inWord) {
|
||||
end = i;
|
||||
tokens.push({
|
||||
term: text.substring(start, end).toLowerCase(),
|
||||
start,
|
||||
end,
|
||||
});
|
||||
inWord = false;
|
||||
}
|
||||
} else {
|
||||
if (!inWord) {
|
||||
start = i;
|
||||
end = i;
|
||||
inWord = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inWord) {
|
||||
tokens.push({
|
||||
term: text.substring(start).toLowerCase(),
|
||||
start,
|
||||
end: text.length,
|
||||
});
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
}
|
||||
|
||||
export class NGramTokenizer implements Tokenizer {
|
||||
constructor(private readonly n: number) {}
|
||||
|
||||
tokenize(text: string): Token[] {
|
||||
const splitted: Token[] = [];
|
||||
for (let i = 0; i < text.length; ) {
|
||||
const nextBreak = Graphemer.nextBreak(text, i);
|
||||
const c = text.substring(i, nextBreak);
|
||||
|
||||
splitted.push({
|
||||
term: c,
|
||||
start: i,
|
||||
end: nextBreak,
|
||||
});
|
||||
|
||||
i = nextBreak;
|
||||
}
|
||||
const tokens: Token[] = [];
|
||||
for (let i = 0; i < splitted.length - this.n + 1; i++) {
|
||||
tokens.push(
|
||||
splitted.slice(i, i + this.n).reduce(
|
||||
(acc, t) => ({
|
||||
term: acc.term + t.term,
|
||||
start: Math.min(acc.start, t.start),
|
||||
end: Math.max(acc.end, t.end),
|
||||
}),
|
||||
{ term: '', start: Infinity, end: -Infinity }
|
||||
)
|
||||
);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
}
|
||||
|
||||
export class GeneralTokenizer implements Tokenizer {
|
||||
constructor() {}
|
||||
|
||||
tokenizeWord(word: string, lang: string): Token[] {
|
||||
if (lang === 'en') {
|
||||
return [{ term: word.toLowerCase(), start: 0, end: word.length }];
|
||||
} else if (lang === 'cjk') {
|
||||
if (word.length < 3) {
|
||||
return [{ term: word, start: 0, end: word.length }];
|
||||
}
|
||||
return new NGramTokenizer(2).tokenize(word);
|
||||
} else if (lang === 'emoji') {
|
||||
return new NGramTokenizer(1).tokenize(word);
|
||||
} else if (lang === '-') {
|
||||
return [];
|
||||
}
|
||||
|
||||
throw new Error('Not implemented');
|
||||
}
|
||||
|
||||
testLang(c: string): string {
|
||||
if (c.match(/[\p{Emoji}]/u)) {
|
||||
return 'emoji';
|
||||
} else if (c.match(/[\p{sc=Han}\p{scx=Hira}\p{scx=Kana}\p{sc=Hang}]/u)) {
|
||||
return 'cjk';
|
||||
} else if (c.match(/[\n\r\p{Z}\p{P}]/u)) {
|
||||
return '-';
|
||||
} else {
|
||||
return 'en';
|
||||
}
|
||||
}
|
||||
|
||||
tokenize(text: string): Token[] {
|
||||
const tokens: Token[] = [];
|
||||
let start = 0;
|
||||
let end = 0;
|
||||
let lang: string | null = null;
|
||||
|
||||
for (let i = 0; i < text.length; ) {
|
||||
const nextBreak = Graphemer.nextBreak(text, i);
|
||||
const c = text.substring(i, nextBreak);
|
||||
|
||||
const l = this.testLang(c);
|
||||
if (lang !== l) {
|
||||
if (lang !== null) {
|
||||
end = i;
|
||||
tokens.push(
|
||||
...this.tokenizeWord(text.substring(start, end), lang).map(
|
||||
token => ({
|
||||
...token,
|
||||
start: token.start + start,
|
||||
end: token.end + start,
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
start = i;
|
||||
end = i;
|
||||
lang = l;
|
||||
}
|
||||
|
||||
i = nextBreak;
|
||||
}
|
||||
if (lang !== null) {
|
||||
tokens.push(
|
||||
...this.tokenizeWord(text.substring(start, text.length), lang).map(
|
||||
token => ({
|
||||
...token,
|
||||
start: token.start + start,
|
||||
end: token.end + start,
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import {
|
||||
asyncScheduler,
|
||||
defer,
|
||||
exhaustMap,
|
||||
finalize,
|
||||
Observable,
|
||||
type ObservableInput,
|
||||
type OperatorFunction,
|
||||
retry,
|
||||
scheduled,
|
||||
Subject,
|
||||
throttle,
|
||||
throwError,
|
||||
timer,
|
||||
} from 'rxjs';
|
||||
|
||||
import { MANUALLY_STOP } from '../../../utils/throw-if-aborted';
|
||||
|
||||
/**
|
||||
* Like exhaustMap, but also includes the trailing value emitted from the source observable while waiting for the preceding inner observable to complete
|
||||
*
|
||||
* Original code adapted from https://github.com/ReactiveX/rxjs/issues/5004
|
||||
* @param {function<T, K>(value: T, ?index: number): ObservableInput<K>} project - A function that, when applied to an item emitted by the
|
||||
* source Observable, returns a projected Observable.
|
||||
*/
|
||||
export function exhaustMapWithTrailing<T, R>(
|
||||
project: (value: T, index: number) => ObservableInput<R>
|
||||
): OperatorFunction<T, R> {
|
||||
return (source$): Observable<R> =>
|
||||
defer(() => {
|
||||
const release$ = new Subject<void>();
|
||||
return source$.pipe(
|
||||
throttle(() => release$, {
|
||||
leading: true,
|
||||
trailing: true,
|
||||
}),
|
||||
exhaustMap((value, index) =>
|
||||
scheduled(project(value, index), asyncScheduler).pipe(
|
||||
finalize(() => {
|
||||
release$.next();
|
||||
})
|
||||
)
|
||||
)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a promise to an observable.
|
||||
*
|
||||
* like `from` but support `AbortSignal`.
|
||||
*/
|
||||
export function fromPromise<T>(
|
||||
promise: Promise<T> | ((signal: AbortSignal) => Promise<T>)
|
||||
): Observable<T> {
|
||||
return new Observable(subscriber => {
|
||||
const abortController = new AbortController();
|
||||
|
||||
const rawPromise =
|
||||
promise instanceof Function ? promise(abortController.signal) : promise;
|
||||
|
||||
rawPromise
|
||||
.then(value => {
|
||||
subscriber.next(value);
|
||||
subscriber.complete();
|
||||
})
|
||||
.catch(error => {
|
||||
subscriber.error(error);
|
||||
});
|
||||
|
||||
return () => abortController.abort(MANUALLY_STOP);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* An operator that retries the source observable when an error occurs.
|
||||
*
|
||||
* https://en.wikipedia.org/wiki/Exponential_backoff
|
||||
*/
|
||||
export function backoffRetry<T>({
|
||||
when,
|
||||
count = 3,
|
||||
delay = 200,
|
||||
maxDelay = 15000,
|
||||
}: {
|
||||
when?: (err: any) => boolean;
|
||||
count?: number;
|
||||
delay?: number;
|
||||
maxDelay?: number;
|
||||
} = {}) {
|
||||
return (obs$: Observable<T>) =>
|
||||
obs$.pipe(
|
||||
retry({
|
||||
count,
|
||||
delay: (err, retryIndex) => {
|
||||
if (when && !when(err)) {
|
||||
return throwError(() => err);
|
||||
}
|
||||
const d = Math.pow(2, retryIndex - 1) * delay;
|
||||
return timer(Math.min(d, maxDelay));
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -37,6 +37,11 @@ Table(PeerClocks)
|
||||
|------|-------|-----------|-----------|
|
||||
| str | str | Date | Date |
|
||||
|
||||
Table(IndexerSync)
|
||||
| docId | clock |
|
||||
|-------|-------|
|
||||
| str | Date |
|
||||
|
||||
Table(BlobSync)
|
||||
| peer | key | uploadedAt |
|
||||
|------|-----|------------|
|
||||
@@ -124,6 +129,43 @@ export interface DocStorageSchema extends DBSchema {
|
||||
lock: Date;
|
||||
};
|
||||
};
|
||||
indexerSync: {
|
||||
key: string;
|
||||
value: {
|
||||
docId: string;
|
||||
indexedClock: Date;
|
||||
};
|
||||
};
|
||||
indexerMetadata: {
|
||||
key: string;
|
||||
value: {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
};
|
||||
indexerRecords: {
|
||||
key: number;
|
||||
value: {
|
||||
table: string;
|
||||
id: string;
|
||||
data: Map<string, string[]>;
|
||||
};
|
||||
indexes: { table: string; id: [string, string] };
|
||||
};
|
||||
invertedIndex: {
|
||||
key: number;
|
||||
value: {
|
||||
table: string;
|
||||
nid: number;
|
||||
pos?: {
|
||||
i: number /* index */;
|
||||
l: number /* length */;
|
||||
rs: [number, number][] /* ranges: [start, end] */;
|
||||
};
|
||||
key: ArrayBuffer;
|
||||
};
|
||||
indexes: { key: [string, ArrayBuffer]; nid: number };
|
||||
};
|
||||
}
|
||||
|
||||
const migrate: OpenDBCallbacks<DocStorageSchema>['upgrade'] = (
|
||||
@@ -199,11 +241,36 @@ const initBlobSync: Migrate = db => {
|
||||
|
||||
blobSync.createIndex('peer', 'peer', { unique: false });
|
||||
};
|
||||
const initIndexer: Migrate = db => {
|
||||
db.createObjectStore('indexerMetadata', {
|
||||
keyPath: 'key',
|
||||
});
|
||||
const indexRecordsStore = db.createObjectStore('indexerRecords', {
|
||||
autoIncrement: true,
|
||||
});
|
||||
indexRecordsStore.createIndex('table', 'table', {
|
||||
unique: false,
|
||||
});
|
||||
indexRecordsStore.createIndex('id', ['table', 'id'], {
|
||||
unique: true,
|
||||
});
|
||||
const invertedIndexStore = db.createObjectStore('invertedIndex', {
|
||||
autoIncrement: true,
|
||||
});
|
||||
invertedIndexStore.createIndex('key', ['table', 'key'], {
|
||||
unique: false,
|
||||
});
|
||||
invertedIndexStore.createIndex('nid', 'nid', { unique: false });
|
||||
db.createObjectStore('indexerSync', {
|
||||
keyPath: 'docId',
|
||||
autoIncrement: false,
|
||||
});
|
||||
};
|
||||
// END REGION
|
||||
|
||||
// 1. all schema changed should be put in migrations
|
||||
// 2. order matters
|
||||
const migrations: Migrate[] = [init, initBlobSync];
|
||||
const migrations: Migrate[] = [init, initBlobSync, initIndexer];
|
||||
|
||||
export const migrator = {
|
||||
version: migrations.length,
|
||||
|
||||
Reference in New Issue
Block a user