diff --git a/Cargo.lock b/Cargo.lock index 3340b75c7b..4db18217b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2643,9 +2643,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memory-indexer" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10feba381e2eeb6582f34379d62ee0658e57f63d776698150b985bd0f38664b3" +checksum = "36308f8c9f537d7624a30cd4d6243c54143221e4e0dc2a699783c206604befbd" dependencies = [ "jieba-rs", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 2da3d23c86..8cea42776c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ resolver = "3" libc = "0.2" log = "0.4" loom = { version = "0.7", features = ["checkpoint"] } - memory-indexer = "0.2" + memory-indexer = "0.2.1" mimalloc = "0.1" mp4parse = "0.17" nanoid = "0.4" diff --git a/packages/common/nbstore/src/impls/cloud/indexer.ts b/packages/common/nbstore/src/impls/cloud/indexer.ts index fee60180fa..5016da553b 100644 --- a/packages/common/nbstore/src/impls/cloud/indexer.ts +++ b/packages/common/nbstore/src/impls/cloud/indexer.ts @@ -143,4 +143,8 @@ export class CloudIndexerStorage extends IndexerStorageBase { override async refreshIfNeed(): Promise { return Promise.resolve(); } + + override async indexVersion(): Promise { + return Promise.resolve(1); + } } diff --git a/packages/common/nbstore/src/impls/idb/indexer/index.ts b/packages/common/nbstore/src/impls/idb/indexer/index.ts index 35bc031e65..f0413c6117 100644 --- a/packages/common/nbstore/src/impls/idb/indexer/index.ts +++ b/packages/common/nbstore/src/impls/idb/indexer/index.ts @@ -218,4 +218,10 @@ export class IndexedDBIndexerStorage extends IndexerStorageBase { this.tableUpdate$.next(table); this.channel.postMessage({ type: 'indexer-updated', table }); } + + // Get the current indexer version + // increase this number to re-index all docs + async indexVersion(): Promise { + return Promise.resolve(1); + } } diff --git a/packages/common/nbstore/src/impls/sqlite/db.ts b/packages/common/nbstore/src/impls/sqlite/db.ts index cb22323c4a..657993607e 100644 --- a/packages/common/nbstore/src/impls/sqlite/db.ts +++ b/packages/common/nbstore/src/impls/sqlite/db.ts @@ -112,6 +112,7 @@ export interface NativeDBApis { query: string ) => Promise<{ start: number; end: number }[]>; ftsFlushIndex: (id: string) => Promise; + ftsIndexVersion: () => Promise; } type NativeDBApisWrapper = NativeDBApis extends infer APIs @@ -119,7 +120,7 @@ type NativeDBApisWrapper = NativeDBApis extends infer APIs [K in keyof APIs]: APIs[K] extends (...args: any[]) => any ? Parameters extends [string, ...infer Rest] ? (...args: Rest) => ReturnType - : never + : (...args: Parameters) => ReturnType : never; } : never; diff --git a/packages/common/nbstore/src/impls/sqlite/indexer/index.ts b/packages/common/nbstore/src/impls/sqlite/indexer/index.ts index e11e8445c2..4a374c5a4d 100644 --- a/packages/common/nbstore/src/impls/sqlite/indexer/index.ts +++ b/packages/common/nbstore/src/impls/sqlite/indexer/index.ts @@ -241,4 +241,8 @@ export class SqliteIndexerStorage extends IndexerStorageBase { async refreshIfNeed(): Promise { await this.connection.apis.ftsFlushIndex(); } + + async indexVersion(): Promise { + return this.connection.apis.ftsIndexVersion(); + } } diff --git a/packages/common/nbstore/src/storage/dummy/indexer.ts b/packages/common/nbstore/src/storage/dummy/indexer.ts index 5a22515b13..8df5ad9675 100644 --- a/packages/common/nbstore/src/storage/dummy/indexer.ts +++ b/packages/common/nbstore/src/storage/dummy/indexer.ts @@ -88,4 +88,7 @@ export class DummyIndexerStorage extends IndexerStorageBase { override async refreshIfNeed(): Promise { return Promise.resolve(); } + override async indexVersion(): Promise { + return Promise.resolve(0); + } } diff --git a/packages/common/nbstore/src/storage/indexer.ts b/packages/common/nbstore/src/storage/indexer.ts index aa031358a8..0ac80e3326 100644 --- a/packages/common/nbstore/src/storage/indexer.ts +++ b/packages/common/nbstore/src/storage/indexer.ts @@ -64,6 +64,7 @@ export interface IndexerStorage extends Storage { refresh(table: T): Promise; refreshIfNeed(): Promise; + indexVersion(): Promise; } type ResultPagination = { @@ -178,4 +179,6 @@ export abstract class IndexerStorageBase implements IndexerStorage { abstract refresh(table: T): Promise; abstract refreshIfNeed(): Promise; + + abstract indexVersion(): Promise; } diff --git a/packages/common/nbstore/src/sync/indexer/index.ts b/packages/common/nbstore/src/sync/indexer/index.ts index d0f6cc3b01..dac3fe8745 100644 --- a/packages/common/nbstore/src/sync/indexer/index.ts +++ b/packages/common/nbstore/src/sync/indexer/index.ts @@ -106,10 +106,6 @@ export interface IndexerSync { } export class IndexerSyncImpl implements IndexerSync { - /** - * increase this number to re-index all docs - */ - readonly INDEXER_VERSION = 2; private abort: AbortController | null = null; private readonly rootDocId = this.doc.spaceId; private readonly status = new IndexerSyncStatus(this.rootDocId); @@ -266,7 +262,8 @@ export class IndexerSyncImpl implements IndexerSync { this.status.errorMessage = null; this.status.statusUpdatedSubject$.next(true); - console.log('indexer sync start'); + const indexVersion = await this.indexer.indexVersion(); + console.log('indexer sync start, version: ', indexVersion); const unsubscribe = this.doc.subscribeDocUpdate(update => { if (!this.status.rootDocReady) { @@ -402,7 +399,7 @@ export class IndexerSyncImpl implements IndexerSync { docIndexedClock && docIndexedClock.timestamp.getTime() === docClock.timestamp.getTime() && - docIndexedClock.indexerVersion === this.INDEXER_VERSION + docIndexedClock.indexerVersion === indexVersion ) { // doc is already indexed, just skip continue; @@ -468,7 +465,7 @@ export class IndexerSyncImpl implements IndexerSync { await this.indexerSync.setDocIndexedClock({ docId, timestamp: docClock.timestamp, - indexerVersion: this.INDEXER_VERSION, + indexerVersion: indexVersion, }); // #endregion } diff --git a/packages/frontend/apps/android/src/plugins/nbstore/definitions.ts b/packages/frontend/apps/android/src/plugins/nbstore/definitions.ts index a0687168b6..6f1faecce1 100644 --- a/packages/frontend/apps/android/src/plugins/nbstore/definitions.ts +++ b/packages/frontend/apps/android/src/plugins/nbstore/definitions.ts @@ -184,4 +184,5 @@ export interface NbStorePlugin { query: string; }) => Promise>; ftsFlushIndex: (options: { id: string }) => Promise; + ftsIndexVersion: () => Promise; } diff --git a/packages/frontend/apps/android/src/plugins/nbstore/index.ts b/packages/frontend/apps/android/src/plugins/nbstore/index.ts index 528f644942..e19a2b1059 100644 --- a/packages/frontend/apps/android/src/plugins/nbstore/index.ts +++ b/packages/frontend/apps/android/src/plugins/nbstore/index.ts @@ -406,4 +406,7 @@ export const NbStoreNativeDBApis: NativeDBApis = { id, }); }, + ftsIndexVersion: function (): Promise { + return NbStore.ftsIndexVersion(); + }, }; diff --git a/packages/frontend/apps/electron/src/helper/nbstore/handlers.ts b/packages/frontend/apps/electron/src/helper/nbstore/handlers.ts index 2735d80788..4a8910bc8f 100644 --- a/packages/frontend/apps/electron/src/helper/nbstore/handlers.ts +++ b/packages/frontend/apps/electron/src/helper/nbstore/handlers.ts @@ -54,4 +54,5 @@ export const nbstoreHandlers: NativeDBApis = { ftsGetDocument: POOL.ftsGetDocument.bind(POOL), ftsGetMatches: POOL.ftsGetMatches.bind(POOL), ftsFlushIndex: POOL.ftsFlushIndex.bind(POOL), + ftsIndexVersion: POOL.ftsIndexVersion.bind(POOL), }; diff --git a/packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts b/packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts index a0687168b6..6f1faecce1 100644 --- a/packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts +++ b/packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts @@ -184,4 +184,5 @@ export interface NbStorePlugin { query: string; }) => Promise>; ftsFlushIndex: (options: { id: string }) => Promise; + ftsIndexVersion: () => Promise; } diff --git a/packages/frontend/apps/ios/src/plugins/nbstore/index.ts b/packages/frontend/apps/ios/src/plugins/nbstore/index.ts index da438efde0..a26115f323 100644 --- a/packages/frontend/apps/ios/src/plugins/nbstore/index.ts +++ b/packages/frontend/apps/ios/src/plugins/nbstore/index.ts @@ -410,4 +410,7 @@ export const NbStoreNativeDBApis: NativeDBApis = { id, }); }, + ftsIndexVersion: function (): Promise { + return NbStore.ftsIndexVersion(); + }, }; diff --git a/packages/frontend/native/index.d.ts b/packages/frontend/native/index.d.ts index 7ff863b118..ad49610f48 100644 --- a/packages/frontend/native/index.d.ts +++ b/packages/frontend/native/index.d.ts @@ -84,6 +84,7 @@ export declare class DocStoragePool { getBlobUploadedAt(universalId: string, peer: string, blobId: string): Promise ftsAddDocument(id: string, indexName: string, docId: string, text: string, index: boolean): Promise ftsFlushIndex(id: string): Promise + ftsIndexVersion(): Promise ftsDeleteDocument(id: string, indexName: string, docId: string): Promise ftsGetDocument(id: string, indexName: string, docId: string): Promise ftsSearch(id: string, indexName: string, query: string): Promise> diff --git a/packages/frontend/native/nbstore/src/blob.rs b/packages/frontend/native/nbstore/src/blob.rs index 2f4dc6f8d3..be574f2878 100644 --- a/packages/frontend/native/nbstore/src/blob.rs +++ b/packages/frontend/native/nbstore/src/blob.rs @@ -74,7 +74,7 @@ impl SqliteDocStorage { mod tests { use sqlx::Row; - use super::*; + use super::{super::Data, *}; async fn get_storage() -> SqliteDocStorage { let storage = SqliteDocStorage::new(":memory:".to_string()); @@ -91,7 +91,7 @@ mod tests { storage .set_blob(SetBlob { key: format!("test_{}", i), - data: vec![0, 0], + data: Into::::into(vec![0, 0]), mime: "text/plain".to_string(), }) .await @@ -131,7 +131,7 @@ mod tests { storage .set_blob(SetBlob { key: format!("test_{}", i), - data: vec![0, 0], + data: Into::::into(vec![0, 0]), mime: "text/plain".to_string(), }) .await @@ -179,7 +179,7 @@ mod tests { storage .set_blob(SetBlob { key: format!("test_{}", i), - data: vec![0, 0], + data: Into::::into(vec![0, 0]), mime: "text/plain".to_string(), }) .await diff --git a/packages/frontend/native/nbstore/src/doc.rs b/packages/frontend/native/nbstore/src/doc.rs index a13dd0d447..7e52bda0dc 100644 --- a/packages/frontend/native/nbstore/src/doc.rs +++ b/packages/frontend/native/nbstore/src/doc.rs @@ -250,6 +250,7 @@ mod tests { use chrono::{DateTime, Utc}; use super::*; + use crate::Data; async fn get_storage() -> SqliteDocStorage { let storage = SqliteDocStorage::new(":memory:".to_string()); @@ -293,7 +294,7 @@ mod tests { storage .set_doc_snapshot(DocRecord { doc_id: "test".to_string(), - bin: vec![0, 0], + bin: Into::::into(vec![0, 0]), timestamp: Utc::now().naive_utc(), }) .await @@ -373,7 +374,7 @@ mod tests { let snapshot = DocRecord { doc_id: "test".to_string(), - bin: vec![0, 0], + bin: Into::::into(vec![0, 0]), timestamp: Utc::now().naive_utc(), }; @@ -391,7 +392,7 @@ mod tests { let snapshot = DocRecord { doc_id: "test".to_string(), - bin: vec![0, 0], + bin: Into::::into(vec![0, 0]), timestamp: Utc::now().naive_utc(), }; @@ -404,7 +405,7 @@ mod tests { let snapshot = DocRecord { doc_id: "test".to_string(), - bin: vec![0, 1], + bin: Into::::into(vec![0, 1]), timestamp: DateTime::from_timestamp_millis(Utc::now().timestamp_millis() - 1000) .unwrap() .naive_utc(), diff --git a/packages/frontend/native/nbstore/src/indexer.rs b/packages/frontend/native/nbstore/src/indexer.rs index 0684850f22..bf82813243 100644 --- a/packages/frontend/native/nbstore/src/indexer.rs +++ b/packages/frontend/native/nbstore/src/indexer.rs @@ -197,6 +197,10 @@ impl SqliteDocStorage { Ok(()) } + pub fn index_version() -> u32 { + memory_indexer::InMemoryIndex::snapshot_version() + } + pub async fn fts_add( &self, index_name: &str, diff --git a/packages/frontend/native/nbstore/src/lib.rs b/packages/frontend/native/nbstore/src/lib.rs index d0036f5d22..ee0ca4bb42 100644 --- a/packages/frontend/native/nbstore/src/lib.rs +++ b/packages/frontend/native/nbstore/src/lib.rs @@ -472,6 +472,11 @@ impl DocStoragePool { Ok(()) } + #[napi] + pub async fn fts_index_version(&self) -> Result { + Ok(SqliteDocStorage::index_version()) + } + #[napi] pub async fn fts_delete_document( &self,