feat(core): run indexer in worker (#7418)

packages/common/infra/src/livedata/ops.ts

@@ -18,6 +18,7 @@ import {
   timer,
 } from 'rxjs';
 
+import { MANUALLY_STOP } from '../utils';
 import type { LiveData } from './livedata';
 
 /**
@@ -107,7 +108,8 @@ export function fromPromise<T>(
       .catch(error => {
         subscriber.error(error);
       });
-    return () => abortController.abort('Aborted');
+
+    return () => abortController.abort(MANUALLY_STOP);
   });
 }
 

packages/common/infra/src/sync/blob/blob.ts

@@ -4,6 +4,7 @@ import { difference } from 'lodash-es';
 
 import { LiveData } from '../../livedata';
 import type { Memento } from '../../storage';
+import { MANUALLY_STOP } from '../../utils';
 import { BlobStorageOverCapacity } from './error';
 
 const logger = new DebugLogger('affine:blob-engine');
@@ -70,7 +71,7 @@ export class BlobEngine {
   }
 
   stop() {
-    this.abort?.abort();
+    this.abort?.abort(MANUALLY_STOP);
     this.abort = null;
   }
 

packages/common/infra/src/sync/doc/old-id.md

@@ -0,0 +1,24 @@
+AFFiNE currently has a lot of data stored using the old ID format. Here, we record the usage of IDs to avoid forgetting.
+
+## Old ID Format
+
+The format is:
+
+- `{workspace-id}:space:{nanoid}` Common
+- `{workspace-id}:space:page:{nanoid}`
+
+> Note: sometimes the `workspace-id` is not same with current workspace id.
+
+## Usage
+
+- Local Storage
+  - indexeddb: Both new and old IDs coexist
+  - sqlite: Both new and old IDs coexist
+  - server-clock: Only new IDs are stored
+  - sync-metadata: Both new and old IDs coexist
+- Server Storage
+  - Only stores new IDs but accepts writes using old IDs
+- Protocols
+  - When the client submits an update, both new and old IDs are used.
+  - When the server broadcasts updates sent by other clients, both new and old IDs are used.
+  - When the server responds to `client-pre-sync` (listing all updated docids), only new IDs are used.

packages/common/infra/src/sync/indexer/indexer.ts

@@ -20,6 +20,9 @@ export interface IndexWriter<S extends Schema>
 
   delete(id: string): void;
 
+  // TODO(@eyhn)
+  // deleteByQuery(query: Query<S>): void;
+
   commit(): Promise<void>;
 
   rollback(): void;

packages/common/infra/src/sync/job/impl/indexeddb/index.ts

@@ -155,6 +155,9 @@ export class IndexedDBJobQueue<J> implements JobQueue<J> {
         .objectStore('jobs')
         .delete(typeof id === 'string' ? parseInt(id) : id);
     }
+
+    trx.commit();
+    this.broadcast.postMessage('job-completed');
   }
 
   async return(jobs: Job[], retry: boolean = false): Promise<void> {
@@ -174,6 +177,10 @@ export class IndexedDBJobQueue<J> implements JobQueue<J> {
           .delete(typeof id === 'string' ? parseInt(id) : id);
       }
     }
+
+    trx.commit();
+
+    this.broadcast.postMessage('job-completed');
   }
 
   async clear(): Promise<void> {

packages/common/infra/src/sync/job/runner.ts

@@ -29,7 +29,7 @@ export class JobRunner<J> {
   }
 
   stop() {
-    this.abort?.abort();
+    this.abort?.abort(MANUALLY_STOP);
     this.abort = null;
   }