refactor(electron): sqlite db data workflow (remove symlink & fs watcher) (#2491)

2026-08-23 04:51:49 +08:00 · 2023-05-29 12:53:15 +08:00
parent 975023463c
commit 96d313d491
58 changed files with 1078 additions and 896 deletions
@@ -0,0 +1,191 @@
+import { debounce } from 'lodash-es';
+import * as Y from 'yjs';
+
+import type { AppContext } from '../context';
+import { logger } from '../logger';
+import type { YOrigin } from '../type';
+import { getWorkspaceMeta } from '../workspace';
+import { BaseSQLiteAdapter } from './base-db-adapter';
+import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
+
+const FLUSH_WAIT_TIME = 5000;
+const FLUSH_MAX_WAIT_TIME = 10000;
+
+export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
+  role = 'secondary';
+  yDoc = new Y.Doc();
+  firstConnected = false;
+
+  updateQueue: Uint8Array[] = [];
+
+  unsubscribers = new Set<() => void>();
+
+  constructor(
+    public override path: string,
+    public upstream: WorkspaceSQLiteDB
+  ) {
+    super(path);
+    this.setupAndListen();
+  }
+
+  close() {
+    this.db?.close();
+    this.db = null;
+  }
+
+  override destroy() {
+    this.unsubscribers.forEach(unsub => unsub());
+    this.db?.close();
+    this.yDoc.destroy();
+    this.close();
+  }
+
+  get workspaceId() {
+    return this.upstream.workspaceId;
+  }
+
+  // do not update db immediately, instead, push to a queue
+  // and flush the queue in a future time
+  addUpdateToUpdateQueue(update: Uint8Array) {
+    this.updateQueue.push(update);
+    this.debouncedFlush();
+  }
+
+  flushUpdateQueue() {
+    logger.debug(
+      'flushUpdateQueue',
+      this.workspaceId,
+      'queue',
+      this.updateQueue.length
+    );
+    const updates = [...this.updateQueue];
+    this.updateQueue = [];
+    this.connect();
+    this.addUpdateToSQLite(updates);
+    this.close();
+  }
+
+  // flush after 5s, but will not wait for more than 10s
+  debouncedFlush = debounce(this.flushUpdateQueue, FLUSH_WAIT_TIME, {
+    maxWait: FLUSH_MAX_WAIT_TIME,
+  });
+
+  runCounter = 0;
+
+  // wrap the fn with connect and close
+  // it only works for sync functions
+  run = (fn: () => void) => {
+    try {
+      if (this.runCounter === 0) {
+        this.connect();
+      }
+      this.runCounter++;
+      fn();
+    } catch (err) {
+      logger.error(err);
+    } finally {
+      this.runCounter--;
+      if (this.runCounter === 0) {
+        this.close();
+      }
+    }
+  };
+
+  setupAndListen() {
+    if (this.firstConnected) {
+      return;
+    }
+    this.firstConnected = true;
+
+    const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => {
+      if (origin === 'renderer') {
+        // update to upstream yDoc should be replicated to self yDoc
+        this.applyUpdate(update, 'upstream');
+      }
+    };
+
+    const onSelfUpdate = (update: Uint8Array, origin: YOrigin) => {
+      // for self update from upstream, we need to push it to external DB
+      if (origin === 'upstream') {
+        this.addUpdateToUpdateQueue(update);
+      }
+
+      if (origin === 'self') {
+        this.upstream.applyUpdate(update, 'external');
+      }
+    };
+
+    // listen to upstream update
+    this.upstream.yDoc.on('update', onUpstreamUpdate);
+    this.yDoc.on('update', onSelfUpdate);
+
+    this.unsubscribers.add(() => {
+      this.upstream.yDoc.off('update', onUpstreamUpdate);
+      this.yDoc.off('update', onSelfUpdate);
+    });
+
+    this.run(() => {
+      // apply all updates from upstream
+      const upstreamUpdate = this.upstream.getDocAsUpdates();
+      // to initialize the yDoc, we need to apply all updates from the db
+      this.applyUpdate(upstreamUpdate, 'upstream');
+
+      this.pull();
+    });
+  }
+
+  applyUpdate = (data: Uint8Array, origin: YOrigin = 'upstream') => {
+    Y.applyUpdate(this.yDoc, data, origin);
+  };
+
+  // TODO: have a better solution to handle blobs
+  syncBlobs() {
+    this.run(() => {
+      // pull blobs
+      const blobsKeys = this.getBlobKeys();
+      const upstreamBlobsKeys = this.upstream.getBlobKeys();
+      // put every missing blob to upstream
+      for (const key of blobsKeys) {
+        if (!upstreamBlobsKeys.includes(key)) {
+          const blob = this.getBlob(key);
+          if (blob) {
+            this.upstream.addBlob(key, blob);
+            logger.debug('syncBlobs', this.workspaceId, key);
+          }
+        }
+      }
+    });
+  }
+
+  /**
+   * pull from external DB file and apply to embedded yDoc
+   * workflow:
+   * - connect to external db
+   * - get updates
+   * - apply updates to local yDoc
+   * - get blobs and put new blobs to upstream
+   * - disconnect
+   */
+  pull() {
+    this.run(() => {
+      // TODO: no need to get all updates, just get the latest ones (using a cursor, etc)?
+      const updates = this.getUpdates().map(update => update.data);
+      Y.transact(this.yDoc, () => {
+        updates.forEach(update => {
+          this.applyUpdate(update, 'self');
+        });
+      });
+      logger.debug('pull external updates', this.path, updates.length);
+
+      this.syncBlobs();
+    });
+  }
+}
+
+export async function getSecondaryWorkspaceDBPath(
+  context: AppContext,
+  workspaceId: string
+) {
+  const meta = await getWorkspaceMeta(context, workspaceId);
+  return meta?.secondaryDBPath;
+}