feat(core): user data db (#7930)

packages/frontend/electron/src/helper/db/ensure-db.ts

@@ -1,18 +1,23 @@
 import { logger } from '../logger';
+import type { SpaceType } from './types';
 import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
 import { openWorkspaceDatabase } from './workspace-db-adapter';
 
 // export for testing
-export const db$Map = new Map<string, Promise<WorkspaceSQLiteDB>>();
+export const db$Map = new Map<
+  `${SpaceType}:${string}`,
+  Promise<WorkspaceSQLiteDB>
+>();
 
-async function getWorkspaceDB(id: string) {
-  let db = await db$Map.get(id);
-  if (!db$Map.has(id)) {
-    const promise = openWorkspaceDatabase(id);
-    db$Map.set(id, promise);
+async function getWorkspaceDB(spaceType: SpaceType, id: string) {
+  const cacheId = `${spaceType}:${id}` as const;
+  let db = await db$Map.get(cacheId);
+  if (!db) {
+    const promise = openWorkspaceDatabase(spaceType, id);
+    db$Map.set(cacheId, promise);
     const _db = (db = await promise);
     const cleanup = () => {
-      db$Map.delete(id);
+      db$Map.delete(cacheId);
       _db
         .destroy()
         .then(() => {
@@ -33,6 +38,6 @@ async function getWorkspaceDB(id: string) {
   return db!;
 }
 
-export function ensureSQLiteDB(id: string) {
-  return getWorkspaceDB(id);
+export function ensureSQLiteDB(spaceType: SpaceType, id: string) {
+  return getWorkspaceDB(spaceType, id);
 }

packages/frontend/electron/src/helper/db/index.ts

@@ -1,92 +1,129 @@
 import { mainRPC } from '../main-rpc';
 import type { MainEventRegister } from '../type';
 import { ensureSQLiteDB } from './ensure-db';
+import type { SpaceType } from './types';
 
 export * from './ensure-db';
 
 export const dbHandlers = {
-  getDocAsUpdates: async (workspaceId: string, subdocId: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.getDocAsUpdates(subdocId);
+  getDocAsUpdates: async (
+    spaceType: SpaceType,
+    workspaceId: string,
+    subdocId: string
+  ) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.getDocAsUpdates(subdocId);
   },
   applyDocUpdate: async (
+    spaceType: SpaceType,
     workspaceId: string,
     update: Uint8Array,
     subdocId: string
   ) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.addUpdateToSQLite(update, subdocId);
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.addUpdateToSQLite(update, subdocId);
   },
-  deleteDoc: async (workspaceId: string, subdocId: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.deleteUpdate(subdocId);
+  deleteDoc: async (
+    spaceType: SpaceType,
+    workspaceId: string,
+    subdocId: string
+  ) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.deleteUpdate(subdocId);
   },
-  addBlob: async (workspaceId: string, key: string, data: Uint8Array) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.addBlob(key, data);
+  addBlob: async (
+    spaceType: SpaceType,
+    workspaceId: string,
+    key: string,
+    data: Uint8Array
+  ) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.addBlob(key, data);
   },
-  getBlob: async (workspaceId: string, key: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.getBlob(key);
+  getBlob: async (spaceType: SpaceType, workspaceId: string, key: string) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.getBlob(key);
   },
-  deleteBlob: async (workspaceId: string, key: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.deleteBlob(key);
+  deleteBlob: async (
+    spaceType: SpaceType,
+    workspaceId: string,
+    key: string
+  ) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.deleteBlob(key);
   },
-  getBlobKeys: async (workspaceId: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.getBlobKeys();
+  getBlobKeys: async (spaceType: SpaceType, workspaceId: string) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.getBlobKeys();
   },
   getDefaultStorageLocation: async () => {
     return await mainRPC.getPath('sessionData');
   },
-  getServerClock: async (workspaceId: string, key: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.serverClock.get(key);
+  getServerClock: async (
+    spaceType: SpaceType,
+    workspaceId: string,
+    key: string
+  ) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.serverClock.get(key);
   },
   setServerClock: async (
+    spaceType: SpaceType,
     workspaceId: string,
     key: string,
     data: Uint8Array
   ) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.serverClock.set(key, data);
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.serverClock.set(key, data);
   },
-  getServerClockKeys: async (workspaceId: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.serverClock.keys();
+  getServerClockKeys: async (spaceType: SpaceType, workspaceId: string) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.serverClock.keys();
   },
-  clearServerClock: async (workspaceId: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.serverClock.clear();
+  clearServerClock: async (spaceType: SpaceType, workspaceId: string) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.serverClock.clear();
   },
-  delServerClock: async (workspaceId: string, key: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.serverClock.del(key);
+  delServerClock: async (
+    spaceType: SpaceType,
+    workspaceId: string,
+    key: string
+  ) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.serverClock.del(key);
   },
-  getSyncMetadata: async (workspaceId: string, key: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.syncMetadata.get(key);
+  getSyncMetadata: async (
+    spaceType: SpaceType,
+    workspaceId: string,
+    key: string
+  ) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.syncMetadata.get(key);
   },
   setSyncMetadata: async (
+    spaceType: SpaceType,
     workspaceId: string,
     key: string,
     data: Uint8Array
   ) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.syncMetadata.set(key, data);
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.syncMetadata.set(key, data);
   },
-  getSyncMetadataKeys: async (workspaceId: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.syncMetadata.keys();
+  getSyncMetadataKeys: async (spaceType: SpaceType, workspaceId: string) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.syncMetadata.keys();
   },
-  clearSyncMetadata: async (workspaceId: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.syncMetadata.clear();
+  clearSyncMetadata: async (spaceType: SpaceType, workspaceId: string) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.syncMetadata.clear();
   },
-  delSyncMetadata: async (workspaceId: string, key: string) => {
-    const workspaceDB = await ensureSQLiteDB(workspaceId);
-    return workspaceDB.adapter.syncMetadata.del(key);
+  delSyncMetadata: async (
+    spaceType: SpaceType,
+    workspaceId: string,
+    key: string
+  ) => {
+    const spaceDB = await ensureSQLiteDB(spaceType, workspaceId);
+    return spaceDB.adapter.syncMetadata.del(key);
   },
 };
 

packages/frontend/electron/src/helper/db/types.ts

@@ -0,0 +1 @@
+export type SpaceType = 'userspace' | 'workspace';

packages/frontend/electron/src/helper/db/workspace-db-adapter.ts

@@ -6,6 +6,7 @@ import { logger } from '../logger';
 import { getWorkspaceMeta } from '../workspace/meta';
 import { SQLiteAdapter } from './db-adapter';
 import { mergeUpdate } from './merge-update';
+import type { SpaceType } from './types';
 
 const TRIM_SIZE = 1;
 
@@ -121,10 +122,13 @@ export class WorkspaceSQLiteDB {
   };
 }
 
-export async function openWorkspaceDatabase(workspaceId: string) {
-  const meta = await getWorkspaceMeta(workspaceId);
-  const db = new WorkspaceSQLiteDB(meta.mainDBPath, workspaceId);
+export async function openWorkspaceDatabase(
+  spaceType: SpaceType,
+  spaceId: string
+) {
+  const meta = await getWorkspaceMeta(spaceType, spaceId);
+  const db = new WorkspaceSQLiteDB(meta.mainDBPath, spaceId);
   await db.init();
-  logger.info(`openWorkspaceDatabase [${workspaceId}]`);
+  logger.info(`openWorkspaceDatabase [${spaceId}]`);
   return db;
 }

packages/frontend/electron/src/helper/dialog/dialog.ts

@@ -6,11 +6,7 @@ import { ensureSQLiteDB } from '../db/ensure-db';
 import { logger } from '../logger';
 import { mainRPC } from '../main-rpc';
 import { storeWorkspaceMeta } from '../workspace';
-import {
-  getWorkspaceDBPath,
-  getWorkspaceMeta,
-  getWorkspacesBasePath,
-} from '../workspace/meta';
+import { getWorkspaceDBPath, getWorkspacesBasePath } from '../workspace/meta';
 
 export type ErrorMessage =
   | 'DB_FILE_ALREADY_LOADED'
@@ -45,17 +41,6 @@ export interface FakeDialogResult {
   filePaths?: string[];
 }
 
-// NOTE:
-// we are using native dialogs because HTML dialogs do not give full file paths
-
-export async function revealDBFile(workspaceId: string) {
-  const meta = await getWorkspaceMeta(workspaceId);
-  if (!meta) {
-    return;
-  }
-  await mainRPC.showItemInFolder(meta.mainDBPath);
-}
-
 // result will be used in the next call to showOpenDialog
 // if it is being read once, it will be reset to undefined
 let fakeDialogResult: FakeDialogResult | undefined = undefined;
@@ -91,7 +76,7 @@ export async function saveDBFileAs(
   workspaceId: string
 ): Promise<SaveDBFileResult> {
   try {
-    const db = await ensureSQLiteDB(workspaceId);
+    const db = await ensureSQLiteDB('workspace', workspaceId);
     const fakedResult = getFakedResult();
 
     const ret =
@@ -215,7 +200,7 @@ export async function loadDBFile(): Promise<LoadDBFileResult> {
 
     // copy the db file to a new workspace id
     const workspaceId = nanoid(10);
-    const internalFilePath = await getWorkspaceDBPath(workspaceId);
+    const internalFilePath = await getWorkspaceDBPath('workspace', workspaceId);
 
     await fs.ensureDir(await getWorkspacesBasePath());
     await fs.copy(originalPath, internalFilePath);

packages/frontend/electron/src/helper/dialog/index.ts

@@ -1,15 +1,11 @@
 import {
   loadDBFile,
-  revealDBFile,
   saveDBFileAs,
   selectDBFileLocation,
   setFakeDialogResult,
 } from './dialog';
 
 export const dialogHandlers = {
-  revealDBFile: async (workspaceId: string) => {
-    return revealDBFile(workspaceId);
-  },
   loadDBFile: async () => {
     return loadDBFile();
   },

packages/frontend/electron/src/helper/workspace/handlers.ts

@@ -8,44 +8,14 @@ import type { WorkspaceMeta } from '../type';
 import {
   getDeletedWorkspacesBasePath,
   getWorkspaceBasePath,
-  getWorkspaceDBPath,
   getWorkspaceMeta,
-  getWorkspaceMetaPath,
-  getWorkspacesBasePath,
 } from './meta';
-import { workspaceSubjects } from './subjects';
-
-export async function listWorkspaces(): Promise<
-  [workspaceId: string, meta: WorkspaceMeta][]
-> {
-  const basePath = await getWorkspacesBasePath();
-  try {
-    await fs.ensureDir(basePath);
-    const dirs = (
-      await fs.readdir(basePath, {
-        withFileTypes: true,
-      })
-    ).filter(d => d.isDirectory());
-    const metaList = (
-      await Promise.all(
-        dirs.map(async dir => {
-          // ? shall we put all meta in a single file instead of one file per workspace?
-          return await getWorkspaceMeta(dir.name);
-        })
-      )
-    ).filter((w): w is WorkspaceMeta => !!w);
-    return metaList.map(meta => [meta.id, meta]);
-  } catch (error) {
-    logger.error('listWorkspaces', error);
-    return [];
-  }
-}
 
 export async function deleteWorkspace(id: string) {
-  const basePath = await getWorkspaceBasePath(id);
+  const basePath = await getWorkspaceBasePath('workspace', id);
   const movedPath = path.join(await getDeletedWorkspacesBasePath(), `${id}`);
   try {
-    const db = await ensureSQLiteDB(id);
+    const db = await ensureSQLiteDB('workspace', id);
     await db.destroy();
     return await fs.move(basePath, movedPath, {
       overwrite: true,
@@ -55,52 +25,20 @@ export async function deleteWorkspace(id: string) {
   }
 }
 
-export async function cloneWorkspace(id: string, newId: string) {
-  const dbPath = await getWorkspaceDBPath(id);
-  const newBasePath = await getWorkspaceBasePath(newId);
-  const newDbPath = await getWorkspaceDBPath(newId);
-  const metaPath = await getWorkspaceMetaPath(newId);
-  // check if new workspace dir exists
-  if (
-    await fs
-      .access(newBasePath)
-      .then(() => true)
-      .catch(() => false)
-  ) {
-    throw new Error(`workspace ${newId} already exists`);
-  }
-
-  try {
-    await fs.ensureDir(newBasePath);
-    const meta = {
-      id: newId,
-      mainDBPath: newDbPath,
-    };
-    await fs.writeJSON(metaPath, meta);
-    await fs.copy(dbPath, newDbPath);
-  } catch (error) {
-    logger.error('cloneWorkspace', error);
-  }
-}
-
 export async function storeWorkspaceMeta(
   workspaceId: string,
   meta: Partial<WorkspaceMeta>
 ) {
   try {
-    const basePath = await getWorkspaceBasePath(workspaceId);
+    const basePath = await getWorkspaceBasePath('workspace', workspaceId);
     await fs.ensureDir(basePath);
     const metaPath = path.join(basePath, 'meta.json');
-    const currentMeta = await getWorkspaceMeta(workspaceId);
+    const currentMeta = await getWorkspaceMeta('workspace', workspaceId);
     const newMeta = {
       ...currentMeta,
       ...meta,
     };
     await fs.writeJSON(metaPath, newMeta);
-    workspaceSubjects.meta$.next({
-      workspaceId,
-      meta: newMeta,
-    });
   } catch (err) {
     logger.error('storeWorkspaceMeta failed', err);
   }

packages/frontend/electron/src/helper/workspace/index.ts

@@ -1,27 +1,11 @@
-import type { MainEventRegister, WorkspaceMeta } from '../type';
-import { cloneWorkspace, deleteWorkspace, listWorkspaces } from './handlers';
-import { getWorkspaceMeta } from './meta';
-import { workspaceSubjects } from './subjects';
+import type { MainEventRegister } from '../type';
+import { deleteWorkspace } from './handlers';
 
 export * from './handlers';
 export * from './subjects';
 
-export const workspaceEvents = {
-  onMetaChange: (
-    fn: (meta: { workspaceId: string; meta: WorkspaceMeta }) => void
-  ) => {
-    const sub = workspaceSubjects.meta$.subscribe(fn);
-    return () => {
-      sub.unsubscribe();
-    };
-  },
-} satisfies Record<string, MainEventRegister>;
+export const workspaceEvents = {} as Record<string, MainEventRegister>;
 
 export const workspaceHandlers = {
-  list: async () => listWorkspaces(),
   delete: async (id: string) => deleteWorkspace(id),
-  getMeta: async (id: string) => {
-    return getWorkspaceMeta(id);
-  },
-  clone: async (id: string, newId: string) => cloneWorkspace(id, newId),
 };

packages/frontend/electron/src/helper/workspace/meta.ts

@@ -2,6 +2,7 @@ import path from 'node:path';
 
 import fs from 'fs-extra';
 
+import type { SpaceType } from '../db/types';
 import { logger } from '../logger';
 import { mainRPC } from '../main-rpc';
 import type { WorkspaceMeta } from '../type';
@@ -20,20 +21,39 @@ export async function getWorkspacesBasePath() {
   return path.join(await getAppDataPath(), 'workspaces');
 }
 
-export async function getWorkspaceBasePath(workspaceId: string) {
-  return path.join(await getAppDataPath(), 'workspaces', workspaceId);
+export async function getWorkspaceBasePath(
+  spaceType: SpaceType,
+  workspaceId: string
+) {
+  return path.join(
+    await getAppDataPath(),
+    spaceType === 'userspace' ? 'userspaces' : 'workspaces',
+    workspaceId
+  );
 }
 
 export async function getDeletedWorkspacesBasePath() {
   return path.join(await getAppDataPath(), 'deleted-workspaces');
 }
 
-export async function getWorkspaceDBPath(workspaceId: string) {
-  return path.join(await getWorkspaceBasePath(workspaceId), 'storage.db');
+export async function getWorkspaceDBPath(
+  spaceType: SpaceType,
+  workspaceId: string
+) {
+  return path.join(
+    await getWorkspaceBasePath(spaceType, workspaceId),
+    'storage.db'
+  );
 }
 
-export async function getWorkspaceMetaPath(workspaceId: string) {
-  return path.join(await getWorkspaceBasePath(workspaceId), 'meta.json');
+export async function getWorkspaceMetaPath(
+  spaceType: SpaceType,
+  workspaceId: string
+) {
+  return path.join(
+    await getWorkspaceBasePath(spaceType, workspaceId),
+    'meta.json'
+  );
 }
 
 /**
@@ -41,11 +61,12 @@ export async function getWorkspaceMetaPath(workspaceId: string) {
  * This function will also migrate the workspace if needed
  */
 export async function getWorkspaceMeta(
+  spaceType: SpaceType,
   workspaceId: string
 ): Promise<WorkspaceMeta> {
   try {
-    const basePath = await getWorkspaceBasePath(workspaceId);
-    const metaPath = await getWorkspaceMetaPath(workspaceId);
+    const basePath = await getWorkspaceBasePath(spaceType, workspaceId);
+    const metaPath = await getWorkspaceMetaPath(spaceType, workspaceId);
     if (
       !(await fs
         .access(metaPath)
@@ -53,11 +74,12 @@ export async function getWorkspaceMeta(
         .catch(() => false))
     ) {
       await fs.ensureDir(basePath);
-      const dbPath = await getWorkspaceDBPath(workspaceId);
+      const dbPath = await getWorkspaceDBPath(spaceType, workspaceId);
       // create one if not exists
       const meta = {
         id: workspaceId,
         mainDBPath: dbPath,
+        type: spaceType,
       };
       await fs.writeJSON(metaPath, meta);
       return meta;