diff --git a/packages/common/env/src/global.ts b/packages/common/env/src/global.ts index 3a4922ca65..12e84c3fc3 100644 --- a/packages/common/env/src/global.ts +++ b/packages/common/env/src/global.ts @@ -26,7 +26,6 @@ export const runtimeFlagsSchema = z.object({ allowLocalWorkspace: z.boolean(), // this is for the electron app serverUrlPrefix: z.string(), - enableMoveDatabase: z.boolean(), appVersion: z.string(), editorVersion: z.string(), appBuildType: z.union([ diff --git a/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/index.tsx b/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/index.tsx index 755612da18..45c591765b 100644 --- a/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/index.tsx +++ b/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/index.tsx @@ -17,7 +17,6 @@ import { ExportPanel } from './export'; import { LabelsPanel } from './labels'; import { MembersPanel } from './members'; import { ProfilePanel } from './profile'; -import { StoragePanel } from './storage'; import type { WorkspaceSettingDetailProps } from './types'; export const WorkspaceSettingDetail = ({ @@ -70,9 +69,6 @@ export const WorkspaceSettingDetail = ({ {environment.isDesktop && ( - {runtimeConfig.enableMoveDatabase ? ( - - ) : null} { - const [path, setPath] = useState(undefined); - useEffect(() => { - if (apis && events && environment.isDesktop) { - apis?.workspace - .getMeta(workspaceId) - .then(meta => { - setPath(meta.secondaryDBPath); - }) - .catch(err => { - console.error(err); - }); - return events.workspace.onMetaChange((newMeta: any) => { - if (newMeta.workspaceId === workspaceId) { - const meta = newMeta.meta; - setPath(meta.secondaryDBPath); - } - }); - } - return; - }, [workspaceId]); - return path; -}; - -interface StoragePanelProps { - workspaceMetadata: WorkspaceMetadata; -} - -export const StoragePanel = ({ workspaceMetadata }: StoragePanelProps) => { - const workspaceId = workspaceMetadata.id; - const t = useAFFiNEI18N(); - const secondaryPath = useDBFileSecondaryPath(workspaceId); - - const [moveToInProgress, setMoveToInProgress] = useState(false); - const onRevealDBFile = useCallback(() => { - apis?.dialog.revealDBFile(workspaceId).catch(err => { - console.error(err); - }); - }, [workspaceId]); - - const handleMoveTo = useCallback(() => { - if (moveToInProgress) { - return; - } - setMoveToInProgress(true); - apis?.dialog - .moveDBFile(workspaceId) - .then(result => { - if (!result?.error && !result?.canceled) { - toast(t['Move folder success']()); - } else if (result?.error) { - toast(t[result.error]()); - } - }) - .catch(() => { - toast(t['UNKNOWN_ERROR']()); - }) - .finally(() => { - setMoveToInProgress(false); - }); - }, [moveToInProgress, t, workspaceId]); - - const rowContent = useMemo( - () => - secondaryPath ? ( - - - - - - - ) : ( - - ), - [handleMoveTo, moveToInProgress, onRevealDBFile, secondaryPath, t] - ); - - return ( - - {rowContent} - - ); -}; diff --git a/packages/frontend/electron/src/helper/db/ensure-db.ts b/packages/frontend/electron/src/helper/db/ensure-db.ts index dc0d857993..a75c5add59 100644 --- a/packages/frontend/electron/src/helper/db/ensure-db.ts +++ b/packages/frontend/electron/src/helper/db/ensure-db.ts @@ -1,145 +1,38 @@ -import type { Subject } from 'rxjs'; -import { - concat, - defer, - from, - fromEvent, - interval, - lastValueFrom, - merge, - Observable, -} from 'rxjs'; -import { - concatMap, - distinctUntilChanged, - filter, - ignoreElements, - last, - map, - shareReplay, - startWith, - switchMap, - take, - takeUntil, - tap, -} from 'rxjs/operators'; - import { logger } from '../logger'; -import { getWorkspaceMeta } from '../workspace/meta'; -import { workspaceSubjects } from '../workspace/subjects'; -import { SecondaryWorkspaceSQLiteDB } from './secondary-db'; import type { WorkspaceSQLiteDB } from './workspace-db-adapter'; import { openWorkspaceDatabase } from './workspace-db-adapter'; // export for testing -export const db$Map = new Map>(); +export const db$Map = new Map>(); -// use defer to prevent `app` is undefined while running tests -const beforeQuit$ = defer(() => fromEvent(process, 'beforeExit')); - -// return a stream that emit a single event when the subject completes -function completed(subject$: Subject) { - return new Observable(subscriber => { - const sub = subject$.subscribe({ - complete: () => { - subscriber.next(); - subscriber.complete(); - }, - }); - return () => sub.unsubscribe(); - }); -} - -function getWorkspaceDB(id: string) { +async function getWorkspaceDB(id: string) { + let db = await db$Map.get(id); if (!db$Map.has(id)) { - db$Map.set( - id, - from(openWorkspaceDatabase(id)).pipe( - tap({ - next: db => { - logger.info( - '[ensureSQLiteDB] db connection established', - db.workspaceId - ); - }, - }), - switchMap(db => - // takeUntil the polling stream, and then destroy the db - concat( - startPollingSecondaryDB(db).pipe( - ignoreElements(), - startWith(db), - takeUntil(merge(beforeQuit$, completed(db.update$))), - last(), - tap({ - next() { - logger.info( - '[ensureSQLiteDB] polling secondary db complete', - db.workspaceId - ); - }, - }) - ), - defer(async () => { - try { - await db.destroy(); - db$Map.delete(id); - return db; - } catch (err) { - logger.error('[ensureSQLiteDB] destroy db failed', err); - throw err; - } - }) - ).pipe(startWith(db)) - ), - shareReplay(1) - ) - ); + const promise = openWorkspaceDatabase(id); + db$Map.set(id, promise); + const _db = (db = await promise); + const cleanup = () => { + db$Map.delete(id); + _db + .destroy() + .then(() => { + logger.info('[ensureSQLiteDB] db connection closed', _db.workspaceId); + }) + .catch(err => { + logger.error('[ensureSQLiteDB] destroy db failed', err); + }); + }; + + db.update$.subscribe({ + complete: cleanup, + }); + + process.on('beforeExit', cleanup); } // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return db$Map.get(id)!; -} - -function startPollingSecondaryDB(db: WorkspaceSQLiteDB) { - return merge( - getWorkspaceMeta(db.workspaceId), - workspaceSubjects.meta$.pipe( - map(({ meta }) => meta), - filter(meta => meta.id === db.workspaceId) - ) - ).pipe( - map(meta => meta?.secondaryDBPath), - filter((p): p is string => !!p), - distinctUntilChanged(), - switchMap(path => { - // on secondary db path change, destroy the old db and create a new one - const secondaryDB = new SecondaryWorkspaceSQLiteDB(path, db); - return new Observable(subscriber => { - subscriber.next(secondaryDB); - return () => { - secondaryDB.destroy().catch(err => { - subscriber.error(err); - }); - }; - }); - }), - switchMap(secondaryDB => { - return interval(300000).pipe( - startWith(0), - concatMap(() => secondaryDB.pull()), - tap({ - error: err => { - logger.error(`[ensureSQLiteDB] polling secondary db error`, err); - }, - complete: () => { - logger.info('[ensureSQLiteDB] polling secondary db complete'); - }, - }) - ); - }) - ); + return db!; } export function ensureSQLiteDB(id: string) { - return lastValueFrom(getWorkspaceDB(id).pipe(take(1))); + return getWorkspaceDB(id); } diff --git a/packages/frontend/electron/src/helper/db/index.ts b/packages/frontend/electron/src/helper/db/index.ts index 1fabc02f43..fbe7ee83ef 100644 --- a/packages/frontend/electron/src/helper/db/index.ts +++ b/packages/frontend/electron/src/helper/db/index.ts @@ -1,10 +1,8 @@ import { mainRPC } from '../main-rpc'; import type { MainEventRegister } from '../type'; import { ensureSQLiteDB } from './ensure-db'; -import { dbSubjects } from './subjects'; export * from './ensure-db'; -export * from './subjects'; export const dbHandlers = { getDocAsUpdates: async (workspaceId: string, subdocId?: string) => { @@ -17,7 +15,12 @@ export const dbHandlers = { subdocId?: string ) => { const workspaceDB = await ensureSQLiteDB(workspaceId); - return workspaceDB.applyUpdate(update, 'renderer', subdocId); + return workspaceDB.addUpdateToSQLite([ + { + data: update, + docId: subdocId, + }, + ]); }, addBlob: async (workspaceId: string, key: string, data: Uint8Array) => { const workspaceDB = await ensureSQLiteDB(workspaceId); @@ -40,17 +43,4 @@ export const dbHandlers = { }, }; -export const dbEvents = { - onExternalUpdate: ( - fn: (update: { - workspaceId: string; - update: Uint8Array; - docId?: string; - }) => void - ) => { - const sub = dbSubjects.externalUpdate$.subscribe(fn); - return () => { - sub.unsubscribe(); - }; - }, -} satisfies Record; +export const dbEvents = {} satisfies Record; diff --git a/packages/frontend/electron/src/helper/db/secondary-db.ts b/packages/frontend/electron/src/helper/db/secondary-db.ts deleted file mode 100644 index 7971facdfa..0000000000 --- a/packages/frontend/electron/src/helper/db/secondary-db.ts +++ /dev/null @@ -1,304 +0,0 @@ -import assert from 'node:assert'; - -import type { InsertRow } from '@affine/native'; -import { debounce } from 'lodash-es'; -import { applyUpdate, Doc as YDoc } from 'yjs'; - -import { logger } from '../logger'; -import type { YOrigin } from '../type'; -import { getWorkspaceMeta } from '../workspace/meta'; -import { BaseSQLiteAdapter } from './base-db-adapter'; -import type { WorkspaceSQLiteDB } from './workspace-db-adapter'; - -const FLUSH_WAIT_TIME = 5000; -const FLUSH_MAX_WAIT_TIME = 10000; - -// todo: trim db when it is too big -export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { - role = 'secondary'; - yDoc = new YDoc(); - firstConnected = false; - destroyed = false; - - updateQueue: { data: Uint8Array; docId?: string }[] = []; - - unsubscribers = new Set<() => void>(); - - constructor( - public override path: string, - public upstream: WorkspaceSQLiteDB - ) { - super(path); - this.init(); - logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId); - } - - getDoc(docId?: string) { - if (!docId) { - return this.yDoc; - } - // this should be pretty fast and we don't need to cache it - for (const subdoc of this.yDoc.subdocs) { - if (subdoc.guid === docId) { - return subdoc; - } - } - return null; - } - - override async destroy() { - await this.flushUpdateQueue(); - this.unsubscribers.forEach(unsub => unsub()); - this.yDoc.destroy(); - await super.destroy(); - this.destroyed = true; - } - - get workspaceId() { - return this.upstream.workspaceId; - } - - // do not update db immediately, instead, push to a queue - // and flush the queue in a future time - async addUpdateToUpdateQueue(update: InsertRow) { - this.updateQueue.push(update); - await this.debouncedFlush(); - } - - async flushUpdateQueue() { - if (this.destroyed) { - return; - } - logger.debug( - 'flushUpdateQueue', - this.workspaceId, - 'queue', - this.updateQueue.length - ); - const updates = [...this.updateQueue]; - this.updateQueue = []; - await this.run(async () => { - await this.addUpdateToSQLite(updates); - }); - } - - // flush after 5s, but will not wait for more than 10s - debouncedFlush = debounce(this.flushUpdateQueue, FLUSH_WAIT_TIME, { - maxWait: FLUSH_MAX_WAIT_TIME, - }); - - runCounter = 0; - - // wrap the fn with connect and close - async run any>( - fn: T - ): Promise< - (T extends (...args: any[]) => infer U ? Awaited : unknown) | undefined - > { - try { - if (this.destroyed) { - return; - } - await this.connectIfNeeded(); - this.runCounter++; - return await fn(); - } catch (err) { - logger.error(err); - throw err; - } finally { - this.runCounter--; - if (this.runCounter === 0) { - // just close db, but not the yDoc - await super.destroy(); - } - } - } - - setupListener(docId?: string) { - logger.debug( - 'SecondaryWorkspaceSQLiteDB:setupListener', - this.workspaceId, - docId - ); - const doc = this.getDoc(docId); - const upstreamDoc = this.upstream.getDoc(docId); - if (!doc || !upstreamDoc) { - logger.warn( - '[SecondaryWorkspaceSQLiteDB] setupListener: doc not found', - docId - ); - return; - } - - const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => { - logger.debug( - 'SecondaryWorkspaceSQLiteDB:onUpstreamUpdate', - origin, - this.workspaceId, - docId, - update.length - ); - if (origin === 'renderer' || origin === 'self') { - // update to upstream yDoc should be replicated to self yDoc - this.applyUpdate(update, 'upstream', docId); - } - }; - - const onSelfUpdate = async (update: Uint8Array, origin: YOrigin) => { - logger.debug( - 'SecondaryWorkspaceSQLiteDB:onSelfUpdate', - origin, - this.workspaceId, - docId, - update.length - ); - // for self update from upstream, we need to push it to external DB - if (origin === 'upstream') { - await this.addUpdateToUpdateQueue({ - data: update, - docId, - }); - } - - if (origin === 'self') { - this.upstream.applyUpdate(update, 'external', docId); - } - }; - - const onSubdocs = ({ added }: { added: Set }) => { - added.forEach(subdoc => { - this.setupListener(subdoc.guid); - }); - }; - - doc.subdocs.forEach(subdoc => { - this.setupListener(subdoc.guid); - }); - - // listen to upstream update - this.upstream.yDoc.on('update', onUpstreamUpdate); - doc.on('update', (update, origin) => { - onSelfUpdate(update, origin).catch(err => { - logger.error(err); - }); - }); - doc.on('subdocs', onSubdocs); - - this.unsubscribers.add(() => { - this.upstream.yDoc.off('update', onUpstreamUpdate); - doc.off('update', (update, origin) => { - onSelfUpdate(update, origin).catch(err => { - logger.error(err); - }); - }); - doc.off('subdocs', onSubdocs); - }); - } - - init() { - if (this.firstConnected) { - return; - } - this.firstConnected = true; - this.setupListener(); - // apply all updates from upstream - // we assume here that the upstream ydoc is already sync'ed - const syncUpstreamDoc = (docId?: string) => { - const update = this.upstream.getDocAsUpdates(docId); - if (update) { - this.applyUpdate(update, 'upstream'); - } - }; - syncUpstreamDoc(); - this.upstream.yDoc.subdocs.forEach(subdoc => { - syncUpstreamDoc(subdoc.guid); - }); - } - - applyUpdate = ( - data: Uint8Array, - origin: YOrigin = 'upstream', - docId?: string - ) => { - const doc = this.getDoc(docId); - if (doc) { - applyUpdate(this.yDoc, data, origin); - } else { - logger.warn( - '[SecondaryWorkspaceSQLiteDB] applyUpdate: doc not found', - docId - ); - } - }; - - // TODO: have a better solution to handle blobs - async syncBlobs() { - await this.run(async () => { - // skip if upstream db is not connected (maybe it is already closed) - const blobsKeys = await this.getBlobKeys(); - if (!this.upstream.db || this.upstream.db?.isClose) { - return; - } - const upstreamBlobsKeys = await this.upstream.getBlobKeys(); - // put every missing blob to upstream - for (const key of blobsKeys) { - if (!upstreamBlobsKeys.includes(key)) { - const blob = await this.getBlob(key); - if (blob) { - await this.upstream.addBlob(key, blob); - logger.debug('syncBlobs', this.workspaceId, key); - } - } - } - }); - } - - /** - * pull from external DB file and apply to embedded yDoc - * workflow: - * - connect to external db - * - get updates - * - apply updates to local yDoc - * - get blobs and put new blobs to upstream - * - disconnect - */ - async pull() { - const start = performance.now(); - assert(this.upstream.db, 'upstream db should be connected'); - const rows = await this.run(async () => { - // TODO: no need to get all updates, just get the latest ones (using a cursor, etc)? - await this.syncBlobs(); - return await this.getAllUpdates(); - }); - - if (!rows || this.destroyed) { - return; - } - - // apply root doc first - rows.forEach(row => { - if (!row.docId) { - this.applyUpdate(row.data, 'self'); - } - }); - - rows.forEach(row => { - if (row.docId) { - this.applyUpdate(row.data, 'self', row.docId); - } - }); - - logger.debug( - 'pull external updates', - this.path, - rows.length, - (performance.now() - start).toFixed(2), - 'ms' - ); - } -} - -export async function getSecondaryWorkspaceDBPath(workspaceId: string) { - const meta = await getWorkspaceMeta(workspaceId); - return meta?.secondaryDBPath; -} diff --git a/packages/frontend/electron/src/helper/db/subjects.ts b/packages/frontend/electron/src/helper/db/subjects.ts deleted file mode 100644 index a1acbbe62f..0000000000 --- a/packages/frontend/electron/src/helper/db/subjects.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Subject } from 'rxjs'; - -export const dbSubjects = { - externalUpdate$: new Subject<{ - workspaceId: string; - update: Uint8Array; - docId?: string; - }>(), -}; diff --git a/packages/frontend/electron/src/helper/db/workspace-db-adapter.ts b/packages/frontend/electron/src/helper/db/workspace-db-adapter.ts index fbb7cb7a67..24d0faf921 100644 --- a/packages/frontend/electron/src/helper/db/workspace-db-adapter.ts +++ b/packages/frontend/electron/src/helper/db/workspace-db-adapter.ts @@ -1,20 +1,16 @@ import type { InsertRow } from '@affine/native'; -import { debounce } from 'lodash-es'; import { Subject } from 'rxjs'; -import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs'; +import { applyUpdate, Doc as YDoc } from 'yjs'; import { logger } from '../logger'; -import type { YOrigin } from '../type'; import { getWorkspaceMeta } from '../workspace/meta'; import { BaseSQLiteAdapter } from './base-db-adapter'; -import { dbSubjects } from './subjects'; +import { mergeUpdate } from './merge-update'; const TRIM_SIZE = 500; export class WorkspaceSQLiteDB extends BaseSQLiteAdapter { role = 'primary'; - yDoc = new YDoc(); - firstConnected = false; update$ = new Subject(); @@ -27,131 +23,30 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter { override async destroy() { await super.destroy(); - this.yDoc.destroy(); // when db is closed, we can safely remove it from ensure-db list this.update$.complete(); - this.firstConnected = false; } - getDoc(docId?: string) { - if (!docId) { - return this.yDoc; - } - // this should be pretty fast and we don't need to cache it - for (const subdoc of this.yDoc.subdocs) { - if (subdoc.guid === docId) { - return subdoc; - } - } - return null; - } - - getWorkspaceName = () => { - return this.yDoc.getMap('meta').get('name') as string; + getWorkspaceName = async () => { + const ydoc = new YDoc(); + const updates = await this.getUpdates(); + updates.forEach(update => { + applyUpdate(ydoc, update.data); + }); + return ydoc.getMap('meta').get('name') as string; }; - setupListener(docId?: string) { - logger.debug('WorkspaceSQLiteDB:setupListener', this.workspaceId, docId); - const doc = this.getDoc(docId); - if (doc) { - const onUpdate = async (update: Uint8Array, origin: YOrigin) => { - logger.debug( - 'WorkspaceSQLiteDB:onUpdate', - this.workspaceId, - docId, - update.length - ); - const insertRows = [{ data: update, docId }]; - if (origin === 'renderer') { - await this.addUpdateToSQLite(insertRows); - } else if (origin === 'external') { - dbSubjects.externalUpdate$.next({ - workspaceId: this.workspaceId, - update, - docId, - }); - await this.addUpdateToSQLite(insertRows); - logger.debug('external update', this.workspaceId); - } - }; - doc.subdocs.forEach(subdoc => { - this.setupListener(subdoc.guid); - }); - const onSubdocs = ({ added }: { added: Set }) => { - logger.info('onSubdocs', this.workspaceId, docId, added); - added.forEach(subdoc => { - this.setupListener(subdoc.guid); - }); - }; - - doc.on('update', (update, origin) => { - onUpdate(update, origin).catch(err => { - logger.error(err); - }); - }); - doc.on('subdocs', onSubdocs); - } else { - logger.error('setupListener: doc not found', docId); - } - } - async init() { const db = await super.connectIfNeeded(); - - if (!this.firstConnected) { - this.setupListener(); - } - - const updates = await this.getAllUpdates(); - - // apply root first (without ID). - // subdoc will be available after root is applied - updates.forEach(update => { - if (!update.docId) { - this.applyUpdate(update.data, 'self'); - } - }); - - // then, for all subdocs, apply the updates - updates.forEach(update => { - if (update.docId) { - this.applyUpdate(update.data, 'self', update.docId); - } - }); - - this.firstConnected = true; - this.update$.next(); - + await this.tryTrim(); return db; } - // unlike getUpdates, this will return updates in yDoc - getDocAsUpdates = (docId?: string) => { - const doc = docId ? this.getDoc(docId) : this.yDoc; - if (doc) { - return encodeStateAsUpdate(doc); - } - return false; - }; - - // non-blocking and use yDoc to validate the update - // after that, the update is added to the db - applyUpdate = ( - data: Uint8Array, - origin: YOrigin = 'renderer', - docId?: string - ) => { - // todo: trim the updates when the number of records is too large - // 1. store the current ydoc state in the db - // 2. then delete the old updates - // yjs-idb will always trim the db for the first time after DB is loaded - const doc = this.getDoc(docId); - if (doc) { - applyUpdate(doc, data, origin); - } else { - logger.warn('[WorkspaceSQLiteDB] applyUpdate: doc not found', docId); - } + // getUpdates then encode + getDocAsUpdates = async (docId?: string) => { + const updates = await this.getUpdates(docId); + return mergeUpdate(updates.map(row => row.data)); }; override async addBlob(key: string, value: Uint8Array) { @@ -167,28 +62,21 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter { override async addUpdateToSQLite(data: InsertRow[]) { this.update$.next(); - data.forEach(row => { - this.trimWhenNecessary(row.docId)?.catch(err => { - logger.error('trimWhenNecessary failed', err); - }); - }); await super.addUpdateToSQLite(data); } - trimWhenNecessary = debounce(async (docId?: string) => { - if (this.firstConnected) { - const count = (await this.db?.getUpdatesCount(docId)) ?? 0; - if (count > TRIM_SIZE) { - logger.debug(`trim ${this.workspaceId}:${docId} ${count}`); - const update = this.getDocAsUpdates(docId); - if (update) { - const insertRows = [{ data: update, docId }]; - await this.db?.replaceUpdates(docId, insertRows); - logger.debug(`trim ${this.workspaceId}:${docId} successfully`); - } + private readonly tryTrim = async (docId?: string) => { + const count = (await this.db?.getUpdatesCount(docId)) ?? 0; + if (count > TRIM_SIZE) { + logger.debug(`trim ${this.workspaceId}:${docId} ${count}`); + const update = await this.getDocAsUpdates(docId); + if (update) { + const insertRows = [{ data: update, docId }]; + await this.db?.replaceUpdates(docId, insertRows); + logger.debug(`trim ${this.workspaceId}:${docId} successfully`); } } - }, 1000); + }; } export async function openWorkspaceDatabase(workspaceId: string) { diff --git a/packages/frontend/electron/src/helper/dialog/dialog.ts b/packages/frontend/electron/src/helper/dialog/dialog.ts index c532556d7b..695da0dea9 100644 --- a/packages/frontend/electron/src/helper/dialog/dialog.ts +++ b/packages/frontend/electron/src/helper/dialog/dialog.ts @@ -1,5 +1,3 @@ -import path from 'node:path'; - import { ValidationResult } from '@affine/native'; import { WorkspaceVersion } from '@toeverything/infra/blocksuite'; import fs from 'fs-extra'; @@ -11,10 +9,9 @@ import { migrateToLatest, migrateToSubdocAndReplaceDatabase, } from '../db/migration'; -import type { WorkspaceSQLiteDB } from '../db/workspace-db-adapter'; import { logger } from '../logger'; import { mainRPC } from '../main-rpc'; -import { listWorkspaces, storeWorkspaceMeta } from '../workspace'; +import { storeWorkspaceMeta } from '../workspace'; import { getWorkspaceDBPath, getWorkspaceMeta, @@ -47,12 +44,6 @@ export interface SelectDBFileLocationResult { canceled?: boolean; } -export interface MoveDBFileResult { - filePath?: string; - error?: ErrorMessage; - canceled?: boolean; -} - // provide a backdoor to set dialog path for testing in playwright export interface FakeDialogResult { canceled?: boolean; @@ -68,7 +59,7 @@ export async function revealDBFile(workspaceId: string) { if (!meta) { return; } - await mainRPC.showItemInFolder(meta.secondaryDBPath ?? meta.mainDBPath); + await mainRPC.showItemInFolder(meta.mainDBPath); } // result will be used in the next call to showOpenDialog @@ -120,7 +111,10 @@ export async function saveDBFileAs( name: '', }, ], - defaultPath: getDefaultDBFileName(db.getWorkspaceName(), workspaceId), + defaultPath: getDefaultDBFileName( + await db.getWorkspaceName(), + workspaceId + ), message: 'Save Workspace as a SQLite Database file', })); const filePath = ret.filePath; @@ -213,11 +207,6 @@ export async function loadDBFile(): Promise { return { error: 'DB_FILE_PATH_INVALID' }; } - if (await dbFileAlreadyLoaded(originalPath)) { - logger.warn('loadDBFile: db file already loaded'); - return { error: 'DB_FILE_ALREADY_LOADED' }; - } - const { SqliteConnection } = await import('@affine/native'); const validationResult = await SqliteConnection.validate(originalPath); @@ -294,100 +283,3 @@ export async function loadDBFile(): Promise { }; } } - -/** - * This function is called when the user clicks the "Move" button in the "Move Workspace Storage" setting. - * - * It will - * - copy the source db file to a new location - * - remove the old db external file - * - update the external db file path in the workspace meta - * - return the new file path - */ -export async function moveDBFile( - workspaceId: string, - dbFileDir?: string -): Promise { - let db: WorkspaceSQLiteDB | null = null; - try { - db = await ensureSQLiteDB(workspaceId); - const meta = await getWorkspaceMeta(workspaceId); - - const oldDir = meta.secondaryDBPath - ? path.dirname(meta.secondaryDBPath) - : null; - const defaultDir = oldDir ?? (await mainRPC.getPath('documents')); - - const newName = getDefaultDBFileName(db.getWorkspaceName(), workspaceId); - - const newDirPath = - dbFileDir ?? - ( - getFakedResult() ?? - (await mainRPC.showOpenDialog({ - properties: ['openDirectory'], - title: 'Move Workspace Storage', - buttonLabel: 'Move', - defaultPath: defaultDir, - message: 'Move Workspace storage file', - })) - ).filePaths?.[0]; - - // skips if - // - user canceled the dialog - // - user selected the same dir - if (!newDirPath || newDirPath === oldDir) { - return { - canceled: true, - }; - } - - const newFilePath = path.join(newDirPath, newName); - - if (await fs.pathExists(newFilePath)) { - return { - error: 'FILE_ALREADY_EXISTS', - }; - } - - logger.info(`[moveDBFile] copy ${meta.mainDBPath} -> ${newFilePath}`); - - await fs.copy(meta.mainDBPath, newFilePath); - - // remove the old db file, but we don't care if it fails - if (meta.secondaryDBPath) { - await fs - .remove(meta.secondaryDBPath) - .then(() => { - logger.info(`[moveDBFile] removed ${meta.secondaryDBPath}`); - }) - .catch(err => { - logger.error( - `[moveDBFile] remove ${meta.secondaryDBPath} failed`, - err - ); - }); - } - - // update meta - await storeWorkspaceMeta(workspaceId, { - secondaryDBPath: newFilePath, - }); - - return { - filePath: newFilePath, - }; - } catch (err) { - await db?.destroy(); - logger.error('[moveDBFile]', err); - return { - error: 'UNKNOWN_ERROR', - }; - } -} - -async function dbFileAlreadyLoaded(path: string) { - const meta = await listWorkspaces(); - const paths = meta.map(m => m[1].secondaryDBPath); - return paths.includes(path); -} diff --git a/packages/frontend/electron/src/helper/dialog/index.ts b/packages/frontend/electron/src/helper/dialog/index.ts index 28079f2d15..f9bf5c21a7 100644 --- a/packages/frontend/electron/src/helper/dialog/index.ts +++ b/packages/frontend/electron/src/helper/dialog/index.ts @@ -1,6 +1,5 @@ import { loadDBFile, - moveDBFile, revealDBFile, saveDBFileAs, selectDBFileLocation, @@ -17,9 +16,6 @@ export const dialogHandlers = { saveDBFileAs: async (workspaceId: string) => { return saveDBFileAs(workspaceId); }, - moveDBFile: (workspaceId: string, dbFileLocation?: string) => { - return moveDBFile(workspaceId, dbFileLocation); - }, selectDBFileLocation: async () => { return selectDBFileLocation(); }, diff --git a/packages/frontend/electron/src/helper/index.ts b/packages/frontend/electron/src/helper/index.ts index 7fe622259c..2d8cff050c 100644 --- a/packages/frontend/electron/src/helper/index.ts +++ b/packages/frontend/electron/src/helper/index.ts @@ -12,7 +12,7 @@ function setupRendererConnection(rendererPort: Electron.MessagePortMain) { try { const start = performance.now(); const result = await handler(...args); - logger.info( + logger.debug( '[async-api]', `${namespace}.${name}`, args.filter( diff --git a/packages/frontend/electron/src/helper/type.ts b/packages/frontend/electron/src/helper/type.ts index 0fa7e8bd07..03acd84f65 100644 --- a/packages/frontend/electron/src/helper/type.ts +++ b/packages/frontend/electron/src/helper/type.ts @@ -1,7 +1,6 @@ export interface WorkspaceMeta { id: string; mainDBPath: string; - secondaryDBPath?: string; // assume there will be only one } export type YOrigin = 'self' | 'external' | 'upstream' | 'renderer'; diff --git a/packages/frontend/electron/src/helper/workspace/meta.ts b/packages/frontend/electron/src/helper/workspace/meta.ts index aa3ff6f210..72525974f3 100644 --- a/packages/frontend/electron/src/helper/workspace/meta.ts +++ b/packages/frontend/electron/src/helper/workspace/meta.ts @@ -52,26 +52,12 @@ export async function getWorkspaceMeta( .then(() => true) .catch(() => false)) ) { - // since not meta is found, we will migrate symlinked db file if needed await fs.ensureDir(basePath); const dbPath = await getWorkspaceDBPath(workspaceId); - - // todo: remove this after migration (in stable version) - const realDBPath = (await fs - .access(dbPath) - .then(() => true) - .catch(() => false)) - ? await fs.realpath(dbPath) - : dbPath; - const isLink = realDBPath !== dbPath; - if (isLink) { - await fs.copy(realDBPath, dbPath); - } // create one if not exists const meta = { id: workspaceId, mainDBPath: dbPath, - secondaryDBPath: isLink ? realDBPath : undefined, }; await fs.writeJSON(metaPath, meta); return meta; diff --git a/packages/frontend/electron/test/db/ensure-db.spec.ts b/packages/frontend/electron/test/db/ensure-db.spec.ts index b4ad5c1f60..e081a99e56 100644 --- a/packages/frontend/electron/test/db/ensure-db.spec.ts +++ b/packages/frontend/electron/test/db/ensure-db.spec.ts @@ -99,47 +99,3 @@ test('db should be removed in db$Map after destroyed', async () => { await setTimeout(100); expect(db$Map.has(workspaceId)).toBe(false); }); - -// we have removed secondary db feature -test.skip('if db has a secondary db path, we should also poll that', async () => { - const { ensureSQLiteDB } = await import( - '@affine/electron/helper/db/ensure-db' - ); - const { storeWorkspaceMeta } = await import( - '@affine/electron/helper/workspace' - ); - const workspaceId = v4(); - await storeWorkspaceMeta(workspaceId, { - secondaryDBPath: path.join(tmpDir, 'secondary.db'), - }); - - const db = await ensureSQLiteDB(workspaceId); - - await setTimeout(10); - - expect(constructorStub).toBeCalledTimes(1); - expect(constructorStub).toBeCalledWith(path.join(tmpDir, 'secondary.db'), db); - - // if secondary meta is changed - await storeWorkspaceMeta(workspaceId, { - secondaryDBPath: path.join(tmpDir, 'secondary2.db'), - }); - - // wait the async `db.destroy()` to be called - await setTimeout(100); - expect(constructorStub).toBeCalledTimes(2); - expect(destroyStub).toBeCalledTimes(1); - - // if secondary meta is changed (but another workspace) - await storeWorkspaceMeta(v4(), { - secondaryDBPath: path.join(tmpDir, 'secondary3.db'), - }); - await vi.advanceTimersByTimeAsync(1500); - expect(constructorStub).toBeCalledTimes(2); - expect(destroyStub).toBeCalledTimes(1); - - // if primary is destroyed, secondary should also be destroyed - await db.destroy(); - await setTimeout(100); - expect(destroyStub).toBeCalledTimes(2); -}); diff --git a/packages/frontend/electron/test/db/workspace-db-adapter.spec.ts b/packages/frontend/electron/test/db/workspace-db-adapter.spec.ts index c16fb46ead..349316be7c 100644 --- a/packages/frontend/electron/test/db/workspace-db-adapter.spec.ts +++ b/packages/frontend/electron/test/db/workspace-db-adapter.spec.ts @@ -1,11 +1,9 @@ import path from 'node:path'; -import { dbSubjects } from '@affine/electron/helper/db/subjects'; import { removeWithRetry } from '@affine-test/kit/utils/utils'; import fs from 'fs-extra'; import { v4 } from 'uuid'; import { afterAll, afterEach, beforeAll, expect, test, vi } from 'vitest'; -import { Doc as YDoc, encodeStateAsUpdate } from 'yjs'; const tmpDir = path.join(__dirname, 'tmp'); const appDataPath = path.join(tmpDir, 'app-data'); @@ -26,31 +24,6 @@ afterAll(() => { vi.doUnmock('@affine/electron/helper/main-rpc'); }); -let testYDoc: YDoc; -let testYSubDoc: YDoc; - -function getTestUpdates() { - testYDoc = new YDoc(); - const yText = testYDoc.getText('test'); - yText.insert(0, 'hello'); - - testYSubDoc = new YDoc(); - testYDoc.getMap('subdocs').set('test-subdoc', testYSubDoc); - - const updates = encodeStateAsUpdate(testYDoc); - - return updates; -} - -function getTestSubDocUpdates() { - const yText = testYSubDoc.getText('test'); - yText.insert(0, 'hello'); - - const updates = encodeStateAsUpdate(testYSubDoc); - - return updates; -} - test('can create new db file if not exists', async () => { const { openWorkspaceDatabase } = await import( '@affine/electron/helper/db/workspace-db-adapter' @@ -66,82 +39,6 @@ test('can create new db file if not exists', async () => { await db.destroy(); }); -test('on applyUpdate (from self), will not trigger update', async () => { - const { openWorkspaceDatabase } = await import( - '@affine/electron/helper/db/workspace-db-adapter' - ); - const workspaceId = v4(); - const onUpdate = vi.fn(); - - const db = await openWorkspaceDatabase(workspaceId); - db.update$.subscribe(onUpdate); - db.applyUpdate(getTestUpdates(), 'self'); - expect(onUpdate).not.toHaveBeenCalled(); - await db.destroy(); -}); - -test('on applyUpdate (from renderer), will trigger update', async () => { - const { openWorkspaceDatabase } = await import( - '@affine/electron/helper/db/workspace-db-adapter' - ); - const workspaceId = v4(); - const onUpdate = vi.fn(); - const onExternalUpdate = vi.fn(); - - const db = await openWorkspaceDatabase(workspaceId); - db.update$.subscribe(onUpdate); - const sub = dbSubjects.externalUpdate$.subscribe(onExternalUpdate); - db.applyUpdate(getTestUpdates(), 'renderer'); - expect(onUpdate).toHaveBeenCalled(); - sub.unsubscribe(); - await db.destroy(); -}); - -test('on applyUpdate (from renderer, subdoc), will trigger update', async () => { - const { openWorkspaceDatabase } = await import( - '@affine/electron/helper/db/workspace-db-adapter' - ); - const workspaceId = v4(); - const onUpdate = vi.fn(); - const insertUpdates = vi.fn(); - - const db = await openWorkspaceDatabase(workspaceId); - db.applyUpdate(getTestUpdates(), 'renderer'); - - db.db!.insertUpdates = insertUpdates; - db.update$.subscribe(onUpdate); - - const subdocUpdates = getTestSubDocUpdates(); - db.applyUpdate(subdocUpdates, 'renderer', testYSubDoc.guid); - - expect(onUpdate).toHaveBeenCalled(); - expect(insertUpdates).toHaveBeenCalledWith([ - { - docId: testYSubDoc.guid, - data: subdocUpdates, - }, - ]); - await db.destroy(); -}); - -test('on applyUpdate (from external), will trigger update & send external update event', async () => { - const { openWorkspaceDatabase } = await import( - '@affine/electron/helper/db/workspace-db-adapter' - ); - const workspaceId = v4(); - const onUpdate = vi.fn(); - const onExternalUpdate = vi.fn(); - - const db = await openWorkspaceDatabase(workspaceId); - db.update$.subscribe(onUpdate); - const sub = dbSubjects.externalUpdate$.subscribe(onExternalUpdate); - db.applyUpdate(getTestUpdates(), 'external'); - expect(onUpdate).toHaveBeenCalled(); - expect(onExternalUpdate).toHaveBeenCalled(); - sub.unsubscribe(); - await db.destroy(); -}); - test('on destroy, check if resources have been released', async () => { const { openWorkspaceDatabase } = await import( '@affine/electron/helper/db/workspace-db-adapter' diff --git a/packages/frontend/electron/test/workspace/handlers.spec.ts b/packages/frontend/electron/test/workspace/handlers.spec.ts index 0ef62453e6..fde95ca9ff 100644 --- a/packages/frontend/electron/test/workspace/handlers.spec.ts +++ b/packages/frontend/electron/test/workspace/handlers.spec.ts @@ -127,7 +127,6 @@ describe('getWorkspaceMeta', () => { expect(await getWorkspaceMeta(workspaceId)).toEqual({ id: workspaceId, mainDBPath: path.join(workspacePath, 'storage.db'), - secondaryDBPath: sourcePath, }); expect( @@ -151,11 +150,4 @@ test('storeWorkspaceMeta', async () => { expect(await fs.readJSON(path.join(workspacePath, 'meta.json'))).toEqual( meta ); - await storeWorkspaceMeta(workspaceId, { - secondaryDBPath: path.join(tmpDir, 'test.db'), - }); - expect(await fs.readJSON(path.join(workspacePath, 'meta.json'))).toEqual({ - ...meta, - secondaryDBPath: path.join(tmpDir, 'test.db'), - }); }); diff --git a/tools/cli/src/webpack/runtime-config.ts b/tools/cli/src/webpack/runtime-config.ts index bd12963cc2..69c36141e3 100644 --- a/tools/cli/src/webpack/runtime-config.ts +++ b/tools/cli/src/webpack/runtime-config.ts @@ -17,7 +17,6 @@ export function getRuntimeConfig(buildFlags: BuildFlags): RuntimeConfig { enablePreloading: true, enableNewSettingModal: true, enableNewSettingUnstableApi: false, - enableMoveDatabase: false, enableCloud: true, enableCaptcha: true, enableEnhanceShareMode: false, @@ -57,7 +56,6 @@ export function getRuntimeConfig(buildFlags: BuildFlags): RuntimeConfig { enablePreloading: true, enableNewSettingModal: true, enableNewSettingUnstableApi: false, - enableMoveDatabase: false, enableCloud: true, enableCaptcha: true, enableEnhanceShareMode: false, @@ -107,9 +105,6 @@ export function getRuntimeConfig(buildFlags: BuildFlags): RuntimeConfig { enableEnhanceShareMode: process.env.ENABLE_ENHANCE_SHARE_MODE ? process.env.ENABLE_ENHANCE_SHARE_MODE === 'true' : currentBuildPreset.enableEnhanceShareMode, - enableMoveDatabase: process.env.ENABLE_MOVE_DATABASE - ? process.env.ENABLE_MOVE_DATABASE === 'true' - : currentBuildPreset.enableMoveDatabase, enablePayment: process.env.ENABLE_PAYMENT ? process.env.ENABLE_PAYMENT !== 'false' : buildFlags.mode === 'development'