feat: create workspace from loading existing exported file (#2122)

Co-authored-by: Himself65 <himself65@outlook.com>
This commit is contained in:
Peng Xiao
2023-05-09 15:30:01 +08:00
committed by GitHub
parent 5432aae85c
commit 7c2574b1ca
93 changed files with 2999 additions and 1406 deletions
@@ -0,0 +1 @@
tmp
@@ -0,0 +1,472 @@
import assert from 'node:assert';
import path from 'node:path';
import fs from 'fs-extra';
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import * as Y from 'yjs';
import type { MainIPCHandlerMap } from '../../../../constraints';
const registeredHandlers = new Map<
string,
((...args: any[]) => Promise<any>)[]
>();
const delay = (ms: number) => new Promise(r => setTimeout(r, ms));
type WithoutFirstParameter<T> = T extends (_: any, ...args: infer P) => infer R
? (...args: P) => R
: T;
// common mock dispatcher for ipcMain.handle AND app.on
// alternatively, we can use single parameter for T & F, eg, dispatch('workspace:list'),
// however this is too hard to be typed correctly
async function dispatch<
T extends keyof MainIPCHandlerMap,
F extends keyof MainIPCHandlerMap[T]
>(
namespace: T,
functionName: F,
// @ts-ignore
...args: Parameters<WithoutFirstParameter<MainIPCHandlerMap[T][F]>>
): // @ts-ignore
ReturnType<MainIPCHandlerMap[T][F]> {
// @ts-ignore
const handlers = registeredHandlers.get(namespace + ':' + functionName);
assert(handlers);
// we only care about the first handler here
return await handlers[0](null, ...args);
}
const SESSION_DATA_PATH = path.join(__dirname, './tmp', 'affine-test');
const browserWindow = {
isDestroyed: () => {
return false;
},
setWindowButtonVisibility: (_v: boolean) => {
// will be stubbed later
},
webContents: {
send: (_type: string, ..._args: any[]) => {
// will be stubbed later
},
},
};
const ipcMain = {
handle: (key: string, callback: (...args: any[]) => Promise<any>) => {
const handlers = registeredHandlers.get(key) || [];
handlers.push(callback);
registeredHandlers.set(key, handlers);
},
};
const nativeTheme = {
themeSource: 'light',
};
function compareBuffer(a: Uint8Array | null, b: Uint8Array | null) {
if (
(a === null && b === null) ||
a === null ||
b === null ||
a.length !== b.length
) {
return false;
}
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) {
return false;
}
}
return true;
}
const electronModule = {
app: {
getPath: (name: string) => {
assert(name === 'sessionData');
return SESSION_DATA_PATH;
},
name: 'affine-test',
on: (name: string, callback: (...args: any[]) => any) => {
const handlers = registeredHandlers.get(name) || [];
handlers.push(callback);
registeredHandlers.set(name, handlers);
},
},
BrowserWindow: {
getAllWindows: () => {
return [browserWindow];
},
},
nativeTheme: nativeTheme,
ipcMain,
shell: {} as Partial<Electron.Shell>,
dialog: {} as Partial<Electron.Dialog>,
};
// dynamically import handlers so that we can inject local variables to mocks
vi.doMock('electron', () => {
return electronModule;
});
beforeEach(async () => {
const { registerHandlers } = await import('../register');
registerHandlers();
// should also register events
const { registerEvents } = await import('../../events');
registerEvents();
});
afterEach(async () => {
const { cleanupSQLiteDBs } = await import('../db/ensure-db');
await cleanupSQLiteDBs();
await fs.remove(SESSION_DATA_PATH);
// reset registered handlers
registeredHandlers.get('before-quit')?.forEach(fn => fn());
});
describe('ensureSQLiteDB', () => {
test('should create db file on connection if it does not exist', async () => {
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
const workspaceDB = await ensureSQLiteDB(id);
const file = workspaceDB.path;
const fileExists = await fs.pathExists(file);
expect(fileExists).toBe(true);
});
test('when db file is removed', async () => {
// stub webContents.send
const sendStub = vi.fn();
browserWindow.webContents.send = sendStub;
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
let workspaceDB = await ensureSQLiteDB(id);
const file = workspaceDB.path;
const fileExists = await fs.pathExists(file);
expect(fileExists).toBe(true);
await fs.remove(file);
// wait for 1000ms for file watcher to detect file removal
await delay(2000);
expect(sendStub).toBeCalledWith('db:onDbFileMissing', id);
// ensureSQLiteDB should recreate the db file
workspaceDB = await ensureSQLiteDB(id);
const fileExists2 = await fs.pathExists(file);
expect(fileExists2).toBe(true);
});
test('when db file is updated', async () => {
// stub webContents.send
const sendStub = vi.fn();
browserWindow.webContents.send = sendStub;
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
const workspaceDB = await ensureSQLiteDB(id);
const file = workspaceDB.path;
const fileExists = await fs.pathExists(file);
expect(fileExists).toBe(true);
// wait to make sure
await delay(500);
// writes some data to the db file
await fs.appendFile(file, 'random-data', { encoding: 'binary' });
// write again
await fs.appendFile(file, 'random-data', { encoding: 'binary' });
// wait for 200ms for file watcher to detect file change
await delay(2000);
expect(sendStub).toBeCalledWith('db:onDbFileUpdate', id);
// should only call once for multiple writes
expect(sendStub).toBeCalledTimes(1);
});
});
describe('workspace handlers', () => {
test('list all workspace ids', async () => {
const ids = ['test-workspace-id', 'test-workspace-id-2'];
const { ensureSQLiteDB } = await import('../db/ensure-db');
await Promise.all(ids.map(id => ensureSQLiteDB(id)));
const list = await dispatch('workspace', 'list');
expect(list.map(([id]) => id)).toEqual(ids);
});
test('delete workspace', async () => {
const ids = ['test-workspace-id', 'test-workspace-id-2'];
const { ensureSQLiteDB } = await import('../db/ensure-db');
await Promise.all(ids.map(id => ensureSQLiteDB(id)));
await dispatch('workspace', 'delete', 'test-workspace-id-2');
const list = await dispatch('workspace', 'list');
expect(list.map(([id]) => id)).toEqual(['test-workspace-id']);
});
});
describe('UI handlers', () => {
test('theme-change', async () => {
await dispatch('ui', 'handleThemeChange', 'dark');
expect(nativeTheme.themeSource).toBe('dark');
await dispatch('ui', 'handleThemeChange', 'light');
expect(nativeTheme.themeSource).toBe('light');
});
test('sidebar-visibility-change (macOS)', async () => {
vi.stubGlobal('process', { platform: 'darwin' });
const setWindowButtonVisibility = vi.fn();
browserWindow.setWindowButtonVisibility = setWindowButtonVisibility;
await dispatch('ui', 'handleSidebarVisibilityChange', true);
expect(setWindowButtonVisibility).toBeCalledWith(true);
await dispatch('ui', 'handleSidebarVisibilityChange', false);
expect(setWindowButtonVisibility).toBeCalledWith(false);
vi.unstubAllGlobals();
});
test('sidebar-visibility-change (non-macOS)', async () => {
vi.stubGlobal('process', { platform: 'linux' });
const setWindowButtonVisibility = vi.fn();
browserWindow.setWindowButtonVisibility = setWindowButtonVisibility;
await dispatch('ui', 'handleSidebarVisibilityChange', true);
expect(setWindowButtonVisibility).not.toBeCalled();
vi.unstubAllGlobals();
});
});
describe('db handlers', () => {
test('apply doc and get doc updates', async () => {
const workspaceId = 'test-workspace-id';
const bin = await dispatch('db', 'getDocAsUpdates', workspaceId);
// ? is this a good test?
expect(bin.every((byte: number) => byte === 0)).toBe(true);
const ydoc = new Y.Doc();
const ytext = ydoc.getText('test');
ytext.insert(0, 'hello world');
const bin2 = Y.encodeStateAsUpdate(ydoc);
await dispatch('db', 'applyDocUpdate', workspaceId, bin2);
const bin3 = await dispatch('db', 'getDocAsUpdates', workspaceId);
const ydoc2 = new Y.Doc();
Y.applyUpdate(ydoc2, bin3);
const ytext2 = ydoc2.getText('test');
expect(ytext2.toString()).toBe('hello world');
});
test('get non existent blob', async () => {
const workspaceId = 'test-workspace-id';
const bin = await dispatch('db', 'getBlob', workspaceId, 'non-existent-id');
expect(bin).toBeNull();
});
test('list blobs (empty)', async () => {
const workspaceId = 'test-workspace-id';
const list = await dispatch('db', 'getPersistedBlobs', workspaceId);
expect(list).toEqual([]);
});
test('CRUD blobs', async () => {
const testBin = new Uint8Array([1, 2, 3, 4, 5]);
const testBin2 = new Uint8Array([6, 7, 8, 9, 10]);
const workspaceId = 'test-workspace-id';
// add blob
await dispatch('db', 'addBlob', workspaceId, 'testBin', testBin);
// get blob
expect(
compareBuffer(
await dispatch('db', 'getBlob', workspaceId, 'testBin'),
testBin
)
).toBe(true);
// add another blob
await dispatch('db', 'addBlob', workspaceId, 'testBin2', testBin2);
expect(
compareBuffer(
await dispatch('db', 'getBlob', workspaceId, 'testBin2'),
testBin2
)
).toBe(true);
// list blobs
let lists = await dispatch('db', 'getPersistedBlobs', workspaceId);
expect(lists).toHaveLength(2);
expect(lists).toContain('testBin');
expect(lists).toContain('testBin2');
// delete blob
await dispatch('db', 'deleteBlob', workspaceId, 'testBin');
lists = await dispatch('db', 'getPersistedBlobs', workspaceId);
expect(lists).toEqual(['testBin2']);
});
});
describe('dialog handlers', () => {
test('revealDBFile', async () => {
const mockShowItemInFolder = vi.fn();
electronModule.shell.showItemInFolder = mockShowItemInFolder;
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
const db = await ensureSQLiteDB(id);
await dispatch('dialog', 'revealDBFile', id);
expect(mockShowItemInFolder).toBeCalledWith(db.path);
});
test('saveDBFileAs (skipped)', async () => {
const mockShowSaveDialog = vi.fn(() => {
return { filePath: undefined };
}) as any;
const mockShowItemInFolder = vi.fn();
electronModule.dialog.showSaveDialog = mockShowSaveDialog;
electronModule.shell.showItemInFolder = mockShowItemInFolder;
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
await ensureSQLiteDB(id);
await dispatch('dialog', 'saveDBFileAs', id);
expect(mockShowSaveDialog).toBeCalled();
expect(mockShowItemInFolder).not.toBeCalled();
});
test('saveDBFileAs', async () => {
const newSavedPath = path.join(SESSION_DATA_PATH, 'saved-to');
const mockShowSaveDialog = vi.fn(() => {
return { filePath: newSavedPath };
}) as any;
const mockShowItemInFolder = vi.fn();
electronModule.dialog.showSaveDialog = mockShowSaveDialog;
electronModule.shell.showItemInFolder = mockShowItemInFolder;
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
await ensureSQLiteDB(id);
await dispatch('dialog', 'saveDBFileAs', id);
expect(mockShowSaveDialog).toBeCalled();
expect(mockShowItemInFolder).toBeCalledWith(newSavedPath);
// check if file is saved to new path
expect(await fs.exists(newSavedPath)).toBe(true);
});
test('loadDBFile (skipped)', async () => {
const mockShowOpenDialog = vi.fn(() => {
return { filePaths: undefined };
}) as any;
electronModule.dialog.showOpenDialog = mockShowOpenDialog;
const res = await dispatch('dialog', 'loadDBFile');
expect(mockShowOpenDialog).toBeCalled();
expect(res.canceled).toBe(true);
});
test('loadDBFile (error, in app-data)', async () => {
const mockShowOpenDialog = vi.fn(() => {
return {
filePaths: [path.join(SESSION_DATA_PATH, 'workspaces')],
};
}) as any;
electronModule.dialog.showOpenDialog = mockShowOpenDialog;
const res = await dispatch('dialog', 'loadDBFile');
expect(mockShowOpenDialog).toBeCalled();
expect(res.error).toBe('DB_FILE_PATH_INVALID');
});
test('loadDBFile (error, not a valid db file)', async () => {
// create a random db file
const basePath = path.join(SESSION_DATA_PATH, 'random-path');
const dbPath = path.join(basePath, 'xxx.db');
await fs.ensureDir(basePath);
await fs.writeFile(dbPath, 'hello world');
const mockShowOpenDialog = vi.fn(() => {
return { filePaths: [dbPath] };
}) as any;
electronModule.dialog.showOpenDialog = mockShowOpenDialog;
const res = await dispatch('dialog', 'loadDBFile');
expect(mockShowOpenDialog).toBeCalled();
expect(res.error).toBe('DB_FILE_INVALID');
});
test('loadDBFile', async () => {
// we use ensureSQLiteDB to create a valid db file
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
const db = await ensureSQLiteDB(id);
// copy db file to dbPath
const basePath = path.join(SESSION_DATA_PATH, 'random-path');
const originDBFilePath = path.join(basePath, 'xxx.db');
await fs.ensureDir(basePath);
await fs.copyFile(db.path, originDBFilePath);
// remove db
await fs.remove(db.path);
// try load originDBFilePath
const mockShowOpenDialog = vi.fn(() => {
return { filePaths: [originDBFilePath] };
}) as any;
electronModule.dialog.showOpenDialog = mockShowOpenDialog;
const res = await dispatch('dialog', 'loadDBFile');
expect(mockShowOpenDialog).toBeCalled();
expect(res.workspaceId).not.toBeUndefined();
const importedDb = await ensureSQLiteDB(res.workspaceId!);
expect(await fs.realpath(importedDb.path)).toBe(originDBFilePath);
expect(importedDb.path).not.toBe(originDBFilePath);
// try load it again, will trigger error (db file already loaded)
const res2 = await dispatch('dialog', 'loadDBFile');
expect(res2.error).toBe('DB_FILE_ALREADY_LOADED');
});
test('moveDBFile', async () => {
const newPath = path.join(SESSION_DATA_PATH, 'affine-test', 'xxx');
const mockShowSaveDialog = vi.fn(() => {
return { filePath: newPath };
}) as any;
electronModule.dialog.showSaveDialog = mockShowSaveDialog;
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
await ensureSQLiteDB(id);
const res = await dispatch('dialog', 'moveDBFile', id);
expect(mockShowSaveDialog).toBeCalled();
expect(res.filePath).toBe(newPath);
});
test('moveDBFile (skipped)', async () => {
const mockShowSaveDialog = vi.fn(() => {
return { filePath: null };
}) as any;
electronModule.dialog.showSaveDialog = mockShowSaveDialog;
const id = 'test-workspace-id';
const { ensureSQLiteDB } = await import('../db/ensure-db');
await ensureSQLiteDB(id);
const res = await dispatch('dialog', 'moveDBFile', id);
expect(mockShowSaveDialog).toBeCalled();
expect(res.filePath).toBe(undefined);
});
});
@@ -0,0 +1,89 @@
import { watch } from 'chokidar';
import { appContext } from '../../context';
import { subjects } from '../../events';
import { logger } from '../../logger';
import { debounce, ts } from '../../utils';
import type { WorkspaceSQLiteDB } from './sqlite';
import { openWorkspaceDatabase } from './sqlite';
const dbMapping = new Map<string, Promise<WorkspaceSQLiteDB>>();
const dbWatchers = new Map<string, () => void>();
// if we removed the file, we will stop watching it
function startWatchingDBFile(db: WorkspaceSQLiteDB) {
if (dbWatchers.has(db.workspaceId)) {
return dbWatchers.get(db.workspaceId);
}
logger.info('watch db file', db.path);
const watcher = watch(db.path);
const debounceOnChange = debounce(() => {
logger.info(
'db file changed on disk',
db.workspaceId,
ts() - db.lastUpdateTime,
'ms'
);
// reconnect db
db.reconnectDB();
subjects.db.dbFileUpdate.next(db.workspaceId);
}, 1000);
watcher.on('change', () => {
const currentTime = ts();
if (currentTime - db.lastUpdateTime > 100) {
debounceOnChange();
}
});
dbWatchers.set(db.workspaceId, () => {
watcher.close();
});
// todo: there is still a possibility that the file is deleted
// but we didn't get the event soon enough and another event tries to
// access the db
watcher.on('unlink', () => {
logger.info('db file missing', db.workspaceId);
subjects.db.dbFileMissing.next(db.workspaceId);
// cleanup
watcher.close().then(() => {
db.destroy();
dbWatchers.delete(db.workspaceId);
dbMapping.delete(db.workspaceId);
});
});
}
export async function ensureSQLiteDB(id: string) {
let workspaceDB = dbMapping.get(id);
if (!workspaceDB) {
logger.info('[ensureSQLiteDB] open db connection', id);
workspaceDB = openWorkspaceDatabase(appContext, id);
dbMapping.set(id, workspaceDB);
startWatchingDBFile(await workspaceDB);
}
return await workspaceDB;
}
export async function disconnectSQLiteDB(id: string) {
const dbp = dbMapping.get(id);
if (dbp) {
const db = await dbp;
logger.info('close db connection', id);
db.destroy();
dbWatchers.get(id)?.();
dbWatchers.delete(id);
dbMapping.delete(id);
}
}
export async function cleanupSQLiteDBs() {
for (const [id] of dbMapping) {
logger.info('close db connection', id);
await disconnectSQLiteDB(id);
}
dbMapping.clear();
dbWatchers.clear();
}
@@ -0,0 +1,33 @@
import { appContext } from '../../context';
import type { NamespaceHandlers } from '../type';
import { ensureSQLiteDB } from './ensure-db';
export const dbHandlers = {
getDocAsUpdates: async (_, id: string) => {
const workspaceDB = await ensureSQLiteDB(id);
return workspaceDB.getDocAsUpdates();
},
applyDocUpdate: async (_, id: string, update: Uint8Array) => {
const workspaceDB = await ensureSQLiteDB(id);
return workspaceDB.applyUpdate(update);
},
addBlob: async (_, workspaceId: string, key: string, data: Uint8Array) => {
const workspaceDB = await ensureSQLiteDB(workspaceId);
return workspaceDB.addBlob(key, data);
},
getBlob: async (_, workspaceId: string, key: string) => {
const workspaceDB = await ensureSQLiteDB(workspaceId);
return workspaceDB.getBlob(key);
},
deleteBlob: async (_, workspaceId: string, key: string) => {
const workspaceDB = await ensureSQLiteDB(workspaceId);
return workspaceDB.deleteBlob(key);
},
getPersistedBlobs: async (_, workspaceId: string) => {
const workspaceDB = await ensureSQLiteDB(workspaceId);
return workspaceDB.getPersistentBlobKeys();
},
getDefaultStorageLocation: async () => {
return appContext.appDataPath;
},
} satisfies NamespaceHandlers;
@@ -0,0 +1,231 @@
import path from 'node:path';
import type { Database } from 'better-sqlite3';
import sqlite from 'better-sqlite3';
import fs from 'fs-extra';
import * as Y from 'yjs';
import type { AppContext } from '../../context';
import { logger } from '../../logger';
import { ts } from '../../utils';
const schemas = [
`CREATE TABLE IF NOT EXISTS "updates" (
id INTEGER PRIMARY KEY AUTOINCREMENT,
data BLOB NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
)`,
`CREATE TABLE IF NOT EXISTS "blobs" (
key TEXT PRIMARY KEY NOT NULL,
data BLOB NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
)`,
];
interface UpdateRow {
id: number;
data: Buffer;
timestamp: string;
}
interface BlobRow {
key: string;
data: Buffer;
timestamp: string;
}
const SQLITE_ORIGIN = Symbol('sqlite-origin');
export class WorkspaceSQLiteDB {
db: Database;
ydoc = new Y.Doc();
firstConnect = false;
lastUpdateTime = ts();
constructor(public path: string, public workspaceId: string) {
this.db = this.reconnectDB();
}
// release resources
destroy = () => {
this.db?.close();
this.ydoc.destroy();
};
getWorkspaceName = () => {
return this.ydoc.getMap('space:meta').get('name') as string;
};
reconnectDB = () => {
logger.log('open db', this.workspaceId);
if (this.db) {
this.db.close();
}
// use cached version?
const db = (this.db = sqlite(this.path));
db.exec(schemas.join(';'));
if (!this.firstConnect) {
this.ydoc.on('update', (update: Uint8Array, origin) => {
if (origin !== SQLITE_ORIGIN) {
this.addUpdateToSQLite(update);
}
});
}
Y.transact(this.ydoc, () => {
const updates = this.getUpdates();
updates.forEach(update => {
// give SQLITE_ORIGIN to skip self update
Y.applyUpdate(this.ydoc, update.data, SQLITE_ORIGIN);
});
});
this.lastUpdateTime = ts();
if (this.firstConnect) {
logger.info('db reconnected', this.workspaceId);
} else {
logger.info('db connected', this.workspaceId);
}
this.firstConnect = true;
return db;
};
getDocAsUpdates = () => {
return Y.encodeStateAsUpdate(this.ydoc);
};
// non-blocking and use yDoc to validate the update
// after that, the update is added to the db
applyUpdate = (data: Uint8Array) => {
Y.applyUpdate(this.ydoc, data);
// todo: trim the updates when the number of records is too large
// 1. store the current ydoc state in the db
// 2. then delete the old updates
// yjs-idb will always trim the db for the first time after DB is loaded
this.lastUpdateTime = ts();
logger.debug('applyUpdate', this.workspaceId, this.lastUpdateTime);
};
addBlob = (key: string, data: Uint8Array) => {
this.lastUpdateTime = ts();
try {
const statement = this.db.prepare(
'INSERT INTO blobs (key, data) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET data = ?'
);
statement.run(key, data, data);
return key;
} catch (error) {
logger.error('addBlob', error);
}
};
getBlob = (key: string) => {
try {
const statement = this.db.prepare('SELECT data FROM blobs WHERE key = ?');
const row = statement.get(key) as BlobRow;
if (!row) {
return null;
}
return row.data;
} catch (error) {
logger.error('getBlob', error);
return null;
}
};
deleteBlob = (key: string) => {
this.lastUpdateTime = ts();
try {
const statement = this.db.prepare('DELETE FROM blobs WHERE key = ?');
statement.run(key);
} catch (error) {
logger.error('deleteBlob', error);
}
};
getPersistentBlobKeys = () => {
try {
const statement = this.db.prepare('SELECT key FROM blobs');
const rows = statement.all() as BlobRow[];
return rows.map(row => row.key);
} catch (error) {
logger.error('getPersistentBlobKeys', error);
return [];
}
};
private getUpdates = () => {
try {
const statement = this.db.prepare('SELECT * FROM updates');
const rows = statement.all() as UpdateRow[];
return rows;
} catch (error) {
logger.error('getUpdates', error);
return [];
}
};
// batch write instead write per key stroke?
private addUpdateToSQLite = (data: Uint8Array) => {
try {
const start = performance.now();
const statement = this.db.prepare(
'INSERT INTO updates (data) VALUES (?)'
);
statement.run(data);
logger.debug(
'addUpdateToSQLite',
this.workspaceId,
'length:',
data.length,
performance.now() - start,
'ms'
);
} catch (error) {
logger.error('addUpdateToSQLite', error);
}
};
}
export async function getWorkspaceDBPath(
context: AppContext,
workspaceId: string
) {
const basePath = path.join(context.appDataPath, 'workspaces', workspaceId);
await fs.ensureDir(basePath);
return path.join(basePath, 'storage.db');
}
export async function openWorkspaceDatabase(
context: AppContext,
workspaceId: string
) {
const dbPath = await getWorkspaceDBPath(context, workspaceId);
return new WorkspaceSQLiteDB(dbPath, workspaceId);
}
export function isValidDBFile(path: string) {
try {
const db = sqlite(path);
// check if db has two tables, one for updates and onefor blobs
const statement = db.prepare(
`SELECT name FROM sqlite_schema WHERE type='table'`
);
const rows = statement.all() as { name: string }[];
const tableNames = rows.map(row => row.name);
if (!tableNames.includes('updates') || !tableNames.includes('blobs')) {
return false;
}
db.close();
return true;
} catch (error) {
logger.error('isValidDBFile', error);
return false;
}
}
@@ -0,0 +1,293 @@
import path from 'node:path';
import { dialog, shell } from 'electron';
import fs from 'fs-extra';
import { nanoid } from 'nanoid';
import { appContext } from '../../context';
import { logger } from '../../logger';
import { ensureSQLiteDB } from '../db/ensure-db';
import { getWorkspaceDBPath, isValidDBFile } from '../db/sqlite';
import { listWorkspaces } from '../workspace/workspace';
// NOTE:
// we are using native dialogs because HTML dialogs do not give full file paths
export async function revealDBFile(workspaceId: string) {
const workspaceDB = await ensureSQLiteDB(workspaceId);
shell.showItemInFolder(workspaceDB.path);
}
// provide a backdoor to set dialog path for testing in playwright
interface FakeDialogResult {
canceled?: boolean;
filePath?: string;
filePaths?: string[];
}
// result will be used in the next call to showOpenDialog
// if it is being read once, it will be reset to undefined
let fakeDialogResult: FakeDialogResult | undefined = undefined;
function getFakedResult() {
const result = fakeDialogResult;
fakeDialogResult = undefined;
return result;
}
export function setFakeDialogResult(result: FakeDialogResult | undefined) {
fakeDialogResult = result;
// for convenience, we will fill filePaths with filePath if it is not set
if (result?.filePaths === undefined && result?.filePath !== undefined) {
result.filePaths = [result.filePath];
}
}
const ErrorMessages = [
'DB_FILE_ALREADY_LOADED',
'DB_FILE_PATH_INVALID',
'DB_FILE_INVALID',
'UNKNOWN_ERROR',
] as const;
type ErrorMessage = (typeof ErrorMessages)[number];
interface SaveDBFileResult {
filePath?: string;
canceled?: boolean;
error?: ErrorMessage;
}
/**
* This function is called when the user clicks the "Save" button in the "Save Workspace" dialog.
*
* It will just copy the file to the given path
*/
export async function saveDBFileAs(
workspaceId: string
): Promise<SaveDBFileResult> {
try {
const db = await ensureSQLiteDB(workspaceId);
const ret =
getFakedResult() ??
(await dialog.showSaveDialog({
properties: ['showOverwriteConfirmation'],
title: 'Save Workspace',
showsTagField: false,
buttonLabel: 'Save',
defaultPath: `${db.getWorkspaceName()}_${workspaceId}.db`,
message: 'Save Workspace as a SQLite Database file',
}));
const filePath = ret.filePath;
if (ret.canceled || !filePath) {
return {
canceled: true,
};
}
await fs.copyFile(db.path, filePath);
logger.log('saved', filePath);
shell.showItemInFolder(filePath);
return { filePath };
} catch (err) {
logger.error('saveDBFileAs', err);
return {
error: 'UNKNOWN_ERROR',
};
}
}
interface SelectDBFileLocationResult {
filePath?: string;
error?: ErrorMessage;
canceled?: boolean;
}
export async function selectDBFileLocation(): Promise<SelectDBFileLocationResult> {
try {
const ret =
getFakedResult() ??
(await dialog.showSaveDialog({
properties: ['showOverwriteConfirmation'],
title: 'Set database location',
showsTagField: false,
buttonLabel: 'Select',
defaultPath: `workspace-storage.db`,
message: "Select a location to store the workspace's database file",
}));
const filePath = ret.filePath;
if (ret.canceled || !filePath) {
return {
canceled: true,
};
}
// the same db file cannot be loaded twice
if (await dbFileAlreadyLoaded(filePath)) {
return {
error: 'DB_FILE_ALREADY_LOADED',
};
}
return { filePath };
} catch (err) {
logger.error('selectDBFileLocation', err);
return {
error: (err as any).message,
};
}
}
interface LoadDBFileResult {
workspaceId?: string;
error?: ErrorMessage;
canceled?: boolean;
}
/**
* This function is called when the user clicks the "Load" button in the "Load Workspace" dialog.
*
* It will
* - symlink the source db file to a new workspace id to app-data
* - return the new workspace id
*
* eg, it will create a new folder in app-data:
* <app-data>/<app-name>/workspaces/<workspace-id>/storage.db
*
* On the renderer side, after the UI got a new workspace id, it will
* update the local workspace id list and then connect to it.
*
*/
export async function loadDBFile(): Promise<LoadDBFileResult> {
try {
const ret =
getFakedResult() ??
(await dialog.showOpenDialog({
properties: ['openFile'],
title: 'Load Workspace',
buttonLabel: 'Load',
filters: [
{
name: 'SQLite Database',
// do we want to support other file format?
extensions: ['db'],
},
],
message: 'Load Workspace from a SQLite Database file',
}));
const filePath = ret.filePaths?.[0];
if (ret.canceled || !filePath) {
logger.info('loadDBFile canceled');
return { canceled: true };
}
// the imported file should not be in app data dir
if (filePath.startsWith(path.join(appContext.appDataPath, 'workspaces'))) {
logger.warn('loadDBFile: db file in app data dir');
return { error: 'DB_FILE_PATH_INVALID' };
}
if (await dbFileAlreadyLoaded(filePath)) {
logger.warn('loadDBFile: db file already loaded');
return { error: 'DB_FILE_ALREADY_LOADED' };
}
if (!isValidDBFile(filePath)) {
// TODO: report invalid db file error?
return { error: 'DB_FILE_INVALID' }; // invalid db file
}
// symlink the db file to a new workspace id
const workspaceId = nanoid(10);
const linkedFilePath = await getWorkspaceDBPath(appContext, workspaceId);
await fs.ensureDir(path.join(appContext.appDataPath, 'workspaces'));
await fs.symlink(filePath, linkedFilePath);
logger.info(`loadDBFile, symlink: ${filePath} -> ${linkedFilePath}`);
return { workspaceId };
} catch (err) {
logger.error('loadDBFile', err);
return {
error: 'UNKNOWN_ERROR',
};
}
}
interface MoveDBFileResult {
filePath?: string;
error?: ErrorMessage;
canceled?: boolean;
}
/**
* This function is called when the user clicks the "Move" button in the "Move Workspace Storage" setting.
*
* It will
* - move the source db file to a new location
* - symlink the new location to the old db file
* - return the new file path
*/
export async function moveDBFile(
workspaceId: string,
dbFileLocation?: string
): Promise<MoveDBFileResult> {
try {
const db = await ensureSQLiteDB(workspaceId);
// get the real file path of db
const realpath = await fs.realpath(db.path);
const isLink = realpath !== db.path;
const newFilePath =
dbFileLocation ||
(
getFakedResult() ||
(await dialog.showSaveDialog({
properties: ['showOverwriteConfirmation'],
title: 'Move Workspace Storage',
showsTagField: false,
buttonLabel: 'Save',
defaultPath: realpath,
message: 'Move Workspace storage file',
}))
).filePath;
// skips if
// - user canceled the dialog
// - user selected the same file
// - user selected the same file in the link file in app data dir
if (!newFilePath || newFilePath === realpath || db.path === newFilePath) {
return {
canceled: true,
};
}
if (isLink) {
// remove the old link to unblock new link
await fs.unlink(db.path);
}
await fs.move(realpath, newFilePath, {
overwrite: true,
});
await fs.ensureSymlink(newFilePath, db.path);
logger.info(`openMoveDBFileDialog symlink: ${realpath} -> ${newFilePath}`);
db.reconnectDB();
return {
filePath: newFilePath,
};
} catch (err) {
logger.error('moveDBFile', err);
return {
error: 'UNKNOWN_ERROR',
};
}
}
async function dbFileAlreadyLoaded(path: string) {
const meta = await listWorkspaces(appContext);
const realpath = await fs.realpath(path);
const paths = meta.map(m => m[1].realpath);
return paths.includes(realpath);
}
@@ -0,0 +1,33 @@
import type { NamespaceHandlers } from '../type';
import {
loadDBFile,
moveDBFile,
revealDBFile,
saveDBFileAs,
selectDBFileLocation,
setFakeDialogResult,
} from './dialog';
export const dialogHandlers = {
revealDBFile: async (_, workspaceId: string) => {
return revealDBFile(workspaceId);
},
loadDBFile: async () => {
return loadDBFile();
},
saveDBFileAs: async (_, workspaceId: string) => {
return saveDBFileAs(workspaceId);
},
moveDBFile: async (_, workspaceId: string, dbFileLocation?: string) => {
return moveDBFile(workspaceId, dbFileLocation);
},
selectDBFileLocation: async () => {
return selectDBFileLocation();
},
setFakeDialogResult: async (
_,
result: Parameters<typeof setFakeDialogResult>[0]
) => {
return setFakeDialogResult(result);
},
} satisfies NamespaceHandlers;
@@ -0,0 +1 @@
export * from './register';
@@ -0,0 +1,63 @@
import { ipcMain } from 'electron';
import { getLogFilePath, logger, revealLogFile } from '../logger';
import { dbHandlers } from './db';
import { dialogHandlers } from './dialog';
import { uiHandlers } from './ui';
import { updaterHandlers } from './updater';
import { workspaceHandlers } from './workspace';
type IsomorphicHandler = (
e: Electron.IpcMainInvokeEvent,
...args: any[]
) => Promise<any>;
type NamespaceHandlers = {
[key: string]: IsomorphicHandler;
};
export const debugHandlers = {
revealLogFile: async () => {
return revealLogFile();
},
logFilePath: async () => {
return getLogFilePath();
},
};
// Note: all of these handlers will be the single-source-of-truth for the apis exposed to the renderer process
export const allHandlers = {
workspace: workspaceHandlers,
ui: uiHandlers,
db: dbHandlers,
dialog: dialogHandlers,
debug: debugHandlers,
updater: updaterHandlers,
} satisfies Record<string, NamespaceHandlers>;
export const registerHandlers = () => {
for (const [namespace, namespaceHandlers] of Object.entries(allHandlers)) {
for (const [key, handler] of Object.entries(namespaceHandlers)) {
const chan = `${namespace}:${key}`;
ipcMain.handle(chan, async (e, ...args) => {
const start = performance.now();
try {
const result = await handler(e, ...args);
logger.info(
'[ipc-api]',
chan,
args.filter(
arg => typeof arg !== 'function' && typeof arg !== 'object'
),
'-',
(performance.now() - start).toFixed(2),
'ms'
);
return result;
} catch (error) {
logger.error('[ipc]', chan, error);
}
});
}
}
};
@@ -0,0 +1,8 @@
export type IsomorphicHandler = (
e: Electron.IpcMainInvokeEvent,
...args: any[]
) => Promise<any>;
export type NamespaceHandlers = {
[key: string]: IsomorphicHandler;
};
@@ -0,0 +1,58 @@
import { app, BrowserWindow, shell } from 'electron';
import { parse } from 'url';
import { logger } from '../../logger';
const redirectUri = 'https://affine.pro/client/auth-callback';
export const oauthEndpoint = `https://accounts.google.com/o/oauth2/v2/auth?client_id=${process.env.AFFINE_GOOGLE_CLIENT_ID}&redirect_uri=${redirectUri}&response_type=code&scope=openid https://www.googleapis.com/auth/userinfo.email profile&access_type=offline&customParameters={"prompt":"select_account"}`;
const tokenEndpoint = 'https://oauth2.googleapis.com/token';
export const getExchangeTokenParams = (code: string) => {
const postData = {
code,
client_id: process.env.AFFINE_GOOGLE_CLIENT_ID || '',
client_secret: process.env.AFFINE_GOOGLE_CLIENT_SECRET || '',
redirect_uri: redirectUri,
grant_type: 'authorization_code',
};
const requestInit: RequestInit = {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
body: new URLSearchParams(postData).toString(),
};
return { requestInit, url: tokenEndpoint };
};
export function getGoogleOauthCode() {
shell.openExternal(oauthEndpoint);
return new Promise<ReturnType<typeof getExchangeTokenParams>>(
(resolve, reject) => {
const handleOpenUrl = async (_: any, url: string) => {
const mainWindow = BrowserWindow.getAllWindows().find(
w => !w.isDestroyed()
);
const urlObj = parse(url.replace('??', '?'), true);
if (!mainWindow || !url.startsWith('affine://auth-callback')) return;
const code = urlObj.query['code'] as string;
if (!code) return;
logger.info('google sign in code received from callback', code);
app.removeListener('open-url', handleOpenUrl);
resolve(getExchangeTokenParams(code));
};
app.on('open-url', handleOpenUrl);
setTimeout(() => {
reject(new Error('Timed out'));
app.removeListener('open-url', handleOpenUrl);
}, 30000);
}
);
}
@@ -0,0 +1,23 @@
import { BrowserWindow, nativeTheme } from 'electron';
import { isMacOS } from '../../../../utils';
import type { NamespaceHandlers } from '../type';
import { getGoogleOauthCode } from './google-auth';
export const uiHandlers = {
handleThemeChange: async (_, theme: (typeof nativeTheme)['themeSource']) => {
nativeTheme.themeSource = theme;
},
handleSidebarVisibilityChange: async (_, visible: boolean) => {
if (isMacOS()) {
const windows = BrowserWindow.getAllWindows();
windows.forEach(w => {
// hide window buttons when sidebar is not visible
w.setWindowButtonVisibility(visible);
});
}
},
getGoogleOauthCode: async () => {
return getGoogleOauthCode();
},
} satisfies NamespaceHandlers;
@@ -0,0 +1,10 @@
import type { NamespaceHandlers } from '../type';
import { updateClient } from './updater';
export const updaterHandlers = {
updateClient: async () => {
return updateClient();
},
} satisfies NamespaceHandlers;
export * from './updater';
@@ -0,0 +1,69 @@
import type { AppUpdater } from 'electron-updater';
import { isMacOS } from '../../../../utils';
import { updaterSubjects } from '../../events/updater';
import { logger } from '../../logger';
const buildType = (process.env.BUILD_TYPE || 'canary').trim().toLowerCase();
const mode = process.env.NODE_ENV;
const isDev = mode === 'development';
let _autoUpdater: AppUpdater | null = null;
export const updateClient = async () => {
_autoUpdater?.quitAndInstall();
};
export const registerUpdater = async () => {
// require it will cause some side effects and will break generate-main-exposed-meta,
// so we wrap it in a function
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { autoUpdater } = await import('electron-updater');
_autoUpdater = autoUpdater;
autoUpdater.autoDownload = false;
autoUpdater.allowPrerelease = buildType !== 'stable';
autoUpdater.autoInstallOnAppQuit = false;
autoUpdater.autoRunAppAfterInstall = true;
autoUpdater.setFeedURL({
channel: buildType,
provider: 'github',
repo: 'AFFiNE',
owner: 'toeverything',
releaseType: buildType === 'stable' ? 'release' : 'prerelease',
});
autoUpdater.autoDownload = false;
autoUpdater.allowPrerelease = buildType !== 'stable';
autoUpdater.autoInstallOnAppQuit = false;
autoUpdater.autoRunAppAfterInstall = true;
autoUpdater.setFeedURL({
channel: buildType,
provider: 'github',
repo: 'AFFiNE',
owner: 'toeverything',
releaseType: buildType === 'stable' ? 'release' : 'prerelease',
});
if (isMacOS()) {
autoUpdater.on('update-available', () => {
autoUpdater.downloadUpdate();
logger.info('Update available, downloading...');
});
autoUpdater.on('download-progress', e => {
logger.info(`Download progress: ${e.percent}`);
});
autoUpdater.on('update-downloaded', e => {
updaterSubjects.clientUpdateReady.next({
version: e.version,
});
logger.info('Update downloaded, ready to install');
});
autoUpdater.on('error', e => {
logger.error('Error while updating client', e);
});
autoUpdater.forceDevUpdateConfig = isDev;
await autoUpdater.checkForUpdatesAndNotify();
}
};
@@ -0,0 +1,8 @@
import { appContext } from '../../context';
import type { NamespaceHandlers } from '../type';
import { deleteWorkspace, listWorkspaces } from './workspace';
export const workspaceHandlers = {
list: async () => listWorkspaces(appContext),
delete: async (_, id: string) => deleteWorkspace(appContext, id),
} satisfies NamespaceHandlers;
@@ -0,0 +1,60 @@
import path from 'node:path';
import fs from 'fs-extra';
import type { AppContext } from '../../context';
import { logger } from '../../logger';
interface WorkspaceMeta {
path: string;
realpath: string;
}
export async function listWorkspaces(
context: AppContext
): Promise<[workspaceId: string, meta: WorkspaceMeta][]> {
const basePath = path.join(context.appDataPath, 'workspaces');
try {
await fs.ensureDir(basePath);
const dirs = await fs.readdir(basePath, {
withFileTypes: true,
});
const meta = await Promise.all(
dirs.map(async dir => {
const dbFilePath = path.join(basePath, dir.name, 'storage.db');
if (dir.isDirectory() && (await fs.exists(dbFilePath))) {
// try read storage.db under it
const realpath = await fs.realpath(dbFilePath);
return [dir.name, { path: dbFilePath, realpath }] as [
string,
WorkspaceMeta
];
} else {
return null;
}
})
);
return meta.filter((w): w is [string, WorkspaceMeta] => !!w);
} catch (error) {
logger.error('listWorkspaces', error);
return [];
}
}
export async function deleteWorkspace(context: AppContext, id: string) {
const basePath = path.join(context.appDataPath, 'workspaces', id);
const movedPath = path.join(
context.appDataPath,
'delete-workspaces',
`${id}`
);
try {
return await fs.move(basePath, movedPath, {
overwrite: true,
});
} catch (error) {
logger.error('deleteWorkspace', error);
}
}