diff --git a/apps/ligo-virgo/src/pages/workspace/docs/Page.tsx b/apps/ligo-virgo/src/pages/workspace/docs/Page.tsx
index 76c48efeb6..886d8f1c1c 100644
--- a/apps/ligo-virgo/src/pages/workspace/docs/Page.tsx
+++ b/apps/ligo-virgo/src/pages/workspace/docs/Page.tsx
@@ -1,11 +1,7 @@
/* eslint-disable filename-rules/match */
import { useEffect, useRef, type UIEvent, useState } from 'react';
import { useParams } from 'react-router';
-import {
- MuiBox as Box,
- MuiCircularProgress as CircularProgress,
- styled,
-} from '@toeverything/components/ui';
+
import { AffineEditor } from '@toeverything/components/affine-editor';
import {
CalendarHeatmap,
@@ -15,10 +11,13 @@ import {
import { CollapsibleTitle } from '@toeverything/components/common';
import {
useShowSpaceSidebar,
- useUserAndSpaces,
usePageClientWidth,
} from '@toeverything/datasource/state';
-import { services } from '@toeverything/datasource/db-service';
+import {
+ MuiBox as Box,
+ MuiCircularProgress as CircularProgress,
+ styled,
+} from '@toeverything/components/ui';
import { WorkspaceName } from './workspace-name';
import { CollapsiblePageTree } from './collapsible-page-tree';
diff --git a/libs/components/account/src/login/fs.tsx b/libs/components/account/src/login/fs.tsx
new file mode 100644
index 0000000000..bc51a90876
--- /dev/null
+++ b/libs/components/account/src/login/fs.tsx
@@ -0,0 +1,99 @@
+/* eslint-disable filename-rules/match */
+import { useState } from 'react';
+
+import { LogoImg } from '@toeverything/components/common';
+import {
+ MuiButton,
+ MuiBox,
+ MuiGrid,
+ MuiSnackbar,
+} from '@toeverything/components/ui';
+import { services } from '@toeverything/datasource/db-service';
+import { useLocalTrigger } from '@toeverything/datasource/state';
+
+import { Error } from './../error';
+
+const requestPermission = async (workspace: string) => {
+ indexedDB.deleteDatabase(workspace);
+ const dirHandler = await window.showDirectoryPicker({
+ id: 'AFFiNE_' + workspace,
+ mode: 'readwrite',
+ startIn: 'documents',
+ });
+ const fileHandle = await dirHandler.getFileHandle('affine.db', {
+ create: true,
+ });
+ const file = await fileHandle.getFile();
+ const initialData = new Uint8Array(await file.arrayBuffer());
+
+ const exporter = async (contents: Uint8Array) => {
+ try {
+ const writable = await fileHandle.createWritable();
+ await writable.write(contents);
+ await writable.close();
+ } catch (e) {
+ console.log(e);
+ }
+ };
+
+ await services.api.editorBlock.setupDataExporter(
+ workspace,
+ new Uint8Array(initialData),
+ exporter
+ );
+};
+
+export const FileSystem = () => {
+ const onSelected = useLocalTrigger();
+ const [error, setError] = useState(false);
+ return (
+
+
+
+
+
+
+
+ {
+ try {
+ await requestPermission('AFFiNE');
+ onSelected();
+ } catch (e) {
+ setError(true);
+ setTimeout(() => setError(false), 3000);
+ }
+ }}
+ style={{
+ textAlign: 'center',
+ width: '300px',
+ margin: '300px auto 20px auto',
+ }}
+ sx={{ mt: 1 }}
+ >
+
+
+
+ Sync to Disk
+
+
+
+
+ );
+};
diff --git a/libs/components/account/src/login/index.tsx b/libs/components/account/src/login/index.tsx
index 3601d97b95..ba7a589c87 100644
--- a/libs/components/account/src/login/index.tsx
+++ b/libs/components/account/src/login/index.tsx
@@ -1,11 +1,13 @@
+/* eslint-disable filename-rules/match */
// import { Authing } from './authing';
import { Firebase } from './firebase';
+import { FileSystem } from './fs';
export function Login() {
return (
<>
{/* */}
-
+ {process.env['NX_LOCAL'] ? : }
>
);
}
diff --git a/libs/components/layout/src/header/LayoutHeader.tsx b/libs/components/layout/src/header/LayoutHeader.tsx
index 0d9560666d..978b7f8dfe 100644
--- a/libs/components/layout/src/header/LayoutHeader.tsx
+++ b/libs/components/layout/src/header/LayoutHeader.tsx
@@ -6,6 +6,7 @@ import {
SideBarViewCloseIcon,
} from '@toeverything/components/icons';
import { useShowSettingsSidebar } from '@toeverything/datasource/state';
+
import { CurrentPageTitle } from './Title';
import { EditorBoardSwitcher } from './EditorBoardSwitcher';
diff --git a/libs/datasource/db-service/src/services/base.ts b/libs/datasource/db-service/src/services/base.ts
index dcf53dcb4c..9e10af9177 100644
--- a/libs/datasource/db-service/src/services/base.ts
+++ b/libs/datasource/db-service/src/services/base.ts
@@ -154,6 +154,14 @@ export abstract class ServiceBaseClass {
await this.database.unregisterTagExporter(workspace, name);
}
+ async setupDataExporter(
+ workspace: string,
+ initialData: Uint8Array,
+ cb: (data: Uint8Array) => Promise
+ ) {
+ await this.database.setupDataExporter(workspace, initialData, cb);
+ }
+
protected async _observe(
workspace: string,
blockId: string,
diff --git a/libs/datasource/db-service/src/services/database/index.ts b/libs/datasource/db-service/src/services/database/index.ts
index 439e6ca90e..9f97b9a35f 100644
--- a/libs/datasource/db-service/src/services/database/index.ts
+++ b/libs/datasource/db-service/src/services/database/index.ts
@@ -192,4 +192,13 @@ export class Database {
}
}
}
+
+ async setupDataExporter(
+ workspace: string,
+ initialData: Uint8Array,
+ callback: (binary: Uint8Array) => Promise
+ ) {
+ const db = await this.getDatabase(workspace);
+ await db.setupDataExporter(initialData, callback);
+ }
}
diff --git a/libs/datasource/jwt-rpc/src/sqlite.ts b/libs/datasource/jwt-rpc/src/sqlite.ts
index 0d47c4615b..3a33daa755 100644
--- a/libs/datasource/jwt-rpc/src/sqlite.ts
+++ b/libs/datasource/jwt-rpc/src/sqlite.ts
@@ -5,7 +5,7 @@ import { Observable } from 'lib0/observable.js';
const PREFERRED_TRIM_SIZE = 500;
const _stmts = {
- create: 'CREATE TABLE updates (key INTEGER PRIMARY KEY AUTOINCREMENT, value BLOB);',
+ create: 'CREATE TABLE IF NOT EXISTS updates (key INTEGER PRIMARY KEY AUTOINCREMENT, value BLOB);',
selectAll: 'SELECT * FROM updates where key >= $idx',
selectCount: 'SELECT count(*) FROM updates',
insert: 'INSERT INTO updates VALUES (null, $data);',
@@ -59,7 +59,7 @@ export class SQLiteProvider extends Observable {
private _size: number;
private _destroyed: boolean;
private _db: Promise;
- private _saver?: (binary: Uint8Array) => void;
+ private _saver?: (binary: Uint8Array) => Promise | undefined;
private _destroy: () => void;
constructor(name: string, doc: Y.Doc, origin?: Uint8Array) {
@@ -82,8 +82,9 @@ export class SQLiteProvider extends Observable {
this.whenSynced = this._db.then(async db => {
this.db = db;
const currState = Y.encodeStateAsUpdate(doc);
- await this._fetchUpdates();
+ await this._fetchUpdates(true);
db.exec(_stmts.insert, { $data: currState });
+ this._storeState();
if (this._destroyed) return this;
this.emit('synced', [this]);
this.synced = true;
@@ -91,21 +92,38 @@ export class SQLiteProvider extends Observable {
});
// Timeout in ms until data is merged and persisted in sqlite.
- const storeTimeout = 1000;
+ const storeTimeout = 500;
let storeTimeoutId: NodeJS.Timer | undefined = undefined;
+ let lastSize = 0;
+
+ const debouncedStoreState = (force = false) => {
+ // debounce store call
+ if (storeTimeoutId) clearTimeout(storeTimeoutId);
+
+ if (force) {
+ if (lastSize !== this._size) {
+ this._storeState();
+ storeTimeoutId = undefined;
+ lastSize = this._size;
+ }
+ } else {
+ storeTimeoutId = setTimeout(() => {
+ this._storeState();
+ storeTimeoutId = undefined;
+ }, storeTimeout);
+ }
+ };
+ const storeStateInterval = setInterval(
+ () => debouncedStoreState(true),
+ 1000
+ );
const storeUpdate = (update: Uint8Array, origin: any) => {
if (this._saver && this.db && origin !== this) {
this.db.exec(_stmts.insert, { $data: update });
if (++this._size >= PREFERRED_TRIM_SIZE) {
- // debounce store call
- if (storeTimeoutId) clearTimeout(storeTimeoutId);
-
- storeTimeoutId = setTimeout(() => {
- this._storeState();
- storeTimeoutId = undefined;
- }, storeTimeout);
+ debouncedStoreState();
}
}
};
@@ -116,34 +134,53 @@ export class SQLiteProvider extends Observable {
this._destroy = () => {
if (storeTimeoutId) clearTimeout(storeTimeoutId);
+ if (storeStateInterval) clearInterval(storeStateInterval);
this.doc.off('update', storeUpdate);
this.doc.off('destroy', this.destroy);
};
}
- registerExporter(saver: (binary: Uint8Array) => void) {
+ registerExporter(saver: (binary: Uint8Array) => Promise | undefined) {
this._saver = saver;
}
- private async _storeState() {
+ private async _storeState(force?: boolean) {
await this._fetchUpdates();
- if (this.db && this._size >= PREFERRED_TRIM_SIZE) {
- this.db.exec(_stmts.insert, {
- $data: Y.encodeStateAsUpdate(this.doc),
- });
+ if (this.db) {
+ if (force || this._size >= PREFERRED_TRIM_SIZE) {
+ this.db.exec(_stmts.insert, {
+ $data: Y.encodeStateAsUpdate(this.doc),
+ });
- clearUpdates(this.db, this._ref);
+ clearUpdates(this.db, this._ref);
- this._size = countUpdates(this.db);
+ this._size = countUpdates(this.db);
+ }
- this._saver?.(this.db?.export());
+ await this._saver?.(this.db?.export());
}
}
- private async _fetchUpdates() {
+ private _waitUpdate(sync = false) {
+ if (sync) {
+ return new Promise((resolve, reject) => {
+ const final = (_: any, origin: any) => {
+ if (origin === this) {
+ this.doc.off('update', final);
+ resolve();
+ }
+ };
+ this.doc.on('update', final);
+ });
+ }
+ return undefined;
+ }
+
+ private async _fetchUpdates(sync = false) {
if (this.db) {
+ const wait = this._waitUpdate(sync);
const updates = getAllUpdates(this.db, this._ref);
Y.transact(
@@ -160,6 +197,7 @@ export class SQLiteProvider extends Observable {
const lastKey = Math.max(...updates.map(([idx]) => idx));
this._ref = lastKey + 1;
this._size = countUpdates(this.db);
+ await wait;
}
}
diff --git a/libs/datasource/jwt/src/adapter/index.ts b/libs/datasource/jwt/src/adapter/index.ts
index ddf4f29ca4..2fa2ec551d 100644
--- a/libs/datasource/jwt/src/adapter/index.ts
+++ b/libs/datasource/jwt/src/adapter/index.ts
@@ -136,6 +136,7 @@ interface BlockInstance {
interface AsyncDatabaseAdapter {
inspector(): Record;
+ reload(): void;
createBlock(
options: Pick, 'type' | 'flavor'> & {
binary?: ArrayBuffer;
@@ -156,6 +157,33 @@ interface AsyncDatabaseAdapter {
getUserId(): string;
}
+export type DataExporter = (binary: Uint8Array) => Promise;
+
+export const getDataExporter = () => {
+ let exporter: DataExporter | undefined = undefined;
+ let importer: (() => Uint8Array | undefined) | undefined = undefined;
+
+ const importData = () => importer?.();
+ const exportData = (binary: Uint8Array) => exporter?.(binary);
+ const hasExporter = () => !!exporter;
+
+ const installExporter = (
+ initialData: Uint8Array | undefined,
+ cb: DataExporter
+ ) => {
+ return new Promise(resolve => {
+ importer = () => initialData;
+ exporter = async (data: Uint8Array) => {
+ exporter = cb;
+ await cb(data);
+ resolve();
+ };
+ });
+ };
+
+ return { importData, exportData, hasExporter, installExporter };
+};
+
export type {
AsyncDatabaseAdapter,
BlockPosition,
diff --git a/libs/datasource/jwt/src/adapter/yjs/index.ts b/libs/datasource/jwt/src/adapter/yjs/index.ts
index 2e7681ae53..a1497de609 100644
--- a/libs/datasource/jwt/src/adapter/yjs/index.ts
+++ b/libs/datasource/jwt/src/adapter/yjs/index.ts
@@ -153,19 +153,21 @@ export class YjsAdapter implements AsyncDatabaseAdapter {
private readonly _doc: Doc; // doc instance
private readonly _awareness: Awareness; // lightweight state synchronization
private readonly _gatekeeper: GateKeeper; // Simple access control
- private readonly _history: YjsHistoryManager;
+ private readonly _history!: YjsHistoryManager;
// Block Collection
// key is a randomly generated global id
- private readonly _blocks: YMap>;
- private readonly _blockUpdated: YMap;
+ private readonly _blocks!: YMap>;
+ private readonly _blockUpdated!: YMap;
// Maximum cache Block 1024, ttl 10 minutes
- private readonly _blockCaches: LRUCache;
+ private readonly _blockCaches!: LRUCache;
- private readonly _binaries: YjsRemoteBinaries;
+ private readonly _binaries!: YjsRemoteBinaries;
private readonly _listener: Map>;
+ private readonly _reload: () => void;
+
static async init(
workspace: string,
options: YjsInitOptions
@@ -184,18 +186,28 @@ export class YjsAdapter implements AsyncDatabaseAdapter {
this._doc = providers.idb.doc;
this._awareness = providers.awareness;
this._gatekeeper = providers.gatekeeper;
-
- const blocks = this._doc.getMap>('blocks');
- this._blocks =
- blocks.get('content') || blocks.set('content', new YMap());
- this._blockUpdated =
- blocks.get('updated') || blocks.set('updated', new YMap());
- this._blockCaches = new LRUCache({ max: 1024, ttl: 1000 * 60 * 10 });
- this._binaries = new YjsRemoteBinaries(
- providers.binariesIdb.doc.getMap(),
- providers.remoteToken
- );
- this._history = new YjsHistoryManager(this._blocks);
+ this._reload = () => {
+ const blocks = this._doc.getMap>('blocks');
+ // @ts-ignore
+ this._blocks =
+ blocks.get('content') || blocks.set('content', new YMap());
+ // @ts-ignore
+ this._blockUpdated =
+ blocks.get('updated') || blocks.set('updated', new YMap());
+ // @ts-ignore
+ this._blockCaches = new LRUCache({
+ max: 1024,
+ ttl: 1000 * 60 * 10,
+ });
+ // @ts-ignore
+ this._binaries = new YjsRemoteBinaries(
+ providers.binariesIdb.doc.getMap(),
+ providers.remoteToken
+ );
+ // @ts-ignore
+ this._history = new YjsHistoryManager(this._blocks);
+ };
+ this._reload();
this._listener = new Map();
@@ -281,6 +293,10 @@ export class YjsAdapter implements AsyncDatabaseAdapter {
});
}
+ reload() {
+ this._reload();
+ }
+
getUserId(): string {
return this._provider.userId;
}
diff --git a/libs/datasource/jwt/src/adapter/yjs/provider.ts b/libs/datasource/jwt/src/adapter/yjs/provider.ts
index d0f7f9d3af..f7068288fd 100644
--- a/libs/datasource/jwt/src/adapter/yjs/provider.ts
+++ b/libs/datasource/jwt/src/adapter/yjs/provider.ts
@@ -22,8 +22,9 @@ export type YjsProvider = (instances: YjsDefaultInstances) => Promise;
export type YjsProviderOptions = {
backend: typeof BucketBackend[keyof typeof BucketBackend];
params?: Record;
- importData?: Uint8Array;
- exportData?: (binary: Uint8Array) => void;
+ importData?: () => Promise | Uint8Array | undefined;
+ exportData?: (binary: Uint8Array) => Promise | undefined;
+ hasExporter?: () => boolean;
};
export const getYjsProviders = (
@@ -31,13 +32,20 @@ export const getYjsProviders = (
): Record => {
return {
sqlite: async (instances: YjsDefaultInstances) => {
- const fs = new SQLiteProvider(
- instances.workspace,
- instances.doc,
- options.importData
- );
- if (options.exportData) fs.registerExporter(options.exportData);
- await fs.whenSynced;
+ const fsHandle = setInterval(async () => {
+ if (options.hasExporter?.()) {
+ clearInterval(fsHandle);
+ const fs = new SQLiteProvider(
+ instances.workspace,
+ instances.doc,
+ await options.importData?.()
+ );
+ if (options.exportData) {
+ fs.registerExporter(options.exportData);
+ }
+ await fs.whenSynced;
+ }
+ }, 500);
},
ws: async (instances: YjsDefaultInstances) => {
if (instances.token) {
diff --git a/libs/datasource/jwt/src/index.ts b/libs/datasource/jwt/src/index.ts
index acd3ce5f53..ecad7aebf9 100644
--- a/libs/datasource/jwt/src/index.ts
+++ b/libs/datasource/jwt/src/index.ts
@@ -14,6 +14,8 @@ import {
HistoryManager,
ContentTypes,
Connectivity,
+ DataExporter,
+ getDataExporter,
} from './adapter';
import {
getYjsProviders,
@@ -66,6 +68,10 @@ type BlockClientOptions = {
content?: BlockExporters;
metadata?: BlockExporters>;
tagger?: BlockExporters;
+ installExporter: (
+ initialData: Uint8Array,
+ exporter: DataExporter
+ ) => Promise;
};
export class BlockClient<
@@ -95,10 +101,15 @@ export class BlockClient<
private readonly _root: { node?: BaseBlock };
+ private readonly _installExporter: (
+ initialData: Uint8Array,
+ exporter: DataExporter
+ ) => Promise;
+
private constructor(
adapter: A,
workspace: string,
- options?: BlockClientOptions
+ options: BlockClientOptions
) {
this._adapter = adapter;
this._workspace = workspace;
@@ -142,6 +153,7 @@ export class BlockClient<
});
this._root = {};
+ this._installExporter = options.installExporter;
}
public addBlockListener(tag: string, listener: BlockListener) {
@@ -590,21 +602,34 @@ export class BlockClient<
return this._adapter.history();
}
+ public async setupDataExporter(initialData: Uint8Array, cb: DataExporter) {
+ await this._installExporter(initialData, cb);
+ this._adapter.reload();
+ }
+
public static async init(
workspace: string,
options: Partial<
YjsInitOptions & YjsProviderOptions & BlockClientOptions
> = {}
): Promise {
+ const { importData, exportData, hasExporter, installExporter } =
+ getDataExporter();
+
const instance = await YjsAdapter.init(workspace, {
provider: getYjsProviders({
backend: BucketBackend.YjsWebSocketAffine,
- exportData: console.log.bind(console),
+ importData,
+ exportData,
+ hasExporter,
...options,
}),
...options,
});
- return new BlockClient(instance, workspace, options);
+ return new BlockClient(instance, workspace, {
+ ...options,
+ installExporter,
+ });
}
}
diff --git a/libs/datasource/state/src/user.ts b/libs/datasource/state/src/user.ts
index bce11b8327..fefb689f0a 100644
--- a/libs/datasource/state/src/user.ts
+++ b/libs/datasource/state/src/user.ts
@@ -55,28 +55,39 @@ const _useUserAndSpace = () => {
const currentSpaceId: string | undefined = useMemo(() => user?.id, [user]);
- return {
- user,
- currentSpaceId,
- loading,
- };
+ return { user, currentSpaceId, loading };
};
+const BRAND_ID = 'AFFiNE';
+
+const _localTrigger = atom(false);
const _useUserAndSpacesForFreeLogin = () => {
+ const [user, setUser] = useAtom(_userAtom);
const [loading, setLoading] = useAtom(_loadingAtom);
+ const [localTrigger] = useAtom(_localTrigger);
useEffect(() => setLoading(false), []);
- const BRAND_ID = 'AFFiNE';
- return {
- user: {
- photo: '',
- id: BRAND_ID,
- nickname: BRAND_ID,
- email: '',
- } as UserInfo,
- currentSpaceId: BRAND_ID,
- loading,
- };
+
+ useEffect(() => {
+ if (localTrigger) {
+ setUser({
+ photo: '',
+ id: BRAND_ID,
+ username: BRAND_ID,
+ nickname: BRAND_ID,
+ email: '',
+ });
+ }
+ }, [localTrigger, setLoading, setUser]);
+
+ const currentSpaceId: string | undefined = useMemo(() => user?.id, [user]);
+
+ return { user, currentSpaceId, loading };
+};
+
+export const useLocalTrigger = () => {
+ const [, setTrigger] = useAtom(_localTrigger);
+ return () => setTrigger(true);
};
export const useUserAndSpaces = process.env['NX_LOCAL']