diff --git a/apps/ligo-virgo/src/pages/workspace/docs/Page.tsx b/apps/ligo-virgo/src/pages/workspace/docs/Page.tsx index 76c48efeb6..886d8f1c1c 100644 --- a/apps/ligo-virgo/src/pages/workspace/docs/Page.tsx +++ b/apps/ligo-virgo/src/pages/workspace/docs/Page.tsx @@ -1,11 +1,7 @@ /* eslint-disable filename-rules/match */ import { useEffect, useRef, type UIEvent, useState } from 'react'; import { useParams } from 'react-router'; -import { - MuiBox as Box, - MuiCircularProgress as CircularProgress, - styled, -} from '@toeverything/components/ui'; + import { AffineEditor } from '@toeverything/components/affine-editor'; import { CalendarHeatmap, @@ -15,10 +11,13 @@ import { import { CollapsibleTitle } from '@toeverything/components/common'; import { useShowSpaceSidebar, - useUserAndSpaces, usePageClientWidth, } from '@toeverything/datasource/state'; -import { services } from '@toeverything/datasource/db-service'; +import { + MuiBox as Box, + MuiCircularProgress as CircularProgress, + styled, +} from '@toeverything/components/ui'; import { WorkspaceName } from './workspace-name'; import { CollapsiblePageTree } from './collapsible-page-tree'; diff --git a/libs/components/account/src/login/fs.tsx b/libs/components/account/src/login/fs.tsx new file mode 100644 index 0000000000..bc51a90876 --- /dev/null +++ b/libs/components/account/src/login/fs.tsx @@ -0,0 +1,99 @@ +/* eslint-disable filename-rules/match */ +import { useState } from 'react'; + +import { LogoImg } from '@toeverything/components/common'; +import { + MuiButton, + MuiBox, + MuiGrid, + MuiSnackbar, +} from '@toeverything/components/ui'; +import { services } from '@toeverything/datasource/db-service'; +import { useLocalTrigger } from '@toeverything/datasource/state'; + +import { Error } from './../error'; + +const requestPermission = async (workspace: string) => { + indexedDB.deleteDatabase(workspace); + const dirHandler = await window.showDirectoryPicker({ + id: 'AFFiNE_' + workspace, + mode: 'readwrite', + startIn: 'documents', + }); + const fileHandle = await dirHandler.getFileHandle('affine.db', { + create: true, + }); + const file = await fileHandle.getFile(); + const initialData = new Uint8Array(await file.arrayBuffer()); + + const exporter = async (contents: Uint8Array) => { + try { + const writable = await fileHandle.createWritable(); + await writable.write(contents); + await writable.close(); + } catch (e) { + console.log(e); + } + }; + + await services.api.editorBlock.setupDataExporter( + workspace, + new Uint8Array(initialData), + exporter + ); +}; + +export const FileSystem = () => { + const onSelected = useLocalTrigger(); + const [error, setError] = useState(false); + return ( + + + + + + + + { + try { + await requestPermission('AFFiNE'); + onSelected(); + } catch (e) { + setError(true); + setTimeout(() => setError(false), 3000); + } + }} + style={{ + textAlign: 'center', + width: '300px', + margin: '300px auto 20px auto', + }} + sx={{ mt: 1 }} + > + + + + Sync to Disk + + + + + ); +}; diff --git a/libs/components/account/src/login/index.tsx b/libs/components/account/src/login/index.tsx index 3601d97b95..ba7a589c87 100644 --- a/libs/components/account/src/login/index.tsx +++ b/libs/components/account/src/login/index.tsx @@ -1,11 +1,13 @@ +/* eslint-disable filename-rules/match */ // import { Authing } from './authing'; import { Firebase } from './firebase'; +import { FileSystem } from './fs'; export function Login() { return ( <> {/* */} - + {process.env['NX_LOCAL'] ? : } ); } diff --git a/libs/components/layout/src/header/LayoutHeader.tsx b/libs/components/layout/src/header/LayoutHeader.tsx index 0d9560666d..978b7f8dfe 100644 --- a/libs/components/layout/src/header/LayoutHeader.tsx +++ b/libs/components/layout/src/header/LayoutHeader.tsx @@ -6,6 +6,7 @@ import { SideBarViewCloseIcon, } from '@toeverything/components/icons'; import { useShowSettingsSidebar } from '@toeverything/datasource/state'; + import { CurrentPageTitle } from './Title'; import { EditorBoardSwitcher } from './EditorBoardSwitcher'; diff --git a/libs/datasource/db-service/src/services/base.ts b/libs/datasource/db-service/src/services/base.ts index dcf53dcb4c..9e10af9177 100644 --- a/libs/datasource/db-service/src/services/base.ts +++ b/libs/datasource/db-service/src/services/base.ts @@ -154,6 +154,14 @@ export abstract class ServiceBaseClass { await this.database.unregisterTagExporter(workspace, name); } + async setupDataExporter( + workspace: string, + initialData: Uint8Array, + cb: (data: Uint8Array) => Promise + ) { + await this.database.setupDataExporter(workspace, initialData, cb); + } + protected async _observe( workspace: string, blockId: string, diff --git a/libs/datasource/db-service/src/services/database/index.ts b/libs/datasource/db-service/src/services/database/index.ts index 439e6ca90e..9f97b9a35f 100644 --- a/libs/datasource/db-service/src/services/database/index.ts +++ b/libs/datasource/db-service/src/services/database/index.ts @@ -192,4 +192,13 @@ export class Database { } } } + + async setupDataExporter( + workspace: string, + initialData: Uint8Array, + callback: (binary: Uint8Array) => Promise + ) { + const db = await this.getDatabase(workspace); + await db.setupDataExporter(initialData, callback); + } } diff --git a/libs/datasource/jwt-rpc/src/sqlite.ts b/libs/datasource/jwt-rpc/src/sqlite.ts index 0d47c4615b..3a33daa755 100644 --- a/libs/datasource/jwt-rpc/src/sqlite.ts +++ b/libs/datasource/jwt-rpc/src/sqlite.ts @@ -5,7 +5,7 @@ import { Observable } from 'lib0/observable.js'; const PREFERRED_TRIM_SIZE = 500; const _stmts = { - create: 'CREATE TABLE updates (key INTEGER PRIMARY KEY AUTOINCREMENT, value BLOB);', + create: 'CREATE TABLE IF NOT EXISTS updates (key INTEGER PRIMARY KEY AUTOINCREMENT, value BLOB);', selectAll: 'SELECT * FROM updates where key >= $idx', selectCount: 'SELECT count(*) FROM updates', insert: 'INSERT INTO updates VALUES (null, $data);', @@ -59,7 +59,7 @@ export class SQLiteProvider extends Observable { private _size: number; private _destroyed: boolean; private _db: Promise; - private _saver?: (binary: Uint8Array) => void; + private _saver?: (binary: Uint8Array) => Promise | undefined; private _destroy: () => void; constructor(name: string, doc: Y.Doc, origin?: Uint8Array) { @@ -82,8 +82,9 @@ export class SQLiteProvider extends Observable { this.whenSynced = this._db.then(async db => { this.db = db; const currState = Y.encodeStateAsUpdate(doc); - await this._fetchUpdates(); + await this._fetchUpdates(true); db.exec(_stmts.insert, { $data: currState }); + this._storeState(); if (this._destroyed) return this; this.emit('synced', [this]); this.synced = true; @@ -91,21 +92,38 @@ export class SQLiteProvider extends Observable { }); // Timeout in ms until data is merged and persisted in sqlite. - const storeTimeout = 1000; + const storeTimeout = 500; let storeTimeoutId: NodeJS.Timer | undefined = undefined; + let lastSize = 0; + + const debouncedStoreState = (force = false) => { + // debounce store call + if (storeTimeoutId) clearTimeout(storeTimeoutId); + + if (force) { + if (lastSize !== this._size) { + this._storeState(); + storeTimeoutId = undefined; + lastSize = this._size; + } + } else { + storeTimeoutId = setTimeout(() => { + this._storeState(); + storeTimeoutId = undefined; + }, storeTimeout); + } + }; + const storeStateInterval = setInterval( + () => debouncedStoreState(true), + 1000 + ); const storeUpdate = (update: Uint8Array, origin: any) => { if (this._saver && this.db && origin !== this) { this.db.exec(_stmts.insert, { $data: update }); if (++this._size >= PREFERRED_TRIM_SIZE) { - // debounce store call - if (storeTimeoutId) clearTimeout(storeTimeoutId); - - storeTimeoutId = setTimeout(() => { - this._storeState(); - storeTimeoutId = undefined; - }, storeTimeout); + debouncedStoreState(); } } }; @@ -116,34 +134,53 @@ export class SQLiteProvider extends Observable { this._destroy = () => { if (storeTimeoutId) clearTimeout(storeTimeoutId); + if (storeStateInterval) clearInterval(storeStateInterval); this.doc.off('update', storeUpdate); this.doc.off('destroy', this.destroy); }; } - registerExporter(saver: (binary: Uint8Array) => void) { + registerExporter(saver: (binary: Uint8Array) => Promise | undefined) { this._saver = saver; } - private async _storeState() { + private async _storeState(force?: boolean) { await this._fetchUpdates(); - if (this.db && this._size >= PREFERRED_TRIM_SIZE) { - this.db.exec(_stmts.insert, { - $data: Y.encodeStateAsUpdate(this.doc), - }); + if (this.db) { + if (force || this._size >= PREFERRED_TRIM_SIZE) { + this.db.exec(_stmts.insert, { + $data: Y.encodeStateAsUpdate(this.doc), + }); - clearUpdates(this.db, this._ref); + clearUpdates(this.db, this._ref); - this._size = countUpdates(this.db); + this._size = countUpdates(this.db); + } - this._saver?.(this.db?.export()); + await this._saver?.(this.db?.export()); } } - private async _fetchUpdates() { + private _waitUpdate(sync = false) { + if (sync) { + return new Promise((resolve, reject) => { + const final = (_: any, origin: any) => { + if (origin === this) { + this.doc.off('update', final); + resolve(); + } + }; + this.doc.on('update', final); + }); + } + return undefined; + } + + private async _fetchUpdates(sync = false) { if (this.db) { + const wait = this._waitUpdate(sync); const updates = getAllUpdates(this.db, this._ref); Y.transact( @@ -160,6 +197,7 @@ export class SQLiteProvider extends Observable { const lastKey = Math.max(...updates.map(([idx]) => idx)); this._ref = lastKey + 1; this._size = countUpdates(this.db); + await wait; } } diff --git a/libs/datasource/jwt/src/adapter/index.ts b/libs/datasource/jwt/src/adapter/index.ts index ddf4f29ca4..2fa2ec551d 100644 --- a/libs/datasource/jwt/src/adapter/index.ts +++ b/libs/datasource/jwt/src/adapter/index.ts @@ -136,6 +136,7 @@ interface BlockInstance { interface AsyncDatabaseAdapter { inspector(): Record; + reload(): void; createBlock( options: Pick, 'type' | 'flavor'> & { binary?: ArrayBuffer; @@ -156,6 +157,33 @@ interface AsyncDatabaseAdapter { getUserId(): string; } +export type DataExporter = (binary: Uint8Array) => Promise; + +export const getDataExporter = () => { + let exporter: DataExporter | undefined = undefined; + let importer: (() => Uint8Array | undefined) | undefined = undefined; + + const importData = () => importer?.(); + const exportData = (binary: Uint8Array) => exporter?.(binary); + const hasExporter = () => !!exporter; + + const installExporter = ( + initialData: Uint8Array | undefined, + cb: DataExporter + ) => { + return new Promise(resolve => { + importer = () => initialData; + exporter = async (data: Uint8Array) => { + exporter = cb; + await cb(data); + resolve(); + }; + }); + }; + + return { importData, exportData, hasExporter, installExporter }; +}; + export type { AsyncDatabaseAdapter, BlockPosition, diff --git a/libs/datasource/jwt/src/adapter/yjs/index.ts b/libs/datasource/jwt/src/adapter/yjs/index.ts index 2e7681ae53..a1497de609 100644 --- a/libs/datasource/jwt/src/adapter/yjs/index.ts +++ b/libs/datasource/jwt/src/adapter/yjs/index.ts @@ -153,19 +153,21 @@ export class YjsAdapter implements AsyncDatabaseAdapter { private readonly _doc: Doc; // doc instance private readonly _awareness: Awareness; // lightweight state synchronization private readonly _gatekeeper: GateKeeper; // Simple access control - private readonly _history: YjsHistoryManager; + private readonly _history!: YjsHistoryManager; // Block Collection // key is a randomly generated global id - private readonly _blocks: YMap>; - private readonly _blockUpdated: YMap; + private readonly _blocks!: YMap>; + private readonly _blockUpdated!: YMap; // Maximum cache Block 1024, ttl 10 minutes - private readonly _blockCaches: LRUCache; + private readonly _blockCaches!: LRUCache; - private readonly _binaries: YjsRemoteBinaries; + private readonly _binaries!: YjsRemoteBinaries; private readonly _listener: Map>; + private readonly _reload: () => void; + static async init( workspace: string, options: YjsInitOptions @@ -184,18 +186,28 @@ export class YjsAdapter implements AsyncDatabaseAdapter { this._doc = providers.idb.doc; this._awareness = providers.awareness; this._gatekeeper = providers.gatekeeper; - - const blocks = this._doc.getMap>('blocks'); - this._blocks = - blocks.get('content') || blocks.set('content', new YMap()); - this._blockUpdated = - blocks.get('updated') || blocks.set('updated', new YMap()); - this._blockCaches = new LRUCache({ max: 1024, ttl: 1000 * 60 * 10 }); - this._binaries = new YjsRemoteBinaries( - providers.binariesIdb.doc.getMap(), - providers.remoteToken - ); - this._history = new YjsHistoryManager(this._blocks); + this._reload = () => { + const blocks = this._doc.getMap>('blocks'); + // @ts-ignore + this._blocks = + blocks.get('content') || blocks.set('content', new YMap()); + // @ts-ignore + this._blockUpdated = + blocks.get('updated') || blocks.set('updated', new YMap()); + // @ts-ignore + this._blockCaches = new LRUCache({ + max: 1024, + ttl: 1000 * 60 * 10, + }); + // @ts-ignore + this._binaries = new YjsRemoteBinaries( + providers.binariesIdb.doc.getMap(), + providers.remoteToken + ); + // @ts-ignore + this._history = new YjsHistoryManager(this._blocks); + }; + this._reload(); this._listener = new Map(); @@ -281,6 +293,10 @@ export class YjsAdapter implements AsyncDatabaseAdapter { }); } + reload() { + this._reload(); + } + getUserId(): string { return this._provider.userId; } diff --git a/libs/datasource/jwt/src/adapter/yjs/provider.ts b/libs/datasource/jwt/src/adapter/yjs/provider.ts index d0f7f9d3af..f7068288fd 100644 --- a/libs/datasource/jwt/src/adapter/yjs/provider.ts +++ b/libs/datasource/jwt/src/adapter/yjs/provider.ts @@ -22,8 +22,9 @@ export type YjsProvider = (instances: YjsDefaultInstances) => Promise; export type YjsProviderOptions = { backend: typeof BucketBackend[keyof typeof BucketBackend]; params?: Record; - importData?: Uint8Array; - exportData?: (binary: Uint8Array) => void; + importData?: () => Promise | Uint8Array | undefined; + exportData?: (binary: Uint8Array) => Promise | undefined; + hasExporter?: () => boolean; }; export const getYjsProviders = ( @@ -31,13 +32,20 @@ export const getYjsProviders = ( ): Record => { return { sqlite: async (instances: YjsDefaultInstances) => { - const fs = new SQLiteProvider( - instances.workspace, - instances.doc, - options.importData - ); - if (options.exportData) fs.registerExporter(options.exportData); - await fs.whenSynced; + const fsHandle = setInterval(async () => { + if (options.hasExporter?.()) { + clearInterval(fsHandle); + const fs = new SQLiteProvider( + instances.workspace, + instances.doc, + await options.importData?.() + ); + if (options.exportData) { + fs.registerExporter(options.exportData); + } + await fs.whenSynced; + } + }, 500); }, ws: async (instances: YjsDefaultInstances) => { if (instances.token) { diff --git a/libs/datasource/jwt/src/index.ts b/libs/datasource/jwt/src/index.ts index acd3ce5f53..ecad7aebf9 100644 --- a/libs/datasource/jwt/src/index.ts +++ b/libs/datasource/jwt/src/index.ts @@ -14,6 +14,8 @@ import { HistoryManager, ContentTypes, Connectivity, + DataExporter, + getDataExporter, } from './adapter'; import { getYjsProviders, @@ -66,6 +68,10 @@ type BlockClientOptions = { content?: BlockExporters; metadata?: BlockExporters>; tagger?: BlockExporters; + installExporter: ( + initialData: Uint8Array, + exporter: DataExporter + ) => Promise; }; export class BlockClient< @@ -95,10 +101,15 @@ export class BlockClient< private readonly _root: { node?: BaseBlock }; + private readonly _installExporter: ( + initialData: Uint8Array, + exporter: DataExporter + ) => Promise; + private constructor( adapter: A, workspace: string, - options?: BlockClientOptions + options: BlockClientOptions ) { this._adapter = adapter; this._workspace = workspace; @@ -142,6 +153,7 @@ export class BlockClient< }); this._root = {}; + this._installExporter = options.installExporter; } public addBlockListener(tag: string, listener: BlockListener) { @@ -590,21 +602,34 @@ export class BlockClient< return this._adapter.history(); } + public async setupDataExporter(initialData: Uint8Array, cb: DataExporter) { + await this._installExporter(initialData, cb); + this._adapter.reload(); + } + public static async init( workspace: string, options: Partial< YjsInitOptions & YjsProviderOptions & BlockClientOptions > = {} ): Promise { + const { importData, exportData, hasExporter, installExporter } = + getDataExporter(); + const instance = await YjsAdapter.init(workspace, { provider: getYjsProviders({ backend: BucketBackend.YjsWebSocketAffine, - exportData: console.log.bind(console), + importData, + exportData, + hasExporter, ...options, }), ...options, }); - return new BlockClient(instance, workspace, options); + return new BlockClient(instance, workspace, { + ...options, + installExporter, + }); } } diff --git a/libs/datasource/state/src/user.ts b/libs/datasource/state/src/user.ts index bce11b8327..fefb689f0a 100644 --- a/libs/datasource/state/src/user.ts +++ b/libs/datasource/state/src/user.ts @@ -55,28 +55,39 @@ const _useUserAndSpace = () => { const currentSpaceId: string | undefined = useMemo(() => user?.id, [user]); - return { - user, - currentSpaceId, - loading, - }; + return { user, currentSpaceId, loading }; }; +const BRAND_ID = 'AFFiNE'; + +const _localTrigger = atom(false); const _useUserAndSpacesForFreeLogin = () => { + const [user, setUser] = useAtom(_userAtom); const [loading, setLoading] = useAtom(_loadingAtom); + const [localTrigger] = useAtom(_localTrigger); useEffect(() => setLoading(false), []); - const BRAND_ID = 'AFFiNE'; - return { - user: { - photo: '', - id: BRAND_ID, - nickname: BRAND_ID, - email: '', - } as UserInfo, - currentSpaceId: BRAND_ID, - loading, - }; + + useEffect(() => { + if (localTrigger) { + setUser({ + photo: '', + id: BRAND_ID, + username: BRAND_ID, + nickname: BRAND_ID, + email: '', + }); + } + }, [localTrigger, setLoading, setUser]); + + const currentSpaceId: string | undefined = useMemo(() => user?.id, [user]); + + return { user, currentSpaceId, loading }; +}; + +export const useLocalTrigger = () => { + const [, setTrigger] = useAtom(_localTrigger); + return () => setTrigger(true); }; export const useUserAndSpaces = process.env['NX_LOCAL']