refactor(infra): migrate to new infra (#5565)

This commit is contained in:
EYHN
2024-01-30 07:16:39 +00:00
parent 1e3499c323
commit 329fc19852
170 changed files with 2007 additions and 4354 deletions
@@ -1,12 +1,12 @@
import 'fake-indexeddb/auto';
import { SyncEngine, SyncEngineStep, SyncPeerStep } from '@affine/workspace';
import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
import { Schema, Workspace } from '@blocksuite/store';
import { SyncEngine, SyncEngineStep, SyncPeerStep } from '@toeverything/infra';
import { beforeEach, describe, expect, test, vi } from 'vitest';
import { Doc } from 'yjs';
import { createIndexedDBStorage } from '..';
import { IndexedDBSyncStorage } from '..';
import { createTestStorage } from './test-storage';
const schema = new Schema();
@@ -29,10 +29,10 @@ describe('SyncEngine', () => {
const syncEngine = new SyncEngine(
workspace.doc,
createIndexedDBStorage(workspace.doc.guid),
new IndexedDBSyncStorage(workspace.doc.guid),
[
createIndexedDBStorage(workspace.doc.guid + '1'),
createIndexedDBStorage(workspace.doc.guid + '2'),
new IndexedDBSyncStorage(workspace.doc.guid + '1'),
new IndexedDBSyncStorage(workspace.doc.guid + '2'),
]
);
syncEngine.start();
@@ -60,7 +60,7 @@ describe('SyncEngine', () => {
});
const syncEngine = new SyncEngine(
workspace.doc,
createIndexedDBStorage(workspace.doc.guid),
new IndexedDBSyncStorage(workspace.doc.guid),
[]
);
syncEngine.start();
@@ -79,7 +79,7 @@ describe('SyncEngine', () => {
});
const syncEngine = new SyncEngine(
workspace.doc,
createIndexedDBStorage(workspace.doc.guid + '1'),
new IndexedDBSyncStorage(workspace.doc.guid + '1'),
[]
);
syncEngine.start();
@@ -98,7 +98,7 @@ describe('SyncEngine', () => {
});
const syncEngine = new SyncEngine(
workspace.doc,
createIndexedDBStorage(workspace.doc.guid + '2'),
new IndexedDBSyncStorage(workspace.doc.guid + '2'),
[]
);
syncEngine.start();
@@ -113,9 +113,9 @@ describe('SyncEngine', () => {
test('status', async () => {
const ydoc = new Doc({ guid: 'test - syncengine - status' });
const localStorage = createTestStorage(createIndexedDBStorage(ydoc.guid));
const localStorage = createTestStorage(new IndexedDBSyncStorage(ydoc.guid));
const remoteStorage = createTestStorage(
createIndexedDBStorage(ydoc.guid + '1')
new IndexedDBSyncStorage(ydoc.guid + '1')
);
localStorage.pausePull();
@@ -1,11 +1,11 @@
import 'fake-indexeddb/auto';
import { SyncPeer, SyncPeerStep } from '@affine/workspace';
import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
import { Schema, Workspace } from '@blocksuite/store';
import { SyncPeer, SyncPeerStep } from '@toeverything/infra';
import { beforeEach, describe, expect, test, vi } from 'vitest';
import { createIndexedDBStorage } from '..';
import { IndexedDBSyncStorage } from '..';
const schema = new Schema();
@@ -27,7 +27,7 @@ describe('SyncPeer', () => {
const syncPeer = new SyncPeer(
workspace.doc,
createIndexedDBStorage(workspace.doc.guid)
new IndexedDBSyncStorage(workspace.doc.guid)
);
await syncPeer.waitForLoaded();
@@ -54,7 +54,7 @@ describe('SyncPeer', () => {
});
const syncPeer = new SyncPeer(
workspace.doc,
createIndexedDBStorage(workspace.doc.guid)
new IndexedDBSyncStorage(workspace.doc.guid)
);
await syncPeer.waitForSynced();
expect(workspace.doc.toJSON()).toEqual({
@@ -73,7 +73,7 @@ describe('SyncPeer', () => {
const syncPeer = new SyncPeer(
workspace.doc,
createIndexedDBStorage(workspace.doc.guid)
new IndexedDBSyncStorage(workspace.doc.guid)
);
expect(syncPeer.status.step).toBe(SyncPeerStep.LoadingRootDoc);
await syncPeer.waitForSynced();
@@ -1,4 +1,4 @@
import type { SyncStorage } from '@affine/workspace';
import type { SyncStorage } from '@toeverything/infra';
export function createTestStorage(origin: SyncStorage) {
const controler = {
@@ -1,4 +1,4 @@
import type { AwarenessProvider } from '@affine/workspace';
import type { AwarenessProvider } from '@toeverything/infra';
import type { Awareness } from 'y-protocols/awareness.js';
import {
applyAwarenessUpdate,
@@ -11,13 +11,35 @@ type ChannelMessage =
| { type: 'connect' }
| { type: 'update'; update: Uint8Array };
export function createBroadcastChannelAwarenessProvider(
workspaceId: string,
awareness: Awareness
): AwarenessProvider {
const channel = new BroadcastChannel('awareness:' + workspaceId);
export class BroadcastChannelAwarenessProvider implements AwarenessProvider {
channel: BroadcastChannel | null = null;
function handleAwarenessUpdate(changes: AwarenessChanges, origin: unknown) {
constructor(
private readonly workspaceId: string,
private readonly awareness: Awareness
) {}
connect(): void {
this.channel = new BroadcastChannel('awareness:' + this.workspaceId);
this.channel.postMessage({
type: 'connect',
} satisfies ChannelMessage);
this.awareness.on('update', (changes: AwarenessChanges, origin: unknown) =>
this.handleAwarenessUpdate(changes, origin)
);
this.channel.addEventListener(
'message',
(event: MessageEvent<ChannelMessage>) => {
this.handleChannelMessage(event);
}
);
}
disconnect(): void {
this.channel?.close();
this.channel = null;
}
handleAwarenessUpdate(changes: AwarenessChanges, origin: unknown) {
if (origin === 'remote') {
return;
}
@@ -26,37 +48,25 @@ export function createBroadcastChannelAwarenessProvider(
res.concat(cur)
);
const update = encodeAwarenessUpdate(awareness, changedClients);
channel.postMessage({
const update = encodeAwarenessUpdate(this.awareness, changedClients);
this.channel?.postMessage({
type: 'update',
update: update,
} satisfies ChannelMessage);
}
function handleChannelMessage(event: MessageEvent<ChannelMessage>) {
handleChannelMessage(event: MessageEvent<ChannelMessage>) {
if (event.data.type === 'update') {
const update = event.data.update;
applyAwarenessUpdate(awareness, update, 'remote');
applyAwarenessUpdate(this.awareness, update, 'remote');
}
if (event.data.type === 'connect') {
channel.postMessage({
this.channel?.postMessage({
type: 'update',
update: encodeAwarenessUpdate(awareness, [awareness.clientID]),
update: encodeAwarenessUpdate(this.awareness, [
this.awareness.clientID,
]),
} satisfies ChannelMessage);
}
}
return {
connect() {
channel.postMessage({
type: 'connect',
} satisfies ChannelMessage);
awareness.on('update', handleAwarenessUpdate);
channel.addEventListener('message', handleChannelMessage);
},
disconnect() {
awareness.off('update', handleAwarenessUpdate);
channel.removeEventListener('message', handleChannelMessage);
},
};
}
@@ -1,34 +1,33 @@
import type { BlobStorage } from '@affine/workspace';
import { type BlobStorage } from '@toeverything/infra';
import { createStore, del, get, keys, set } from 'idb-keyval';
import { bufferToBlob } from '../utils/buffer-to-blob';
export const createIndexeddbBlobStorage = (
workspaceId: string
): BlobStorage => {
const db = createStore(`${workspaceId}_blob`, 'blob');
const mimeTypeDb = createStore(`${workspaceId}_blob_mime`, 'blob_mime');
return {
name: 'indexeddb',
readonly: false,
get: async (key: string) => {
const res = await get<ArrayBuffer>(key, db);
if (res) {
return bufferToBlob(res);
}
return null;
},
set: async (key: string, value: Blob) => {
await set(key, await value.arrayBuffer(), db);
await set(key, value.type, mimeTypeDb);
return key;
},
delete: async (key: string) => {
await del(key, db);
await del(key, mimeTypeDb);
},
list: async () => {
return keys<string>(db);
},
};
};
export class IndexedDBBlobStorage implements BlobStorage {
constructor(private readonly workspaceId: string) {}
name = 'indexeddb';
readonly = false;
db = createStore(`${this.workspaceId}_blob`, 'blob');
mimeTypeDb = createStore(`${this.workspaceId}_blob_mime`, 'blob_mime');
async get(key: string) {
const res = await get<ArrayBuffer>(key, this.db);
if (res) {
return bufferToBlob(res);
}
return null;
}
async set(key: string, value: Blob) {
await set(key, await value.arrayBuffer(), this.db);
await set(key, value.type, this.mimeTypeDb);
return key;
}
async delete(key: string) {
await del(key, this.db);
await del(key, this.mimeTypeDb);
}
async list() {
return keys<string>(this.db);
}
}
@@ -1,38 +1,36 @@
import { apis } from '@affine/electron-api';
import type { BlobStorage } from '@affine/workspace';
import { assertExists } from '@blocksuite/global/utils';
import { type BlobStorage } from '@toeverything/infra';
import { bufferToBlob } from '../utils/buffer-to-blob';
export const createSQLiteBlobStorage = (workspaceId: string): BlobStorage => {
assertExists(apis);
return {
name: 'sqlite',
readonly: false,
get: async (key: string) => {
assertExists(apis);
const buffer = await apis.db.getBlob(workspaceId, key);
if (buffer) {
return bufferToBlob(buffer);
}
return null;
},
set: async (key: string, value: Blob) => {
assertExists(apis);
await apis.db.addBlob(
workspaceId,
key,
new Uint8Array(await value.arrayBuffer())
);
return key;
},
delete: async (key: string) => {
assertExists(apis);
return apis.db.deleteBlob(workspaceId, key);
},
list: async () => {
assertExists(apis);
return apis.db.getBlobKeys(workspaceId);
},
};
};
export class SQLiteBlobStorage implements BlobStorage {
constructor(private readonly workspaceId: string) {}
name = 'sqlite';
readonly = false;
async get(key: string) {
assertExists(apis);
const buffer = await apis.db.getBlob(this.workspaceId, key);
if (buffer) {
return bufferToBlob(buffer);
}
return null;
}
async set(key: string, value: Blob) {
assertExists(apis);
await apis.db.addBlob(
this.workspaceId,
key,
new Uint8Array(await value.arrayBuffer())
);
return key;
}
delete(key: string) {
assertExists(apis);
return apis.db.deleteBlob(this.workspaceId, key);
}
list() {
assertExists(apis);
return apis.db.getBlobKeys(this.workspaceId);
}
}
@@ -1,4 +1,4 @@
import type { BlobStorage } from '@affine/workspace';
import { type BlobStorage } from '@toeverything/infra';
export const predefinedStaticFiles = [
'029uztLz2CzJezK7UUhrbGiWUdZ0J7NVs_qR6RDsvb8=',
@@ -36,37 +36,36 @@ export const predefinedStaticFiles = [
'v2yF7lY2L5rtorTtTmYFsoMb9dBPKs5M1y9cUKxcI1M=',
];
export const createStaticBlobStorage = (): BlobStorage => {
return {
name: 'static',
readonly: true,
get: async (key: string) => {
const isStaticResource =
predefinedStaticFiles.includes(key) || key.startsWith('/static/');
if (!isStaticResource) {
return null;
}
const path = key.startsWith('/static/') ? key : `/static/${key}`;
const response = await fetch(path);
if (response.ok) {
return await response.blob();
}
export class StaticBlobStorage implements BlobStorage {
name = 'static';
readonly = true;
async get(key: string) {
const isStaticResource =
predefinedStaticFiles.includes(key) || key.startsWith('/static/');
if (!isStaticResource) {
return null;
},
set: async key => {
// ignore
return key;
},
delete: async () => {
// ignore
},
list: async () => {
// ignore
return [];
},
};
};
}
const path = key.startsWith('/static/') ? key : `/static/${key}`;
const response = await fetch(path);
if (response.ok) {
return await response.blob();
}
return null;
}
async set(key: string) {
// ignore
return key;
}
async delete() {
// ignore
}
async list() {
// ignore
return [];
}
}
@@ -1,10 +0,0 @@
import { createIndexeddbBlobStorage } from './blob-indexeddb';
import { createSQLiteBlobStorage } from './blob-sqlite';
export function createLocalBlobStorage(workspaceId: string) {
if (environment.isDesktop) {
return createSQLiteBlobStorage(workspaceId);
} else {
return createIndexeddbBlobStorage(workspaceId);
}
}
@@ -1,11 +1,9 @@
export * from './awareness';
export * from './blob';
export * from './blob-indexeddb';
export * from './blob-sqlite';
export * from './blob-static';
export * from './consts';
export * from './list';
export * from './sync';
export * from './sync-indexeddb';
export * from './sync-sqlite';
export * from './workspace-factory';
+121 -100
View File
@@ -1,130 +1,151 @@
import { apis } from '@affine/electron-api';
import { WorkspaceFlavour } from '@affine/env/workspace';
import type { WorkspaceListProvider } from '@affine/workspace';
import { globalBlockSuiteSchema } from '@affine/workspace';
import { Workspace as BlockSuiteWorkspace } from '@blocksuite/store';
import type { WorkspaceListProvider } from '@toeverything/infra';
import {
type BlobStorage,
type WorkspaceInfo,
type WorkspaceMetadata,
} from '@toeverything/infra';
import { globalBlockSuiteSchema } from '@toeverything/infra';
import { difference } from 'lodash-es';
import { nanoid } from 'nanoid';
import { applyUpdate, encodeStateAsUpdate } from 'yjs';
import { createLocalBlobStorage } from './blob';
import { IndexedDBBlobStorage } from './blob-indexeddb';
import { SQLiteBlobStorage } from './blob-sqlite';
import {
LOCAL_WORKSPACE_CREATED_BROADCAST_CHANNEL_KEY,
LOCAL_WORKSPACE_LOCAL_STORAGE_KEY,
} from './consts';
import { createLocalStorage } from './sync';
import { IndexedDBSyncStorage } from './sync-indexeddb';
import { SQLiteSyncStorage } from './sync-sqlite';
export function createLocalWorkspaceListProvider(): WorkspaceListProvider {
const notifyChannel = new BroadcastChannel(
export class LocalWorkspaceListProvider implements WorkspaceListProvider {
name = WorkspaceFlavour.LOCAL;
notifyChannel = new BroadcastChannel(
LOCAL_WORKSPACE_CREATED_BROADCAST_CHANNEL_KEY
);
return {
name: WorkspaceFlavour.LOCAL,
getList() {
return Promise.resolve(
JSON.parse(
localStorage.getItem(LOCAL_WORKSPACE_LOCAL_STORAGE_KEY) ?? '[]'
).map((id: string) => ({ id, flavour: WorkspaceFlavour.LOCAL }))
);
},
subscribe(callback) {
let lastWorkspaceIDs: string[] = [];
async getList() {
return JSON.parse(
localStorage.getItem(LOCAL_WORKSPACE_LOCAL_STORAGE_KEY) ?? '[]'
).map((id: string) => ({ id, flavour: WorkspaceFlavour.LOCAL }));
}
function scan() {
const allWorkspaceIDs: string[] = JSON.parse(
localStorage.getItem(LOCAL_WORKSPACE_LOCAL_STORAGE_KEY) ?? '[]'
);
const added = difference(allWorkspaceIDs, lastWorkspaceIDs);
const deleted = difference(lastWorkspaceIDs, allWorkspaceIDs);
lastWorkspaceIDs = allWorkspaceIDs;
callback({
added: added.map(id => ({ id, flavour: WorkspaceFlavour.LOCAL })),
deleted: deleted.map(id => ({ id, flavour: WorkspaceFlavour.LOCAL })),
});
}
async delete(workspaceId: string) {
const allWorkspaceIDs: string[] = JSON.parse(
localStorage.getItem(LOCAL_WORKSPACE_LOCAL_STORAGE_KEY) ?? '[]'
);
localStorage.setItem(
LOCAL_WORKSPACE_LOCAL_STORAGE_KEY,
JSON.stringify(allWorkspaceIDs.filter(x => x !== workspaceId))
);
scan();
if (apis && environment.isDesktop) {
await apis.workspace.delete(workspaceId);
}
// rescan if other tabs notify us
notifyChannel.addEventListener('message', scan);
return () => {
notifyChannel.removeEventListener('message', scan);
};
},
async create(initial) {
const id = nanoid();
// notify all browser tabs, so they can update their workspace list
this.notifyChannel.postMessage(workspaceId);
}
const blobStorage = createLocalBlobStorage(id);
const syncStorage = createLocalStorage(id);
async create(
initial: (
workspace: BlockSuiteWorkspace,
blobStorage: BlobStorage
) => Promise<void>
): Promise<WorkspaceMetadata> {
const id = nanoid();
const workspace = new BlockSuiteWorkspace({
id: id,
idGenerator: () => nanoid(),
schema: globalBlockSuiteSchema,
});
const blobStorage = environment.isDesktop
? new SQLiteBlobStorage(id)
: new IndexedDBBlobStorage(id);
const syncStorage = environment.isDesktop
? new SQLiteSyncStorage(id)
: new IndexedDBSyncStorage(id);
// apply initial state
await initial(workspace, blobStorage);
const workspace = new BlockSuiteWorkspace({
id: id,
idGenerator: () => nanoid(),
schema: globalBlockSuiteSchema,
});
// save workspace to local storage
await syncStorage.push(id, encodeStateAsUpdate(workspace.doc));
for (const subdocs of workspace.doc.getSubdocs()) {
await syncStorage.push(subdocs.guid, encodeStateAsUpdate(subdocs));
}
// apply initial state
await initial(workspace, blobStorage);
// save workspace id to local storage
// save workspace to local storage
await syncStorage.push(id, encodeStateAsUpdate(workspace.doc));
for (const subdocs of workspace.doc.getSubdocs()) {
await syncStorage.push(subdocs.guid, encodeStateAsUpdate(subdocs));
}
// save workspace id to local storage
const allWorkspaceIDs: string[] = JSON.parse(
localStorage.getItem(LOCAL_WORKSPACE_LOCAL_STORAGE_KEY) ?? '[]'
);
allWorkspaceIDs.push(id);
localStorage.setItem(
LOCAL_WORKSPACE_LOCAL_STORAGE_KEY,
JSON.stringify(allWorkspaceIDs)
);
// notify all browser tabs, so they can update their workspace list
this.notifyChannel.postMessage(id);
return { id, flavour: WorkspaceFlavour.LOCAL };
}
subscribe(
callback: (changed: {
added?: WorkspaceMetadata[] | undefined;
deleted?: WorkspaceMetadata[] | undefined;
}) => void
): () => void {
let lastWorkspaceIDs: string[] = [];
function scan() {
const allWorkspaceIDs: string[] = JSON.parse(
localStorage.getItem(LOCAL_WORKSPACE_LOCAL_STORAGE_KEY) ?? '[]'
);
allWorkspaceIDs.push(id);
localStorage.setItem(
LOCAL_WORKSPACE_LOCAL_STORAGE_KEY,
JSON.stringify(allWorkspaceIDs)
);
// notify all browser tabs, so they can update their workspace list
notifyChannel.postMessage(id);
return id;
},
async delete(workspaceId) {
const allWorkspaceIDs: string[] = JSON.parse(
localStorage.getItem(LOCAL_WORKSPACE_LOCAL_STORAGE_KEY) ?? '[]'
);
localStorage.setItem(
LOCAL_WORKSPACE_LOCAL_STORAGE_KEY,
JSON.stringify(allWorkspaceIDs.filter(x => x !== workspaceId))
);
if (apis && environment.isDesktop) {
await apis.workspace.delete(workspaceId);
}
// notify all browser tabs, so they can update their workspace list
notifyChannel.postMessage(workspaceId);
},
async getInformation(id) {
// get information from root doc
const storage = createLocalStorage(id);
const data = await storage.pull(id, new Uint8Array([]));
if (!data) {
return;
}
const bs = new BlockSuiteWorkspace({
id,
schema: globalBlockSuiteSchema,
const added = difference(allWorkspaceIDs, lastWorkspaceIDs);
const deleted = difference(lastWorkspaceIDs, allWorkspaceIDs);
lastWorkspaceIDs = allWorkspaceIDs;
callback({
added: added.map(id => ({ id, flavour: WorkspaceFlavour.LOCAL })),
deleted: deleted.map(id => ({ id, flavour: WorkspaceFlavour.LOCAL })),
});
}
applyUpdate(bs.doc, data.data);
scan();
return {
name: bs.meta.name,
avatar: bs.meta.avatar,
};
},
};
// rescan if other tabs notify us
this.notifyChannel.addEventListener('message', scan);
return () => {
this.notifyChannel.removeEventListener('message', scan);
};
}
async getInformation(id: string): Promise<WorkspaceInfo | undefined> {
// get information from root doc
const storage = environment.isDesktop
? new SQLiteSyncStorage(id)
: new IndexedDBSyncStorage(id);
const data = await storage.pull(id, new Uint8Array([]));
if (!data) {
return;
}
const bs = new BlockSuiteWorkspace({
id,
schema: globalBlockSuiteSchema,
});
applyUpdate(bs.doc, data.data);
return {
name: bs.meta.name,
avatar: bs.meta.avatar,
};
}
}
@@ -1,9 +1,7 @@
import type { SyncStorage } from '@affine/workspace';
import { mergeUpdates, type SyncStorage } from '@toeverything/infra';
import { type DBSchema, type IDBPDatabase, openDB } from 'idb';
import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs';
import { mergeUpdates } from '../utils/merge-updates';
export const dbVersion = 1;
export const DEFAULT_DB_NAME = 'affine-local';
@@ -38,81 +36,83 @@ type ChannelMessage = {
payload: { docId: string; update: Uint8Array };
};
export function createIndexedDBStorage(
workspaceId: string,
dbName = DEFAULT_DB_NAME,
mergeCount = 1
): SyncStorage {
let dbPromise: Promise<IDBPDatabase<BlockSuiteBinaryDB>> | null = null;
const getDb = async () => {
if (dbPromise === null) {
dbPromise = openDB<BlockSuiteBinaryDB>(dbName, dbVersion, {
export class IndexedDBSyncStorage implements SyncStorage {
name = 'indexeddb';
dbName = DEFAULT_DB_NAME;
mergeCount = 1;
dbPromise: Promise<IDBPDatabase<BlockSuiteBinaryDB>> | null = null;
// indexeddb could be shared between tabs, so we use broadcast channel to notify other tabs
channel = new BroadcastChannel('indexeddb:' + this.workspaceId);
constructor(private readonly workspaceId: string) {}
getDb() {
if (this.dbPromise === null) {
this.dbPromise = openDB<BlockSuiteBinaryDB>(this.dbName, dbVersion, {
upgrade: upgradeDB,
});
}
return dbPromise;
};
return this.dbPromise;
}
// indexeddb could be shared between tabs, so we use broadcast channel to notify other tabs
const channel = new BroadcastChannel('indexeddb:' + workspaceId);
async pull(
docId: string,
state: Uint8Array
): Promise<{ data: Uint8Array; state?: Uint8Array | undefined } | null> {
const db = await this.getDb();
const store = db
.transaction('workspace', 'readonly')
.objectStore('workspace');
const data = await store.get(docId);
return {
name: 'indexeddb',
async pull(docId, state) {
const db = await getDb();
const store = db
.transaction('workspace', 'readonly')
.objectStore('workspace');
const data = await store.get(docId);
if (!data) {
return null;
}
if (!data) {
return null;
const { updates } = data;
const update = mergeUpdates(updates.map(({ update }) => update));
const diff = state.length ? diffUpdate(update, state) : update;
return { data: diff, state: encodeStateVectorFromUpdate(update) };
}
async push(docId: string, data: Uint8Array): Promise<void> {
const db = await this.getDb();
const store = db
.transaction('workspace', 'readwrite')
.objectStore('workspace');
// TODO: maybe we do not need to get data every time
const { updates } = (await store.get(docId)) ?? { updates: [] };
let rows: UpdateMessage[] = [
...updates,
{ timestamp: Date.now(), update: data },
];
if (this.mergeCount && rows.length >= this.mergeCount) {
const merged = mergeUpdates(rows.map(({ update }) => update));
rows = [{ timestamp: Date.now(), update: merged }];
}
await store.put({
id: docId,
updates: rows,
});
this.channel.postMessage({
type: 'db-updated',
payload: { docId, update: data },
} satisfies ChannelMessage);
}
async subscribe(cb: (docId: string, data: Uint8Array) => void) {
function onMessage(event: MessageEvent<ChannelMessage>) {
const { type, payload } = event.data;
if (type === 'db-updated') {
const { docId, update } = payload;
cb(docId, update);
}
const { updates } = data;
const update = mergeUpdates(updates.map(({ update }) => update));
const diff = state.length ? diffUpdate(update, state) : update;
return { data: diff, state: encodeStateVectorFromUpdate(update) };
},
async push(docId, update) {
const db = await getDb();
const store = db
.transaction('workspace', 'readwrite')
.objectStore('workspace');
// TODO: maybe we do not need to get data every time
const { updates } = (await store.get(docId)) ?? { updates: [] };
let rows: UpdateMessage[] = [
...updates,
{ timestamp: Date.now(), update },
];
if (mergeCount && rows.length >= mergeCount) {
const merged = mergeUpdates(rows.map(({ update }) => update));
rows = [{ timestamp: Date.now(), update: merged }];
}
await store.put({
id: docId,
updates: rows,
});
channel.postMessage({
type: 'db-updated',
payload: { docId, update },
} satisfies ChannelMessage);
},
async subscribe(cb, _disconnect) {
function onMessage(event: MessageEvent<ChannelMessage>) {
const { type, payload } = event.data;
if (type === 'db-updated') {
const { docId, update } = payload;
cb(docId, update);
}
}
channel.addEventListener('message', onMessage);
return () => {
channel.removeEventListener('message', onMessage);
};
},
};
}
this.channel.addEventListener('message', onMessage);
return () => {
this.channel.removeEventListener('message', onMessage);
};
}
}
@@ -1,44 +1,46 @@
import { apis } from '@affine/electron-api';
import type { SyncStorage } from '@affine/workspace';
import { type SyncStorage } from '@toeverything/infra';
import { encodeStateVectorFromUpdate } from 'yjs';
export function createSQLiteStorage(workspaceId: string): SyncStorage {
if (!apis?.db) {
throw new Error('sqlite datasource is not available');
export class SQLiteSyncStorage implements SyncStorage {
name = 'sqlite';
constructor(private readonly workspaceId: string) {
if (!apis?.db) {
throw new Error('sqlite datasource is not available');
}
}
return {
name: 'sqlite',
async pull(docId, _state) {
if (!apis?.db) {
throw new Error('sqlite datasource is not available');
}
const update = await apis.db.getDocAsUpdates(
workspaceId,
workspaceId === docId ? undefined : docId
);
async pull(docId: string, _state: Uint8Array) {
if (!apis?.db) {
throw new Error('sqlite datasource is not available');
}
const update = await apis.db.getDocAsUpdates(
this.workspaceId,
this.workspaceId === docId ? undefined : docId
);
if (update) {
return {
data: update,
state: encodeStateVectorFromUpdate(update),
};
}
if (update) {
return {
data: update,
state: encodeStateVectorFromUpdate(update),
};
}
return null;
},
async push(docId, data) {
if (!apis?.db) {
throw new Error('sqlite datasource is not available');
}
return apis.db.applyDocUpdate(
workspaceId,
data,
workspaceId === docId ? undefined : docId
);
},
async subscribe(_cb, _disconnect) {
return () => {};
},
};
return null;
}
async push(docId: string, data: Uint8Array) {
if (!apis?.db) {
throw new Error('sqlite datasource is not available');
}
return apis.db.applyDocUpdate(
this.workspaceId,
data,
this.workspaceId === docId ? undefined : docId
);
}
async subscribe() {
return () => {};
}
}
@@ -1,7 +0,0 @@
import { createIndexedDBStorage } from './sync-indexeddb';
import { createSQLiteStorage } from './sync-sqlite';
export const createLocalStorage = (workspaceId: string) =>
environment.isDesktop
? createSQLiteStorage(workspaceId)
: createIndexedDBStorage(workspaceId);
@@ -1,54 +1,50 @@
import { setupEditorFlags } from '@affine/env/global';
import type { WorkspaceFactory } from '@affine/workspace';
import { WorkspaceEngine } from '@affine/workspace';
import { BlobEngine } from '@affine/workspace';
import { SyncEngine } from '@affine/workspace';
import { globalBlockSuiteSchema } from '@affine/workspace';
import { Workspace } from '@affine/workspace';
import { Workspace as BlockSuiteWorkspace } from '@blocksuite/store';
import { nanoid } from 'nanoid';
import type { ServiceCollection, WorkspaceFactory } from '@toeverything/infra';
import {
AwarenessContext,
AwarenessProvider,
LocalBlobStorage,
LocalSyncStorage,
RemoteBlobStorage,
WorkspaceIdContext,
WorkspaceScope,
} from '@toeverything/infra';
import { createBroadcastChannelAwarenessProvider } from './awareness';
import { createLocalBlobStorage } from './blob';
import { createStaticBlobStorage } from './blob-static';
import { createLocalStorage } from './sync';
import { BroadcastChannelAwarenessProvider } from './awareness';
import { IndexedDBBlobStorage } from './blob-indexeddb';
import { SQLiteBlobStorage } from './blob-sqlite';
import { StaticBlobStorage } from './blob-static';
import { IndexedDBSyncStorage } from './sync-indexeddb';
import { SQLiteSyncStorage } from './sync-sqlite';
export const localWorkspaceFactory: WorkspaceFactory = {
name: 'local',
openWorkspace(metadata) {
const blobEngine = new BlobEngine(createLocalBlobStorage(metadata.id), [
createStaticBlobStorage(),
]);
const bs = new BlockSuiteWorkspace({
id: metadata.id,
blobStorages: [
() => ({
crud: blobEngine,
}),
],
idGenerator: () => nanoid(),
schema: globalBlockSuiteSchema,
});
const syncEngine = new SyncEngine(
bs.doc,
createLocalStorage(metadata.id),
[]
);
const awarenessProvider = createBroadcastChannelAwarenessProvider(
metadata.id,
bs.awarenessStore.awareness
);
const engine = new WorkspaceEngine(blobEngine, syncEngine, [
awarenessProvider,
]);
export class LocalWorkspaceFactory implements WorkspaceFactory {
name = 'local';
configureWorkspace(services: ServiceCollection): void {
if (environment.isDesktop) {
services
.scope(WorkspaceScope)
.addImpl(LocalBlobStorage, SQLiteBlobStorage, [WorkspaceIdContext])
.addImpl(LocalSyncStorage, SQLiteSyncStorage, [WorkspaceIdContext]);
} else {
services
.scope(WorkspaceScope)
.addImpl(LocalBlobStorage, IndexedDBBlobStorage, [WorkspaceIdContext])
.addImpl(LocalSyncStorage, IndexedDBSyncStorage, [WorkspaceIdContext]);
}
setupEditorFlags(bs);
return new Workspace(metadata, engine, bs);
},
async getWorkspaceBlob(id, blobKey) {
const blobStorage = createLocalBlobStorage(id);
services
.scope(WorkspaceScope)
.addImpl(RemoteBlobStorage('static'), StaticBlobStorage)
.addImpl(
AwarenessProvider('broadcast-channel'),
BroadcastChannelAwarenessProvider,
[WorkspaceIdContext, AwarenessContext]
);
}
async getWorkspaceBlob(id: string, blobKey: string): Promise<Blob | null> {
const blobStorage = environment.isDesktop
? new SQLiteBlobStorage(id)
: new IndexedDBBlobStorage(id);
return await blobStorage.get(blobKey);
},
};
}
}