feat(core): new worker workspace engine (#9257)

This commit is contained in:
EYHN
2025-01-17 00:22:18 +08:00
committed by GitHub
parent 7dc470e7ea
commit a2ffdb4047
219 changed files with 4267 additions and 7194 deletions
+54 -23
View File
@@ -12,23 +12,22 @@ import {
DefaultServerService,
ServersService,
ValidatorProvider,
WebSocketAuthProvider,
} from '@affine/core/modules/cloud';
import { DocsService } from '@affine/core/modules/doc';
import { GlobalContextService } from '@affine/core/modules/global-context';
import { I18nProvider } from '@affine/core/modules/i18n';
import { LifecycleService } from '@affine/core/modules/lifecycle';
import { configureLocalStorageStateStorageImpls } from '@affine/core/modules/storage';
import {
configureLocalStorageStateStorageImpls,
NbstoreProvider,
} from '@affine/core/modules/storage';
import { PopupWindowProvider } from '@affine/core/modules/url';
import { ClientSchemeProvider } from '@affine/core/modules/url/providers/client-schema';
import { configureIndexedDBUserspaceStorageProvider } from '@affine/core/modules/userspace';
import { configureBrowserWorkbenchModule } from '@affine/core/modules/workbench';
import { WorkspacesService } from '@affine/core/modules/workspace';
import {
configureBrowserWorkspaceFlavours,
configureIndexedDBWorkspaceEngineStorageProvider,
} from '@affine/core/modules/workspace-engine';
import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine';
import { I18n } from '@affine/i18n';
import { WorkerClient } from '@affine/nbstore/worker/client';
import {
defaultBlockMarkdownAdapterMatchers,
docLinkBaseURLMiddleware,
@@ -44,16 +43,17 @@ import { Browser } from '@capacitor/browser';
import { Haptics } from '@capacitor/haptics';
import { Keyboard, KeyboardStyle } from '@capacitor/keyboard';
import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra';
import { OpClient } from '@toeverything/infra/op';
import { AsyncCall } from 'async-call-rpc';
import { useTheme } from 'next-themes';
import { Suspense, useEffect } from 'react';
import { RouterProvider } from 'react-router-dom';
import { BlocksuiteMenuConfigProvider } from './bs-menu-config';
import { configureFetchProvider } from './fetch';
import { ModalConfigProvider } from './modal-config';
import { Cookie } from './plugins/cookie';
import { Hashcash } from './plugins/hashcash';
import { Intelligents } from './plugins/intelligents';
import { NbStoreNativeDBApis } from './plugins/nbstore';
import { enableNavigationGesture$ } from './web-navigation-control';
const future = {
@@ -65,9 +65,52 @@ configureCommonModules(framework);
configureBrowserWorkbenchModule(framework);
configureLocalStorageStateStorageImpls(framework);
configureBrowserWorkspaceFlavours(framework);
configureIndexedDBWorkspaceEngineStorageProvider(framework);
configureIndexedDBUserspaceStorageProvider(framework);
configureMobileModules(framework);
framework.impl(NbstoreProvider, {
openStore(_key, options) {
const worker = new Worker(
new URL(
/* webpackChunkName: "nbstore-worker" */ './worker.ts',
import.meta.url
)
);
const { port1: nativeDBApiChannelServer, port2: nativeDBApiChannelClient } =
new MessageChannel();
AsyncCall<typeof NbStoreNativeDBApis>(NbStoreNativeDBApis, {
channel: {
on(listener) {
const f = (e: MessageEvent<any>) => {
listener(e.data);
};
nativeDBApiChannelServer.addEventListener('message', f);
return () => {
nativeDBApiChannelServer.removeEventListener('message', f);
};
},
send(data) {
nativeDBApiChannelServer.postMessage(data);
},
},
log: false,
});
nativeDBApiChannelServer.start();
worker.postMessage(
{
type: 'native-db-api-channel',
port: nativeDBApiChannelClient,
},
[nativeDBApiChannelClient]
);
const client = new WorkerClient(new OpClient(worker), options);
return {
store: client,
dispose: () => {
worker.terminate();
nativeDBApiChannelServer.close();
},
};
},
});
framework.impl(PopupWindowProvider, {
open: (url: string) => {
Browser.open({
@@ -81,18 +124,6 @@ framework.impl(ClientSchemeProvider, {
return 'affine';
},
});
configureFetchProvider(framework);
framework.impl(WebSocketAuthProvider, {
getAuthToken: async url => {
const cookies = await Cookie.getCookies({
url,
});
return {
userId: cookies['affine_user_id'],
token: cookies['affine_session'],
};
},
});
framework.impl(ValidatorProvider, {
async validate(_challenge, resource) {
const res = await Hashcash.hash({ challenge: resource });
-191
View File
@@ -1,191 +0,0 @@
/**
* this file is modified from part of https://github.com/ionic-team/capacitor/blob/74c3e9447e1e32e73f818d252eb12f453d849e8d/ios/Capacitor/Capacitor/assets/native-bridge.js#L466
*
* for support arraybuffer response type
*/
import { RawFetchProvider } from '@affine/core/modules/cloud/provider/fetch';
import { CapacitorHttp } from '@capacitor/core';
import type { Framework } from '@toeverything/infra';
const readFileAsBase64 = (file: File) =>
new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onloadend = () => {
const data = reader.result;
if (data === null) {
reject(new Error('Failed to read file'));
} else {
resolve(btoa(data as string));
}
};
reader.onerror = reject;
reader.readAsBinaryString(file);
});
const convertFormData = async (formData: FormData) => {
const newFormData = [];
for (const pair of formData.entries()) {
const [key, value] = pair;
if (value instanceof File) {
const base64File = await readFileAsBase64(value);
newFormData.push({
key,
value: base64File,
type: 'base64File',
contentType: value.type,
fileName: value.name,
});
} else {
newFormData.push({ key, value, type: 'string' });
}
}
return newFormData;
};
const convertBody = async (body: unknown, contentType: string) => {
if (body instanceof ReadableStream || body instanceof Uint8Array) {
let encodedData;
if (body instanceof ReadableStream) {
const reader = body.getReader();
const chunks = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
const concatenated = new Uint8Array(
chunks.reduce((acc, chunk) => acc + chunk.length, 0)
);
let position = 0;
for (const chunk of chunks) {
concatenated.set(chunk, position);
position += chunk.length;
}
encodedData = concatenated;
} else {
encodedData = body;
}
let data = new TextDecoder().decode(encodedData);
let type;
if (contentType === 'application/json') {
try {
data = JSON.parse(data);
} catch {
// ignore
}
type = 'json';
} else if (contentType === 'multipart/form-data') {
type = 'formData';
} else if (
contentType === null || contentType === void 0
? void 0
: contentType.startsWith('image')
) {
type = 'image';
} else if (contentType === 'application/octet-stream') {
type = 'binary';
} else {
type = 'text';
}
return {
data,
type,
headers: { 'Content-Type': contentType || 'application/octet-stream' },
};
} else if (body instanceof URLSearchParams) {
return {
data: body.toString(),
type: 'text',
};
} else if (body instanceof FormData) {
const formData = await convertFormData(body);
return {
data: formData,
type: 'formData',
};
} else if (body instanceof File) {
const fileData = await readFileAsBase64(body);
return {
data: fileData,
type: 'file',
headers: { 'Content-Type': body.type },
};
}
return { data: body, type: 'json' };
};
function base64ToUint8Array(base64: string) {
const binaryString = atob(base64);
const binaryArray = [...binaryString].map(function (char) {
return char.charCodeAt(0);
});
return new Uint8Array(binaryArray);
}
export function configureFetchProvider(framework: Framework) {
framework.override(RawFetchProvider, {
fetch: async (input, init) => {
const request = new Request(input, init);
const { method } = request;
const tag = `CapacitorHttp fetch ${Date.now()} ${input}`;
console.time(tag);
try {
const { body } = request;
const optionHeaders = Object.fromEntries(request.headers.entries());
const {
data: requestData,
type,
headers,
} = await convertBody(
(init === null || init === void 0 ? void 0 : init.body) ||
body ||
undefined,
optionHeaders['Content-Type'] || optionHeaders['content-type']
);
const accept = optionHeaders['Accept'] || optionHeaders['accept'];
const nativeResponse = await CapacitorHttp.request({
url: request.url,
method: method,
data: requestData,
dataType: type as any,
responseType:
accept === 'application/octet-stream' ? 'arraybuffer' : undefined,
headers: Object.assign(Object.assign({}, headers), optionHeaders),
});
const contentType =
nativeResponse.headers['Content-Type'] ||
nativeResponse.headers['content-type'];
let data =
accept === 'application/octet-stream'
? base64ToUint8Array(nativeResponse.data)
: contentType === null || contentType === void 0
? void 0
: contentType.startsWith('application/json')
? JSON.stringify(nativeResponse.data)
: contentType === 'application/octet-stream'
? base64ToUint8Array(nativeResponse.data)
: nativeResponse.data;
// use null data for 204 No Content HTTP response
if (nativeResponse.status === 204) {
data = null;
}
// intercept & parse response before returning
const response = new Response(new Blob([data], { type: contentType }), {
headers: nativeResponse.headers,
status: nativeResponse.status,
});
/*
* copy url to response, `cordova-plugin-ionic` uses this url from the response
* we need `Object.defineProperty` because url is an inherited getter on the Response
* see: https://stackoverflow.com/a/57382543
* */
Object.defineProperty(response, 'url', {
value: nativeResponse.url,
});
console.timeEnd(tag);
return response;
} catch (error) {
console.timeEnd(tag);
throw error;
}
},
});
}
+9 -9
View File
@@ -1,5 +1,8 @@
import './setup';
import '@affine/component/theme';
import '@affine/core/mobile/styles/mobile.css';
import { bindNativeDBApis } from '@affine/nbstore/sqlite';
import {
init,
reactRouterV6BrowserTracingIntegration,
@@ -15,18 +18,15 @@ import {
} from 'react-router-dom';
import { App } from './app';
import { NbStoreNativeDBApis } from './plugins/nbstore';
bindNativeDBApis(NbStoreNativeDBApis);
// TODO(@L-Sun) Uncomment this when the `show` method implement by `@capacitor/keyboard` in ios
// import './virtual-keyboard';
function main() {
if (BUILD_CONFIG.debug || window.SENTRY_RELEASE) {
// workaround for Capacitor HttpPlugin
// capacitor-http-plugin will replace window.XMLHttpRequest with its own implementation
// but XMLHttpRequest.prototype is not defined which is used by sentry
// see: https://github.com/ionic-team/capacitor/blob/74c3e9447e1e32e73f818d252eb12f453d849e8d/core/native-bridge.ts#L581
if ('CapacitorWebXMLHttpRequest' in window) {
window.XMLHttpRequest.prototype = (
window.CapacitorWebXMLHttpRequest as any
).prototype;
}
// https://docs.sentry.io/platforms/javascript/guides/react/#configure
init({
dsn: process.env.SENTRY_DSN,
@@ -1,6 +0,0 @@
export interface CookiePlugin {
/**
* Returns the screen's current orientation.
*/
getCookies(options: { url: string }): Promise<Record<string, string>>;
}
@@ -1,8 +0,0 @@
import { registerPlugin } from '@capacitor/core';
import type { CookiePlugin } from './definitions';
const Cookie = registerPlugin<CookiePlugin>('Cookie');
export * from './definitions';
export { Cookie };
@@ -70,12 +70,12 @@ export interface NbStorePlugin {
timestamps: number[];
}) => Promise<{ count: number }>;
deleteDoc: (options: { id: string; docId: string }) => Promise<void>;
getDocClocks: (options: { id: string; after?: number | null }) => Promise<
{
getDocClocks: (options: { id: string; after?: number | null }) => Promise<{
clocks: {
docId: string;
timestamp: number;
}[]
>;
}[];
}>;
getDocClock: (options: { id: string; docId: string }) => Promise<
| {
docId: string;
@@ -95,47 +95,47 @@ export interface NbStorePlugin {
getPeerRemoteClocks: (options: {
id: string;
peer: string;
}) => Promise<Array<DocClock>>;
}) => Promise<{ clocks: Array<DocClock> }>;
getPeerRemoteClock: (options: {
id: string;
peer: string;
docId: string;
}) => Promise<DocClock>;
}) => Promise<DocClock | null>;
setPeerRemoteClock: (options: {
id: string;
peer: string;
docId: string;
clock: number;
timestamp: number;
}) => Promise<void>;
getPeerPushedClocks: (options: {
id: string;
peer: string;
}) => Promise<Array<DocClock>>;
}) => Promise<{ clocks: Array<DocClock> }>;
getPeerPushedClock: (options: {
id: string;
peer: string;
docId: string;
}) => Promise<DocClock>;
}) => Promise<DocClock | null>;
setPeerPushedClock: (options: {
id: string;
peer: string;
docId: string;
clock: number;
timestamp: number;
}) => Promise<void>;
getPeerPulledRemoteClocks: (options: {
id: string;
peer: string;
}) => Promise<Array<DocClock>>;
}) => Promise<{ clocks: Array<DocClock> }>;
getPeerPulledRemoteClock: (options: {
id: string;
peer: string;
docId: string;
}) => Promise<DocClock>;
}) => Promise<DocClock | null>;
setPeerPulledRemoteClock: (options: {
id: string;
peer: string;
docId: string;
clock: number;
timestamp: number;
}) => Promise<void>;
clearClocks: (options: { id: string }) => Promise<void>;
}
@@ -96,10 +96,12 @@ export const NbStoreNativeDBApis: NativeDBApis = {
id: string,
after?: Date | undefined | null
): Promise<DocClock[]> {
const clocks = await NbStore.getDocClocks({
id,
after: after?.getTime(),
});
const clocks = (
await NbStore.getDocClocks({
id,
after: after?.getTime(),
})
).clocks;
return clocks.map(c => ({
docId: c.docId,
timestamp: new Date(c.timestamp),
@@ -176,30 +178,30 @@ export const NbStoreNativeDBApis: NativeDBApis = {
id: string,
peer: string
): Promise<DocClock[]> {
const clocks = await NbStore.getPeerRemoteClocks({
id,
peer,
});
const clocks = (
await NbStore.getPeerRemoteClocks({
id,
peer,
})
).clocks;
return clocks.map(c => ({
docId: c.docId,
timestamp: new Date(c.timestamp),
}));
},
getPeerRemoteClock: async function (
id: string,
peer: string,
docId: string
): Promise<DocClock> {
getPeerRemoteClock: async function (id: string, peer: string, docId: string) {
const clock = await NbStore.getPeerRemoteClock({
id,
peer,
docId,
});
return {
docId: clock.docId,
timestamp: new Date(clock.timestamp),
};
return clock
? {
docId: clock.docId,
timestamp: new Date(clock.timestamp),
}
: null;
},
setPeerRemoteClock: async function (
id: string,
@@ -211,17 +213,19 @@ export const NbStoreNativeDBApis: NativeDBApis = {
id,
peer,
docId,
clock: clock.getTime(),
timestamp: clock.getTime(),
});
},
getPeerPulledRemoteClocks: async function (
id: string,
peer: string
): Promise<DocClock[]> {
const clocks = await NbStore.getPeerPulledRemoteClocks({
id,
peer,
});
const clocks = (
await NbStore.getPeerPulledRemoteClocks({
id,
peer,
})
).clocks;
return clocks.map(c => ({
docId: c.docId,
timestamp: new Date(c.timestamp),
@@ -231,16 +235,18 @@ export const NbStoreNativeDBApis: NativeDBApis = {
id: string,
peer: string,
docId: string
): Promise<DocClock> {
) {
const clock = await NbStore.getPeerPulledRemoteClock({
id,
peer,
docId,
});
return {
docId: clock.docId,
timestamp: new Date(clock.timestamp),
};
return clock
? {
docId: clock.docId,
timestamp: new Date(clock.timestamp),
}
: null;
},
setPeerPulledRemoteClock: async function (
id: string,
@@ -252,17 +258,19 @@ export const NbStoreNativeDBApis: NativeDBApis = {
id,
peer,
docId,
clock: clock.getTime(),
timestamp: clock.getTime(),
});
},
getPeerPushedClocks: async function (
id: string,
peer: string
): Promise<DocClock[]> {
const clocks = await NbStore.getPeerPushedClocks({
id,
peer,
});
const clocks = (
await NbStore.getPeerPushedClocks({
id,
peer,
})
).clocks;
return clocks.map(c => ({
docId: c.docId,
timestamp: new Date(c.timestamp),
@@ -272,16 +280,18 @@ export const NbStoreNativeDBApis: NativeDBApis = {
id: string,
peer: string,
docId: string
): Promise<DocClock> {
): Promise<DocClock | null> {
const clock = await NbStore.getPeerPushedClock({
id,
peer,
docId,
});
return {
docId: clock.docId,
timestamp: new Date(clock.timestamp),
};
return clock
? {
docId: clock.docId,
timestamp: new Date(clock.timestamp),
}
: null;
},
setPeerPushedClock: async function (
id: string,
@@ -293,7 +303,7 @@ export const NbStoreNativeDBApis: NativeDBApis = {
id,
peer,
docId,
clock: clock.getTime(),
timestamp: clock.getTime(),
});
},
clearClocks: async function (id: string): Promise<void> {
+189 -4
View File
@@ -1,6 +1,191 @@
import '@affine/core/bootstrap/browser';
import '@affine/component/theme';
import '@affine/core/mobile/styles/mobile.css';
// TODO(@L-Sun) Uncomment this when the `show` method implement by `@capacitor/keyboard` in ios
// import './virtual-keyboard';
/**
* the below code includes the custom fetch and websocket implementation for ios webview.
* should be included in the entry file of the app or webworker.
*/
/*
* we override the browser's fetch function with our custom fetch function to
* overcome the restrictions of cross-domain and third-party cookies in ios webview.
*
* the custom fetch function will convert the request to `affine-http://` or `affine-https://`
* and send the request to the server.
*/
const rawFetch = globalThis.fetch;
globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit) => {
const url = new URL(
typeof input === 'string'
? input
: input instanceof URL
? input.toString()
: input.url,
globalThis.location.origin
);
if (url.protocol === 'capacitor:') {
return rawFetch(input, init);
}
if (url.protocol === 'http:') {
url.protocol = 'affine-http:';
}
if (url.protocol === 'https:') {
url.protocol = 'affine-https:';
}
return rawFetch(url, input instanceof Request ? input : init);
};
/**
* we create a custom websocket class to simulate the browser's websocket connection
* through the custom url scheme handler.
*
* to overcome the restrictions of cross-domain and third-party cookies in ios webview,
* the front-end opens a websocket connection and sends a message by sending a request
* to `affine-ws://` or `affine-wss://`.
*
* the scheme has two endpoints:
*
* `affine-ws:///open?uuid={uuid}&url={wsUrl}`: opens a websocket connection and returns
* the received data via the SSE protocol.
* If the front-end closes the http connection, the websocket connection will also be closed.
*
* `affine-ws:///send?uuid={uuid}`: sends the request body data to the websocket connection
* with the specified uuid.
*/
class WrappedWebSocket {
static CLOSED = WebSocket.CLOSED;
static CLOSING = WebSocket.CLOSING;
static CONNECTING = WebSocket.CONNECTING;
static OPEN = WebSocket.OPEN;
readonly isWss: boolean;
readonly uuid = crypto.randomUUID();
readyState: number = WebSocket.CONNECTING;
events: Record<string, ((event: any) => void)[]> = {};
onopen: ((event: any) => void) | undefined = undefined;
onclose: ((event: any) => void) | undefined = undefined;
onerror: ((event: any) => void) | undefined = undefined;
onmessage: ((event: any) => void) | undefined = undefined;
eventSource: EventSource;
constructor(
readonly url: string,
_protocols?: string | string[] // not supported yet
) {
const parsedUrl = new URL(url);
this.isWss = parsedUrl.protocol === 'wss:';
this.eventSource = new EventSource(
`${this.isWss ? 'affine-wss' : 'affine-ws'}:///open?uuid=${this.uuid}&url=${encodeURIComponent(this.url)}`
);
this.eventSource.addEventListener('open', () => {
this.emitOpen(new Event('open'));
});
this.eventSource.addEventListener('error', () => {
this.eventSource.close();
this.emitError(new Event('error'));
this.emitClose(new CloseEvent('close'));
});
this.eventSource.addEventListener('message', data => {
const decodedData = JSON.parse(data.data);
if (decodedData.type === 'message') {
this.emitMessage(
new MessageEvent('message', { data: decodedData.data })
);
}
});
}
send(data: string) {
rawFetch(
`${this.isWss ? 'affine-wss' : 'affine-ws'}:///send?uuid=${this.uuid}`,
{
method: 'POST',
headers: {
'Content-Type': 'text/plain',
},
body: data,
}
).catch(e => {
console.error('Failed to send message', e);
});
}
close() {
this.eventSource.close();
this.emitClose(new CloseEvent('close'));
}
addEventListener(type: string, listener: (event: any) => void) {
this.events[type] = this.events[type] || [];
this.events[type].push(listener);
}
removeEventListener(type: string, listener: (event: any) => void) {
this.events[type] = this.events[type] || [];
this.events[type] = this.events[type].filter(l => l !== listener);
}
private emitOpen(event: Event) {
this.readyState = WebSocket.OPEN;
this.events['open']?.forEach(listener => {
try {
listener(event);
} catch (e) {
console.error(e);
}
});
try {
this.onopen?.(event);
} catch (e) {
console.error(e);
}
}
private emitClose(event: CloseEvent) {
this.readyState = WebSocket.CLOSED;
this.events['close']?.forEach(listener => {
try {
listener(event);
} catch (e) {
console.error(e);
}
});
try {
this.onclose?.(event);
} catch (e) {
console.error(e);
}
}
private emitMessage(event: MessageEvent) {
this.events['message']?.forEach(listener => {
try {
listener(event);
} catch (e) {
console.error(e);
}
});
try {
this.onmessage?.(event);
} catch (e) {
console.error(e);
}
}
private emitError(event: Event) {
this.events['error']?.forEach(listener => {
try {
listener(event);
} catch (e) {
console.error(e);
}
});
try {
this.onerror?.(event);
} catch (e) {
console.error(e);
}
}
}
globalThis.WebSocket = WrappedWebSocket as any;
+52
View File
@@ -0,0 +1,52 @@
import './setup';
import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel';
import { cloudStorages } from '@affine/nbstore/cloud';
import {
bindNativeDBApis,
type NativeDBApis,
sqliteStorages,
} from '@affine/nbstore/sqlite';
import {
WorkerConsumer,
type WorkerOps,
} from '@affine/nbstore/worker/consumer';
import { type MessageCommunicapable, OpConsumer } from '@toeverything/infra/op';
import { AsyncCall } from 'async-call-rpc';
globalThis.addEventListener('message', e => {
if (e.data.type === 'native-db-api-channel') {
const port = e.ports[0] as MessagePort;
const rpc = AsyncCall<NativeDBApis>(
{},
{
channel: {
on(listener) {
const f = (e: MessageEvent<any>) => {
listener(e.data);
};
port.addEventListener('message', f);
return () => {
port.removeEventListener('message', f);
};
},
send(data) {
port.postMessage(data);
},
},
}
);
bindNativeDBApis(rpc);
port.start();
}
});
const consumer = new OpConsumer<WorkerOps>(globalThis as MessageCommunicapable);
const worker = new WorkerConsumer([
...sqliteStorages,
...broadcastChannelStorages,
...cloudStorages,
]);
worker.bindConsumer(consumer);