Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-19 02:56:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): new worker workspace engine (#9257)

Browse Source
This commit is contained in:
EYHN
2025-01-17 00:22:18 +08:00
committed by GitHub
parent 7dc470e7ea
commit a2ffdb4047
219 changed files with 4267 additions and 7194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/common/nbstore/src/impls/broadcast-channel/channel.ts
+2 -7
View File
@@ -15,12 +15,7 @@ export class BroadcastChannelConnection extends AutoReconnectConnection<Broadcas
return new BroadcastChannel(this.channelName);
}
override doDisconnect() {
this.close();
}
private close(error?: Error) {
this.maybeConnection?.close();
this.setStatus('closed', error);
override doDisconnect(channel: BroadcastChannel) {
channel.close();
}
}
packages/common/nbstore/src/impls/cloud/awareness.ts
+18 -21
View File
@@ -1,5 +1,3 @@
import type { SocketOptions } from 'socket.io-client';
import { share } from '../../connection';
import {
type AwarenessRecord,
@@ -13,7 +11,6 @@ import {
} from './socket';
interface CloudAwarenessStorageOptions {
socketOptions?: SocketOptions;
serverBaseUrl: string;
type: SpaceType;
id: string;
@@ -26,12 +23,7 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
super();
}
connection = share(
new SocketConnection(
`${this.options.serverBaseUrl}/`,
this.options.socketOptions
)
);
connection = share(new SocketConnection(`${this.options.serverBaseUrl}/`));
private get socket() {
return this.connection.inner;
@@ -52,9 +44,14 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
onUpdate: (update: AwarenessRecord, origin?: string) => void,
onCollect: () => Promise<AwarenessRecord | null>
): () => void {
// TODO: handle disconnect
// leave awareness
const leave = () => {
if (this.connection.status !== 'connected') return;
this.socket.off('space:collect-awareness', handleCollectAwareness);
this.socket.off(
'space:broadcast-awareness-update',
handleBroadcastAwarenessUpdate
);
this.socket.emit('space:leave-awareness', {
spaceType: this.options.type,
spaceId: this.options.id,
@@ -64,6 +61,11 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
// join awareness, and collect awareness from others
const joinAndCollect = async () => {
this.socket.on('space:collect-awareness', handleCollectAwareness);
this.socket.on(
'space:broadcast-awareness-update',
handleBroadcastAwarenessUpdate
);
await this.socket.emitWithAck('space:join-awareness', {
spaceType: this.options.type,
spaceId: this.options.id,
@@ -77,7 +79,11 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
});
};
joinAndCollect().catch(err => console.error('awareness join failed', err));
if (this.connection.status === 'connected') {
joinAndCollect().catch(err =>
console.error('awareness join failed', err)
);
}
const unsubscribeConnectionStatusChanged = this.connection.onStatusChanged(
status => {
@@ -141,18 +147,9 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
}
};
this.socket.on('space:collect-awareness', handleCollectAwareness);
this.socket.on(
'space:broadcast-awareness-update',
handleBroadcastAwarenessUpdate
);
return () => {
leave();
this.socket.off('space:collect-awareness', handleCollectAwareness);
this.socket.off(
'space:broadcast-awareness-update',
handleBroadcastAwarenessUpdate
);
unsubscribeConnectionStatusChanged();
};
}
packages/common/nbstore/src/impls/cloud/doc-static.ts
+19 -14
View File
@@ -45,23 +45,28 @@ export class StaticCloudDocStorage extends DocStorageBase<CloudDocStorageOptions
protected override async getDocSnapshot(
docId: string
): Promise<DocRecord | null> {
const arrayBuffer = await this.connection.fetchArrayBuffer(
`/api/workspaces/${this.spaceId}/docs/${docId}`,
{
priority: 'high',
headers: {
Accept: 'application/octet-stream', // this is necessary for ios native fetch to return arraybuffer
},
try {
const arrayBuffer = await this.connection.fetchArrayBuffer(
`/api/workspaces/${this.spaceId}/docs/${docId}`,
{
priority: 'high',
headers: {
Accept: 'application/octet-stream', // this is necessary for ios native fetch to return arraybuffer
},
}
);
if (!arrayBuffer) {
return null;
}
);
if (!arrayBuffer) {
return {
docId: docId,
bin: new Uint8Array(arrayBuffer),
timestamp: new Date(),
};
} catch (error) {
console.error(error);
return null;
}
return {
docId: docId,
bin: new Uint8Array(arrayBuffer),
timestamp: new Date(),
};
}
protected override setDocSnapshot(
_snapshot: DocRecord,
packages/common/nbstore/src/impls/cloud/doc.ts
+67 -70
View File
@@ -1,10 +1,5 @@
import type { Socket, SocketOptions } from 'socket.io-client';
import type { Socket } from 'socket.io-client';
import {
type Connection,
type ConnectionStatus,
share,
} from '../../connection';
import {
type DocClock,
type DocClocks,
@@ -12,6 +7,7 @@ import {
type DocStorageOptions,
type DocUpdate,
} from '../../storage';
import { getIdConverter, type IdConverter } from '../../utils/id-converter';
import type { SpaceType } from '../../utils/universal-id';
import {
base64ToUint8Array,
@@ -21,7 +17,6 @@ import {
} from './socket';
interface CloudDocStorageOptions extends DocStorageOptions {
socketOptions?: SocketOptions;
serverBaseUrl: string;
type: SpaceType;
}
@@ -32,7 +27,12 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
get socket() {
return this.connection.inner;
}
get idConverter() {
if (!this.connection.idConverter) {
throw new Error('Id converter not initialized');
}
return this.connection.idConverter;
}
readonly spaceType = this.options.type;
onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => {
@@ -41,7 +41,7 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
this.spaceId === message.spaceId
) {
this.emit('update', {
docId: message.docId,
docId: this.idConverter.oldIdToNewId(message.docId),
bin: base64ToUint8Array(message.update),
timestamp: new Date(message.timestamp),
editor: message.editor,
@@ -58,10 +58,13 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
const response = await this.socket.emitWithAck('space:load-doc', {
spaceType: this.spaceType,
spaceId: this.spaceId,
docId,
docId: this.idConverter.newIdToOldId(docId),
});
if ('error' in response) {
if (response.error.name === 'DOC_NOT_FOUND') {
return null;
}
// TODO: use [UserFriendlyError]
throw new Error(response.error.message);
}
@@ -77,11 +80,14 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
const response = await this.socket.emitWithAck('space:load-doc', {
spaceType: this.spaceType,
spaceId: this.spaceId,
docId,
docId: this.idConverter.newIdToOldId(docId),
stateVector: state ? await uint8ArrayToBase64(state) : void 0,
});
if ('error' in response) {
if (response.error.name === 'DOC_NOT_FOUND') {
return null;
}
// TODO: use [UserFriendlyError]
throw new Error(response.error.message);
}
@@ -98,8 +104,8 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
const response = await this.socket.emitWithAck('space:push-doc-update', {
spaceType: this.spaceType,
spaceId: this.spaceId,
docId: update.docId,
updates: await uint8ArrayToBase64(update.bin),
docId: this.idConverter.newIdToOldId(update.docId),
update: await uint8ArrayToBase64(update.bin),
});
if ('error' in response) {
@@ -120,7 +126,7 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
const response = await this.socket.emitWithAck('space:load-doc', {
spaceType: this.spaceType,
spaceId: this.spaceId,
docId,
docId: this.idConverter.newIdToOldId(docId),
});
if ('error' in response) {
@@ -150,7 +156,7 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
}
return Object.entries(response.data).reduce((ret, [docId, timestamp]) => {
ret[docId] = new Date(timestamp);
ret[this.idConverter.oldIdToNewId(docId)] = new Date(timestamp);
return ret;
}, {} as DocClocks);
}
@@ -159,7 +165,7 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
this.socket.emit('space:delete-doc', {
spaceType: this.spaceType,
spaceId: this.spaceId,
docId,
docId: this.idConverter.newIdToOldId(docId),
});
}
@@ -174,83 +180,74 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
}
}
class CloudDocStorageConnection implements Connection<Socket> {
connection = share(
new SocketConnection(
`${this.options.serverBaseUrl}/`,
this.options.socketOptions
)
);
private disposeConnectionStatusListener?: () => void;
private get socket() {
return this.connection.inner;
}
class CloudDocStorageConnection extends SocketConnection {
constructor(
private readonly options: CloudDocStorageOptions,
private readonly onServerUpdate: ServerEventsMap['space:broadcast-doc-update']
) {}
get status() {
return this.connection.status;
) {
super(`${options.serverBaseUrl}/`);
}
get inner() {
return this.connection.inner;
}
idConverter: IdConverter | null = null;
connect(): void {
if (!this.disposeConnectionStatusListener) {
this.disposeConnectionStatusListener = this.connection.onStatusChanged(
status => {
if (status === 'connected') {
this.join().catch(err => {
console.error('doc storage join failed', err);
});
this.socket.on('space:broadcast-doc-update', this.onServerUpdate);
}
}
);
}
return this.connection.connect();
}
override async doConnect(signal?: AbortSignal) {
const socket = await super.doConnect(signal);
async join() {
try {
const res = await this.socket.emitWithAck('space:join', {
const res = await socket.emitWithAck('space:join', {
spaceType: this.options.type,
spaceId: this.options.id,
clientVersion: BUILD_CONFIG.appVersion,
});
if ('error' in res) {
this.connection.setStatus('closed', new Error(res.error.message));
throw new Error(res.error.message);
}
if (!this.idConverter) {
this.idConverter = await this.getIdConverter(socket);
}
socket.on('space:broadcast-doc-update', this.onServerUpdate);
return socket;
} catch (e) {
this.connection.setStatus('error', e as Error);
socket.close();
throw e;
}
}
disconnect() {
if (this.disposeConnectionStatusListener) {
this.disposeConnectionStatusListener();
}
this.socket.emit('space:leave', {
override doDisconnect(socket: Socket) {
socket.emit('space:leave', {
spaceType: this.options.type,
spaceId: this.options.id,
});
this.socket.off('space:broadcast-doc-update', this.onServerUpdate);
this.connection.disconnect();
socket.off('space:broadcast-doc-update', this.onServerUpdate);
super.disconnect();
}
waitForConnected(signal?: AbortSignal): Promise<void> {
return this.connection.waitForConnected(signal);
}
onStatusChanged(
cb: (status: ConnectionStatus, error?: Error) => void
): () => void {
return this.connection.onStatusChanged(cb);
async getIdConverter(socket: Socket) {
return getIdConverter(
{
getDocBuffer: async id => {
const response = await socket.emitWithAck('space:load-doc', {
spaceType: this.options.type,
spaceId: this.options.id,
docId: id,
});
if ('error' in response) {
if (response.error.name === 'DOC_NOT_FOUND') {
return null;
}
// TODO: use [UserFriendlyError]
throw new Error(response.error.message);
}
return base64ToUint8Array(response.data.missing);
},
},
this.options.id
);
}
}
packages/common/nbstore/src/impls/cloud/http.ts
+6 -2
View File
@@ -23,6 +23,7 @@ export class HttpConnection extends DummyConnection {
...init,
signal: abortController.signal,
headers: {
...this.requestHeaders,
...init?.headers,
'x-affine-version': BUILD_CONFIG.appVersion,
},
@@ -35,7 +36,7 @@ export class HttpConnection extends DummyConnection {
let reason: string | any = '';
if (res.headers.get('Content-Type')?.includes('application/json')) {
try {
reason = await res.json();
reason = JSON.stringify(await res.json());
} catch {
// ignore
}
@@ -63,7 +64,10 @@ export class HttpConnection extends DummyConnection {
this.fetch
);
constructor(private readonly serverBaseUrl: string) {
constructor(
private readonly serverBaseUrl: string,
private readonly requestHeaders?: Record<string, string>
) {
super();
}
}
packages/common/nbstore/src/impls/cloud/socket.ts
+50 -30
View File
@@ -4,10 +4,8 @@ import {
type SocketOptions,
} from 'socket.io-client';
import {
AutoReconnectConnection,
type ConnectionStatus,
} from '../../connection';
import { AutoReconnectConnection } from '../../connection';
import { throwIfAborted } from '../../utils/throw-if-aborted';
// TODO(@forehalo): use [UserFriendlyError]
interface EventError {
@@ -82,7 +80,7 @@ interface ClientEvents {
};
'space:push-doc-update': [
{ spaceType: string; spaceId: string; docId: string; updates: string },
{ spaceType: string; spaceId: string; docId: string; update: string },
{ timestamp: number },
];
'space:load-doc-timestamps': [
@@ -153,12 +151,24 @@ export function base64ToUint8Array(base64: string) {
return new Uint8Array(binaryArray);
}
const SOCKET_IOMANAGER_CACHE = new Map<string, SocketIOManager>();
function getSocketIOManager(endpoint: string) {
let manager = SOCKET_IOMANAGER_CACHE.get(endpoint);
if (!manager) {
manager = new SocketIOManager(endpoint, {
autoConnect: false,
transports: ['websocket'],
secure: new URL(endpoint).protocol === 'https:',
// we will handle reconnection by ourselves
reconnection: false,
});
SOCKET_IOMANAGER_CACHE.set(endpoint, manager);
}
return manager;
}
export class SocketConnection extends AutoReconnectConnection<Socket> {
manager = new SocketIOManager(this.endpoint, {
autoConnect: false,
transports: ['websocket'],
secure: new URL(this.endpoint).protocol === 'https:',
});
manager = getSocketIOManager(this.endpoint);
constructor(
private readonly endpoint: string,
@@ -171,32 +181,42 @@ export class SocketConnection extends AutoReconnectConnection<Socket> {
return `socket:${this.endpoint}`;
}
override async doConnect() {
const conn = this.manager.socket('/', this.socketOptions);
override async doConnect(signal?: AbortSignal) {
const socket = this.manager.socket('/', this.socketOptions);
try {
throwIfAborted(signal);
await Promise.race([
new Promise<void>((resolve, reject) => {
socket.once('connect', () => {
resolve();
});
socket.once('connect_error', err => {
reject(err);
});
socket.open();
}),
new Promise<void>((_resolve, reject) => {
signal?.addEventListener('abort', () => {
reject(new Error('Aborted'));
});
}),
]);
} catch (err) {
socket.close();
throw err;
}
await new Promise<void>((resolve, reject) => {
conn.once('connect', () => {
resolve();
});
conn.once('connect_error', err => {
reject(err);
});
conn.open();
});
socket.on('disconnect', this.handleDisconnect);
return conn;
return socket;
}
override doDisconnect(conn: Socket) {
conn.off('disconnect', this.handleDisconnect);
conn.close();
}
/**
* Socket connection allow explicitly set status by user
*
* used when join space failed
*/
override setStatus(status: ConnectionStatus, error?: Error) {
super.setStatus(status, error);
}
handleDisconnect = (reason: SocketIO.DisconnectReason) => {
this.error = new Error(reason);
};
}
packages/common/nbstore/src/impls/idb/db.ts
+19 -15
View File
@@ -25,22 +25,14 @@ export class IDBConnection extends AutoReconnectConnection<{
}
override async doConnect() {
// indexeddb will responsible for version control, so the db.version always match migrator.version
const db = await openDB<DocStorageSchema>(this.dbName, migrator.version, {
upgrade: migrator.migrate,
});
db.addEventListener('versionchange', this.handleVersionChange);
return {
db: await openDB<DocStorageSchema>(this.dbName, migrator.version, {
upgrade: migrator.migrate,
blocking: () => {
// if, for example, an tab with newer version is opened, this function will be called.
// we should close current connection to allow the new version to upgrade the db.
this.setStatus(
'closed',
new Error('Blocking a new version. Closing the connection.')
);
},
blocked: () => {
// fallback to retry auto retry
this.setStatus('error', new Error('Blocked by other tabs.'));
},
}),
db,
channel: new BroadcastChannel('idb:' + this.dbName),
};
}
@@ -49,7 +41,19 @@ export class IDBConnection extends AutoReconnectConnection<{
db: IDBPDatabase<DocStorageSchema>;
channel: BroadcastChannel;
}) {
db.db.removeEventListener('versionchange', this.handleVersionChange);
db.channel.close();
db.db.close();
}
handleVersionChange = (e: IDBVersionChangeEvent) => {
if (e.newVersion !== migrator.version) {
this.error = new Error(
'Database version mismatch, expected ' +
migrator.version +
' but got ' +
e.newVersion
);
}
};
}
packages/common/nbstore/src/impls/idb/doc.ts
+29 -20
View File
@@ -29,26 +29,35 @@ export class IndexedDBDocStorage extends DocStorageBase<IDBConnectionOptions> {
override locker = new IndexedDBLocker(this.connection);
private _lastTimestamp = new Date(0);
private generateTimestamp() {
const timestamp = new Date();
if (timestamp.getTime() <= this._lastTimestamp.getTime()) {
timestamp.setTime(this._lastTimestamp.getTime() + 1);
}
this._lastTimestamp = timestamp;
return timestamp;
}
override async pushDocUpdate(update: DocUpdate, origin?: string) {
const trx = this.db.transaction(['updates', 'clocks'], 'readwrite');
const timestamp = this.generateTimestamp();
await trx.objectStore('updates').add({
...update,
createdAt: timestamp,
});
let timestamp = new Date();
await trx.objectStore('clocks').put({ docId: update.docId, timestamp });
let retry = 0;
while (true) {
try {
const trx = this.db.transaction(['updates', 'clocks'], 'readwrite');
await trx.objectStore('updates').add({
...update,
createdAt: timestamp,
});
await trx.objectStore('clocks').put({ docId: update.docId, timestamp });
trx.commit();
} catch (e) {
if (e instanceof Error && e.name === 'ConstraintError') {
retry++;
if (retry < 10) {
timestamp = new Date(timestamp.getTime() + 1);
continue;
}
}
throw e;
}
break;
}
this.emit(
'update',
@@ -191,9 +200,9 @@ export class IndexedDBDocStorage extends DocStorageBase<IDBConnectionOptions> {
};
}
handleChannelMessage(event: MessageEvent<ChannelMessage>) {
handleChannelMessage = (event: MessageEvent<ChannelMessage>) => {
if (event.data.type === 'update') {
this.emit('update', event.data.update, event.data.origin);
}
}
};
}
packages/common/nbstore/src/impls/idb/index.ts
-6
View File
@@ -2,7 +2,6 @@ import type { StorageConstructor } from '..';
import { IndexedDBBlobStorage } from './blob';
import { IndexedDBDocStorage } from './doc';
import { IndexedDBSyncStorage } from './sync';
import { IndexedDBV1BlobStorage, IndexedDBV1DocStorage } from './v1';
export * from './blob';
export * from './doc';
@@ -13,8 +12,3 @@ export const idbStorages = [
IndexedDBBlobStorage,
IndexedDBSyncStorage,
] satisfies StorageConstructor[];
export const idbv1Storages = [
IndexedDBV1DocStorage,
IndexedDBV1BlobStorage,
] satisfies StorageConstructor[];
packages/common/nbstore/src/impls/idb/v1/blob.ts
+9
View File
@@ -19,6 +19,9 @@ export class IndexedDBV1BlobStorage extends BlobStorageBase {
}
override async get(key: string) {
if (!this.db) {
return null;
}
const trx = this.db.transaction('blob', 'readonly');
const blob = await trx.store.get(key);
if (!blob) {
@@ -34,6 +37,9 @@ export class IndexedDBV1BlobStorage extends BlobStorageBase {
}
override async delete(key: string, permanently: boolean) {
if (!this.db) {
return;
}
if (permanently) {
const trx = this.db.transaction('blob', 'readwrite');
await trx.store.delete(key);
@@ -41,6 +47,9 @@ export class IndexedDBV1BlobStorage extends BlobStorageBase {
}
override async list() {
if (!this.db) {
return [];
}
const trx = this.db.transaction('blob', 'readonly');
const it = trx.store.iterate();
packages/common/nbstore/src/impls/idb/v1/db.ts
+26 -20
View File
@@ -15,23 +15,26 @@ export interface DocDBSchema extends DBSchema {
};
}
export class DocIDBConnection extends AutoReconnectConnection<
IDBPDatabase<DocDBSchema>
> {
export class DocIDBConnection extends AutoReconnectConnection<IDBPDatabase<DocDBSchema> | null> {
override get shareId() {
return 'idb(old):affine-local';
}
override async doConnect() {
return openDB<DocDBSchema>('affine-local', 1, {
upgrade: db => {
db.createObjectStore('workspace', { keyPath: 'id' });
},
});
const dbs = await indexedDB.databases();
if (dbs.some(d => d.name === 'affine-local')) {
return openDB<DocDBSchema>('affine-local', 1, {
upgrade: db => {
db.createObjectStore('workspace', { keyPath: 'id' });
},
});
} else {
return null;
}
}
override doDisconnect(conn: IDBPDatabase<DocDBSchema>) {
conn.close();
override doDisconnect(conn: IDBPDatabase<DocDBSchema> | null) {
conn?.close();
}
}
@@ -46,9 +49,7 @@ export interface BlobIDBConnectionOptions {
id: string;
}
export class BlobIDBConnection extends AutoReconnectConnection<
IDBPDatabase<BlobDBSchema>
> {
export class BlobIDBConnection extends AutoReconnectConnection<IDBPDatabase<BlobDBSchema> | null> {
constructor(private readonly options: BlobIDBConnectionOptions) {
super();
}
@@ -58,14 +59,19 @@ export class BlobIDBConnection extends AutoReconnectConnection<
}
override async doConnect() {
return openDB<BlobDBSchema>(`${this.options.id}_blob`, 1, {
upgrade: db => {
db.createObjectStore('blob');
},
});
const dbs = await indexedDB.databases();
if (dbs.some(d => d.name === `${this.options.id}_blob`)) {
return openDB<BlobDBSchema>(`${this.options.id}_blob`, 1, {
upgrade: db => {
db.createObjectStore('blob');
},
});
} else {
return null;
}
}
override doDisconnect(conn: IDBPDatabase<BlobDBSchema>) {
conn.close();
override doDisconnect(conn: IDBPDatabase<BlobDBSchema> | null) {
conn?.close();
}
}
packages/common/nbstore/src/impls/idb/v1/doc.ts
+127 -21
View File
@@ -1,9 +1,20 @@
import { once } from 'lodash-es';
import {
applyUpdate,
type Array as YArray,
Doc as YDoc,
type Map as YMap,
} from 'yjs';
import { share } from '../../../connection';
import {
type DocClocks,
type DocRecord,
DocStorageBase,
type DocStorageOptions,
type DocUpdate,
} from '../../../storage';
import { getIdConverter } from '../../../utils/id-converter';
import { DocIDBConnection } from './db';
/**
@@ -14,6 +25,13 @@ export class IndexedDBV1DocStorage extends DocStorageBase {
readonly connection = share(new DocIDBConnection());
constructor(opts: DocStorageOptions) {
super({
...opts,
readonlyMode: true,
});
}
get db() {
return this.connection.inner;
}
@@ -23,26 +41,11 @@ export class IndexedDBV1DocStorage extends DocStorageBase {
}
override async getDoc(docId: string) {
const trx = this.db.transaction('workspace', 'readonly');
const record = await trx.store.get(docId);
if (!record?.updates.length) {
if (!this.db) {
return null;
}
if (record.updates.length === 1) {
return {
docId,
bin: record.updates[0].update,
timestamp: new Date(record.updates[0].timestamp),
};
}
return {
docId,
bin: await this.mergeUpdates(record.updates.map(update => update.update)),
timestamp: new Date(record.updates.at(-1)?.timestamp ?? Date.now()),
};
const oldId = (await this.getIdConverter()).newIdToOldId(docId);
return this.rawGetDoc(oldId);
}
protected override async getDocSnapshot() {
@@ -55,12 +58,60 @@ export class IndexedDBV1DocStorage extends DocStorageBase {
}
override async deleteDoc(docId: string) {
if (!this.db) {
return;
}
const oldId = (await this.getIdConverter()).newIdToOldId(docId);
const trx = this.db.transaction('workspace', 'readwrite');
await trx.store.delete(docId);
await trx.store.delete(oldId);
}
override async getDocTimestamps() {
return {};
override async getDocTimestamps(): Promise<DocClocks> {
if (!this.db) {
return {};
}
const idConverter = await this.getIdConverter();
const oldIds: string[] = [this.spaceId];
const rootDocBuffer = await this.rawGetDoc(this.spaceId);
if (rootDocBuffer) {
const ydoc = new YDoc({
guid: this.spaceId,
});
applyUpdate(ydoc, rootDocBuffer.bin);
// get all ids from rootDoc.meta.pages.[*].id, trust this id as normalized id
const normalizedDocIds = (
(ydoc.getMap('meta') as YMap<any> | undefined)?.get('pages') as
| YArray<YMap<any>>
| undefined
)
?.map(i => i.get('id') as string)
.filter(i => !!i);
const spaces = ydoc.getMap('spaces') as YMap<any> | undefined;
for (const pageId of normalizedDocIds ?? []) {
const subdoc = spaces?.get(pageId);
if (subdoc && subdoc instanceof YDoc) {
oldIds.push(subdoc.guid);
}
}
}
const trx = this.db.transaction('workspace', 'readonly');
const allKeys = await trx.store.getAllKeys();
oldIds.push(...allKeys.filter(k => k.startsWith(`db$${this.spaceId}$`)));
oldIds.push(
...allKeys.filter(k =>
k.match(new RegExp(`^userdata\\$[\\w-]+\\$${this.spaceId}$`))
)
);
return Object.fromEntries(
oldIds.map(id => [idConverter.oldIdToNewId(id), new Date(1)])
);
}
override async getDocTimestamp(_docId: string) {
@@ -78,4 +129,59 @@ export class IndexedDBV1DocStorage extends DocStorageBase {
protected override async markUpdatesMerged(): Promise<number> {
return 0;
}
private async rawGetDoc(id: string) {
if (!this.db) {
return null;
}
const trx = this.db.transaction('workspace', 'readonly');
const record = await trx.store.get(id);
if (!record?.updates.length) {
return null;
}
if (record.updates.length === 1) {
return {
docId: id,
bin: record.updates[0].update,
timestamp: new Date(record.updates[0].timestamp),
};
}
return {
docId: id,
bin: await this.mergeUpdates(record.updates.map(update => update.update)),
timestamp: new Date(record.updates.at(-1)?.timestamp ?? Date.now()),
};
}
private readonly getIdConverter = once(async () => {
const idConverter = getIdConverter(
{
getDocBuffer: async id => {
if (!this.db) {
return null;
}
const trx = this.db.transaction('workspace', 'readonly');
const record = await trx.store.get(id);
if (!record?.updates.length) {
return null;
}
if (record.updates.length === 1) {
return record.updates[0].update;
}
return await this.mergeUpdates(
record.updates.map(update => update.update)
);
},
},
this.spaceId
);
return await idConverter;
});
}
packages/common/nbstore/src/impls/idb/v1/index.ts
+9
View File
@@ -1,2 +1,11 @@
import type { StorageConstructor } from '../..';
import { IndexedDBV1BlobStorage } from './blob';
import { IndexedDBV1DocStorage } from './doc';
export * from './blob';
export * from './doc';
export const idbV1Storages = [
IndexedDBV1DocStorage,
IndexedDBV1BlobStorage,
] satisfies StorageConstructor[];
packages/common/nbstore/src/impls/index.ts
+5 -2
View File
@@ -1,8 +1,10 @@
import type { Storage } from '../storage';
import type { broadcastChannelStorages } from './broadcast-channel';
import type { cloudStorages } from './cloud';
import type { idbStorages, idbv1Storages } from './idb';
import type { idbStorages } from './idb';
import type { idbV1Storages } from './idb/v1';
import type { sqliteStorages } from './sqlite';
import type { sqliteV1Storages } from './sqlite/v1';
export type StorageConstructor = {
new (...args: any[]): Storage;
@@ -11,9 +13,10 @@ export type StorageConstructor = {
type Storages =
| typeof cloudStorages
| typeof idbv1Storages
| typeof idbV1Storages
| typeof idbStorages
| typeof sqliteStorages
| typeof sqliteV1Storages
| typeof broadcastChannelStorages;
// oxlint-disable-next-line no-redeclare
packages/common/nbstore/src/impls/sqlite/db.ts
+3 -3
View File
@@ -41,7 +41,7 @@ export type NativeDBApis = {
id: string,
peer: string,
docId: string
): Promise<DocClock>;
): Promise<DocClock | null>;
setPeerRemoteClock(
id: string,
peer: string,
@@ -53,7 +53,7 @@ export type NativeDBApis = {
id: string,
peer: string,
docId: string
): Promise<DocClock>;
): Promise<DocClock | null>;
setPeerPulledRemoteClock(
id: string,
peer: string,
@@ -65,7 +65,7 @@ export type NativeDBApis = {
id: string,
peer: string,
docId: string
): Promise<DocClock>;
): Promise<DocClock | null>;
setPeerPushedClock(
id: string,
peer: string,
packages/common/nbstore/src/impls/sqlite/index.ts
-1
View File
@@ -7,7 +7,6 @@ export * from './blob';
export { bindNativeDBApis, type NativeDBApis } from './db';
export * from './doc';
export * from './sync';
export * from './v1';
export const sqliteStorages = [
SqliteDocStorage,
packages/common/nbstore/src/impls/sqlite/v1/blob.ts
+1
View File
@@ -7,6 +7,7 @@ import { apis } from './db';
* @deprecated readonly
*/
export class SqliteV1BlobStorage extends BlobStorageBase {
static identifier = 'SqliteV1BlobStorage';
override connection = new DummyConnection();
constructor(private readonly options: { type: SpaceType; id: string }) {
packages/common/nbstore/src/impls/sqlite/v1/doc.ts
+47 -2
View File
@@ -4,6 +4,8 @@ import {
DocStorageBase,
type DocUpdate,
} from '../../../storage';
import { getIdConverter, type IdConverter } from '../../../utils/id-converter';
import { isEmptyUpdate } from '../../../utils/is-empty-update';
import type { SpaceType } from '../../../utils/universal-id';
import { apis } from './db';
@@ -14,8 +16,14 @@ export class SqliteV1DocStorage extends DocStorageBase<{
type: SpaceType;
id: string;
}> {
static identifier = 'SqliteV1DocStorage';
cachedIdConverter: Promise<IdConverter> | null = null;
override connection = new DummyConnection();
constructor(options: { type: SpaceType; id: string }) {
super({ ...options, readonlyMode: true });
}
private get db() {
if (!apis) {
throw new Error('Not in electron context.');
@@ -26,17 +34,21 @@ export class SqliteV1DocStorage extends DocStorageBase<{
override async pushDocUpdate(update: DocUpdate) {
// no more writes
return { docId: update.docId, timestamp: new Date() };
}
override async getDoc(docId: string) {
const idConverter = await this.getIdConverter();
const bin = await this.db.getDocAsUpdates(
this.options.type,
this.options.id,
docId
idConverter.newIdToOldId(docId)
);
if (isEmptyUpdate(bin)) {
return null;
}
return {
docId,
bin,
@@ -71,4 +83,37 @@ export class SqliteV1DocStorage extends DocStorageBase<{
protected override async markUpdatesMerged(): Promise<number> {
return 0;
}
private async getIdConverter() {
if (this.cachedIdConverter) {
return await this.cachedIdConverter;
}
this.cachedIdConverter = getIdConverter(
{
getDocBuffer: async id => {
if (!this.db) {
return null;
}
const updates = await this.db.getDocAsUpdates(
this.options.type,
this.options.id,
id
);
if (isEmptyUpdate(updates)) {
return null;
}
if (!updates) {
return null;
}
return updates;
},
},
this.spaceId
);
return await this.cachedIdConverter;
}
}
packages/common/nbstore/src/impls/sqlite/v1/index.ts
+9
View File
@@ -1,3 +1,12 @@
import type { StorageConstructor } from '../..';
import { SqliteV1BlobStorage } from './blob';
import { SqliteV1DocStorage } from './doc';
export * from './blob';
export { bindNativeDBV1Apis } from './db';
export * from './doc';
export const sqliteV1Storages = [
SqliteV1DocStorage,
SqliteV1BlobStorage,
] satisfies StorageConstructor[];