mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-02-13 12:55:00 +00:00
feat(nbstore): improve nbstore (#9512)
This commit is contained in:
EYHN
@@ -1,33 +0,0 @@
|
||||
import { type BlobRecord, BlobStorageBase, share } from '@affine/nbstore';
|
||||
|
||||
import { NativeDBConnection } from './db';
|
||||
|
||||
export class SqliteBlobStorage extends BlobStorageBase {
|
||||
override connection = share(
|
||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||
);
|
||||
|
||||
get db() {
|
||||
return this.connection.inner;
|
||||
}
|
||||
|
||||
override async get(key: string) {
|
||||
return this.db.getBlob(key);
|
||||
}
|
||||
|
||||
override async set(blob: BlobRecord) {
|
||||
await this.db.setBlob(blob);
|
||||
}
|
||||
|
||||
override async delete(key: string, permanently: boolean) {
|
||||
await this.db.deleteBlob(key, permanently);
|
||||
}
|
||||
|
||||
override async release() {
|
||||
await this.db.releaseBlobs();
|
||||
}
|
||||
|
||||
override async list() {
|
||||
return this.db.listBlobs();
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
import path from 'node:path';
|
||||
|
||||
import { DocStorage as NativeDocStorage } from '@affine/native';
|
||||
import { AutoReconnectConnection, type SpaceType } from '@affine/nbstore';
|
||||
import fs from 'fs-extra';
|
||||
|
||||
import { logger } from '../logger';
|
||||
import { getSpaceDBPath } from '../workspace/meta';
|
||||
|
||||
export class NativeDBConnection extends AutoReconnectConnection<NativeDocStorage> {
|
||||
constructor(
|
||||
private readonly peer: string,
|
||||
private readonly type: SpaceType,
|
||||
private readonly id: string
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async getDBPath() {
|
||||
return await getSpaceDBPath(this.peer, this.type, this.id);
|
||||
}
|
||||
|
||||
override get shareId(): string {
|
||||
return `sqlite:${this.peer}:${this.type}:${this.id}`;
|
||||
}
|
||||
|
||||
override async doConnect() {
|
||||
const dbPath = await this.getDBPath();
|
||||
await fs.ensureDir(path.dirname(dbPath));
|
||||
const conn = new NativeDocStorage(dbPath);
|
||||
await conn.connect();
|
||||
logger.info('[nbstore] connection established', this.shareId);
|
||||
return conn;
|
||||
}
|
||||
|
||||
override doDisconnect(conn: NativeDocStorage) {
|
||||
conn
|
||||
.close()
|
||||
.then(() => {
|
||||
logger.info('[nbstore] connection closed', this.shareId);
|
||||
})
|
||||
.catch(err => {
|
||||
logger.error('[nbstore] connection close failed', this.shareId, err);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
import {
|
||||
type DocClocks,
|
||||
type DocRecord,
|
||||
DocStorageBase,
|
||||
type DocUpdate,
|
||||
share,
|
||||
} from '@affine/nbstore';
|
||||
|
||||
import { NativeDBConnection } from './db';
|
||||
|
||||
export class SqliteDocStorage extends DocStorageBase {
|
||||
override connection = share(
|
||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||
);
|
||||
|
||||
get db() {
|
||||
return this.connection.inner;
|
||||
}
|
||||
|
||||
override async pushDocUpdate(update: DocUpdate) {
|
||||
const timestamp = await this.db.pushUpdate(update.docId, update.bin);
|
||||
|
||||
return { docId: update.docId, timestamp };
|
||||
}
|
||||
|
||||
override async deleteDoc(docId: string) {
|
||||
await this.db.deleteDoc(docId);
|
||||
}
|
||||
|
||||
override async getDocTimestamps(after?: Date) {
|
||||
const clocks = await this.db.getDocClocks(after);
|
||||
|
||||
return clocks.reduce((ret, cur) => {
|
||||
ret[cur.docId] = cur.timestamp;
|
||||
return ret;
|
||||
}, {} as DocClocks);
|
||||
}
|
||||
|
||||
override async getDocTimestamp(docId: string) {
|
||||
return this.db.getDocClock(docId);
|
||||
}
|
||||
|
||||
protected override async getDocSnapshot(docId: string) {
|
||||
const snapshot = await this.db.getDocSnapshot(docId);
|
||||
|
||||
if (!snapshot) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
docId,
|
||||
bin: snapshot.data,
|
||||
timestamp: snapshot.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
protected override async setDocSnapshot(
|
||||
snapshot: DocRecord
|
||||
): Promise<boolean> {
|
||||
return this.db.setDocSnapshot({
|
||||
docId: snapshot.docId,
|
||||
data: Buffer.from(snapshot.bin),
|
||||
timestamp: new Date(snapshot.timestamp),
|
||||
});
|
||||
}
|
||||
|
||||
protected override async getDocUpdates(docId: string) {
|
||||
return this.db.getDocUpdates(docId).then(updates =>
|
||||
updates.map(update => ({
|
||||
docId,
|
||||
bin: update.data,
|
||||
timestamp: update.createdAt,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
protected override markUpdatesMerged(docId: string, updates: DocRecord[]) {
|
||||
return this.db.markUpdatesMerged(
|
||||
docId,
|
||||
updates.map(update => update.timestamp)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,128 +1,43 @@
|
||||
import {
|
||||
type BlobRecord,
|
||||
type DocClock,
|
||||
type DocUpdate,
|
||||
} from '@affine/nbstore';
|
||||
import path from 'node:path';
|
||||
|
||||
import { ensureStorage, getStorage } from './storage';
|
||||
import { DocStoragePool } from '@affine/native';
|
||||
import { parseUniversalId } from '@affine/nbstore';
|
||||
import type { NativeDBApis } from '@affine/nbstore/sqlite';
|
||||
import fs from 'fs-extra';
|
||||
|
||||
export const nbstoreHandlers = {
|
||||
connect: async (id: string) => {
|
||||
await ensureStorage(id);
|
||||
},
|
||||
|
||||
close: async (id: string) => {
|
||||
const store = getStorage(id);
|
||||
|
||||
if (store) {
|
||||
store.disconnect();
|
||||
// The store may be shared with other tabs, so we don't delete it from cache
|
||||
// the underlying connection will handle the close correctly
|
||||
// STORE_CACHE.delete(`${spaceType}:${spaceId}`);
|
||||
}
|
||||
},
|
||||
|
||||
pushDocUpdate: async (id: string, update: DocUpdate) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('doc').pushDocUpdate(update);
|
||||
},
|
||||
|
||||
getDoc: async (id: string, docId: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('doc').getDoc(docId);
|
||||
},
|
||||
|
||||
deleteDoc: async (id: string, docId: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('doc').deleteDoc(docId);
|
||||
},
|
||||
|
||||
getDocTimestamps: async (id: string, after?: Date) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('doc').getDocTimestamps(after);
|
||||
},
|
||||
|
||||
getDocTimestamp: async (id: string, docId: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('doc').getDocTimestamp(docId);
|
||||
},
|
||||
|
||||
setBlob: async (id: string, blob: BlobRecord) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('blob').set(blob);
|
||||
},
|
||||
|
||||
getBlob: async (id: string, key: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('blob').get(key);
|
||||
},
|
||||
|
||||
deleteBlob: async (id: string, key: string, permanently: boolean) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('blob').delete(key, permanently);
|
||||
},
|
||||
|
||||
listBlobs: async (id: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('blob').list();
|
||||
},
|
||||
|
||||
releaseBlobs: async (id: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('blob').release();
|
||||
},
|
||||
|
||||
getPeerRemoteClocks: async (id: string, peer: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').getPeerRemoteClocks(peer);
|
||||
},
|
||||
|
||||
getPeerRemoteClock: async (id: string, peer: string, docId: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').getPeerRemoteClock(peer, docId);
|
||||
},
|
||||
|
||||
setPeerRemoteClock: async (id: string, peer: string, clock: DocClock) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').setPeerRemoteClock(peer, clock);
|
||||
},
|
||||
|
||||
getPeerPulledRemoteClocks: async (id: string, peer: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').getPeerPulledRemoteClocks(peer);
|
||||
},
|
||||
|
||||
getPeerPulledRemoteClock: async (id: string, peer: string, docId: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').getPeerPulledRemoteClock(peer, docId);
|
||||
},
|
||||
|
||||
setPeerPulledRemoteClock: async (
|
||||
id: string,
|
||||
peer: string,
|
||||
clock: DocClock
|
||||
) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').setPeerPulledRemoteClock(peer, clock);
|
||||
},
|
||||
|
||||
getPeerPushedClocks: async (id: string, peer: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').getPeerPushedClocks(peer);
|
||||
},
|
||||
|
||||
getPeerPushedClock: async (id: string, peer: string, docId: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').getPeerPushedClock(peer, docId);
|
||||
},
|
||||
|
||||
setPeerPushedClock: async (id: string, peer: string, clock: DocClock) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').setPeerPushedClock(peer, clock);
|
||||
},
|
||||
|
||||
clearClocks: async (id: string) => {
|
||||
const store = await ensureStorage(id);
|
||||
return store.get('sync').clearClocks();
|
||||
import { getSpaceDBPath } from '../workspace/meta';
|
||||
|
||||
const POOL = new DocStoragePool();
|
||||
|
||||
export const nbstoreHandlers: NativeDBApis = {
|
||||
connect: async (universalId: string) => {
|
||||
const { peer, type, id } = parseUniversalId(universalId);
|
||||
const dbPath = await getSpaceDBPath(peer, type, id);
|
||||
await fs.ensureDir(path.dirname(dbPath));
|
||||
await POOL.connect(universalId, dbPath);
|
||||
},
|
||||
disconnect: POOL.disconnect.bind(POOL),
|
||||
pushUpdate: POOL.pushUpdate.bind(POOL),
|
||||
getDocSnapshot: POOL.getDocSnapshot.bind(POOL),
|
||||
setDocSnapshot: POOL.setDocSnapshot.bind(POOL),
|
||||
getDocUpdates: POOL.getDocUpdates.bind(POOL),
|
||||
markUpdatesMerged: POOL.markUpdatesMerged.bind(POOL),
|
||||
deleteDoc: POOL.deleteDoc.bind(POOL),
|
||||
getDocClocks: POOL.getDocClocks.bind(POOL),
|
||||
getDocClock: POOL.getDocClock.bind(POOL),
|
||||
getBlob: POOL.getBlob.bind(POOL),
|
||||
setBlob: POOL.setBlob.bind(POOL),
|
||||
deleteBlob: POOL.deleteBlob.bind(POOL),
|
||||
releaseBlobs: POOL.releaseBlobs.bind(POOL),
|
||||
listBlobs: POOL.listBlobs.bind(POOL),
|
||||
getPeerRemoteClocks: POOL.getPeerRemoteClocks.bind(POOL),
|
||||
getPeerRemoteClock: POOL.getPeerRemoteClock.bind(POOL),
|
||||
setPeerRemoteClock: POOL.setPeerRemoteClock.bind(POOL),
|
||||
getPeerPulledRemoteClocks: POOL.getPeerPulledRemoteClocks.bind(POOL),
|
||||
getPeerPulledRemoteClock: POOL.getPeerPulledRemoteClock.bind(POOL),
|
||||
setPeerPulledRemoteClock: POOL.setPeerPulledRemoteClock.bind(POOL),
|
||||
getPeerPushedClocks: POOL.getPeerPushedClocks.bind(POOL),
|
||||
getPeerPushedClock: POOL.getPeerPushedClock.bind(POOL),
|
||||
setPeerPushedClock: POOL.setPeerPushedClock.bind(POOL),
|
||||
clearClocks: POOL.clearClocks.bind(POOL),
|
||||
};
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
export { nbstoreHandlers } from './handlers';
|
||||
export * from './storage';
|
||||
export { dbEvents as dbEventsV1, dbHandlers as dbHandlersV1 } from './v1';
|
||||
export { universalId } from '@affine/nbstore';
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
import { parseUniversalId, SpaceStorage } from '@affine/nbstore';
|
||||
import { applyUpdate, Doc as YDoc } from 'yjs';
|
||||
|
||||
import { logger } from '../logger';
|
||||
import { SqliteBlobStorage } from './blob';
|
||||
import { NativeDBConnection } from './db';
|
||||
import { SqliteDocStorage } from './doc';
|
||||
import { SqliteSyncStorage } from './sync';
|
||||
|
||||
export class SqliteSpaceStorage extends SpaceStorage {
|
||||
get connection() {
|
||||
const docStore = this.get('doc');
|
||||
|
||||
if (!docStore) {
|
||||
throw new Error('doc store not found');
|
||||
}
|
||||
|
||||
const connection = docStore.connection;
|
||||
|
||||
if (!(connection instanceof NativeDBConnection)) {
|
||||
throw new Error('doc store connection is not a Sqlite connection');
|
||||
}
|
||||
|
||||
return connection;
|
||||
}
|
||||
|
||||
async getDBPath() {
|
||||
return this.connection.getDBPath();
|
||||
}
|
||||
|
||||
async getWorkspaceName() {
|
||||
const docStore = this.tryGet('doc');
|
||||
|
||||
if (!docStore) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const doc = await docStore.getDoc(docStore.spaceId);
|
||||
if (!doc) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const ydoc = new YDoc();
|
||||
applyUpdate(ydoc, doc.bin);
|
||||
return ydoc.getMap('meta').get('name') as string;
|
||||
}
|
||||
|
||||
async checkpoint() {
|
||||
await this.connection.inner.checkpoint();
|
||||
}
|
||||
}
|
||||
|
||||
const STORE_CACHE = new Map<string, SqliteSpaceStorage>();
|
||||
|
||||
process.on('beforeExit', () => {
|
||||
STORE_CACHE.forEach(store => {
|
||||
store.destroy().catch(err => {
|
||||
logger.error('[nbstore] destroy store failed', err);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
export function getStorage(universalId: string) {
|
||||
return STORE_CACHE.get(universalId);
|
||||
}
|
||||
|
||||
export async function ensureStorage(universalId: string) {
|
||||
const { peer, type, id } = parseUniversalId(universalId);
|
||||
let store = STORE_CACHE.get(universalId);
|
||||
|
||||
if (!store) {
|
||||
const opts = {
|
||||
peer,
|
||||
type,
|
||||
id,
|
||||
};
|
||||
|
||||
store = new SqliteSpaceStorage([
|
||||
new SqliteDocStorage(opts),
|
||||
new SqliteBlobStorage(opts),
|
||||
new SqliteSyncStorage(opts),
|
||||
]);
|
||||
|
||||
store.connect();
|
||||
|
||||
await store.waitForConnected();
|
||||
|
||||
STORE_CACHE.set(universalId, store);
|
||||
}
|
||||
|
||||
return store;
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
import {
|
||||
BasicSyncStorage,
|
||||
type DocClock,
|
||||
type DocClocks,
|
||||
share,
|
||||
} from '@affine/nbstore';
|
||||
|
||||
import { NativeDBConnection } from './db';
|
||||
|
||||
export class SqliteSyncStorage extends BasicSyncStorage {
|
||||
override connection = share(
|
||||
new NativeDBConnection(this.peer, this.spaceType, this.spaceId)
|
||||
);
|
||||
|
||||
get db() {
|
||||
return this.connection.inner;
|
||||
}
|
||||
|
||||
override async getPeerRemoteClocks(peer: string) {
|
||||
const records = await this.db.getPeerRemoteClocks(peer);
|
||||
return records.reduce((clocks, { docId, timestamp }) => {
|
||||
clocks[docId] = timestamp;
|
||||
return clocks;
|
||||
}, {} as DocClocks);
|
||||
}
|
||||
|
||||
override async getPeerRemoteClock(peer: string, docId: string) {
|
||||
return this.db.getPeerRemoteClock(peer, docId);
|
||||
}
|
||||
|
||||
override async setPeerRemoteClock(peer: string, clock: DocClock) {
|
||||
await this.db.setPeerRemoteClock(peer, clock.docId, clock.timestamp);
|
||||
}
|
||||
|
||||
override async getPeerPulledRemoteClock(peer: string, docId: string) {
|
||||
return this.db.getPeerPulledRemoteClock(peer, docId);
|
||||
}
|
||||
|
||||
override async getPeerPulledRemoteClocks(peer: string) {
|
||||
const records = await this.db.getPeerPulledRemoteClocks(peer);
|
||||
return records.reduce((clocks, { docId, timestamp }) => {
|
||||
clocks[docId] = timestamp;
|
||||
return clocks;
|
||||
}, {} as DocClocks);
|
||||
}
|
||||
|
||||
override async setPeerPulledRemoteClock(peer: string, clock: DocClock) {
|
||||
await this.db.setPeerPulledRemoteClock(peer, clock.docId, clock.timestamp);
|
||||
}
|
||||
|
||||
override async getPeerPushedClocks(peer: string) {
|
||||
const records = await this.db.getPeerPushedClocks(peer);
|
||||
return records.reduce((clocks, { docId, timestamp }) => {
|
||||
clocks[docId] = timestamp;
|
||||
return clocks;
|
||||
}, {} as DocClocks);
|
||||
}
|
||||
|
||||
override async getPeerPushedClock(peer: string, docId: string) {
|
||||
return this.db.getPeerPushedClock(peer, docId);
|
||||
}
|
||||
|
||||
override async setPeerPushedClock(peer: string, clock: DocClock) {
|
||||
await this.db.setPeerPushedClock(peer, clock.docId, clock.timestamp);
|
||||
}
|
||||
|
||||
override async clearClocks() {
|
||||
await this.db.clearClocks();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user