Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-21 03:56:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(nbstore): improve nbstore (#9512)

Browse Source
This commit is contained in:
EYHN
2025-01-06 09:38:03 +00:00
parent a2563d2180
commit 46c8c4a408
103 changed files with 3337 additions and 3423 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/common/nbstore/src/sync/doc/index.ts
+59 -26
View File
@@ -1,17 +1,23 @@
import type { Observable } from 'rxjs';
import { combineLatest, map } from 'rxjs';
import { combineLatest, map, of } from 'rxjs';
import type { DocStorage, SyncStorage } from '../../storage';
import { DummyDocStorage } from '../../storage/dummy/doc';
import { DummySyncStorage } from '../../storage/dummy/sync';
import { MANUALLY_STOP } from '../../utils/throw-if-aborted';
import type { PeerStorageOptions } from '../types';
import { DocSyncPeer } from './peer';
export interface DocSyncState {
total: number;
syncing: number;
synced: boolean;
retrying: boolean;
errorMessage: string | null;
}
export interface DocSyncDocState {
synced: boolean;
syncing: boolean;
retrying: boolean;
errorMessage: string | null;
@@ -24,43 +30,70 @@ export interface DocSync {
}
export class DocSyncImpl implements DocSync {
private readonly peers: DocSyncPeer[] = this.remotes.map(
remote => new DocSyncPeer(this.local, this.sync, remote)
private readonly peers: DocSyncPeer[] = Object.entries(
this.storages.remotes
).map(
([peerId, remote]) =>
new DocSyncPeer(peerId, this.storages.local, this.sync, remote)
);
private abort: AbortController | null = null;
readonly state$: Observable<DocSyncState> = combineLatest(
this.peers.map(peer => peer.peerState$)
).pipe(
map(allPeers => ({
total: allPeers.reduce((acc, peer) => acc + peer.total, 0),
syncing: allPeers.reduce((acc, peer) => acc + peer.syncing, 0),
retrying: allPeers.some(peer => peer.retrying),
errorMessage:
allPeers.find(peer => peer.errorMessage)?.errorMessage ?? null,
}))
);
constructor(
readonly local: DocStorage,
readonly sync: SyncStorage,
readonly remotes: DocStorage[]
) {}
docState$(docId: string): Observable<DocSyncDocState> {
return combineLatest(this.peers.map(peer => peer.docState$(docId))).pipe(
get state$() {
return combineLatest(this.peers.map(peer => peer.peerState$)).pipe(
map(allPeers => ({
total: allPeers.reduce((acc, peer) => Math.max(acc, peer.total), 0),
syncing: allPeers.reduce((acc, peer) => Math.max(acc, peer.syncing), 0),
synced: allPeers.every(peer => peer.synced),
retrying: allPeers.some(peer => peer.retrying),
errorMessage:
allPeers.find(peer => peer.errorMessage)?.errorMessage ?? null,
retrying: allPeers.some(peer => peer.retrying),
syncing: allPeers.some(peer => peer.syncing),
}))
) as Observable<DocSyncState>;
}
constructor(
readonly storages: PeerStorageOptions<DocStorage>,
readonly sync: SyncStorage
) {}
/**
* for testing
*/
static get dummy() {
return new DocSyncImpl(
{
local: new DummyDocStorage(),
remotes: {},
},
new DummySyncStorage()
);
}
docState$(docId: string): Observable<DocSyncDocState> {
if (this.peers.length === 0) {
return of({
errorMessage: null,
retrying: false,
syncing: false,
synced: true,
});
}
return combineLatest(this.peers.map(peer => peer.docState$(docId))).pipe(
map(allPeers => {
return {
errorMessage:
allPeers.find(peer => peer.errorMessage)?.errorMessage ?? null,
retrying: allPeers.some(peer => peer.retrying),
syncing: allPeers.some(peer => peer.syncing),
synced: allPeers.every(peer => peer.synced),
};
})
);
}
start() {
if (this.abort) {
this.abort.abort();
this.abort.abort(MANUALLY_STOP);
}
const abort = new AbortController();
this.abort = abort;
packages/common/nbstore/src/sync/doc/peer.ts
+76 -45
View File
@@ -43,6 +43,7 @@ interface Status {
remoteClocks: ClockMap;
syncing: boolean;
retrying: boolean;
skipped: boolean;
errorMessage: string | null;
}
@@ -50,11 +51,13 @@ interface PeerState {
total: number;
syncing: number;
retrying: boolean;
synced: boolean;
errorMessage: string | null;
}
interface PeerDocState {
syncing: boolean;
synced: boolean;
retrying: boolean;
errorMessage: string | null;
}
@@ -92,10 +95,11 @@ export class DocSyncPeer {
/**
* random unique id for recognize self in "update" event
*/
private readonly uniqueId = `sync:${this.local.universalId}:${this.remote.universalId}:${nanoid()}`;
private readonly uniqueId = `sync:${this.peerId}:${nanoid()}`;
private readonly prioritySettings = new Map<string, number>();
constructor(
readonly peerId: string,
readonly local: DocStorage,
readonly syncMetadata: SyncStorage,
readonly remote: DocStorage,
@@ -110,43 +114,59 @@ export class DocSyncPeer {
remoteClocks: new ClockMap(new Map()),
syncing: false,
retrying: false,
skipped: false,
errorMessage: null,
};
private readonly statusUpdatedSubject$ = new Subject<string | true>();
peerState$ = new Observable<PeerState>(subscribe => {
const next = () => {
if (!this.status.syncing) {
// if syncing = false, jobMap is empty
subscribe.next({
total: this.status.docs.size,
syncing: this.status.docs.size,
retrying: this.status.retrying,
errorMessage: this.status.errorMessage,
});
} else {
const syncing = this.status.jobMap.size;
subscribe.next({
total: this.status.docs.size,
syncing: syncing,
retrying: this.status.retrying,
errorMessage: this.status.errorMessage,
});
}
};
next();
return this.statusUpdatedSubject$.subscribe(() => {
get peerState$() {
return new Observable<PeerState>(subscribe => {
const next = () => {
if (this.status.skipped) {
subscribe.next({
total: 0,
syncing: 0,
synced: true,
retrying: false,
errorMessage: null,
});
} else if (!this.status.syncing) {
// if syncing = false, jobMap is empty
subscribe.next({
total: this.status.docs.size,
syncing: this.status.docs.size,
synced: false,
retrying: this.status.retrying,
errorMessage: this.status.errorMessage,
});
} else {
const syncing = this.status.jobMap.size;
subscribe.next({
total: this.status.docs.size,
syncing: syncing,
retrying: this.status.retrying,
errorMessage: this.status.errorMessage,
synced: syncing === 0,
});
}
};
next();
return this.statusUpdatedSubject$.subscribe(() => {
next();
});
});
});
}
docState$(docId: string) {
return new Observable<PeerDocState>(subscribe => {
const next = () => {
const syncing =
!this.status.connectedDocs.has(docId) ||
this.status.jobMap.has(docId);
subscribe.next({
syncing:
!this.status.connectedDocs.has(docId) ||
this.status.jobMap.has(docId),
syncing: syncing,
synced: !syncing,
retrying: this.status.retrying,
errorMessage: this.status.errorMessage,
});
@@ -161,22 +181,21 @@ export class DocSyncPeer {
private readonly jobs = createJobErrorCatcher({
connect: async (docId: string, signal?: AbortSignal) => {
const pushedClock =
(await this.syncMetadata.getPeerPushedClock(this.remote.peer, docId))
(await this.syncMetadata.getPeerPushedClock(this.peerId, docId))
?.timestamp ?? null;
const clock = await this.local.getDocTimestamp(docId);
throwIfAborted(signal);
if (pushedClock === null || pushedClock !== clock?.timestamp) {
if (
!this.remote.isReadonly &&
(pushedClock === null || pushedClock !== clock?.timestamp)
) {
await this.jobs.pullAndPush(docId, signal);
} else {
// no need to push
const pulled =
(
await this.syncMetadata.getPeerPulledRemoteClock(
this.remote.peer,
docId
)
)?.timestamp ?? null;
(await this.syncMetadata.getPeerPulledRemoteClock(this.peerId, docId))
?.timestamp ?? null;
if (pulled === null || pulled !== this.status.remoteClocks.get(docId)) {
await this.jobs.pull(docId, signal);
}
@@ -214,7 +233,7 @@ export class DocSyncPeer {
});
}
throwIfAborted(signal);
await this.syncMetadata.setPeerPushedClock(this.remote.peer, {
await this.syncMetadata.setPeerPushedClock(this.peerId, {
docId,
timestamp: maxClock,
});
@@ -249,7 +268,7 @@ export class DocSyncPeer {
this.uniqueId
);
throwIfAborted(signal);
await this.syncMetadata.setPeerPulledRemoteClock(this.remote.peer, {
await this.syncMetadata.setPeerPulledRemoteClock(this.peerId, {
docId,
timestamp: remoteClock,
});
@@ -273,7 +292,7 @@ export class DocSyncPeer {
});
}
throwIfAborted(signal);
await this.syncMetadata.setPeerPushedClock(this.remote.peer, {
await this.syncMetadata.setPeerPushedClock(this.peerId, {
docId,
timestamp: localClock,
});
@@ -294,7 +313,7 @@ export class DocSyncPeer {
remoteClock,
});
}
await this.syncMetadata.setPeerPushedClock(this.remote.peer, {
await this.syncMetadata.setPeerPushedClock(this.peerId, {
docId,
timestamp: localDocRecord.timestamp,
});
@@ -322,7 +341,7 @@ export class DocSyncPeer {
this.uniqueId
);
throwIfAborted(signal);
await this.syncMetadata.setPeerPulledRemoteClock(this.remote.peer, {
await this.syncMetadata.setPeerPulledRemoteClock(this.peerId, {
docId,
timestamp: remoteClock,
});
@@ -360,7 +379,7 @@ export class DocSyncPeer {
);
throwIfAborted(signal);
await this.syncMetadata.setPeerPulledRemoteClock(this.remote.peer, {
await this.syncMetadata.setPeerPulledRemoteClock(this.peerId, {
docId,
timestamp: remoteClock,
});
@@ -372,7 +391,7 @@ export class DocSyncPeer {
updateRemoteClock: async (docId: string, remoteClock: Date) => {
const updated = this.status.remoteClocks.setIfBigger(docId, remoteClock);
if (updated) {
await this.syncMetadata.setPeerRemoteClock(this.remote.peer, {
await this.syncMetadata.setPeerRemoteClock(this.peerId, {
docId,
timestamp: remoteClock,
});
@@ -455,6 +474,7 @@ export class DocSyncPeer {
jobMap: new Map(),
remoteClocks: new ClockMap(new Map()),
syncing: false,
skipped: false,
// tell ui to show retrying status
retrying: true,
// error message from last retry
@@ -482,6 +502,17 @@ export class DocSyncPeer {
private async retryLoop(signal?: AbortSignal) {
throwIfAborted(signal);
if (this.local.isReadonly) {
// Local is readonly, skip sync
this.status.skipped = true;
this.statusUpdatedSubject$.next(true);
await new Promise((_, reject) => {
signal?.addEventListener('abort', reason => {
reject(reason);
});
});
return;
}
const abort = new AbortController();
signal?.addEventListener('abort', reason => {
@@ -536,8 +567,8 @@ export class DocSyncPeer {
if (
origin === this.uniqueId ||
origin?.startsWith(
`sync:${this.local.peer}:${this.remote.peer}:`
// skip if local and remote is same
`sync:${this.peerId}:`
// skip if peerId is same
)
) {
return;
@@ -572,7 +603,7 @@ export class DocSyncPeer {
// get cached clocks from metadata
const cachedClocks = await this.syncMetadata.getPeerRemoteClocks(
this.remote.peer
this.peerId
);
throwIfAborted(signal);
for (const [id, v] of Object.entries(cachedClocks)) {