feat(core): migrate more pull to realtime (#14936)

#### PR Dependency Tree


* **PR #14936** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Refactor**
* Consolidated realtime subscription patterns for consistent, more
reliable live updates across comments, notifications, transcription
tasks, and embedding progress.
* Standardized realtime room naming and subscription keys for
deterministic delivery.

* **New Features**
* Introduced a reusable live-query mechanism powering realtime snapshot
+ event workflows used by comments, notifications, transcript tasks, and
embedding progress.

* **Tests**
* Added tests covering live-query behavior and deterministic
subscription key generation.

[![Review Change
Stack](https://storage.googleapis.com/coderabbit_public_assets/review-stack-in-coderabbit-ui.svg)](https://app.coderabbit.ai/change-stack/toeverything/AFFiNE/pull/14936)
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-05-11 00:33:25 +08:00
committed by GitHub
parent 8cf00738c2
commit db0ff0a9df
26 changed files with 812 additions and 441 deletions
@@ -3,12 +3,17 @@ import { z } from 'zod';
import { decodeWithJson, encodeWithJson } from '../../base/graphql';
import { AccessController } from '../permission';
import type { RealtimePublisher, RealtimeRegistry } from '../realtime';
import {
realtimeCommentRoom,
type RealtimePublisher,
type RealtimeRegistry,
registerRealtimeLiveQuery,
} from '../realtime';
import type { CommentCursor } from './resolver';
import { CommentService } from './service';
export function commentRoom(workspaceId: string, docId: string) {
return `workspace:${workspaceId}:doc:${docId}:comment`;
return realtimeCommentRoom(workspaceId, docId);
}
@Injectable()
@@ -27,53 +32,57 @@ export class CommentRealtimeProvider implements OnModuleInit {
first: z.number().optional(),
});
this.registry?.registerRequest({
name: 'comment.changes.get',
input,
handle: async (user, payload) => {
await this.assertRead(user.id, payload.workspaceId, payload.docId);
const cursor: CommentCursor = decodeWithJson(payload.after) ?? {};
const changes = await this.service.listCommentChanges(
payload.workspaceId,
payload.docId,
{
commentUpdatedAt: cursor.commentUpdatedAt,
replyUpdatedAt: cursor.replyUpdatedAt,
take: payload.first,
registerRealtimeLiveQuery(this.registry, {
request: {
name: 'comment.changes.get',
input,
handle: async (user, payload) => {
await this.assertRead(user.id, payload.workspaceId, payload.docId);
const cursor: CommentCursor = decodeWithJson(payload.after) ?? {};
const limit = payload.first;
const changes = await this.service.listCommentChanges(
payload.workspaceId,
payload.docId,
{
commentUpdatedAt: cursor.commentUpdatedAt,
replyUpdatedAt: cursor.replyUpdatedAt,
take: limit ? limit + 1 : undefined,
}
);
const pageChanges = limit ? changes.slice(0, limit) : changes;
const endCursor = cursor;
for (const change of pageChanges) {
if (change.commentId) {
endCursor.replyUpdatedAt = change.item.updatedAt;
} else {
endCursor.commentUpdatedAt = change.item.updatedAt;
}
}
);
const endCursor = cursor;
for (const change of changes) {
if (change.commentId) {
endCursor.replyUpdatedAt = change.item.updatedAt;
} else {
endCursor.commentUpdatedAt = change.item.updatedAt;
}
}
return {
changes: changes.map(change => ({
id: change.id,
action: change.action,
item: change.item,
commentId: change.commentId ?? undefined,
})) as never,
startCursor: '',
endCursor: encodeWithJson(endCursor),
hasNextPage: changes.length > 0,
};
return {
changes: pageChanges.map(change => ({
id: change.id,
action: change.action,
item: change.item,
commentId: change.commentId ?? null,
})),
startCursor: '',
endCursor: encodeWithJson(endCursor),
hasNextPage: limit ? changes.length > limit : false,
};
},
},
});
this.registry?.registerTopic({
name: 'comment.changed',
input: z.object({
workspaceId: z.string(),
docId: z.string(),
}),
authorize: async (user, payload) => {
await this.assertRead(user.id, payload.workspaceId, payload.docId);
topic: {
name: 'comment.changed',
input: z.object({
workspaceId: z.string(),
docId: z.string(),
}),
authorize: async (user, payload) => {
await this.assertRead(user.id, payload.workspaceId, payload.docId);
},
room: (_user, payload) =>
commentRoom(payload.workspaceId, payload.docId),
},
room: (_user, payload) => commentRoom(payload.workspaceId, payload.docId),
});
}
@@ -1,3 +0,0 @@
export function notificationCountRoom(userId: string) {
return `user:${userId}:notification`;
}
@@ -1,8 +1,11 @@
import { Injectable, OnModuleInit, Optional } from '@nestjs/common';
import { z } from 'zod';
import type { RealtimeRegistry } from '../realtime';
import { notificationCountRoom } from './realtime-room';
import {
realtimeNotificationRoom,
type RealtimeRegistry,
registerRealtimeLiveQuery,
} from '../realtime';
import { NotificationService } from './service';
@Injectable()
@@ -13,23 +16,25 @@ export class NotificationRealtimeProvider implements OnModuleInit {
) {}
onModuleInit() {
this.registry?.registerRequest({
name: 'notification.count.get',
input: z.object({}).strict(),
handle: async user => ({
count: await this.service.countByUserId(user.id),
}),
});
this.registry?.registerTopic({
name: 'notification.count.changed',
input: z.object({}).strict(),
authorize: async () => {},
room: user => {
if (!user) {
throw new Error('User is required for notification count room');
}
return notificationCountRoom(user.id);
const input = z.object({}).strict();
registerRealtimeLiveQuery(this.registry, {
request: {
name: 'notification.count.get',
input,
handle: async user => ({
count: await this.service.countByUserId(user.id),
}),
},
topic: {
name: 'notification.count.changed',
input,
authorize: async () => {},
room: user => {
if (!user) {
throw new Error('User is required for notification count room');
}
return realtimeNotificationRoom(user.id);
},
},
});
}
@@ -18,12 +18,12 @@ import {
import { DocReader } from '../doc';
import { Mailer } from '../mail';
import type { RealtimePublisher } from '../realtime';
import { realtimeNotificationRoom } from '../realtime';
import { generateDocPath } from '../utils/doc';
import {
generateWorkspaceSettingsPath,
WorkspaceSettingsTab,
} from '../utils/workspace';
import { notificationCountRoom } from './realtime-room';
@Injectable()
export class NotificationService {
@@ -499,7 +499,7 @@ export class NotificationService {
'notification.count.changed',
{},
{ count: await this.countByUserId(userId), reason },
{ room: notificationCountRoom(userId) }
{ room: realtimeNotificationRoom(userId) }
);
} catch (error) {
this.logger.error(
@@ -1,11 +1,18 @@
import { getRealtimeInputKey } from '@affine/realtime';
import test from 'ava';
import { z } from 'zod';
import type { CurrentUser } from '../../auth';
import { RealtimeGateway } from '../gateway';
import type { RealtimePublisher } from '../publisher';
import {
realtimeCommentRoom,
realtimeNotificationRoom,
realtimeTranscriptTaskRoom,
realtimeWorkspaceEmbeddingProgressRoom,
registerRealtimeLiveQuery,
} from '../index';
import { RealtimePublisher } from '../publisher';
import { RealtimeRegistry } from '../registry';
import { stableStringify } from '../stable-stringify';
const user: CurrentUser = {
id: 'u1',
@@ -109,7 +116,7 @@ test('gateway authorizes subscription and joins room', async t => {
t.deepEqual(joined, ['workspace:space']);
t.deepEqual(result, {
data: {
subscriptionId: `socket-1:comment.changed:${stableStringify({
subscriptionId: `socket-1:comment.changed:${getRealtimeInputKey({
workspaceId: 'space',
docId: 'doc',
})}`,
@@ -126,22 +133,101 @@ test('gateway authorizes subscription and joins room', async t => {
);
});
test('stableStringify is deterministic for subscription input keys', t => {
test('getRealtimeInputKey is deterministic for subscription input keys', t => {
t.is(
stableStringify({ docId: 'doc', workspaceId: 'space' }),
stableStringify({ workspaceId: 'space', docId: 'doc' })
getRealtimeInputKey({ docId: 'doc', workspaceId: 'space' }),
getRealtimeInputKey({ workspaceId: 'space', docId: 'doc' })
);
});
test('stableStringify follows JSON semantics for subscription input keys', t => {
t.is(stableStringify({ after: undefined }), stableStringify({}));
t.is(stableStringify([undefined]), '[null]');
test('getRealtimeInputKey follows JSON semantics for subscription input keys', t => {
t.is(getRealtimeInputKey({ after: undefined }), getRealtimeInputKey({}));
t.is(getRealtimeInputKey([undefined]), '[null]');
t.is(
stableStringify(new Date('2026-01-02T03:04:05.000Z')),
getRealtimeInputKey(new Date('2026-01-02T03:04:05.000Z')),
'"2026-01-02T03:04:05.000Z"'
);
});
test('room helpers produce stable realtime room names', t => {
t.is(realtimeNotificationRoom('u1'), 'user:u1:notification');
t.is(realtimeCommentRoom('space', 'doc'), 'workspace:space:doc:doc:comment');
t.is(
realtimeWorkspaceEmbeddingProgressRoom('space'),
'workspace:space:embedding-progress'
);
t.is(
realtimeTranscriptTaskRoom('space', 'task'),
'copilot:transcript:space:task'
);
});
test('registerRealtimeLiveQuery registers paired request and topic handlers', async t => {
const registry = new RealtimeRegistry();
registerRealtimeLiveQuery(registry, {
request: {
name: 'notification.count.get',
input: z.object({}).strict(),
handle: async () => ({ count: 7 }),
},
topic: {
name: 'notification.count.changed',
input: z.object({}).strict(),
authorize: async () => {},
room: currentUser => `user:${currentUser?.id}:notification`,
},
});
t.deepEqual(
await registry.getRequest('notification.count.get').handle(user, {}),
{
count: 7,
}
);
t.is(
registry.getTopic('notification.count.changed').room(user, {}),
'user:u1:notification'
);
});
test('publisher emits realtime event with shared input key', t => {
const registry = new RealtimeRegistry();
registry.registerTopic({
name: 'comment.changed',
input: z.object({ workspaceId: z.string(), docId: z.string() }),
authorize: async () => {},
room: (_currentUser, input) =>
realtimeCommentRoom(input.workspaceId, input.docId),
});
const emitted: unknown[] = [];
const publisher = new RealtimePublisher(registry, {
broadcast: () => {},
} as never);
publisher.attachServer({
to: (room: string) => ({
emit: (event: string, payload: unknown) =>
emitted.push({ room, event, payload }),
}),
} as never);
publisher.publishLocal({
topic: 'comment.changed',
input: { docId: 'doc', workspaceId: 'space' },
event: { changed: true },
});
t.like(emitted[0], {
room: 'workspace:space:doc:doc:comment',
event: 'realtime:event',
payload: {
topic: 'comment.changed',
inputKey: getRealtimeInputKey({ workspaceId: 'space', docId: 'doc' }),
event: { changed: true },
},
});
});
test('gateway removes subscriptions on socket disconnect', async t => {
const registry = new RealtimeRegistry();
registry.registerTopic({
@@ -3,6 +3,7 @@ import type {
RealtimeSubscribeEnvelope,
RealtimeUnsubscribeEnvelope,
} from '@affine/realtime';
import { getRealtimeInputKey } from '@affine/realtime';
import { applyDecorators, Logger, UseInterceptors } from '@nestjs/common';
import {
ConnectedSocket,
@@ -25,7 +26,6 @@ import {
import { CurrentUser } from '../auth';
import { RealtimePublisher } from './publisher';
import { RealtimeRegistry } from './registry';
import { stableStringify } from './stable-stringify';
import type { RealtimePublishPayload } from './types';
const SubscribeMessage = (event: string) =>
@@ -87,7 +87,7 @@ export class RealtimeGateway implements OnGatewayInit, OnGatewayDisconnect {
await handler.authorize(user, input as never);
const room = handler.room(user, input as never);
await client.join(room);
const subscriptionId = `${client.id}:${envelope.topic}:${stableStringify(input)}`;
const subscriptionId = `${client.id}:${envelope.topic}:${getRealtimeInputKey(input)}`;
this.subscriptions.set(subscriptionId, {
socketId: client.id,
room,
@@ -11,6 +11,16 @@ import { RealtimeRegistry } from './registry';
})
export class RealtimeModule {}
export { registerRealtimeLiveQuery } from './provider';
export { RealtimePublisher } from './publisher';
export { RealtimeRegistry } from './registry';
export {
realtimeCommentRoom,
realtimeNotificationRoom,
realtimeTranscriptTaskRoom,
realtimeUserRoom,
realtimeWorkspaceDocRoom,
realtimeWorkspaceEmbeddingProgressRoom,
realtimeWorkspaceRoom,
} from './rooms';
export type { RealtimeRequestHandler, RealtimeTopicHandler } from './types';
@@ -0,0 +1,23 @@
import type { RealtimeRequestName, RealtimeTopicName } from '@affine/realtime';
import type { RealtimeRegistry } from './registry';
import type { RealtimeRequestHandler, RealtimeTopicHandler } from './types';
export type RealtimeLiveQueryDefinition<
Request extends RealtimeRequestName,
Topic extends RealtimeTopicName,
> = {
request: RealtimeRequestHandler<Request>;
topic: RealtimeTopicHandler<Topic>;
};
export function registerRealtimeLiveQuery<
Request extends RealtimeRequestName,
Topic extends RealtimeTopicName,
>(
registry: RealtimeRegistry | undefined,
definition: RealtimeLiveQueryDefinition<Request, Topic>
) {
registry?.registerRequest(definition.request);
registry?.registerTopic(definition.topic);
}
@@ -1,10 +1,13 @@
import type { RealtimeEvent, RealtimeTopicName } from '@affine/realtime';
import {
getRealtimeInputKey,
type RealtimeEvent,
type RealtimeTopicName,
} from '@affine/realtime';
import { Injectable, Logger } from '@nestjs/common';
import type { Server } from 'socket.io';
import { EventBus } from '../../base';
import { RealtimeRegistry } from './registry';
import { stableStringify } from './stable-stringify';
import type { RealtimePublishPayload } from './types';
@Injectable()
@@ -46,7 +49,7 @@ export class RealtimePublisher {
const room = payload.room ?? handler.room(null, payload.input as never);
const envelope: RealtimeEvent = {
topic: payload.topic,
inputKey: stableStringify(payload.input),
inputKey: getRealtimeInputKey(payload.input),
sentAt: Date.now(),
event: payload.event as never,
};
@@ -0,0 +1,34 @@
export function realtimeUserRoom(userId: string, scope: string) {
return `user:${userId}:${scope}`;
}
export function realtimeWorkspaceRoom(workspaceId: string, scope: string) {
return `workspace:${workspaceId}:${scope}`;
}
export function realtimeWorkspaceDocRoom(
workspaceId: string,
docId: string,
scope: string
) {
return `workspace:${workspaceId}:doc:${docId}:${scope}`;
}
export function realtimeTranscriptTaskRoom(
workspaceId: string,
taskId: string
) {
return `copilot:transcript:${workspaceId}:${taskId}`;
}
export function realtimeNotificationRoom(userId: string) {
return realtimeUserRoom(userId, 'notification');
}
export function realtimeCommentRoom(workspaceId: string, docId: string) {
return realtimeWorkspaceDocRoom(workspaceId, docId, 'comment');
}
export function realtimeWorkspaceEmbeddingProgressRoom(workspaceId: string) {
return realtimeWorkspaceRoom(workspaceId, 'embedding-progress');
}
@@ -1,31 +0,0 @@
export function stableStringify(value: unknown): string {
if (
value === undefined ||
typeof value === 'function' ||
typeof value === 'symbol'
) {
return 'null';
}
if (value === null || typeof value !== 'object') {
return JSON.stringify(value);
}
if (Array.isArray(value)) {
return `[${value.map(stableStringify).join(',')}]`;
}
if (value instanceof Date) {
return JSON.stringify(value.toJSON());
}
const record = value as Record<string, unknown>;
return `{${Object.keys(record)
.filter(key => {
const property = record[key];
return (
property !== undefined &&
typeof property !== 'function' &&
typeof property !== 'symbol'
);
})
.sort()
.map(key => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
.join(',')}}`;
}
@@ -3,15 +3,17 @@ import { z } from 'zod';
import { OnEvent } from '../../../base';
import { AccessController } from '../../../core/permission';
import type {
import {
RealtimePublisher,
RealtimeRegistry,
realtimeWorkspaceEmbeddingProgressRoom,
registerRealtimeLiveQuery,
} from '../../../core/realtime';
import { Models } from '../../../models';
import { CopilotContextService } from './service';
export function workspaceEmbeddingRoom(workspaceId: string) {
return `workspace:${workspaceId}:embedding-progress`;
return realtimeWorkspaceEmbeddingProgressRoom(workspaceId);
}
@Injectable()
@@ -27,27 +29,28 @@ export class CopilotEmbeddingRealtimeProvider implements OnModuleInit {
onModuleInit() {
const input = z.object({ workspaceId: z.string() });
this.registry?.registerRequest({
name: 'workspace.embedding.progress.get',
input,
handle: async (user, payload) => {
await this.assertCopilot(user.id, payload.workspaceId);
if (!this.context.canEmbedding) {
return { total: 0, embedded: 0 };
}
return await this.models.copilotWorkspace.getEmbeddingStatus(
payload.workspaceId
);
registerRealtimeLiveQuery(this.registry, {
request: {
name: 'workspace.embedding.progress.get',
input,
handle: async (user, payload) => {
await this.assertCopilot(user.id, payload.workspaceId);
if (!this.context.canEmbedding) {
return { total: 0, embedded: 0 };
}
return await this.models.copilotWorkspace.getEmbeddingStatus(
payload.workspaceId
);
},
},
});
this.registry?.registerTopic({
name: 'workspace.embedding.progress.changed',
input,
authorize: async (user, payload) => {
await this.assertCopilot(user.id, payload.workspaceId);
topic: {
name: 'workspace.embedding.progress.changed',
input,
authorize: async (user, payload) => {
await this.assertCopilot(user.id, payload.workspaceId);
},
room: (_user, payload) => workspaceEmbeddingRoom(payload.workspaceId),
},
room: (_user, payload) => workspaceEmbeddingRoom(payload.workspaceId),
});
}
@@ -3,8 +3,12 @@ import { z } from 'zod';
import { CopilotTranscriptionJobNotFound } from '../../../base';
import { AccessController } from '../../../core/permission';
import type { RealtimeRegistry } from '../../../core/realtime';
import { CopilotTranscriptionService, transcriptTaskRoom } from './service';
import {
type RealtimeRegistry,
realtimeTranscriptTaskRoom,
registerRealtimeLiveQuery,
} from '../../../core/realtime';
import { CopilotTranscriptionService } from './service';
@Injectable()
export class CopilotTranscriptRealtimeProvider implements OnModuleInit {
@@ -15,47 +19,51 @@ export class CopilotTranscriptRealtimeProvider implements OnModuleInit {
) {}
onModuleInit() {
this.registry?.registerRequest({
name: 'copilot.transcript.task.get',
input: z
.object({
workspaceId: z.string(),
blobId: z.string().optional(),
taskId: z.string().optional(),
})
.refine(input => input.blobId || input.taskId),
handle: async (user, input) => {
await this.assertCopilot(user.id, input.workspaceId);
return {
task: await this.transcript.queryTask(
user.id,
input.workspaceId,
input.taskId,
input.blobId
),
};
},
const requestInput = z
.object({
workspaceId: z.string(),
blobId: z.string().optional(),
taskId: z.string().optional(),
})
.refine(input => input.blobId || input.taskId);
const topicInput = z.object({
workspaceId: z.string(),
taskId: z.string(),
});
this.registry?.registerTopic({
name: 'copilot.transcript.task.changed',
input: z.object({
workspaceId: z.string(),
taskId: z.string(),
}),
authorize: async (user, input) => {
await this.assertCopilot(user.id, input.workspaceId);
const task = await this.transcript.queryTask(
user.id,
input.workspaceId,
input.taskId
);
if (!task) {
throw new CopilotTranscriptionJobNotFound();
}
registerRealtimeLiveQuery(this.registry, {
request: {
name: 'copilot.transcript.task.get',
input: requestInput,
handle: async (user, input) => {
await this.assertCopilot(user.id, input.workspaceId);
return {
task: await this.transcript.queryTask(
user.id,
input.workspaceId,
input.taskId,
input.blobId
),
};
},
},
topic: {
name: 'copilot.transcript.task.changed',
input: topicInput,
authorize: async (user, input) => {
await this.assertCopilot(user.id, input.workspaceId);
const task = await this.transcript.queryTask(
user.id,
input.workspaceId,
input.taskId
);
if (!task) {
throw new CopilotTranscriptionJobNotFound();
}
},
room: (_user, input) =>
realtimeTranscriptTaskRoom(input.workspaceId, input.taskId),
},
room: (_user, input) =>
transcriptTaskRoom(input.workspaceId, input.taskId),
});
}
@@ -9,7 +9,10 @@ import {
OnJob,
sniffMime,
} from '../../../base';
import type { RealtimePublisher } from '../../../core/realtime';
import {
type RealtimePublisher,
realtimeTranscriptTaskRoom,
} from '../../../core/realtime';
import { Models } from '../../../models';
import { CopilotAccessPolicy } from '../access';
import { PromptService } from '../prompt';
@@ -33,10 +36,6 @@ const TRANSCRIPT_ACTION_ID = 'transcript.audio.gemini';
const TRANSCRIPT_ACTION_VERSION = 'v1';
const TRANSCRIPT_STRATEGY = 'gemini';
export function transcriptTaskRoom(workspaceId: string, taskId: string) {
return `copilot:transcript:${workspaceId}:${taskId}`;
}
export type TranscriptionJob = {
id: string;
status: AiJobStatus;
@@ -468,7 +467,7 @@ export class CopilotTranscriptionService {
'copilot.transcript.task.changed',
{ workspaceId, taskId },
{ taskId, status, error },
{ room: transcriptTaskRoom(workspaceId, taskId) }
{ room: realtimeTranscriptTaskRoom(workspaceId, taskId) }
);
}
}