diff --git a/packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts b/packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts index e4121cf522..9b04783d3d 100644 --- a/packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts +++ b/packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts @@ -117,9 +117,12 @@ test('retry when connect failed', async () => { class TestConnection extends AutoReconnectConnection { override retryDelay = 300; connectCount = 0; + retryDelayFor(retryCount: number) { + return this.getRetryDelay(retryCount); + } override async doConnect() { this.connectCount++; - if (this.connectCount === 3) { + if (this.connectCount >= 3) { return { hello: 'world' }; } throw new Error('not connected, count: ' + this.connectCount); @@ -127,9 +130,15 @@ test('retry when connect failed', async () => { override doDisconnect() { return Promise.resolve(); } + triggerError(error: Error) { + this.error = error; + } } const connection = new TestConnection(); + expect([0, 1, 2, 8].map(count => connection.retryDelayFor(count))).toEqual([ + 300, 600, 1200, 60000, + ]); connection.connect(); await vitest.waitFor(() => { @@ -149,6 +158,12 @@ test('retry when connect failed', async () => { expect(connection.status).toBe('connected'); expect(connection.error).toBeUndefined(); }); + + connection.triggerError(new Error('disconnected')); + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(4); + expect(connection.status).toBe('connected'); + }); }); test('retry when error', async () => { diff --git a/packages/common/nbstore/src/connection/connection.ts b/packages/common/nbstore/src/connection/connection.ts index 9804fadcdf..2d97c1c03a 100644 --- a/packages/common/nbstore/src/connection/connection.ts +++ b/packages/common/nbstore/src/connection/connection.ts @@ -31,7 +31,9 @@ export abstract class AutoReconnectConnection< private _status: ConnectionStatus = 'idle'; private _error: Error | undefined = undefined; retryDelay = 3000; + maxRetryDelay = 60000; connectingTimeout = 15000; + private retryCount = 0; private refCount = 0; private connectingAbort?: AbortController; private reconnectingAbort?: AbortController; @@ -103,6 +105,7 @@ export abstract class AutoReconnectConnection< clearTimeout(timeout); if (!signal.aborted) { this._inner = value; + this.retryCount = 0; this.setStatus('connected'); } else { try { @@ -150,16 +153,21 @@ export abstract class AutoReconnectConnection< this.reconnectingAbort = new AbortController(); const signal = this.reconnectingAbort.signal; + const retryDelay = this.getRetryDelay(this.retryCount++); const timeout = setTimeout(() => { if (!signal.aborted) { this.innerConnect(); } - }, this.retryDelay); + }, retryDelay); signal.addEventListener('abort', () => { clearTimeout(timeout); }); } + protected getRetryDelay(retryCount: number) { + return Math.min(this.retryDelay * 2 ** retryCount, this.maxRetryDelay); + } + connect() { this.refCount++; if (this.refCount === 1) { @@ -175,6 +183,7 @@ export abstract class AutoReconnectConnection< } if (this.refCount === 0) { this.innerDisconnect(); + this.retryCount = 0; this.setStatus('closed'); } } diff --git a/packages/frontend/core/src/components/providers/workspace-side-effects.tsx b/packages/frontend/core/src/components/providers/workspace-side-effects.tsx index 51b7ecd73c..fd5277ce04 100644 --- a/packages/frontend/core/src/components/providers/workspace-side-effects.tsx +++ b/packages/frontend/core/src/components/providers/workspace-side-effects.tsx @@ -150,13 +150,23 @@ export const WorkspaceSideEffects = () => { useEffect(() => { if (!realtimeConnectionError) return; - notify.warning( + const message = { + authentication: + t['com.affine.realtime.connection-error.authentication'](), + network: t['com.affine.realtime.connection-error.network'](), + server: t['com.affine.realtime.connection-error.server'](), + timeout: t['com.affine.realtime.connection-error.timeout'](), + }[realtimeConnectionError.type]; + const id = notify.warning( { title: t['com.affine.realtime.connection-error.title'](), - message: t['com.affine.realtime.connection-error.message'](), + message, }, { id: `realtime-connection-error:${realtimeConnectionError.endpoint}` } ); + return () => { + notify.dismiss(id); + }; }, [realtimeConnectionError, t]); useEffect(() => { diff --git a/packages/frontend/core/src/modules/cloud/services/realtime.ts b/packages/frontend/core/src/modules/cloud/services/realtime.ts index d02136f75a..2ef28505d1 100644 --- a/packages/frontend/core/src/modules/cloud/services/realtime.ts +++ b/packages/frontend/core/src/modules/cloud/services/realtime.ts @@ -8,13 +8,62 @@ import type { NbstoreService } from '../../storage'; import type { Server } from '../entities/server'; import type { ServersService } from './servers'; +const CONNECTION_FAILURE_THRESHOLD = 3; +const INITIAL_RETRY_DELAY = 3000; +const MAX_RETRY_DELAY = 60000; + +function getConnectionErrorType(error: unknown) { + if (!(error instanceof Error)) { + return 'server'; + } + + const detail = `${error.name} ${error.message}`.toLowerCase(); + if ( + detail.includes('auth') || + detail.includes('forbidden') || + detail.includes('unauthorized') || + detail.includes('jwt') + ) { + return 'authentication'; + } + if (detail.includes('timeout') || detail.includes('timed out')) { + return 'timeout'; + } + if ( + detail.includes('network') || + detail.includes('transport') || + detail.includes('websocket') || + detail.includes('xhr') || + detail.includes('connect') + ) { + return 'network'; + } + return 'server'; +} + +function waitForRetry(delay: number, signal: AbortSignal) { + return new Promise(resolve => { + const onAbort = () => { + clearTimeout(timeout); + resolve(); + }; + const timeout = setTimeout(() => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, delay); + signal.addEventListener('abort', onAbort, { once: true }); + }); +} + @OnEvent(ApplicationStarted, service => service.onApplicationStarted) export class RealtimeService extends Service { readonly connectionError$ = new LiveData<{ endpoint: string; error: unknown; + type: ReturnType; } | null>(null); private contextGeneration = 0; + private probeAbort?: AbortController; private readonly currentServer$ = this.globalContextService.globalContext.serverId.$.selector(id => @@ -50,11 +99,17 @@ export class RealtimeService extends Service { const subscription = this.currentServer$.subscribe(context => { const generation = ++this.contextGeneration; - this.configure(context, generation).catch(error => { + this.probeAbort?.abort(); + const probeAbort = new AbortController(); + this.probeAbort = probeAbort; + this.configure(context, generation, probeAbort.signal).catch(error => { console.error('Failed to configure realtime context', error); }); }); - this.disposables.push(() => subscription.unsubscribe()); + this.disposables.push(() => { + subscription.unsubscribe(); + this.probeAbort?.abort(); + }); } onApplicationStarted() {} @@ -65,7 +120,8 @@ export class RealtimeService extends Service { authenticated: boolean; isSelfHosted: boolean; }, - generation: number + generation: number, + signal: AbortSignal ) { await this.nbstoreService.realtime.configure(context); if (generation !== this.contextGeneration) return; @@ -74,19 +130,34 @@ export class RealtimeService extends Service { return; } - try { - await this.nbstoreService.realtime.request( - 'user.profile.get', - {}, - { timeoutMs: 10_000 } - ); - if (generation === this.contextGeneration) { - this.connectionError$.next(null); - } - } catch (error) { - if (generation === this.contextGeneration) { - this.connectionError$.next({ endpoint: context.endpoint, error }); + let failures = 0; + let retryDelay = INITIAL_RETRY_DELAY; + while (generation === this.contextGeneration && !signal.aborted) { + try { + await this.nbstoreService.realtime.request( + 'user.profile.get', + {}, + { timeoutMs: 10_000, signal } + ); + if (generation === this.contextGeneration && !signal.aborted) { + this.connectionError$.next(null); + } + return; + } catch (error) { + if (signal.aborted || generation !== this.contextGeneration) { + return; + } + failures++; + if (failures >= CONNECTION_FAILURE_THRESHOLD) { + this.connectionError$.next({ + endpoint: context.endpoint, + error, + type: getConnectionErrorType(error), + }); + } } + await waitForRetry(retryDelay, signal); + retryDelay = Math.min(retryDelay * 2, MAX_RETRY_DELAY); } } } diff --git a/packages/frontend/i18n/src/i18n.gen.ts b/packages/frontend/i18n/src/i18n.gen.ts index 92b8d60c9e..fe9c232372 100644 --- a/packages/frontend/i18n/src/i18n.gen.ts +++ b/packages/frontend/i18n/src/i18n.gen.ts @@ -9681,9 +9681,21 @@ export function useAFFiNEI18N(): { */ ["com.affine.realtime.connection-error.title"](): string; /** - * `Check that your server proxy forwards /socket.io over WebSocket or HTTP polling.` + * `The real-time connection could not authenticate your session. Sign in again if the problem continues.` */ - ["com.affine.realtime.connection-error.message"](): string; + ["com.affine.realtime.connection-error.authentication"](): string; + /** + * `The server cannot be reached. Check your network connection and server proxy.` + */ + ["com.affine.realtime.connection-error.network"](): string; + /** + * `The server rejected the real-time request. Try again later.` + */ + ["com.affine.realtime.connection-error.server"](): string; + /** + * `The server did not respond in time. AFFiNE will keep trying to reconnect.` + */ + ["com.affine.realtime.connection-error.timeout"](): string; /** * `An internal error occurred.` */ diff --git a/packages/frontend/i18n/src/resources/de.json b/packages/frontend/i18n/src/resources/de.json index 2c238d9b78..f340116500 100644 --- a/packages/frontend/i18n/src/resources/de.json +++ b/packages/frontend/i18n/src/resources/de.json @@ -2307,7 +2307,6 @@ "com.affine.settings.devices.sign-out-failed": "Gerät konnte nicht abgemeldet werden", "com.affine.settings.devices.sign-out-all-failed": "Geräte konnten nicht abgemeldet werden", "com.affine.realtime.connection-error.title": "Echtzeitverbindung fehlgeschlagen", - "com.affine.realtime.connection-error.message": "Stelle sicher, dass dein Server-Proxy /socket.io über WebSocket oder HTTP-Polling weiterleitet.", "error.INTERNAL_SERVER_ERROR": "Es ist ein interner Fehler aufgetreten.", "error.NETWORK_ERROR": "Netzwerkfehler.", "error.TOO_MANY_REQUEST": "Zu viele Anfragen.", diff --git a/packages/frontend/i18n/src/resources/en.json b/packages/frontend/i18n/src/resources/en.json index e8ed6572fa..838842292f 100644 --- a/packages/frontend/i18n/src/resources/en.json +++ b/packages/frontend/i18n/src/resources/en.json @@ -2410,7 +2410,10 @@ "com.affine.settings.devices.sign-out-failed": "Failed to sign out device", "com.affine.settings.devices.sign-out-all-failed": "Failed to sign out devices", "com.affine.realtime.connection-error.title": "Real-time connection failed", - "com.affine.realtime.connection-error.message": "Check that your server proxy forwards /socket.io over WebSocket or HTTP polling.", + "com.affine.realtime.connection-error.authentication": "The real-time connection could not authenticate your session. Sign in again if the problem continues.", + "com.affine.realtime.connection-error.network": "The server cannot be reached. Check your network connection and server proxy.", + "com.affine.realtime.connection-error.server": "The server rejected the real-time request. Try again later.", + "com.affine.realtime.connection-error.timeout": "The server did not respond in time. AFFiNE will keep trying to reconnect.", "error.INTERNAL_SERVER_ERROR": "An internal error occurred.", "error.NETWORK_ERROR": "Network error.", "error.TOO_MANY_REQUEST": "Too many requests.", diff --git a/packages/frontend/i18n/src/resources/zh-Hans.json b/packages/frontend/i18n/src/resources/zh-Hans.json index d6b3333b96..90232f5acc 100644 --- a/packages/frontend/i18n/src/resources/zh-Hans.json +++ b/packages/frontend/i18n/src/resources/zh-Hans.json @@ -2307,7 +2307,6 @@ "error.INTERNAL_SERVER_ERROR": "发生内部错误。", "error.NETWORK_ERROR": "网络错误。", "com.affine.realtime.connection-error.title": "实时同步连接失败", - "com.affine.realtime.connection-error.message": "请检查服务器反向代理是否通过 WebSocket 或 HTTP polling 正确转发 /socket.io。", "error.TOO_MANY_REQUEST": "请求过多。", "error.SSRF_BLOCKED_ERROR": "无效的 URL", "error.RESPONSE_TOO_LARGE_ERROR": "响应过大（{{receivedBytes}} 字节），限制为 {{limitBytes}} 字节",