feat(core): improve reconnect handling (#15458)

#### PR Dependency Tree


* **PR #15458** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Bug Fixes**
* Improved realtime connection recovery with progressively longer retry
delays, capped to prevent excessive waiting.
* Connection retries now reset after successful reconnection and stop
cleanly when no longer needed.
* Reduced false error alerts by confirming repeated failures before
notifying users.
* Added cancellation handling when connection context changes or the
service is closed.

* **User Experience**
* Realtime connection alerts now identify authentication, network,
server, and timeout issues with localized messages.
* Active alerts are automatically dismissed when the connection
recovers.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts

@@ -117,9 +117,12 @@ test('retry when connect failed', async () => {
   class TestConnection extends AutoReconnectConnection {
     override retryDelay = 300;
     connectCount = 0;
+    retryDelayFor(retryCount: number) {
+      return this.getRetryDelay(retryCount);
+    }
     override async doConnect() {
       this.connectCount++;
-      if (this.connectCount === 3) {
+      if (this.connectCount >= 3) {
         return { hello: 'world' };
       }
       throw new Error('not connected, count: ' + this.connectCount);
@@ -127,9 +130,15 @@ test('retry when connect failed', async () => {
     override doDisconnect() {
       return Promise.resolve();
     }
+    triggerError(error: Error) {
+      this.error = error;
+    }
   }
 
   const connection = new TestConnection();
+  expect([0, 1, 2, 8].map(count => connection.retryDelayFor(count))).toEqual([
+    300, 600, 1200, 60000,
+  ]);
   connection.connect();
 
   await vitest.waitFor(() => {
@@ -149,6 +158,12 @@ test('retry when connect failed', async () => {
     expect(connection.status).toBe('connected');
     expect(connection.error).toBeUndefined();
   });
+
+  connection.triggerError(new Error('disconnected'));
+  await vitest.waitFor(() => {
+    expect(connection.connectCount).toBe(4);
+    expect(connection.status).toBe('connected');
+  });
 });
 
 test('retry when error', async () => {

packages/common/nbstore/src/connection/connection.ts

@@ -31,7 +31,9 @@ export abstract class AutoReconnectConnection<
   private _status: ConnectionStatus = 'idle';
   private _error: Error | undefined = undefined;
   retryDelay = 3000;
+  maxRetryDelay = 60000;
   connectingTimeout = 15000;
+  private retryCount = 0;
   private refCount = 0;
   private connectingAbort?: AbortController;
   private reconnectingAbort?: AbortController;
@@ -103,6 +105,7 @@ export abstract class AutoReconnectConnection<
           clearTimeout(timeout);
           if (!signal.aborted) {
             this._inner = value;
+            this.retryCount = 0;
             this.setStatus('connected');
           } else {
             try {
@@ -150,16 +153,21 @@ export abstract class AutoReconnectConnection<
 
     this.reconnectingAbort = new AbortController();
     const signal = this.reconnectingAbort.signal;
+    const retryDelay = this.getRetryDelay(this.retryCount++);
     const timeout = setTimeout(() => {
       if (!signal.aborted) {
         this.innerConnect();
       }
-    }, this.retryDelay);
+    }, retryDelay);
     signal.addEventListener('abort', () => {
       clearTimeout(timeout);
     });
   }
 
+  protected getRetryDelay(retryCount: number) {
+    return Math.min(this.retryDelay * 2 ** retryCount, this.maxRetryDelay);
+  }
+
   connect() {
     this.refCount++;
     if (this.refCount === 1) {
@@ -175,6 +183,7 @@ export abstract class AutoReconnectConnection<
     }
     if (this.refCount === 0) {
       this.innerDisconnect();
+      this.retryCount = 0;
       this.setStatus('closed');
     }
   }

packages/frontend/core/src/components/providers/workspace-side-effects.tsx

@@ -150,13 +150,23 @@ export const WorkspaceSideEffects = () => {
 
   useEffect(() => {
     if (!realtimeConnectionError) return;
-    notify.warning(
+    const message = {
+      authentication:
+        t['com.affine.realtime.connection-error.authentication'](),
+      network: t['com.affine.realtime.connection-error.network'](),
+      server: t['com.affine.realtime.connection-error.server'](),
+      timeout: t['com.affine.realtime.connection-error.timeout'](),
+    }[realtimeConnectionError.type];
+    const id = notify.warning(
       {
         title: t['com.affine.realtime.connection-error.title'](),
-        message: t['com.affine.realtime.connection-error.message'](),
+        message,
       },
       { id: `realtime-connection-error:${realtimeConnectionError.endpoint}` }
     );
+    return () => {
+      notify.dismiss(id);
+    };
   }, [realtimeConnectionError, t]);
 
   useEffect(() => {

packages/frontend/core/src/modules/cloud/services/realtime.ts

@@ -8,13 +8,62 @@ import type { NbstoreService } from '../../storage';
 import type { Server } from '../entities/server';
 import type { ServersService } from './servers';
 
+const CONNECTION_FAILURE_THRESHOLD = 3;
+const INITIAL_RETRY_DELAY = 3000;
+const MAX_RETRY_DELAY = 60000;
+
+function getConnectionErrorType(error: unknown) {
+  if (!(error instanceof Error)) {
+    return 'server';
+  }
+
+  const detail = `${error.name} ${error.message}`.toLowerCase();
+  if (
+    detail.includes('auth') ||
+    detail.includes('forbidden') ||
+    detail.includes('unauthorized') ||
+    detail.includes('jwt')
+  ) {
+    return 'authentication';
+  }
+  if (detail.includes('timeout') || detail.includes('timed out')) {
+    return 'timeout';
+  }
+  if (
+    detail.includes('network') ||
+    detail.includes('transport') ||
+    detail.includes('websocket') ||
+    detail.includes('xhr') ||
+    detail.includes('connect')
+  ) {
+    return 'network';
+  }
+  return 'server';
+}
+
+function waitForRetry(delay: number, signal: AbortSignal) {
+  return new Promise<void>(resolve => {
+    const onAbort = () => {
+      clearTimeout(timeout);
+      resolve();
+    };
+    const timeout = setTimeout(() => {
+      signal.removeEventListener('abort', onAbort);
+      resolve();
+    }, delay);
+    signal.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
 @OnEvent(ApplicationStarted, service => service.onApplicationStarted)
 export class RealtimeService extends Service {
   readonly connectionError$ = new LiveData<{
     endpoint: string;
     error: unknown;
+    type: ReturnType<typeof getConnectionErrorType>;
   } | null>(null);
   private contextGeneration = 0;
+  private probeAbort?: AbortController;
 
   private readonly currentServer$ =
     this.globalContextService.globalContext.serverId.$.selector(id =>
@@ -50,11 +99,17 @@ export class RealtimeService extends Service {
 
     const subscription = this.currentServer$.subscribe(context => {
       const generation = ++this.contextGeneration;
-      this.configure(context, generation).catch(error => {
+      this.probeAbort?.abort();
+      const probeAbort = new AbortController();
+      this.probeAbort = probeAbort;
+      this.configure(context, generation, probeAbort.signal).catch(error => {
         console.error('Failed to configure realtime context', error);
       });
     });
-    this.disposables.push(() => subscription.unsubscribe());
+    this.disposables.push(() => {
+      subscription.unsubscribe();
+      this.probeAbort?.abort();
+    });
   }
 
   onApplicationStarted() {}
@@ -65,7 +120,8 @@ export class RealtimeService extends Service {
       authenticated: boolean;
       isSelfHosted: boolean;
     },
-    generation: number
+    generation: number,
+    signal: AbortSignal
   ) {
     await this.nbstoreService.realtime.configure(context);
     if (generation !== this.contextGeneration) return;
@@ -74,19 +130,34 @@ export class RealtimeService extends Service {
       return;
     }
 
-    try {
-      await this.nbstoreService.realtime.request(
-        'user.profile.get',
-        {},
-        { timeoutMs: 10_000 }
-      );
-      if (generation === this.contextGeneration) {
-        this.connectionError$.next(null);
-      }
-    } catch (error) {
-      if (generation === this.contextGeneration) {
-        this.connectionError$.next({ endpoint: context.endpoint, error });
+    let failures = 0;
+    let retryDelay = INITIAL_RETRY_DELAY;
+    while (generation === this.contextGeneration && !signal.aborted) {
+      try {
+        await this.nbstoreService.realtime.request(
+          'user.profile.get',
+          {},
+          { timeoutMs: 10_000, signal }
+        );
+        if (generation === this.contextGeneration && !signal.aborted) {
+          this.connectionError$.next(null);
+        }
+        return;
+      } catch (error) {
+        if (signal.aborted || generation !== this.contextGeneration) {
+          return;
+        }
+        failures++;
+        if (failures >= CONNECTION_FAILURE_THRESHOLD) {
+          this.connectionError$.next({
+            endpoint: context.endpoint,
+            error,
+            type: getConnectionErrorType(error),
+          });
+        }
       }
+      await waitForRetry(retryDelay, signal);
+      retryDelay = Math.min(retryDelay * 2, MAX_RETRY_DELAY);
     }
   }
 }

packages/frontend/i18n/src/i18n.gen.ts

@@ -9681,9 +9681,21 @@ export function useAFFiNEI18N(): {
       */
     ["com.affine.realtime.connection-error.title"](): string;
     /**
-      * `Check that your server proxy forwards /socket.io over WebSocket or HTTP polling.`
+      * `The real-time connection could not authenticate your session. Sign in again if the problem continues.`
       */
-    ["com.affine.realtime.connection-error.message"](): string;
+    ["com.affine.realtime.connection-error.authentication"](): string;
+    /**
+      * `The server cannot be reached. Check your network connection and server proxy.`
+      */
+    ["com.affine.realtime.connection-error.network"](): string;
+    /**
+      * `The server rejected the real-time request. Try again later.`
+      */
+    ["com.affine.realtime.connection-error.server"](): string;
+    /**
+      * `The server did not respond in time. AFFiNE will keep trying to reconnect.`
+      */
+    ["com.affine.realtime.connection-error.timeout"](): string;
     /**
       * `An internal error occurred.`
       */

packages/frontend/i18n/src/resources/de.json

@@ -2307,7 +2307,6 @@
   "com.affine.settings.devices.sign-out-failed": "Gerät konnte nicht abgemeldet werden",
   "com.affine.settings.devices.sign-out-all-failed": "Geräte konnten nicht abgemeldet werden",
   "com.affine.realtime.connection-error.title": "Echtzeitverbindung fehlgeschlagen",
-  "com.affine.realtime.connection-error.message": "Stelle sicher, dass dein Server-Proxy /socket.io über WebSocket oder HTTP-Polling weiterleitet.",
   "error.INTERNAL_SERVER_ERROR": "Es ist ein interner Fehler aufgetreten.",
   "error.NETWORK_ERROR": "Netzwerkfehler.",
   "error.TOO_MANY_REQUEST": "Zu viele Anfragen.",

packages/frontend/i18n/src/resources/en.json

@@ -2410,7 +2410,10 @@
   "com.affine.settings.devices.sign-out-failed": "Failed to sign out device",
   "com.affine.settings.devices.sign-out-all-failed": "Failed to sign out devices",
   "com.affine.realtime.connection-error.title": "Real-time connection failed",
-  "com.affine.realtime.connection-error.message": "Check that your server proxy forwards /socket.io over WebSocket or HTTP polling.",
+  "com.affine.realtime.connection-error.authentication": "The real-time connection could not authenticate your session. Sign in again if the problem continues.",
+  "com.affine.realtime.connection-error.network": "The server cannot be reached. Check your network connection and server proxy.",
+  "com.affine.realtime.connection-error.server": "The server rejected the real-time request. Try again later.",
+  "com.affine.realtime.connection-error.timeout": "The server did not respond in time. AFFiNE will keep trying to reconnect.",
   "error.INTERNAL_SERVER_ERROR": "An internal error occurred.",
   "error.NETWORK_ERROR": "Network error.",
   "error.TOO_MANY_REQUEST": "Too many requests.",

packages/frontend/i18n/src/resources/zh-Hans.json

@@ -2307,7 +2307,6 @@
   "error.INTERNAL_SERVER_ERROR": "发生内部错误。",
   "error.NETWORK_ERROR": "网络错误。",
   "com.affine.realtime.connection-error.title": "实时同步连接失败",
-  "com.affine.realtime.connection-error.message": "请检查服务器反向代理是否通过 WebSocket 或 HTTP polling 正确转发 /socket.io。",
   "error.TOO_MANY_REQUEST": "请求过多。",
   "error.SSRF_BLOCKED_ERROR": "无效的 URL",
   "error.RESPONSE_TOO_LARGE_ERROR": "响应过大（{{receivedBytes}} 字节），限制为 {{limitBytes}} 字节",