feat(server): improve oidc compatibility (#14686)

fix #13938 
fix #14683 
fix #14532

#### PR Dependency Tree


* **PR #14686** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Flexible OIDC claim mapping for email/name, automatic OIDC discovery
retry with exponential backoff, and explicit OAuth flow modes (popup vs
redirect) propagated through the auth flow.

* **Bug Fixes**
* Stricter OIDC email validation, clearer error messages listing
attempted claim candidates, and improved callback redirect handling for
various flow scenarios.

* **Tests**
* Added unit tests covering OIDC behaviors, backoff scheduler/promise
utilities, and frontend OAuth flow parsing/redirect logic.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/frontend/core/src/__tests__/oauth-flow.spec.ts

@@ -0,0 +1,63 @@
+import {
+  attachOAuthFlowToAuthUrl,
+  parseOAuthCallbackState,
+  resolveOAuthFlowMode,
+  resolveOAuthRedirect,
+} from '@affine/core/desktop/pages/auth/oauth-flow';
+import { describe, expect, test } from 'vitest';
+
+describe('oauth flow mode', () => {
+  test('defaults to redirect for missing or unknown values', () => {
+    expect(resolveOAuthFlowMode()).toBe('redirect');
+    expect(resolveOAuthFlowMode(null)).toBe('redirect');
+    expect(resolveOAuthFlowMode('unknown')).toBe('redirect');
+  });
+
+  test('persists flow in oauth state instead of web storage', () => {
+    const url = attachOAuthFlowToAuthUrl(
+      'https://example.com/auth?state=%7B%22state%22%3A%22nonce%22%2C%22provider%22%3A%22Google%22%2C%22client%22%3A%22web%22%7D',
+      'redirect'
+    );
+
+    expect(
+      parseOAuthCallbackState(new URL(url).searchParams.get('state')!)
+    ).toEqual({
+      client: 'web',
+      flow: 'redirect',
+      provider: 'Google',
+      state: 'nonce',
+    });
+  });
+
+  test('falls back to popup when callback state has no flow', () => {
+    expect(
+      parseOAuthCallbackState(
+        JSON.stringify({ client: 'web', provider: 'Google', state: 'nonce' })
+      ).flow
+    ).toBe('popup');
+  });
+
+  test('keeps same-origin redirects direct', () => {
+    expect(resolveOAuthRedirect('/workspace', 'https://app.affine.pro')).toBe(
+      '/workspace'
+    );
+
+    expect(
+      resolveOAuthRedirect(
+        'https://app.affine.pro/workspace?from=oauth',
+        'https://app.affine.pro'
+      )
+    ).toBe('https://app.affine.pro/workspace?from=oauth');
+  });
+
+  test('wraps external redirects with redirect-proxy', () => {
+    expect(
+      resolveOAuthRedirect(
+        'https://github.com/toeverything/AFFiNE',
+        'https://app.affine.pro'
+      )
+    ).toBe(
+      'https://app.affine.pro/redirect-proxy?redirect_uri=https%3A%2F%2Fgithub.com%2Ftoeverything%2FAFFiNE'
+    );
+  });
+});

packages/frontend/core/src/components/affine/auth/oauth.tsx

@@ -73,11 +73,13 @@ export function OAuth({ redirectUrl }: { redirectUrl?: string }) {
               params.set('redirect_uri', redirectUrl);
             }
 
+            params.set('flow', 'redirect');
+
             const oauthUrl =
               serverService.server.baseUrl +
               `/oauth/login?${params.toString()}`;
 
-            urlService.openPopupWindow(oauthUrl);
+            urlService.openExternal(oauthUrl);
           };
 
       const ret = open();

packages/frontend/core/src/desktop/pages/auth/oauth-callback.tsx

@@ -13,10 +13,16 @@ import {
   buildOpenAppUrlRoute,
 } from '../../../modules/open-in-app';
 import { supportedClient } from './common';
+import {
+  type OAuthFlowMode,
+  parseOAuthCallbackState,
+  resolveOAuthRedirect,
+} from './oauth-flow';
 
 interface LoaderData {
   state: string;
   code: string;
+  flow: OAuthFlowMode;
   provider: string;
 }
 
@@ -31,12 +37,18 @@ export const loader: LoaderFunction = async ({ request }) => {
   }
 
   try {
-    const { state, client, provider } = JSON.parse(stateStr);
+    const { state, client, flow, provider } = parseOAuthCallbackState(stateStr);
+
+    if (!state || !provider) {
+      return redirect('/sign-in?error=Invalid oauth callback parameters');
+    }
+
     stateStr = state;
 
     const payload: LoaderData = {
       state,
       code,
+      flow,
       provider,
     };
 
@@ -79,8 +91,13 @@ export const Component = () => {
     triggeredRef.current = true;
     auth
       .signInOauth(data.code, data.state, data.provider)
-      .then(() => {
-        window.close();
+      .then(({ redirectUri }) => {
+        if (data.flow === 'popup') {
+          window.close();
+          return;
+        }
+
+        location.replace(resolveOAuthRedirect(redirectUri, location.origin));
       })
       .catch(e => {
         nav(`/sign-in?error=${encodeURIComponent(e.message)}`);

packages/frontend/core/src/desktop/pages/auth/oauth-flow.ts

@@ -0,0 +1,73 @@
+export const oauthFlowModes = ['popup', 'redirect'] as const;
+
+export type OAuthFlowMode = (typeof oauthFlowModes)[number];
+
+export function resolveOAuthFlowMode(
+  mode?: string | null,
+  fallback: OAuthFlowMode = 'redirect'
+): OAuthFlowMode {
+  return mode === 'popup' || mode === 'redirect' ? mode : fallback;
+}
+
+export function attachOAuthFlowToAuthUrl(url: string, flow: OAuthFlowMode) {
+  const authUrl = new URL(url);
+  const state = authUrl.searchParams.get('state');
+  if (!state) return url;
+
+  try {
+    const payload = JSON.parse(state) as Record<string, unknown>;
+    authUrl.searchParams.set('state', JSON.stringify({ ...payload, flow }));
+    return authUrl.toString();
+  } catch {
+    return url;
+  }
+}
+
+export function readOAuthFlowModeFromCallbackState(state: string | null) {
+  if (!state) return 'popup';
+
+  try {
+    const payload = JSON.parse(state) as { flow?: string };
+    return resolveOAuthFlowMode(payload.flow, 'popup');
+  } catch {
+    return 'popup';
+  }
+}
+
+export function parseOAuthCallbackState(state: string) {
+  const parsed = JSON.parse(state) as {
+    client?: string;
+    provider?: string;
+    state?: string;
+  };
+
+  return {
+    client: parsed.client,
+    flow: readOAuthFlowModeFromCallbackState(state),
+    provider: parsed.provider,
+    state: parsed.state,
+  };
+}
+
+export function resolveOAuthRedirect(
+  redirectUri: string | null | undefined,
+  currentOrigin: string
+) {
+  if (!redirectUri) return '/';
+  if (redirectUri.startsWith('/') && !redirectUri.startsWith('//')) {
+    return redirectUri;
+  }
+
+  let target: URL;
+  try {
+    target = new URL(redirectUri);
+  } catch {
+    return '/';
+  }
+
+  if (target.origin === currentOrigin) return target.toString();
+
+  const redirectProxy = new URL('/redirect-proxy', currentOrigin);
+  redirectProxy.searchParams.set('redirect_uri', target.toString());
+  return redirectProxy.toString();
+}

packages/frontend/core/src/desktop/pages/auth/oauth-login.tsx

@@ -12,6 +12,7 @@ import {
 import { z } from 'zod';
 
 import { supportedClient } from './common';
+import { attachOAuthFlowToAuthUrl, resolveOAuthFlowMode } from './oauth-flow';
 
 const supportedProvider = z.nativeEnum(OAuthProviderType);
 const CSRF_COOKIE_NAME = 'affine_csrf_token';
@@ -36,12 +37,14 @@ const oauthParameters = z.object({
   provider: supportedProvider,
   client: supportedClient,
   redirectUri: z.string().optional().nullable(),
+  flow: z.string().optional().nullable(),
 });
 
 interface LoaderData {
   provider: OAuthProviderType;
   client: string;
   redirectUri?: string;
+  flow: string;
 }
 
 export const loader: LoaderFunction = async ({ request }) => {
@@ -50,6 +53,7 @@ export const loader: LoaderFunction = async ({ request }) => {
   const provider = searchParams.get('provider');
   const client = searchParams.get('client') ?? 'web';
   const redirectUri = searchParams.get('redirect_uri');
+  const flow = searchParams.get('flow');
 
   // sign out first, web only
   if (client === 'web') {
@@ -64,6 +68,7 @@ export const loader: LoaderFunction = async ({ request }) => {
     provider,
     client,
     redirectUri,
+    flow,
   });
 
   if (paramsParseResult.success) {
@@ -71,6 +76,7 @@ export const loader: LoaderFunction = async ({ request }) => {
       provider,
       client,
       redirectUri,
+      flow: resolveOAuthFlowMode(flow),
     };
   }
 
@@ -90,7 +96,10 @@ export const Component = () => {
       .oauthPreflight(data.provider, data.client, data.redirectUri)
       .then(({ url }) => {
         // this is the url of oauth provider auth page, can't navigate with react-router
-        location.href = url;
+        location.href = attachOAuthFlowToAuthUrl(
+          url,
+          resolveOAuthFlowMode(data.flow)
+        );
       })
       .catch(e => {
         nav(`/sign-in?error=${encodeURIComponent(e.message)}`);