refactor(server): indexer & worker & sync perf (#15504)

packages/frontend/admin/src/config.json

@@ -95,6 +95,10 @@
       "type": "Boolean",
       "desc": "Whether request abuse source facts should trust Cloudflare headers from the origin edge."
     },
+    "signInRateLimit": {
+      "type": "Object",
+      "desc": "Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds."
+    },
     "inviteQuotaShadowMode": {
       "type": "Boolean",
       "desc": "Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions."
@@ -298,13 +302,6 @@
       "desc": "Whether allow guest users to create demo workspaces."
     }
   },
-  "docService": {
-    "endpoint": {
-      "type": "String",
-      "desc": "The endpoint of the doc service.",
-      "env": "DOC_SERVICE_ENDPOINT"
-    }
-  },
   "telemetry": {
     "allowedOrigin": {
       "type": "Array",
@@ -406,12 +403,12 @@
     },
     "provider.type": {
       "type": "String",
-      "desc": "Indexer search service provider name",
+      "desc": "Indexer search provider. Self-hosted uses the embedded provider by default; remote providers require an endpoint.",
       "env": "AFFINE_INDEXER_SEARCH_PROVIDER"
     },
     "provider.endpoint": {
       "type": "String",
-      "desc": "Indexer search service endpoint",
+      "desc": "Remote indexer endpoint. Not used by the embedded provider.",
       "env": "AFFINE_INDEXER_SEARCH_ENDPOINT"
     },
     "provider.apiKey": {

packages/frontend/admin/src/modules/settings/config.ts

@@ -163,6 +163,23 @@ export const KNOWN_CONFIG_GROUPS = [
       },
     ],
   } as ConfigGroup<'copilot'>,
+  {
+    name: 'Indexer',
+    module: 'indexer',
+    fields: [
+      {
+        key: 'provider.type',
+        type: 'Enum',
+        options: ['embedded', 'manticoresearch', 'elasticsearch'],
+        desc: 'Search provider. Embedded keeps external credentials for later reuse.',
+      },
+      'provider.endpoint',
+      'provider.apiKey',
+      'provider.username',
+      'provider.password',
+      'autoIndex.batchSize',
+    ],
+  } as ConfigGroup<'indexer'>,
 ];
 
 export const UNKNOWN_CONFIG_GROUPS = ALL_CONFIGURABLE_MODULES.filter(

packages/frontend/admin/src/modules/settings/index.spec.tsx

@@ -24,13 +24,26 @@ vi.mock('../header', () => ({
 vi.mock('./config-input-row', () => ({
   ConfigRow: ({
     field,
+    defaultValue,
+    onChange,
     onErrorChange,
   }: {
     field: string;
+    defaultValue?: unknown;
+    onChange?: (field: string, value: unknown) => void;
     onErrorChange?: (field: string, error?: string) => void;
   }) => (
     <div data-testid={`field-${field}`}>
-      <div>{field}</div>
+      <div>{`${field}:${defaultValue}`}</div>
+      <button type="button" onClick={() => onChange?.(field, 'embedded')}>
+        set-embedded-{field}
+      </button>
+      <button
+        type="button"
+        onClick={() => onChange?.(field, 'manticoresearch')}
+      >
+        set-manticoresearch-{field}
+      </button>
       <button
         type="button"
         onClick={() => {
@@ -65,6 +78,16 @@ vi.mock('./config', () => ({
         type: 'Boolean',
       },
     },
+    indexer: {
+      'provider.type': {
+        desc: 'Provider',
+        type: 'String',
+      },
+      'provider.endpoint': {
+        desc: 'Endpoint',
+        type: 'String',
+      },
+    },
   },
   ALL_SETTING_GROUPS: [
     {
@@ -77,6 +100,18 @@ vi.mock('./config', () => ({
       module: 'auth',
       fields: ['allowSignup'],
     },
+    {
+      name: 'Indexer',
+      module: 'indexer',
+      fields: [
+        {
+          key: 'provider.type',
+          type: 'Enum',
+          options: ['embedded', 'manticoresearch', 'elasticsearch'],
+        },
+        'provider.endpoint',
+      ],
+    },
   ],
 }));
 
@@ -93,6 +128,13 @@ describe('SettingsPage', () => {
         auth: {
           allowSignup: true,
         },
+        indexer: {
+          enabled: false,
+          provider: {
+            type: 'elasticsearch',
+            endpoint: 'http://search.example',
+          },
+        },
       },
       patchedAppConfig: {
         server: {
@@ -101,6 +143,13 @@ describe('SettingsPage', () => {
         auth: {
           allowSignup: true,
         },
+        indexer: {
+          enabled: false,
+          provider: {
+            type: 'elasticsearch',
+            endpoint: 'http://search.example',
+          },
+        },
       },
       update: vi.fn(),
       saveGroup: vi.fn().mockResolvedValue(undefined),
@@ -148,6 +197,46 @@ describe('SettingsPage', () => {
     expect(authItem?.dataset.state).toBe('open');
   });
 
+  test('encodes embedded without replacing external provider settings', () => {
+    const update = vi.fn();
+    useAppConfigMock.mockReturnValue({
+      ...useAppConfigMock(),
+      update,
+    });
+    render(
+      <MemoryRouter initialEntries={['/admin/settings']}>
+        <Routes>
+          <Route path="/admin/settings" element={<SettingsPage />} />
+        </Routes>
+      </MemoryRouter>
+    );
+
+    fireEvent.click(screen.getAllByRole('button', { name: /Indexer/i })[0]);
+    expect(screen.getByText('indexer/provider.type:embedded')).toBeTruthy();
+    expect(screen.queryByTestId('field-indexer/provider.endpoint')).toBeNull();
+    fireEvent.click(
+      screen.getByRole('button', {
+        name: 'set-embedded-indexer/provider.type',
+      })
+    );
+    expect(update).toHaveBeenCalledWith('indexer/enabled', false);
+    expect(update).not.toHaveBeenCalledWith(
+      'indexer/provider.type',
+      'embedded'
+    );
+
+    fireEvent.click(
+      screen.getByRole('button', {
+        name: 'set-manticoresearch-indexer/provider.type',
+      })
+    );
+    expect(update).toHaveBeenCalledWith('indexer/enabled', true);
+    expect(update).toHaveBeenCalledWith(
+      'indexer/provider.type',
+      'manticoresearch'
+    );
+  });
+
   test('disables save when group has validation errors even if group is dirty', () => {
     useAppConfigMock.mockReset();
     useAppConfigMock.mockReturnValue({

packages/frontend/admin/src/modules/settings/index.tsx

@@ -173,6 +173,19 @@ const AdminPanel = ({
                     key={`${module}-${version}`}
                   >
                     {fields.map(field => {
+                      const fieldKey =
+                        typeof field === 'string' ? field : String(field.key);
+                      const effectiveIndexerProvider = sourceConfig?.enabled
+                        ? sourceConfig?.provider?.type
+                        : 'embedded';
+                      if (
+                        module === 'indexer' &&
+                        effectiveIndexerProvider === 'embedded' &&
+                        fieldKey.startsWith('provider.') &&
+                        fieldKey !== 'provider.type'
+                      ) {
+                        return null;
+                      }
                       let props: ConfigInputProps;
                       if (typeof field === 'string') {
                         const descriptor =
@@ -194,11 +207,26 @@ const AdminPanel = ({
                           type: field.type ?? descriptor.type,
                           // @ts-expect-error for enum type
                           options: field.options,
-                          defaultValue: get(
-                            sourceConfig,
-                            field.key + (field.sub ? '.' + field.sub : '')
-                          ),
-                          onChange: onUpdate,
+                          defaultValue:
+                            module === 'indexer' &&
+                            field.key === 'provider.type'
+                              ? effectiveIndexerProvider
+                              : get(
+                                  sourceConfig,
+                                  field.key + (field.sub ? '.' + field.sub : '')
+                                ),
+                          onChange:
+                            module === 'indexer' &&
+                            field.key === 'provider.type'
+                              ? (_path, value) => {
+                                  if (value === 'embedded') {
+                                    onUpdate('indexer/enabled', false);
+                                  } else {
+                                    onUpdate('indexer/enabled', true);
+                                    onUpdate('indexer/provider.type', value);
+                                  }
+                                }
+                              : onUpdate,
                         };
                       }