Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-09-02 14:49:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): improve indexer perf (#15512)

Browse Source 
#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-08-23 17:54:49 +08:00
committed by GitHub
parent a8eef53966
commit b530198a3b
107 changed files with 8612 additions and 5306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/frontend/admin/src/config.json
+8 -22
View File
@@ -35,10 +35,6 @@
"type": "Object",
"desc": "The config for doc job queue"
},
"queues.indexer": {
"type": "Object",
"desc": "The config for indexer job queue"
},
"queues.notification": {
"type": "Object",
"desc": "The config for notification job queue"
@@ -398,39 +394,29 @@
"indexer": {
"enabled": {
"type": "Boolean",
"desc": "Enable indexer plugin",
"env": "AFFINE_INDEXER_ENABLED"
"desc": "Enable indexer plugin"
},
"provider.type": {
"type": "String",
"desc": "Indexer search provider. Self-hosted uses the embedded provider by default; remote providers require an endpoint.",
"env": "AFFINE_INDEXER_SEARCH_PROVIDER"
"desc": "Indexer search provider. Self-hosted uses the embedded provider by default; remote providers require an endpoint."
},
"provider.endpoint": {
"type": "String",
"desc": "Remote indexer endpoint. Not used by the embedded provider.",
"env": "AFFINE_INDEXER_SEARCH_ENDPOINT"
"desc": "Remote indexer endpoint. Not used by the embedded provider."
},
"provider.apiKey": {
"type": "String",
"desc": "Indexer search service api key. Optional for elasticsearch",
"link": "https://www.elastic.co/guide/server/current/api-key.html",
"env": "AFFINE_INDEXER_SEARCH_API_KEY"
"desc": "Indexer search service api key. Optional for remote providers",
"link": "https://www.elastic.co/guide/server/current/api-key.html"
},
"provider.username": {
"type": "String",
"desc": "Indexer search service auth username, if not set, basic auth will be disabled. Optional for elasticsearch",
"link": "https://www.elastic.co/guide/en/elasticsearch/reference/current/http-clients.html",
"env": "AFFINE_INDEXER_SEARCH_USERNAME"
"desc": "Indexer search service auth username, if not set, basic auth will be disabled. Optional for remote providers",
"link": "https://www.elastic.co/guide/en/elasticsearch/reference/current/http-clients.html"
},
"provider.password": {
"type": "String",
"desc": "Indexer search service auth password, if not set, basic auth will be disabled. Optional for elasticsearch",
"env": "AFFINE_INDEXER_SEARCH_PASSWORD"
},
"autoIndex.batchSize": {
"type": "Number",
"desc": "Number of workspaces automatically indexed per batch"
"desc": "Indexer search service auth password, if not set, basic auth will be disabled. Optional for remote providers"
}
},
"oauth": {
packages/frontend/admin/src/modules/settings/config.ts
+2 -3
View File
@@ -170,14 +170,13 @@ export const KNOWN_CONFIG_GROUPS = [
{
key: 'provider.type',
type: 'Enum',
options: ['embedded', 'manticoresearch', 'elasticsearch'],
desc: 'Search provider. Embedded keeps external credentials for later reuse.',
options: ['embedded', 'elasticsearch', 'manticoresearch'],
desc: 'Search provider. Embedded and Elasticsearch provide full search semantics; Manticore Search provides basic search semantics.',
},
'provider.endpoint',
'provider.apiKey',
'provider.username',
'provider.password',
'autoIndex.batchSize',
],
} as ConfigGroup<'indexer'>,
];
packages/frontend/admin/src/modules/settings/index.spec.tsx
+12 -16
View File
@@ -38,11 +38,8 @@ vi.mock('./config-input-row', () => ({
<button type="button" onClick={() => onChange?.(field, 'embedded')}>
set-embedded-{field}
</button>
<button
type="button"
onClick={() => onChange?.(field, 'manticoresearch')}
>
set-manticoresearch-{field}
<button type="button" onClick={() => onChange?.(field, 'elasticsearch')}>
set-elasticsearch-{field}
</button>
<button
type="button"
@@ -107,7 +104,7 @@ vi.mock('./config', () => ({
{
key: 'provider.type',
type: 'Enum',
options: ['embedded', 'manticoresearch', 'elasticsearch'],
options: ['embedded', 'elasticsearch', 'manticoresearch'],
},
'provider.endpoint',
],
@@ -197,7 +194,7 @@ describe('SettingsPage', () => {
expect(authItem?.dataset.state).toBe('open');
});
test('encodes embedded without replacing external provider settings', () => {
test('enables the selected provider without replacing external settings', () => {
const update = vi.fn();
useAppConfigMock.mockReturnValue({
...useAppConfigMock(),
@@ -212,28 +209,27 @@ describe('SettingsPage', () => {
);
fireEvent.click(screen.getAllByRole('button', { name: /Indexer/i })[0]);
expect(screen.getByText('indexer/provider.type:embedded')).toBeTruthy();
expect(screen.queryByTestId('field-indexer/provider.endpoint')).toBeNull();
expect(
screen.getByText('indexer/provider.type:elasticsearch')
).toBeTruthy();
expect(screen.getByTestId('field-indexer/provider.endpoint')).toBeTruthy();
fireEvent.click(
screen.getByRole('button', {
name: 'set-embedded-indexer/provider.type',
})
);
expect(update).toHaveBeenCalledWith('indexer/enabled', false);
expect(update).not.toHaveBeenCalledWith(
'indexer/provider.type',
'embedded'
);
expect(update).toHaveBeenCalledWith('indexer/enabled', true);
expect(update).toHaveBeenCalledWith('indexer/provider.type', 'embedded');
fireEvent.click(
screen.getByRole('button', {
name: 'set-manticoresearch-indexer/provider.type',
name: 'set-elasticsearch-indexer/provider.type',
})
);
expect(update).toHaveBeenCalledWith('indexer/enabled', true);
expect(update).toHaveBeenCalledWith(
'indexer/provider.type',
'manticoresearch'
'elasticsearch'
);
});
packages/frontend/admin/src/modules/settings/index.tsx
+4 -9
View File
@@ -175,9 +175,8 @@ const AdminPanel = ({
{fields.map(field => {
const fieldKey =
typeof field === 'string' ? field : String(field.key);
const effectiveIndexerProvider = sourceConfig?.enabled
? sourceConfig?.provider?.type
: 'embedded';
const effectiveIndexerProvider =
sourceConfig?.provider?.type ?? 'embedded';
if (
module === 'indexer' &&
effectiveIndexerProvider === 'embedded' &&
@@ -219,12 +218,8 @@ const AdminPanel = ({
module === 'indexer' &&
field.key === 'provider.type'
? (_path, value) => {
if (value === 'embedded') {
onUpdate('indexer/enabled', false);
} else {
onUpdate('indexer/enabled', true);
onUpdate('indexer/provider.type', value);
}
onUpdate('indexer/enabled', true);
onUpdate('indexer/provider.type', value);
}
: onUpdate,
};
packages/frontend/core/src/modules/blob-management/entity/unused-blobs.spec.ts
+78
View File
@@ -0,0 +1,78 @@
/** @vitest-environment happy-dom */
import { Framework } from '@toeverything/infra';
import { describe, expect, it, vi } from 'vitest';
import { DocsSearchService } from '../../docs-search';
import { WorkspaceService } from '../../workspace';
import { WorkspaceFlavoursService } from '../../workspace/services/flavours';
import { UnusedBlobs } from './unused-blobs';
describe('UnusedBlobs', () => {
it('reads every used blob page from the local index', async () => {
const aggregate = vi
.fn()
.mockResolvedValueOnce({
pagination: { hasMore: true },
buckets: [{ key: 'used-1' }],
})
.mockResolvedValueOnce({
pagination: { hasMore: false },
buckets: [{ key: 'used-2' }],
});
const flavoursService = {
flavours$: {
value: [
{
flavour: 'local',
listBlobs: vi
.fn()
.mockResolvedValue([
{ key: 'used-1' },
{ key: 'used-2' },
{ key: 'unused' },
]),
},
],
},
};
const workspaceService = {
workspace: {
id: 'workspace',
flavour: 'local',
avatar$: { value: null },
engine: { doc: { waitForSynced: vi.fn() } },
},
};
const docsSearchService = {
indexer: { aggregate, waitForCompleted: vi.fn() },
};
const framework = new Framework();
framework
.service(
WorkspaceFlavoursService,
flavoursService as unknown as WorkspaceFlavoursService
)
.service(
WorkspaceService,
workspaceService as unknown as WorkspaceService
)
.service(
DocsSearchService,
docsSearchService as unknown as DocsSearchService
)
.entity(UnusedBlobs, [
WorkspaceFlavoursService,
WorkspaceService,
DocsSearchService,
]);
const entity = framework.provider().createEntity(UnusedBlobs);
await expect(entity.getUnusedBlobs()).resolves.toEqual([{ key: 'unused' }]);
expect(aggregate).toHaveBeenCalledTimes(2);
expect(aggregate.mock.calls.map(call => call[3])).toEqual([
{ pagination: { limit: 1000, skip: 0 }, prefer: 'local' },
{ pagination: { limit: 1000, skip: 1000 }, prefer: 'local' },
]);
});
});
packages/frontend/core/src/modules/blob-management/entity/unused-blobs.ts
+22 -19
View File
@@ -97,27 +97,30 @@ export class UnusedBlobs extends Entity {
}
private async getUsedBlobs(): Promise<string[]> {
const result = await this.docsSearchService.indexer.aggregate(
'block',
{
type: 'boolean',
occur: 'must',
queries: [
{
type: 'exists',
field: 'blob',
},
],
},
'blob',
{
pagination: {
limit: Number.MAX_SAFE_INTEGER,
const limit = 1000;
const usedBlobs: string[] = [];
for (let skip = 0; ; skip += limit) {
const result = await this.docsSearchService.indexer.aggregate(
'block',
{
type: 'boolean',
occur: 'must',
queries: [
{
type: 'exists',
field: 'blob',
},
],
},
'blob',
{ pagination: { limit, skip }, prefer: 'local' }
);
usedBlobs.push(...result.buckets.map(bucket => bucket.key));
if (!result.pagination.hasMore) return usedBlobs;
if (result.buckets.length === 0) {
throw new Error('Local blob index pagination did not advance');
}
);
return result.buckets.map(bucket => bucket.key);
}
}
async hydrateBlob(
packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts
+49 -35
View File
@@ -1,3 +1,4 @@
import { UserFriendlyError } from '@affine/error';
import {
catchErrorInto,
effect,
@@ -7,7 +8,6 @@ import {
LiveData,
onComplete,
onStart,
smartRetry,
} from '@toeverything/infra';
import { tap } from 'rxjs';
@@ -47,42 +47,57 @@ export class DocBacklinks extends Entity {
exhaustMapWithTrailing(() =>
fromPromise(async () => {
const searchFromCloud =
this.featureFlagService.flags.enable_battery_save_mode &&
this.featureFlagService.flags.enable_battery_save_mode.value &&
this.workspaceService.workspace.flavour !== 'local';
const { buckets } = await this.docsSearchService.indexer.aggregate(
'block',
{
type: 'boolean',
occur: 'must',
queries: [
{
type: 'match',
field: 'refDocId',
match: this.docService.doc.id,
},
],
},
'docId',
{
hits: {
fields: [
'docId',
'blockId',
'parentBlockId',
'parentFlavour',
'additional',
'markdownPreview',
const aggregate = (prefer: 'local' | 'remote') =>
this.docsSearchService.indexer.aggregate(
'block',
{
type: 'boolean',
occur: 'must',
queries: [
{
type: 'match',
field: 'refDocId',
match: this.docService.doc.id,
},
],
pagination: {
limit: BUILD_CONFIG.isElectron ? 100 : 5, // the max number of backlinks to show for each doc
},
'docId',
{
hits: {
fields: [
'docId',
'blockId',
'parentBlockId',
'parentFlavour',
'additional',
'markdownPreview',
],
pagination: {
limit: BUILD_CONFIG.isElectron ? 100 : 5,
},
},
},
pagination: {
limit: 100,
},
prefer: searchFromCloud ? 'remote' : 'local',
}
);
pagination: {
limit: 100,
},
prefer,
}
);
const { buckets } = searchFromCloud
? await aggregate('remote').catch(error => {
const cause = UserFriendlyError.fromAny(error);
if (
cause.is('SEARCH_PROVIDER_UNAVAILABLE') ||
cause.is('NETWORK_ERROR') ||
(cause.is('INVALID_INDEXER_INPUT') &&
cause.data?.reason === 'unsupported_query')
) {
return aggregate('local');
}
throw error;
})
: await aggregate('local');
return buckets.flatMap(bucket => {
const title =
this.docsService.list.doc$(bucket.key).value?.title$.value ?? '';
@@ -140,7 +155,6 @@ export class DocBacklinks extends Entity {
});
});
}).pipe(
smartRetry(),
tap(backlinks => {
this.backlinks$.value = backlinks;
}),
packages/frontend/i18n/src/i18n.gen.ts
+18
View File
@@ -10502,6 +10502,24 @@ export function useAFFiNEI18N(): {
["error.INVALID_APP_CONFIG_INPUT"](options: {
readonly message: string;
}): string;
/**
* `Search index for Space {{spaceId}} is not ready yet.`
*/
["error.SEARCH_INDEX_NOT_READY"](options: {
readonly spaceId: string;
}): string;
/**
* `Search permissions are still syncing. Please try again shortly.`
*/
["error.SEARCH_PERMISSION_SYNCING"](): string;
/**
* `Search provider is temporarily unavailable.`
*/
["error.SEARCH_PROVIDER_UNAVAILABLE"](): string;
/**
* `Search index is temporarily unavailable.`
*/
["error.SEARCH_INDEX_FAILED"](): string;
/**
* `Search provider not found.`
*/
packages/frontend/i18n/src/resources/en.json
+4
View File
@@ -2584,6 +2584,10 @@
"error.MENTION_USER_ONESELF_DENIED": "You can not mention yourself.",
"error.INVALID_APP_CONFIG": "Invalid app config for module `{{module}}` with key `{{key}}`. {{hint}}.",
"error.INVALID_APP_CONFIG_INPUT": "Invalid app config input: {{message}}",
"error.SEARCH_INDEX_NOT_READY": "Search index for Space {{spaceId}} is not ready yet.",
"error.SEARCH_PERMISSION_SYNCING": "Search permissions are still syncing. Please try again shortly.",
"error.SEARCH_PROVIDER_UNAVAILABLE": "Search provider is temporarily unavailable.",
"error.SEARCH_INDEX_FAILED": "Search index is temporarily unavailable.",
"error.SEARCH_PROVIDER_NOT_FOUND": "Search provider not found.",
"error.INVALID_SEARCH_PROVIDER_REQUEST": "Invalid request argument to search provider: {{reason}}",
"error.INVALID_INDEXER_INPUT": "Invalid indexer input: {{reason}}",