mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-17 01:56:27 +08:00
feat(server): add cloud indexer with Elasticsearch and Manticoresearch providers (#11835)
close CLOUD-137 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced advanced workspace-scoped search and aggregation capabilities with support for complex queries, highlights, and pagination. - Added pluggable search providers: Elasticsearch and Manticoresearch. - New GraphQL queries, schema types, and resolver support for search and aggregation. - Enhanced configuration options for search providers in self-hosted and cloud deployments. - Added Docker Compose services and environment variables for Elasticsearch and Manticoresearch. - Integrated indexer service into deployment and CI workflows. - **Bug Fixes** - Improved error handling with new user-friendly error messages for search provider and indexer issues. - **Documentation** - Updated configuration examples and environment variable references for indexer and search providers. - **Tests** - Added extensive end-to-end and provider-specific tests covering indexing, searching, aggregation, deletion, and error cases. - Included snapshot tests and test fixtures for search providers. - **Chores** - Updated deployment scripts, Helm charts, and Kubernetes manifests to include indexer-related environment variables and secrets. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
fengmk2
@@ -0,0 +1,12 @@
|
||||
import { serverConfigQuery, ServerFeature } from '@affine/graphql';
|
||||
|
||||
import { app, e2e } from '../test';
|
||||
|
||||
e2e('should indexer feature enabled by default', async t => {
|
||||
const { serverConfig } = await app.gql({ query: serverConfigQuery });
|
||||
t.is(
|
||||
serverConfig.features.includes(ServerFeature.Indexer),
|
||||
true,
|
||||
JSON.stringify(serverConfig, null, 2)
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
# Snapshot report for `src/__tests__/e2e/indexer/aggregate.spec.ts`
|
||||
|
||||
The actual snapshot is saved in `aggregate.spec.ts.snap`.
|
||||
|
||||
Generated by [AVA](https://avajs.dev).
|
||||
|
||||
## should aggregate by docId
|
||||
|
||||
> Snapshot 1
|
||||
|
||||
[
|
||||
{
|
||||
count: 3,
|
||||
hits: {
|
||||
nodes: [
|
||||
{
|
||||
fields: {
|
||||
blockId: [
|
||||
'block-2',
|
||||
],
|
||||
flavour: [
|
||||
'affine:page',
|
||||
],
|
||||
},
|
||||
highlights: {
|
||||
content: [
|
||||
'test3 <b>hello</b> title top1',
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
fields: {
|
||||
blockId: [
|
||||
'block-0',
|
||||
],
|
||||
flavour: [
|
||||
'affine:text',
|
||||
],
|
||||
},
|
||||
highlights: {
|
||||
content: [
|
||||
'test1 <b>hello world</b> top2',
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
key: 'doc-0',
|
||||
},
|
||||
{
|
||||
count: 1,
|
||||
hits: {
|
||||
nodes: [
|
||||
{
|
||||
fields: {
|
||||
blockId: [
|
||||
'block-3',
|
||||
],
|
||||
flavour: [
|
||||
'affine:text',
|
||||
],
|
||||
},
|
||||
highlights: {
|
||||
content: [
|
||||
'test4 <b>hello world</b>',
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
key: 'doc-1',
|
||||
},
|
||||
{
|
||||
count: 1,
|
||||
hits: {
|
||||
nodes: [
|
||||
{
|
||||
fields: {
|
||||
blockId: [
|
||||
'block-4',
|
||||
],
|
||||
flavour: [
|
||||
'affine:text',
|
||||
],
|
||||
},
|
||||
highlights: {
|
||||
content: [
|
||||
'test5 <b>hello</b>',
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
key: 'doc-2',
|
||||
},
|
||||
]
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,36 @@
|
||||
# Snapshot report for `src/__tests__/e2e/indexer/search.spec.ts`
|
||||
|
||||
The actual snapshot is saved in `search.spec.ts.snap`.
|
||||
|
||||
Generated by [AVA](https://avajs.dev).
|
||||
|
||||
## should search with query
|
||||
|
||||
> Snapshot 1
|
||||
|
||||
[
|
||||
{
|
||||
fields: {
|
||||
ref: [
|
||||
'{"foo": "bar1"}',
|
||||
'{"foo": "bar3"}',
|
||||
],
|
||||
refDocId: [
|
||||
'doc-0',
|
||||
'doc-2',
|
||||
],
|
||||
},
|
||||
highlights: null,
|
||||
},
|
||||
{
|
||||
fields: {
|
||||
ref: [
|
||||
'{"foo": "bar1"}',
|
||||
],
|
||||
refDocId: [
|
||||
'doc-0',
|
||||
],
|
||||
},
|
||||
highlights: null,
|
||||
},
|
||||
]
|
||||
Binary file not shown.
@@ -0,0 +1,159 @@
|
||||
import { indexerAggregateQuery, SearchTable } from '@affine/graphql';
|
||||
|
||||
import { IndexerService } from '../../../plugins/indexer/service';
|
||||
import { Mockers } from '../../mocks';
|
||||
import { app, e2e } from '../test';
|
||||
|
||||
e2e('should aggregate by docId', async t => {
|
||||
const owner = await app.signup();
|
||||
|
||||
const workspace = await app.create(Mockers.Workspace, {
|
||||
owner: { id: owner.id },
|
||||
});
|
||||
|
||||
const indexerService = app.get(IndexerService);
|
||||
|
||||
await indexerService.write(
|
||||
SearchTable.block,
|
||||
[
|
||||
{
|
||||
docId: 'doc-0',
|
||||
workspaceId: workspace.id,
|
||||
content: 'test1 hello world top2',
|
||||
flavour: 'affine:text',
|
||||
blockId: 'block-0',
|
||||
createdByUserId: owner.id,
|
||||
updatedByUserId: owner.id,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
{
|
||||
docId: 'doc-0',
|
||||
workspaceId: workspace.id,
|
||||
content: 'test2 hello hello top3',
|
||||
flavour: 'affine:text',
|
||||
blockId: 'block-1',
|
||||
createdByUserId: owner.id,
|
||||
updatedByUserId: owner.id,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
{
|
||||
docId: 'doc-0',
|
||||
workspaceId: workspace.id,
|
||||
content: 'test3 hello title top1',
|
||||
flavour: 'affine:page',
|
||||
blockId: 'block-2',
|
||||
createdByUserId: owner.id,
|
||||
updatedByUserId: owner.id,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
{
|
||||
docId: 'doc-1',
|
||||
workspaceId: workspace.id,
|
||||
content: 'test4 hello world',
|
||||
flavour: 'affine:text',
|
||||
blockId: 'block-3',
|
||||
refDocId: 'doc-0',
|
||||
ref: ['{"foo": "bar1"}'],
|
||||
createdByUserId: owner.id,
|
||||
updatedByUserId: owner.id,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
{
|
||||
docId: 'doc-2',
|
||||
workspaceId: workspace.id,
|
||||
content: 'test5 hello',
|
||||
flavour: 'affine:text',
|
||||
blockId: 'block-4',
|
||||
refDocId: 'doc-0',
|
||||
ref: ['{"foo": "bar2"}'],
|
||||
createdByUserId: owner.id,
|
||||
updatedByUserId: owner.id,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
],
|
||||
{
|
||||
refresh: true,
|
||||
}
|
||||
);
|
||||
|
||||
const result = await app.gql({
|
||||
query: indexerAggregateQuery,
|
||||
variables: {
|
||||
id: workspace.id,
|
||||
input: {
|
||||
table: SearchTable.block,
|
||||
query: {
|
||||
// @ts-expect-error allow to use string as enum
|
||||
type: 'boolean',
|
||||
// @ts-expect-error allow to use string as enum
|
||||
occur: 'must',
|
||||
queries: [
|
||||
{
|
||||
// @ts-expect-error allow to use string as enum
|
||||
type: 'match',
|
||||
field: 'content',
|
||||
match: 'hello world',
|
||||
},
|
||||
{
|
||||
// @ts-expect-error allow to use string as enum
|
||||
type: 'boolean',
|
||||
// @ts-expect-error allow to use string as enum
|
||||
occur: 'should',
|
||||
queries: [
|
||||
{
|
||||
// @ts-expect-error allow to use string as enum
|
||||
type: 'match',
|
||||
field: 'content',
|
||||
match: 'hello world',
|
||||
},
|
||||
{
|
||||
// @ts-expect-error allow to use string as enum
|
||||
type: 'boost',
|
||||
boost: 1.5,
|
||||
query: {
|
||||
// @ts-expect-error allow to use string as enum
|
||||
type: 'match',
|
||||
field: 'flavour',
|
||||
match: 'affine:page',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
field: 'docId',
|
||||
options: {
|
||||
pagination: {
|
||||
limit: 50,
|
||||
skip: 0,
|
||||
},
|
||||
hits: {
|
||||
pagination: {
|
||||
limit: 2,
|
||||
skip: 0,
|
||||
},
|
||||
fields: ['blockId', 'flavour'],
|
||||
highlights: [
|
||||
{
|
||||
field: 'content',
|
||||
before: '<b>',
|
||||
end: '</b>',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
t.truthy(result.workspace.aggregate, 'failed to aggregate');
|
||||
t.is(result.workspace.aggregate.pagination.count, 5);
|
||||
t.is(result.workspace.aggregate.pagination.hasMore, true);
|
||||
t.truthy(result.workspace.aggregate.pagination.nextCursor);
|
||||
t.snapshot(result.workspace.aggregate.buckets);
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
import {
|
||||
indexerSearchQuery,
|
||||
SearchQueryOccur,
|
||||
SearchQueryType,
|
||||
SearchTable,
|
||||
} from '@affine/graphql';
|
||||
|
||||
import { IndexerService } from '../../../plugins/indexer/service';
|
||||
import { Mockers } from '../../mocks';
|
||||
import { app, e2e } from '../test';
|
||||
|
||||
e2e('should search with query', async t => {
|
||||
const owner = await app.signup();
|
||||
|
||||
const workspace = await app.create(Mockers.Workspace, {
|
||||
owner: { id: owner.id },
|
||||
});
|
||||
|
||||
const indexerService = app.get(IndexerService);
|
||||
|
||||
await indexerService.write(
|
||||
SearchTable.block,
|
||||
[
|
||||
{
|
||||
docId: 'doc-0',
|
||||
workspaceId: workspace.id,
|
||||
content: 'test1',
|
||||
flavour: 'markdown',
|
||||
blockId: 'block-0',
|
||||
createdByUserId: owner.id,
|
||||
updatedByUserId: owner.id,
|
||||
createdAt: new Date('2025-04-22T00:00:00.000Z'),
|
||||
updatedAt: new Date('2025-04-22T00:00:00.000Z'),
|
||||
},
|
||||
{
|
||||
docId: 'doc-1',
|
||||
workspaceId: workspace.id,
|
||||
content: 'test2',
|
||||
flavour: 'markdown',
|
||||
blockId: 'block-1',
|
||||
refDocId: ['doc-0'],
|
||||
ref: ['{"foo": "bar1"}'],
|
||||
createdByUserId: owner.id,
|
||||
updatedByUserId: owner.id,
|
||||
createdAt: new Date('2021-04-22T00:00:00.000Z'),
|
||||
updatedAt: new Date('2021-04-22T00:00:00.000Z'),
|
||||
},
|
||||
{
|
||||
docId: 'doc-2',
|
||||
workspaceId: workspace.id,
|
||||
content: 'test3',
|
||||
flavour: 'markdown',
|
||||
blockId: 'block-2',
|
||||
refDocId: ['doc-0', 'doc-2'],
|
||||
ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'],
|
||||
createdByUserId: owner.id,
|
||||
updatedByUserId: owner.id,
|
||||
createdAt: new Date('2025-03-22T00:00:00.000Z'),
|
||||
updatedAt: new Date('2025-03-22T00:00:00.000Z'),
|
||||
},
|
||||
],
|
||||
{
|
||||
refresh: true,
|
||||
}
|
||||
);
|
||||
|
||||
const result = await app.gql({
|
||||
query: indexerSearchQuery,
|
||||
variables: {
|
||||
id: workspace.id,
|
||||
input: {
|
||||
table: SearchTable.block,
|
||||
query: {
|
||||
type: SearchQueryType.boolean,
|
||||
occur: SearchQueryOccur.must,
|
||||
queries: [
|
||||
{
|
||||
type: SearchQueryType.boolean,
|
||||
occur: SearchQueryOccur.should,
|
||||
queries: ['doc-0', 'doc-1', 'doc-2'].map(id => ({
|
||||
type: SearchQueryType.match,
|
||||
field: 'docId',
|
||||
match: id,
|
||||
})),
|
||||
},
|
||||
{
|
||||
type: SearchQueryType.exists,
|
||||
field: 'refDocId',
|
||||
},
|
||||
],
|
||||
},
|
||||
options: {
|
||||
fields: ['refDocId', 'ref'],
|
||||
pagination: {
|
||||
limit: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
t.truthy(result.workspace.search, 'failed to search');
|
||||
t.is(result.workspace.search.pagination.count, 2);
|
||||
t.is(result.workspace.search.pagination.hasMore, true);
|
||||
t.truthy(result.workspace.search.pagination.nextCursor);
|
||||
t.is(result.workspace.search.nodes.length, 2);
|
||||
t.snapshot(result.workspace.search.nodes);
|
||||
});
|
||||
Reference in New Issue
Block a user