mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-02-12 20:38:52 +00:00
feat(server): add cloud indexer with Elasticsearch and Manticoresearch providers (#11835)
close CLOUD-137 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced advanced workspace-scoped search and aggregation capabilities with support for complex queries, highlights, and pagination. - Added pluggable search providers: Elasticsearch and Manticoresearch. - New GraphQL queries, schema types, and resolver support for search and aggregation. - Enhanced configuration options for search providers in self-hosted and cloud deployments. - Added Docker Compose services and environment variables for Elasticsearch and Manticoresearch. - Integrated indexer service into deployment and CI workflows. - **Bug Fixes** - Improved error handling with new user-friendly error messages for search provider and indexer issues. - **Documentation** - Updated configuration examples and environment variable references for indexer and search providers. - **Tests** - Added extensive end-to-end and provider-specific tests covering indexing, searching, aggregation, deletion, and error cases. - Included snapshot tests and test fixtures for search providers. - **Chores** - Updated deployment scripts, Helm charts, and Kubernetes manifests to include indexer-related environment variables and secrets. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
fengmk2
@@ -54,6 +54,48 @@ export interface AddContextFileInput {
|
||||
contextId: Scalars['String']['input'];
|
||||
}
|
||||
|
||||
export interface AggregateBucketHitsObjectType {
|
||||
__typename?: 'AggregateBucketHitsObjectType';
|
||||
nodes: Array<SearchNodeObjectType>;
|
||||
}
|
||||
|
||||
export interface AggregateBucketObjectType {
|
||||
__typename?: 'AggregateBucketObjectType';
|
||||
count: Scalars['Int']['output'];
|
||||
/** The hits object */
|
||||
hits: AggregateBucketHitsObjectType;
|
||||
key: Scalars['String']['output'];
|
||||
}
|
||||
|
||||
export interface AggregateHitsOptions {
|
||||
fields: Array<Scalars['String']['input']>;
|
||||
highlights?: InputMaybe<Array<SearchHighlight>>;
|
||||
pagination?: InputMaybe<AggregateHitsPagination>;
|
||||
}
|
||||
|
||||
export interface AggregateHitsPagination {
|
||||
limit?: InputMaybe<Scalars['Int']['input']>;
|
||||
skip?: InputMaybe<Scalars['Int']['input']>;
|
||||
}
|
||||
|
||||
export interface AggregateInput {
|
||||
field: Scalars['String']['input'];
|
||||
options: AggregateOptions;
|
||||
query: SearchQuery;
|
||||
table: SearchTable;
|
||||
}
|
||||
|
||||
export interface AggregateOptions {
|
||||
hits: AggregateHitsOptions;
|
||||
pagination?: InputMaybe<SearchPagination>;
|
||||
}
|
||||
|
||||
export interface AggregateResultObjectType {
|
||||
__typename?: 'AggregateResultObjectType';
|
||||
buckets: Array<AggregateBucketObjectType>;
|
||||
pagination: SearchResultPagination;
|
||||
}
|
||||
|
||||
export enum AiJobStatus {
|
||||
claimed = 'claimed',
|
||||
failed = 'failed',
|
||||
@@ -612,11 +654,13 @@ export type ErrorDataUnion =
|
||||
| HttpRequestErrorDataType
|
||||
| InvalidEmailDataType
|
||||
| InvalidHistoryTimestampDataType
|
||||
| InvalidIndexerInputDataType
|
||||
| InvalidLicenseToActivateDataType
|
||||
| InvalidLicenseUpdateParamsDataType
|
||||
| InvalidOauthCallbackCodeDataType
|
||||
| InvalidPasswordLengthDataType
|
||||
| InvalidRuntimeConfigTypeDataType
|
||||
| InvalidSearchProviderRequestDataType
|
||||
| MemberNotFoundInSpaceDataType
|
||||
| MentionUserDocAccessDeniedDataType
|
||||
| MissingOauthQueryParameterDataType
|
||||
@@ -707,6 +751,7 @@ export enum ErrorNames {
|
||||
INVALID_EMAIL = 'INVALID_EMAIL',
|
||||
INVALID_EMAIL_TOKEN = 'INVALID_EMAIL_TOKEN',
|
||||
INVALID_HISTORY_TIMESTAMP = 'INVALID_HISTORY_TIMESTAMP',
|
||||
INVALID_INDEXER_INPUT = 'INVALID_INDEXER_INPUT',
|
||||
INVALID_INVITATION = 'INVALID_INVITATION',
|
||||
INVALID_LICENSE_SESSION_ID = 'INVALID_LICENSE_SESSION_ID',
|
||||
INVALID_LICENSE_TO_ACTIVATE = 'INVALID_LICENSE_TO_ACTIVATE',
|
||||
@@ -715,6 +760,7 @@ export enum ErrorNames {
|
||||
INVALID_OAUTH_CALLBACK_STATE = 'INVALID_OAUTH_CALLBACK_STATE',
|
||||
INVALID_PASSWORD_LENGTH = 'INVALID_PASSWORD_LENGTH',
|
||||
INVALID_RUNTIME_CONFIG_TYPE = 'INVALID_RUNTIME_CONFIG_TYPE',
|
||||
INVALID_SEARCH_PROVIDER_REQUEST = 'INVALID_SEARCH_PROVIDER_REQUEST',
|
||||
INVALID_SUBSCRIPTION_PARAMETERS = 'INVALID_SUBSCRIPTION_PARAMETERS',
|
||||
LICENSE_EXPIRED = 'LICENSE_EXPIRED',
|
||||
LICENSE_NOT_FOUND = 'LICENSE_NOT_FOUND',
|
||||
@@ -741,6 +787,7 @@ export enum ErrorNames {
|
||||
RUNTIME_CONFIG_NOT_FOUND = 'RUNTIME_CONFIG_NOT_FOUND',
|
||||
SAME_EMAIL_PROVIDED = 'SAME_EMAIL_PROVIDED',
|
||||
SAME_SUBSCRIPTION_RECURRING = 'SAME_SUBSCRIPTION_RECURRING',
|
||||
SEARCH_PROVIDER_NOT_FOUND = 'SEARCH_PROVIDER_NOT_FOUND',
|
||||
SIGN_UP_FORBIDDEN = 'SIGN_UP_FORBIDDEN',
|
||||
SPACE_ACCESS_DENIED = 'SPACE_ACCESS_DENIED',
|
||||
SPACE_NOT_FOUND = 'SPACE_NOT_FOUND',
|
||||
@@ -852,6 +899,11 @@ export interface InvalidHistoryTimestampDataType {
|
||||
timestamp: Scalars['String']['output'];
|
||||
}
|
||||
|
||||
export interface InvalidIndexerInputDataType {
|
||||
__typename?: 'InvalidIndexerInputDataType';
|
||||
reason: Scalars['String']['output'];
|
||||
}
|
||||
|
||||
export interface InvalidLicenseToActivateDataType {
|
||||
__typename?: 'InvalidLicenseToActivateDataType';
|
||||
reason: Scalars['String']['output'];
|
||||
@@ -881,6 +933,12 @@ export interface InvalidRuntimeConfigTypeDataType {
|
||||
want: Scalars['String']['output'];
|
||||
}
|
||||
|
||||
export interface InvalidSearchProviderRequestDataType {
|
||||
__typename?: 'InvalidSearchProviderRequestDataType';
|
||||
reason: Scalars['String']['output'];
|
||||
type: Scalars['String']['output'];
|
||||
}
|
||||
|
||||
export interface InvitationAcceptedNotificationBodyType {
|
||||
__typename?: 'InvitationAcceptedNotificationBodyType';
|
||||
/** The user who created the notification, maybe null when user is deleted or sent by system */
|
||||
@@ -1950,6 +2008,83 @@ export interface SameSubscriptionRecurringDataType {
|
||||
recurring: Scalars['String']['output'];
|
||||
}
|
||||
|
||||
export interface SearchHighlight {
|
||||
before: Scalars['String']['input'];
|
||||
end: Scalars['String']['input'];
|
||||
field: Scalars['String']['input'];
|
||||
}
|
||||
|
||||
export interface SearchInput {
|
||||
options: SearchOptions;
|
||||
query: SearchQuery;
|
||||
table: SearchTable;
|
||||
}
|
||||
|
||||
export interface SearchNodeObjectType {
|
||||
__typename?: 'SearchNodeObjectType';
|
||||
/** The search result fields, see UnionSearchItemObjectType */
|
||||
fields: Scalars['JSONObject']['output'];
|
||||
/** The search result fields, see UnionSearchItemObjectType */
|
||||
highlights: Maybe<Scalars['JSONObject']['output']>;
|
||||
}
|
||||
|
||||
export interface SearchOptions {
|
||||
fields: Array<Scalars['String']['input']>;
|
||||
highlights?: InputMaybe<Array<SearchHighlight>>;
|
||||
pagination?: InputMaybe<SearchPagination>;
|
||||
}
|
||||
|
||||
export interface SearchPagination {
|
||||
cursor?: InputMaybe<Scalars['String']['input']>;
|
||||
limit?: InputMaybe<Scalars['Int']['input']>;
|
||||
skip?: InputMaybe<Scalars['Int']['input']>;
|
||||
}
|
||||
|
||||
export interface SearchQuery {
|
||||
boost?: InputMaybe<Scalars['Float']['input']>;
|
||||
field?: InputMaybe<Scalars['String']['input']>;
|
||||
match?: InputMaybe<Scalars['String']['input']>;
|
||||
occur?: InputMaybe<SearchQueryOccur>;
|
||||
queries?: InputMaybe<Array<SearchQuery>>;
|
||||
query?: InputMaybe<SearchQuery>;
|
||||
type: SearchQueryType;
|
||||
}
|
||||
|
||||
/** Search query occur */
|
||||
export enum SearchQueryOccur {
|
||||
must = 'must',
|
||||
must_not = 'must_not',
|
||||
should = 'should',
|
||||
}
|
||||
|
||||
/** Search query type */
|
||||
export enum SearchQueryType {
|
||||
all = 'all',
|
||||
boolean = 'boolean',
|
||||
boost = 'boost',
|
||||
exists = 'exists',
|
||||
match = 'match',
|
||||
}
|
||||
|
||||
export interface SearchResultObjectType {
|
||||
__typename?: 'SearchResultObjectType';
|
||||
nodes: Array<SearchNodeObjectType>;
|
||||
pagination: SearchResultPagination;
|
||||
}
|
||||
|
||||
export interface SearchResultPagination {
|
||||
__typename?: 'SearchResultPagination';
|
||||
count: Scalars['Int']['output'];
|
||||
hasMore: Scalars['Boolean']['output'];
|
||||
nextCursor: Maybe<Scalars['String']['output']>;
|
||||
}
|
||||
|
||||
/** Search table */
|
||||
export enum SearchTable {
|
||||
block = 'block',
|
||||
doc = 'doc',
|
||||
}
|
||||
|
||||
export interface ServerConfigType {
|
||||
__typename?: 'ServerConfigType';
|
||||
/** fetch latest available upgradable release of server */
|
||||
@@ -1981,6 +2116,7 @@ export enum ServerDeploymentType {
|
||||
export enum ServerFeature {
|
||||
Captcha = 'Captcha',
|
||||
Copilot = 'Copilot',
|
||||
Indexer = 'Indexer',
|
||||
OAuth = 'OAuth',
|
||||
Payment = 'Payment',
|
||||
}
|
||||
@@ -2382,6 +2518,8 @@ export interface WorkspaceRolePermissions {
|
||||
|
||||
export interface WorkspaceType {
|
||||
__typename?: 'WorkspaceType';
|
||||
/** Search a specific table with aggregate */
|
||||
aggregate: AggregateResultObjectType;
|
||||
/** List blobs of workspace */
|
||||
blobs: Array<ListedBlob>;
|
||||
/** Blobs size of workspace */
|
||||
@@ -2437,12 +2575,18 @@ export interface WorkspaceType {
|
||||
quota: WorkspaceQuotaType;
|
||||
/** Role of current signed in user in workspace */
|
||||
role: Permission;
|
||||
/** Search a specific table */
|
||||
search: SearchResultObjectType;
|
||||
/** The team subscription of the workspace, if exists. */
|
||||
subscription: Maybe<SubscriptionType>;
|
||||
/** if workspace is team workspace */
|
||||
team: Scalars['Boolean']['output'];
|
||||
}
|
||||
|
||||
export interface WorkspaceTypeAggregateArgs {
|
||||
input: AggregateInput;
|
||||
}
|
||||
|
||||
export interface WorkspaceTypeDocArgs {
|
||||
docId: Scalars['String']['input'];
|
||||
}
|
||||
@@ -2476,6 +2620,10 @@ export interface WorkspaceTypePublicPageArgs {
|
||||
pageId: Scalars['String']['input'];
|
||||
}
|
||||
|
||||
export interface WorkspaceTypeSearchArgs {
|
||||
input: SearchInput;
|
||||
}
|
||||
|
||||
export interface WorkspaceUserType {
|
||||
__typename?: 'WorkspaceUserType';
|
||||
avatarUrl: Maybe<Scalars['String']['output']>;
|
||||
@@ -3997,6 +4145,66 @@ export type ListHistoryQuery = {
|
||||
};
|
||||
};
|
||||
|
||||
export type IndexerAggregateQueryVariables = Exact<{
|
||||
id: Scalars['String']['input'];
|
||||
input: AggregateInput;
|
||||
}>;
|
||||
|
||||
export type IndexerAggregateQuery = {
|
||||
__typename?: 'Query';
|
||||
workspace: {
|
||||
__typename?: 'WorkspaceType';
|
||||
aggregate: {
|
||||
__typename?: 'AggregateResultObjectType';
|
||||
buckets: Array<{
|
||||
__typename?: 'AggregateBucketObjectType';
|
||||
key: string;
|
||||
count: number;
|
||||
hits: {
|
||||
__typename?: 'AggregateBucketHitsObjectType';
|
||||
nodes: Array<{
|
||||
__typename?: 'SearchNodeObjectType';
|
||||
fields: any;
|
||||
highlights: any | null;
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
pagination: {
|
||||
__typename?: 'SearchResultPagination';
|
||||
count: number;
|
||||
hasMore: boolean;
|
||||
nextCursor: string | null;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
export type IndexerSearchQueryVariables = Exact<{
|
||||
id: Scalars['String']['input'];
|
||||
input: SearchInput;
|
||||
}>;
|
||||
|
||||
export type IndexerSearchQuery = {
|
||||
__typename?: 'Query';
|
||||
workspace: {
|
||||
__typename?: 'WorkspaceType';
|
||||
search: {
|
||||
__typename?: 'SearchResultObjectType';
|
||||
nodes: Array<{
|
||||
__typename?: 'SearchNodeObjectType';
|
||||
fields: any;
|
||||
highlights: any | null;
|
||||
}>;
|
||||
pagination: {
|
||||
__typename?: 'SearchResultPagination';
|
||||
count: number;
|
||||
hasMore: boolean;
|
||||
nextCursor: string | null;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
export type GetInvoicesCountQueryVariables = Exact<{ [key: string]: never }>;
|
||||
|
||||
export type GetInvoicesCountQuery = {
|
||||
@@ -4924,6 +5132,16 @@ export type Queries =
|
||||
variables: ListHistoryQueryVariables;
|
||||
response: ListHistoryQuery;
|
||||
}
|
||||
| {
|
||||
name: 'indexerAggregateQuery';
|
||||
variables: IndexerAggregateQueryVariables;
|
||||
response: IndexerAggregateQuery;
|
||||
}
|
||||
| {
|
||||
name: 'indexerSearchQuery';
|
||||
variables: IndexerSearchQueryVariables;
|
||||
response: IndexerSearchQuery;
|
||||
}
|
||||
| {
|
||||
name: 'getInvoicesCountQuery';
|
||||
variables: GetInvoicesCountQueryVariables;
|
||||
|
||||
Reference in New Issue
Block a user