Compare commits

..

1 Commits

Author SHA1 Message Date
renovate[bot]
6987321864 chore: bump up all non-major npm dependencies 2026-03-21 18:58:26 +00:00
14 changed files with 62 additions and 414 deletions

View File

@@ -1,12 +0,0 @@
import { ModuleRef } from '@nestjs/core';
import { PrismaClient } from '@prisma/client';
import { IndexerService } from '../../plugins/indexer';
export class RebuildManticoreMixedScriptIndexes1763800000000 {
static async up(_db: PrismaClient, ref: ModuleRef) {
await ref.get(IndexerService, { strict: false }).rebuildManticoreIndexes();
}
static async down(_db: PrismaClient) {}
}

View File

@@ -3,4 +3,3 @@ export * from './1703756315970-unamed-account';
export * from './1721299086340-refresh-unnamed-user';
export * from './1745211351719-create-indexer-tables';
export * from './1751966744168-correct-session-update-time';
export * from './1763800000000-rebuild-manticore-mixed-script-indexes';

View File

@@ -4,75 +4,6 @@ The actual snapshot is saved in `manticoresearch.spec.ts.snap`.
Generated by [AVA](https://avajs.dev).
## should search doc title match chinese word segmentation
> Snapshot 1
[
{
_id: '5373363211628325828',
_source: {
doc_id: 'doc-chinese',
workspace_id: 'workspace-test-doc-title-chinese',
},
fields: {
doc_id: [
'doc-chinese',
],
title: [
'AFFiNE 是一个基于云端的笔记应用',
],
},
highlights: undefined,
},
]
## should search block content match korean ngram
> Snapshot 1
[
{
_id: '1227635764506850985',
_source: {
doc_id: 'doc-korean',
workspace_id: 'workspace-test-block-content-korean',
},
fields: {
block_id: [
'block-korean',
],
content: [
'다람쥐 헌 쳇바퀴에 타고파',
],
},
highlights: undefined,
},
]
## should search block content match japanese kana ngram
> Snapshot 1
[
{
_id: '381498385699454292',
_source: {
doc_id: 'doc-japanese',
workspace_id: 'workspace-test-block-content-japanese',
},
fields: {
block_id: [
'block-japanese',
],
content: [
'いろはにほへと ちりぬるを',
],
},
highlights: undefined,
},
]
## should write document work
> Snapshot 1
@@ -958,7 +889,7 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 1
{
equals: {
term: {
workspace_id: 'workspaceId1',
},
}
@@ -966,7 +897,7 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 2
{
equals: {
term: {
workspace_id: 'workspaceId1',
},
}

View File

@@ -33,8 +33,8 @@ const user = await module.create(Mockers.User);
const workspace = await module.create(Mockers.Workspace);
test.before(async () => {
await searchProvider.recreateTable(SearchTable.block, blockSQL);
await searchProvider.recreateTable(SearchTable.doc, docSQL);
await searchProvider.createTable(SearchTable.block, blockSQL);
await searchProvider.createTable(SearchTable.doc, docSQL);
await searchProvider.write(
SearchTable.block,
@@ -163,135 +163,6 @@ test('should provider is manticoresearch', t => {
t.is(searchProvider.type, SearchProviderType.Manticoresearch);
});
test('should search doc title match chinese word segmentation', async t => {
const workspaceId = 'workspace-test-doc-title-chinese';
const docId = 'doc-chinese';
const title = 'AFFiNE 是一个基于云端的笔记应用';
await searchProvider.write(
SearchTable.doc,
[
{
workspace_id: workspaceId,
doc_id: docId,
title,
},
],
{
refresh: true,
}
);
const result = await searchProvider.search(SearchTable.doc, {
_source: ['workspace_id', 'doc_id'],
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ match: { title: '笔记' } },
],
},
},
fields: ['doc_id', 'title'],
sort: ['_score'],
});
t.true(result.total >= 1);
t.snapshot(
result.nodes
.filter(node => node._source.doc_id === docId)
.map(node => omit(node, ['_score']))
);
});
test('should search block content match korean ngram', async t => {
const workspaceId = 'workspace-test-block-content-korean';
const docId = 'doc-korean';
const blockId = 'block-korean';
const content = '다람쥐 헌 쳇바퀴에 타고파';
await searchProvider.write(
SearchTable.block,
[
{
workspace_id: workspaceId,
doc_id: docId,
block_id: blockId,
content,
flavour: 'affine:paragraph',
},
],
{
refresh: true,
}
);
const result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id'],
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ match: { content: '쥐' } },
],
},
},
fields: ['block_id', 'content'],
sort: ['_score'],
});
t.true(result.total >= 1);
t.snapshot(
result.nodes
.filter(node => node.fields.block_id?.[0] === blockId)
.map(node => omit(node, ['_score']))
);
});
test('should search block content match japanese kana ngram', async t => {
const workspaceId = 'workspace-test-block-content-japanese';
const docId = 'doc-japanese';
const blockId = 'block-japanese';
const content = 'いろはにほへと ちりぬるを';
await searchProvider.write(
SearchTable.block,
[
{
workspace_id: workspaceId,
doc_id: docId,
block_id: blockId,
content,
flavour: 'affine:paragraph',
},
],
{
refresh: true,
}
);
const result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id'],
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ match: { content: 'へ' } },
],
},
},
fields: ['block_id', 'content'],
sort: ['_score'],
});
t.true(result.total >= 1);
t.snapshot(
result.nodes
.filter(node => node.fields.block_id?.[0] === blockId)
.map(node => omit(node, ['_score']))
);
});
// #region write
test('should write document work', async t => {
@@ -318,7 +189,7 @@ test('should write document work', async t => {
let result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id'],
query: { term: { doc_id: { value: docId } } },
query: { match: { doc_id: docId } },
fields: [
'flavour',
'flavour_indexed',
@@ -361,7 +232,7 @@ test('should write document work', async t => {
result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id'],
query: { term: { doc_id: { value: docId } } },
query: { match: { doc_id: docId } },
fields: ['flavour', 'block_id', 'content', 'ref_doc_id'],
sort: ['_score'],
});
@@ -392,7 +263,7 @@ test('should write document work', async t => {
result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id'],
query: { term: { doc_id: { value: docId } } },
query: { match: { doc_id: docId } },
fields: ['flavour', 'block_id', 'content', 'ref_doc_id'],
sort: ['_score'],
});
@@ -448,8 +319,8 @@ test('should handle ref_doc_id as string[]', async t => {
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ term: { doc_id: { value: docId } } },
{ match: { workspace_id: workspaceId } },
{ match: { doc_id: docId } },
],
},
},
@@ -500,8 +371,8 @@ test('should handle ref_doc_id as string[]', async t => {
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ term: { doc_id: { value: docId } } },
{ match: { workspace_id: workspaceId } },
{ match: { doc_id: docId } },
],
},
},
@@ -545,8 +416,8 @@ test('should handle content as string[]', async t => {
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ term: { doc_id: { value: docId } } },
{ match: { workspace_id: workspaceId } },
{ match: { doc_id: docId } },
],
},
},
@@ -584,8 +455,8 @@ test('should handle content as string[]', async t => {
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ term: { doc_id: { value: docId } } },
{ match: { workspace_id: workspaceId } },
{ match: { doc_id: docId } },
],
},
},
@@ -626,8 +497,8 @@ test('should handle blob as string[]', async t => {
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ term: { doc_id: { value: docId } } },
{ match: { workspace_id: workspaceId } },
{ match: { doc_id: docId } },
],
},
},
@@ -663,8 +534,8 @@ test('should handle blob as string[]', async t => {
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ term: { doc_id: { value: docId } } },
{ match: { workspace_id: workspaceId } },
{ match: { doc_id: docId } },
],
},
},
@@ -700,8 +571,8 @@ test('should handle blob as string[]', async t => {
query: {
bool: {
must: [
{ term: { workspace_id: { value: workspaceId } } },
{ term: { doc_id: { value: docId } } },
{ match: { workspace_id: workspaceId } },
{ match: { doc_id: docId } },
],
},
},
@@ -811,10 +682,8 @@ test('should search query all and get next cursor work', async t => {
'id',
],
query: {
term: {
workspace_id: {
value: workspaceId,
},
match: {
workspace_id: workspaceId,
},
},
fields: ['flavour', 'workspace_id', 'doc_id', 'block_id'],
@@ -839,10 +708,8 @@ test('should search query all and get next cursor work', async t => {
'id',
],
query: {
term: {
workspace_id: {
value: workspaceId,
},
match: {
workspace_id: workspaceId,
},
},
fields: ['flavour', 'workspace_id', 'doc_id', 'block_id'],
@@ -867,10 +734,8 @@ test('should search query all and get next cursor work', async t => {
'id',
],
query: {
term: {
workspace_id: {
value: workspaceId,
},
match: {
workspace_id: workspaceId,
},
},
fields: ['flavour', 'workspace_id', 'doc_id', 'block_id'],
@@ -915,20 +780,16 @@ test('should filter by workspace_id work', async t => {
bool: {
must: [
{
term: {
workspace_id: {
value: workspaceId,
},
match: {
workspace_id: workspaceId,
},
},
{
bool: {
must: [
{
term: {
doc_id: {
value: docId,
},
match: {
doc_id: docId,
},
},
],

View File

@@ -8,12 +8,11 @@ import { createModule } from '../../../__tests__/create-module';
import { Mockers } from '../../../__tests__/mocks';
import { ConfigModule } from '../../../base/config';
import { ServerConfigModule } from '../../../core/config';
import { Models } from '../../../models';
import { SearchProviderFactory } from '../factory';
import { IndexerModule, IndexerService } from '../index';
import { ManticoresearchProvider } from '../providers';
import { UpsertDoc } from '../service';
import { blockSQL, docSQL, SearchTable } from '../tables';
import { SearchTable } from '../tables';
import {
AggregateInput,
SearchInput,
@@ -36,7 +35,6 @@ const module = await createModule({
const indexerService = module.get(IndexerService);
const searchProviderFactory = module.get(SearchProviderFactory);
const manticoresearch = module.get(ManticoresearchProvider);
const models = module.get(Models);
const user = await module.create(Mockers.User);
const workspace = await module.create(Mockers.Workspace, {
snapshot: true,
@@ -52,8 +50,7 @@ test.after.always(async () => {
});
test.before(async () => {
await manticoresearch.recreateTable(SearchTable.block, blockSQL);
await manticoresearch.recreateTable(SearchTable.doc, docSQL);
await indexerService.createTables();
});
test.afterEach.always(async () => {
@@ -2314,29 +2311,3 @@ test('should search docs by keyword work', async t => {
});
// #endregion
test('should rebuild manticore indexes and requeue workspaces', async t => {
const workspace1 = await module.create(Mockers.Workspace, {
indexed: true,
});
const workspace2 = await module.create(Mockers.Workspace, {
indexed: true,
});
const queueCount = module.queue.count('indexer.indexWorkspace');
await indexerService.rebuildManticoreIndexes();
const queuedWorkspaceIds = new Set(
module.queue.add
.getCalls()
.filter(call => call.args[0] === 'indexer.indexWorkspace')
.slice(queueCount)
.map(call => call.args[1].workspaceId)
);
t.true(queuedWorkspaceIds.has(workspace1.id));
t.true(queuedWorkspaceIds.has(workspace2.id));
t.is((await models.workspace.get(workspace1.id))?.indexed, false);
t.is((await models.workspace.get(workspace2.id))?.indexed, false);
});

View File

@@ -38,17 +38,6 @@ const SupportIndexedAttributes = [
'parent_block_id',
];
const SupportExactTermFields = new Set([
'workspace_id',
'doc_id',
'block_id',
'flavour',
'parent_flavour',
'parent_block_id',
'created_by_user_id',
'updated_by_user_id',
]);
const ConvertEmptyStringToNullValueFields = new Set([
'ref_doc_id',
'ref',
@@ -66,20 +55,23 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
table: SearchTable,
mapping: string
): Promise<void> {
const text = await this.#executeSQL(mapping);
const url = `${this.config.provider.endpoint}/cli`;
const response = await fetch(url, {
method: 'POST',
body: mapping,
headers: {
'Content-Type': 'text/plain',
},
});
// manticoresearch cli response is not json, so we need to handle it manually
const text = (await response.text()).trim();
if (!response.ok) {
this.logger.error(`failed to create table ${table}, response: ${text}`);
throw new InternalServerError();
}
this.logger.log(`created table ${table}, response: ${text}`);
}
async dropTable(table: SearchTable): Promise<void> {
const text = await this.#executeSQL(`DROP TABLE IF EXISTS ${table}`);
this.logger.log(`dropped table ${table}, response: ${text}`);
}
async recreateTable(table: SearchTable, mapping: string): Promise<void> {
await this.dropTable(table);
await this.createTable(table, mapping);
}
override async write(
table: SearchTable,
documents: Record<string, unknown>[],
@@ -260,12 +252,6 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
// 1750389254 => new Date(1750389254 * 1000)
return new Date(value * 1000);
}
if (value && typeof value === 'string') {
const timestamp = Date.parse(value);
if (!Number.isNaN(timestamp)) {
return new Date(timestamp);
}
}
return value;
}
@@ -316,10 +302,8 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
// workspace_id: 'workspaceId1'
// }
// }
let termField = options?.termMappingField ?? 'term';
let field = Object.keys(query.term)[0];
let termField =
options?.termMappingField ??
(SupportExactTermFields.has(field) ? 'equals' : 'term');
let value = query.term[field];
if (typeof value === 'object' && 'value' in value) {
if ('boost' in value) {
@@ -448,28 +432,4 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
}
return value;
}
async #executeSQL(sql: string) {
const url = `${this.config.provider.endpoint}/cli`;
const headers: Record<string, string> = {
'Content-Type': 'text/plain',
};
if (this.config.provider.apiKey) {
headers.Authorization = `ApiKey ${this.config.provider.apiKey}`;
} else if (this.config.provider.password) {
headers.Authorization = `Basic ${Buffer.from(`${this.config.provider.username}:${this.config.provider.password}`).toString('base64')}`;
}
const response = await fetch(url, {
method: 'POST',
body: sql,
headers,
});
const text = (await response.text()).trim();
if (!response.ok) {
this.logger.error(`failed to execute SQL "${sql}", response: ${text}`);
throw new InternalServerError();
}
return text;
}
}

View File

@@ -14,7 +14,6 @@ import {
AggregateQueryDSL,
BaseQueryDSL,
HighlightDSL,
ManticoresearchProvider,
OperationOptions,
SearchNode,
SearchProvider,
@@ -131,63 +130,6 @@ export class IndexerService {
}
}
async rebuildManticoreIndexes() {
let searchProvider: SearchProvider | undefined;
try {
searchProvider = this.factory.get();
} catch (err) {
if (err instanceof SearchProviderNotFound) {
this.logger.debug('No search provider found, skip rebuilding tables');
return;
}
throw err;
}
if (!(searchProvider instanceof ManticoresearchProvider)) {
this.logger.debug(
`Search provider ${searchProvider.type} does not need manticore rebuild`
);
return;
}
const mappings = SearchTableMappingStrings[searchProvider.type];
for (const table of Object.keys(mappings) as SearchTable[]) {
await searchProvider.recreateTable(table, mappings[table]);
}
let lastWorkspaceSid = 0;
while (true) {
const workspaces = await this.models.workspace.list(
{ sid: { gt: lastWorkspaceSid } },
{ id: true, sid: true },
100
);
if (!workspaces.length) {
break;
}
for (const workspace of workspaces) {
await this.models.workspace.update(
workspace.id,
{ indexed: false },
false
);
await this.queue.add(
'indexer.indexWorkspace',
{
workspaceId: workspace.id,
},
{
jobId: `indexWorkspace/${workspace.id}`,
priority: 100,
}
);
}
lastWorkspaceSid = workspaces[workspaces.length - 1].sid;
}
}
async write<T extends SearchTable>(
table: T,
documents: UpsertTypeByTable<T>[],

View File

@@ -150,8 +150,6 @@ CREATE TABLE IF NOT EXISTS block (
updated_at timestamp
)
morphology = 'jieba_chinese, lemmatize_en_all, lemmatize_de_all, lemmatize_ru_all, libstemmer_ar, libstemmer_ca, stem_cz, libstemmer_da, libstemmer_nl, libstemmer_fi, libstemmer_fr, libstemmer_el, libstemmer_hi, libstemmer_hu, libstemmer_id, libstemmer_ga, libstemmer_it, libstemmer_lt, libstemmer_ne, libstemmer_no, libstemmer_pt, libstemmer_ro, libstemmer_es, libstemmer_sv, libstemmer_ta, libstemmer_tr'
charset_table = 'non_cjk, chinese'
ngram_len = '1'
ngram_chars = 'U+1100..U+11FF, U+3130..U+318F, U+A960..U+A97F, U+AC00..U+D7AF, U+D7B0..U+D7FF, U+3040..U+30FF, U+0E00..U+0E7F'
charset_table = 'non_cjk, cjk'
index_field_lengths = '1'
`;

View File

@@ -109,8 +109,6 @@ CREATE TABLE IF NOT EXISTS doc (
updated_at timestamp
)
morphology = 'jieba_chinese, lemmatize_en_all, lemmatize_de_all, lemmatize_ru_all, libstemmer_ar, libstemmer_ca, stem_cz, libstemmer_da, libstemmer_nl, libstemmer_fi, libstemmer_fr, libstemmer_el, libstemmer_hi, libstemmer_hu, libstemmer_id, libstemmer_ga, libstemmer_it, libstemmer_lt, libstemmer_ne, libstemmer_no, libstemmer_pt, libstemmer_ro, libstemmer_es, libstemmer_sv, libstemmer_ta, libstemmer_tr'
charset_table = 'non_cjk, chinese'
ngram_len = '1'
ngram_chars = 'U+1100..U+11FF, U+3130..U+318F, U+A960..U+A97F, U+AC00..U+D7AF, U+D7B0..U+D7FF, U+3040..U+30FF, U+0E00..U+0E7F'
charset_table = 'non_cjk, cjk'
index_field_lengths = '1'
`;

View File

@@ -77,8 +77,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/RevenueCat/purchases-ios-spm.git",
"state" : {
"revision" : "abb0d68c3e7ba97b16ab51c38fcaca16b0e358c8",
"version" : "5.66.0"
"revision" : "2913a336eb37dc06795cdbaa5b5de330b6707669",
"version" : "5.65.0"
}
},
{
@@ -113,8 +113,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/apple/swift-collections",
"state" : {
"revision" : "6675bc0ff86e61436e615df6fc5174e043e57924",
"version" : "1.4.1"
"revision" : "8d9834a6189db730f6264db7556a7ffb751e99ee",
"version" : "1.4.0"
}
},
{

View File

@@ -17,7 +17,7 @@ let package = Package(
],
dependencies: [
.package(path: "../AffineResources"),
.package(url: "https://github.com/RevenueCat/purchases-ios-spm.git", from: "5.66.0"),
.package(url: "https://github.com/RevenueCat/purchases-ios-spm.git", from: "5.60.0"),
],
targets: [
.target(

View File

@@ -17,7 +17,7 @@ let package = Package(
.package(path: "../AffineGraphQL"),
.package(path: "../AffineResources"),
.package(url: "https://github.com/apollographql/apollo-ios.git", from: "1.23.0"),
.package(url: "https://github.com/apple/swift-collections.git", from: "1.4.1"),
.package(url: "https://github.com/apple/swift-collections.git", from: "1.4.0"),
.package(url: "https://github.com/SnapKit/SnapKit.git", from: "5.7.1"),
.package(url: "https://github.com/SwifterSwift/SwifterSwift.git", from: "6.2.0"),
.package(url: "https://github.com/Recouse/EventSource.git", from: "0.1.7"),

View File

@@ -25539,8 +25539,8 @@ __metadata:
linkType: hard
"i18next@npm:^25.0.0":
version: 25.10.3
resolution: "i18next@npm:25.10.3"
version: 25.10.2
resolution: "i18next@npm:25.10.2"
dependencies:
"@babel/runtime": "npm:^7.29.2"
peerDependencies:
@@ -25548,7 +25548,7 @@ __metadata:
peerDependenciesMeta:
typescript:
optional: true
checksum: 10/230bf2af3ab81dafb0d5209739f665636c6b5b93f8bbf9749d284eb1014708035e32c42e2e6dd401b23a5fef3fe904eea35b9f1c11ffaf42f25741c5b0db883e
checksum: 10/98ee6718975cf561987cf0db4e638146fb4c3f38cfa3fc6c313b4ce4a707e94ffe7ad17a4c0a8d1b15c9f9abd5837d495d147e104106c6237b7f9fa1a73110d4
languageName: node
linkType: hard
@@ -31922,11 +31922,11 @@ __metadata:
linkType: hard
"react-hook-form@npm:^7.54.1":
version: 7.72.0
resolution: "react-hook-form@npm:7.72.0"
version: 7.71.2
resolution: "react-hook-form@npm:7.71.2"
peerDependencies:
react: ^16.8.0 || ^17 || ^18 || ^19
checksum: 10/25268c510aa5943adfeb2cffe497f5b4b506974f60b4404b42f1b73eaa6db63a1efdb51f3e5f0fa6c0239cbd0f66a2dddf06ec45141d1894a266e77e2dbefb2d
checksum: 10/4cc90868016f8463463ea5d1812f9405832e02fdb12f18ff0262c6437e7a9cdfe6443a7f58decf903cb8f20bfd68c0ed419283b0d6be886f3e84dc4d14b0efa6
languageName: node
linkType: hard