diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json index 9341acc80a..9583934ef0 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json +++ b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json @@ -1,26 +1,13 @@ -{ "index" : {"_id" : "workspaceId1/docId1/title/blockId1", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId1", "content" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "flavour" : "title", "blob" : "blob1", "ref_doc_id" : "refDocId1", "ref" : "ref1", "parent_flavour" : "parentFlavour1", "parent_block_id" : "parentBlockId1", "additional" : "additional1", "markdown_preview" : "markdownPreview1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour2/blockId2", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId2", "content" : "title2 world, test searching morphology", "flavour" : "flavour2", "blob" : "blob2", "ref_doc_id" : "refDocId2", "ref" : "ref2", "parent_flavour" : "parentFlavour2", "parent_block_id" : "parentBlockId2", "additional" : "additional2", "markdown_preview" : "markdownPreview2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour3/blockId3", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId3", "content" : "title3 hello update", "flavour" : "flavour3", "blob" : "blob3", "ref_doc_id" : "refDocId3", "ref" : "ref3", "parent_flavour" : "parentFlavour3", "parent_block_id" : "parentBlockId3", "additional" : "additional3", "markdown_preview" : "markdownPreview3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour4/blockId4", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId4", "content" : "title4 hello", "flavour" : "flavour4", "blob" : "blob4", "ref_doc_id" : "refDocId4", "ref" : "ref4", "parent_flavour" : "parentFlavour4", "parent_block_id" : "parentBlockId4", "additional" : "additional4", "markdown_preview" : "markdownPreview4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour5/blockId5", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId5", "content" : "title5 hello", "flavour" : "flavour5", "blob" : "blob5", "ref_doc_id" : "refDocId5", "ref" : "ref5", "parent_flavour" : "parentFlavour5", "parent_block_id" : "parentBlockId5", "additional" : "additional5", "markdown_preview" : "markdownPreview5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour6/blockId6", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId6", "content" : "title6 hello", "flavour" : "flavour6", "blob" : "blob6", "ref_doc_id" : "refDocId6", "ref" : "ref6", "parent_flavour" : "parentFlavour6", "parent_block_id" : "parentBlockId6", "additional" : "additional6", "markdown_preview" : "markdownPreview6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId1/flavour7/blockId7", "_index" : "block"} } {"workspace_id" : "workspaceId2", "doc_id" : "docId1", "block_id" : "blockId7", "content" : "title7 hello", "flavour" : "flavour7", "blob" : "blob7", "ref_doc_id" : "refDocId7", "ref" : "ref7", "parent_flavour" : "parentFlavour7", "parent_block_id" : "parentBlockId7", "additional" : "additional7", "markdown_preview" : "markdownPreview7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId9", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId9", "block_id" : "blockId9", "content" : "title9 hello affine issue hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour9", "parent_block_id" : "parentBlockId9", "additional" : "additional9", "markdown_preview" : "markdownPreview9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId10", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId10", "content" : "this is docId2 title content hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour10", "parent_block_id" : "parentBlockId10", "additional" : "additional10", "markdown_preview" : "markdownPreview10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId11", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId11", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour11", "parent_block_id" : "parentBlockId11", "additional" : "additional11", "markdown_preview" : "markdownPreview11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId12", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId12", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour12", "parent_block_id" : "parentBlockId12", "additional" : "additional12", "markdown_preview" : "markdownPreview12", "created_by_user_id" : "userId12", "updated_by_user_id" : "userId12", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} -{ "index" : {"_id" : "workspaceId1/docId3/affine:page/blockId13", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId13", "content" : "this is docId3 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour13", "parent_block_id" : "parentBlockId13", "additional" : "additional13", "markdown_preview" : "markdownPreview13", "created_by_user_id" : "userId13", "updated_by_user_id" : "userId13", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} -{ "index" : {"_id" : "workspaceId1/docId3/affine:database/blockId14", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId14", "content" : "this is docId3 title content world", "flavour" : "affine:database", "parent_flavour": "affine:database", "parent_block_id" : "parentBlockId14", "additional" : "additional14", "markdown_preview" : "markdownPreview14", "created_by_user_id" : "userId14", "updated_by_user_id" : "userId14", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json index 69b755c804..e252622d95 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json +++ b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json @@ -1,22 +1,11 @@ -{ "index" : {"_id" : "workspaceId1/docId1", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "title" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "summary" : "summary1", "journal" : "journal1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId2", "title" : "title2 world, test searching morphology", "summary" : "summary2", "journal" : "journal2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId3", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId3", "title" : "title3 hello update", "summary" : "summary3", "journal" : "journal3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId4", "_index" : "doc"} } {"workspace_id" : "workspaceId2", "doc_id" : "docId4", "title" : "title4 hello", "summary" : "summary4", "journal" : "journal4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId5", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId5", "title" : "title5 hello", "summary" : "summary5", "journal" : "journal5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId6", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId6", "title" : "title6 hello", "summary" : "summary6", "journal" : "journal6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId7", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId7", "title" : "title7 hello", "summary" : "summary7", "journal" : "journal7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId8", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId8", "title" : "title8 hello", "summary" : "summary8", "journal" : "journal8", "created_by_user_id" : "userId8", "updated_by_user_id" : "userId8", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId3/docId9", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId9", "title" : "title9 hello", "summary" : "summary9", "journal" : "journal9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId3/docId10", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId10", "title" : "title10 hello", "summary" : "summary10", "journal" : "journal10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId3/docId10", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId11", "title" : "title11 hello, old value", "summary" : "summary11", "journal" : "journal11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md index ea75f1a483..4f23bb22d4 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md @@ -4,6 +4,51 @@ The actual snapshot is saved in `elasticsearch.spec.ts.snap`. Generated by [AVA](https://avajs.dev). +## should batch write bugfix + +> Snapshot 1 + + [ + { + _id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b1', + _source: { + doc_id: 'a', + workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch', + }, + fields: { + block_id: [ + 'b1', + ], + doc_id: [ + 'a', + ], + workspace_id: [ + 'workspaceId-batch-write-bugfix-for-elasticsearch', + ], + }, + highlights: undefined, + }, + { + _id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b2', + _source: { + doc_id: 'a', + workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch', + }, + fields: { + block_id: [ + 'b2', + ], + doc_id: [ + 'a', + ], + workspace_id: [ + 'workspaceId-batch-write-bugfix-for-elasticsearch', + ], + }, + highlights: undefined, + }, + ] + ## should search block table query match url work > Snapshot 1 @@ -549,7 +594,7 @@ Generated by [AVA](https://avajs.dev). [ { - _id: 'workspaceId1/docId2/affine:page/blockId9', + _id: 'workspaceId1/docId9/blockId9', _source: { doc_id: 'docId9', workspace_id: 'workspaceId1', diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap index a98a9c1b9a..dc889bfecf 100644 Binary files a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap and b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap differ diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md index f7a619b17d..5807a24e7d 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md @@ -310,6 +310,57 @@ Generated by [AVA](https://avajs.dev). }, ] +## should batch write bugfix + +> Snapshot 1 + + [ + { + _id: '8950102031541144623', + _source: { + doc_id: 'a', + workspace_id: 'workspaceId-batch-write-bugfix-for-manticoresearch', + }, + fields: { + block_id: [ + 'b1', + ], + content: [ + '2025-05-26', + ], + doc_id: [ + 'a', + ], + workspace_id: [ + 'workspaceId-batch-write-bugfix-for-manticoresearch', + ], + }, + highlights: undefined, + }, + { + _id: '8950103131052772834', + _source: { + doc_id: 'a', + workspace_id: 'workspaceId-batch-write-bugfix-for-manticoresearch', + }, + fields: { + block_id: [ + 'b2', + ], + content: [ + '', + ], + doc_id: [ + 'a', + ], + workspace_id: [ + 'workspaceId-batch-write-bugfix-for-manticoresearch', + ], + }, + highlights: undefined, + }, + ] + ## should search query all and get next cursor work > Snapshot 1 diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap index ee2736d984..95b68a72a8 100644 Binary files a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap and b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap differ diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts index 3f39b499c2..6f73dfc6ab 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts @@ -154,22 +154,24 @@ _test.before(async () => { path.join(import.meta.dirname, '../__fixtures__/test-blocks.json'), 'utf-8' ); - // @ts-expect-error access protected method - await searchProvider.requestBulk( - SearchTable.block, - blocks.trim().split('\n'), - { - // make sure the data is visible to search - refresh: 'true', - } - ); + const blockDocuments = blocks + .trim() + .split('\n') + .map(line => JSON.parse(line)); + await searchProvider.write(SearchTable.block, blockDocuments, { + refresh: true, + }); + const docs = await readFile( path.join(import.meta.dirname, '../__fixtures__/test-docs.json'), 'utf-8' ); - // @ts-expect-error access protected method - await searchProvider.requestBulk(SearchTable.doc, docs.trim().split('\n'), { - refresh: 'true', + const docDocuments = docs + .trim() + .split('\n') + .map(line => JSON.parse(line)); + await searchProvider.write(SearchTable.doc, docDocuments, { + refresh: true, }); }); @@ -616,6 +618,70 @@ test('should handle blob as string[]', async t => { }); }); +test('should batch write bugfix', async t => { + const workspaceId = 'workspaceId-batch-write-bugfix-for-elasticsearch'; + + await searchProvider.write( + SearchTable.block, + [ + { + workspace_id: workspaceId, + doc_id: 'a', + block_id: 'b1', + content: '2025-05-26', + flavour: 'affine:page', + additional: '{"displayMode":"edgeless"}', + created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + created_at: '2025-05-26T05:16:23.128Z', + updated_at: '2025-05-26T05:15:53.091Z', + flavour_indexed: 'affine:page', + }, + { + workspace_id: workspaceId, + doc_id: 'a', + block_id: 'b2', + content: '', + flavour: 'affine:surface', + parent_flavour: 'affine:page', + parent_block_id: 'TcOGF6HSa7', + additional: '', + created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + created_at: '2025-05-26T05:16:23.128Z', + updated_at: '2025-05-26T05:15:53.091Z', + flavour_indexed: 'affine:surface', + parent_flavour_indexed: 'affine:page', + parent_block_id_indexed: 'TcOGF6HSa7', + }, + ], + { + refresh: true, + } + ); + + const result = await searchProvider.search(SearchTable.block, { + _source: ['workspace_id', 'doc_id'], + query: { + bool: { + must: [ + { + term: { + workspace_id: { + value: workspaceId, + }, + }, + }, + ], + }, + }, + fields: ['workspace_id', 'doc_id', 'block_id'], + sort: ['_score'], + }); + + t.snapshot(result.nodes.map(node => omit(node, ['_score']))); +}); + // #endregion // #region search diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts index 6ecf13b5b0..75b3f1f690 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts @@ -11,7 +11,7 @@ import { ConfigModule } from '../../../../base/config'; import { IndexerModule } from '../../'; import { SearchProviderType } from '../../config'; import { ManticoresearchProvider } from '../../providers'; -import { SearchTable } from '../../tables'; +import { blockSQL, docSQL, SearchTable } from '../../tables'; const module = await createModule({ imports: [ @@ -32,15 +32,8 @@ const user = await module.create(Mockers.User); const workspace = await module.create(Mockers.Workspace); test.before(async () => { - const tablesDir = path.join(import.meta.dirname, '../../tables'); - await searchProvider.createTable( - SearchTable.block, - path.join(tablesDir, 'block.sql') - ); - await searchProvider.createTable( - SearchTable.doc, - path.join(tablesDir, 'doc.sql') - ); + await searchProvider.createTable(SearchTable.block, blockSQL); + await searchProvider.createTable(SearchTable.doc, docSQL); await searchProvider.write( SearchTable.block, @@ -122,22 +115,24 @@ test.before(async () => { path.join(import.meta.dirname, '../__fixtures__/test-blocks.json'), 'utf-8' ); - // @ts-expect-error access protected method - await searchProvider.requestBulk( - SearchTable.block, - blocks.trim().split('\n'), - { - // make sure the data is visible to search - refresh: 'true', - } - ); + const blockDocuments = blocks + .trim() + .split('\n') + .map(line => JSON.parse(line)); + await searchProvider.write(SearchTable.block, blockDocuments, { + refresh: true, + }); + const docs = await readFile( path.join(import.meta.dirname, '../__fixtures__/test-docs.json'), 'utf-8' ); - // @ts-expect-error access protected method - await searchProvider.requestBulk(SearchTable.doc, docs.trim().split('\n'), { - refresh: 'true', + const docDocuments = docs + .trim() + .split('\n') + .map(line => JSON.parse(line)); + await searchProvider.write(SearchTable.doc, docDocuments, { + refresh: true, }); }); @@ -587,6 +582,70 @@ test('should handle blob as string[]', async t => { t.snapshot(result.nodes.map(node => omit(node, ['_score']))); }); +test('should batch write bugfix', async t => { + const workspaceId = 'workspaceId-batch-write-bugfix-for-manticoresearch'; + + await searchProvider.write( + SearchTable.block, + [ + { + workspace_id: workspaceId, + doc_id: 'a', + block_id: 'b1', + content: '2025-05-26', + flavour: 'affine:page', + additional: '{"displayMode":"edgeless"}', + created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + created_at: '2025-05-26T05:16:23.128Z', + updated_at: '2025-05-26T05:15:53.091Z', + flavour_indexed: 'affine:page', + }, + { + workspace_id: workspaceId, + doc_id: 'a', + block_id: 'b2', + content: '', + flavour: 'affine:surface', + parent_flavour: 'affine:page', + parent_block_id: 'TcOGF6HSa7', + additional: '', + created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + created_at: '2025-05-26T05:16:23.128Z', + updated_at: '2025-05-26T05:15:53.091Z', + flavour_indexed: 'affine:surface', + parent_flavour_indexed: 'affine:page', + parent_block_id_indexed: 'TcOGF6HSa7', + }, + ], + { + refresh: true, + } + ); + + const result = await searchProvider.search(SearchTable.block, { + _source: ['workspace_id', 'doc_id'], + query: { + bool: { + must: [ + { + term: { + workspace_id: { + value: workspaceId, + }, + }, + }, + ], + }, + }, + fields: ['workspace_id', 'doc_id', 'block_id', 'content'], + sort: ['_score'], + }); + + t.snapshot(result.nodes.map(node => omit(node, ['_score']))); +}); + // #endregion // #region search diff --git a/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts b/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts index 0fc2227196..696a39c739 100644 --- a/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts +++ b/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts @@ -112,11 +112,11 @@ export class ElasticsearchProvider extends SearchProvider { ); records.push(JSON.stringify(document)); } - const query: Record = {}; + const url = new URL(`${this.config.provider.endpoint}/_bulk`); if (options?.refresh) { - query.refresh = 'true'; + url.searchParams.set('refresh', 'true'); } - await this.requestBulk(table, records, query); + await this.requestBulk(url.toString(), records); this.logger.debug( `wrote ${documents.length} documents to ${table} in ${Date.now() - start}ms` ); @@ -220,17 +220,7 @@ export class ElasticsearchProvider extends SearchProvider { /** * @see https://www.elastic.co/docs/api/doc/elasticsearch-serverless/operation/operation-bulk-2 */ - protected async requestBulk( - table: SearchTable, - records: string[], - query?: Record - ) { - const url = new URL(`${this.config.provider.endpoint}/${table}/_bulk`); - if (query) { - Object.entries(query).forEach(([key, value]) => { - url.searchParams.set(key, value); - }); - } + protected async requestBulk(url: string, records: string[]) { return await this.request( 'POST', url.toString(), diff --git a/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts b/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts index 9eeb1432bc..822515a433 100644 --- a/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts +++ b/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts @@ -99,6 +99,7 @@ export class ManticoresearchProvider extends ElasticsearchProvider { ), })); } + await super.write(table, documents, options); }