From 409e71ff8b3ea05253a522d9d50bec6e783c26f1 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Tue, 27 May 2025 06:43:26 +0000 Subject: [PATCH] fix(server): use /_bulk endpoint instead (#12542) ## Summary by CodeRabbit - **Bug Fixes** - Improved reliability of batch write operations to search providers, ensuring documents are correctly indexed and retrievable. - **Tests** - Added new test cases and snapshots to verify batch write functionality and confirm resolution of prior batch processing issues. --- .../__tests__/__fixtures__/test-blocks.json | 13 --- .../__tests__/__fixtures__/test-docs.json | 11 -- .../__snapshots__/elasticsearch.spec.ts.md | 47 +++++++- .../__snapshots__/elasticsearch.spec.ts.snap | Bin 4695 -> 4996 bytes .../__snapshots__/manticoresearch.spec.ts.md | 51 +++++++++ .../manticoresearch.spec.ts.snap | Bin 5893 -> 6154 bytes .../__tests__/providers/elasticsearch.spec.ts | 90 +++++++++++++-- .../providers/manticoresearch.spec.ts | 103 ++++++++++++++---- .../indexer/providers/elasticsearch.ts | 18 +-- .../indexer/providers/manticoresearch.ts | 1 + 10 files changed, 261 insertions(+), 73 deletions(-) diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json index 9341acc80a..9583934ef0 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json +++ b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json @@ -1,26 +1,13 @@ -{ "index" : {"_id" : "workspaceId1/docId1/title/blockId1", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId1", "content" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "flavour" : "title", "blob" : "blob1", "ref_doc_id" : "refDocId1", "ref" : "ref1", "parent_flavour" : "parentFlavour1", "parent_block_id" : "parentBlockId1", "additional" : "additional1", "markdown_preview" : "markdownPreview1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour2/blockId2", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId2", "content" : "title2 world, test searching morphology", "flavour" : "flavour2", "blob" : "blob2", "ref_doc_id" : "refDocId2", "ref" : "ref2", "parent_flavour" : "parentFlavour2", "parent_block_id" : "parentBlockId2", "additional" : "additional2", "markdown_preview" : "markdownPreview2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour3/blockId3", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId3", "content" : "title3 hello update", "flavour" : "flavour3", "blob" : "blob3", "ref_doc_id" : "refDocId3", "ref" : "ref3", "parent_flavour" : "parentFlavour3", "parent_block_id" : "parentBlockId3", "additional" : "additional3", "markdown_preview" : "markdownPreview3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour4/blockId4", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId4", "content" : "title4 hello", "flavour" : "flavour4", "blob" : "blob4", "ref_doc_id" : "refDocId4", "ref" : "ref4", "parent_flavour" : "parentFlavour4", "parent_block_id" : "parentBlockId4", "additional" : "additional4", "markdown_preview" : "markdownPreview4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour5/blockId5", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId5", "content" : "title5 hello", "flavour" : "flavour5", "blob" : "blob5", "ref_doc_id" : "refDocId5", "ref" : "ref5", "parent_flavour" : "parentFlavour5", "parent_block_id" : "parentBlockId5", "additional" : "additional5", "markdown_preview" : "markdownPreview5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId1/flavour6/blockId6", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId6", "content" : "title6 hello", "flavour" : "flavour6", "blob" : "blob6", "ref_doc_id" : "refDocId6", "ref" : "ref6", "parent_flavour" : "parentFlavour6", "parent_block_id" : "parentBlockId6", "additional" : "additional6", "markdown_preview" : "markdownPreview6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId1/flavour7/blockId7", "_index" : "block"} } {"workspace_id" : "workspaceId2", "doc_id" : "docId1", "block_id" : "blockId7", "content" : "title7 hello", "flavour" : "flavour7", "blob" : "blob7", "ref_doc_id" : "refDocId7", "ref" : "ref7", "parent_flavour" : "parentFlavour7", "parent_block_id" : "parentBlockId7", "additional" : "additional7", "markdown_preview" : "markdownPreview7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId9", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId9", "block_id" : "blockId9", "content" : "title9 hello affine issue hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour9", "parent_block_id" : "parentBlockId9", "additional" : "additional9", "markdown_preview" : "markdownPreview9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId10", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId10", "content" : "this is docId2 title content hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour10", "parent_block_id" : "parentBlockId10", "additional" : "additional10", "markdown_preview" : "markdownPreview10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId11", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId11", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour11", "parent_block_id" : "parentBlockId11", "additional" : "additional11", "markdown_preview" : "markdownPreview11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId12", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId12", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour12", "parent_block_id" : "parentBlockId12", "additional" : "additional12", "markdown_preview" : "markdownPreview12", "created_by_user_id" : "userId12", "updated_by_user_id" : "userId12", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} -{ "index" : {"_id" : "workspaceId1/docId3/affine:page/blockId13", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId13", "content" : "this is docId3 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour13", "parent_block_id" : "parentBlockId13", "additional" : "additional13", "markdown_preview" : "markdownPreview13", "created_by_user_id" : "userId13", "updated_by_user_id" : "userId13", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} -{ "index" : {"_id" : "workspaceId1/docId3/affine:database/blockId14", "_index" : "block"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId14", "content" : "this is docId3 title content world", "flavour" : "affine:database", "parent_flavour": "affine:database", "parent_block_id" : "parentBlockId14", "additional" : "additional14", "markdown_preview" : "markdownPreview14", "created_by_user_id" : "userId14", "updated_by_user_id" : "userId14", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"} diff --git a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json index 69b755c804..e252622d95 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json +++ b/packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json @@ -1,22 +1,11 @@ -{ "index" : {"_id" : "workspaceId1/docId1", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId1", "title" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "summary" : "summary1", "journal" : "journal1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId2", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId2", "title" : "title2 world, test searching morphology", "summary" : "summary2", "journal" : "journal2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId1/docId3", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId3", "title" : "title3 hello update", "summary" : "summary3", "journal" : "journal3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId4", "_index" : "doc"} } {"workspace_id" : "workspaceId2", "doc_id" : "docId4", "title" : "title4 hello", "summary" : "summary4", "journal" : "journal4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId5", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId5", "title" : "title5 hello", "summary" : "summary5", "journal" : "journal5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId6", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId6", "title" : "title6 hello", "summary" : "summary6", "journal" : "journal6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId7", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId7", "title" : "title7 hello", "summary" : "summary7", "journal" : "journal7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId2/docId8", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId8", "title" : "title8 hello", "summary" : "summary8", "journal" : "journal8", "created_by_user_id" : "userId8", "updated_by_user_id" : "userId8", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId3/docId9", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId9", "title" : "title9 hello", "summary" : "summary9", "journal" : "journal9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId3/docId10", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId10", "title" : "title10 hello", "summary" : "summary10", "journal" : "journal10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"} -{ "index" : {"_id" : "workspaceId3/docId10", "_index" : "doc"} } {"workspace_id" : "workspaceId1", "doc_id" : "docId11", "title" : "title11 hello, old value", "summary" : "summary11", "journal" : "journal11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"} diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md index ea75f1a483..4f23bb22d4 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md @@ -4,6 +4,51 @@ The actual snapshot is saved in `elasticsearch.spec.ts.snap`. Generated by [AVA](https://avajs.dev). +## should batch write bugfix + +> Snapshot 1 + + [ + { + _id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b1', + _source: { + doc_id: 'a', + workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch', + }, + fields: { + block_id: [ + 'b1', + ], + doc_id: [ + 'a', + ], + workspace_id: [ + 'workspaceId-batch-write-bugfix-for-elasticsearch', + ], + }, + highlights: undefined, + }, + { + _id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b2', + _source: { + doc_id: 'a', + workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch', + }, + fields: { + block_id: [ + 'b2', + ], + doc_id: [ + 'a', + ], + workspace_id: [ + 'workspaceId-batch-write-bugfix-for-elasticsearch', + ], + }, + highlights: undefined, + }, + ] + ## should search block table query match url work > Snapshot 1 @@ -549,7 +594,7 @@ Generated by [AVA](https://avajs.dev). [ { - _id: 'workspaceId1/docId2/affine:page/blockId9', + _id: 'workspaceId1/docId9/blockId9', _source: { doc_id: 'docId9', workspace_id: 'workspaceId1', diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap index a98a9c1b9a8a9fc0aff50dfda1e5745e1b0ebe51..dc889bfecf3023b6d557a78ce3ecf67802df1312 100644 GIT binary patch literal 4996 zcmV-~6MO7IRzV02Z9KJJmln(}s zp><4~IMQkFPMl6UOFC?a5FU9DXzc=VfCS3J<(Ys@AO!HoZVGO~!;}=_kd%aw5{zA1 z3^Z+dSxJKy>Jelx!r-_f5)XZwcof4x;LqzY+u z!_53(b~K#~C*p;^!SMQAs-T7wqXYe^4XJ!4K9V2I7V=lzkxa%5@io2$0{mcqh)je= z080QY12B#FrVyzsmy2&C#CWQ$tpy2!P>>KIXXyn)gnWVhOIRJ!cK|#JKqkOE0>ntM zF5kI{i`2dXq2^;s2EwoyR2rumZD zK1=_SIG0~#cDM!AINqRqiSwrcv#aqqq%gLy`rRXt}JSts#h!;M`|x>n+*W218^TW(vxvb{j)92YHCPk z`_4=@H)t}RnT%?~^#^d7INyQ8#92egsic%{nL4$ZEkS@m0&McsaiM=8z&{<*bH5_M z2LzZw0fhqRQecDv*HYjv3OwOC{ZckV|C)!@jCXSM>Af?S38l)~t6JSbt!sW~(KW$% zGMOr*vYB{V%bO=y`nfNgDX5u(meFg;2=?XFctK6Bjq7SnZnw#>FZ?+pr#1uPscCLj^?!dKU(sIoZ7D?F0{;f!JOK^ z)*{o{*1XY?q}h&YjczrB97s_8z>puf#qTj$Kko&K}<1i^g3n zpB+}i=~QMYJXk1<(y5FZ&n=FRj5Ngi`%@WJNviqOK&By;&yT7NUA;9wpMVG!pBI zL|a*U_wq=0jHPsSMPf@MJ+Vl1X(YNl($mYb*h4hZ-4ltmn3XV0tc4}A`g^pLR#r*0 zn_*&&ERm%|Bhk~fS9C>Ut*oA|UJbl766Vq`k4Vy{Tn!^U~;r&Tw;@b;>m7 zvX%N&<~*W|`KsI0V--NX`kWRu+_Z8~_>y`}p89tE-=_LS^~2euTHjtj5+6|OFD(`z z#H~cLy@Y$SU5ug1T`2MJb;pK09f?jht~(kMoqR7)LOK$i8nIY*dqaZ`F7k_|u@s$V z?3dD!=+xO}UU&C$^C@QLnq|ApvN1EQ$HYa= zw5VB2Pp`dZOK#N6?J;X^@osVXdrKuswOIPH$DwSiy_L~!`+H-JTrEcT3P$%{K+3B# z8x%d=p>s>!*J|&Au3iU0OP|dYuQYT#8rf#A+44~*(tV}R)@zr~(xN7ZPhR?{rM_Ci zl{LCqG$xmG%UWUSTeD>8Q^l6sd#}`IT^gZ^UQXbINiS#8%lUup<=iX+cX)X@yF_5O zmzQ&k57_GE1&$-!K@447p^DEd}XA|Ij z0$k>4Z;1~Q;Clr45drq|aJx+f3ix=N%0FJ4s=BNr4$rjGcbn!?7xU?K>TDvVtOZQN z#__T9GQ^&OjE-*MgkZ*SlK%`ff0f6IYw5ANHsW&f6YoonE~j1xxgx6Y%s z>}C%kIue~$IV>*EBoil@ILX9g$He{jZ#ksd4{UpoaZs1V(5}tp+BKQ&E7PqS$Xm0{ za^f?K^tlnh_KESi-W-QYLCqJGWVTPy#8Ub$8e;rb8ntn>$BoYa>q6&w73lnD1UQEP zIZrwZ4-?>V0=(c&U=-@q6+PMZ@qAh*4*T491FyH_l?P8h>T{mxeql*>+|VQb${Tv~E%{}8cc#E+`P~H` z@w*B9b-%N~JNzC3zv>~dD0>Nfw(Kr&PIeRcM%h{5owA3(|0Y)!*cb2;xFO&!@C5-k zfj0)61->=lA@J@%Wq}WP3Ops~F7T3|o4|h=bQbvXpohSB1uF~uY|uqunb6+`nWFGe zcC1aG*In zXzL-F?&UypdC*fez&~@KFyF$rYJlI=0D;VVmGSeC%5temU?oVEg(=!UScFmWywp8WM5fw)JXQ1CC7|p z*{U{`rbZ(@aO6Hs&=_KhOUE9aFJp^nFk0gDW!x$rR|A{f`sr9U&}HaN=nK{q8`z>W zD@|;;9=k@0j<0EP{fbWfYI5dRt8vc!-6L|&JZGLWr{4f@8^7jfUk+eL(RcAM`$=Nm zL;i`=(w&)G=j6A`#D5k6*aF@#0j?szZ3OtoVi2r{|NJ`wPzubUK$HS2DX@+L*YT@% zAs2V_feOK?FH$#C{ympnSOgyT3Sj+-2>hG_*?f<8Mc_RSWb-{vkbv3Tv1H!&2z5w6w*-7j z0JMQ>h4#fK>bgvJ1nFCE3pEWmW1s%6mP>YwLJz?$XQo+~^?PB{}PRwiD zUs?c0esp9cn=6F#1$B5>&4v3@>4KUot-F8G*!P<0wC{B_fUN*_Ol;pvaow{iilyjc zS$4i&a@+YLP71PykVO@CzF2tG8mF~*`_9)hV-K(T0|CTixbwAOjJ;^t` z`%;~^p06n$t*NM~?xw)~UfW&2pulTKbh~Tr>FP*2yD^;2gjc6B$?W=kII%Gt59_hT ztV^6q%`cVeNOTqpvJJ2D!jlY{WXL2#CK)oxkV%GAVn|sj^K8DFV|?N5w47QP&1J&s z@JL}}IA6%6G6P|4Ahzd6hlk_2jp2*q=~2}oJ}GM97+nOQ*=v164}It`iJ1j33gCdg z&ViX8Bie(QtF5;@>Gr3hg?uP_BNeTYirz>?YotO|qO zTtT;m0N*6QE>CibtrTddz$yw1@gx<;OHxs9ls89L9r+Hm1mR!lLg#y|!Su!I^urF$ zRCbD4vaz&E0H%6{&({mULSejAE_SCw7MxQJ3MxekX0io?hU`(@kjfYGMG}@4Y3vN7 zZyzxR9t7Ygz}$&3@YAjXCX+1~gN8loZ7dF%QX5?esGoV!dAQ7theL23jx0G z$wQ%y0;f{o&yO7k&pYB>YSE~}u4|^(JCTzuFc$h}{Bti+H1MvItnt}CW>|A%e{W5mT^Tv@?j zG=l%OdJpg)AF?g~l>ootp|*QYDT~~jI^i>2`uXD6;T`7E2k|McgLq}-Q(L2Mr?zgT zz@1)$oUvec#X&9;;Z+LkV>f_|`%so#8V}Cb7wBtq?CiGV)*XEo8d+l6foOI{TJx6l zVqxu!b9v53@ng{#akf?8WfOffuY>uiq^OSz*^zL*FPl@t{i$5OV2=_G)gO_azR#IE zed2w^qcrSj=WCujexg8t00G#+&JLcn_I)&gW17^L9nBQ9Ut*CKpRr6O;$W)4t5IP{ z-N?H!UN!4X{o*)kHkZv=cnbj@BIR8w@KoDLfF}v?90#m&p4@**w7xgLMUWcb;*-?8 z-J4zFb1?;;E7JJ)9(rFUl5tLR;~|6ddI9(rA68s#EAm4DcuoN3i@<3ju$qqvo<1(^ z(r%7r-Ltkh+Sw6B8_mjae1oEmXnm1FS3bqXioEEedu|n-bZ%VI@&RAw zp+1{dztsoa$-`_nmkYamz-}LKfQQ!DpfNvinjcu>=b*K=m*@Pz1|DWRr664A2X6EO zJ9+2~8?>6;3nT-I%t24Ey}VZjR`9TyM{faa(Rc=w#oSR{gTk+oE1EmoWhZlIY$lpE z6Cb@c?R9O_2bJ42Cve=hX}N1G{9vGBn>IPNNhb860PtAABV^)V0>DdwO4jBfqWxC@ zc%K9D*Ll&VAh0y(5qhvH2z-hI*}|ST27${s&@>)&dl0yr1KEO=p9lg^aUecenH~rN z2RYD5o(8wy)PZBNw{3`p9f{7y#ZB6^jA4Zto9k6oyGwL2yZKZ*EBgJL_wBg5Qp8_T zu{qu`WlE5X59Hz_E_dVC^=ycTZpkPk}XuEaXdh3)yasJWm0(LUhn$w@D_{ zF96jZ|BuS?pp)mjhVq>yRGjNa9BS7|j+7 z*FTUI{0@w)PHUN(TIFk@k1n5C{H zu2z1lrM!MTksnKDH)Pi3^6Hf-b%TcZwgpj_OXSr|ah>+ujV@lX0BTEs;dJ84>_lFx z|Ikt|L~9H@o^`Oc7mhzx9zf$s!>c_H8n;mSsJ z6^(-0lMFL{m=7{kp64+~1bvQZZ^~F?R86O|NB95oX*Vlm$JIzWl^KhS7K^z;S7T#3 zl~EJ<6^UG~F)=)x%BV_GEu=;=jj2LmLT&6FP?}piH(*T*hm1JQU(w7)$XYmfG}E9^c>w7)YN?~TUVSb5*7XkVO_^!7&Mozeby zG}al7t%~*!uqt*Di}v+L^V3=?lp2u=gBegNpZQ&6_oO+M}^pG}ck-klmt#-88<6_1xDM?QL9j8s?S`rV`M zx(9pMyzXus?BTUQ3F#i}(TK&WyBZsHaEV{6jFsp#W51N{!5*D$Mx91F<45m+0b&$1 zN;M`L6$T---_h>Dp8gJY$*%eajH_6aab>A(Gq3vWwK218yv3|-H*3wS-gZL}UMGgB zKvxr8u1>SZUbCMzd*6Ml%&WLrYqsq*+s4hZeiIin%VK6P{R8%%EwwSTw%_cz)qBL1 z_bs<5_hQjyzeC$L`zT|5_WQ=0xLyqP3WjMK9C0lR#b5j8tJ^72D1>#BomYp7W?UM?3^jKZQ@vt{{F#aG$2 zSJqjtMyQgP6F6q z%c=7N^Sr#AHb2ni<>jpP1MB?2tzKTvvwq+oy}X>){J;SZFGr}Afw?kphdk@q%z8Gn zp3TYR+1P}hx@R*J+E`r|n-(&T&dhP6l@Uhn@CqZBR+3UUdFYBUmvFLhos*$;4S>4< zY@eB-b*6)%rRch;w6~sjv$x)LvA331v9~TDz@-GZ#?#&sw-ewy1bBe}2YI;NrUC_g zyiMhwu1!@_p@^e1tprUAi>ZtGbPjbkky6$MreR~+BDXm9ClmS<1$IymTjYNz@LTF( zi_8~*#e#<|5)*(Pp_-ZD^z!&*@4!In!v4sw?%MXo)7QT7^kWC^+xf=MTi)1t+re*c zJNWf49(;WBp{?5w?A&~C^CQkZ@K44GBM)2Oqqgc64KjNeL=HjnnX(fNN}=)9x~o&S^o7ZM=vNoV0H0_-5bi{4Z|M1aFQ%r{*s zOQkV;Q+Q!D)fu;{B}8&z?k6nelGf0|+lw43WkQP-_`DZuS#akLFV?<9fuGT8Zn_f} z4DB2rvQB5sV`MwH`H@+U%yMLwBQ}mOc96wx3lddzyDw&0K``uXgp$qFiQH6CE6JQMiuJUHx_X)6%0KfER-3J5! zZ)6@v)y$&+F%<9a5WD*#RmH-0q*8uXV_rI zQdj|NurW=az%LSq7lc_GrqycCty+71)!N-owKDO)M}a?3;0yt16M)qMFfIU_1g|%Y z|4{+hCII^c;D7-9UH}$}K##~Z4cVH0Rs{Z11il~wcZk5&Qoh?;eA82HO+ykeM*SDi+xWnuoxm>?QC8 zvb(@}*-hX(WoLnR$sPj#n_OLBU%*S?#(=xPmj&Deo(wn(d~d)*;5~ur0w3}ecuvq= z;Etf1z<(Ze7Wmqrhrka8s|)N!v-9|kstfm_1BgJED-82INf@Mainp5gQwps@xxx2CeAIsDx| zR|6zD&|Dt$#Twvd4m6JkJy-*LlLMW`gPu7e({mi?k9g3pYJmUXK&SKj?BQDAlv-d} zZ6$C%-`m->KtBgs$b-~cV2lI(F%SB3E%4VIXb}(Eazv&_IM8Aq^jt0QBMubdNBEms z;J<2tKwTwp3Ex|!4p_>8KE{Iv>VVZ8D9VGz>VWkeXekf6u@2b8fzIMVkJkZDa-d}# zNP4Lb_*orrxDJr&fra&z4L%lsXX18&KO5;!x>GVv8;*vMuIFz8{ZCY%5*;;WcpQx%A zxA6&oyY;YOuZRB!P?RQTa^~2nN8OdH%QU(Hm$Bt(TQKp&iraC z#SVP$SYii46M&9V{H|)eK#vEUZ^jGqrFel%=+^+;$H)Ba;ehV~*v-R&E}r>MoI-bI zZM~D#45~~Y5>X!Q|6S_$R zzT%a@`jiOl@Je9)i3t3h1KE6!cSPV_4rKE^PLqIzQYDD@Jwn|Q&?f<(k$|iOd|3ki zRsy~y0sAE2bqRQ%#Zb6RSet*c%m=LCKsNv6LLYD$2eSDmENoKXKsNv6CLi!O9LVOM zu;Y&3;y}EALihWCS2)m|=~;8LR?tan1+{t^+B04~T{@VWpOh9 zT2pnUdWZs#du@09f&#A})9tRs=cu`Kb~2L9MAoD-$?S$gWNQ;dxct(@UM5F^TXC; z`cilLVFza_JLN3dSQ-|9Q@qmW8w6mPFkLE_)ag(K=Td`$N{ND*Y|)@0yHq!(3dKT+ zgykTOoq_bNW5&S40Q>}4JTnIVh3kaLWQ)e6VVC+COH-yiMi&C=XI^w3u5{z!*wOOv z4g%amfbV(oP-v&Xrzr4eCys*`A9IyjR@8CVnCb0K&MsBThBCWXin9*bhfR_Ng=@E(dpvf|)FItp>E$lF^amFjI zr{uWJ_+}U5@$xFh;|&D3i2z%@C;MvzI6#08d8lYJ9#1a?^=EdzOFwHIKe~fm`h@Ou zozRz8Kf!gK+X=46D6rjY!tbTP3son)OoU%kfUQ+On)U%LxjY}7KV?v!I*!|T^oP)h z6W0z$v-8rL=cG4_Xy=_PYd(q}%cqF5MSxe%^v?Wx%)eGjYjZK1ixh^kc{MVe$`^|E zjNwqjG1>L|yt(TqK3Y0f!;X2r?z#IX3P;EtAOJh$+0C=ozNaQ|Op}JP6Pcp+ODxgi zbCy?$IGQT*OR6xYPVz2|SI;_AZE3)6L25jUKcU96kk|)(uA~67LJv^j_Z~KYOeEvX=HwCM^L7Dvj88GHu{HUD06Z@M zOGV&p5m>|L2hW|>O4DV|W!=lRINjNtqRnPyJh4&HX0#r#&=;Rl;G!V9tUdRN&TG$J z(Pi!N5VU97f;JnHHaD?t6!x_zH(?h@&cY5zABC_jy6Gjk2z!@{uzTEum5K1C1ia1a z`Mhq)wyLW>V9W=6iHG`ZtNOh@;2S(FVC!+W57^@a4)M@h8#L|*&h`Uq{T#H;cJsU+ z*vP|dCl-X;{J@=lU>6UaXM@(TZv@GJB6HBwY&Rc}fz>>0{_zWjEuYSWvY0!nYl8UA za#eF@o9txnOwB~o-o(dmO#5BO^nUd*Eef2pV_M~!7e5-PI;KsIW0DDdG5|as@JOHd zrvUJBpqjOLglPX60N&$3{HMKWa}ekZdZZy-5(GZOfov(zlR@Ab4sO~?@gH!{^)r%FHFjk}Suo&j zRI^A;=|3$xB_+`py@szUy<2U)N-(G6mr7s$prCnI`}{szw%A ZW}X22T&VW0|4844{|B?1003Oq9u5Ei diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md index f7a619b17d..5807a24e7d 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md @@ -310,6 +310,57 @@ Generated by [AVA](https://avajs.dev). }, ] +## should batch write bugfix + +> Snapshot 1 + + [ + { + _id: '8950102031541144623', + _source: { + doc_id: 'a', + workspace_id: 'workspaceId-batch-write-bugfix-for-manticoresearch', + }, + fields: { + block_id: [ + 'b1', + ], + content: [ + '2025-05-26', + ], + doc_id: [ + 'a', + ], + workspace_id: [ + 'workspaceId-batch-write-bugfix-for-manticoresearch', + ], + }, + highlights: undefined, + }, + { + _id: '8950103131052772834', + _source: { + doc_id: 'a', + workspace_id: 'workspaceId-batch-write-bugfix-for-manticoresearch', + }, + fields: { + block_id: [ + 'b2', + ], + content: [ + '', + ], + doc_id: [ + 'a', + ], + workspace_id: [ + 'workspaceId-batch-write-bugfix-for-manticoresearch', + ], + }, + highlights: undefined, + }, + ] + ## should search query all and get next cursor work > Snapshot 1 diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap index ee2736d98481b08d5a2ae7bc3c3bbf6dab8e708d..95b68a72a835e914de0cf7b344917b699de404c1 100644 GIT binary patch literal 6154 zcmV+l81?5tRzVM#BoJr@T3+d-@9lRCNCq-IT_1@M zATirBeP^ak&-BpUlgtp%570$L6xK&X6v=|1f}&y+A%Yv1hYI?EuplnG!mbi=VL?E_ zU0_qUZ};4;d#h_+31dF^gY;CLdYw9_esxZrs=Ix#KbIfaRQ&Wm8KrC~XG|n znZTApwqyh{`GN5fW3&|5k}qt^7Dv-##o>IZc**umCS6LOEnO!=1^tish}sL_6af7I zwgUJpfE@rH0nka5cA~ty|AWRriID2gwzf8qW)KCWg^<_w6zU=}h_{{i%zkamX3U%( zGL@qa|VK&h6G zjM-AOS`<|F`qRZ}QdzP}Znep(cLjhG0GtD$2;h@q8TFWA@infDJ^+r4{s|lzJ-{Yn z{(>>MF_RxKYx5ClB>{RxV)Z@(tP_b0AC{Y#A@FPgObS)oSM-qbN61` z+vRsuK^Kwdm=vdQa@ap&!uym@=bb>4hL`U`;9AeftGlbeFMSx$gh-MQIYY_F{M zM_B9qaC$V8GXhgm7)Tcb#Zn3IE0|37V@CJYb2s5)%?aca!H%Wlw2{1x{e;~j$Y>@^xC2plWsjx)e zn9WpvcShrhcubE)wNxyciY6k7RMHfLyX+vdv6vq(3>an2wq+=TStFMzGWGk1vqQtV z?9gzjXeO!6)9pq)rlrDhO$+PcaLQ~gC~>;6PMS?Oe8hX11Y9WrJ0##gB)}+r%_ zKT8HS$iQY9__PdM&ld4Giu_mx?va6KWMGj8INrk)@jHrKzK7&99^hsVaJL6|oGsEQ zoLQwT#+5(4nfyQ~%(A^;3|995TYCoc`JNSlp8j;9=R%f&Q;NYXQ4C=d-6<8v(gTK# z^q6YmshLp07!1+QWHX_`d?7TF9xY`D@&%)4qzePXHmLcthqEvp7A(zCSuBjPAEk2rBU4=2WWIPp0*oVar@qHD;(Ufq?y`hsNe(>w+@DI5lCiVK6! zQTRZ7i2{6F1jJuefSUwBEZ?gD4=BKY3&2=zQh^Q?I9e3|@+uWLSrq{CfC^;Q`hYBd zR0S?mft%F&pe*070uSw}@elg~m?a@Fo7sf8<)3dMrlXQ7a5II=>5+K@bVtKZKxcYj zCpcf9W&bNy_hGeGx1}BRO>X%%+x9_P3P2Za9f-MMSXuRYQE$mOfa?ML$jvh=%?$jL zsYEiNMUzqbBb-RaBJr~20uJgPJuC3{5$}2e^jlk&Zxi6x?6$>;x)o39S_v4DfQzO2 zsJp;z$b@|nwge`WE{4hi6H3#07p%zFNSqb- zvcNQ=4u%V4foXKL3~ZO{t;7q#45=va?QbdYTME2CDDaswuwJCV6Eg52kph2N2Ckd2 z0yo==T(I(z_D_zyuN2Kpdz)sQW??V#snCSGEE)37j^z^sN)ft>h+0?M2653lT zfsYa3X0b|mC7@9P4wHZtb5x1;b9>C`>sf04t2>PUH%&d(xT?nmJoVVlQ4jeZ3AkSZ zo)W3Zs}k^rNJU!dc~xBibF%;|D#93mX0kg}%|LVB0+TlnOd<db^^Xq?W+$dLV;5Qaquj6`p1>qiu`oUHh-k5>rbVN1JyQP!=9#^6 z0#oirM2l$Qn4U=JNoyfk!aIREWtJx}@>U7BXb&cCXSLdCLcSsauS>w&WI&UFQ-vol z@{kOS$iU?+sq@6;zpS7Jj|W)d0b=4P1*!fbLekS&K=hgggvx!$l>1Ow?gd8V8e3RF zZ6wqLFW}(IwhX0WEVdA`%a&f#(}gnDr@3Qo;+;-tUiYz{>E({qN2IM@;3BVJE&Z$) z_@Y;^mhSKZcX$P6^8;SsSKfMNvyVvK3UII}Bw)D$#6%$hs}w<;aEtIyAx3PaZ8_-#JG2DM*FpS>`w*T!WOobQv>IX8->Y0I+qKiM>ByT zqZAl5CQ5;U@j@|QU_!Jjkf&~(7s$J~3VF&80o(=PIRH%rpc}g5gs`W4fB>02WS=3x zcL?w^0z4&#K5sqD(UOr^DiKR+dP<9=bUmTP*fnQkk+RkN(?bW)LWs;mq-C>0BylHW zsaRYOM`Kzdl8VGNJvEI^rg@$G0-h1W<{B9|TLy|UaM`qi{N!CSaNi#2 zd{zvbXL^A39$>Qv_>>2@an7(w{|sr<+I*j6*T%A5g-pE)rE|GZdNdOnGD;z;SD~_A z2@vLPF2X#+0nJukd_k0tFwYpuQ3SFyx}ZZ8 zcXzY`EEmPyQN(+%DDLhC1-MxRvUl&n&!3Ax_9+F}Edp7O3ixMyykDLbQ$u(@Zf>U7 zk3LWGRIlTydL8Fv?xT)Zu)4>>>cMQTWE2AZlL5Q@^3eIn{TQ8N$y7KNkHod49*-s? zu}E|QqI0>E;fR(>=&4jJoY2_AynTz#e-NYdNiuMn3}mbm^3RFU`Bp3P@Ch+GC#?wm zbsk{M1AKhW=sbPTSlq6VWkm}0PloKI0xY_b3&;0!;P?d(2aCGr0>}Gu%?>Kvy%Rs?MmyFEDbUCkIc3&AU zwWYU>qzjue`7NUx#|p;gtg*$UxY8j^FU4TWk#a#{#E z$(}bpmZ^@T8P;|r>*~1m5$TH_;95`Za^<-DyVC>g@(7mos}*3o$gFZ(LLJNcg9`Ad zXjy+n0bXZ_8d#zwDsVVs4Efoh=w1~#MFl1q!bY|>8eaV!QNZ?nD)0+Iz_#~E6?mFP zcKd+qe88i=8OLK!$#~M0Lb{ij3t}=*4uzW4|Ci3VSTR3h1ajHYO@ZN3X{@+n`SM(L z)JPYWrN_pWm)HML#wccoMwe%c#c^Z#s=kmGNu-t+bLrx6Xdpi_k{-C(@dGqrp{fK^M6j`1GF%GJvL<5%_3qKtS| zN>%}~XG-9v%GAnTrHpKAtybu!O4iC=Wj}Q^e?Mh=3ZQmdRk@IGK$mS>rO*%)`U1=u z0M4>rff*4wGQ6hyVxsEExXE&4G!pXf^k>=8(N18-J~R??68-5hSJ^EDxX_BCx|;w` zh>zZ1-?M}ANI*ma&X9l%ySHg{6#1wG&?|qpOTe!r;E!yPE=Q3j8R(FKRaTgLfi=39 zIEs8m2ClL^=ugVPt89_C&umYc&=ZkZl%6+eNj;T_#lz*e?!_L?p7d1DES)R$x5klF zBCbUvx*pSFu~Z_VmAA$p;dSHIz0{3xEFO<(2|W@ICsXlMynH$IIbJuGc=tv(Vu@%n z8cT)alqX>=rCWx=IxnXim+qBr#FKh7k_gjS|7bWBPwI)XZ0zE7Vo;6(*!wXWCE8+83DaIwl1b@;bD*QmhtDsZm~yvC9@Ia=E20}LNWV?y!04Pxu^IIh3n zM7+=x;&DB0<%h}!oB8JGWz9Tq+_#(EZO%uU1s3IBo9kPYhqmzVAnzv}KdFV!@oWo^ z1WvlIbO2xNsPCkcZtnno z%n&)wUVhsF{Jx`}E6qnFsT1&Z))KWj9EY92;S5nbOVry5oWc-wFhq^Roxo@(a7ia{ zRVQ#uCvbNs@S9HHg-$^20+w_E(XQHxI@#7v>jK`x5FNl0jdcN=8KSqbL|1kJ+Zdt) zS)!Y}fbTLyiyfZYF5ppy=pfd=`j;-?wJxAzQ7z%YY;8v^0+umE0hZ|R76EG+qQh9C z{32kSA?jg?KCuY6f+0GbCAwu1u!A8wf*s)_i-6xO0-k3GgKTX|H_+5wOLVm3RI?k< z8KPrYqW5+KXE8*_vP8LVpui9GX<6j9o99^ zXM}lPJi$4r1sh_QC8zxKP9SM3ybA22(0HS0v_?UrgyuL9jvv=`Abz)2oG*>g)QH2G zNK>N@cOgxURZ>0n)T(RIYfnv7QWbk@vXZLWQ&W{xUnP|u9@8kZa&p7G&4<4tRUr;L z_sVoQeBstOT&9Cp?TkjJhvup^a~_?$u+3@iTCCAULV8FGMfc$d9ft)qio?#Ii#lnp z>J+WS9IW_TxN0VT5^h@l+%$%>u7!2{gu0iI@~^qL(rD~Qr? zv)T5p^(A}x*4I(on6`IuV%iE`f%Ub`Ti^QHS?49c1;;zD!sp#F3Xga1R@`}4QUtua zN~zDgJL^2s_n!*z2hk&a4Qko%U)ORyRs}SM$YHtmslXYc$N4s?zUFsRG+n z;AR!rr2>zsz;i0#@d1l{K-dTL`G5^fc!$HfE&6~-hR9*vUgZP6Qa4bj^y-NyfEABBtx*=aOIM1&an|`CNwlZUVfY02AV;GAduEX_bJZ#7||+ z7Z|SPf?;{WL*|56UdAxT9ZxgKw&5V!RGQ6e;nsW0k`v#B) zU_X_BUy9zzpx;>d-HhMK;H#edO=MK`CuHf*Xued@A^Ovp$QDaQOM|As+C_bCTyW3L z=+Rgr9?>H4gr15;^h8)o)PHW~u-WR*83dr;*BKY;kNhkFUL-*CzSEsGT-_PXmn!;0 zf0ifP(w=E-FC~X1Rl9{e%}uGOPgVlPMM^?%NBv`6C2@`YGdsWmACqIOz^8|X3dT^n zWLN=hmhhMIV*&FnS70z(D3+W8@BhNQ813g*jdsu10o(%MegIDiWZy?TG65P0@OA>M z6dUX8F`RcGKR#MA|EkZlK{I2a6d!X1G37~Jt z94P}Q$iO<*LU#lRc}p^Iz6^X_2I!^u$JruXGxWLWNi76AbL&AOswERzJQaz>tZ>HX zc|ADDEo6-bHF!Ba*t3d{c!8o9*v5W0!RuJ?cY6VP&&|gMwJ9vAbeIAh$&xiVYOH)M zW)n-=;vn6r0J{|6-&oRCN8>Lm!2hyj9gZ65hg=q`z&qKYVfly$JVQgBa(Q6EW-FIF6yW4N5 zcT^VA!n~uhwrZn=^iUx^#<>LBwt6BxGL|#w4H2_dQFFZOIMZwH@HN@dp_~yYj`tT! z+0r<5auS3!&KCpU)SqO}|QHZ5jK1hu4%oHM3zk9^!8l47sxnoJe@1r zaxZqv^%3do1h|E5!eI?aza_x$Su%AVRv6b}u9xepi&1#N$%OPh0Jn=3tz3ouo6Gj8 zc-0De5V+T7jXK|$HQ&zQ^q78Cd~$E#4*Lx$W-YP2yRKI@*L`cAhH17jmtlI%m96sE c0nC|m1;DIcZ(o?bwjFf;KhX*6-$#J}0Q*?c0RR91 literal 5893 zcmV+g7y9TyRzV- z1hYNUH8X8`ribpHB*Q~KK}AJfSXU#9i)2BOhk_!C7{SHmp`v~uEQrglu%8lfVL?E_ zU0`#ot9z<%)$KdeGl^q9_=D8k`@6U9J@=e@?m73I>K&uQ`NGKN(r5nJEa%F3bKCOL zcwsu94Q?&w%4RTI7@3|hC(FUDh2rL1X)-fa8ZVSf7wyPqGv&<~Fm^04M?Yl$b|-7F&Fk%cDO9he!Va4v+5V zV6i~a9Nm;Hj99&CL|Q|DK@nTMj{qA)Y*i+}IYhu#pC`Z%;wGDEEfhdrC_OC%uSA+dDiUCf1$l1>QRmrks|<-wnRCnLfENDAgzR) zRB6~s$cN~^#FEHw0Qe4o-vW3YzT_>AnMx+2iMVbg;zm3bOBrd45$<#DYVFrFJ5 z&*#R*%O$HwZJ+Bfk_p|2By~L!jYJIVZ6S#>j15vV!_bKT5(&6W0&bOn|C9htW(X*| zXnvXuoFxNWWZ>^);2O4x&(-7yGH{O!JS_vue84e2rb)onCYBM!V#A2MRT-v1~_+Mv``pW9UK_W6bH^{2{j#1JYQ~H41P&18QSI_b9;q45*z2?NWee8Bhlc>QI3`wI0+dQe2C2GH}Uh zgX6+m z4ZUX$RODq~MdcypYPE=Z;v}A)nBwV)&w1&IJNBS<4ZFx2^`vifNfP)e9)a5w4uN&W zgTQAfd_{bb0(?TGh`+1=HwYB5e6IrhQ~~~5pp4}<73ficBUFJxUaJD{Pz4HkLN1s>ep25r98H13f=W$ zw?bz|-~^l#ZF2lotNY-3tJ{{2#wNFXo8$N(tpE@e9UX{uU|3W0dQor5G=OUWeBaA6 zE6q>&r;SuPrN`58`VdK_6R~8)asfx{9?_KY*NA^30fy}(%gqG%C3|dfYu&1+biD*j zNWg_sW39WyZOD{!6Sf&9oGFDX3=_`Kbr8M44Mq7H`hWPxdPg$(SF8|}nP!3?Pi@a=C1@EZcWF9`6dGO$r3z}sZtLm~nG zCmFbU-U8h25a1xs%3o9wt}5;W>lOEKzA)^qwX0X8a$xBRy`GH4jdUt%BopaWB9V;6 z8w-7?Nul?xC%_p5IFA5V2?gHw3ISdxz%mI~DHe8_8HA6nMorY|A9@qpfv4=oow7@N zp0nUqZlB)a5JPL2l{;#lIA7)|&Ki-vEde`4x4#ccz@wtuU%D|oJKx*i8dc|UE9DTw zsz~T>hy*@PfE&dk;g^6`2{>2+RxeN_y3ZW2mTzE%^>5T={Qt$o;}lQvIFBbDJ2>JY z-y;F{OTd#N5qU)dUKfc-C%vy~sKC4|z^aHa2B1~!t*YiBId3SFznn6OEfs0mr<94e zOJ@!uQ9Y4G>=RjqOu@6K&EiZjlg|e;liA>ySq@H`+seU_>0+r+WWv}!Tn%Q}v?v!d z7uGIjl;K#d-rYFT0H2y{cs;6TKBb7+& zQA3Xz(P&Cfu#CK^MET76(?=)JL5R$|H(A-#y@_~`2_un=M&b!Q6*FQ!((PSOyw$Ekt>1OdbcBxp+5HN zX-ADxwL&`x>EzwP#2poNW1l+-SkBrjBsksABf*FK-Xyr1M}qtKB+!WTj30QxFWA)S z4NQ+HHsJ^bSS5;0I7tECD~e6{iUQmyicPp%0e&KiO?XlPc8OvWd@2x__Z3KGRV)eN z`2o0eb0Td-63JLn zPe+sSbSx2zFG1~G8Du1;8>y&aBqAxDb$ISw?fg5jc0OJPPLzSHeT(opv39=2j?Q>o ztew+#6ypXTFy#Y2v0&{yck3bP6v&Dog@R*UD-kWoN z8(07SEl2-;l&62MZ>WEN?gt(g>ECDlz>9u?{tYNVmq`DvP=KgN|IR4D1tR_X1qHZL zq<_Dq0Jn?u?_&z^Yo$5;dqURRyI5nE=CoMT4Af|(oQsD>Bb{3YOOvBzwQQY5Jdr7n zj0dNSd5)HC>Gn#Gq23~zIdRpT(qs40GkN#5uqw?y_EppBfujBc#Z&V5^$&l3`xM41Y9fuUzC8`gqlWr zNCF;XVI9p_XLGzqY;$mjZuK3oNb`b2tp+mLY_6OuOlI;{-J4{NbLeR2Rp3a`%#_XS zrc9-7SoYo-FLso7O=OCjvxTjbo2H89mYliOLfq&;blSOdb(`)34uDtk-5(wGwQI|R z99pRE+OmU?6@a-Pxj8j=3d2VCW=->zKwQ9FIWT|U9KeuV<;5Sdk8 zTc~SWe?S2q5pC-)E5K_EsD%X`paO?5#!!F_92-=DcdEb)18ik`qakSD5Cyf}rvg6{ z1hx5}P=Tjt^qU4;tpSf{^Nv}YkGEd2 zHe|$UKWm39a4KXB(h<%!PtYnl40q3IIw@n&YOY)3NrckrP&#RMNMDhpFEx&%JrBl1 zYrPb$&-eJ_KN)Kx`oFegy{EEadGk(nR~oFS`o^H4%FdPcz7PtYTM4!uSY7#--oH&0 zvgW|*0eWEZ2sLC|MBOd;)Al+)If&(BTbBvugMlmXlrcJ)GsMHN(fO z9imMc_Nun5Dae6YhMR3u&v(@}vg2*_OgG!Ip7*NfsU!I3DMz|w{jsVVw7FlO<5;E8 zYdD%Ee=>m6>`eIykt4%zxi7ZW92qy*j*M19{)0YN939;R*8NZ`A;;54pS83G73Bq>cIx)HSvg$;hrFfQIB!$_v1@mMNC<00b_Bbkn-D!g$QZy3)vH4KgT zss{Y~72py@WNoO=D8LH}&}pY2&@6*t6}V7kn!5a3zAIJW8Wp%#1zu(0ZLT+M(g0Hf zKCA&(XuvHR@Q?<)s4-33U2p1Y0s312qXj6mFz{urH$Bh-JkkQZ&;qmvfKUJ!3IL-4 zrfI+HO&11$j|YGo0>Be2{6N>6j%x)@Xa&qx;DT0Qdn<5TEAVJ5)AS(Mn_g@M{@e;2 z&<32+#=sAAz3J9A;CpSrBW=L*Z9q#qaAZ4hVms6HaA70@V=4Nwg%Q?i#+XoiZ-dyn zJeuq8w=`aOR^xF!Zk3O@jJEU5(M#HS-ng6Fy=~5i+XWWopW7Q-lm~V2^R)I6iXY#> zr+BV|NAb^ecvJkA4gtjzPXb}@k=|sDSm6GfZ~sJHm3ONo%=k+ z5AWhr{9RoBVl~&)oti-SQqF*duV#89l(;djw9p^*z8DJpw1)Ob>8DkHAUyr5@nRJ&m1o z((OIK4;YZ^?&a4#z;AmRxzaQuNxgv9TMz1TISzY)Ll{su3mWVN-pPP^7*OkYFEH5) zT+|C(-V5B+3*6lc{Hhmtz86sYfCKt~cwc=-z3kgg>;vAzfc9fSQ+>b|2J|KtbXgy; zodLa>1>M*Oe3Jn!cX?|2fJYe6{;YrXPkq3veL&B$df;2w-VR>|tYknz7W6mEfb|UM zU=~zZ223-c0T%ShWx%Bj=nxik(=y;z26QMp!-tmvzgh-7#{ffYZ%RMV)?W`g!gZ_J z4@4Qzku2!F{lIAq=qMJH?+1zuXoZU#`ho2XD9k#nztazVzaMz8AK29oyv8(GDRfxZ zr5HzeUOd4)sI4}{E=x}Px!pk0x9~b}&O+zSqSc-SjS^bmL^yt2&xr)Q-r~;2rlm2L zGm)0YUG73!ny8lgoTW9_qTgAXs+KCw(sZ>{b(R{{QmtA_FOO-IS*5sT(bgkS<*G1; zoO@(BT)uF79v;iV>vmqVi-s2}H4C1dH?u8h?KI8>fKM!%z zJ;f`9`P5b5bX@XI`3W?_^JI-B!TDheRgDC*^Zt6iVwKg2Ry|_>9xC7 zRuHA(WwY&B>r3|Yt*^tmF>UYS#IzOt0_$tLzp?eTv%%MnmK^Q;3ZHgIDm>b~Tk)n{ zSrO3ga-}it?r88KqyJKX--$kC)S_1W{tYeHqf|g=KrYL5NCi$7eZXk53Y^V=Sj)Bb zG8Nda0ynC_T`KUX3OuUBsGR2wTmP~%ybO(M)FD+V5@znrm zMCU^b%PH=Ors9!kDr&^iNj;THM5C2YO1{Jk77aF;QY^ol0PiQjHgP&}^$Y%;5^%UU zop`a7;d-tztSorgTJUQ2ur>Fvz2ISM!3DX*ZBHskxIt&STYqg60vA1qNf_CCJW`N z{LshBV%yR)hvTK>vZU&dkSBTx74^wVz_dt6Xm-@!Hxv@j*gtCk9OYwCjAi)D*jUjV z%alz!pv`9fa$zcH<#Gi_bH!5G&G3OQEQ-;7PR(fdT?gPM0QUoUQo#Ef@yP^eA;4P+ zutsdGv)6F`k;3$3+4`%RsrCT}|NF*sWtRKp&E||#`_k+VQO^|~xo93e*D_4$)dK;( zDfsaSc_#txBEbC&u+4kJp}7ultVKYouEo((&4Dr{0b3;C%Mx&h1pJ#oTs7kVg9N-R z0f))Ju`;lMwa{GwLjJN0oFfC*$pB4?e~fL?H&35SG_A+uaU&It=rmTVw&TgX9nUL< zu7i-1ooNKl^b+Bot0yA{-F1y*x`)N z@kX$}SI8O-YVdPLuzMGu@B<}3u$}$3wBNPi@A3mQ&rM^4+7uQl9jpL{v9K0bkJT^g zZ)Tw#F6d4LxJv>4orQL~UjC8-{4WdZarH<)cE4N&-p)qOx#HR6OI6_WDsVFkb;Ykq zKT?68v#>Ysx#ZCmwOv14HcRE%l(KNSFcr4;{qRI)TiDwB9lCCp%F}i2npc`+E$b%g ztgdC8Ia2jbZrapH3sBz-rUy${EPv)oHzf|0=(4FFQXBuKLETXP!DoN6r=*c z(G18HQSgBPkYzy2g~v9v^6iGE^IG>`Gki>BrJe|egIi5=^GcnjF$8zrdBtyT|4O5) zvalZEU6pm!URunI6*E(uB-pNX+cFbVd6Q;{SZ@`*j+c%zx7QtVN^Wv2Zw5=#!=-Yr zJWXA>!O=o-B2%u7pnAeR$UfmtWr`&;XqM~)ZrLnOFj*QOshnvyeAQ{gtppGk=et(q z=0t~0<)d$~x;PuaR%=~75j$^g_3>fSBb2zWWV3xx46$sq^rbH{-VUW6rRSa{5=5w;$@L7 zI!)}nwW|0U-r~D8R~`?JOqa@qiHaysWTvKSUQXT*l(sI8qzo4d`FTpp?z7RSon1^! zmn@Acf7|ww-ou#8vbV|nK+R-+xn?p;2iae(X>6KF^rJS%u`q=*+B2PAAoJ|$Oupo( zz0j*xBhqyQxQTs*%Nme=O@QCBFm(}j7|&*&$@R6(D6-^uLi!ef+r@%b>B9NDl+LAi z#SVH9xYwH2I@dQ%x05*|roX&8vnTWp`#T%VUSfN9JySL}e0`CoX}x11)AU { path.join(import.meta.dirname, '../__fixtures__/test-blocks.json'), 'utf-8' ); - // @ts-expect-error access protected method - await searchProvider.requestBulk( - SearchTable.block, - blocks.trim().split('\n'), - { - // make sure the data is visible to search - refresh: 'true', - } - ); + const blockDocuments = blocks + .trim() + .split('\n') + .map(line => JSON.parse(line)); + await searchProvider.write(SearchTable.block, blockDocuments, { + refresh: true, + }); + const docs = await readFile( path.join(import.meta.dirname, '../__fixtures__/test-docs.json'), 'utf-8' ); - // @ts-expect-error access protected method - await searchProvider.requestBulk(SearchTable.doc, docs.trim().split('\n'), { - refresh: 'true', + const docDocuments = docs + .trim() + .split('\n') + .map(line => JSON.parse(line)); + await searchProvider.write(SearchTable.doc, docDocuments, { + refresh: true, }); }); @@ -616,6 +618,70 @@ test('should handle blob as string[]', async t => { }); }); +test('should batch write bugfix', async t => { + const workspaceId = 'workspaceId-batch-write-bugfix-for-elasticsearch'; + + await searchProvider.write( + SearchTable.block, + [ + { + workspace_id: workspaceId, + doc_id: 'a', + block_id: 'b1', + content: '2025-05-26', + flavour: 'affine:page', + additional: '{"displayMode":"edgeless"}', + created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + created_at: '2025-05-26T05:16:23.128Z', + updated_at: '2025-05-26T05:15:53.091Z', + flavour_indexed: 'affine:page', + }, + { + workspace_id: workspaceId, + doc_id: 'a', + block_id: 'b2', + content: '', + flavour: 'affine:surface', + parent_flavour: 'affine:page', + parent_block_id: 'TcOGF6HSa7', + additional: '', + created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + created_at: '2025-05-26T05:16:23.128Z', + updated_at: '2025-05-26T05:15:53.091Z', + flavour_indexed: 'affine:surface', + parent_flavour_indexed: 'affine:page', + parent_block_id_indexed: 'TcOGF6HSa7', + }, + ], + { + refresh: true, + } + ); + + const result = await searchProvider.search(SearchTable.block, { + _source: ['workspace_id', 'doc_id'], + query: { + bool: { + must: [ + { + term: { + workspace_id: { + value: workspaceId, + }, + }, + }, + ], + }, + }, + fields: ['workspace_id', 'doc_id', 'block_id'], + sort: ['_score'], + }); + + t.snapshot(result.nodes.map(node => omit(node, ['_score']))); +}); + // #endregion // #region search diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts index 6ecf13b5b0..75b3f1f690 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts @@ -11,7 +11,7 @@ import { ConfigModule } from '../../../../base/config'; import { IndexerModule } from '../../'; import { SearchProviderType } from '../../config'; import { ManticoresearchProvider } from '../../providers'; -import { SearchTable } from '../../tables'; +import { blockSQL, docSQL, SearchTable } from '../../tables'; const module = await createModule({ imports: [ @@ -32,15 +32,8 @@ const user = await module.create(Mockers.User); const workspace = await module.create(Mockers.Workspace); test.before(async () => { - const tablesDir = path.join(import.meta.dirname, '../../tables'); - await searchProvider.createTable( - SearchTable.block, - path.join(tablesDir, 'block.sql') - ); - await searchProvider.createTable( - SearchTable.doc, - path.join(tablesDir, 'doc.sql') - ); + await searchProvider.createTable(SearchTable.block, blockSQL); + await searchProvider.createTable(SearchTable.doc, docSQL); await searchProvider.write( SearchTable.block, @@ -122,22 +115,24 @@ test.before(async () => { path.join(import.meta.dirname, '../__fixtures__/test-blocks.json'), 'utf-8' ); - // @ts-expect-error access protected method - await searchProvider.requestBulk( - SearchTable.block, - blocks.trim().split('\n'), - { - // make sure the data is visible to search - refresh: 'true', - } - ); + const blockDocuments = blocks + .trim() + .split('\n') + .map(line => JSON.parse(line)); + await searchProvider.write(SearchTable.block, blockDocuments, { + refresh: true, + }); + const docs = await readFile( path.join(import.meta.dirname, '../__fixtures__/test-docs.json'), 'utf-8' ); - // @ts-expect-error access protected method - await searchProvider.requestBulk(SearchTable.doc, docs.trim().split('\n'), { - refresh: 'true', + const docDocuments = docs + .trim() + .split('\n') + .map(line => JSON.parse(line)); + await searchProvider.write(SearchTable.doc, docDocuments, { + refresh: true, }); }); @@ -587,6 +582,70 @@ test('should handle blob as string[]', async t => { t.snapshot(result.nodes.map(node => omit(node, ['_score']))); }); +test('should batch write bugfix', async t => { + const workspaceId = 'workspaceId-batch-write-bugfix-for-manticoresearch'; + + await searchProvider.write( + SearchTable.block, + [ + { + workspace_id: workspaceId, + doc_id: 'a', + block_id: 'b1', + content: '2025-05-26', + flavour: 'affine:page', + additional: '{"displayMode":"edgeless"}', + created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + created_at: '2025-05-26T05:16:23.128Z', + updated_at: '2025-05-26T05:15:53.091Z', + flavour_indexed: 'affine:page', + }, + { + workspace_id: workspaceId, + doc_id: 'a', + block_id: 'b2', + content: '', + flavour: 'affine:surface', + parent_flavour: 'affine:page', + parent_block_id: 'TcOGF6HSa7', + additional: '', + created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de', + created_at: '2025-05-26T05:16:23.128Z', + updated_at: '2025-05-26T05:15:53.091Z', + flavour_indexed: 'affine:surface', + parent_flavour_indexed: 'affine:page', + parent_block_id_indexed: 'TcOGF6HSa7', + }, + ], + { + refresh: true, + } + ); + + const result = await searchProvider.search(SearchTable.block, { + _source: ['workspace_id', 'doc_id'], + query: { + bool: { + must: [ + { + term: { + workspace_id: { + value: workspaceId, + }, + }, + }, + ], + }, + }, + fields: ['workspace_id', 'doc_id', 'block_id', 'content'], + sort: ['_score'], + }); + + t.snapshot(result.nodes.map(node => omit(node, ['_score']))); +}); + // #endregion // #region search diff --git a/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts b/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts index 0fc2227196..696a39c739 100644 --- a/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts +++ b/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts @@ -112,11 +112,11 @@ export class ElasticsearchProvider extends SearchProvider { ); records.push(JSON.stringify(document)); } - const query: Record = {}; + const url = new URL(`${this.config.provider.endpoint}/_bulk`); if (options?.refresh) { - query.refresh = 'true'; + url.searchParams.set('refresh', 'true'); } - await this.requestBulk(table, records, query); + await this.requestBulk(url.toString(), records); this.logger.debug( `wrote ${documents.length} documents to ${table} in ${Date.now() - start}ms` ); @@ -220,17 +220,7 @@ export class ElasticsearchProvider extends SearchProvider { /** * @see https://www.elastic.co/docs/api/doc/elasticsearch-serverless/operation/operation-bulk-2 */ - protected async requestBulk( - table: SearchTable, - records: string[], - query?: Record - ) { - const url = new URL(`${this.config.provider.endpoint}/${table}/_bulk`); - if (query) { - Object.entries(query).forEach(([key, value]) => { - url.searchParams.set(key, value); - }); - } + protected async requestBulk(url: string, records: string[]) { return await this.request( 'POST', url.toString(), diff --git a/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts b/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts index 9eeb1432bc..822515a433 100644 --- a/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts +++ b/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts @@ -99,6 +99,7 @@ export class ManticoresearchProvider extends ElasticsearchProvider { ), })); } + await super.write(table, documents, options); }