mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-02-13 21:05:19 +00:00
fix(server): use /_bulk endpoint instead (#12542)
<!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **Bug Fixes** - Improved reliability of batch write operations to search providers, ensuring documents are correctly indexed and retrievable. - **Tests** - Added new test cases and snapshots to verify batch write functionality and confirm resolution of prior batch processing issues. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
fengmk2
@@ -1,26 +1,13 @@
|
||||
{ "index" : {"_id" : "workspaceId1/docId1/title/blockId1", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId1", "content" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "flavour" : "title", "blob" : "blob1", "ref_doc_id" : "refDocId1", "ref" : "ref1", "parent_flavour" : "parentFlavour1", "parent_block_id" : "parentBlockId1", "additional" : "additional1", "markdown_preview" : "markdownPreview1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId1/flavour2/blockId2", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId2", "content" : "title2 world, test searching morphology", "flavour" : "flavour2", "blob" : "blob2", "ref_doc_id" : "refDocId2", "ref" : "ref2", "parent_flavour" : "parentFlavour2", "parent_block_id" : "parentBlockId2", "additional" : "additional2", "markdown_preview" : "markdownPreview2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId1/flavour3/blockId3", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId3", "content" : "title3 hello update", "flavour" : "flavour3", "blob" : "blob3", "ref_doc_id" : "refDocId3", "ref" : "ref3", "parent_flavour" : "parentFlavour3", "parent_block_id" : "parentBlockId3", "additional" : "additional3", "markdown_preview" : "markdownPreview3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId1/flavour4/blockId4", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId4", "content" : "title4 hello", "flavour" : "flavour4", "blob" : "blob4", "ref_doc_id" : "refDocId4", "ref" : "ref4", "parent_flavour" : "parentFlavour4", "parent_block_id" : "parentBlockId4", "additional" : "additional4", "markdown_preview" : "markdownPreview4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId1/flavour5/blockId5", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId5", "content" : "title5 hello", "flavour" : "flavour5", "blob" : "blob5", "ref_doc_id" : "refDocId5", "ref" : "ref5", "parent_flavour" : "parentFlavour5", "parent_block_id" : "parentBlockId5", "additional" : "additional5", "markdown_preview" : "markdownPreview5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId1/flavour6/blockId6", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId6", "content" : "title6 hello", "flavour" : "flavour6", "blob" : "blob6", "ref_doc_id" : "refDocId6", "ref" : "ref6", "parent_flavour" : "parentFlavour6", "parent_block_id" : "parentBlockId6", "additional" : "additional6", "markdown_preview" : "markdownPreview6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId2/docId1/flavour7/blockId7", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId2", "doc_id" : "docId1", "block_id" : "blockId7", "content" : "title7 hello", "flavour" : "flavour7", "blob" : "blob7", "ref_doc_id" : "refDocId7", "ref" : "ref7", "parent_flavour" : "parentFlavour7", "parent_block_id" : "parentBlockId7", "additional" : "additional7", "markdown_preview" : "markdownPreview7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId9", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId9", "block_id" : "blockId9", "content" : "title9 hello affine issue hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour9", "parent_block_id" : "parentBlockId9", "additional" : "additional9", "markdown_preview" : "markdownPreview9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId10", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId10", "content" : "this is docId2 title content hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour10", "parent_block_id" : "parentBlockId10", "additional" : "additional10", "markdown_preview" : "markdownPreview10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId11", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId11", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour11", "parent_block_id" : "parentBlockId11", "additional" : "additional11", "markdown_preview" : "markdownPreview11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId2/affine:page/blockId12", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId12", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour12", "parent_block_id" : "parentBlockId12", "additional" : "additional12", "markdown_preview" : "markdownPreview12", "created_by_user_id" : "userId12", "updated_by_user_id" : "userId12", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId3/affine:page/blockId13", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId13", "content" : "this is docId3 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour13", "parent_block_id" : "parentBlockId13", "additional" : "additional13", "markdown_preview" : "markdownPreview13", "created_by_user_id" : "userId13", "updated_by_user_id" : "userId13", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId3/affine:database/blockId14", "_index" : "block"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId14", "content" : "this is docId3 title content world", "flavour" : "affine:database", "parent_flavour": "affine:database", "parent_block_id" : "parentBlockId14", "additional" : "additional14", "markdown_preview" : "markdownPreview14", "created_by_user_id" : "userId14", "updated_by_user_id" : "userId14", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"}
|
||||
|
||||
@@ -1,22 +1,11 @@
|
||||
{ "index" : {"_id" : "workspaceId1/docId1", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "title" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "summary" : "summary1", "journal" : "journal1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId2", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "title" : "title2 world, test searching morphology", "summary" : "summary2", "journal" : "journal2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId1/docId3", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "title" : "title3 hello update", "summary" : "summary3", "journal" : "journal3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId2/docId4", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId2", "doc_id" : "docId4", "title" : "title4 hello", "summary" : "summary4", "journal" : "journal4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId2/docId5", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId5", "title" : "title5 hello", "summary" : "summary5", "journal" : "journal5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId2/docId6", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId6", "title" : "title6 hello", "summary" : "summary6", "journal" : "journal6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId2/docId7", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId7", "title" : "title7 hello", "summary" : "summary7", "journal" : "journal7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId2/docId8", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId8", "title" : "title8 hello", "summary" : "summary8", "journal" : "journal8", "created_by_user_id" : "userId8", "updated_by_user_id" : "userId8", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId3/docId9", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId9", "title" : "title9 hello", "summary" : "summary9", "journal" : "journal9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId3/docId10", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId10", "title" : "title10 hello", "summary" : "summary10", "journal" : "journal10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
|
||||
{ "index" : {"_id" : "workspaceId3/docId10", "_index" : "doc"} }
|
||||
{"workspace_id" : "workspaceId1", "doc_id" : "docId11", "title" : "title11 hello, old value", "summary" : "summary11", "journal" : "journal11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"}
|
||||
|
||||
@@ -4,6 +4,51 @@ The actual snapshot is saved in `elasticsearch.spec.ts.snap`.
|
||||
|
||||
Generated by [AVA](https://avajs.dev).
|
||||
|
||||
## should batch write bugfix
|
||||
|
||||
> Snapshot 1
|
||||
|
||||
[
|
||||
{
|
||||
_id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b1',
|
||||
_source: {
|
||||
doc_id: 'a',
|
||||
workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch',
|
||||
},
|
||||
fields: {
|
||||
block_id: [
|
||||
'b1',
|
||||
],
|
||||
doc_id: [
|
||||
'a',
|
||||
],
|
||||
workspace_id: [
|
||||
'workspaceId-batch-write-bugfix-for-elasticsearch',
|
||||
],
|
||||
},
|
||||
highlights: undefined,
|
||||
},
|
||||
{
|
||||
_id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b2',
|
||||
_source: {
|
||||
doc_id: 'a',
|
||||
workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch',
|
||||
},
|
||||
fields: {
|
||||
block_id: [
|
||||
'b2',
|
||||
],
|
||||
doc_id: [
|
||||
'a',
|
||||
],
|
||||
workspace_id: [
|
||||
'workspaceId-batch-write-bugfix-for-elasticsearch',
|
||||
],
|
||||
},
|
||||
highlights: undefined,
|
||||
},
|
||||
]
|
||||
|
||||
## should search block table query match url work
|
||||
|
||||
> Snapshot 1
|
||||
@@ -549,7 +594,7 @@ Generated by [AVA](https://avajs.dev).
|
||||
|
||||
[
|
||||
{
|
||||
_id: 'workspaceId1/docId2/affine:page/blockId9',
|
||||
_id: 'workspaceId1/docId9/blockId9',
|
||||
_source: {
|
||||
doc_id: 'docId9',
|
||||
workspace_id: 'workspaceId1',
|
||||
|
||||
Binary file not shown.
@@ -310,6 +310,57 @@ Generated by [AVA](https://avajs.dev).
|
||||
},
|
||||
]
|
||||
|
||||
## should batch write bugfix
|
||||
|
||||
> Snapshot 1
|
||||
|
||||
[
|
||||
{
|
||||
_id: '8950102031541144623',
|
||||
_source: {
|
||||
doc_id: 'a',
|
||||
workspace_id: 'workspaceId-batch-write-bugfix-for-manticoresearch',
|
||||
},
|
||||
fields: {
|
||||
block_id: [
|
||||
'b1',
|
||||
],
|
||||
content: [
|
||||
'2025-05-26',
|
||||
],
|
||||
doc_id: [
|
||||
'a',
|
||||
],
|
||||
workspace_id: [
|
||||
'workspaceId-batch-write-bugfix-for-manticoresearch',
|
||||
],
|
||||
},
|
||||
highlights: undefined,
|
||||
},
|
||||
{
|
||||
_id: '8950103131052772834',
|
||||
_source: {
|
||||
doc_id: 'a',
|
||||
workspace_id: 'workspaceId-batch-write-bugfix-for-manticoresearch',
|
||||
},
|
||||
fields: {
|
||||
block_id: [
|
||||
'b2',
|
||||
],
|
||||
content: [
|
||||
'',
|
||||
],
|
||||
doc_id: [
|
||||
'a',
|
||||
],
|
||||
workspace_id: [
|
||||
'workspaceId-batch-write-bugfix-for-manticoresearch',
|
||||
],
|
||||
},
|
||||
highlights: undefined,
|
||||
},
|
||||
]
|
||||
|
||||
## should search query all and get next cursor work
|
||||
|
||||
> Snapshot 1
|
||||
|
||||
Binary file not shown.
@@ -154,22 +154,24 @@ _test.before(async () => {
|
||||
path.join(import.meta.dirname, '../__fixtures__/test-blocks.json'),
|
||||
'utf-8'
|
||||
);
|
||||
// @ts-expect-error access protected method
|
||||
await searchProvider.requestBulk(
|
||||
SearchTable.block,
|
||||
blocks.trim().split('\n'),
|
||||
{
|
||||
// make sure the data is visible to search
|
||||
refresh: 'true',
|
||||
}
|
||||
);
|
||||
const blockDocuments = blocks
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => JSON.parse(line));
|
||||
await searchProvider.write(SearchTable.block, blockDocuments, {
|
||||
refresh: true,
|
||||
});
|
||||
|
||||
const docs = await readFile(
|
||||
path.join(import.meta.dirname, '../__fixtures__/test-docs.json'),
|
||||
'utf-8'
|
||||
);
|
||||
// @ts-expect-error access protected method
|
||||
await searchProvider.requestBulk(SearchTable.doc, docs.trim().split('\n'), {
|
||||
refresh: 'true',
|
||||
const docDocuments = docs
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => JSON.parse(line));
|
||||
await searchProvider.write(SearchTable.doc, docDocuments, {
|
||||
refresh: true,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -616,6 +618,70 @@ test('should handle blob as string[]', async t => {
|
||||
});
|
||||
});
|
||||
|
||||
test('should batch write bugfix', async t => {
|
||||
const workspaceId = 'workspaceId-batch-write-bugfix-for-elasticsearch';
|
||||
|
||||
await searchProvider.write(
|
||||
SearchTable.block,
|
||||
[
|
||||
{
|
||||
workspace_id: workspaceId,
|
||||
doc_id: 'a',
|
||||
block_id: 'b1',
|
||||
content: '2025-05-26',
|
||||
flavour: 'affine:page',
|
||||
additional: '{"displayMode":"edgeless"}',
|
||||
created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de',
|
||||
updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de',
|
||||
created_at: '2025-05-26T05:16:23.128Z',
|
||||
updated_at: '2025-05-26T05:15:53.091Z',
|
||||
flavour_indexed: 'affine:page',
|
||||
},
|
||||
{
|
||||
workspace_id: workspaceId,
|
||||
doc_id: 'a',
|
||||
block_id: 'b2',
|
||||
content: '',
|
||||
flavour: 'affine:surface',
|
||||
parent_flavour: 'affine:page',
|
||||
parent_block_id: 'TcOGF6HSa7',
|
||||
additional: '',
|
||||
created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de',
|
||||
updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de',
|
||||
created_at: '2025-05-26T05:16:23.128Z',
|
||||
updated_at: '2025-05-26T05:15:53.091Z',
|
||||
flavour_indexed: 'affine:surface',
|
||||
parent_flavour_indexed: 'affine:page',
|
||||
parent_block_id_indexed: 'TcOGF6HSa7',
|
||||
},
|
||||
],
|
||||
{
|
||||
refresh: true,
|
||||
}
|
||||
);
|
||||
|
||||
const result = await searchProvider.search(SearchTable.block, {
|
||||
_source: ['workspace_id', 'doc_id'],
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
term: {
|
||||
workspace_id: {
|
||||
value: workspaceId,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
fields: ['workspace_id', 'doc_id', 'block_id'],
|
||||
sort: ['_score'],
|
||||
});
|
||||
|
||||
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
|
||||
});
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region search
|
||||
|
||||
@@ -11,7 +11,7 @@ import { ConfigModule } from '../../../../base/config';
|
||||
import { IndexerModule } from '../../';
|
||||
import { SearchProviderType } from '../../config';
|
||||
import { ManticoresearchProvider } from '../../providers';
|
||||
import { SearchTable } from '../../tables';
|
||||
import { blockSQL, docSQL, SearchTable } from '../../tables';
|
||||
|
||||
const module = await createModule({
|
||||
imports: [
|
||||
@@ -32,15 +32,8 @@ const user = await module.create(Mockers.User);
|
||||
const workspace = await module.create(Mockers.Workspace);
|
||||
|
||||
test.before(async () => {
|
||||
const tablesDir = path.join(import.meta.dirname, '../../tables');
|
||||
await searchProvider.createTable(
|
||||
SearchTable.block,
|
||||
path.join(tablesDir, 'block.sql')
|
||||
);
|
||||
await searchProvider.createTable(
|
||||
SearchTable.doc,
|
||||
path.join(tablesDir, 'doc.sql')
|
||||
);
|
||||
await searchProvider.createTable(SearchTable.block, blockSQL);
|
||||
await searchProvider.createTable(SearchTable.doc, docSQL);
|
||||
|
||||
await searchProvider.write(
|
||||
SearchTable.block,
|
||||
@@ -122,22 +115,24 @@ test.before(async () => {
|
||||
path.join(import.meta.dirname, '../__fixtures__/test-blocks.json'),
|
||||
'utf-8'
|
||||
);
|
||||
// @ts-expect-error access protected method
|
||||
await searchProvider.requestBulk(
|
||||
SearchTable.block,
|
||||
blocks.trim().split('\n'),
|
||||
{
|
||||
// make sure the data is visible to search
|
||||
refresh: 'true',
|
||||
}
|
||||
);
|
||||
const blockDocuments = blocks
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => JSON.parse(line));
|
||||
await searchProvider.write(SearchTable.block, blockDocuments, {
|
||||
refresh: true,
|
||||
});
|
||||
|
||||
const docs = await readFile(
|
||||
path.join(import.meta.dirname, '../__fixtures__/test-docs.json'),
|
||||
'utf-8'
|
||||
);
|
||||
// @ts-expect-error access protected method
|
||||
await searchProvider.requestBulk(SearchTable.doc, docs.trim().split('\n'), {
|
||||
refresh: 'true',
|
||||
const docDocuments = docs
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => JSON.parse(line));
|
||||
await searchProvider.write(SearchTable.doc, docDocuments, {
|
||||
refresh: true,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -587,6 +582,70 @@ test('should handle blob as string[]', async t => {
|
||||
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
|
||||
});
|
||||
|
||||
test('should batch write bugfix', async t => {
|
||||
const workspaceId = 'workspaceId-batch-write-bugfix-for-manticoresearch';
|
||||
|
||||
await searchProvider.write(
|
||||
SearchTable.block,
|
||||
[
|
||||
{
|
||||
workspace_id: workspaceId,
|
||||
doc_id: 'a',
|
||||
block_id: 'b1',
|
||||
content: '2025-05-26',
|
||||
flavour: 'affine:page',
|
||||
additional: '{"displayMode":"edgeless"}',
|
||||
created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de',
|
||||
updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de',
|
||||
created_at: '2025-05-26T05:16:23.128Z',
|
||||
updated_at: '2025-05-26T05:15:53.091Z',
|
||||
flavour_indexed: 'affine:page',
|
||||
},
|
||||
{
|
||||
workspace_id: workspaceId,
|
||||
doc_id: 'a',
|
||||
block_id: 'b2',
|
||||
content: '',
|
||||
flavour: 'affine:surface',
|
||||
parent_flavour: 'affine:page',
|
||||
parent_block_id: 'TcOGF6HSa7',
|
||||
additional: '',
|
||||
created_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de',
|
||||
updated_by_user_id: '46ce597c-098a-4c61-a106-ce79827ec1de',
|
||||
created_at: '2025-05-26T05:16:23.128Z',
|
||||
updated_at: '2025-05-26T05:15:53.091Z',
|
||||
flavour_indexed: 'affine:surface',
|
||||
parent_flavour_indexed: 'affine:page',
|
||||
parent_block_id_indexed: 'TcOGF6HSa7',
|
||||
},
|
||||
],
|
||||
{
|
||||
refresh: true,
|
||||
}
|
||||
);
|
||||
|
||||
const result = await searchProvider.search(SearchTable.block, {
|
||||
_source: ['workspace_id', 'doc_id'],
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
term: {
|
||||
workspace_id: {
|
||||
value: workspaceId,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
fields: ['workspace_id', 'doc_id', 'block_id', 'content'],
|
||||
sort: ['_score'],
|
||||
});
|
||||
|
||||
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
|
||||
});
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region search
|
||||
|
||||
@@ -112,11 +112,11 @@ export class ElasticsearchProvider extends SearchProvider {
|
||||
);
|
||||
records.push(JSON.stringify(document));
|
||||
}
|
||||
const query: Record<string, string> = {};
|
||||
const url = new URL(`${this.config.provider.endpoint}/_bulk`);
|
||||
if (options?.refresh) {
|
||||
query.refresh = 'true';
|
||||
url.searchParams.set('refresh', 'true');
|
||||
}
|
||||
await this.requestBulk(table, records, query);
|
||||
await this.requestBulk(url.toString(), records);
|
||||
this.logger.debug(
|
||||
`wrote ${documents.length} documents to ${table} in ${Date.now() - start}ms`
|
||||
);
|
||||
@@ -220,17 +220,7 @@ export class ElasticsearchProvider extends SearchProvider {
|
||||
/**
|
||||
* @see https://www.elastic.co/docs/api/doc/elasticsearch-serverless/operation/operation-bulk-2
|
||||
*/
|
||||
protected async requestBulk(
|
||||
table: SearchTable,
|
||||
records: string[],
|
||||
query?: Record<string, string>
|
||||
) {
|
||||
const url = new URL(`${this.config.provider.endpoint}/${table}/_bulk`);
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
url.searchParams.set(key, value);
|
||||
});
|
||||
}
|
||||
protected async requestBulk(url: string, records: string[]) {
|
||||
return await this.request(
|
||||
'POST',
|
||||
url.toString(),
|
||||
|
||||
@@ -99,6 +99,7 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
|
||||
),
|
||||
}));
|
||||
}
|
||||
|
||||
await super.write(table, documents, options);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user