Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-13 08:06:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): return empty summary field value (#12517)

Browse Source 
close AF-2658

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **Tests**
  - Added new test cases and snapshots to enhance coverage for search results involving empty or missing fields like summary, title, and ref_doc_id.
  - Verified consistent handling of empty string values and absence of fields across different search providers.

- **Bug Fixes**
  - Improved handling of empty string values for specific fields by converting them to null to ensure consistent search result formatting.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
fengmk2
2025-05-27 04:42:54 +00:00
parent 3676f3b769
commit 382c237dac
10 changed files with 459 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md
+49
View File
@@ -494,6 +494,55 @@ Generated by [AVA](https://avajs.dev).
},
}
## should return empty string field:summary value
> Snapshot 1
[
{
_id: 'workspaceId-search-query-return-empty-string-field-summary-value-for-elasticsearch/doc0',
_source: {
doc_id: 'doc0',
workspace_id: 'workspaceId-search-query-return-empty-string-field-summary-value-for-elasticsearch',
},
fields: {
doc_id: [
'doc0',
],
summary: [
'',
],
title: [
'',
],
},
highlights: undefined,
},
]
## should not return not exists field:ref_doc_id
> Snapshot 1
[
{
_id: 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-elasticsearch/doc0/block0',
_source: {
doc_id: 'doc0',
workspace_id: 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-elasticsearch',
},
fields: {
block_id: [
'block0',
],
doc_id: [
'doc0',
],
},
highlights: undefined,
},
]
## should aggregate query work
> Snapshot 1
packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap
BIN
View File
Binary file not shown.
packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md
+67
View File
@@ -9,6 +9,9 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 1
{
block_id: [
'',
],
content: [
'hello world',
],
@@ -29,6 +32,9 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 2
{
block_id: [
'',
],
content: [
'hello world',
],
@@ -43,6 +49,9 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 3
{
block_id: [
'',
],
content: [
'hello world',
],
@@ -239,6 +248,9 @@ Generated by [AVA](https://avajs.dev).
blob: [
'blob1',
],
content: [
'',
],
flavour: [
'affine:page',
],
@@ -262,6 +274,9 @@ Generated by [AVA](https://avajs.dev).
'blob1',
'blob2',
],
content: [
'',
],
flavour: [
'affine:page',
],
@@ -284,6 +299,9 @@ Generated by [AVA](https://avajs.dev).
blob: [
'blob3',
],
content: [
'',
],
flavour: [
'affine:page',
],
@@ -687,6 +705,55 @@ Generated by [AVA](https://avajs.dev).
},
]
## should return empty string field:summary value
> Snapshot 1
[
{
_id: '274027293861775228',
_source: {
doc_id: 'doc0',
workspace_id: 'workspaceId-search-query-return-empty-string-field-summary-value-for-manticoresearch',
},
fields: {
doc_id: [
'doc0',
],
summary: [
'',
],
title: [
'',
],
},
highlights: undefined,
},
]
## should not return not exists field:ref_doc_id
> Snapshot 1
[
{
_id: '2457631367295327017',
_source: {
doc_id: 'doc0',
workspace_id: 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-manticoresearch',
},
fields: {
block_id: [
'block0',
],
doc_id: [
'doc0',
],
},
highlights: undefined,
},
]
## should aggregate query return top score first
> Snapshot 1
packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap
BIN
View File
Binary file not shown.
packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts
+107
View File
@@ -1303,6 +1303,113 @@ test('should search doc title support stemmer filter', async t => {
t.snapshot(omit(result.nodes[0], ['_score']));
});
test('should return empty string field:summary value', async t => {
const workspaceId =
'workspaceId-search-query-return-empty-string-field-summary-value-for-elasticsearch';
const docId = 'doc0';
await searchProvider.write(
SearchTable.doc,
[
{
workspace_id: workspaceId,
doc_id: docId,
title: '',
summary: '',
created_by_user_id: user.id,
updated_by_user_id: user.id,
created_at: new Date(),
updated_at: new Date(),
},
],
{
refresh: true,
}
);
let result = await searchProvider.search(SearchTable.doc, {
_source: ['workspace_id', 'doc_id'],
query: {
bool: {
must: [
{
term: { workspace_id: { value: workspaceId } },
},
{
term: {
doc_id: {
value: docId,
},
},
},
],
},
},
fields: ['doc_id', 'title', 'summary'],
sort: ['_score'],
});
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
});
test('should not return not exists field:ref_doc_id', async t => {
const workspaceId =
'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-elasticsearch';
const docId = 'doc0';
const blockId = 'block0';
await searchProvider.write(
SearchTable.block,
[
{
workspace_id: workspaceId,
doc_id: docId,
block_id: blockId,
content: 'hello world on search title blockId1-text',
flavour: 'affine:text',
created_by_user_id: user.id,
updated_by_user_id: user.id,
created_at: new Date(),
updated_at: new Date(),
},
],
{
refresh: true,
}
);
let result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id'],
query: {
bool: {
must: [
{
term: { workspace_id: { value: workspaceId } },
},
{
term: {
doc_id: {
value: docId,
},
},
},
],
},
},
fields: [
'doc_id',
'block_id',
'ref_doc_id',
'parent_block_id',
'additional',
'parent_flavour',
],
sort: ['_score'],
});
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
});
// #endregion
// #region aggregate
packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts
+107
View File
@@ -1065,6 +1065,113 @@ test('should search query match ref_doc_id work', async t => {
t.is(result.total, 2);
});
test('should return empty string field:summary value', async t => {
const workspaceId =
'workspaceId-search-query-return-empty-string-field-summary-value-for-manticoresearch';
const docId = 'doc0';
await searchProvider.write(
SearchTable.doc,
[
{
workspace_id: workspaceId,
doc_id: docId,
title: '',
summary: '',
created_by_user_id: user.id,
updated_by_user_id: user.id,
created_at: new Date(),
updated_at: new Date(),
},
],
{
refresh: true,
}
);
let result = await searchProvider.search(SearchTable.doc, {
_source: ['workspace_id', 'doc_id'],
query: {
bool: {
must: [
{
term: { workspace_id: { value: workspaceId } },
},
{
term: {
doc_id: {
value: docId,
},
},
},
],
},
},
fields: ['doc_id', 'title', 'summary'],
sort: ['_score'],
});
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
});
test('should not return not exists field:ref_doc_id', async t => {
const workspaceId =
'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-manticoresearch';
const docId = 'doc0';
const blockId = 'block0';
await searchProvider.write(
SearchTable.block,
[
{
workspace_id: workspaceId,
doc_id: docId,
block_id: blockId,
content: 'hello world on search title blockId1-text',
flavour: 'affine:text',
created_by_user_id: user.id,
updated_by_user_id: user.id,
created_at: new Date(),
updated_at: new Date(),
},
],
{
refresh: true,
}
);
let result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id'],
query: {
bool: {
must: [
{
term: { workspace_id: { value: workspaceId } },
},
{
term: {
doc_id: {
value: docId,
},
},
},
],
},
},
fields: [
'doc_id',
'block_id',
'ref_doc_id',
'parent_block_id',
'additional',
'parent_flavour',
],
sort: ['_score'],
});
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
});
// #endregion
// #region aggregate
packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts
+13 -1
View File
@@ -38,6 +38,15 @@ const SupportIndexedAttributes = [
'parent_block_id',
];
const ConvertEmptyStringToNullValueFields = new Set([
'ref_doc_id',
'ref',
'blob',
'additional',
'parent_block_id',
'parent_flavour',
]);
@Injectable()
export class ManticoresearchProvider extends ElasticsearchProvider {
override type = SearchProviderType.Manticoresearch;
@@ -344,7 +353,10 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
return fields.reduce(
(acc, field) => {
let value = source[field];
if (value !== null && value !== undefined && value !== '') {
if (ConvertEmptyStringToNullValueFields.has(field) && value === '') {
value = null;
}
if (value !== null && value !== undefined) {
// special handle `ref_doc_id`, `ref`, `blob` as string[]
if (
(field === 'ref_doc_id' || field === 'ref' || field === 'blob') &&