Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-14 21:27:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): convert date and timestamp value to Date instance (#12867)

Browse Source 
#### PR Dependency Tree


* **PR #12867** 👈
  * **PR #12863**
    * **PR #12837**
    * **PR #12866**

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **New Features**
- Date fields in search results are now returned as JavaScript Date
objects instead of strings or numeric timestamps.
- **Bug Fixes**
- Improved consistency and correctness of date field types across
different search providers.
- **Tests**
- Added tests to verify that date fields are correctly returned as Date
objects.
- Updated existing test snapshots to reflect the new Date object format
for date fields.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
fengmk2
2025-06-20 17:21:29 +08:00
committed by GitHub
parent 13b64c6780
commit bebe4349a9
9 changed files with 153 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md
View File

@@ -67,7 +67,7 @@ Generated by [AVA](https://avajs.dev).
'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2',
],
created_at: [
'2025-03-08T06:04:13.278Z',
Date 2025-03-08 06:04:13 278ms UTC {},
],
doc_id: [
'docId2',
@@ -89,7 +89,7 @@ Generated by [AVA](https://avajs.dev).
'docId1',
],
updated_at: [
'2025-03-08T06:04:13.278Z',
Date 2025-03-08 06:04:13 278ms UTC {},
],
},
highlights: {
@@ -119,7 +119,7 @@ Generated by [AVA](https://avajs.dev).
'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2',
],
created_at: [
'2025-03-08T06:04:13.278Z',
Date 2025-03-08 06:04:13 278ms UTC {},
],
doc_id: [
'docId2',
@@ -141,7 +141,7 @@ Generated by [AVA](https://avajs.dev).
'docId1',
],
updated_at: [
'2025-03-08T06:04:13.278Z',
Date 2025-03-08 06:04:13 278ms UTC {},
],
},
highlights: {

BIN
packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap
View File

Binary file not shown.

4
packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md
View File

@@ -485,7 +485,7 @@ Generated by [AVA](https://avajs.dev).
'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2',
],
created_at: [
1741413853,
Date 2025-03-08 06:04:13 UTC {},
],
doc_id: [
'docId2',
@@ -507,7 +507,7 @@ Generated by [AVA](https://avajs.dev).
'docId1',
],
updated_at: [
1741413853,
Date 2025-03-08 06:04:13 UTC {},
],
},
highlights: {

BIN
packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap
View File

Binary file not shown.

43
packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts
View File

@@ -1477,6 +1477,49 @@ test('should not return not exists field:ref_doc_id', async t => {
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
});
test('should created_at and updated_at is date type', async t => {
const result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id', 'created_at', 'updated_at'],
sort: [
'_score',
{
updated_at: 'desc',
},
'doc_id',
'block_id',
],
query: {
match_all: {},
},
fields: ['created_at', 'updated_at'],
size: 2,
});
t.truthy(result.total);
t.truthy(result.nodes[0].fields.created_at);
t.truthy(result.nodes[0].fields.updated_at);
t.true(
result.nodes[0].fields.created_at[0] instanceof Date,
'created_at should be date type, but got ' +
result.nodes[0].fields.created_at[0]
);
t.true(
result.nodes[0].fields.updated_at[0] instanceof Date,
'updated_at should be date type, but got ' +
result.nodes[0].fields.updated_at[0]
);
t.true(
result.nodes[0]._source.created_at instanceof Date,
'created_at should be date type, but got ' +
result.nodes[0]._source.created_at
);
t.true(
result.nodes[0]._source.updated_at instanceof Date,
'updated_at should be date type, but got ' +
result.nodes[0]._source.updated_at
);
});
// #endregion
// #region aggregate

43
packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts
View File

@@ -1232,6 +1232,49 @@ test('should not return not exists field:ref_doc_id', async t => {
t.snapshot(result.nodes.map(node => omit(node, ['_score'])));
});
test('should created_at and updated_at is date type', async t => {
const result = await searchProvider.search(SearchTable.block, {
_source: ['workspace_id', 'doc_id', 'created_at', 'updated_at'],
sort: [
'_score',
{
updated_at: 'desc',
},
'doc_id',
'block_id',
],
query: {
match_all: {},
},
fields: ['created_at', 'updated_at'],
size: 2,
});
t.truthy(result.total);
t.truthy(result.nodes[0].fields.created_at);
t.truthy(result.nodes[0].fields.updated_at);
t.true(
result.nodes[0].fields.created_at[0] instanceof Date,
'created_at should be date type, but got ' +
result.nodes[0].fields.created_at[0]
);
t.true(
result.nodes[0].fields.updated_at[0] instanceof Date,
'updated_at should be date type, but got ' +
result.nodes[0].fields.updated_at[0]
);
t.true(
result.nodes[0]._source.created_at instanceof Date,
'created_at should be date type, but got ' +
result.nodes[0]._source.created_at
);
t.true(
result.nodes[0]._source.updated_at instanceof Date,
'updated_at should be date type, but got ' +
result.nodes[0]._source.updated_at
);
});
// #endregion
// #region aggregate

41
packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts
View File

@@ -5,7 +5,7 @@ import {
InvalidSearchProviderRequest,
} from '../../../base';
import { SearchProviderType } from '../config';
import { SearchTable, SearchTableUniqueId } from '../tables';
import { DateFieldNames, SearchTable, SearchTableUniqueId } from '../tables';
import {
AggregateQueryDSL,
AggregateResult,
@@ -161,8 +161,8 @@ export class ElasticsearchProvider extends SearchProvider {
nodes: data.hits.hits.map(hit => ({
_id: hit._id,
_score: hit._score,
_source: hit._source,
fields: hit.fields,
_source: this.formatDateFields(hit._source),
fields: this.formatDateFields(hit.fields),
highlights: hit.highlight,
})),
};
@@ -187,8 +187,8 @@ export class ElasticsearchProvider extends SearchProvider {
nodes: bucket.result.hits.hits.map(hit => ({
_id: hit._id,
_score: hit._score,
_source: hit._source,
fields: hit.fields,
_source: this.formatDateFields(hit._source),
fields: this.formatDateFields(hit.fields),
highlights: hit.highlight,
})),
},
@@ -196,6 +196,37 @@ export class ElasticsearchProvider extends SearchProvider {
};
}
protected formatDateFields<T extends Record<string, unknown[] | unknown>>(
fieldsOrSource: T
): T {
for (const fieldName of DateFieldNames) {
let values = fieldsOrSource[fieldName];
if (!values) {
continue;
}
if (Array.isArray(values)) {
// { created_at: ['2025-06-20T03:02:43.442Z'] } => { created_at: [new Date('2025-06-20T03:02:43.442Z')] }
values = values.map(this.formatDateValue);
} else {
// { created_at: '2025-06-20T03:02:43.442Z' } => { created_at: new Date('2025-06-20T03:02:43.442Z') }
values = this.formatDateValue(values);
}
// @ts-expect-error ignore type check
fieldsOrSource[fieldName] = values;
}
return fieldsOrSource;
}
/**
* elasticsearch return date value as string, we need to convert it to Date object
*/
protected formatDateValue(value: unknown) {
if (value && typeof value === 'string') {
return new Date(value);
}
return value;
}
protected async requestSearch(table: SearchTable, body: Record<string, any>) {
const url = `${this.config.provider.endpoint}/${table}/_search`;
const jsonBody = JSON.stringify(body);

27
packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts
View File

@@ -141,8 +141,12 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
nodes: data.hits.hits.map(hit => ({
_id: hit._id,
_score: hit._score,
_source: this.#formatSource(dsl._source, hit._source),
fields: this.#formatFieldsFromSource(dsl.fields, hit._source),
_source: this.formatDateFields(
this.#formatSource(dsl._source, hit._source)
),
fields: this.formatDateFields(
this.#formatFieldsFromSource(dsl.fields, hit._source)
),
highlights: this.#formatHighlights(
dsl.highlight?.fields,
hit.highlight
@@ -176,8 +180,12 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
const node = {
_id: hit._id,
_score: hit._score,
_source: this.#formatSource(topHits._source, hit._source),
fields: this.#formatFieldsFromSource(topHits.fields, hit._source),
_source: this.formatDateFields(
this.#formatSource(topHits._source, hit._source)
),
fields: this.formatDateFields(
this.#formatFieldsFromSource(topHits.fields, hit._source)
),
highlights: this.#formatHighlights(
topHits.highlight?.fields,
hit.highlight
@@ -236,6 +244,17 @@ export class ManticoresearchProvider extends ElasticsearchProvider {
return data;
}
/**
* manticoresearch return date value as timestamp, we need to convert it to Date object
*/
protected override formatDateValue(value: unknown) {
if (value && typeof value === 'number') {
// 1750389254 => new Date(1750389254 * 1000)
return new Date(value * 1000);
}
return value;
}
private parseESQuery(
query: Record<string, any>,
options?: {

2
packages/backend/server/src/plugins/indexer/tables/index.ts
View File

@@ -11,5 +11,7 @@ export const SearchTableUniqueId = {
[SearchTable.doc]: getDocUniqueId,
};
export const DateFieldNames = ['created_at', 'updated_at'];
export * from './block';
export * from './doc';