feat(server): improve indexer (#14698)

fix #13862 


#### PR Dependency Tree


* **PR #14698** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Enhanced search support for Chinese, Japanese, and Korean languages
with improved text segmentation and character matching.
* Added index management capabilities with table recreation
functionality.

* **Bug Fixes**
* Improved search accuracy for non-Latin scripts through updated
morphology and n-gram configuration.

* **Chores**
  * Added database migration for search index optimization.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/plugins/indexer/service.ts

@@ -14,6 +14,7 @@ import {
   AggregateQueryDSL,
   BaseQueryDSL,
   HighlightDSL,
+  ManticoresearchProvider,
   OperationOptions,
   SearchNode,
   SearchProvider,
@@ -130,6 +131,63 @@ export class IndexerService {
     }
   }
 
+  async rebuildManticoreIndexes() {
+    let searchProvider: SearchProvider | undefined;
+    try {
+      searchProvider = this.factory.get();
+    } catch (err) {
+      if (err instanceof SearchProviderNotFound) {
+        this.logger.debug('No search provider found, skip rebuilding tables');
+        return;
+      }
+      throw err;
+    }
+
+    if (!(searchProvider instanceof ManticoresearchProvider)) {
+      this.logger.debug(
+        `Search provider ${searchProvider.type} does not need manticore rebuild`
+      );
+      return;
+    }
+
+    const mappings = SearchTableMappingStrings[searchProvider.type];
+    for (const table of Object.keys(mappings) as SearchTable[]) {
+      await searchProvider.recreateTable(table, mappings[table]);
+    }
+
+    let lastWorkspaceSid = 0;
+    while (true) {
+      const workspaces = await this.models.workspace.list(
+        { sid: { gt: lastWorkspaceSid } },
+        { id: true, sid: true },
+        100
+      );
+      if (!workspaces.length) {
+        break;
+      }
+
+      for (const workspace of workspaces) {
+        await this.models.workspace.update(
+          workspace.id,
+          { indexed: false },
+          false
+        );
+        await this.queue.add(
+          'indexer.indexWorkspace',
+          {
+            workspaceId: workspace.id,
+          },
+          {
+            jobId: `indexWorkspace/${workspace.id}`,
+            priority: 100,
+          }
+        );
+      }
+
+      lastWorkspaceSid = workspaces[workspaces.length - 1].sid;
+    }
+  }
+
   async write<T extends SearchTable>(
     table: T,
     documents: UpsertTypeByTable<T>[],