feat(server): increase embedding jobs concurrency & handle empty content after trim (#12574)

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **Improvements**
  - Increased the default concurrency for background tasks, enhancing processing efficiency.
  - Improved handling of empty or unsupported documents to ensure consistent processing.
  - Optimized document filtering to exclude certain documents from processing, improving performance.

- **Bug Fixes**
  - Enhanced detection of empty document summaries, reducing errors during processing.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/models/copilot-workspace.ts

@@ -42,23 +42,26 @@ export class CopilotWorkspaceConfigModel extends BaseModel {
    */
   @Transactional()
   async findDocsToEmbed(workspaceId: string): Promise<string[]> {
+    const ignoredDocIds = (await this.listIgnoredDocIds(workspaceId)).map(
+      d => d.docId
+    );
+
     const docIds = await this.db.snapshot
       .findMany({
         where: {
           workspaceId,
-          embedding: {
-            none: {},
-          },
+          AND: [
+            { id: { notIn: ignoredDocIds } },
+            { id: { not: workspaceId } },
+            { id: { not: { contains: '$' } } },
+          ],
+          embedding: { none: {} },
         },
         select: { id: true },
       })
       .then(r => r.map(doc => doc.id));
 
-    const skipDocIds = await this.listIgnoredDocIds(workspaceId).then(
-      r => new Set(r.map(r => r.docId))
-    );
-
-    return docIds.filter(id => !skipDocIds.has(id));
+    return docIds;
   }
 
   @Transactional()