feat(server): auto index all workspaces to indexer (#12205)

close CLOUD-207

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Introduced automated periodic indexing of workspaces with a new job type and a scheduled cron job running every 30 seconds.
  - Added a unique sequential identifier (`sid`) and an "indexed" flag to workspaces to track indexing status.
- **Improvements**
  - Enhanced workspace indexing to handle missing workspaces and snapshots distinctly and selectively index documents.
  - Added ability to query workspaces after a given identifier with result limits.
- **Bug Fixes**
  - Improved error handling and logging during workspace indexing operations.
- **Tests**
  - Expanded test coverage for workspace indexing and auto-indexing, including scheduling and edge cases.
- **Chores**
  - Updated data models and schema to support new workspace fields and indexing features.
  - Enhanced mock data utilities to allow custom timestamps.
  - Improved type safety and flexibility in document snapshot retrieval.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/models/doc.ts

@@ -171,15 +171,20 @@ export class DocModel extends BaseModel {
     };
   }
 
-  async getSnapshot(workspaceId: string, docId: string) {
-    return await this.db.snapshot.findUnique({
+  async getSnapshot<Select extends Prisma.SnapshotSelect>(
+    workspaceId: string,
+    docId: string,
+    options?: { select?: Select }
+  ) {
+    return (await this.db.snapshot.findUnique({
       where: {
         workspaceId_id: {
           workspaceId,
           id: docId,
         },
       },
-    });
+      select: options?.select,
+    })) as Prisma.SnapshotGetPayload<{ select: Select }> | null;
   }
 
   async getAuthors(workspaceId: string, docId: string) {

packages/backend/server/src/models/workspace.ts

@@ -22,6 +22,7 @@ export type UpdateWorkspaceInput = Pick<
   | 'enableDocEmbedding'
   | 'name'
   | 'avatarKey'
+  | 'indexed'
 >;
 
 @Injectable()
@@ -54,7 +55,7 @@ export class WorkspaceModel extends BaseModel {
       },
       data,
     });
-    this.logger.log(
+    this.logger.debug(
       `Updated workspace ${workspaceId} with data ${JSON.stringify(data)}`
     );
     return workspace;
@@ -76,6 +77,18 @@ export class WorkspaceModel extends BaseModel {
     });
   }
 
+  async listAfterSid(sid: number, limit: number) {
+    return await this.db.workspace.findMany({
+      where: {
+        sid: { gt: sid },
+      },
+      take: limit,
+      orderBy: {
+        sid: 'asc',
+      },
+    });
+  }
+
   async delete(workspaceId: string) {
     const rawResult = await this.db.workspace.deleteMany({
       where: {