feat(core): migration for created by and updated by fields (#12171)

packages/common/graphql/src/graphql/get-doc-created-by-updated-by-list.gql

@@ -0,0 +1,18 @@
+query getDocCreatedByUpdatedByList($workspaceId: String!, $pagination: PaginationInput!) {
+  workspace(id: $workspaceId) {
+    docs(pagination: $pagination) {
+      totalCount
+      pageInfo {
+      endCursor
+        hasNextPage
+      }
+      edges {
+        node {
+          id
+          creatorId
+          lastUpdaterId
+        }
+      }
+    }
+  }
+}

packages/common/graphql/src/graphql/index.ts

@@ -981,6 +981,29 @@ export const getCurrentUserQuery = {
   deprecations: ["'token' is deprecated: use [/api/auth/sign-in?native=true] instead"],
 };
 
+export const getDocCreatedByUpdatedByListQuery = {
+  id: 'getDocCreatedByUpdatedByListQuery' as const,
+  op: 'getDocCreatedByUpdatedByList',
+  query: `query getDocCreatedByUpdatedByList($workspaceId: String!, $pagination: PaginationInput!) {
+  workspace(id: $workspaceId) {
+    docs(pagination: $pagination) {
+      totalCount
+      pageInfo {
+        endCursor
+        hasNextPage
+      }
+      edges {
+        node {
+          id
+          creatorId
+          lastUpdaterId
+        }
+      }
+    }
+  }
+}`,
+};
+
 export const getDocDefaultRoleQuery = {
   id: 'getDocDefaultRoleQuery' as const,
   op: 'getDocDefaultRole',

packages/common/graphql/src/schema.ts

@@ -3603,6 +3603,36 @@ export type GetCurrentUserQuery = {
   } | null;
 };
 
+export type GetDocCreatedByUpdatedByListQueryVariables = Exact<{
+  workspaceId: Scalars['String']['input'];
+  pagination: PaginationInput;
+}>;
+
+export type GetDocCreatedByUpdatedByListQuery = {
+  __typename?: 'Query';
+  workspace: {
+    __typename?: 'WorkspaceType';
+    docs: {
+      __typename?: 'PaginatedDocType';
+      totalCount: number;
+      pageInfo: {
+        __typename?: 'PageInfo';
+        endCursor: string | null;
+        hasNextPage: boolean;
+      };
+      edges: Array<{
+        __typename?: 'DocTypeEdge';
+        node: {
+          __typename?: 'DocType';
+          id: string;
+          creatorId: string | null;
+          lastUpdaterId: string | null;
+        };
+      }>;
+    };
+  };
+};
+
 export type GetDocDefaultRoleQueryVariables = Exact<{
   workspaceId: Scalars['String']['input'];
   docId: Scalars['String']['input'];
@@ -4736,6 +4766,11 @@ export type Queries =
       variables: GetCurrentUserQueryVariables;
       response: GetCurrentUserQuery;
     }
+  | {
+      name: 'getDocCreatedByUpdatedByListQuery';
+      variables: GetDocCreatedByUpdatedByListQueryVariables;
+      response: GetDocCreatedByUpdatedByListQuery;
+    }
   | {
       name: 'getDocDefaultRoleQuery';
       variables: GetDocDefaultRoleQueryVariables;

packages/frontend/core/src/desktop/pages/workspace/all-page-new/all-page.tsx

@@ -15,6 +15,7 @@ import {
 } from '../../../../modules/workbench';
 import { AllDocSidebarTabs } from '../layouts/all-doc-sidebar-tabs';
 import * as styles from './all-page.css';
+import { MigrationAllDocsDataNotification } from './migration-data';
 export const AllPage = () => {
   const t = useI18n();
 
@@ -57,6 +58,7 @@ export const AllPage = () => {
       <ViewHeader></ViewHeader>
       <ViewBody>
         <div className={styles.body}>
+          <MigrationAllDocsDataNotification />
           <div>
             <Filters
               filters={explorerPreference.filters ?? []}

packages/frontend/core/src/desktop/pages/workspace/all-page-new/migration-data.css.ts

@@ -0,0 +1,31 @@
+import { cssVar } from '@toeverything/theme';
+import { cssVarV2 } from '@toeverything/theme/v2';
+import { style } from '@vanilla-extract/css';
+
+export const migrationDataNotificationContainer = style({
+  border: `1px solid ${cssVarV2('layer/insideBorder/border')}`,
+  padding: '12px 240px 12px 12px',
+  borderRadius: '8px',
+});
+
+export const migrationDataNotificationTitle = style({
+  fontSize: cssVar('fontBase'),
+  fontWeight: '600',
+  lineHeight: '24px',
+  color: cssVarV2('text/primary'),
+  paddingBottom: '8px',
+});
+
+export const migrationDataNotificationContent = style({
+  fontSize: cssVar('fontSm'),
+  color: cssVarV2('text/secondary'),
+  lineHeight: '22px',
+  paddingBottom: '16px',
+});
+
+export const migrationDataNotificationError = style({
+  fontSize: cssVar('fontSm'),
+  color: cssVarV2('status/error'),
+  lineHeight: '22px',
+  paddingBottom: '16px',
+});

packages/frontend/core/src/desktop/pages/workspace/all-page-new/migration-data.tsx

@@ -0,0 +1,58 @@
+import { Button } from '@affine/component';
+import { DocCreatedByUpdatedBySyncService } from '@affine/core/modules/cloud';
+import { UserFriendlyError } from '@affine/error';
+import { useI18n } from '@affine/i18n';
+import { useLiveData, useService } from '@toeverything/infra';
+import { useCallback, useMemo } from 'react';
+
+import * as styles from './migration-data.css';
+
+export const MigrationAllDocsDataNotification = () => {
+  const t = useI18n();
+  const docCreatedByUpdatedBySyncService = useService(
+    DocCreatedByUpdatedBySyncService
+  );
+  const needSync = useLiveData(docCreatedByUpdatedBySyncService.needSync$);
+  const syncing = useLiveData(docCreatedByUpdatedBySyncService.syncing$);
+  const error = useLiveData(docCreatedByUpdatedBySyncService.error$);
+  const errorMessage = useMemo(() => {
+    if (error) {
+      const userFriendlyError = UserFriendlyError.fromAny(error);
+      return t[`error.${userFriendlyError.name}`](userFriendlyError.data);
+    }
+    return null;
+  }, [error, t]);
+  const progress = useLiveData(docCreatedByUpdatedBySyncService.progress$);
+
+  const handleSync = useCallback(() => {
+    docCreatedByUpdatedBySyncService.sync();
+  }, [docCreatedByUpdatedBySyncService]);
+
+  if (!needSync) {
+    return null;
+  }
+
+  return (
+    <div className={styles.migrationDataNotificationContainer}>
+      <div className={styles.migrationDataNotificationTitle}>
+        {t['com.affine.migration-all-docs-notification.title']()}
+      </div>
+      <div className={styles.migrationDataNotificationContent}>
+        {t['com.affine.migration-all-docs-notification.content']()}
+      </div>
+
+      {error && (
+        <div className={styles.migrationDataNotificationError}>
+          {t['com.affine.migration-all-docs-notification.error']({
+            errorMessage: errorMessage ?? '',
+          })}
+        </div>
+      )}
+
+      <Button loading={syncing} onClick={handleSync}>
+        {t['com.affine.migration-all-docs-notification.button']()}
+        {syncing ? ` (${Math.floor(progress * 100)}%)...` : ''}
+      </Button>
+    </div>
+  );
+};

packages/frontend/core/src/modules/cloud/index.ts

@@ -10,6 +10,7 @@ export { ServerScope } from './scopes/server';
 export { AuthService } from './services/auth';
 export { CaptchaService } from './services/captcha';
 export { DefaultServerService } from './services/default-server';
+export { DocCreatedByUpdatedBySyncService } from './services/doc-created-by-updated-by-sync';
 export { EventSourceService } from './services/eventsource';
 export { FetchService } from './services/fetch';
 export { GraphQLService } from './services/graphql';
@@ -98,6 +99,10 @@ import { UserQuotaStore } from './stores/user-quota';
 import { UserSettingsStore } from './stores/user-settings';
 import { DocCreatedByService } from './services/doc-created-by';
 import { DocUpdatedByService } from './services/doc-updated-by';
+import { DocCreatedByUpdatedBySyncService } from './services/doc-created-by-updated-by-sync';
+import { WorkspacePermissionService } from '../permissions';
+import { DocsService } from '../doc';
+import { DocCreatedByUpdatedBySyncStore } from './stores/doc-created-by-updated-by-sync';
 
 export function configureCloudModule(framework: Framework) {
   configureDefaultAuthProvider(framework);
@@ -181,5 +186,15 @@ export function configureCloudModule(framework: Framework) {
     .entity(WorkspaceInvoices, [WorkspaceService, WorkspaceServerService])
     .service(SelfhostLicenseService, [SelfhostLicenseStore, WorkspaceService])
     .store(SelfhostLicenseStore, [WorkspaceServerService])
-    .service(BlocksuiteWriterInfoService, [WorkspaceServerService]);
+    .service(BlocksuiteWriterInfoService, [WorkspaceServerService])
+    .service(DocCreatedByUpdatedBySyncService, [
+      WorkspaceService,
+      DocsService,
+      WorkspacePermissionService,
+      DocCreatedByUpdatedBySyncStore,
+    ])
+    .store(DocCreatedByUpdatedBySyncStore, [
+      WorkspaceServerService,
+      WorkspaceService,
+    ]);
 }

packages/frontend/core/src/modules/cloud/services/doc-created-by-updated-by-sync.ts

@@ -0,0 +1,150 @@
+import {
+  catchErrorInto,
+  effect,
+  fromPromise,
+  LiveData,
+  onStart,
+  Service,
+  throwIfAborted,
+} from '@toeverything/infra';
+import { clamp } from 'lodash-es';
+import { combineLatest, exhaustMap, finalize, map } from 'rxjs';
+
+import type { DocsService } from '../../doc';
+import type { WorkspacePermissionService } from '../../permissions';
+import type { WorkspaceService } from '../../workspace';
+import type { DocCreatedByUpdatedBySyncStore } from '../stores/doc-created-by-updated-by-sync';
+
+/**
+ * This service is used to sync createdBy and updatedBy data from the cloud to local doc properties.
+ *
+ * # When sync is needed
+ *
+ * 1. When the user is an owner or admin
+ * 2. When the root doc sync is complete
+ * 3. When a doc is missing createdBy data
+ * 4. When workspace has not been marked as `DocCreatedByUpdatedBySynced`
+ */
+export class DocCreatedByUpdatedBySyncService extends Service {
+  constructor(
+    private readonly workspaceService: WorkspaceService,
+    private readonly docsService: DocsService,
+    private readonly workspacePermissionService: WorkspacePermissionService,
+    private readonly docCreatedByUpdatedBySyncStore: DocCreatedByUpdatedBySyncStore
+  ) {
+    super();
+  }
+
+  syncing$ = new LiveData(false);
+  error$ = new LiveData<any>(null);
+  // sync progress 0.0 - 1.0
+  progress$ = new LiveData<number>(0);
+
+  sync = effect(
+    exhaustMap(() => {
+      return fromPromise(async (signal?: AbortSignal) => {
+        let afterCursor: string | null = null;
+        let finishedCount = 0;
+        while (true) {
+          const result =
+            await this.docCreatedByUpdatedBySyncStore.getDocCreatedByUpdatedByList(
+              afterCursor
+            );
+          throwIfAborted(signal);
+
+          for (const edge of result.workspace.docs.edges) {
+            const docId = edge.node.id;
+            const docRecord = this.docsService.list.doc$(docId).value;
+            if (docRecord) {
+              if (edge.node.creatorId) {
+                docRecord.setCreatedBy(edge.node.creatorId);
+              }
+              if (edge.node.lastUpdaterId) {
+                docRecord.setUpdatedBy(edge.node.lastUpdaterId);
+              }
+            }
+            finishedCount++;
+          }
+          this.progress$.value = clamp(
+            finishedCount / result.workspace.docs.totalCount,
+            0,
+            1
+          );
+          if (!result.workspace.docs.pageInfo.hasNextPage) {
+            break;
+          }
+          afterCursor = result.workspace.docs.pageInfo.endCursor;
+        }
+
+        this.docCreatedByUpdatedBySyncStore.setDocCreatedByUpdatedBySynced(
+          true
+        );
+      }).pipe(
+        catchErrorInto(this.error$),
+        onStart(() => {
+          this.syncing$.value = true;
+          this.progress$.value = 0;
+          this.error$.value = null;
+        }),
+        finalize(() => {
+          this.syncing$.value = false;
+        })
+      );
+    })
+  );
+
+  private readonly workspaceRootDocSynced$ =
+    this.workspaceService.workspace.engine.doc
+      .docState$(this.workspaceService.workspace.id)
+      .pipe(map(doc => doc.synced));
+
+  private readonly isOwnerOrAdmin$ =
+    this.workspacePermissionService.permission.isOwnerOrAdmin$;
+
+  private readonly missingCreatedBy$ = this.docsService
+    .propertyValues$('createdBy')
+    .pipe(
+      map(allDocsCreatedBy => {
+        let missingCreatedBy = false;
+        console.log(allDocsCreatedBy);
+        for (const createdBy of allDocsCreatedBy.values()) {
+          if (!createdBy) {
+            missingCreatedBy = true;
+            break;
+          }
+        }
+        return missingCreatedBy;
+      })
+    );
+
+  private readonly markedSynced$ =
+    this.docCreatedByUpdatedBySyncStore.watchDocCreatedByUpdatedBySynced();
+
+  needSync$ = LiveData.from(
+    combineLatest([
+      this.workspaceRootDocSynced$,
+      this.isOwnerOrAdmin$,
+      this.missingCreatedBy$,
+      this.markedSynced$,
+    ]).pipe(
+      map(
+        ([
+          workspaceRootDocSynced,
+          isOwnerOrAdmin,
+          missingCreatedBy,
+          markedSynced,
+        ]) =>
+          workspaceRootDocSynced &&
+          isOwnerOrAdmin &&
+          missingCreatedBy &&
+          !markedSynced
+      )
+    ),
+    false
+  );
+
+  override dispose(): void {
+    super.dispose();
+    this.sync.unsubscribe();
+  }
+}

packages/frontend/core/src/modules/cloud/stores/doc-created-by-updated-by-sync.ts

@@ -0,0 +1,47 @@
+import { getDocCreatedByUpdatedByListQuery } from '@affine/graphql';
+import { Store, yjsGetPath } from '@toeverything/infra';
+import type { Observable } from 'rxjs';
+
+import type { WorkspaceService } from '../../workspace';
+import type { WorkspaceServerService } from '../services/workspace-server';
+
+export class DocCreatedByUpdatedBySyncStore extends Store {
+  constructor(
+    private readonly workspaceServerService: WorkspaceServerService,
+    private readonly workspaceService: WorkspaceService
+  ) {
+    super();
+  }
+
+  async getDocCreatedByUpdatedByList(afterCursor?: string | null) {
+    if (!this.workspaceServerService.server) {
+      throw new Error('Server not found');
+    }
+
+    return await this.workspaceServerService.server.gql({
+      query: getDocCreatedByUpdatedByListQuery,
+      variables: {
+        workspaceId: this.workspaceService.workspace.id,
+        pagination: {
+          first: 100,
+          after: afterCursor,
+        },
+      },
+    });
+  }
+
+  watchDocCreatedByUpdatedBySynced() {
+    const rootYDoc = this.workspaceService.workspace.rootYDoc;
+    return yjsGetPath(
+      rootYDoc.getMap('affine:workspace-properties'),
+      'docCreatedByUpdatedBySynced'
+    ) as Observable<boolean>;
+  }
+
+  setDocCreatedByUpdatedBySynced(synced: boolean) {
+    const rootYDoc = this.workspaceService.workspace.rootYDoc;
+    rootYDoc
+      .getMap('affine:workspace-properties')
+      .set('docCreatedByUpdatedBySynced', synced);
+  }
+}

packages/frontend/core/src/modules/permissions/entities/permission.ts

@@ -20,6 +20,9 @@ export class WorkspacePermission extends Entity {
   );
   isOwner$ = this.cache$.map(cache => cache?.isOwner ?? null);
   isAdmin$ = this.cache$.map(cache => cache?.isAdmin ?? null);
+  isOwnerOrAdmin$ = this.cache$.map(
+    cache => (cache?.isOwner ?? null) || (cache?.isAdmin ?? null)
+  );
   isTeam$ = this.cache$.map(cache => cache?.isTeam ?? null);
   isRevalidating$ = new LiveData(false);
 

packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts

@@ -46,7 +46,13 @@ import {
 } from '@toeverything/infra';
 import { isEqual } from 'lodash-es';
 import { map, Observable, switchMap, tap } from 'rxjs';
-import { Doc as YDoc, encodeStateAsUpdate } from 'yjs';
+import {
+  applyUpdate,
+  type Array as YArray,
+  Doc as YDoc,
+  encodeStateAsUpdate,
+  type Map as YMap,
+} from 'yjs';
 
 import type { Server, ServersService } from '../../cloud';
 import {
@@ -209,6 +215,13 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
         });
       }
 
+      const accountId = this.authService.session.account$.value?.id;
+      await this.writeInitialDocProperties(
+        workspaceId,
+        docStorage,
+        accountId ?? ''
+      );
+
       docStorage.connection.disconnect();
       blobStorage.connection.disconnect();
 
@@ -533,6 +546,45 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
     };
   }
 
+  async writeInitialDocProperties(
+    workspaceId: string,
+    docStorage: DocStorage,
+    creatorId: string
+  ) {
+    try {
+      const rootDocBuffer = await docStorage.getDoc(workspaceId);
+      const rootDoc = new YDoc({ guid: workspaceId });
+      if (rootDocBuffer) {
+        applyUpdate(rootDoc, rootDocBuffer.bin);
+      }
+
+      const docIds = (
+        rootDoc.getMap('meta').get('pages') as YArray<YMap<string>>
+      )
+        ?.map(page => page.get('id'))
+        .filter(Boolean) as string[];
+
+      const propertiesDBBuffer = await docStorage.getDoc('db$docProperties');
+      const propertiesDB = new YDoc({ guid: 'db$docProperties' });
+      if (propertiesDBBuffer) {
+        applyUpdate(propertiesDB, propertiesDBBuffer.bin);
+      }
+
+      for (const docId of docIds) {
+        const docProperties = propertiesDB.getMap(docId);
+        docProperties.set('id', docId);
+        docProperties.set('createdBy', creatorId);
+      }
+
+      await docStorage.pushDocUpdate({
+        docId: 'db$docProperties',
+        bin: encodeStateAsUpdate(propertiesDB),
+      });
+    } catch (error) {
+      logger.error('error to write initial doc properties', error);
+    }
+  }
+
   private waitForLoaded() {
     return this.isRevalidating$.waitFor(loading => !loading);
   }

packages/frontend/i18n/src/i18n.gen.ts

@@ -7846,6 +7846,24 @@ export function useAFFiNEI18N(): {
       * `Stop`
       */
     ["com.affine.recording.stop"](): string;
+    /**
+      * `All docs data needs to be migrated in order to fully utilize its features.`
+      */
+    ["com.affine.migration-all-docs-notification.title"](): string;
+    /**
+      * `We upgraded the data format, requiring local data migration to utilize features. Created by and Updated by info is now stored locally. Without the update, you will see the actual status. Perform the upgrade under better network conditions.`
+      */
+    ["com.affine.migration-all-docs-notification.content"](): string;
+    /**
+      * `Migration failed: {{errorMessage}}`
+      */
+    ["com.affine.migration-all-docs-notification.error"](options: {
+        readonly errorMessage: string;
+    }): string;
+    /**
+      * `Migrate data`
+      */
+    ["com.affine.migration-all-docs-notification.button"](): string;
     /**
       * `An internal error occurred.`
       */

packages/frontend/i18n/src/resources/en.json

@@ -1965,6 +1965,10 @@
   "com.affine.recording.start": "Start",
   "com.affine.recording.dismiss": "Dismiss",
   "com.affine.recording.stop": "Stop",
+  "com.affine.migration-all-docs-notification.title": "All docs data needs to be migrated in order to fully utilize its features.",
+  "com.affine.migration-all-docs-notification.content": "We upgraded the data format, requiring local data migration to utilize features. Created by and Updated by info is now stored locally. Without the update, you will see the actual status. Perform the upgrade under better network conditions.",
+  "com.affine.migration-all-docs-notification.error": "Migration failed: {{errorMessage}}",
+  "com.affine.migration-all-docs-notification.button": "Migrate data",
   "error.INTERNAL_SERVER_ERROR": "An internal error occurred.",
   "error.NETWORK_ERROR": "Network error.",
   "error.TOO_MANY_REQUEST": "Too many requests.",