feat(server): refactor mail queue (#15204)

packages/backend/server/src/__tests__/auth/auth.e2e.ts

@@ -110,7 +110,7 @@ test('set and change password', async t => {
   const u1 = await app.signupV1(u1Email);
   await sendSetPasswordEmail(app, u1Email, '/password-change');
 
-  const setPasswordMail = app.mails.last('ChangePassword');
+  const setPasswordMail = app.mails.last('SetPassword');
   const link = new URL(setPasswordMail.props.url);
   const setPasswordToken = link.searchParams.get('token');
 

packages/backend/server/src/__tests__/e2e/user/account.spec.ts

@@ -4,7 +4,9 @@ import {
   getCurrentUserQuery,
   getWorkspaceQuery,
 } from '@affine/graphql';
+import { WorkspaceMemberStatus } from '@prisma/client';
 
+import { WorkspaceRole } from '../../../models';
 import { app, e2e, Mockers } from '../test';
 
 const admin = await app.create(Mockers.User, {
@@ -107,6 +109,31 @@ e2e('should ban account', async t => {
   t.is(banUser.disabled, true);
 });
 
+e2e('should ban account with pending workspace invitation', async t => {
+  const owner = await app.create(Mockers.User);
+  const user = await app.create(Mockers.User);
+  const workspace = await app.create(Mockers.Workspace, { owner });
+
+  await app.models.workspaceInvitation.set(
+    workspace.id,
+    user.id,
+    WorkspaceRole.Collaborator,
+    WorkspaceMemberStatus.Pending,
+    { inviterId: owner.id }
+  );
+
+  await app.login(admin);
+
+  const { banUser } = await app.gql({
+    query: disableUserMutation,
+    variables: {
+      id: user.id,
+    },
+  });
+
+  t.is(banUser.disabled, true);
+});
+
 e2e('should not login banned account', async t => {
   const user = await app.create(Mockers.User);
 

packages/backend/server/src/__tests__/mocks/mailer.mock.ts

@@ -2,17 +2,17 @@ import { interval, map, take, takeUntil } from 'rxjs';
 import Sinon from 'sinon';
 
 import { Mailer } from '../../core/mail';
+import type { SendMailCommand } from '../../core/mail/types';
 import { MailName } from '../../mails';
 
 export class MockMailer {
   send = Sinon.createStubInstance(Mailer).send.resolves(true);
-  trySend(command: Jobs['notification.sendMail']) {
+  skip = Sinon.createStubInstance(Mailer).skip.resolves(false);
+  trySend(command: SendMailCommand) {
     return this.send(command, true);
   }
 
-  last<Mail extends MailName>(
-    name: Mail
-  ): Extract<Jobs['notification.sendMail'], { name: Mail }> {
+  last<Mail extends MailName>(name: Mail): SendMailCommand & { name: Mail } {
     const last = this.send.lastCall.args[0];
 
     if (!last) {
@@ -23,22 +23,26 @@ export class MockMailer {
       throw new Error(`Mail name mismatch: ${last.name} !== ${name}`);
     }
 
-    return last as any;
+    return last as SendMailCommand & { name: Mail };
   }
 
   waitFor<Mail extends MailName>(
     name: Mail,
     timeout: number = 1000
-  ): Promise<Extract<Jobs['notification.sendMail'], { name: Mail }>> {
-    const { promise, reject, resolve } = Promise.withResolvers<any>();
+  ): Promise<SendMailCommand & { name: Mail }> {
+    const { promise, reject, resolve } = Promise.withResolvers<
+      SendMailCommand & { name: Mail }
+    >();
 
     interval(10)
       .pipe(
         take(Math.floor(timeout / 10)),
         takeUntil(promise),
         map(() => {
-          const last = this.send.lastCall.args[0];
-          return last.name === name ? last : undefined;
+          const last = this.send.lastCall?.args[0];
+          return last?.name === name
+            ? (last as SendMailCommand & { name: Mail })
+            : undefined;
         })
       )
       .subscribe({

packages/backend/server/src/base/job/queue/config.ts

@@ -102,4 +102,12 @@ defineModuleConfig('job', {
     },
     schema,
   },
+
+  'queues.inviteAbuse': {
+    desc: 'The config for invite abuse disposition job queue',
+    default: {
+      concurrency: 1,
+    },
+    schema,
+  },
 });

packages/backend/server/src/base/job/queue/def.ts

@@ -30,6 +30,7 @@ export enum Queue {
   INDEXER = 'indexer',
   CALENDAR = 'calendar',
   BACKENDRUNTIME = 'backendRuntime',
+  INVITE_ABUSE = 'inviteAbuse',
 }
 
 export const QUEUES = Object.values(Queue);

packages/backend/server/src/core/auth/config.ts

@@ -12,6 +12,9 @@ export interface AuthConfig {
   requireEmailDomainVerification: boolean;
   requireEmailVerification: boolean;
   newAccountShareActionDelay: number;
+  trustedCloudflareHeaders: boolean;
+  inviteQuotaShadowMode: boolean;
+  inviteQuotaFailOpenOnRuntimeError: boolean;
   passwordRequirements: ConfigItem<{
     min: number;
     max: number;
@@ -46,6 +49,21 @@ defineModuleConfig('auth', {
     default: 24 * 60 * 60,
     shape: z.number().int().min(0),
   },
+  trustedCloudflareHeaders: {
+    desc: 'Whether request abuse source facts should trust Cloudflare headers from the origin edge.',
+    default: false,
+    shape: z.boolean(),
+  },
+  inviteQuotaShadowMode: {
+    desc: 'Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions.',
+    default: false,
+    shape: z.boolean(),
+  },
+  inviteQuotaFailOpenOnRuntimeError: {
+    desc: 'Whether workspace invite quota should fail open when native runtime admission is unavailable. Keep disabled for production.',
+    default: false,
+    shape: z.boolean(),
+  },
   passwordRequirements: {
     desc: 'The password strength requirements when set new password.',
     default: {

packages/backend/server/src/core/auth/controller.ts

@@ -15,6 +15,7 @@ import type { Request, Response } from 'express';
 
 import {
   ActionForbidden,
+  Config,
   EmailTokenNotFound,
   getRequestCookie,
   InvalidAuthState,
@@ -25,6 +26,7 @@ import {
 } from '../../base';
 import { Models } from '../../models';
 import { validators } from '../utils/validators';
+import { getAbuseRequestSource } from '../workspaces/abuse';
 import { Public } from './guard';
 import { MagicLinkAuthService } from './magic-link';
 import { AuthMethodsService } from './methods';
@@ -79,7 +81,8 @@ export class AuthController {
     private readonly openApp: OpenAppAuthService,
     private readonly authMethods: AuthMethodsService,
     private readonly sessionExchange: SessionExchangeService,
-    private readonly models: Models
+    private readonly models: Models,
+    private readonly config: Config
   ) {
     if (env.dev) {
       // set DNS servers in dev mode
@@ -135,6 +138,7 @@ export class AuthController {
       );
     } else {
       await this.sendMagicLink(
+        req,
         res,
         credential.email,
         credential.callbackUrl,
@@ -163,12 +167,15 @@ export class AuthController {
   }
 
   async sendMagicLink(
+    req: Request,
     res: Response,
     email: string,
     callbackUrl = '/magic-link',
     clientNonce?: string
   ) {
-    const payload = await this.magicLink.send(email, callbackUrl, clientNonce);
+    const payload = await this.magicLink.send(email, callbackUrl, clientNonce, {
+      source: getAbuseRequestSource(req, this.config),
+    });
     res.status(HttpStatus.OK).send(payload);
   }
 

packages/backend/server/src/core/auth/magic-link.ts

@@ -12,6 +12,7 @@ import {
   WrongSignInCredentials,
 } from '../../base';
 import { Models, TokenType } from '../../models';
+import type { MailDeliveryMetadata } from '../mail/types';
 import { validators } from '../utils/validators';
 import { verifyEmailDomainRecords } from './email-domain';
 import type { VerifiedIdentity } from './identity';
@@ -29,7 +30,12 @@ export class MagicLinkAuthService {
     private readonly crypto: CryptoHelper
   ) {}
 
-  async send(email: string, callbackUrl = '/magic-link', clientNonce?: string) {
+  async send(
+    email: string,
+    callbackUrl = '/magic-link',
+    clientNonce?: string,
+    metadata?: Pick<MailDeliveryMetadata, 'source'>
+  ) {
     validators.assertValidEmail(email);
 
     if (!this.url.isAllowedCallbackUrl(callbackUrl)) {
@@ -57,21 +63,33 @@ export class MagicLinkAuthService {
     }
 
     const ttlInSec = 30 * 60;
-    const token = await this.models.verificationToken.create(
-      TokenType.SignIn,
-      email,
-      ttlInSec
-    );
+    const { token, expiresAt: tokenExpiresAt } =
+      await this.models.verificationToken.createWithExpiresAt(
+        TokenType.SignIn,
+        email,
+        ttlInSec
+      );
 
     const otp = this.crypto.otp();
-    await this.models.magicLinkOtp.upsert(email, otp, token, clientNonce);
+    const { expiresAt: otpExpiresAt } = await this.models.magicLinkOtp.upsert(
+      email,
+      otp,
+      token,
+      clientNonce
+    );
 
     const magicLink = this.url.link(callbackUrl, { token: otp, email });
     if (env.dev) {
       this.logger.debug(`Magic link: ${magicLink}`);
     }
 
-    await this.auth.sendSignInEmail(email, magicLink, otp, !user);
+    await this.auth.sendSignInEmail(email, magicLink, otp, !user, {
+      ...metadata,
+      expiresAt:
+        tokenExpiresAt.getTime() < otpExpiresAt.getTime()
+          ? tokenExpiresAt
+          : otpExpiresAt,
+    });
 
     return { email };
   }

packages/backend/server/src/core/auth/resolver.ts

@@ -1,5 +1,6 @@
 import {
   Args,
+  Context,
   Field,
   Mutation,
   ObjectType,
@@ -11,6 +12,7 @@ import {
 
 import {
   ActionForbidden,
+  Config,
   EmailAlreadyUsed,
   EmailTokenNotFound,
   EmailVerificationRequired,
@@ -21,10 +23,13 @@ import {
   Throttle,
   URLHelper,
 } from '../../base';
+import type { GraphqlContext } from '../../base/graphql';
 import { Models, TokenType } from '../../models';
 import { Admin } from '../common';
+import type { MailDeliveryMetadata } from '../mail/types';
 import { UserType } from '../user/types';
 import { validators } from '../utils/validators';
+import { getAbuseRequestSource } from '../workspaces/abuse';
 import { Public } from './guard';
 import { AuthService } from './service';
 import { CurrentUser } from './session';
@@ -47,9 +52,20 @@ export class AuthResolver {
   constructor(
     private readonly url: URLHelper,
     private readonly auth: AuthService,
-    private readonly models: Models
+    private readonly models: Models,
+    private readonly config: Config
   ) {}
 
+  private mailMetadata(
+    context: GraphqlContext,
+    expiresAt?: Date
+  ): MailDeliveryMetadata {
+    return {
+      source: getAbuseRequestSource(context.req, this.config),
+      expiresAt,
+    };
+  }
+
   @SkipThrottle()
   @Public()
   @Query(() => UserType, {
@@ -145,23 +161,30 @@ export class AuthResolver {
     @CurrentUser() user: CurrentUser,
     @Args('callbackUrl') callbackUrl: string,
     @Args('email', {
+      type: () => String,
       nullable: true,
       deprecationReason: 'fetched from signed in user',
     })
-    _email?: string
+    _email: string | undefined,
+    @Context() context: GraphqlContext
   ) {
     if (!user.emailVerified) {
       throw new EmailVerificationRequired();
     }
 
-    const token = await this.models.verificationToken.create(
-      TokenType.ChangePassword,
-      user.id
-    );
+    const { token, expiresAt } =
+      await this.models.verificationToken.createWithExpiresAt(
+        TokenType.ChangePassword,
+        user.id
+      );
 
     const url = this.url.safeLink(callbackUrl, { userId: user.id, token });
 
-    return await this.auth.sendChangePasswordEmail(user.email, url);
+    return await this.auth.sendChangePasswordEmail(
+      user.email,
+      url,
+      this.mailMetadata(context, expiresAt)
+    );
   }
 
   @Mutation(() => Boolean)
@@ -169,12 +192,26 @@ export class AuthResolver {
     @CurrentUser() user: CurrentUser,
     @Args('callbackUrl') callbackUrl: string,
     @Args('email', {
+      type: () => String,
       nullable: true,
       deprecationReason: 'fetched from signed in user',
     })
-    _email?: string
+    _email: string | undefined,
+    @Context() context: GraphqlContext
   ) {
-    return this.sendChangePasswordEmail(user, callbackUrl);
+    const { token, expiresAt } =
+      await this.models.verificationToken.createWithExpiresAt(
+        TokenType.ChangePassword,
+        user.id
+      );
+
+    const url = this.url.safeLink(callbackUrl, { userId: user.id, token });
+
+    return await this.auth.sendSetPasswordEmail(
+      user.email,
+      url,
+      this.mailMetadata(context, expiresAt)
+    );
   }
 
   // The change email step is:
@@ -187,20 +224,26 @@ export class AuthResolver {
   @Mutation(() => Boolean)
   async sendChangeEmail(
     @CurrentUser() user: CurrentUser,
-    @Args('callbackUrl') callbackUrl: string
+    @Args('callbackUrl') callbackUrl: string,
+    @Context() context: GraphqlContext
   ) {
     if (!user.emailVerified) {
       throw new EmailVerificationRequired();
     }
 
-    const token = await this.models.verificationToken.create(
-      TokenType.ChangeEmail,
-      user.id
-    );
+    const { token, expiresAt } =
+      await this.models.verificationToken.createWithExpiresAt(
+        TokenType.ChangeEmail,
+        user.id
+      );
 
     const url = this.url.safeLink(callbackUrl, { token });
 
-    return await this.auth.sendChangeEmail(user.email, url);
+    return await this.auth.sendChangeEmail(
+      user.email,
+      url,
+      this.mailMetadata(context, expiresAt)
+    );
   }
 
   @Mutation(() => Boolean)
@@ -208,7 +251,8 @@ export class AuthResolver {
     @CurrentUser() user: CurrentUser,
     @Args('token') token: string,
     @Args('email') email: string,
-    @Args('callbackUrl') callbackUrl: string
+    @Args('callbackUrl') callbackUrl: string,
+    @Context() context: GraphqlContext
   ) {
     if (!token) {
       throw new EmailTokenNotFound();
@@ -237,31 +281,42 @@ export class AuthResolver {
       }
     }
 
-    const verifyEmailToken = await this.models.verificationToken.create(
-      TokenType.VerifyEmail,
-      user.id
-    );
+    const { token: verifyEmailToken, expiresAt } =
+      await this.models.verificationToken.createWithExpiresAt(
+        TokenType.VerifyEmail,
+        user.id
+      );
 
     const url = this.url.safeLink(callbackUrl, {
       token: verifyEmailToken,
       email,
     });
-    return await this.auth.sendVerifyChangeEmail(email, url);
+    return await this.auth.sendVerifyChangeEmail(
+      email,
+      url,
+      this.mailMetadata(context, expiresAt)
+    );
   }
 
   @Mutation(() => Boolean)
   async sendVerifyEmail(
     @CurrentUser() user: CurrentUser,
-    @Args('callbackUrl') callbackUrl: string
+    @Args('callbackUrl') callbackUrl: string,
+    @Context() context: GraphqlContext
   ) {
-    const token = await this.models.verificationToken.create(
-      TokenType.VerifyEmail,
-      user.id
-    );
+    const { token, expiresAt } =
+      await this.models.verificationToken.createWithExpiresAt(
+        TokenType.VerifyEmail,
+        user.id
+      );
 
     const url = this.url.safeLink(callbackUrl, { token });
 
-    return await this.auth.sendVerifyEmail(user.email, url);
+    return await this.auth.sendVerifyEmail(
+      user.email,
+      url,
+      this.mailMetadata(context, expiresAt)
+    );
   }
 
   @Mutation(() => Boolean)

packages/backend/server/src/core/auth/service.ts

@@ -7,6 +7,7 @@ import { assign, pick } from 'lodash-es';
 import { Config, SignUpForbidden } from '../../base';
 import { Models, type User, type UserSession } from '../../models';
 import { Mailer } from '../mail/mailer';
+import type { MailDeliveryMetadata } from '../mail/types';
 import { createDevUsers } from './dev';
 import type { VerifiedIdentity } from './identity';
 import {
@@ -312,49 +313,74 @@ export class AuthService implements OnApplicationBootstrap {
     });
   }
 
-  async sendChangePasswordEmail(email: string, callbackUrl: string) {
+  async sendChangePasswordEmail(
+    email: string,
+    callbackUrl: string,
+    metadata?: MailDeliveryMetadata
+  ) {
     return await this.mailer.send({
       name: 'ChangePassword',
       to: email,
       props: {
         url: callbackUrl,
       },
+      metadata,
     });
   }
-  async sendSetPasswordEmail(email: string, callbackUrl: string) {
+  async sendSetPasswordEmail(
+    email: string,
+    callbackUrl: string,
+    metadata?: MailDeliveryMetadata
+  ) {
     return await this.mailer.send({
       name: 'SetPassword',
       to: email,
       props: {
         url: callbackUrl,
       },
+      metadata,
     });
   }
-  async sendChangeEmail(email: string, callbackUrl: string) {
+  async sendChangeEmail(
+    email: string,
+    callbackUrl: string,
+    metadata?: MailDeliveryMetadata
+  ) {
     return await this.mailer.send({
       name: 'ChangeEmail',
       to: email,
       props: {
         url: callbackUrl,
       },
+      metadata,
     });
   }
-  async sendVerifyChangeEmail(email: string, callbackUrl: string) {
+  async sendVerifyChangeEmail(
+    email: string,
+    callbackUrl: string,
+    metadata?: MailDeliveryMetadata
+  ) {
     return await this.mailer.send({
       name: 'VerifyChangeEmail',
       to: email,
       props: {
         url: callbackUrl,
       },
+      metadata,
     });
   }
-  async sendVerifyEmail(email: string, callbackUrl: string) {
+  async sendVerifyEmail(
+    email: string,
+    callbackUrl: string,
+    metadata?: MailDeliveryMetadata
+  ) {
     return await this.mailer.send({
       name: 'VerifyEmail',
       to: email,
       props: {
         url: callbackUrl,
       },
+      metadata,
     });
   }
   async sendNotificationChangeEmail(email: string) {
@@ -371,7 +397,8 @@ export class AuthService implements OnApplicationBootstrap {
     email: string,
     link: string,
     otp: string,
-    signUp: boolean
+    signUp: boolean,
+    metadata?: MailDeliveryMetadata
   ) {
     return await this.mailer.send({
       name: signUp ? 'SignUp' : 'SignIn',
@@ -381,6 +408,7 @@ export class AuthService implements OnApplicationBootstrap {
         otp,
         serverName: this.getServerName(),
       },
+      metadata,
     });
   }
 }

packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts

@@ -15,6 +15,7 @@ interface Context {
   runtime: {
     cleanupExpiredRuntimeStates: Sinon.SinonStub;
     cleanupExpiredRuntimeGates: Sinon.SinonStub;
+    cleanupExpiredRollingQuota: Sinon.SinonStub;
   };
 }
 
@@ -24,6 +25,7 @@ test.before(async t => {
   t.context.runtime = {
     cleanupExpiredRuntimeStates: Sinon.stub(),
     cleanupExpiredRuntimeGates: Sinon.stub(),
+    cleanupExpiredRollingQuota: Sinon.stub(),
   };
   t.context.module = await createTestingModule({
     imports: [ScheduleModule.forRoot(), BackendRuntimeModule],
@@ -39,6 +41,7 @@ test.before(async t => {
 test.beforeEach(t => {
   t.context.runtime.cleanupExpiredRuntimeStates.reset();
   t.context.runtime.cleanupExpiredRuntimeGates.reset();
+  t.context.runtime.cleanupExpiredRollingQuota.reset();
 });
 
 test.after.always(async t => {
@@ -49,9 +52,11 @@ test('backend-runtime housekeeping cleans runtime state and gate batches', async
   t.context.runtime.cleanupExpiredRuntimeStates.onCall(0).resolves(1000);
   t.context.runtime.cleanupExpiredRuntimeStates.onCall(1).resolves(2);
   t.context.runtime.cleanupExpiredRuntimeGates.resolves(1);
+  t.context.runtime.cleanupExpiredRollingQuota.resolves(1);
 
   await t.context.job.cleanExpiredRuntimeHousekeeping();
 
   t.is(t.context.runtime.cleanupExpiredRuntimeStates.callCount, 2);
   t.is(t.context.runtime.cleanupExpiredRuntimeGates.callCount, 1);
+  t.is(t.context.runtime.cleanupExpiredRollingQuota.callCount, 1);
 });

packages/backend/server/src/core/backend-runtime/index.ts

@@ -10,4 +10,15 @@ import { BackendRuntimeProvider } from './provider';
 })
 export class BackendRuntimeModule {}
 
-export { BackendRuntimeProvider } from './provider';
+export {
+  BackendRuntimeProvider,
+  type RuntimeInviteAbuseAction,
+  type RuntimeInviteAbuseClaimedAction,
+  type RuntimeMailDeliveryQuotaDecision,
+  type RuntimeMailDeliveryQuotaInput,
+  type RuntimeQuotaSourceInput,
+  type RuntimeQuotaTargetDomainInput,
+  type RuntimeWorkspaceInviteQuotaDecision,
+  type RuntimeWorkspaceInviteQuotaInput,
+  type RuntimeWorkspaceInviteQuotaUsage,
+} from './provider';

packages/backend/server/src/core/backend-runtime/job.ts

@@ -38,9 +38,12 @@ export class BackendRuntimeHousekeepingJob {
     const gates = await this.cleanBatches(() =>
       this.rt.cleanupExpiredRuntimeGates(1000)
     );
+    const rollingQuota = await this.cleanBatches(() =>
+      this.rt.cleanupExpiredRollingQuota(1000)
+    );
 
     this.logger.log(
-      `cleaned runtime housekeeping states=${states} gates=${gates}`
+      `cleaned runtime housekeeping states=${states} gates=${gates} rollingQuota=${rollingQuota}`
     );
   }
 

packages/backend/server/src/core/backend-runtime/provider.ts

@@ -10,6 +10,187 @@ import { BackendRuntime, type BackendRuntimeHealth } from '../../native';
 
 type RuntimeInstance = InstanceType<typeof BackendRuntime>;
 
+export type RuntimeQuotaTargetDomainInput = {
+  domain: string;
+  count: number;
+};
+
+export type RuntimeQuotaSourceInput = {
+  trusted: boolean;
+  ip?: string;
+  country?: string;
+  asn?: number;
+  rayId?: string;
+};
+
+export type RuntimeWorkspaceInviteQuotaInput = {
+  actorUserId: string;
+  workspaceId: string;
+  requestId?: string;
+  targetCount: number;
+  targetDomains: RuntimeQuotaTargetDomainInput[];
+  source?: RuntimeQuotaSourceInput;
+};
+
+export type RuntimeWorkspaceInviteQuotaUsage = {
+  targetCount: number;
+  targetDomains: RuntimeQuotaTargetDomainInput[];
+};
+
+export type RuntimeInviteAbuseAction =
+  | 'ban_actor'
+  | 'quarantine_actor'
+  | 'quarantine_workspace'
+  | 'quarantine_source_cohort';
+
+const RUNTIME_INVITE_ABUSE_ACTIONS = new Set<RuntimeInviteAbuseAction>([
+  'ban_actor',
+  'quarantine_actor',
+  'quarantine_workspace',
+  'quarantine_source_cohort',
+]);
+
+export type RuntimeInviteAbuseClaimedAction = {
+  action: RuntimeInviteAbuseAction;
+  subjectKey: string;
+  evidenceId: string;
+  actionId: string;
+  actorUserId: string;
+  workspaceId: string;
+};
+
+type NativeRuntimeInviteAbuseClaimedAction = Omit<
+  RuntimeInviteAbuseClaimedAction,
+  'action'
+> & {
+  action: string;
+};
+
+export type RuntimeWorkspaceInviteQuotaDecision = {
+  allowed: boolean;
+  reservationId?: string;
+  retryAfterSeconds?: number;
+  reason?: string;
+  scopeKey?: string;
+  windowSeconds?: number;
+  limit?: number;
+  current?: number;
+  requested?: number;
+  actionRequired?: {
+    action: RuntimeInviteAbuseAction;
+    subjectKey: string;
+    evidenceId: string;
+    actionId: string;
+  };
+};
+
+type NativeRuntimeInviteAbuseActionRequired = Omit<
+  NonNullable<RuntimeWorkspaceInviteQuotaDecision['actionRequired']>,
+  'action'
+> & {
+  action: string;
+};
+
+type NativeRuntimeWorkspaceInviteQuotaDecision = Omit<
+  RuntimeWorkspaceInviteQuotaDecision,
+  'actionRequired'
+> & {
+  actionRequired?: NativeRuntimeInviteAbuseActionRequired;
+};
+
+export type RuntimeMailDeliveryQuotaInput = {
+  requestId?: string;
+  mailName: string;
+  recipient: {
+    email: string;
+    domain: string;
+    userId?: string;
+  };
+  metadata: {
+    actorUserId?: string;
+    workspaceId?: string;
+    notificationId?: string;
+    abuseSubjectKey?: string;
+  };
+  source?: RuntimeQuotaSourceInput;
+};
+
+export type RuntimeMailDeliveryQuotaDecision = {
+  allowed: boolean;
+  reservationId?: string;
+  mailClass: string;
+  retryAfterSeconds?: number;
+  reason?: string;
+  scopeKey?: string;
+  windowSeconds?: number;
+  limit?: number;
+  current?: number;
+  requested?: number;
+};
+
+type RuntimeQuotaMethods = RuntimeInstance & {
+  assertWorkspaceInviteQuotaV1(
+    input: RuntimeWorkspaceInviteQuotaInput
+  ): Promise<NativeRuntimeWorkspaceInviteQuotaDecision>;
+  commitWorkspaceInviteQuotaV1(
+    reservationId: string,
+    usage: RuntimeWorkspaceInviteQuotaUsage
+  ): Promise<boolean>;
+  releaseWorkspaceInviteQuotaV1(reservationId: string): Promise<boolean>;
+  assertMailDeliveryQuotaV1(
+    input: RuntimeMailDeliveryQuotaInput
+  ): Promise<RuntimeMailDeliveryQuotaDecision>;
+  commitMailDeliveryQuotaV1(reservationId: string): Promise<boolean>;
+  releaseMailDeliveryQuotaV1(reservationId: string): Promise<boolean>;
+  cleanupExpiredRollingQuota(limit: number): Promise<number>;
+  isInviteAbuseUserQuarantinedOrBanned(userId: string): Promise<boolean>;
+  isInviteAbuseWorkspaceQuarantined(workspaceId: string): Promise<boolean>;
+  claimInviteAbuseAction(actionId: string, workerId: string): Promise<boolean>;
+  claimRetryableInviteAbuseActions(
+    workerId: string,
+    limit: number
+  ): Promise<NativeRuntimeInviteAbuseClaimedAction[]>;
+  markInviteAbuseAction(
+    actionId: string,
+    workerId: string,
+    status: 'succeeded' | 'failed',
+    error?: string | null
+  ): Promise<boolean>;
+};
+
+function normalizeInviteAbuseAction(action: string): RuntimeInviteAbuseAction {
+  if (RUNTIME_INVITE_ABUSE_ACTIONS.has(action as RuntimeInviteAbuseAction)) {
+    return action as RuntimeInviteAbuseAction;
+  }
+  throw new Error(`Unknown invite abuse action: ${action}`);
+}
+
+function normalizeWorkspaceInviteQuotaDecision(
+  decision: NativeRuntimeWorkspaceInviteQuotaDecision
+): RuntimeWorkspaceInviteQuotaDecision {
+  const { actionRequired, ...rest } = decision;
+  if (!actionRequired) {
+    return rest;
+  }
+
+  return {
+    ...rest,
+    actionRequired: {
+      ...actionRequired,
+      action: normalizeInviteAbuseAction(actionRequired.action),
+    },
+  };
+}
+
+function normalizeClaimedInviteAbuseAction(
+  action: NativeRuntimeInviteAbuseClaimedAction
+): RuntimeInviteAbuseClaimedAction {
+  return {
+    ...action,
+    action: normalizeInviteAbuseAction(action.action),
+  };
+}
+
 @Injectable()
 export class BackendRuntimeProvider
   implements OnApplicationBootstrap, OnApplicationShutdown
@@ -66,6 +247,102 @@ export class BackendRuntimeProvider
     );
   }
 
+  async assertWorkspaceInviteQuotaV1(
+    input: RuntimeWorkspaceInviteQuotaInput
+  ): Promise<RuntimeWorkspaceInviteQuotaDecision> {
+    return normalizeWorkspaceInviteQuotaDecision(
+      await this.measured('assertWorkspaceInviteQuotaV1', rt =>
+        this.quotaRuntime(rt).assertWorkspaceInviteQuotaV1(input)
+      )
+    );
+  }
+
+  async commitWorkspaceInviteQuotaV1(
+    reservationId: string,
+    usage: RuntimeWorkspaceInviteQuotaUsage
+  ): Promise<boolean> {
+    return await this.measured('commitWorkspaceInviteQuotaV1', rt =>
+      this.quotaRuntime(rt).commitWorkspaceInviteQuotaV1(reservationId, usage)
+    );
+  }
+
+  async releaseWorkspaceInviteQuotaV1(reservationId: string): Promise<boolean> {
+    return await this.measured('releaseWorkspaceInviteQuotaV1', rt =>
+      this.quotaRuntime(rt).releaseWorkspaceInviteQuotaV1(reservationId)
+    );
+  }
+
+  async assertMailDeliveryQuotaV1(
+    input: RuntimeMailDeliveryQuotaInput
+  ): Promise<RuntimeMailDeliveryQuotaDecision> {
+    return await this.measured('assertMailDeliveryQuotaV1', rt =>
+      this.quotaRuntime(rt).assertMailDeliveryQuotaV1(input)
+    );
+  }
+
+  async commitMailDeliveryQuotaV1(reservationId: string): Promise<boolean> {
+    return await this.measured('commitMailDeliveryQuotaV1', rt =>
+      this.quotaRuntime(rt).commitMailDeliveryQuotaV1(reservationId)
+    );
+  }
+
+  async releaseMailDeliveryQuotaV1(reservationId: string): Promise<boolean> {
+    return await this.measured('releaseMailDeliveryQuotaV1', rt =>
+      this.quotaRuntime(rt).releaseMailDeliveryQuotaV1(reservationId)
+    );
+  }
+
+  async cleanupExpiredRollingQuota(limit: number) {
+    return await this.measured('cleanupExpiredRollingQuota', rt =>
+      this.quotaRuntime(rt).cleanupExpiredRollingQuota(limit)
+    );
+  }
+
+  async isInviteAbuseUserQuarantinedOrBanned(userId: string) {
+    return await this.measured('isInviteAbuseUserQuarantinedOrBanned', rt =>
+      this.quotaRuntime(rt).isInviteAbuseUserQuarantinedOrBanned(userId)
+    );
+  }
+
+  async isInviteAbuseWorkspaceQuarantined(workspaceId: string) {
+    return await this.measured('isInviteAbuseWorkspaceQuarantined', rt =>
+      this.quotaRuntime(rt).isInviteAbuseWorkspaceQuarantined(workspaceId)
+    );
+  }
+
+  async claimInviteAbuseAction(actionId: string, workerId: string) {
+    return await this.measured('claimInviteAbuseAction', rt =>
+      this.quotaRuntime(rt).claimInviteAbuseAction(actionId, workerId)
+    );
+  }
+
+  async claimRetryableInviteAbuseActions(
+    workerId: string,
+    limit: number
+  ): Promise<RuntimeInviteAbuseClaimedAction[]> {
+    return (
+      await this.measured('claimRetryableInviteAbuseActions', rt =>
+        this.quotaRuntime(rt).claimRetryableInviteAbuseActions(workerId, limit)
+      )
+    ).map(normalizeClaimedInviteAbuseAction);
+  }
+
+  async markInviteAbuseAction(
+    actionId: string,
+    workerId: string,
+    status: 'succeeded' | 'failed',
+    error?: string | null
+  ) {
+    return await this.measured('markInviteAbuseAction', rt =>
+      this.quotaRuntime(rt).markInviteAbuseAction(
+        actionId,
+        workerId,
+        status,
+        error
+      )
+    );
+  }
+
   private async measured<T>(
     method: string,
     fn: (runtime: RuntimeInstance) => Promise<T>
@@ -78,6 +355,10 @@ export class BackendRuntimeProvider
     )();
   }
 
+  private quotaRuntime(runtime: RuntimeInstance): RuntimeQuotaMethods {
+    return runtime as unknown as RuntimeQuotaMethods;
+  }
+
   private async runMigrationsOnce() {
     if (this.migrationsStarted) {
       return;

packages/backend/server/src/core/content-policy.ts

@@ -0,0 +1,14 @@
+import { scanContentPolicyV1 } from '../native';
+
+export function scanContentPolicy(value: string | null | undefined) {
+  return scanContentPolicyV1({
+    value: value ?? '',
+    checks: ['url_or_domain'],
+  });
+}
+
+export function containsUrlOrDomain(value: string | null | undefined) {
+  return scanContentPolicy(value).matches.some(
+    match => match.type === 'url_or_domain'
+  );
+}

packages/backend/server/src/core/mail/__tests__/job.spec.ts

@@ -1,62 +1,70 @@
+import { Prisma, PrismaClient } from '@prisma/client';
 import test from 'ava';
 import Sinon from 'sinon';
 
 import { Mockers } from '../../../__tests__/mocks';
 import { createTestingModule } from '../../../__tests__/utils';
-import { Cache } from '../../../base';
 import { Models } from '../../../models';
 import { MailJob } from '../job';
 import { MailSender } from '../sender';
 
 let module: Awaited<ReturnType<typeof createTestingModule>>;
-let cache: Cache;
 let mailJob: MailJob;
 let sender: MailSender;
 let models: Models;
+let db: PrismaClient;
 
 test.before(async () => {
   module = await createTestingModule();
-  cache = module.get(Cache);
   mailJob = module.get(MailJob);
   sender = module.get(MailSender);
   models = module.get(Models);
+  db = module.get(PrismaClient);
 });
 
 test.after.always(async () => {
   await module.close();
 });
 
-test.afterEach(() => {
+test.afterEach.always(async () => {
   Sinon.restore();
+  await db.mailDelivery.deleteMany();
 });
 
-test('should clear pending mail records when user is deleted', async t => {
+async function createDelivery(
+  input: {
+    name: 'SignIn' | 'VerifyEmail' | 'MemberInvitation';
+    to: string;
+    props: Record<string, unknown>;
+  },
+  overrides: Partial<Parameters<Models['mailDelivery']['create']>[0]> = {}
+) {
+  const mailClass =
+    input.name === 'MemberInvitation' ? 'workspace_invitation' : 'auth';
+  return await models.mailDelivery.create({
+    mailName: input.name,
+    mailClass,
+    priority: mailClass === 'auth' ? 'critical' : 'normal',
+    recipientEmail: input.to,
+    payload: input as Prisma.JsonObject,
+    ...overrides,
+  });
+}
+
+async function delivery(id: string) {
+  return await db.mailDelivery.findUniqueOrThrow({ where: { id } });
+}
+
+test('should cancel pending mail deliveries when user is deleted', async t => {
   const user = await module.create(Mockers.User);
   const another = await module.create(Mockers.User);
-  const sendMailKey = 'mailjob:sendMail';
-  const retryMailKey = 'mailjob:sendMail:retry';
-  const userKey = `${sendMailKey}:SignIn:${user.email}`;
-  const userRetryKey = `${sendMailKey}:VerifyEmail:${user.email}`;
-  const anotherKey = `${sendMailKey}:SignIn:${another.email}`;
-  const senderRetryKey = `${retryMailKey}:MemberInvitation:invited@affine.pro`;
-
-  await cache.mapSet(sendMailKey, userKey, 1);
-  await cache.mapSet(sendMailKey, anotherKey, 1);
-  await cache.mapSet(
-    retryMailKey,
-    userRetryKey,
-    JSON.stringify({
-      startTime: Date.now(),
-      name: 'VerifyEmail',
-      to: user.email,
-      props: { url: 'https://affine.pro/verify' },
-    })
-  );
-  await cache.mapSet(
-    retryMailKey,
-    senderRetryKey,
-    JSON.stringify({
-      startTime: Date.now(),
+  const recipientDelivery = await createDelivery({
+    name: 'SignIn',
+    to: user.email,
+    props: { url: 'https://affine.pro/sign-in', otp: '123456' },
+  });
+  const senderDelivery = await createDelivery(
+    {
       name: 'MemberInvitation',
       to: 'invited@affine.pro',
       props: {
@@ -64,153 +72,126 @@ test('should clear pending mail records when user is deleted', async t => {
         workspace: { $$workspaceId: 'workspace-id' },
         url: 'https://affine.pro/invite',
       },
-    })
+    },
+    { actorUserId: user.id }
   );
+  const anotherDelivery = await createDelivery({
+    name: 'SignIn',
+    to: another.email,
+    props: { url: 'https://affine.pro/sign-in', otp: '123456' },
+  });
 
   await mailJob.onUserDeleted({ ...user, ownedWorkspaces: [] });
 
-  t.true(module.queue.removeWhere.calledOnce);
-  t.is(module.queue.removeWhere.firstCall.args[0], 'notification.sendMail');
-  const shouldRemove = module.queue.removeWhere.firstCall.args[1];
-  t.true(
-    await shouldRemove({
-      to: user.email,
-    } as Jobs['notification.sendMail'])
-  );
-  t.true(
-    await shouldRemove({
-      name: 'MemberInvitation',
-      to: 'invited@affine.pro',
-      props: {
-        user: { $$userId: user.id },
-        workspace: { $$workspaceId: 'workspace-id' },
-        url: 'https://affine.pro/invite',
-      },
-    } as Jobs['notification.sendMail'])
-  );
-  t.false(
-    await shouldRemove({
-      to: another.email,
-    } as Jobs['notification.sendMail'])
-  );
-  t.is(await cache.mapGet(sendMailKey, userKey), undefined);
-  t.is(await cache.mapGet(retryMailKey, userRetryKey), undefined);
-  t.is(await cache.mapGet(retryMailKey, senderRetryKey), undefined);
-  t.is(await cache.mapGet(sendMailKey, anotherKey), 1);
+  t.is((await delivery(recipientDelivery.id)).status, 'canceled');
+  t.is((await delivery(senderDelivery.id)).status, 'canceled');
+  t.is((await delivery(anotherDelivery.id)).status, 'queued');
+  t.is((await delivery(recipientDelivery.id)).recipientEmail, null);
+  t.is((await delivery(senderDelivery.id)).payload, null);
 });
 
 test('should skip queued mail for disabled recipient', async t => {
   const user = await module.create(Mockers.User, { disabled: true });
-  const send = Sinon.stub(sender, 'send').resolves(true);
-
-  await mailJob.sendMail({
-    startTime: Date.now(),
+  const send = Sinon.stub(sender, 'send').resolves({
+    status: 'accepted',
+    retryable: false,
+  });
+  const row = await createDelivery({
     name: 'SignIn',
     to: user.email,
-    props: {
-      url: 'https://affine.pro/sign-in',
-      otp: '123456',
-    },
+    props: { url: 'https://affine.pro/sign-in', otp: '123456' },
   });
 
+  await mailJob.processReadyDeliveries();
+
+  const updated = await delivery(row.id);
   t.false(send.called);
-  t.truthy(await models.user.get(user.id, { withDisabled: true }));
+  t.is(updated.status, 'skipped');
+  t.is(updated.lastErrorCode, 'disabled_recipient');
+  t.is(updated.recipientEmail, null);
+  t.is(updated.payload, null);
 });
 
-test('should drop expired mail retry', async t => {
-  const send = Sinon.stub(sender, 'send').resolves(true);
-
-  await mailJob.sendMail({
-    startTime: Date.now() - 25 * 60 * 60 * 1000,
-    name: 'SignIn',
-    to: 'expired-retry@example.com',
-    props: {
-      url: 'https://affine.pro/sign-in',
-      otp: '123456',
-    },
+test('should not create sendable row for expired mail', async t => {
+  const send = Sinon.stub(sender, 'send').resolves({
+    status: 'accepted',
+    retryable: false,
   });
-
-  t.false(send.called);
-});
-
-test('should drop time-sensitive mail after its business expiration', async t => {
-  const send = Sinon.stub(sender, 'send').resolves(true);
-
-  await mailJob.sendMail({
-    startTime: Date.now() - 31 * 60 * 1000,
-    name: 'SignIn',
-    to: 'expired-sign-in@example.com',
-    props: {
-      url: 'https://affine.pro/sign-in',
-      otp: '123456',
+  const row = await createDelivery(
+    {
+      name: 'SignIn',
+      to: 'expired-retry@example.com',
+      props: { url: 'https://affine.pro/sign-in', otp: '123456' },
     },
-  });
+    { expiresAt: new Date(Date.now() - 1) }
+  );
 
+  await mailJob.processReadyDeliveries();
+
+  const updated = await delivery(row.id);
   t.false(send.called);
+  t.is(updated.status, 'failed');
+  t.is(updated.lastErrorCode, 'expired');
+  t.is(updated.retentionState, 'anonymized');
 });
 
-test('should use explicit mail expiration when provided', async t => {
-  const send = Sinon.stub(sender, 'send').resolves(true);
+test('should not claim delivery when max attempts is zero', async t => {
+  const send = Sinon.stub(sender, 'send').resolves({
+    status: 'accepted',
+    retryable: false,
+  });
+  const row = await createDelivery(
+    {
+      name: 'SignIn',
+      to: 'max-attempts@example.com',
+      props: { url: 'https://affine.pro/sign-in', otp: '123456' },
+    },
+    { maxAttempts: 0 }
+  );
 
-  await mailJob.sendMail({
-    startTime: Date.now(),
-    expiresAt: Date.now() - 1,
-    name: 'MemberInvitation',
-    to: 'expired-invitation@example.com',
-    props: {
-      user: {
-        $$userId: 'owner-id',
+  await mailJob.processReadyDeliveries();
+
+  t.false(send.called);
+  t.is((await delivery(row.id)).status, 'queued');
+});
+
+test('should retry retryable send failures without mutating stored dynamic props', async t => {
+  const owner = await module.create(Mockers.User);
+  const member = await module.create(Mockers.User);
+  const workspace = await module.create(Mockers.Workspace, {
+    owner: { id: owner.id },
+    name: 'Safe Workspace',
+  });
+  Sinon.stub(sender, 'send').resolves({
+    status: 'failed',
+    retryable: true,
+    errorCode: 'transport_failed',
+    error: 'temporary failure',
+  });
+  const row = await createDelivery(
+    {
+      name: 'MemberInvitation',
+      to: member.email,
+      props: {
+        user: { $$userId: owner.id },
+        workspace: { $$workspaceId: workspace.id },
+        url: 'https://affine.pro/invite/test',
       },
-      workspace: {
-        $$workspaceId: 'workspace-id',
-      },
-      url: 'https://affine.pro/invite/test',
+    },
+    { actorUserId: owner.id, workspaceId: workspace.id }
+  );
+
+  await mailJob.processReadyDeliveries();
+
+  const updated = await delivery(row.id);
+  t.is(updated.status, 'retry_wait');
+  t.is(updated.attemptCount, 1);
+  t.like(updated.payload as object, {
+    props: {
+      user: { $$userId: owner.id },
+      workspace: { $$workspaceId: workspace.id },
     },
   });
-
-  t.false(send.called);
-});
-
-test('should drop mail retry after max attempts', async t => {
-  const send = Sinon.stub(sender, 'send').resolves(true);
-
-  await mailJob.sendMail({
-    startTime: Date.now(),
-    retryCount: 12,
-    name: 'SignIn',
-    to: 'max-retry@example.com',
-    props: {
-      url: 'https://affine.pro/sign-in',
-      otp: '123456',
-    },
-  });
-
-  t.false(send.called);
-});
-
-test('should requeue legacy stringified retry mail', async t => {
-  const retryMailKey = 'mailjob:sendMail:retry';
-  const job: Jobs['notification.sendMail'] = {
-    startTime: Date.now(),
-    name: 'SignIn',
-    to: 'legacy-retry@example.com',
-    props: {
-      url: 'https://affine.pro/sign-in',
-      otp: '123456',
-    },
-  };
-  const cacheKey = `${retryMailKey}:SignIn:${job.to}`;
-
-  Sinon.stub(cache, 'mapRandomKey')
-    .onFirstCall()
-    .resolves(cacheKey)
-    .onSecondCall()
-    .resolves(undefined);
-  await cache.mapSet(retryMailKey, cacheKey, JSON.stringify(job));
-  await mailJob.sendRetryMails();
-
-  t.true(module.queue.add.calledWith('notification.sendMail', job));
-  t.is(await cache.mapGet(retryMailKey, cacheKey), undefined);
 });
 
 test('should skip member invitation mail when rendered workspace name contains domain', async t => {
@@ -218,53 +199,123 @@ test('should skip member invitation mail when rendered workspace name contains d
   const member = await module.create(Mockers.User);
   const workspace = await module.create(Mockers.Workspace, {
     owner: { id: owner.id },
-    name: 'BTC https://spam.example',
+    name: 'BTC example.com',
   });
-  const send = Sinon.stub(sender, 'send').resolves(true);
-
-  await mailJob.sendMail({
-    startTime: Date.now(),
-    name: 'MemberInvitation',
-    to: member.email,
-    props: {
-      user: {
-        $$userId: owner.id,
+  const send = Sinon.stub(sender, 'send').resolves({
+    status: 'accepted',
+    retryable: false,
+  });
+  const row = await createDelivery(
+    {
+      name: 'MemberInvitation',
+      to: member.email,
+      props: {
+        user: { $$userId: owner.id },
+        workspace: { $$workspaceId: workspace.id },
+        url: 'https://affine.pro/invite/test',
       },
-      workspace: {
-        $$workspaceId: workspace.id,
-      },
-      url: 'https://affine.pro/invite/test',
     },
-  });
+    { actorUserId: owner.id, workspaceId: workspace.id }
+  );
 
+  await mailJob.processReadyDeliveries();
+
+  const updated = await delivery(row.id);
   t.false(send.called);
+  t.is(updated.status, 'skipped');
+  t.is(updated.lastErrorCode, 'dynamic_props_missing');
 });
 
-test('should keep dynamic mail props untouched for retry', async t => {
-  const owner = await module.create(Mockers.User);
-  const member = await module.create(Mockers.User);
-  const workspace = await module.create(Mockers.Workspace, {
-    owner: { id: owner.id },
-    name: 'Safe Workspace',
+test('should mark accepted mail as sent and anonymize sendable payload', async t => {
+  const user = await module.create(Mockers.User);
+  Sinon.stub(sender, 'send').resolves({
+    status: 'accepted',
+    retryable: false,
+    providerMessageId: 'message-id',
+    providerResponse: '250 ok',
+  });
+  const row = await createDelivery({
+    name: 'SignIn',
+    to: user.email,
+    props: { url: 'https://affine.pro/sign-in', otp: '123456' },
   });
-  Sinon.stub(sender, 'send').resolves(false);
-  const job: Jobs['notification.sendMail'] = {
-    startTime: Date.now(),
-    name: 'MemberInvitation',
-    to: member.email,
-    props: {
-      user: {
-        $$userId: owner.id,
-      },
-      workspace: {
-        $$workspaceId: workspace.id,
-      },
-      url: 'https://affine.pro/invite/test',
-    },
-  };
 
-  await mailJob.sendMail(job);
+  await mailJob.processReadyDeliveries();
 
-  t.deepEqual(job.props.user, { $$userId: owner.id });
-  t.deepEqual(job.props.workspace, { $$workspaceId: workspace.id });
+  const updated = await delivery(row.id);
+  t.is(updated.status, 'sent');
+  t.is(updated.providerMessageId, 'message-id');
+  t.is(updated.recipientEmail, null);
+  t.is(updated.payload, null);
+  t.is(updated.retentionState, 'anonymized');
+});
+
+test('should claim critical priority before lower priority rows', async t => {
+  Sinon.stub(sender, 'send').resolves({
+    status: 'accepted',
+    retryable: false,
+  });
+  const low = await createDelivery(
+    {
+      name: 'SignIn',
+      to: 'low-priority@example.com',
+      props: { url: 'https://affine.pro/sign-in', otp: '123456' },
+    },
+    { priority: 'low' }
+  );
+  const critical = await createDelivery({
+    name: 'SignIn',
+    to: 'critical-priority@example.com',
+    props: { url: 'https://affine.pro/sign-in', otp: '123456' },
+  });
+
+  await mailJob.processReadyDeliveries(1);
+
+  t.is((await delivery(critical.id)).status, 'sent');
+  t.is((await delivery(low.id)).status, 'queued');
+});
+
+test('should reclaim expired sending lease', async t => {
+  Sinon.stub(sender, 'send').resolves({
+    status: 'accepted',
+    retryable: false,
+  });
+  const row = await createDelivery({
+    name: 'SignIn',
+    to: 'reclaim@example.com',
+    props: { url: 'https://affine.pro/sign-in', otp: '123456' },
+  });
+  await db.mailDelivery.update({
+    where: { id: row.id },
+    data: {
+      status: 'sending',
+      lockedBy: 'dead-worker',
+      lockedUntil: new Date(Date.now() - 1000),
+    },
+  });
+
+  await mailJob.processReadyDeliveries();
+
+  t.is((await delivery(row.id)).status, 'sent');
+});
+
+test('should delete retained anonymized terminal rows on worker tick', async t => {
+  const row = await createDelivery(
+    {
+      name: 'SignIn',
+      to: 'retention@example.com',
+      props: { url: 'https://affine.pro/sign-in', otp: '123456' },
+    },
+    { status: 'skipped' }
+  );
+  await db.mailDelivery.update({
+    where: { id: row.id },
+    data: {
+      settledAt: new Date(Date.now() - 31 * 24 * 60 * 60 * 1000),
+    },
+  });
+
+  await mailJob.sendPendingMails();
+
+  t.is(await db.mailDelivery.count({ where: { id: row.id } }), 0);
 });

packages/backend/server/src/core/mail/__tests__/mailer.spec.ts

@@ -0,0 +1,316 @@
+import { createHash, createHmac } from 'node:crypto';
+
+import { PrismaClient } from '@prisma/client';
+import ava, { TestFn } from 'ava';
+import Sinon from 'sinon';
+
+import {
+  createTestingModule,
+  type TestingModule,
+} from '../../../__tests__/utils';
+import { CryptoHelper } from '../../../base';
+import { Models } from '../../../models';
+import { BackendRuntimeProvider } from '../../backend-runtime';
+import { Mailer } from '../mailer';
+import { MailSender } from '../sender';
+
+interface Context {
+  module: TestingModule;
+  mailer: Mailer;
+  models: Models;
+  db: PrismaClient;
+  crypto: CryptoHelper;
+  runtime: {
+    assertMailDeliveryQuotaV1: Sinon.SinonStub;
+    commitMailDeliveryQuotaV1: Sinon.SinonStub;
+    releaseMailDeliveryQuotaV1: Sinon.SinonStub;
+  };
+}
+
+const test = ava as TestFn<Context>;
+
+test.before(async t => {
+  t.context.runtime = {
+    assertMailDeliveryQuotaV1: Sinon.stub(),
+    commitMailDeliveryQuotaV1: Sinon.stub(),
+    releaseMailDeliveryQuotaV1: Sinon.stub(),
+  };
+  t.context.module = await createTestingModule({
+    tapModule: builder => {
+      builder
+        .overrideProvider(Mailer)
+        .useClass(Mailer)
+        .overrideProvider(MailSender)
+        .useValue({ configured: true })
+        .overrideProvider(BackendRuntimeProvider)
+        .useValue(t.context.runtime);
+    },
+  });
+  t.context.mailer = t.context.module.get(Mailer);
+  t.context.models = t.context.module.get(Models);
+  t.context.db = t.context.module.get(PrismaClient);
+  t.context.crypto = t.context.module.get(CryptoHelper);
+});
+
+test.beforeEach(t => {
+  t.context.runtime.assertMailDeliveryQuotaV1.reset();
+  t.context.runtime.commitMailDeliveryQuotaV1.reset();
+  t.context.runtime.releaseMailDeliveryQuotaV1.reset();
+  t.context.runtime.assertMailDeliveryQuotaV1.resolves({
+    allowed: true,
+    reservationId: '00000000-0000-0000-0000-000000000001',
+    mailClass: 'auth',
+  });
+  t.context.runtime.commitMailDeliveryQuotaV1.resolves(true);
+  t.context.runtime.releaseMailDeliveryQuotaV1.resolves(true);
+});
+
+test.afterEach.always(async t => {
+  Sinon.restore();
+  await t.context.db.mailDelivery.deleteMany();
+});
+
+test.after.always(async t => {
+  await t.context.module.close();
+});
+
+test('trySend creates a delivery row and commits quota reservation', async t => {
+  const sent = await t.context.mailer.trySend({
+    name: 'SignIn',
+    to: 'auth-user@example.com',
+    props: {
+      url: 'https://affine.pro/sign-in',
+      otp: '123456',
+    },
+    metadata: {
+      dedupeKey: 'signin:auth-user@example.com:1',
+      recipientUserId: 'user-1',
+      source: { trusted: false },
+    },
+  });
+
+  const row = await t.context.db.mailDelivery.findFirstOrThrow({
+    where: { dedupeKey: 'signin:auth-user@example.com:1' },
+  });
+
+  t.true(sent);
+  t.is(row.status, 'queued');
+  t.is(row.recipientUserId, 'user-1');
+  t.is(row.mailClass, 'auth');
+  t.is(
+    row.recipientHash,
+    createHmac('sha256', t.context.crypto.keyPair.sha256.privateKey)
+      .update('auth-user@example.com')
+      .digest('hex')
+  );
+  t.not(
+    row.recipientHash,
+    createHash('sha256').update('auth-user@example.com').digest('hex')
+  );
+  t.true(
+    t.context.runtime.commitMailDeliveryQuotaV1.calledOnceWithExactly(
+      '00000000-0000-0000-0000-000000000001'
+    )
+  );
+});
+
+test('cancelByRecipient matches the keyed recipient hash', async t => {
+  await t.context.mailer.trySend({
+    name: 'SignIn',
+    to: '  Delete-Me@Example.COM  ',
+    props: {
+      url: 'https://affine.pro/sign-in',
+      otp: '123456',
+    },
+    metadata: {
+      dedupeKey: 'signin:delete-me@example.com:1',
+      source: { trusted: false },
+    },
+  });
+
+  await t.context.models.mailDelivery.cancelByRecipient(
+    'delete-me@example.com'
+  );
+
+  const row = await t.context.db.mailDelivery.findFirstOrThrow({
+    where: { dedupeKey: 'signin:delete-me@example.com:1' },
+  });
+  t.is(
+    row.recipientHash,
+    createHmac('sha256', t.context.crypto.keyPair.sha256.privateKey)
+      .update('delete-me@example.com')
+      .digest('hex')
+  );
+  t.is(row.status, 'canceled');
+  t.is(row.lastErrorCode, 'recipient_deleted');
+});
+
+test('dedupe replay does not consume another mail quota reservation', async t => {
+  const command = {
+    name: 'SignIn' as const,
+    to: 'dedupe@example.com',
+    props: {
+      url: 'https://affine.pro/sign-in',
+      otp: '123456',
+    },
+    metadata: {
+      dedupeKey: 'signin:dedupe@example.com:1',
+      source: { trusted: false },
+    },
+  };
+
+  t.true(await t.context.mailer.trySend(command));
+  t.true(await t.context.mailer.trySend(command));
+
+  t.is(t.context.runtime.assertMailDeliveryQuotaV1.callCount, 1);
+  t.is(t.context.runtime.commitMailDeliveryQuotaV1.callCount, 1);
+  t.is(
+    await t.context.db.mailDelivery.count({
+      where: { dedupeKey: command.metadata.dedupeKey },
+    }),
+    1
+  );
+});
+
+test('quota denial records skipped deliveries without committing quota', async t => {
+  for (const quota of [
+    {
+      email: 'limited@example.com',
+      mailClass: 'auth',
+      reason: 'recipient_rate_limited',
+    },
+    {
+      email: 'unmapped@example.com',
+      mailClass: 'unknown',
+      reason: 'unmapped_mail_name',
+    },
+  ] as const) {
+    t.context.runtime.assertMailDeliveryQuotaV1.resolves({
+      allowed: false,
+      mailClass: quota.mailClass,
+      reason: quota.reason,
+    });
+
+    const dedupeKey = `signin:${quota.email}:1`;
+    const sent = await t.context.mailer.trySend({
+      name: 'SignIn',
+      to: quota.email,
+      props: {
+        url: 'https://affine.pro/sign-in',
+        otp: '123456',
+      },
+      metadata: {
+        dedupeKey,
+        source: { trusted: false },
+      },
+    });
+
+    const row = await t.context.db.mailDelivery.findFirstOrThrow({
+      where: { dedupeKey },
+    });
+
+    t.false(sent, quota.reason);
+    t.is(row.status, 'skipped', quota.reason);
+    t.is(row.mailClass, quota.mailClass, quota.reason);
+    t.is(row.lastErrorCode, quota.reason, quota.reason);
+    t.is(row.recipientEmail, null, quota.reason);
+    t.false(t.context.runtime.commitMailDeliveryQuotaV1.called, quota.reason);
+    t.context.runtime.assertMailDeliveryQuotaV1.resetHistory();
+  }
+});
+
+test('skip records terminal delivery without quota admission', async t => {
+  const sent = await t.context.mailer.skip(
+    {
+      name: 'MemberInvitation',
+      to: 'skip@example.com',
+      props: {
+        url: 'https://affine.pro/invite',
+        user: { $$userId: 'actor-1' },
+        workspace: { $$workspaceId: 'workspace-1' },
+      },
+      metadata: {
+        dedupeKey: 'invite:skip@example.com:1',
+        recipientUserId: 'user-1',
+        actorUserId: 'actor-1',
+        workspaceId: 'workspace-1',
+        source: { trusted: false },
+      },
+    },
+    {
+      mailClass: 'workspace_invitation',
+      reason: 'workspace_name_contains_domain',
+    }
+  );
+
+  const row = await t.context.db.mailDelivery.findFirstOrThrow({
+    where: { dedupeKey: 'invite:skip@example.com:1' },
+  });
+
+  t.false(sent);
+  t.is(row.status, 'skipped');
+  t.is(row.mailClass, 'workspace_invitation');
+  t.is(row.lastErrorCode, 'workspace_name_contains_domain');
+  t.is(row.maxAttempts, 0);
+  t.false(t.context.runtime.assertMailDeliveryQuotaV1.called);
+});
+
+test('send releases quota reservation and rethrows when ledger write fails', async t => {
+  Sinon.stub(t.context.models.mailDelivery, 'create').rejects(
+    new Error('ledger failed')
+  );
+
+  await t.throwsAsync(
+    t.context.mailer.send({
+      name: 'SignIn',
+      to: 'throw@example.com',
+      props: {
+        url: 'https://affine.pro/sign-in',
+        otp: '123456',
+      },
+      metadata: {
+        source: { trusted: false },
+      },
+    }),
+    { message: 'ledger failed' }
+  );
+
+  t.true(
+    t.context.runtime.releaseMailDeliveryQuotaV1.calledOnceWithExactly(
+      '00000000-0000-0000-0000-000000000001'
+    )
+  );
+});
+
+test('send cancels queued delivery when quota commit fails', async t => {
+  t.context.runtime.commitMailDeliveryQuotaV1.rejects(
+    new Error('commit failed')
+  );
+
+  await t.throwsAsync(
+    t.context.mailer.send({
+      name: 'SignIn',
+      to: 'commit-failed@example.com',
+      props: {
+        url: 'https://affine.pro/sign-in',
+        otp: '123456',
+      },
+      metadata: {
+        dedupeKey: 'signin:commit-failed@example.com:1',
+        source: { trusted: false },
+      },
+    }),
+    { message: 'commit failed' }
+  );
+
+  const row = await t.context.db.mailDelivery.findFirstOrThrow({
+    where: { dedupeKey: 'signin:commit-failed@example.com:1' },
+  });
+  t.is(row.status, 'canceled');
+  t.is(row.lastErrorCode, 'quota_commit_failed');
+  t.true(
+    t.context.runtime.releaseMailDeliveryQuotaV1.calledOnceWithExactly(
+      '00000000-0000-0000-0000-000000000001'
+    )
+  );
+});

packages/backend/server/src/core/mail/__tests__/resolver.spec.ts

@@ -0,0 +1,257 @@
+import { PrismaClient } from '@prisma/client';
+import test from 'ava';
+
+import { createApp, type TestingApp } from '../../../__tests__/e2e/test';
+import { Mockers } from '../../../__tests__/mocks';
+
+let app: TestingApp;
+
+function startOfUtcHour(value: Date) {
+  return new Date(
+    Date.UTC(
+      value.getUTCFullYear(),
+      value.getUTCMonth(),
+      value.getUTCDate(),
+      value.getUTCHours()
+    )
+  );
+}
+
+test.before(async () => {
+  app = await createApp();
+});
+
+test.beforeEach(async () => {
+  await app.get(PrismaClient).mailDelivery.deleteMany();
+});
+
+test.after.always(async () => {
+  await app.close();
+});
+
+test.afterEach.always(async () => {
+  await app.get(PrismaClient).mailDelivery.deleteMany();
+});
+
+test('sendTestEmail rejects non-admin users before SMTP or ledger', async t => {
+  const user = await app.create(Mockers.User);
+
+  await app.login(user);
+
+  const response = await app.request('post', '/graphql').send({
+    query: /* GraphQL */ `
+      mutation SendTestEmail($config: JSONObject!) {
+        sendTestEmail(config: $config)
+      }
+    `,
+    variables: {
+      config: {
+        name: '',
+        host: 'smtp.example.com',
+        port: 587,
+        username: 'user',
+        password: 'password',
+        ignoreTLS: false,
+        sender: 'AFFiNE <noreply@example.com>',
+      },
+    },
+  });
+
+  t.is(response.status, 200);
+  t.truthy(response.body.errors);
+  t.is(await app.get(PrismaClient).mailDelivery.count(), 0);
+});
+
+test('adminMailDeliveries returns timeline series for status type and outcome', async t => {
+  const db = app.get(PrismaClient);
+  const admin = await app.create(Mockers.User, {
+    feature: 'administrator',
+  });
+  const now = new Date();
+  const createdAt = new Date(startOfUtcHour(now).getTime() - 60 * 60 * 1000);
+  const base = {
+    priority: 'normal',
+    recipientHash: 'hash',
+    recipientDomain: 'example.com',
+    sendAfter: now,
+    retentionState: 'anonymized',
+    anonymizedAt: now,
+    createdAt,
+    updatedAt: now,
+  };
+
+  await db.mailDelivery.createMany({
+    data: [
+      {
+        ...base,
+        mailName: 'SignIn',
+        mailClass: 'auth',
+        status: 'sent',
+        settledAt: now,
+        sentAt: now,
+      },
+      {
+        ...base,
+        mailName: 'MemberInvitation',
+        mailClass: 'workspace_invitation',
+        status: 'failed',
+        settledAt: now,
+        failedAt: now,
+        lastErrorCode: 'transport_failed',
+      },
+      {
+        ...base,
+        mailName: 'Mention',
+        mailClass: 'notification',
+        status: 'queued',
+        recipientEmail: 'queued@example.com',
+        payload: {},
+        retentionState: 'full',
+        anonymizedAt: null,
+      },
+    ],
+  });
+
+  await app.login(admin);
+  const response = await app.request('post', '/graphql').send({
+    query: /* GraphQL */ `
+      query AdminMailDeliveries($input: AdminMailDeliveriesInput) {
+        adminMailDeliveries(input: $input) {
+          window {
+            bucket
+            effectiveSize
+          }
+          summary {
+            total
+            sent
+            failed
+            queued
+            successRate
+          }
+          byStatus {
+            key
+            total
+            points {
+              bucket
+              count
+            }
+          }
+          byType {
+            key
+            total
+          }
+          byOutcome {
+            key
+            total
+          }
+        }
+      }
+    `,
+    variables: {
+      input: {
+        hours: 24,
+      },
+    },
+  });
+
+  t.is(response.status, 200);
+  t.falsy(response.body.errors);
+  const analytics = response.body.data.adminMailDeliveries;
+  t.is(analytics.window.bucket, 'Hour');
+  t.is(analytics.window.effectiveSize, 24);
+  t.like(analytics.summary, {
+    total: 3,
+    sent: 1,
+    failed: 1,
+    queued: 1,
+    successRate: 0.5,
+  });
+  t.true(
+    analytics.byStatus.some((series: { key: string; total: number }) => {
+      return series.key === 'sent' && series.total === 1;
+    })
+  );
+  const sent = analytics.byStatus.find(
+    (series: { key: string; points: { bucket: string; count: number }[] }) =>
+      series.key === 'sent'
+  );
+  t.deepEqual(
+    sent?.points.filter((point: { count: number }) => point.count > 0),
+    [{ bucket: createdAt.toISOString(), count: 1 }]
+  );
+  t.true(
+    analytics.byType.some(
+      (series: { key: string; total: number }) =>
+        series.key === 'workspace_invitation' && series.total === 1
+    )
+  );
+  t.true(
+    analytics.byOutcome.some(
+      (series: { key: string; total: number }) =>
+        series.key === 'pending' && series.total === 1
+    )
+  );
+});
+
+test('adminMailDeliveries uses day buckets for seven day window', async t => {
+  const admin = await app.create(Mockers.User, {
+    feature: 'administrator',
+  });
+
+  await app.login(admin);
+  const response = await app.request('post', '/graphql').send({
+    query: /* GraphQL */ `
+      query AdminMailDeliveries($input: AdminMailDeliveriesInput) {
+        adminMailDeliveries(input: $input) {
+          window {
+            bucket
+            effectiveSize
+          }
+        }
+      }
+    `,
+    variables: {
+      input: {
+        hours: 168,
+      },
+    },
+  });
+
+  t.is(response.status, 200);
+  t.falsy(response.body.errors);
+  t.like(response.body.data.adminMailDeliveries.window, {
+    bucket: 'Day',
+    effectiveSize: 7,
+  });
+});
+
+test('adminMailDeliveries rejects unsupported window sizes', async t => {
+  const admin = await app.create(Mockers.User, {
+    feature: 'administrator',
+  });
+
+  await app.login(admin);
+  const response = await app.request('post', '/graphql').send({
+    query: /* GraphQL */ `
+      query AdminMailDeliveries($input: AdminMailDeliveriesInput) {
+        adminMailDeliveries(input: $input) {
+          window {
+            requestedSize
+          }
+        }
+      }
+    `,
+    variables: {
+      input: {
+        hours: 48,
+      },
+    },
+  });
+
+  t.is(response.status, 200);
+  t.truthy(response.body.errors);
+  t.regex(
+    response.body.errors[0].message,
+    /Mail delivery analytics window must be 24 or 168 hours/
+  );
+});

packages/backend/server/src/core/mail/config.ts

@@ -16,6 +16,11 @@ declare global {
       };
 
       fallbackDomains: ConfigItem<string[]>;
+      deliveryWorker: {
+        batchSize: number;
+        leaseMs: number;
+        retentionDays: number;
+      };
       fallbackSMTP: {
         name: string;
         host: string;
@@ -71,6 +76,21 @@ defineModuleConfig('mailer', {
     default: [],
     shape: z.array(z.string()),
   },
+  'deliveryWorker.batchSize': {
+    desc: 'Number of mail delivery rows claimed by each worker tick.',
+    default: 50,
+    shape: z.number().int().min(1).max(1000),
+  },
+  'deliveryWorker.leaseMs': {
+    desc: 'Mail delivery worker lease duration in milliseconds.',
+    default: 2 * 60 * 1000,
+    shape: z.number().int().min(1000),
+  },
+  'deliveryWorker.retentionDays': {
+    desc: 'Days to retain anonymized terminal mail delivery ledger rows.',
+    default: 30,
+    shape: z.number().int().min(1),
+  },
   'fallbackSMTP.name': {
     desc: 'Hostname used for fallback SMTP HELO/EHLO (e.g. mail.example.com). Leave empty to use the system hostname.',
     default: '',

packages/backend/server/src/core/mail/index.ts

@@ -2,6 +2,7 @@ import './config';
 
 import { Module } from '@nestjs/common';
 
+import { BackendRuntimeModule } from '../backend-runtime';
 import { DocStorageModule } from '../doc';
 import { StorageModule } from '../storage';
 import { MailJob } from './job';
@@ -10,7 +11,7 @@ import { MailResolver } from './resolver';
 import { MailSender } from './sender';
 
 @Module({
-  imports: [DocStorageModule, StorageModule],
+  imports: [BackendRuntimeModule, DocStorageModule, StorageModule],
   providers: [MailSender, Mailer, MailJob, MailResolver],
   exports: [Mailer],
 })

packages/backend/server/src/core/mail/job.ts

@@ -1,106 +1,184 @@
+import { randomUUID } from 'node:crypto';
+
 import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 import { getStreamAsBuffer } from 'get-stream';
 
-import { Cache, JOB_SIGNAL, JobQueue, OnEvent, OnJob, sleep } from '../../base';
-import { type MailName, MailProps, Renderers } from '../../mails';
+import { Config, metrics, OnEvent } from '../../base';
+import { type MailName, Renderers } from '../../mails';
 import { UserProps, WorkspaceProps } from '../../mails/components';
-import { Models } from '../../models';
+import { MailDeliveryRow, Models } from '../../models';
+import { containsUrlOrDomain } from '../content-policy';
 import { DocReader } from '../doc/reader';
 import { WorkspaceBlobStorage } from '../storage';
-import { containsUrlOrDomain } from '../workspaces/abuse';
 import { MailSender, SendOptions } from './sender';
+import { SendMailPayload } from './types';
 
-type DynamicallyFetchedProps<Props> = {
-  [Key in keyof Props]: Props[Key] extends infer Prop
-    ? Prop extends UserProps
-      ? {
-          $$userId: string;
-        } & Omit<Prop, 'email' | 'avatar'>
-      : Prop extends WorkspaceProps
-        ? {
-            $$workspaceId: string;
-          } & Omit<Prop, 'name' | 'avatar'>
-        : Prop
-    : never;
-};
-
-type SendMailJob<Mail extends MailName = MailName, Props = MailProps<Mail>> = {
-  name: Mail;
-  to: string;
-  // NOTE(@forehalo):
-  //   workspace avatar currently send as base64 img instead of a avatar url,
-  //   so the content might be too large to be put in job payload.
-  props: DynamicallyFetchedProps<Props>;
-};
-
-declare global {
-  interface Jobs {
-    'notification.sendMail': {
-      startTime: number;
-      retryCount?: number;
-      expiresAt?: number;
-    } & {
-      [K in MailName]: SendMailJob<K>;
-    }[MailName];
-  }
-}
-
-const sendMailKey = 'mailjob:sendMail';
-const retryMailKey = 'mailjob:sendMail:retry';
-const sendMailCacheKey = (name: string, to: string) =>
-  `${sendMailKey}:${name}:${to}`;
-const retryMaxPerTick = 20;
-const retryFirstTime = 3;
-const retryMaxAttempts = 12;
-const retryMaxAge = 24 * 60 * 60 * 1000;
-const magicLinkExpiresIn = 30 * 60 * 1000;
-
-const mailExpiresIn: Partial<Record<MailName, number>> = {
-  SignIn: magicLinkExpiresIn,
-  SignUp: magicLinkExpiresIn,
-  SetPassword: magicLinkExpiresIn,
-  ChangePassword: magicLinkExpiresIn,
-  VerifyEmail: magicLinkExpiresIn,
-  ChangeEmail: magicLinkExpiresIn,
-  VerifyChangeEmail: magicLinkExpiresIn,
+type DynamicProp = Record<string, unknown> & {
+  $$workspaceId?: string;
+  $$userId?: string;
 };
 
 @Injectable()
 export class MailJob {
-  private readonly logger = new Logger('MailJob');
+  private readonly logger = new Logger('MailDeliveryWorker');
+  private readonly workerId = `mail-delivery-${process.pid}-${randomUUID()}`;
 
   constructor(
-    private readonly cache: Cache,
-    private readonly queue: JobQueue,
     private readonly sender: MailSender,
     private readonly doc: DocReader,
     private readonly workspaceBlob: WorkspaceBlobStorage,
-    private readonly models: Models
+    private readonly models: Models,
+    private readonly config: Config
   ) {}
 
-  private calculateRetryDelay(startTime: number) {
-    const elapsed = Date.now() - startTime;
-    return Math.min(30 * 1000, Math.round(elapsed / 2000) * 1000);
+  @OnEvent('user.deleted')
+  async onUserDeleted(user: Events['user.deleted']) {
+    await Promise.all([
+      this.models.mailDelivery.cancelByRecipient(user.email),
+      this.models.mailDelivery.cancelMemberInvitationByActor(user.id),
+    ]);
   }
 
-  private getRetryExhaustedReason({
-    startTime,
-    retryCount,
-    expiresAt,
-    name,
-  }: Jobs['notification.sendMail']) {
-    const expiredAt =
-      expiresAt ?? startTime + (mailExpiresIn[name] ?? retryMaxAge);
-    if (Date.now() > expiredAt) {
-      return 'expired';
+  @Cron(CronExpression.EVERY_MINUTE)
+  async sendPendingMails() {
+    await this.processReadyDeliveries();
+    await this.cleanupRetainedDeliveries();
+    await this.recordDeliveryMetrics();
+  }
+
+  async processReadyDeliveries(
+    batchSize = this.config.mailer.deliveryWorker.batchSize
+  ) {
+    const rows = await this.models.mailDelivery.claimReady(this.workerId, {
+      batchSize,
+      leaseMs: this.config.mailer.deliveryWorker.leaseMs,
+    });
+
+    for (const row of rows) {
+      await this.processDelivery(row);
+    }
+    return rows.length;
+  }
+
+  private async cleanupRetainedDeliveries() {
+    const retentionMs =
+      this.config.mailer.deliveryWorker.retentionDays * 24 * 60 * 60 * 1000;
+    const before = new Date(Date.now() - retentionMs);
+    await this.models.mailDelivery.deleteAnonymizedBefore(
+      before,
+      this.config.mailer.deliveryWorker.batchSize
+    );
+  }
+
+  private async recordDeliveryMetrics() {
+    const snapshot = await this.models.mailDelivery.metricsSnapshot();
+    metrics.mail.gauge('retry_wait_backlog').record(snapshot.retryWait);
+    metrics.mail.gauge('failed_recent').record(snapshot.failedRecent);
+    metrics.mail.gauge('expired_lease_backlog').record(snapshot.expiredLeases);
+    metrics.mail.histogram('ready_delay_ms').record(snapshot.readyDelayMs);
+  }
+
+  private async processDelivery(row: MailDeliveryRow) {
+    if (!row.recipientEmail || !row.payload) {
+      await this.models.mailDelivery.markSkipped(
+        row.id,
+        this.workerId,
+        'missing_payload'
+      );
+      return;
+    }
+    if (row.expiresAt && row.expiresAt.getTime() <= Date.now()) {
+      await this.models.mailDelivery.markFailed(
+        row.id,
+        this.workerId,
+        'expired'
+      );
+      return;
+    }
+    if (await this.shouldSkipRecipient(row.recipientEmail)) {
+      await this.models.mailDelivery.markSkipped(
+        row.id,
+        this.workerId,
+        'disabled_recipient'
+      );
+      return;
     }
 
-    if ((retryCount ?? 0) > retryMaxAttempts) {
-      return 'max attempts reached';
+    const payload = row.payload as SendMailPayload;
+    const rendered = await this.renderPayload(payload);
+    if (!rendered) {
+      await this.models.mailDelivery.markSkipped(
+        row.id,
+        this.workerId,
+        'dynamic_props_missing'
+      );
+      return;
+    }
+    if (row.expiresAt && row.expiresAt.getTime() <= Date.now()) {
+      await this.models.mailDelivery.markFailed(
+        row.id,
+        this.workerId,
+        'expired'
+      );
+      return;
     }
 
-    return;
+    const attempt = await this.models.mailDelivery.markAttemptStarted(
+      row.id,
+      this.workerId
+    );
+    if (!attempt) {
+      return;
+    }
+
+    const result = await this.sender.send(payload.name, {
+      to: row.recipientEmail,
+      ...rendered.options,
+      ...rendered.content,
+    });
+
+    if (result.status === 'accepted') {
+      await this.models.mailDelivery.markSent(row.id, this.workerId, {
+        providerMessageId: result.providerMessageId,
+        providerResponse: result.providerResponse,
+      });
+      return;
+    }
+
+    const nextSendAfter = this.nextSendAfter(attempt.attemptCount);
+    const canRetry =
+      result.retryable &&
+      attempt.attemptCount < attempt.maxAttempts &&
+      (!attempt.expiresAt || nextSendAfter < attempt.expiresAt);
+    if (canRetry) {
+      const retry = await this.models.mailDelivery.markRetry(
+        row.id,
+        this.workerId,
+        {
+          sendAfter: nextSendAfter,
+          errorCode: result.errorCode,
+          error: result.error,
+        }
+      );
+      if (retry) {
+        return;
+      }
+    }
+
+    await this.models.mailDelivery.markFailed(
+      row.id,
+      this.workerId,
+      attempt.expiresAt && nextSendAfter >= attempt.expiresAt
+        ? 'expired'
+        : (result.errorCode ?? result.status),
+      result.error ?? result.providerResponse ?? undefined
+    );
+  }
+
+  private nextSendAfter(attemptCount: number) {
+    const delayMs = Math.min(30 * 60 * 1000, 2 ** attemptCount * 30 * 1000);
+    return new Date(Date.now() + delayMs);
   }
 
   private async shouldSkipRecipient(to: string) {
@@ -111,72 +189,16 @@ export class MailJob {
     return user?.disabled === true;
   }
 
-  private async deleteRecipientMailCache(to: string) {
-    const suffix = `:${to}`;
-
-    await Promise.all(
-      [sendMailKey, retryMailKey].map(async map => {
-        const keys = await this.cache.mapKeys(map);
-        await Promise.all(
-          keys
-            .filter(key => key.endsWith(suffix))
-            .map(key => this.cache.mapDelete(map, key))
-        );
-      })
-    );
-  }
-
-  private isInvitationMailSentByUser(
-    job: Jobs['notification.sendMail'],
-    userId: string
-  ) {
-    return (
-      job.name === 'MemberInvitation' &&
-      'user' in job.props &&
-      typeof job.props.user === 'object' &&
-      job.props.user !== null &&
-      '$$userId' in job.props.user &&
-      job.props.user.$$userId === userId
-    );
-  }
-
-  private async deleteInvitationMailCacheBySender(userId: string) {
-    const keys = await this.cache.mapKeys(retryMailKey);
-    await Promise.all(
-      keys.map(async key => {
-        const job = await this.cache.mapGet<
-          Jobs['notification.sendMail'] | string
-        >(retryMailKey, key);
-        const jobData =
-          typeof job === 'string'
-            ? (JSON.parse(job) as Jobs['notification.sendMail'])
-            : job;
-        if (jobData && this.isInvitationMailSentByUser(jobData, userId)) {
-          await this.cache.mapDelete(retryMailKey, key);
-        }
-      })
-    );
-  }
-
-  private async sendMailInternal({
-    startTime,
-    name,
-    to,
-    props,
-  }: Jobs['notification.sendMail']) {
-    if (await this.shouldSkipRecipient(to)) {
-      this.logger.debug(`Skip mail [${name}] to disabled user [${to}]`);
-      return;
-    }
-
+  private async renderPayload(payload: SendMailPayload) {
     let options: Partial<SendOptions> = {};
-    const renderedProps = { ...props };
+    const renderedProps = { ...payload.props };
 
     for (const key in renderedProps) {
-      // @ts-expect-error allow
-      const val = renderedProps[key];
+      const val = renderedProps[key as keyof typeof renderedProps] as
+        | DynamicProp
+        | undefined;
       if (val && typeof val === 'object') {
-        if ('$$workspaceId' in val) {
+        if (typeof val.$$workspaceId === 'string') {
           const workspaceProps = await this.fetchWorkspaceProps(
             val.$$workspaceId
           );
@@ -186,11 +208,11 @@ export class MailJob {
           }
 
           if (
-            name === 'MemberInvitation' &&
+            payload.name === 'MemberInvitation' &&
             containsUrlOrDomain(workspaceProps.name)
           ) {
             this.logger.warn(
-              `Skip mail [${name}] to [${to}], reason=workspace name contains url or domain`
+              `Skip mail [${payload.name}] to [${payload.to}], reason=workspace name contains url or domain`
             );
             return;
           }
@@ -206,57 +228,42 @@ export class MailJob {
             ];
             workspaceProps.avatar = 'cid:workspaceAvatar';
           }
-          // @ts-expect-error replacement
-          renderedProps[key] = workspaceProps;
-        } else if ('$$userId' in val) {
+          Object.assign(val, workspaceProps);
+          delete val.$$workspaceId;
+        } else if (typeof val.$$userId === 'string') {
           const userProps = await this.fetchUserProps(val.$$userId);
 
           if (!userProps) {
             return;
           }
 
-          // @ts-expect-error replacement
-          renderedProps[key] = userProps;
+          Object.assign(val, userProps);
+          delete val.$$userId;
         }
       }
     }
 
     if (
-      name === 'MemberInvitation' &&
+      payload.name === 'MemberInvitation' &&
       'workspace' in renderedProps &&
       containsUrlOrDomain(
         (renderedProps.workspace as WorkspaceProps | undefined)?.name
       )
     ) {
       this.logger.warn(
-        `Skip mail [${name}] to [${to}], reason=workspace name contains url or domain`
+        `Skip mail [${payload.name}] to [${payload.to}], reason=workspace name contains url or domain`
       );
       return;
     }
 
-    try {
-      const result = await this.sender.send(name, {
-        to,
-        ...(await Renderers[name](
-          // @ts-expect-error the job trigger part has been typechecked
-          renderedProps
-        )),
-        ...options,
-      });
-      if (!result) {
-        // wait for a while before retrying
-        const retryDelay = this.calculateRetryDelay(startTime);
-        await sleep(retryDelay);
-        return JOB_SIGNAL.Retry;
-      }
-      return undefined;
-    } catch (e) {
-      this.logger.error(`Failed to send mail [${name}] to [${to}]`, e);
-      // wait for a while before retrying
-      const retryDelay = this.calculateRetryDelay(startTime);
-      await sleep(retryDelay);
-      return JOB_SIGNAL.Retry;
-    }
+    return {
+      options,
+      content: await (
+        Renderers[payload.name] as (
+          props: unknown
+        ) => ReturnType<(typeof Renderers)[MailName]>
+      )(renderedProps),
+    };
   }
 
   private async fetchWorkspaceProps(workspaceId: string) {
@@ -294,76 +301,4 @@ export class MailJob {
 
     return { email: user.email } satisfies UserProps;
   }
-
-  @OnJob('notification.sendMail')
-  async sendMail(job: Jobs['notification.sendMail']) {
-    const cacheKey = sendMailCacheKey(job.name, job.to);
-    job.retryCount = (job.retryCount ?? 0) + 1;
-    const exhaustedReason = this.getRetryExhaustedReason(job);
-    if (exhaustedReason) {
-      this.logger.warn(
-        `Drop mail [${job.name}] to [${job.to}], reason=${exhaustedReason}`
-      );
-      await Promise.all([
-        this.cache.mapDelete(sendMailKey, cacheKey),
-        this.cache.mapDelete(retryMailKey, cacheKey),
-      ]);
-      return;
-    }
-
-    const retried = await this.cache.mapIncrease(sendMailKey, cacheKey, 1);
-    if (retried <= retryFirstTime) {
-      const ret = await this.sendMailInternal(job);
-      if (!ret) await this.cache.mapDelete(sendMailKey, cacheKey);
-      return ret;
-    }
-    await this.cache.mapSet(retryMailKey, cacheKey, job);
-    await this.cache.mapDelete(sendMailKey, cacheKey);
-    return undefined;
-  }
-
-  @OnEvent('user.deleted')
-  async onUserDeleted(user: Events['user.deleted']) {
-    await Promise.all([
-      this.deleteRecipientMailCache(user.email),
-      this.deleteInvitationMailCacheBySender(user.id),
-      this.queue.removeWhere(
-        'notification.sendMail',
-        job =>
-          job.to === user.email || this.isInvitationMailSentByUser(job, user.id)
-      ),
-    ]);
-  }
-
-  @Cron(CronExpression.EVERY_MINUTE)
-  async sendRetryMails() {
-    // pick random one from the retry map
-    let processed = 0;
-    let key = await this.cache.mapRandomKey(retryMailKey);
-    while (key && processed < retryMaxPerTick) {
-      try {
-        const job = await this.cache.mapGet<
-          Jobs['notification.sendMail'] | string
-        >(retryMailKey, key);
-        if (job) {
-          const jobData =
-            typeof job === 'string'
-              ? (JSON.parse(job) as Jobs['notification.sendMail'])
-              : job;
-          await this.queue.add('notification.sendMail', jobData);
-          // wait for a while before retrying
-          const retryDelay = this.calculateRetryDelay(jobData.startTime);
-          await sleep(retryDelay);
-        }
-        await this.cache.mapDelete(retryMailKey, key);
-      } catch (e) {
-        this.logger.error(
-          `Failed to re-queue retry mail job for key [${key}]`,
-          e
-        );
-      }
-      key = await this.cache.mapRandomKey(retryMailKey);
-      processed++;
-    }
-  }
 }

packages/backend/server/src/core/mail/mailer.ts

@@ -1,13 +1,50 @@
 import { Injectable } from '@nestjs/common';
 
-import { EmailServiceNotConfigured, JobQueue } from '../../base';
+import { EmailServiceNotConfigured } from '../../base';
+import type { MailName } from '../../mails';
+import { Models } from '../../models';
+import { BackendRuntimeProvider } from '../backend-runtime';
 import { MailSender } from './sender';
+import type { SendMailCommand } from './types';
+
+type MailCommand = SendMailCommand;
+type SkipMailOptions = {
+  mailClass: string;
+  reason: string;
+  priority?: 'critical' | 'high' | 'normal' | 'low';
+};
+
+function recipientDomain(email: string) {
+  const parts = email.trim().toLowerCase().split('@');
+  return parts.length === 2 ? parts[1] : '';
+}
+
+function defaultPriority(mailClass: string): 'critical' | 'high' | 'normal' {
+  if (mailClass === 'auth') {
+    return 'critical';
+  }
+  if (mailClass === 'billing_license') {
+    return 'high';
+  }
+  return 'normal';
+}
+
+function serializePayload(command: MailCommand) {
+  return JSON.parse(
+    JSON.stringify({
+      name: command.name,
+      to: command.to,
+      props: command.props,
+    })
+  );
+}
 
 @Injectable()
 export class Mailer {
   constructor(
-    private readonly queue: JobQueue,
-    private readonly sender: MailSender
+    private readonly sender: MailSender,
+    private readonly models: Models,
+    private readonly runtime: BackendRuntimeProvider
   ) {}
 
   /**
@@ -15,14 +52,43 @@ export class Mailer {
    *
    * @note never throw
    */
-  async trySend(command: Omit<Jobs['notification.sendMail'], 'startTime'>) {
+  async trySend(command: MailCommand) {
     return this.send(command, true);
   }
 
-  async send(
-    command: Omit<Jobs['notification.sendMail'], 'startTime'>,
-    suppressError = false
-  ) {
+  async skip(command: MailCommand, options: SkipMailOptions) {
+    const metadata = command.metadata ?? {};
+    const deduped = metadata.dedupeKey
+      ? await this.models.mailDelivery.findByDedupeKey(metadata.dedupeKey)
+      : null;
+    if (deduped) {
+      return false;
+    }
+
+    await this.models.mailDelivery.create({
+      mailName: command.name,
+      mailClass: options.mailClass,
+      priority:
+        options.priority ??
+        metadata.priority ??
+        defaultPriority(options.mailClass),
+      status: 'skipped',
+      dedupeKey: metadata.dedupeKey,
+      recipientEmail: command.to,
+      recipientUserId: metadata.recipientUserId,
+      actorUserId: metadata.actorUserId,
+      workspaceId: metadata.workspaceId,
+      notificationId: metadata.notificationId,
+      abuseSubjectKey: metadata.abuseSubjectKey,
+      payload: serializePayload(command),
+      expiresAt: metadata.expiresAt,
+      maxAttempts: 0,
+      lastErrorCode: options.reason,
+    });
+    return false;
+  }
+
+  async send(command: MailCommand, suppressError = false) {
     if (!this.sender.configured) {
       if (suppressError) {
         return false;
@@ -30,15 +96,95 @@ export class Mailer {
       throw new EmailServiceNotConfigured();
     }
 
+    let reservationId: string | undefined;
+    let deliveryId: string | undefined;
     try {
-      await this.queue.add(
-        'notification.sendMail',
-        Object.assign({}, command, {
-          startTime: Date.now(),
-        }) as Jobs['notification.sendMail']
-      );
+      const metadata = command.metadata ?? {};
+      const deduped = metadata.dedupeKey
+        ? await this.models.mailDelivery.findByDedupeKey(metadata.dedupeKey)
+        : null;
+      if (deduped) {
+        return !['failed', 'canceled', 'skipped'].includes(deduped.status);
+      }
+
+      const decision = await this.runtime.assertMailDeliveryQuotaV1({
+        mailName: command.name as MailName,
+        recipient: {
+          email: command.to,
+          domain: recipientDomain(command.to),
+          userId: metadata.recipientUserId,
+        },
+        metadata: {
+          actorUserId: metadata.actorUserId,
+          workspaceId: metadata.workspaceId,
+          notificationId: metadata.notificationId,
+          abuseSubjectKey: metadata.abuseSubjectKey,
+        },
+        source: metadata.source,
+      });
+      reservationId = decision.reservationId;
+
+      if (!decision.allowed) {
+        await this.models.mailDelivery.create({
+          mailName: command.name,
+          mailClass: decision.mailClass,
+          priority: metadata.priority ?? defaultPriority(decision.mailClass),
+          status: 'skipped',
+          dedupeKey: metadata.dedupeKey,
+          recipientEmail: command.to,
+          recipientUserId: metadata.recipientUserId,
+          actorUserId: metadata.actorUserId,
+          workspaceId: metadata.workspaceId,
+          notificationId: metadata.notificationId,
+          abuseSubjectKey: metadata.abuseSubjectKey,
+          quotaDecision: decision,
+          payload: serializePayload(command),
+          expiresAt: metadata.expiresAt,
+          maxAttempts: metadata.maxAttempts,
+          lastErrorCode: decision.reason ?? 'quota_denied',
+        });
+        return false;
+      }
+
+      const delivery = await this.models.mailDelivery.create({
+        mailName: command.name,
+        mailClass: decision.mailClass,
+        priority: metadata.priority ?? defaultPriority(decision.mailClass),
+        dedupeKey: metadata.dedupeKey,
+        recipientEmail: command.to,
+        recipientUserId: metadata.recipientUserId,
+        actorUserId: metadata.actorUserId,
+        workspaceId: metadata.workspaceId,
+        notificationId: metadata.notificationId,
+        abuseSubjectKey: metadata.abuseSubjectKey,
+        quotaReservationId: decision.reservationId,
+        quotaDecision: decision,
+        payload: serializePayload(command),
+        expiresAt: metadata.expiresAt,
+        maxAttempts: metadata.maxAttempts,
+      });
+      deliveryId = delivery.id;
+      if (decision.reservationId) {
+        if (delivery.quotaReservationId === decision.reservationId) {
+          await this.runtime.commitMailDeliveryQuotaV1(decision.reservationId);
+        } else {
+          await this.runtime.releaseMailDeliveryQuotaV1(decision.reservationId);
+        }
+      }
       return true;
-    } catch {
+    } catch (error) {
+      if (deliveryId) {
+        await this.models.mailDelivery.cancelById(
+          deliveryId,
+          'quota_commit_failed'
+        );
+      }
+      if (reservationId) {
+        await this.runtime.releaseMailDeliveryQuotaV1(reservationId);
+      }
+      if (!suppressError) {
+        throw error;
+      }
       return false;
     }
   }

packages/backend/server/src/core/mail/resolver.ts

@@ -1,15 +1,212 @@
-import { Args, Mutation, Resolver } from '@nestjs/graphql';
+import {
+  Args,
+  Field,
+  Float,
+  InputType,
+  Int,
+  Mutation,
+  ObjectType,
+  Query,
+  Resolver,
+} from '@nestjs/graphql';
 import { GraphQLJSONObject } from 'graphql-scalars';
 
 import { BadRequest } from '../../base';
 import { Renderers } from '../../mails';
+import { Models } from '../../models';
 import { CurrentUser } from '../auth/session';
 import { Admin } from '../common';
+import {
+  TimeBucket,
+  TimeWindow,
+} from '../workspaces/resolvers/analytics-types';
 import { MailSender } from './sender';
 
+@InputType()
+class AdminMailDeliveriesInput {
+  @Field(() => Int, { defaultValue: 24 })
+  hours!: number;
+}
+
+@ObjectType()
+class AdminMailDeliveryPoint {
+  @Field(() => Date)
+  bucket!: Date;
+
+  @Field(() => Int)
+  count!: number;
+}
+
+@ObjectType()
+class AdminMailDeliverySeries {
+  @Field()
+  key!: string;
+
+  @Field()
+  label!: string;
+
+  @Field(() => Int)
+  total!: number;
+
+  @Field(() => [AdminMailDeliveryPoint])
+  points!: AdminMailDeliveryPoint[];
+}
+
+@ObjectType()
+class AdminMailDeliverySummary {
+  @Field(() => Int)
+  total!: number;
+
+  @Field(() => Int)
+  sent!: number;
+
+  @Field(() => Int)
+  failed!: number;
+
+  @Field(() => Int)
+  skipped!: number;
+
+  @Field(() => Int)
+  canceled!: number;
+
+  @Field(() => Int)
+  queued!: number;
+
+  @Field(() => Int)
+  sending!: number;
+
+  @Field(() => Int)
+  retryWait!: number;
+
+  @Field(() => Float)
+  successRate!: number;
+}
+
+@ObjectType()
+class AdminMailDeliveryAnalytics {
+  @Field(() => TimeWindow)
+  window!: TimeWindow;
+
+  @Field(() => AdminMailDeliverySummary)
+  summary!: AdminMailDeliverySummary;
+
+  @Field(() => [AdminMailDeliverySeries])
+  byStatus!: AdminMailDeliverySeries[];
+
+  @Field(() => [AdminMailDeliverySeries])
+  byType!: AdminMailDeliverySeries[];
+
+  @Field(() => [AdminMailDeliverySeries])
+  byOutcome!: AdminMailDeliverySeries[];
+}
+
+const MAIL_STATUSES = [
+  'sent',
+  'failed',
+  'skipped',
+  'canceled',
+  'queued',
+  'sending',
+  'retry_wait',
+] as const;
+
+const STATUS_LABELS = {
+  sent: 'Sent',
+  failed: 'Failed',
+  skipped: 'Skipped',
+  canceled: 'Canceled',
+  queued: 'Queued',
+  sending: 'Sending',
+  retry_wait: 'Retry wait',
+} satisfies Record<(typeof MAIL_STATUSES)[number], string>;
+
+function startOfUtcHour(value: Date) {
+  return new Date(
+    Date.UTC(
+      value.getUTCFullYear(),
+      value.getUTCMonth(),
+      value.getUTCDate(),
+      value.getUTCHours()
+    )
+  );
+}
+
+function startOfUtcDay(value: Date) {
+  return new Date(
+    Date.UTC(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate())
+  );
+}
+
+function addUtcHours(value: Date, hours: number) {
+  return new Date(value.getTime() + hours * 60 * 60 * 1000);
+}
+
+function addUtcDays(value: Date, days: number) {
+  return new Date(value.getTime() + days * 24 * 60 * 60 * 1000);
+}
+
+function normalizeMailWindow(hours: number | undefined) {
+  if (hours !== undefined && hours !== 24 && hours !== 24 * 7) {
+    throw new BadRequest(
+      'Mail delivery analytics window must be 24 or 168 hours.'
+    );
+  }
+  const requestedHours = hours ?? 24;
+  const bucket = requestedHours > 24 ? ('day' as const) : ('hour' as const);
+  const now = new Date();
+  const to =
+    bucket === 'hour'
+      ? addUtcHours(startOfUtcHour(now), 1)
+      : addUtcDays(startOfUtcDay(now), 1);
+  const effectiveSize =
+    bucket === 'hour' ? requestedHours : requestedHours / 24;
+  const from =
+    bucket === 'hour'
+      ? addUtcHours(to, -effectiveSize)
+      : addUtcDays(to, -effectiveSize);
+
+  return {
+    from,
+    to,
+    bucket,
+    requestedHours,
+    effectiveSize,
+  };
+}
+
+function buildBuckets(input: ReturnType<typeof normalizeMailWindow>) {
+  return Array.from({ length: input.effectiveSize }, (_, index) =>
+    input.bucket === 'hour'
+      ? addUtcHours(input.from, index)
+      : addUtcDays(input.from, index)
+  );
+}
+
+function bucketKey(value: Date) {
+  return value.toISOString();
+}
+
+function buildSeries(
+  keys: { key: string; label: string }[],
+  buckets: Date[],
+  counts: Map<string, number>
+): AdminMailDeliverySeries[] {
+  return keys.map(({ key, label }) => {
+    let total = 0;
+    const points = buckets.map(bucket => {
+      const count = counts.get(`${key}:${bucketKey(bucket)}`) ?? 0;
+      total += count;
+      return { bucket, count };
+    });
+    return { key, label, total, points };
+  });
+}
+
 @Admin()
 @Resolver(() => Boolean)
 export class MailResolver {
+  constructor(private readonly models: Models) {}
+
   @Mutation(() => Boolean)
   async sendTestEmail(
     @CurrentUser() user: CurrentUser,
@@ -46,4 +243,111 @@ export class MailResolver {
 
     return true;
   }
+
+  @Query(() => AdminMailDeliveryAnalytics, {
+    description: 'Aggregate mail delivery timeline facts for admin panel',
+  })
+  async adminMailDeliveries(
+    @Args('input', { nullable: true, type: () => AdminMailDeliveriesInput })
+    input?: AdminMailDeliveriesInput
+  ) {
+    const window = normalizeMailWindow(input?.hours);
+    const buckets = buildBuckets(window);
+    const rows = await this.models.mailDelivery.adminAggregate({
+      from: window.from,
+      to: window.to,
+      bucket: window.bucket,
+    });
+
+    const statusCounts = new Map<string, number>();
+    const classCounts = new Map<string, number>();
+    const outcomeCounts = new Map<string, number>();
+    const classTotals = new Map<string, number>();
+    const summary = {
+      total: 0,
+      sent: 0,
+      failed: 0,
+      skipped: 0,
+      canceled: 0,
+      queued: 0,
+      sending: 0,
+      retryWait: 0,
+      successRate: 0,
+    };
+
+    for (const row of rows) {
+      const bucket = bucketKey(row.bucket);
+      summary.total += row.count;
+      if (row.status === 'retry_wait') {
+        summary.retryWait += row.count;
+      } else {
+        summary[row.status] += row.count;
+      }
+
+      statusCounts.set(
+        `${row.status}:${bucket}`,
+        (statusCounts.get(`${row.status}:${bucket}`) ?? 0) + row.count
+      );
+      classCounts.set(
+        `${row.mailClass}:${bucket}`,
+        (classCounts.get(`${row.mailClass}:${bucket}`) ?? 0) + row.count
+      );
+      classTotals.set(
+        row.mailClass,
+        (classTotals.get(row.mailClass) ?? 0) + row.count
+      );
+
+      const outcome =
+        row.status === 'sent'
+          ? 'successful'
+          : row.status === 'queued' ||
+              row.status === 'sending' ||
+              row.status === 'retry_wait'
+            ? 'pending'
+            : 'unsuccessful';
+      outcomeCounts.set(
+        `${outcome}:${bucket}`,
+        (outcomeCounts.get(`${outcome}:${bucket}`) ?? 0) + row.count
+      );
+    }
+
+    const terminal =
+      summary.sent + summary.failed + summary.skipped + summary.canceled;
+    summary.successRate = terminal ? summary.sent / terminal : 0;
+
+    const topClasses = [...classTotals.entries()]
+      .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+      .slice(0, 6)
+      .map(([key]) => ({ key, label: key }));
+
+    return {
+      window: {
+        from: window.from,
+        to: window.to,
+        timezone: 'UTC',
+        bucket: window.bucket === 'hour' ? TimeBucket.Hour : TimeBucket.Day,
+        requestedSize: window.requestedHours,
+        effectiveSize: window.effectiveSize,
+      },
+      summary,
+      byStatus: buildSeries(
+        MAIL_STATUSES.map(status => ({
+          key: status,
+          label: STATUS_LABELS[status],
+        })),
+        buckets,
+        statusCounts
+      ),
+      byType: buildSeries(topClasses, buckets, classCounts),
+      byOutcome: buildSeries(
+        [
+          { key: 'successful', label: 'Successful' },
+          { key: 'unsuccessful', label: 'Unsuccessful' },
+          { key: 'pending', label: 'Pending' },
+        ],
+        buckets,
+        outcomeCounts
+      ),
+    };
+  }
 }

packages/backend/server/src/core/mail/sender.ts

@@ -17,8 +17,24 @@ export type SendOptions = Omit<SendMailOptions, 'to' | 'subject' | 'html'> & {
   html: string;
 };
 
+export type MailSendResult =
+  | {
+      status: 'accepted';
+      providerMessageId?: string;
+      providerResponse?: string;
+      retryable: false;
+    }
+  | {
+      status: 'rejected' | 'failed';
+      providerResponse?: string;
+      retryable: boolean;
+      errorCode?: string;
+      error?: string;
+    };
+
 function configToSMTPOptions(
-  config: AppConfig['mailer']['SMTP']
+  config: AppConfig['mailer']['SMTP'],
+  timeoutMs = 30 * 1000
 ): SMTPTransport.Options {
   const name = resolveSMTPHeloHostname(config.name);
 
@@ -33,6 +49,9 @@ function configToSMTPOptions(
       user: config.username,
       pass: config.password,
     },
+    connectionTimeout: timeoutMs,
+    greetingTimeout: timeoutMs,
+    socketTimeout: timeoutMs,
   };
 }
 
@@ -68,7 +87,11 @@ export class MailSender {
 
   private setup() {
     const { SMTP, fallbackDomains, fallbackSMTP } = this.config.mailer;
-    const opts = configToSMTPOptions(SMTP);
+    const timeoutMs = Math.max(
+      1000,
+      Math.min(30 * 1000, this.config.mailer.deliveryWorker.leaseMs - 1000)
+    );
+    const opts = configToSMTPOptions(SMTP, timeoutMs);
 
     if (SMTP.host) {
       this.smtp = createTransport(opts);
@@ -76,7 +99,9 @@ export class MailSender {
         this.logger.warn(
           `Fallback SMTP is configured for domains: ${fallbackDomains.join(', ')}`
         );
-        this.fallbackSMTP = createTransport(configToSMTPOptions(fallbackSMTP));
+        this.fallbackSMTP = createTransport(
+          configToSMTPOptions(fallbackSMTP, timeoutMs)
+        );
       }
     } else if (env.dev) {
       createTestAccount((err, account) => {
@@ -107,17 +132,25 @@ export class MailSender {
     return [this.smtp, SMTP.sender] as const;
   }
 
-  async send(name: string, options: SendOptions) {
+  async send(name: string, options: SendOptions): Promise<MailSendResult> {
     const [, domain, ...rest] = options.to.split('@');
     if (rest.length || !domain) {
       this.logger.error(`Invalid email address: ${options.to}`);
-      return null;
+      return {
+        status: 'rejected',
+        retryable: false,
+        errorCode: 'invalid_recipient',
+      };
     }
 
     const [smtpClient, from] = this.getSender(domain);
     if (!smtpClient) {
       this.logger.warn(`Mailer SMTP transport is not configured to send mail.`);
-      return null;
+      return {
+        status: 'failed',
+        retryable: true,
+        errorCode: 'smtp_not_configured',
+      };
     }
 
     metrics.mail.counter('send_total').add(1, { name });
@@ -129,7 +162,12 @@ export class MailSender {
         this.logger.error(
           `Mail [${name}] rejected with response: ${result.response}`
         );
-        return false;
+        return {
+          status: 'rejected',
+          providerResponse: result.response,
+          retryable: false,
+          errorCode: 'provider_rejected',
+        };
       }
 
       metrics.mail.counter('accepted_total').add(1, { name });
@@ -140,11 +178,22 @@ export class MailSender {
         );
       }
 
-      return true;
+      return {
+        status: 'accepted',
+        providerMessageId:
+          typeof result.messageId === 'string' ? result.messageId : undefined,
+        providerResponse: result.response,
+        retryable: false,
+      };
     } catch (e) {
       metrics.mail.counter('failed_total').add(1, { name });
       this.logger.error(`Failed to send mail [${name}].`, e);
-      return false;
+      return {
+        status: 'failed',
+        retryable: true,
+        errorCode: 'transport_failed',
+        error: e instanceof Error ? e.message : String(e),
+      };
     }
   }
 }

packages/backend/server/src/core/mail/types.ts

@@ -0,0 +1,45 @@
+import type { MailName, MailProps } from '../../mails';
+import type { UserProps, WorkspaceProps } from '../../mails/components';
+import type { RuntimeQuotaSourceInput } from '../backend-runtime';
+
+export type MailDeliveryMetadata = {
+  dedupeKey?: string;
+  recipientUserId?: string;
+  actorUserId?: string;
+  workspaceId?: string;
+  notificationId?: string;
+  abuseSubjectKey?: string;
+  source?: RuntimeQuotaSourceInput;
+  expiresAt?: Date;
+  maxAttempts?: number;
+  priority?: 'critical' | 'high' | 'normal' | 'low';
+};
+
+export type DynamicallyFetchedProps<Props> = {
+  [Key in keyof Props]: Props[Key] extends infer Prop
+    ? Prop extends UserProps
+      ? {
+          $$userId: string;
+        } & Omit<Prop, 'email' | 'avatar'>
+      : Prop extends WorkspaceProps
+        ? {
+            $$workspaceId: string;
+          } & Omit<Prop, 'name' | 'avatar'>
+        : Prop
+    : never;
+};
+
+export type SendMailPayload<
+  Mail extends MailName = MailName,
+  Props = MailProps<Mail>,
+> = {
+  name: Mail;
+  to: string;
+  props: DynamicallyFetchedProps<Props>;
+};
+
+export type SendMailCommand = {
+  [K in MailName]: SendMailPayload<K>;
+}[MailName] & {
+  metadata?: MailDeliveryMetadata;
+};

packages/backend/server/src/core/notification/__tests__/service.spec.ts

@@ -4,7 +4,7 @@ import { mock } from 'node:test';
 import test from 'ava';
 
 import { createModule } from '../../../__tests__/create-module';
-import { Mockers } from '../../../__tests__/mocks';
+import { Mockers, MockMailer } from '../../../__tests__/mocks';
 import { Due, NotificationNotFound } from '../../../base';
 import {
   DocMode,
@@ -18,6 +18,7 @@ import {
 import { DocStorageModule } from '../../doc';
 import { FeatureModule } from '../../features';
 import { MailModule } from '../../mail';
+import { Mailer } from '../../mail/mailer';
 import { PermissionModule } from '../../permission';
 import { StorageModule } from '../../storage';
 import { NotificationModule } from '../index';
@@ -33,9 +34,13 @@ const module = await createModule({
     NotificationModule,
   ],
   providers: [NotificationService],
+  tapModule: builder => {
+    builder.overrideProvider(Mailer).useValue(new MockMailer());
+  },
 });
 const notificationService = module.get(NotificationService);
 const models = module.get(Models);
+const mailer = module.get(Mailer) as unknown as MockMailer;
 
 let owner: User;
 let member: User;
@@ -82,9 +87,9 @@ test('should create invitation notification and email', async t => {
   t.is(notification!.body.inviteId, inviteId);
 
   // should send invitation email
-  const invitationMail = module.queue.last('notification.sendMail');
-  t.is(invitationMail.payload.to, member.email);
-  t.is(invitationMail.payload.name, 'MemberInvitation');
+  const invitationMail = mailer.last('MemberInvitation');
+  t.is(invitationMail.to, member.email);
+  t.is(invitationMail.name, 'MemberInvitation');
 });
 
 test('should not send invitation email when workspace name contains domain', async t => {
@@ -92,10 +97,11 @@ test('should not send invitation email when workspace name contains domain', asy
     owner: {
       id: owner.id,
     },
-    name: 'BTC https://spam.example',
+    name: 'BTC example.com',
   });
   const inviteId = randomUUID();
-  const invitationMailCount = module.queue.count('notification.sendMail');
+  const invitationMailCount = mailer.send.callCount;
+  const skippedMailCount = mailer.skip.callCount;
 
   const notification = await notificationService.createInvitation({
     userId: member.id,
@@ -107,7 +113,9 @@ test('should not send invitation email when workspace name contains domain', asy
   });
 
   t.truthy(notification);
-  t.is(module.queue.count('notification.sendMail'), invitationMailCount);
+  t.is(mailer.send.callCount, invitationMailCount);
+  t.is(mailer.skip.callCount, skippedMailCount + 1);
+  t.is(mailer.skip.lastCall.args[1].reason, 'workspace_name_contains_domain');
 });
 
 test('should not send invitation email if user setting is not to receive invitation email', async t => {
@@ -116,7 +124,8 @@ test('should not send invitation email if user setting is not to receive invitat
     userId: member.id,
     receiveInvitationEmail: false,
   });
-  const invitationMailCount = module.queue.count('notification.sendMail');
+  const invitationMailCount = mailer.send.callCount;
+  const skippedMailCount = mailer.skip.callCount;
 
   const notification = await notificationService.createInvitation({
     userId: member.id,
@@ -130,7 +139,9 @@ test('should not send invitation email if user setting is not to receive invitat
   t.truthy(notification);
 
   // no new invitation email should be sent
-  t.is(module.queue.count('notification.sendMail'), invitationMailCount);
+  t.is(mailer.send.callCount, invitationMailCount);
+  t.is(mailer.skip.callCount, skippedMailCount + 1);
+  t.is(mailer.skip.lastCall.args[1].reason, 'recipient_notification_disabled');
 });
 
 test('should not create invitation notification if user is already a member', async t => {
@@ -174,9 +185,9 @@ test('should create invitation accepted notification and email', async t => {
   t.is(notification!.body.inviteId, inviteId);
 
   // should send email
-  const invitationAcceptedMail = module.queue.last('notification.sendMail');
-  t.is(invitationAcceptedMail.payload.to, owner.email);
-  t.is(invitationAcceptedMail.payload.name, 'MemberAccepted');
+  const invitationAcceptedMail = mailer.last('MemberAccepted');
+  t.is(invitationAcceptedMail.to, owner.email);
+  t.is(invitationAcceptedMail.name, 'MemberAccepted');
 });
 
 test('should not send invitation accepted email if user settings is not receive invitation email', async t => {
@@ -189,9 +200,7 @@ test('should not send invitation accepted email if user settings is not receive
     userId: owner.id,
     receiveInvitationEmail: false,
   });
-  const invitationAcceptedMailCount = module.queue.count(
-    'notification.sendMail'
-  );
+  const invitationAcceptedMailCount = mailer.send.callCount;
 
   const notification = await notificationService.createInvitationAccepted({
     userId: owner.id,
@@ -205,10 +214,7 @@ test('should not send invitation accepted email if user settings is not receive
   t.truthy(notification);
 
   // no new invitation accepted email should be sent
-  t.is(
-    module.queue.count('notification.sendMail'),
-    invitationAcceptedMailCount
-  );
+  t.is(mailer.send.callCount, invitationAcceptedMailCount);
 });
 
 test('should not create invitation accepted notification if user is not an active member', async t => {
@@ -286,11 +292,11 @@ test('should create invitation review request notification if user is not an act
   t.is(notification!.body.inviteId, inviteId);
 
   // should send email
-  const invitationReviewRequestMail = module.queue.last(
-    'notification.sendMail'
+  const invitationReviewRequestMail = mailer.last(
+    'LinkInvitationReviewRequest'
   );
-  t.is(invitationReviewRequestMail.payload.to, owner.email);
-  t.is(invitationReviewRequestMail.payload.name, 'LinkInvitationReviewRequest');
+  t.is(invitationReviewRequestMail.to, owner.email);
+  t.is(invitationReviewRequestMail.name, 'LinkInvitationReviewRequest');
 });
 
 test('should not create invitation review request notification if user is an active member', async t => {
@@ -336,11 +342,9 @@ test('should create invitation review approved notification if user is an active
   t.is(notification!.body.inviteId, inviteId);
 
   // should send email
-  const invitationReviewApprovedMail = module.queue.last(
-    'notification.sendMail'
-  );
-  t.is(invitationReviewApprovedMail.payload.to, member.email);
-  t.is(invitationReviewApprovedMail.payload.name, 'LinkInvitationApprove');
+  const invitationReviewApprovedMail = mailer.last('LinkInvitationApprove');
+  t.is(invitationReviewApprovedMail.to, member.email);
+  t.is(invitationReviewApprovedMail.name, 'LinkInvitationApprove');
 });
 
 test('should not create invitation review approved notification if user is not an active member', async t => {
@@ -382,11 +386,9 @@ test('should create invitation review declined notification if user is not an ac
   t.is(notification!.body.createdByUserId, owner.id);
 
   // should send email
-  const invitationReviewDeclinedMail = module.queue.last(
-    'notification.sendMail'
-  );
-  t.is(invitationReviewDeclinedMail.payload.to, member.email);
-  t.is(invitationReviewDeclinedMail.payload.name, 'LinkInvitationDecline');
+  const invitationReviewDeclinedMail = mailer.last('LinkInvitationDecline');
+  t.is(invitationReviewDeclinedMail.to, member.email);
+  t.is(invitationReviewDeclinedMail.name, 'LinkInvitationDecline');
 });
 
 test('should not create invitation review declined notification if user is an active member', async t => {
@@ -618,12 +620,12 @@ test('should send mention email by user setting', async t => {
   t.truthy(notification);
 
   // should send mention email
-  const mentionMail = module.queue.last('notification.sendMail');
-  t.is(mentionMail.payload.to, member.email);
-  t.is(mentionMail.payload.name, 'Mention');
+  const mentionMail = mailer.last('Mention');
+  t.is(mentionMail.to, member.email);
+  t.is(mentionMail.name, 'Mention');
 
   // update user setting to not receive mention email
-  const mentionMailCount = module.queue.count('notification.sendMail');
+  const mentionMailCount = mailer.send.callCount;
   await module.create(Mockers.UserSettings, {
     userId: member.id,
     receiveMentionEmail: false,
@@ -644,7 +646,7 @@ test('should send mention email by user setting', async t => {
   });
 
   // should not send mention email
-  t.is(module.queue.count('notification.sendMail'), mentionMailCount);
+  t.is(mailer.send.callCount, mentionMailCount);
 });
 
 test('should send mention email with use client doc title if server doc title is empty', async t => {
@@ -672,11 +674,10 @@ test('should send mention email with use client doc title if server doc title is
 
   t.truthy(notification);
 
-  const mentionMail = module.queue.last('notification.sendMail');
-  t.is(mentionMail.payload.to, member.email);
-  t.is(mentionMail.payload.name, 'Mention');
-  // @ts-expect-error - payload is not typed
-  t.is(mentionMail.payload.props.doc.title, 'doc-title-1');
+  const mentionMail = mailer.last('Mention');
+  t.is(mentionMail.to, member.email);
+  t.is(mentionMail.name, 'Mention');
+  t.is(mentionMail.props.doc.title, 'doc-title-1');
 });
 
 test('should send comment notification and email', async t => {
@@ -699,9 +700,9 @@ test('should send comment notification and email', async t => {
 
   t.truthy(notification);
 
-  const commentMail = module.queue.last('notification.sendMail');
-  t.is(commentMail.payload.to, member.email);
-  t.is(commentMail.payload.name, 'Comment');
+  const commentMail = mailer.last('Comment');
+  t.is(commentMail.to, member.email);
+  t.is(commentMail.name, 'Comment');
 });
 
 test('should send comment mention notification and email', async t => {
@@ -729,9 +730,9 @@ test('should send comment mention notification and email', async t => {
 
   t.truthy(notification);
 
-  const commentMentionMail = module.queue.last('notification.sendMail');
-  t.is(commentMentionMail.payload.to, member.email);
-  t.is(commentMentionMail.payload.name, 'CommentMention');
+  const commentMentionMail = mailer.last('CommentMention');
+  t.is(commentMentionMail.to, member.email);
+  t.is(commentMentionMail.name, 'CommentMention');
 });
 
 test('should send comment email by user setting', async t => {
@@ -753,12 +754,12 @@ test('should send comment email by user setting', async t => {
 
   t.truthy(notification);
 
-  const commentMail = module.queue.last('notification.sendMail');
-  t.is(commentMail.payload.to, member.email);
-  t.is(commentMail.payload.name, 'Comment');
+  const commentMail = mailer.last('Comment');
+  t.is(commentMail.to, member.email);
+  t.is(commentMail.name, 'Comment');
 
   // update user setting to not receive comment email
-  const commentMailCount = module.queue.count('notification.sendMail');
+  const commentMailCount = mailer.send.callCount;
   await module.create(Mockers.UserSettings, {
     userId: member.id,
     receiveCommentEmail: false,
@@ -779,5 +780,5 @@ test('should send comment email by user setting', async t => {
   });
 
   // should not send comment email
-  t.is(module.queue.count('notification.sendMail'), commentMailCount);
+  t.is(mailer.send.callCount, commentMailCount);
 });

packages/backend/server/src/core/notification/service.ts

@@ -15,15 +15,17 @@ import {
   UnionNotificationBody,
   Workspace,
 } from '../../models';
+import { BackendRuntimeProvider } from '../backend-runtime';
+import { containsUrlOrDomain } from '../content-policy';
 import { DocReader } from '../doc';
 import { Mailer } from '../mail';
+import type { SendMailCommand } from '../mail/types';
 import { realtimeNotificationRoom, RealtimePublisher } from '../realtime';
 import { generateDocPath } from '../utils/doc';
 import {
   generateWorkspaceSettingsPath,
   WorkspaceSettingsTab,
 } from '../utils/workspace';
-import { containsUrlOrDomain } from '../workspaces/abuse';
 
 @Injectable()
 export class NotificationService {
@@ -34,7 +36,8 @@ export class NotificationService {
     private readonly docReader: DocReader,
     private readonly mailer: Mailer,
     private readonly url: URLHelper,
-    private readonly realtime: RealtimePublisher
+    private readonly realtime: RealtimePublisher,
+    private readonly runtime: BackendRuntimeProvider
   ) {}
 
   async cleanExpiredNotifications() {
@@ -55,23 +58,21 @@ export class NotificationService {
     const notification = isMention
       ? await this.models.notification.createCommentMention(input)
       : await this.models.notification.createComment(input);
-    await this.sendCommentEmail(input, isMention);
+    await this.sendCommentEmail(input, isMention, notification.id);
     await this.publishCountChanged(input.userId, 'created');
     return notification;
   }
 
   private async sendCommentEmail(
     input: CommentNotificationCreate,
-    isMention?: boolean
+    isMention?: boolean,
+    notificationId?: string
   ) {
-    const userSetting = await this.models.userSettings.get(input.userId);
-    if (!userSetting.receiveCommentEmail) {
-      return;
-    }
     const receiver = await this.models.user.getWorkspaceUser(input.userId);
     if (!receiver) {
       return;
     }
+    const userSetting = await this.models.userSettings.get(input.userId);
     const doc = await this.models.doc.getMeta(
       input.body.workspaceId,
       input.body.doc.id
@@ -88,7 +89,7 @@ export class NotificationService {
         replyId: input.body.replyId,
       })
     );
-    await this.mailer.trySend({
+    const command: SendMailCommand = {
       name: isMention ? 'CommentMention' : 'Comment',
       to: receiver.email,
       props: {
@@ -100,26 +101,44 @@ export class NotificationService {
           url,
         },
       },
-    });
+      metadata: {
+        dedupeKey: notificationId
+          ? `notification:${notificationId}:mail:${isMention ? 'CommentMention' : 'Comment'}`
+          : undefined,
+        recipientUserId: receiver.id,
+        actorUserId: input.body.createdByUserId,
+        workspaceId: input.body.workspaceId,
+        notificationId,
+        source: { trusted: false },
+      },
+    };
+    if (!userSetting.receiveCommentEmail) {
+      await this.mailer.skip(command, {
+        mailClass: 'collaboration_notice',
+        reason: 'recipient_notification_disabled',
+      });
+      return;
+    }
+    await this.trySendMail(command, 'collaboration_notice');
     this.logger.debug(`Comment email sent to user ${receiver.id}`);
   }
 
   async createMention(input: MentionNotificationCreate) {
     const notification = await this.models.notification.createMention(input);
-    await this.sendMentionEmail(input);
+    await this.sendMentionEmail(input, notification.id);
     await this.publishCountChanged(input.userId, 'created');
     return notification;
   }
 
-  private async sendMentionEmail(input: MentionNotificationCreate) {
-    const userSetting = await this.models.userSettings.get(input.userId);
-    if (!userSetting.receiveMentionEmail) {
-      return;
-    }
+  private async sendMentionEmail(
+    input: MentionNotificationCreate,
+    notificationId?: string
+  ) {
     const receiver = await this.models.user.getWorkspaceUser(input.userId);
     if (!receiver) {
       return;
     }
+    const userSetting = await this.models.userSettings.get(input.userId);
     const doc = await this.models.doc.getMeta(
       input.body.workspaceId,
       input.body.doc.id
@@ -134,7 +153,7 @@ export class NotificationService {
         elementId: input.body.doc.elementId,
       })
     );
-    await this.mailer.trySend({
+    const command: SendMailCommand = {
       name: 'Mention',
       to: receiver.email,
       props: {
@@ -146,7 +165,25 @@ export class NotificationService {
           url,
         },
       },
-    });
+      metadata: {
+        dedupeKey: notificationId
+          ? `notification:${notificationId}:mail:Mention`
+          : undefined,
+        recipientUserId: receiver.id,
+        actorUserId: input.body.createdByUserId,
+        workspaceId: input.body.workspaceId,
+        notificationId,
+        source: { trusted: false },
+      },
+    };
+    if (!userSetting.receiveMentionEmail) {
+      await this.mailer.skip(command, {
+        mailClass: 'collaboration_notice',
+        reason: 'recipient_notification_disabled',
+      });
+      return;
+    }
+    await this.trySendMail(command, 'collaboration_notice');
     this.logger.debug(`Mention email sent to user ${receiver.id}`);
   }
 
@@ -161,36 +198,25 @@ export class NotificationService {
       input,
       NotificationType.Invitation
     );
-    await this.sendInvitationEmail(input);
+    await this.sendInvitationEmail(input, notification.id);
     await this.publishCountChanged(input.userId, 'created');
     return notification;
   }
 
-  private async sendInvitationEmail(input: InvitationNotificationCreate) {
-    const workspace = await this.docReader.getWorkspaceContent(
-      input.body.workspaceId
-    );
-    if (containsUrlOrDomain(workspace?.name)) {
-      this.logger.warn(
-        `Skip invitation email for workspace ${input.body.workspaceId}, reason=workspace name contains url or domain`
-      );
-      return;
-    }
-
+  private async sendInvitationEmail(
+    input: InvitationNotificationCreate,
+    notificationId?: string
+  ) {
     const inviteUrl = this.url.link(`/invite/${input.body.inviteId}`);
     if (env.dev) {
       // make it easier to test in dev mode
       this.logger.debug(`Invite link: ${inviteUrl}`);
     }
-    const userSetting = await this.models.userSettings.get(input.userId);
-    if (!userSetting.receiveInvitationEmail) {
-      return;
-    }
     const receiver = await this.models.user.getWorkspaceUser(input.userId);
     if (!receiver) {
       return;
     }
-    await this.mailer.trySend({
+    const command: SendMailCommand = {
       name: 'MemberInvitation',
       to: receiver.email,
       props: {
@@ -202,7 +228,40 @@ export class NotificationService {
         },
         url: inviteUrl,
       },
-    });
+      metadata: {
+        dedupeKey: notificationId
+          ? `notification:${notificationId}:mail:MemberInvitation`
+          : `invite:${input.body.inviteId}:mail:MemberInvitation`,
+        recipientUserId: receiver.id,
+        actorUserId: input.body.createdByUserId,
+        workspaceId: input.body.workspaceId,
+        notificationId,
+        source: { trusted: false },
+      },
+    };
+    const workspace = await this.docReader.getWorkspaceContent(
+      input.body.workspaceId
+    );
+    if (containsUrlOrDomain(workspace?.name)) {
+      this.logger.warn(
+        `Skip invitation email for workspace ${input.body.workspaceId}, reason=workspace name contains url or domain`
+      );
+      await this.mailer.skip(command, {
+        mailClass: 'workspace_invitation',
+        reason: 'workspace_name_contains_domain',
+      });
+      return;
+    }
+
+    const userSetting = await this.models.userSettings.get(input.userId);
+    if (!userSetting.receiveInvitationEmail) {
+      await this.mailer.skip(command, {
+        mailClass: 'workspace_invitation',
+        reason: 'recipient_notification_disabled',
+      });
+      return;
+    }
+    await this.trySendMail(command, 'workspace_invitation');
     this.logger.debug(
       `Invitation email sent to user ${receiver.id} for workspace ${input.body.workspaceId}`
     );
@@ -223,26 +282,23 @@ export class NotificationService {
       input,
       NotificationType.InvitationAccepted
     );
-    await this.sendInvitationAcceptedEmail(input);
+    await this.sendInvitationAcceptedEmail(input, notification.id);
     await this.publishCountChanged(input.userId, 'created');
     return notification;
   }
 
   private async sendInvitationAcceptedEmail(
-    input: InvitationNotificationCreate
+    input: InvitationNotificationCreate,
+    notificationId?: string
   ) {
     const inviterUserId = input.userId;
     const inviteeUserId = input.body.createdByUserId;
     const workspaceId = input.body.workspaceId;
-    const userSetting = await this.models.userSettings.get(inviterUserId);
-    if (!userSetting.receiveInvitationEmail) {
-      return;
-    }
     const inviter = await this.models.user.getWorkspaceUser(inviterUserId);
     if (!inviter) {
       return;
     }
-    await this.mailer.trySend({
+    const command: SendMailCommand = {
       name: 'MemberAccepted',
       to: inviter.email,
       props: {
@@ -259,7 +315,26 @@ export class NotificationService {
           })
         ),
       },
-    });
+      metadata: {
+        dedupeKey: notificationId
+          ? `notification:${notificationId}:mail:MemberAccepted`
+          : undefined,
+        recipientUserId: inviter.id,
+        actorUserId: inviteeUserId,
+        workspaceId,
+        notificationId,
+        source: { trusted: false },
+      },
+    };
+    const userSetting = await this.models.userSettings.get(inviterUserId);
+    if (!userSetting.receiveInvitationEmail) {
+      await this.mailer.skip(command, {
+        mailClass: 'workspace_lifecycle',
+        reason: 'recipient_notification_disabled',
+      });
+      return;
+    }
+    await this.trySendMail(command, 'workspace_lifecycle');
     this.logger.debug(
       `Invitation accepted email sent to user ${inviter.id} for workspace ${workspaceId}`
     );
@@ -297,13 +372,14 @@ export class NotificationService {
       input,
       NotificationType.InvitationReviewRequest
     );
-    await this.sendInvitationReviewRequestEmail(input);
+    await this.sendInvitationReviewRequestEmail(input, notification.id);
     await this.publishCountChanged(input.userId, 'created');
     return notification;
   }
 
   private async sendInvitationReviewRequestEmail(
-    input: InvitationNotificationCreate
+    input: InvitationNotificationCreate,
+    notificationId?: string
   ) {
     const inviteeUserId = input.body.createdByUserId;
     const reviewerUserId = input.userId;
@@ -312,24 +388,37 @@ export class NotificationService {
     if (!reviewer) {
       return;
     }
-    await this.mailer.trySend({
-      name: 'LinkInvitationReviewRequest',
-      to: reviewer.email,
-      props: {
-        user: {
-          $$userId: inviteeUserId,
+    await this.trySendMail(
+      {
+        name: 'LinkInvitationReviewRequest',
+        to: reviewer.email,
+        props: {
+          user: {
+            $$userId: inviteeUserId,
+          },
+          workspace: {
+            $$workspaceId: workspaceId,
+          },
+          url: this.url.link(
+            generateWorkspaceSettingsPath({
+              workspaceId,
+              tab: WorkspaceSettingsTab.members,
+            })
+          ),
         },
-        workspace: {
-          $$workspaceId: workspaceId,
+        metadata: {
+          dedupeKey: notificationId
+            ? `notification:${notificationId}:mail:LinkInvitationReviewRequest`
+            : undefined,
+          recipientUserId: reviewer.id,
+          actorUserId: inviteeUserId,
+          workspaceId,
+          notificationId,
+          source: { trusted: false },
         },
-        url: this.url.link(
-          generateWorkspaceSettingsPath({
-            workspaceId,
-            tab: WorkspaceSettingsTab.members,
-          })
-        ),
       },
-    });
+      'workspace_invitation'
+    );
     this.logger.debug(
       `Invitation review request email sent to user ${reviewer.id} for workspace ${workspaceId}`
     );
@@ -346,13 +435,14 @@ export class NotificationService {
       input,
       NotificationType.InvitationReviewApproved
     );
-    await this.sendInvitationReviewApprovedEmail(input);
+    await this.sendInvitationReviewApprovedEmail(input, notification.id);
     await this.publishCountChanged(input.userId, 'created');
     return notification;
   }
 
   private async sendInvitationReviewApprovedEmail(
-    input: InvitationNotificationCreate
+    input: InvitationNotificationCreate,
+    notificationId?: string
   ) {
     const workspaceId = input.body.workspaceId;
     const receiverUserId = input.userId;
@@ -360,16 +450,29 @@ export class NotificationService {
     if (!receiver) {
       return;
     }
-    await this.mailer.trySend({
-      name: 'LinkInvitationApprove',
-      to: receiver.email,
-      props: {
-        workspace: {
-          $$workspaceId: workspaceId,
+    await this.trySendMail(
+      {
+        name: 'LinkInvitationApprove',
+        to: receiver.email,
+        props: {
+          workspace: {
+            $$workspaceId: workspaceId,
+          },
+          url: this.url.link(`/workspace/${workspaceId}`),
+        },
+        metadata: {
+          dedupeKey: notificationId
+            ? `notification:${notificationId}:mail:LinkInvitationApprove`
+            : undefined,
+          recipientUserId: receiver.id,
+          actorUserId: input.body.createdByUserId,
+          workspaceId,
+          notificationId,
+          source: { trusted: false },
         },
-        url: this.url.link(`/workspace/${workspaceId}`),
       },
-    });
+      'workspace_invitation'
+    );
     this.logger.debug(
       `Invitation review approved email sent to user ${receiver.id} for workspace ${workspaceId}`
     );
@@ -386,13 +489,14 @@ export class NotificationService {
     await this.ensureWorkspaceContentExists(workspaceId);
     const notification =
       await this.models.notification.createInvitationReviewDeclined(input);
-    await this.sendInvitationReviewDeclinedEmail(input);
+    await this.sendInvitationReviewDeclinedEmail(input, notification.id);
     await this.publishCountChanged(input.userId, 'created');
     return notification;
   }
 
   private async sendInvitationReviewDeclinedEmail(
-    input: InvitationReviewDeclinedNotificationCreate
+    input: InvitationReviewDeclinedNotificationCreate,
+    notificationId?: string
   ) {
     const workspaceId = input.body.workspaceId;
     const receiverUserId = input.userId;
@@ -400,15 +504,28 @@ export class NotificationService {
     if (!receiver) {
       return;
     }
-    await this.mailer.trySend({
-      name: 'LinkInvitationDecline',
-      to: receiver.email,
-      props: {
-        workspace: {
-          $$workspaceId: workspaceId,
+    await this.trySendMail(
+      {
+        name: 'LinkInvitationDecline',
+        to: receiver.email,
+        props: {
+          workspace: {
+            $$workspaceId: workspaceId,
+          },
+        },
+        metadata: {
+          dedupeKey: notificationId
+            ? `notification:${notificationId}:mail:LinkInvitationDecline`
+            : undefined,
+          recipientUserId: receiver.id,
+          actorUserId: input.body.createdByUserId,
+          workspaceId,
+          notificationId,
+          source: { trusted: false },
         },
       },
-    });
+      'workspace_invitation'
+    );
     this.logger.debug(
       `Invitation review declined email sent to user ${receiver.id} for workspace ${workspaceId}`
     );
@@ -539,4 +656,32 @@ export class NotificationService {
     );
     return !!isActive;
   }
+
+  private async trySendMail(command: SendMailCommand, mailClass: string) {
+    const actorUserId = command.metadata?.actorUserId;
+    if (
+      actorUserId &&
+      (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(actorUserId))
+    ) {
+      await this.mailer.skip(command, {
+        mailClass,
+        reason: 'actor_quarantined',
+      });
+      return false;
+    }
+
+    const workspaceId = command.metadata?.workspaceId;
+    if (
+      workspaceId &&
+      (await this.runtime.isInviteAbuseWorkspaceQuarantined(workspaceId))
+    ) {
+      await this.mailer.skip(command, {
+        mailClass,
+        reason: 'workspace_quarantined',
+      });
+      return false;
+    }
+
+    return await this.mailer.trySend(command);
+  }
 }

packages/backend/server/src/core/workspaces/__tests__/abuse.spec.ts

@@ -1,36 +1,390 @@
+import { createHash } from 'node:crypto';
+
+import {
+  createInviteLinkMutation,
+  inviteByEmailsMutation,
+  WorkspaceInviteLinkExpireTime,
+} from '@affine/graphql';
+import { PrismaClient, WorkspaceMemberStatus } from '@prisma/client';
 import test from 'ava';
+import type { Request } from 'express';
+import Sinon from 'sinon';
 
-import { canUserExecuteLimitedActions, containsUrlOrDomain } from '../abuse';
+import { createApp, type TestingApp } from '../../../__tests__/e2e/test';
+import { Mockers } from '../../../__tests__/mocks';
+import { Config } from '../../../base';
+import { ActionForbidden, TooManyRequest } from '../../../base/error';
+import { Models, WorkspaceRole } from '../../../models';
+import {
+  getAbuseRequestSource,
+  InviteAbuseDispositionService,
+  InviteQuotaAssertService,
+} from '../abuse';
 
-test('should detect links and bare domains in workspace names', t => {
-  t.true(containsUrlOrDomain('BTC https://spam.example'));
-  t.true(containsUrlOrDomain('Join spam.example now'));
-  t.true(containsUrlOrDomain('Join spam.example, ltd'));
-  t.true(containsUrlOrDomain('Join spam.example。'));
-  t.true(containsUrlOrDomain('www.spam.example'));
+let app: TestingApp;
+const quota = {
+  assertWorkspaceInviteQuota: Sinon.stub(),
+  commitWorkspaceInviteQuota: Sinon.stub(),
+  releaseWorkspaceInviteQuota: Sinon.stub(),
+};
+
+function workspaceSubjectKey(workspaceId: string) {
+  return `workspace:v1:${createHash('sha256').update(workspaceId).digest('hex').slice(0, 24)}`;
+}
+
+test.before(async () => {
+  app = await createApp({
+    tapModule: builder => {
+      builder.overrideProvider(InviteQuotaAssertService).useValue(quota);
+    },
+  });
 });
 
-test('should not detect email addresses or partial domain words', t => {
-  t.false(containsUrlOrDomain('Contact user@spam.example'));
-  t.false(containsUrlOrDomain('spam.example_btc'));
+test.beforeEach(() => {
+  quota.assertWorkspaceInviteQuota.reset();
+  quota.commitWorkspaceInviteQuota.reset();
+  quota.releaseWorkspaceInviteQuota.reset();
 });
 
-test('should check account age for share actions', t => {
-  const minimumAccountAgeMs = 24 * 60 * 60 * 1000;
+test.after.always(async () => {
+  await app.close();
+});
 
-  t.false(
-    canUserExecuteLimitedActions({ createdAt: new Date() }, minimumAccountAgeMs)
-  );
-  t.true(
-    canUserExecuteLimitedActions(
-      {
-        createdAt: new Date(Date.now() - minimumAccountAgeMs - 1),
+test('invite quota rejection has no invite side effects', async t => {
+  quota.assertWorkspaceInviteQuota.rejects(new TooManyRequest());
+  const models = app.get(Models);
+  const owner = await app.create(Mockers.User);
+  const workspace = await app.create(Mockers.Workspace, {
+    owner: { id: owner.id },
+  });
+  const targetEmail = `quota-${Date.now()}@example.com`;
+
+  await app.login(owner);
+  await t.throwsAsync(
+    app.gql({
+      query: inviteByEmailsMutation,
+      variables: {
+        workspaceId: workspace.id,
+        emails: [targetEmail],
       },
-      minimumAccountAgeMs
-    )
+    })
+  );
+
+  t.is(await models.user.getUserByEmail(targetEmail), null);
+  t.is(await models.workspaceUser.count(workspace.id), 1);
+  t.is(app.mails.send.callCount, 0);
+  t.is(app.queue.count('notification.sendInvitation'), 0);
+  t.is(quota.commitWorkspaceInviteQuota.callCount, 0);
+  t.is(quota.releaseWorkspaceInviteQuota.callCount, 0);
+});
+
+test('abuse request source trusts Cloudflare facts only when configured', t => {
+  const config = app.get(Config);
+  const previousTrusted = config.auth.trustedCloudflareHeaders;
+  const req = {
+    ip: '10.0.0.1',
+    get(name: string) {
+      return (
+        {
+          'CF-Connecting-IP': '114.51.41.91',
+          'CF-IPCountry': 'JP',
+          'CF-Ray': 'ray-id',
+          'x-affine-cf-asn': '4294967295',
+          'X-Forwarded-For': '198.51.100.9',
+        } satisfies Record<string, string>
+      )[name];
+    },
+  } as Request;
+
+  try {
+    config.auth.trustedCloudflareHeaders = false;
+    t.deepEqual(getAbuseRequestSource(req, config), { trusted: false });
+
+    config.auth.trustedCloudflareHeaders = true;
+    t.deepEqual(getAbuseRequestSource(req, config), {
+      trusted: true,
+      ip: '114.51.41.91',
+      country: 'JP',
+      asn: 4294967295,
+      rayId: 'ray-id',
+    });
+  } finally {
+    config.auth.trustedCloudflareHeaders = previousTrusted;
+  }
+});
+
+test('invite quota rejection keeps mapped response when disposition fails', async t => {
+  const service = new InviteQuotaAssertService(
+    app.get(Config),
+    {
+      getWorkspaceSeatQuota: Sinon.stub().resolves({
+        memberLimit: 10,
+        memberCount: 1,
+      }),
+    } as any,
+    {
+      assertWorkspaceInviteQuotaV1: Sinon.stub().resolves({
+        allowed: false,
+        reason: 'abuse_subject',
+        actionRequired: {
+          action: 'quarantine_actor',
+          actionId: '1',
+          subjectKey: 'subject',
+          evidenceId: '1',
+        },
+      }),
+    } as any,
+    {
+      execute: Sinon.stub().rejects(new Error('disposition failed')),
+    } as any
+  );
+
+  await t.throwsAsync(
+    service.assertWorkspaceInviteQuota({
+      actorUserId: 'actor',
+      workspaceId: 'workspace',
+      targetCount: 1,
+      targetDomains: [{ domain: 'example.com', count: 1 }],
+    }),
+    { instanceOf: ActionForbidden }
   );
 });
 
-test('should skip account age check when share action delay is disabled', t => {
-  t.true(canUserExecuteLimitedActions({ createdAt: new Date() }, 0));
+test('abuse disposition applies action scope to invitation artifacts', async t => {
+  const models = app.get(Models);
+  const db = app.get(PrismaClient);
+  const disposition = app.get(InviteAbuseDispositionService);
+
+  for (const scenario of [
+    {
+      name: 'actor',
+      subjectKey: 'actor-subject',
+      subjectKind: 'actor_email',
+      action: 'quarantine_actor',
+    },
+    {
+      name: 'workspace',
+      subjectKey: 'workspace-subject',
+      subjectKind: 'workspace',
+      action: 'quarantine_workspace',
+    },
+  ] as const) {
+    const actor = await app.create(Mockers.User);
+    const invitee = await app.create(Mockers.User);
+    const workspace = await app.create(Mockers.Workspace, { owner: actor });
+    const anotherWorkspace = await app.create(Mockers.Workspace, {
+      owner: actor,
+    });
+    await models.workspaceInvitation.set(
+      workspace.id,
+      invitee.id,
+      WorkspaceRole.Collaborator,
+      WorkspaceMemberStatus.Pending,
+      { inviterId: actor.id }
+    );
+    await models.workspaceInvitation.set(
+      anotherWorkspace.id,
+      invitee.id,
+      WorkspaceRole.Collaborator,
+      WorkspaceMemberStatus.Pending,
+      { inviterId: actor.id }
+    );
+    const canceledDelivery = await models.mailDelivery.create({
+      mailName: 'MemberInvitation',
+      mailClass: 'workspace_invitation',
+      priority: 'normal',
+      recipientEmail: invitee.email,
+      actorUserId: actor.id,
+      workspaceId: workspace.id,
+      abuseSubjectKey: scenario.subjectKey,
+      payload: {
+        name: 'MemberInvitation',
+        to: invitee.email,
+        props: {
+          url: 'https://affine.pro/invite',
+          user: { $$userId: actor.id },
+          workspace: { $$workspaceId: workspace.id },
+        },
+      },
+    });
+    const otherDelivery =
+      scenario.name === 'workspace'
+        ? await models.mailDelivery.create({
+            mailName: 'MemberInvitation',
+            mailClass: 'workspace_invitation',
+            priority: 'normal',
+            recipientEmail: invitee.email,
+            actorUserId: actor.id,
+            workspaceId: anotherWorkspace.id,
+            abuseSubjectKey: 'other-workspace-subject',
+            payload: {
+              name: 'MemberInvitation',
+              to: invitee.email,
+              props: {
+                url: 'https://affine.pro/invite',
+                user: { $$userId: actor.id },
+                workspace: { $$workspaceId: anotherWorkspace.id },
+              },
+            },
+          })
+        : null;
+    const [{ id: actionId }] = await db.$queryRaw<Array<{ id: bigint }>>`
+      WITH subject AS (
+        INSERT INTO runtime_invite_abuse_subjects (
+          subject_key,
+          kind,
+          user_id,
+          actor_email_hash,
+          status,
+          first_seen_at,
+          last_seen_at
+        )
+        VALUES (${scenario.subjectKey}, ${scenario.subjectKind}, ${actor.id}, 'hash', 'quarantined', now(), now())
+        ON CONFLICT (subject_key) DO NOTHING
+        RETURNING subject_key
+      ),
+      evidence AS (
+        INSERT INTO runtime_invite_abuse_evidence (
+          subject_key,
+          workspace_id,
+          user_id,
+          actor_email_hash,
+          decision,
+          reason
+        )
+        VALUES (${scenario.subjectKey}, ${workspace.id}, ${actor.id}, 'hash', ${scenario.action}, 'test')
+        RETURNING id
+      )
+      INSERT INTO runtime_invite_abuse_actions (
+        subject_key,
+        evidence_id,
+        action,
+        status
+      )
+      SELECT ${scenario.subjectKey}, evidence.id, ${scenario.action}, 'pending'
+      FROM evidence
+      RETURNING id
+    `;
+
+    await disposition.execute({
+      actorUserId: actor.id,
+      workspaceId: workspace.id,
+      actionRequired: {
+        action: scenario.action,
+        subjectKey: scenario.subjectKey,
+        evidenceId: '1',
+        actionId: actionId.toString(),
+      },
+    });
+
+    if (scenario.name === 'actor') {
+      t.is(
+        await db.workspaceInvitation.count({
+          where: { inviterUserId: actor.id },
+        }),
+        0,
+        scenario.action
+      );
+    } else {
+      t.is(
+        await db.workspaceInvitation.count({
+          where: { workspaceId: workspace.id },
+        }),
+        0,
+        scenario.action
+      );
+      t.is(
+        await db.workspaceInvitation.count({
+          where: { workspaceId: anotherWorkspace.id },
+        }),
+        1,
+        scenario.action
+      );
+    }
+    t.is(
+      (
+        await db.mailDelivery.findUniqueOrThrow({
+          where: { id: canceledDelivery.id },
+        })
+      ).status,
+      'canceled',
+      scenario.action
+    );
+    if (otherDelivery) {
+      t.is(
+        (
+          await db.mailDelivery.findUniqueOrThrow({
+            where: { id: otherDelivery.id },
+          })
+        ).status,
+        'queued',
+        scenario.action
+      );
+    }
+  }
+});
+
+test('workspace quarantine blocks invite link creation', async t => {
+  const db = app.get(PrismaClient);
+  const config = app.get(Config);
+  const owner = await app.create(Mockers.User);
+  const workspace = await app.create(Mockers.Workspace, { owner });
+  const subjectKey = workspaceSubjectKey(workspace.id);
+  await db.$executeRaw`
+    INSERT INTO runtime_invite_abuse_subjects (
+      subject_key,
+      kind,
+      status,
+      first_seen_at,
+      last_seen_at
+    )
+    VALUES (${subjectKey}, 'workspace', 'quarantined', now(), now())
+    ON CONFLICT (subject_key)
+    DO UPDATE SET
+      status = 'quarantined',
+      updated_at = now()
+  `;
+
+  const previousDelay = config.auth.newAccountShareActionDelay;
+  config.auth.newAccountShareActionDelay = 0;
+  try {
+    await app.login(owner);
+    await t.throwsAsync(
+      app.gql({
+        query: createInviteLinkMutation,
+        variables: {
+          workspaceId: workspace.id,
+          expireTime: WorkspaceInviteLinkExpireTime.OneDay,
+        },
+      })
+    );
+  } finally {
+    config.auth.newAccountShareActionDelay = previousDelay;
+  }
+});
+
+test('domain workspace name blocks invite link creation', async t => {
+  const config = app.get(Config);
+  const owner = await app.create(Mockers.User);
+  const workspace = await app.create(Mockers.Workspace, {
+    owner,
+    name: 'Join example.com',
+  });
+
+  const previousDelay = config.auth.newAccountShareActionDelay;
+  config.auth.newAccountShareActionDelay = 0;
+  try {
+    await app.login(owner);
+    await t.throwsAsync(
+      app.gql({
+        query: createInviteLinkMutation,
+        variables: {
+          workspaceId: workspace.id,
+          expireTime: WorkspaceInviteLinkExpireTime.OneDay,
+        },
+      })
+    );
+  } finally {
+    config.auth.newAccountShareActionDelay = previousDelay;
+  }
 });

packages/backend/server/src/core/workspaces/abuse.ts

@@ -1,8 +1,41 @@
-const URL_OR_DOMAIN_PATTERN =
-  /(?:https?:\/\/|www\.|(?<![@\w-])(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}(?=$|[^\p{L}\p{N}._-]))/iu;
+import { createHash, randomUUID } from 'node:crypto';
 
-export function containsUrlOrDomain(value: string | null | undefined) {
-  return URL_OR_DOMAIN_PATTERN.test(value ?? '');
+import { Injectable, Logger } from '@nestjs/common';
+import { Cron, CronExpression } from '@nestjs/schedule';
+import type { Request } from 'express';
+
+import {
+  ActionForbidden,
+  Config,
+  getRequestClientIp,
+  JobQueue,
+  metrics,
+  OnJob,
+  TooManyRequest,
+  type UserFriendlyError,
+} from '../../base';
+import { Models } from '../../models';
+import {
+  BackendRuntimeProvider,
+  type RuntimeQuotaSourceInput,
+  type RuntimeQuotaTargetDomainInput,
+  type RuntimeWorkspaceInviteQuotaDecision,
+} from '../backend-runtime';
+import { QuotaService } from '../quota/service';
+
+type ActionRequired = NonNullable<
+  RuntimeWorkspaceInviteQuotaDecision['actionRequired']
+>;
+
+export type InviteQuotaAdmission = {
+  reservationId?: string;
+  decision: RuntimeWorkspaceInviteQuotaDecision;
+};
+
+declare global {
+  interface Jobs {
+    'inviteAbuse.executePendingActions': {};
+  }
 }
 
 export function canUserExecuteLimitedActions(
@@ -12,3 +45,344 @@ export function canUserExecuteLimitedActions(
   if (minimumAccountAgeMs <= 0) return true;
   return Date.now() - user.createdAt.getTime() >= minimumAccountAgeMs;
 }
+
+function parseAsn(value: string | undefined) {
+  if (!value) {
+    return;
+  }
+  const asn = Number(value);
+  return Number.isSafeInteger(asn) && asn > 0 && asn <= 0xffffffff
+    ? asn
+    : undefined;
+}
+
+export function getAbuseRequestSource(
+  req: Request | undefined,
+  config: Config
+): RuntimeQuotaSourceInput {
+  if (!req || !config.auth.trustedCloudflareHeaders) {
+    return { trusted: false };
+  }
+
+  return {
+    trusted: true,
+    ip: getRequestClientIp(req),
+    country: req.get('CF-IPCountry')?.trim() || undefined,
+    asn: parseAsn(req.get('x-affine-cf-asn')),
+    rayId: req.get('CF-Ray')?.trim() || undefined,
+  };
+}
+
+function hashLogValue(value: string | undefined) {
+  if (!value) {
+    return undefined;
+  }
+  return createHash('sha256').update(value).digest('hex').slice(0, 24);
+}
+
+@Injectable()
+export class InviteAbuseDispositionService {
+  private readonly logger = new Logger(InviteAbuseDispositionService.name);
+  private readonly workerOwnerId = randomUUID();
+  private readonly inlineWorkerId = `node-inline:${this.workerOwnerId}`;
+
+  constructor(
+    private readonly models: Models,
+    private readonly runtime: BackendRuntimeProvider,
+    private readonly queue: JobQueue
+  ) {}
+
+  async execute(input: {
+    actionRequired?: ActionRequired;
+    actorUserId: string;
+    workspaceId: string;
+    alreadyClaimed?: boolean;
+    workerId?: string;
+  }) {
+    const { actionRequired } = input;
+    if (!actionRequired) {
+      return;
+    }
+    const workerId = input.workerId ?? this.inlineWorkerId;
+
+    try {
+      if (!input.alreadyClaimed) {
+        const claimed = await this.runtime.claimInviteAbuseAction(
+          actionRequired.actionId,
+          workerId
+        );
+        if (!claimed) {
+          return;
+        }
+      }
+
+      switch (actionRequired.action) {
+        case 'ban_actor':
+          await this.cancelPendingActorArtifacts(input, actionRequired);
+          await this.models.session.deleteUserSessions(input.actorUserId);
+          await this.models.user.ban(input.actorUserId);
+          break;
+        case 'quarantine_actor':
+          await this.cancelPendingActorArtifacts(input, actionRequired);
+          await this.models.session.deleteUserSessions(input.actorUserId);
+          break;
+        case 'quarantine_workspace':
+          await this.cancelPendingWorkspaceArtifacts(input.workspaceId);
+          break;
+        case 'quarantine_source_cohort':
+          await this.models.mailDelivery.cancelByAbuseSubject(
+            actionRequired.subjectKey,
+            'source_cohort_quarantined'
+          );
+          break;
+        default:
+          throw new Error(
+            `Unknown invite abuse action: ${actionRequired.action}`
+          );
+      }
+
+      await this.runtime.markInviteAbuseAction(
+        actionRequired.actionId,
+        workerId,
+        'succeeded'
+      );
+    } catch (error) {
+      const message =
+        error instanceof Error ? error.message : 'Failed to execute action.';
+      await this.runtime.markInviteAbuseAction(
+        actionRequired.actionId,
+        workerId,
+        'failed',
+        message
+      );
+      this.logger.error('Failed to execute invite abuse disposition', {
+        userId: input.actorUserId,
+        workspaceId: input.workspaceId,
+        action: actionRequired.action,
+        actionId: actionRequired.actionId,
+        evidenceId: actionRequired.evidenceId,
+        subjectKey: actionRequired.subjectKey,
+        error: message,
+      });
+      throw error;
+    }
+  }
+
+  @Cron(CronExpression.EVERY_MINUTE)
+  async enqueuePendingActions() {
+    await this.queue.add(
+      'inviteAbuse.executePendingActions',
+      {},
+      { jobId: 'invite-abuse-execute-pending-actions' }
+    );
+  }
+
+  @OnJob('inviteAbuse.executePendingActions')
+  async executePendingActions() {
+    const workerId = `node:${this.workerOwnerId}`;
+    const actions = await this.runtime.claimRetryableInviteAbuseActions(
+      workerId,
+      20
+    );
+    for (const action of actions) {
+      await this.execute({
+        actorUserId: action.actorUserId,
+        workspaceId: action.workspaceId,
+        alreadyClaimed: true,
+        workerId,
+        actionRequired: {
+          action: action.action,
+          subjectKey: action.subjectKey,
+          evidenceId: action.evidenceId,
+          actionId: action.actionId,
+        },
+      });
+    }
+  }
+
+  private async cancelPendingActorArtifacts(
+    input: {
+      actorUserId: string;
+      workspaceId: string;
+    },
+    action: ActionRequired
+  ) {
+    await this.models.mailDelivery.cancelByActor(input.actorUserId);
+    await this.models.mailDelivery.cancelByWorkspace(input.workspaceId);
+    await this.models.mailDelivery.cancelByAbuseSubject(action.subjectKey);
+
+    await this.models.workspaceInvitation.cancelPendingByActor(
+      input.actorUserId
+    );
+  }
+
+  private async cancelPendingWorkspaceArtifacts(workspaceId: string) {
+    await this.models.mailDelivery.cancelByWorkspace(workspaceId);
+    await this.models.workspaceInvitation.cancelPendingByWorkspace(workspaceId);
+  }
+}
+
+@Injectable()
+export class InviteQuotaAssertService {
+  private readonly logger = new Logger(InviteQuotaAssertService.name);
+
+  constructor(
+    private readonly config: Config,
+    private readonly quota: QuotaService,
+    private readonly runtime: BackendRuntimeProvider,
+    private readonly disposition: InviteAbuseDispositionService
+  ) {}
+
+  async assertWorkspaceInviteQuota(input: {
+    actorUserId: string;
+    workspaceId: string;
+    requestId?: string;
+    targetCount: number;
+    targetDomains: RuntimeQuotaTargetDomainInput[];
+    source?: RuntimeQuotaSourceInput;
+  }): Promise<InviteQuotaAdmission> {
+    const start = Date.now();
+    const seatQuota = await this.quota.getWorkspaceSeatQuota(input.workspaceId);
+    let decision: RuntimeWorkspaceInviteQuotaDecision;
+    try {
+      decision = await this.runtime.assertWorkspaceInviteQuotaV1(input);
+    } catch (error) {
+      const message =
+        error instanceof Error ? error.message : 'native assert failed';
+      metrics.workspace.counter('invite_quota_runtime_fallback').add(1, {
+        mode: this.config.auth.inviteQuotaFailOpenOnRuntimeError
+          ? 'fail_open'
+          : 'fail_closed',
+      });
+      this.logger.error('Workspace invite quota native assert failed', {
+        userId: input.actorUserId,
+        workspaceId: input.workspaceId,
+        targetCount: input.targetCount,
+        targetDomainsSummary: input.targetDomains,
+        sourceTrusted: input.source?.trusted ?? false,
+        country: input.source?.country,
+        asn: input.source?.asn,
+        requestId: input.requestId,
+        cfRay: input.source?.rayId,
+        error: message,
+      });
+      if (this.config.auth.inviteQuotaFailOpenOnRuntimeError) {
+        return { decision: { allowed: true, requested: input.targetCount } };
+      }
+      throw new TooManyRequest();
+    }
+    metrics.workspace
+      .counter('invite_quota_requested_targets')
+      .add(input.targetCount);
+    metrics.workspace
+      .histogram('invite_quota_counter_latency_ms')
+      .record(Date.now() - start);
+
+    if (!decision.allowed) {
+      metrics.workspace.counter('invite_quota_rejected').add(1, {
+        reason: decision.reason ?? 'unknown',
+      });
+      metrics.workspace.counter('invite_quota_reject_by_reason').add(1, {
+        reason: decision.reason ?? 'unknown',
+      });
+      if (this.config.auth.inviteQuotaShadowMode) {
+        this.logger.warn('Workspace invite quota shadow rejected', {
+          userId: input.actorUserId,
+          workspaceId: input.workspaceId,
+          targetCount: input.targetCount,
+          targetDomainsSummary: input.targetDomains,
+          sourceTrusted: input.source?.trusted ?? false,
+          country: input.source?.country,
+          asn: input.source?.asn,
+          reason: decision.reason,
+          scopeKeyHash: hashLogValue(decision.scopeKey),
+          limit: decision.limit,
+          current: decision.current,
+          requested: decision.requested,
+          memberLimit: seatQuota.memberLimit,
+          memberCount: seatQuota.memberCount,
+          retryAfter: decision.retryAfterSeconds,
+          requestId: input.requestId,
+          cfRay: input.source?.rayId,
+          wouldDispose: decision.actionRequired?.action,
+        });
+        return { decision };
+      }
+
+      try {
+        await this.disposition.execute({
+          actionRequired: decision.actionRequired,
+          actorUserId: input.actorUserId,
+          workspaceId: input.workspaceId,
+        });
+      } catch (error) {
+        this.logger.error('Workspace invite quota disposition failed', {
+          userId: input.actorUserId,
+          workspaceId: input.workspaceId,
+          action: decision.actionRequired?.action,
+          actionId: decision.actionRequired?.actionId,
+          reason: decision.reason,
+          error: error instanceof Error ? error.message : 'Unknown error',
+        });
+      }
+      this.logger.warn('Workspace invite quota rejected', {
+        userId: input.actorUserId,
+        workspaceId: input.workspaceId,
+        targetCount: input.targetCount,
+        targetDomainsSummary: input.targetDomains,
+        sourceTrusted: input.source?.trusted ?? false,
+        country: input.source?.country,
+        asn: input.source?.asn,
+        reason: decision.reason,
+        scopeKeyHash: hashLogValue(decision.scopeKey),
+        limit: decision.limit,
+        current: decision.current,
+        requested: decision.requested,
+        memberLimit: seatQuota.memberLimit,
+        memberCount: seatQuota.memberCount,
+        retryAfter: decision.retryAfterSeconds,
+        requestId: input.requestId,
+        cfRay: input.source?.rayId,
+      });
+      throw this.mapDecision(decision);
+    }
+
+    metrics.workspace.counter('invite_quota_allowed').add(1);
+    return {
+      reservationId: decision.reservationId,
+      decision,
+    };
+  }
+
+  async commitWorkspaceInviteQuota(
+    reservationId: string | undefined,
+    usage: {
+      targetCount: number;
+      targetDomains: RuntimeQuotaTargetDomainInput[];
+    }
+  ) {
+    if (!reservationId) {
+      return false;
+    }
+    return await this.runtime.commitWorkspaceInviteQuotaV1(
+      reservationId,
+      usage
+    );
+  }
+
+  async releaseWorkspaceInviteQuota(reservationId: string | undefined) {
+    if (!reservationId) {
+      return false;
+    }
+    return await this.runtime.releaseWorkspaceInviteQuotaV1(reservationId);
+  }
+
+  private mapDecision(
+    decision: RuntimeWorkspaceInviteQuotaDecision
+  ): UserFriendlyError {
+    if (decision.reason === 'abuse_subject' || decision.actionRequired) {
+      return new ActionForbidden('This feature is temporarily unavailable.');
+    }
+    return new TooManyRequest();
+  }
+}

packages/backend/server/src/core/workspaces/event.ts

@@ -79,6 +79,11 @@ export class WorkspaceEvents {
           $$workspaceId: workspaceId,
         },
       },
+      metadata: {
+        workspaceId,
+        recipientUserId: userId,
+        source: { trusted: false },
+      },
     });
   }
 

packages/backend/server/src/core/workspaces/index.ts

@@ -9,6 +9,10 @@ import { PermissionModule } from '../permission';
 import { QuotaModule } from '../quota';
 import { StorageModule } from '../storage';
 import { UserModule } from '../user';
+import {
+  InviteAbuseDispositionService,
+  InviteQuotaAssertService,
+} from './abuse';
 import { WorkspacesController } from './controller';
 import { WorkspaceEvents } from './event';
 import { WorkspaceRealtimeModule } from './realtime.module';
@@ -46,6 +50,8 @@ import { WorkspaceStatsJob } from './stats.job';
     DocHistoryResolver,
     WorkspaceBlobResolver,
     WorkspaceService,
+    InviteAbuseDispositionService,
+    InviteQuotaAssertService,
     WorkspaceEvents,
     AdminWorkspaceResolver,
     WorkspaceStatsJob,
@@ -54,6 +60,11 @@ import { WorkspaceStatsJob } from './stats.job';
 })
 export class WorkspaceModule {}
 
+export {
+  getAbuseRequestSource,
+  InviteAbuseDispositionService,
+  InviteQuotaAssertService,
+} from './abuse';
 export { WorkspaceRealtimeModule } from './realtime.module';
 export { WorkspaceService } from './service';
 export { InvitationType, WorkspaceType } from './types';

packages/backend/server/src/core/workspaces/resolvers/analytics-types.ts

@@ -2,6 +2,7 @@ import { Field, Int, ObjectType, registerEnumType } from '@nestjs/graphql';
 
 export enum TimeBucket {
   Minute = 'Minute',
+  Hour = 'Hour',
   Day = 'Day',
 }
 

packages/backend/server/src/core/workspaces/resolvers/doc.ts

@@ -35,6 +35,7 @@ import {
 import { PageInfo } from '../../../base/graphql/pagination';
 import { Models, PublicDocMode } from '../../../models';
 import { CurrentUser } from '../../auth';
+import { BackendRuntimeProvider } from '../../backend-runtime';
 import { Editor } from '../../doc';
 import {
   DOC_ACTIONS,
@@ -303,13 +304,34 @@ export class WorkspaceDocResolver {
     private readonly models: Models,
     private readonly cache: Cache,
     private readonly event: EventBus,
-    private readonly config: Config
+    private readonly config: Config,
+    private readonly runtime: BackendRuntimeProvider
   ) {}
 
   private async assertCanShare(
     userId: string,
     context: { workspaceId: string; docId: string; action: 'publishDoc' }
   ) {
+    if (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(userId)) {
+      this.logger.warn('Share action blocked for quarantined actor', {
+        userId,
+        ...context,
+      });
+      throw new ActionForbidden(
+        'This feature is temporarily unavailable for you.'
+      );
+    }
+    if (
+      await this.runtime.isInviteAbuseWorkspaceQuarantined(context.workspaceId)
+    ) {
+      this.logger.warn('Share action blocked for quarantined workspace', {
+        userId,
+        ...context,
+      });
+      throw new ActionForbidden(
+        'This feature is temporarily unavailable for you.'
+      );
+    }
     const user = await this.models.user.get(userId);
     const newAccountAgeMs = this.config.auth.newAccountShareActionDelay * 1000;
     if (!user || !canUserExecuteLimitedActions(user, newAccountAgeMs)) {

packages/backend/server/src/core/workspaces/resolvers/member.ts

@@ -1,6 +1,7 @@
 import { Logger } from '@nestjs/common';
 import {
   Args,
+  Context,
   Int,
   Mutation,
   Parent,
@@ -24,6 +25,7 @@ import {
   CanNotRevokeYourself,
   Config,
   EventBus,
+  getRequestTrackerId,
   InvalidInvitation,
   isValidCacheTtl,
   mapAnyError,
@@ -38,8 +40,11 @@ import {
   URLHelper,
   UserNotFound,
 } from '../../../base';
+import type { GraphqlContext } from '../../../base/graphql';
 import { Models } from '../../../models';
 import { CurrentUser, Public } from '../../auth';
+import { BackendRuntimeProvider } from '../../backend-runtime';
+import { containsUrlOrDomain } from '../../content-policy';
 import {
   PermissionAccess,
   WorkspacePolicyService,
@@ -48,7 +53,11 @@ import {
 import { QuotaService } from '../../quota';
 import { UserType } from '../../user';
 import { validators } from '../../utils/validators';
-import { canUserExecuteLimitedActions, containsUrlOrDomain } from '../abuse';
+import {
+  canUserExecuteLimitedActions,
+  getAbuseRequestSource,
+  InviteQuotaAssertService,
+} from '../abuse';
 import { WorkspaceService } from '../service';
 import {
   InvitationType,
@@ -59,6 +68,27 @@ import {
   WorkspaceType,
 } from '../types';
 
+type InviteCandidate = {
+  index: number;
+  email: string;
+  normalizedEmail: string;
+  domain: string;
+  target?: { id: string };
+};
+
+function emailDomain(email: string) {
+  const parts = email.split('@');
+  return parts.length === 2 ? parts[1] : '';
+}
+
+function aggregateTargetDomains(candidates: InviteCandidate[]) {
+  const domains = new Map<string, number>();
+  for (const candidate of candidates) {
+    domains.set(candidate.domain, (domains.get(candidate.domain) ?? 0) + 1);
+  }
+  return Array.from(domains, ([domain, count]) => ({ domain, count }));
+}
+
 /**
  * Workspace team resolver
  * Public apis rate limit: 10 req/m
@@ -78,16 +108,39 @@ export class WorkspaceMemberResolver {
     private readonly policy: WorkspacePolicyService,
     private readonly workspaceService: WorkspaceService,
     private readonly quota: QuotaService,
-    private readonly config: Config
+    private readonly config: Config,
+    private readonly inviteQuota: InviteQuotaAssertService,
+    private readonly runtime: BackendRuntimeProvider
   ) {}
 
   private async assertCanInviteOrShare(
     userId: string,
     context: {
       workspaceId: string;
-      action: 'inviteMembers' | 'createInviteLink';
+      action: 'createInviteLink';
     }
   ) {
+    if (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(userId)) {
+      this.logger.warn('Share action blocked for quarantined actor', {
+        userId,
+        ...context,
+      });
+      throw new ActionForbidden(
+        'This feature is temporarily unavailable for you.'
+      );
+    }
+    if (
+      await this.runtime.isInviteAbuseWorkspaceQuarantined(context.workspaceId)
+    ) {
+      this.logger.warn('Share action blocked for quarantined workspace', {
+        userId,
+        ...context,
+      });
+      throw new ActionForbidden(
+        'This feature is temporarily unavailable for you.'
+      );
+    }
+    // Member invites are owned by native quota; this guard stays for invite links until share/link actions migrate.
     const user = await this.models.user.get(userId);
     const newAccountAgeMs = this.config.auth.newAccountShareActionDelay * 1000;
     if (!user || !canUserExecuteLimitedActions(user, newAccountAgeMs)) {
@@ -182,6 +235,7 @@ export class WorkspaceMemberResolver {
   @Mutation(() => [InviteResult])
   async inviteMembers(
     @CurrentUser() me: CurrentUser,
+    @Context() context: GraphqlContext,
     @Args('workspaceId') workspaceId: string,
     @Args({ name: 'emails', type: () => [String] }) emails: string[]
   ): Promise<InviteResult[]> {
@@ -189,16 +243,54 @@ export class WorkspaceMemberResolver {
       .user(me.id)
       .workspace(workspaceId)
       .assert('Workspace.Users.Manage');
-    await this.assertCanInviteOrShare(me.id, {
-      workspaceId,
-      action: 'inviteMembers',
-    });
     await this.assertWorkspaceNameCanInvite(workspaceId);
 
     if (emails.length > 512) {
       throw new TooManyRequest();
     }
 
+    const results: InviteResult[] = emails.map(email => ({ email }));
+    const candidates: InviteCandidate[] = [];
+    const seen = new Set<string>();
+    for (const [index, email] of emails.entries()) {
+      try {
+        const normalizedEmail = email.trim().toLowerCase();
+        validators.assertValidEmail(normalizedEmail);
+        if (seen.has(normalizedEmail)) {
+          throw new ActionForbidden('Duplicate invite email.');
+        }
+        seen.add(normalizedEmail);
+
+        const target = await this.models.user.getUserByEmail(normalizedEmail);
+        if (target) {
+          const originRecord = await this.models.workspaceUser.get(
+            workspaceId,
+            target.id
+          );
+          if (originRecord) {
+            throw new AlreadyInSpace({ spaceId: workspaceId });
+          }
+        }
+
+        candidates.push({
+          index,
+          email,
+          normalizedEmail,
+          domain: emailDomain(normalizedEmail),
+          target: target ? { id: target.id } : undefined,
+        });
+      } catch (error) {
+        results[index] = {
+          email,
+          error: mapAnyError(error),
+        };
+      }
+    }
+
+    if (candidates.length === 0) {
+      return results;
+    }
+
     // lock to prevent concurrent invite
     const lockFlag = `invite:${workspaceId}`;
     await using lock = await this.mutex.acquire(lockFlag);
@@ -206,49 +298,68 @@ export class WorkspaceMemberResolver {
       throw new TooManyRequest();
     }
 
+    const admission = await this.inviteQuota.assertWorkspaceInviteQuota({
+      actorUserId: me.id,
+      workspaceId,
+      requestId: getRequestTrackerId(context.req),
+      targetCount: candidates.length,
+      targetDomains: aggregateTargetDomains(candidates),
+      source: getAbuseRequestSource(context.req, this.config),
+    });
     const quota = await this.quota.getWorkspaceSeatQuota(workspaceId);
     const isTeam = await this.workspaceService.isTeamWorkspace(workspaceId);
+    const successfulCandidates: InviteCandidate[] = [];
+    let reservationSettled = false;
 
-    const results: InviteResult[] = [];
+    try {
+      for (const candidate of candidates) {
+        try {
+          let target = candidate.target;
+          if (!target) {
+            target = await this.models.user.create({
+              email: candidate.normalizedEmail,
+              registered: false,
+            });
+          }
 
-    for (const [idx, email] of emails.entries()) {
-      try {
-        validators.assertValidEmail(email);
-        let target = await this.models.user.getUserByEmail(email);
-        if (target) {
-          const originRecord = await this.models.workspaceUser.get(
+          const existingMember = await this.models.workspaceUser.get(
             workspaceId,
             target.id
           );
-          // only invite if the user is not already in the workspace
-          if (originRecord) {
+          if (existingMember) {
             throw new AlreadyInSpace({ spaceId: workspaceId });
           }
-        } else {
-          target = await this.models.user.create({
-            email,
-            registered: false,
-          });
-        }
 
-        // no need to check quota, directly go allocating seat path
-        if (isTeam) {
-          const role = await this.models.workspaceUser.set(
-            workspaceId,
-            target.id,
-            WorkspaceRole.Collaborator,
-            {
-              status: WorkspaceMemberStatus.AllocatingSeat,
-              source: WorkspaceMemberSource.Email,
-              inviterId: me.id,
+          if (!isTeam) {
+            const needMoreSeat =
+              quota.memberCount + successfulCandidates.length + 1 >
+              quota.memberLimit;
+            if (needMoreSeat) {
+              throw new NoMoreSeat({ spaceId: workspaceId });
             }
-          );
-          await this.allocateAvailableTeamSeats(workspaceId, quota.memberLimit);
-          results.push({ email, inviteId: role.id });
-        } else {
-          const needMoreSeat = quota.memberCount + idx + 1 > quota.memberLimit;
-          if (needMoreSeat) {
-            throw new NoMoreSeat({ spaceId: workspaceId });
+          }
+
+          // no need to check quota, directly go allocating seat path
+          if (isTeam) {
+            const role = await this.models.workspaceUser.set(
+              workspaceId,
+              target.id,
+              WorkspaceRole.Collaborator,
+              {
+                status: WorkspaceMemberStatus.AllocatingSeat,
+                source: WorkspaceMemberSource.Email,
+                inviterId: me.id,
+              }
+            );
+            await this.allocateAvailableTeamSeats(
+              workspaceId,
+              quota.memberLimit
+            );
+            results[candidate.index] = {
+              email: candidate.email,
+              inviteId: role.id,
+            };
+            successfulCandidates.push(candidate);
           } else {
             const role = await this.models.workspaceUser.set(
               workspaceId,
@@ -264,17 +375,39 @@ export class WorkspaceMemberResolver {
               inviteId: role.id,
               inviterId: me.id,
             });
-            results.push({
-              email,
+            results[candidate.index] = {
+              email: candidate.email,
               inviteId: role.id,
-            });
+            };
+            successfulCandidates.push(candidate);
           }
+        } catch (error) {
+          results[candidate.index] = {
+            email: candidate.email,
+            error: mapAnyError(error),
+          };
         }
-      } catch (error) {
-        results.push({
-          email,
-          error: mapAnyError(error),
-        });
+      }
+
+      if (successfulCandidates.length > 0) {
+        await this.inviteQuota.commitWorkspaceInviteQuota(
+          admission.reservationId,
+          {
+            targetCount: successfulCandidates.length,
+            targetDomains: aggregateTargetDomains(successfulCandidates),
+          }
+        );
+      } else {
+        await this.inviteQuota.releaseWorkspaceInviteQuota(
+          admission.reservationId
+        );
+      }
+      reservationSettled = true;
+    } finally {
+      if (!reservationSettled) {
+        await this.inviteQuota.releaseWorkspaceInviteQuota(
+          admission.reservationId
+        );
       }
     }
 

packages/backend/server/src/core/workspaces/service.ts

@@ -7,8 +7,10 @@ import {
   DEFAULT_WORKSPACE_NAME,
   Models,
 } from '../../models';
+import { BackendRuntimeProvider } from '../backend-runtime';
 import { DocReader } from '../doc';
 import { Mailer } from '../mail';
+import type { SendMailCommand } from '../mail/types';
 import { WorkspaceRole } from '../permission';
 import { QuotaStateService } from '../quota/state';
 import { WorkspaceBlobStorage } from '../storage';
@@ -32,7 +34,8 @@ export class WorkspaceService {
     private readonly blobStorage: WorkspaceBlobStorage,
     private readonly mailer: Mailer,
     private readonly queue: JobQueue,
-    private readonly quotaState: QuotaStateService
+    private readonly quotaState: QuotaStateService,
+    private readonly runtime: BackendRuntimeProvider
   ) {}
 
   async getInviteInfo(inviteId: string): Promise<InviteInfo> {
@@ -111,7 +114,7 @@ export class WorkspaceService {
     const admins = await this.models.workspaceUser.getAdmins(workspaceId);
 
     const link = this.url.link(`/workspace/${workspaceId}`);
-    await this.mailer.trySend({
+    await this.trySendWorkspaceMail({
       name: 'TeamWorkspaceUpgraded',
       to: owner.email,
       props: {
@@ -121,11 +124,16 @@ export class WorkspaceService {
         isOwner: true,
         url: link,
       },
+      metadata: {
+        workspaceId,
+        recipientUserId: owner.id,
+        source: { trusted: false },
+      },
     });
 
     await Promise.allSettled(
       admins.map(async user => {
-        await this.mailer.trySend({
+        await this.trySendWorkspaceMail({
           name: 'TeamWorkspaceUpgraded',
           to: user.email,
           props: {
@@ -135,6 +143,11 @@ export class WorkspaceService {
             isOwner: false,
             url: link,
           },
+          metadata: {
+            workspaceId,
+            recipientUserId: user.id,
+            source: { trusted: false },
+          },
         });
       })
     );
@@ -192,7 +205,7 @@ export class WorkspaceService {
     }
 
     if (ws.role === WorkspaceRole.Admin) {
-      await this.mailer.trySend({
+      await this.trySendWorkspaceMail({
         name: 'TeamBecomeAdmin',
         to: user.email,
         props: {
@@ -201,9 +214,14 @@ export class WorkspaceService {
           },
           url: this.url.link(`/workspace/${ws.id}`),
         },
+        metadata: {
+          workspaceId: ws.id,
+          recipientUserId: user.id,
+          source: { trusted: false },
+        },
       });
     } else {
-      await this.mailer.trySend({
+      await this.trySendWorkspaceMail({
         name: 'TeamBecomeCollaborator',
         to: user.email,
         props: {
@@ -212,12 +230,17 @@ export class WorkspaceService {
           },
           url: this.url.link(`/workspace/${ws.id}`),
         },
+        metadata: {
+          workspaceId: ws.id,
+          recipientUserId: user.id,
+          source: { trusted: false },
+        },
       });
     }
   }
 
   async sendOwnershipTransferredEmail(email: string, ws: { id: string }) {
-    await this.mailer.trySend({
+    await this.trySendWorkspaceMail({
       name: 'OwnershipTransferred',
       to: email,
       props: {
@@ -225,11 +248,15 @@ export class WorkspaceService {
           $$workspaceId: ws.id,
         },
       },
+      metadata: {
+        workspaceId: ws.id,
+        source: { trusted: false },
+      },
     });
   }
 
   async sendOwnershipReceivedEmail(email: string, ws: { id: string }) {
-    await this.mailer.trySend({
+    await this.trySendWorkspaceMail({
       name: 'OwnershipReceived',
       to: email,
       props: {
@@ -237,12 +264,16 @@ export class WorkspaceService {
           $$workspaceId: ws.id,
         },
       },
+      metadata: {
+        workspaceId: ws.id,
+        source: { trusted: false },
+      },
     });
   }
 
   async sendLeaveEmail(workspaceId: string, userId: string) {
     const owner = await this.models.workspaceUser.getOwner(workspaceId);
-    await this.mailer.trySend({
+    await this.trySendWorkspaceMail({
       name: 'MemberLeave',
       to: owner.email,
       props: {
@@ -253,9 +284,41 @@ export class WorkspaceService {
           $$userId: userId,
         },
       },
+      metadata: {
+        workspaceId,
+        recipientUserId: owner.id,
+        actorUserId: userId,
+        source: { trusted: false },
+      },
     });
   }
 
+  private async trySendWorkspaceMail(command: SendMailCommand) {
+    const actorUserId = command.metadata?.actorUserId;
+    if (
+      actorUserId &&
+      (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(actorUserId))
+    ) {
+      await this.mailer.skip(command, {
+        mailClass: 'workspace_lifecycle',
+        reason: 'actor_quarantined',
+      });
+      return false;
+    }
+    const workspaceId = command.metadata?.workspaceId;
+    if (
+      workspaceId &&
+      (await this.runtime.isInviteAbuseWorkspaceQuarantined(workspaceId))
+    ) {
+      await this.mailer.skip(command, {
+        mailClass: 'workspace_lifecycle',
+        reason: 'workspace_quarantined',
+      });
+      return false;
+    }
+    return await this.mailer.trySend(command);
+  }
+
   async allocateSeats(workspaceId: string, quantity: number) {
     const pendings = await this.models.workspaceUser.allocateSeats(
       workspaceId,

packages/backend/server/src/models/index.ts

@@ -29,6 +29,7 @@ import { DocUserModel } from './doc-user';
 import { FeatureModel } from './feature';
 import { HistoryModel } from './history';
 import { MagicLinkOtpModel } from './magic-link-otp';
+import { MailDeliveryModel } from './mail-delivery';
 import { NotificationModel } from './notification';
 import { PermissionProjectionModel } from './permission-projection';
 import {
@@ -57,6 +58,7 @@ const MODELS = {
   session: SessionModel,
   verificationToken: VerificationTokenModel,
   magicLinkOtp: MagicLinkOtpModel,
+  mailDelivery: MailDeliveryModel,
   feature: FeatureModel,
   workspace: WorkspaceModel,
   userFeature: UserFeatureModel,
@@ -165,6 +167,7 @@ export * from './doc-user';
 export * from './feature';
 export * from './history';
 export * from './magic-link-otp';
+export * from './mail-delivery';
 export * from './notification';
 export * from './permission-projection';
 export * from './permission-write';

packages/backend/server/src/models/magic-link-otp.ts

@@ -36,6 +36,7 @@ export class MagicLinkOtpModel extends BaseModel {
       create: { email, otpHash, token, clientNonce, expiresAt, attempts: 0 },
       update: { otpHash, token, clientNonce, expiresAt, attempts: 0 },
     });
+    return { expiresAt };
   }
 
   @Transactional()

packages/backend/server/src/models/mail-delivery.ts

@@ -0,0 +1,665 @@
+import { createHmac } from 'node:crypto';
+
+import { Injectable } from '@nestjs/common';
+import { Prisma } from '@prisma/client';
+
+import { CryptoHelper } from '../base';
+import { BaseModel } from './base';
+
+export type MailDeliveryStatus =
+  | 'queued'
+  | 'sending'
+  | 'retry_wait'
+  | 'sent'
+  | 'skipped'
+  | 'failed'
+  | 'canceled';
+
+export type MailDeliveryPriority = 'critical' | 'high' | 'normal' | 'low';
+
+export type MailDeliveryRow = {
+  id: string;
+  mailName: string;
+  mailClass: string;
+  priority: MailDeliveryPriority;
+  status: MailDeliveryStatus;
+  dedupeKey: string | null;
+  recipientEmail: string | null;
+  recipientHash: string;
+  recipientDomain: string;
+  recipientUserId: string | null;
+  actorUserId: string | null;
+  workspaceId: string | null;
+  notificationId: string | null;
+  abuseSubjectKey: string | null;
+  quotaReservationId: string | null;
+  quotaDecision: Prisma.JsonValue | null;
+  payload: Prisma.JsonValue | null;
+  sendAfter: Date;
+  expiresAt: Date | null;
+  attemptCount: number;
+  maxAttempts: number;
+  lockedBy: string | null;
+  lockedUntil: Date | null;
+  firstAttemptAt: Date | null;
+  lastAttemptAt: Date | null;
+  sentAt: Date | null;
+  settledAt: Date | null;
+  canceledAt: Date | null;
+  failedAt: Date | null;
+  providerMessageId: string | null;
+  providerResponse: string | null;
+  lastErrorCode: string | null;
+  lastError: string | null;
+  retentionState: 'full' | 'anonymized';
+  anonymizedAt: Date | null;
+  createdAt: Date;
+  updatedAt: Date;
+};
+
+export type MailDeliveryCreateInput = {
+  mailName: string;
+  mailClass: string;
+  priority: MailDeliveryPriority;
+  status?: Extract<MailDeliveryStatus, 'queued' | 'skipped' | 'failed'>;
+  dedupeKey?: string;
+  recipientEmail: string;
+  recipientUserId?: string;
+  actorUserId?: string;
+  workspaceId?: string;
+  notificationId?: string;
+  abuseSubjectKey?: string;
+  quotaReservationId?: string;
+  quotaDecision?: Prisma.JsonValue;
+  payload: Prisma.JsonValue;
+  sendAfter?: Date;
+  expiresAt?: Date;
+  maxAttempts?: number;
+  lastErrorCode?: string;
+  lastError?: string;
+};
+
+export type MailDeliveryAdminAggregate = {
+  bucket: Date;
+  mailName: string;
+  mailClass: string;
+  status: MailDeliveryStatus;
+  count: number;
+};
+
+export type MailDeliveryMetricsSnapshot = {
+  retryWait: number;
+  failedRecent: number;
+  expiredLeases: number;
+  readyDelayMs: number;
+};
+
+const TERMINAL_STATUSES = new Set<MailDeliveryStatus>([
+  'sent',
+  'skipped',
+  'failed',
+  'canceled',
+]);
+
+function recipientDomain(email: string) {
+  const parts = email.trim().toLowerCase().split('@');
+  return parts.length === 2 ? parts[1] : '';
+}
+
+function redact(value: string | undefined | null) {
+  if (!value) {
+    return null;
+  }
+  return value
+    .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '[email]')
+    .replace(/\b\d{6,}\b/g, '[code]')
+    .replace(/https?:\/\/\S+/gi, '[url]')
+    .slice(0, 1000);
+}
+
+function mapRow(row: Record<string, unknown>): MailDeliveryRow {
+  return {
+    id: row.id as string,
+    mailName: row.mailName as string,
+    mailClass: row.mailClass as string,
+    priority: row.priority as MailDeliveryPriority,
+    status: row.status as MailDeliveryStatus,
+    dedupeKey: row.dedupeKey as string | null,
+    recipientEmail: row.recipientEmail as string | null,
+    recipientHash: row.recipientHash as string,
+    recipientDomain: row.recipientDomain as string,
+    recipientUserId: row.recipientUserId as string | null,
+    actorUserId: row.actorUserId as string | null,
+    workspaceId: row.workspaceId as string | null,
+    notificationId: row.notificationId as string | null,
+    abuseSubjectKey: row.abuseSubjectKey as string | null,
+    quotaReservationId: row.quotaReservationId as string | null,
+    quotaDecision: row.quotaDecision as Prisma.JsonValue | null,
+    payload: row.payload as Prisma.JsonValue | null,
+    sendAfter: row.sendAfter as Date,
+    expiresAt: row.expiresAt as Date | null,
+    attemptCount: row.attemptCount as number,
+    maxAttempts: row.maxAttempts as number,
+    lockedBy: row.lockedBy as string | null,
+    lockedUntil: row.lockedUntil as Date | null,
+    firstAttemptAt: row.firstAttemptAt as Date | null,
+    lastAttemptAt: row.lastAttemptAt as Date | null,
+    sentAt: row.sentAt as Date | null,
+    settledAt: row.settledAt as Date | null,
+    canceledAt: row.canceledAt as Date | null,
+    failedAt: row.failedAt as Date | null,
+    providerMessageId: row.providerMessageId as string | null,
+    providerResponse: row.providerResponse as string | null,
+    lastErrorCode: row.lastErrorCode as string | null,
+    lastError: row.lastError as string | null,
+    retentionState: row.retentionState as 'full' | 'anonymized',
+    anonymizedAt: row.anonymizedAt as Date | null,
+    createdAt: row.createdAt as Date,
+    updatedAt: row.updatedAt as Date,
+  };
+}
+
+const SELECT_FIELDS = Prisma.sql`
+  id::text AS "id",
+  mail_name AS "mailName",
+  mail_class AS "mailClass",
+  priority,
+  status,
+  dedupe_key AS "dedupeKey",
+  recipient_email AS "recipientEmail",
+  recipient_hash AS "recipientHash",
+  recipient_domain AS "recipientDomain",
+  recipient_user_id AS "recipientUserId",
+  actor_user_id AS "actorUserId",
+  workspace_id AS "workspaceId",
+  notification_id AS "notificationId",
+  abuse_subject_key AS "abuseSubjectKey",
+  quota_reservation_id::text AS "quotaReservationId",
+  quota_decision AS "quotaDecision",
+  payload,
+  send_after AS "sendAfter",
+  expires_at AS "expiresAt",
+  attempt_count AS "attemptCount",
+  max_attempts AS "maxAttempts",
+  locked_by AS "lockedBy",
+  locked_until AS "lockedUntil",
+  first_attempt_at AS "firstAttemptAt",
+  last_attempt_at AS "lastAttemptAt",
+  sent_at AS "sentAt",
+  settled_at AS "settledAt",
+  canceled_at AS "canceledAt",
+  failed_at AS "failedAt",
+  provider_message_id AS "providerMessageId",
+  provider_response AS "providerResponse",
+  last_error_code AS "lastErrorCode",
+  last_error AS "lastError",
+  retention_state AS "retentionState",
+  anonymized_at AS "anonymizedAt",
+  created_at AS "createdAt",
+  updated_at AS "updatedAt"
+`;
+
+@Injectable()
+export class MailDeliveryModel extends BaseModel {
+  constructor(private readonly crypto: CryptoHelper) {
+    super();
+  }
+
+  private recipientHash(email: string) {
+    return createHmac('sha256', this.crypto.keyPair.sha256.privateKey)
+      .update(email.trim().toLowerCase())
+      .digest('hex');
+  }
+
+  async create(input: MailDeliveryCreateInput): Promise<MailDeliveryRow> {
+    const now = new Date();
+    const sendAfter = input.sendAfter ?? now;
+    const status = input.status ?? 'queued';
+    const expired =
+      input.expiresAt &&
+      (input.expiresAt.getTime() <= now.getTime() ||
+        sendAfter.getTime() >= input.expiresAt.getTime());
+    const finalStatus = expired ? 'failed' : status;
+    const terminal = TERMINAL_STATUSES.has(finalStatus);
+    const lastErrorCode = expired
+      ? 'expired'
+      : (input.lastErrorCode ?? (status === 'skipped' ? 'skipped' : null));
+    const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
+      INSERT INTO mail_deliveries (
+        mail_name,
+        mail_class,
+        priority,
+        status,
+        dedupe_key,
+        recipient_email,
+        recipient_hash,
+        recipient_domain,
+        recipient_user_id,
+        actor_user_id,
+        workspace_id,
+        notification_id,
+        abuse_subject_key,
+        quota_reservation_id,
+        quota_decision,
+        payload,
+        send_after,
+        expires_at,
+        max_attempts,
+        settled_at,
+        failed_at,
+        last_error_code,
+        last_error,
+        retention_state,
+        anonymized_at
+      )
+      VALUES (
+        ${input.mailName},
+        ${input.mailClass},
+        ${input.priority},
+        ${finalStatus},
+        ${input.dedupeKey ?? null},
+        ${terminal ? null : input.recipientEmail},
+        ${this.recipientHash(input.recipientEmail)},
+        ${recipientDomain(input.recipientEmail)},
+        ${input.recipientUserId ?? null},
+        ${input.actorUserId ?? null},
+        ${input.workspaceId ?? null},
+        ${input.notificationId ?? null},
+        ${input.abuseSubjectKey ?? null},
+        ${input.quotaReservationId ?? null}::uuid,
+        ${input.quotaDecision ?? null},
+        ${terminal ? null : input.payload},
+        ${sendAfter},
+        ${input.expiresAt ?? null},
+        ${input.maxAttempts ?? 3},
+        ${terminal ? now : null},
+        ${finalStatus === 'failed' ? now : null},
+        ${lastErrorCode},
+        ${redact(input.lastError)},
+        ${terminal ? 'anonymized' : 'full'},
+        ${terminal ? now : null}
+      )
+      ON CONFLICT (dedupe_key) WHERE dedupe_key IS NOT NULL
+      DO NOTHING
+      RETURNING ${SELECT_FIELDS}
+    `;
+
+    if (rows[0]) {
+      return mapRow(rows[0]);
+    }
+    if (!input.dedupeKey) {
+      throw new Error('Failed to create mail delivery.');
+    }
+    const existing = await this.findByDedupeKey(input.dedupeKey);
+    if (!existing) {
+      throw new Error('Failed to find deduped mail delivery.');
+    }
+    return existing;
+  }
+
+  async findByDedupeKey(dedupeKey: string) {
+    const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
+      SELECT ${SELECT_FIELDS}
+      FROM mail_deliveries
+      WHERE dedupe_key = ${dedupeKey}
+      LIMIT 1
+    `;
+    return rows[0] ? mapRow(rows[0]) : null;
+  }
+
+  async claimReady(
+    workerId: string,
+    options: { batchSize: number; leaseMs: number }
+  ) {
+    await this.markExpiredPending(options.batchSize);
+    const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
+      UPDATE mail_deliveries
+      SET status = 'sending',
+          locked_by = ${workerId},
+          locked_until = now() + (${options.leaseMs}::text || ' milliseconds')::interval,
+          updated_at = now()
+      WHERE id IN (
+        SELECT id
+        FROM mail_deliveries
+        WHERE (
+            status IN ('queued', 'retry_wait')
+            OR (status = 'sending' AND locked_until < now())
+          )
+          AND send_after <= now()
+          AND (expires_at IS NULL OR expires_at > now())
+          AND (locked_until IS NULL OR locked_until < now())
+          AND max_attempts > attempt_count
+        ORDER BY
+          CASE priority
+            WHEN 'critical' THEN 0
+            WHEN 'high' THEN 1
+            WHEN 'normal' THEN 2
+            ELSE 3
+          END,
+          send_after,
+          created_at
+        FOR UPDATE SKIP LOCKED
+        LIMIT ${options.batchSize}
+      )
+      RETURNING ${SELECT_FIELDS}
+    `;
+    return rows.map(mapRow);
+  }
+
+  async markAttemptStarted(id: string, workerId: string) {
+    const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
+      UPDATE mail_deliveries
+      SET attempt_count = attempt_count + 1,
+          first_attempt_at = COALESCE(first_attempt_at, now()),
+          last_attempt_at = now(),
+          updated_at = now()
+      WHERE id = ${id}::uuid
+        AND status = 'sending'
+        AND locked_by = ${workerId}
+        AND (expires_at IS NULL OR expires_at > now())
+        AND max_attempts > attempt_count
+      RETURNING ${SELECT_FIELDS}
+    `;
+    return rows[0] ? mapRow(rows[0]) : null;
+  }
+
+  async markSent(
+    id: string,
+    workerId: string,
+    result: {
+      providerMessageId?: string | null;
+      providerResponse?: string | null;
+    }
+  ) {
+    return await this.markTerminal(id, workerId, 'sent', {
+      providerMessageId: result.providerMessageId,
+      providerResponse: redact(result.providerResponse),
+    });
+  }
+
+  async markRetry(
+    id: string,
+    workerId: string,
+    input: {
+      sendAfter: Date;
+      errorCode?: string | null;
+      error?: string | null;
+    }
+  ) {
+    const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
+      UPDATE mail_deliveries
+      SET status = 'retry_wait',
+          send_after = ${input.sendAfter},
+          locked_by = NULL,
+          locked_until = NULL,
+          last_error_code = ${input.errorCode ?? null},
+          last_error = ${redact(input.error)},
+          updated_at = now()
+      WHERE id = ${id}::uuid
+        AND status = 'sending'
+        AND locked_by = ${workerId}
+        AND (expires_at IS NULL OR ${input.sendAfter} < expires_at)
+        AND attempt_count < max_attempts
+      RETURNING ${SELECT_FIELDS}
+    `;
+    return rows[0] ? mapRow(rows[0]) : null;
+  }
+
+  async markSkipped(
+    id: string,
+    workerId: string,
+    reason: string,
+    detail?: string
+  ) {
+    return await this.markTerminal(id, workerId, 'skipped', {
+      lastErrorCode: reason,
+      lastError: redact(detail),
+    });
+  }
+
+  async markFailed(
+    id: string,
+    workerId: string,
+    errorCode: string,
+    error?: string
+  ) {
+    return await this.markTerminal(id, workerId, 'failed', {
+      lastErrorCode: errorCode,
+      lastError: redact(error),
+    });
+  }
+
+  async markCanceled(id: string, workerId: string, reason = 'canceled') {
+    return await this.markTerminal(id, workerId, 'canceled', {
+      lastErrorCode: reason,
+    });
+  }
+
+  async cancelByRecipient(
+    recipientEmail: string,
+    reason = 'recipient_deleted'
+  ) {
+    return await this.cancelWhere(
+      Prisma.sql`recipient_hash = ${this.recipientHash(recipientEmail)}`,
+      reason
+    );
+  }
+
+  async cancelById(id: string, reason = 'canceled') {
+    return await this.cancelWhere(Prisma.sql`id = ${id}::uuid`, reason);
+  }
+
+  async cancelMemberInvitationByActor(
+    actorUserId: string,
+    reason = 'actor_deleted'
+  ) {
+    return await this.cancelWhere(
+      Prisma.sql`mail_name = 'MemberInvitation' AND actor_user_id = ${actorUserId}`,
+      reason
+    );
+  }
+
+  async cancelByActor(actorUserId: string, reason = 'actor_quarantined') {
+    return await this.cancelWhere(
+      Prisma.sql`actor_user_id = ${actorUserId}`,
+      reason
+    );
+  }
+
+  async cancelByWorkspace(
+    workspaceId: string,
+    reason = 'workspace_quarantined'
+  ) {
+    return await this.cancelWhere(
+      Prisma.sql`workspace_id = ${workspaceId}`,
+      reason
+    );
+  }
+
+  async cancelByAbuseSubject(
+    subjectKey: string,
+    reason = 'abuse_subject_quarantined'
+  ) {
+    return await this.cancelWhere(
+      Prisma.sql`abuse_subject_key = ${subjectKey}`,
+      reason
+    );
+  }
+
+  async markExpiredPending(limit: number) {
+    const result = await this.db.$executeRaw`
+      UPDATE mail_deliveries
+      SET status = 'failed',
+          failed_at = now(),
+          settled_at = now(),
+          locked_by = NULL,
+          locked_until = NULL,
+          recipient_email = NULL,
+          payload = NULL,
+          retention_state = 'anonymized',
+          anonymized_at = now(),
+          last_error_code = 'expired',
+          updated_at = now()
+      WHERE id IN (
+        SELECT id
+        FROM mail_deliveries
+        WHERE (
+            status IN ('queued', 'retry_wait')
+            OR (status = 'sending' AND locked_until < now())
+          )
+          AND expires_at IS NOT NULL
+          AND expires_at <= now()
+        LIMIT ${limit}
+      )
+    `;
+    return Number(result);
+  }
+
+  async adminAggregate(input: {
+    from: Date;
+    to: Date;
+    bucket: 'hour' | 'day';
+  }) {
+    const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
+      WITH events AS (
+        SELECT
+          date_trunc(${input.bucket}, created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AS bucket,
+          mail_name,
+          mail_class,
+          status,
+          COUNT(*)::int AS count
+        FROM mail_deliveries
+        WHERE created_at >= ${input.from}
+          AND created_at < ${input.to}
+        GROUP BY
+          bucket,
+          mail_name,
+          mail_class,
+          status
+      )
+      SELECT
+        bucket AS "bucket",
+        mail_name AS "mailName",
+        mail_class AS "mailClass",
+        status,
+        count
+      FROM events
+      ORDER BY bucket ASC, count DESC, "mailName" ASC
+    `;
+    return rows.map(row => ({
+      bucket: row.bucket as Date,
+      mailName: row.mailName as string,
+      mailClass: row.mailClass as string,
+      status: row.status as MailDeliveryStatus,
+      count: row.count as number,
+    })) satisfies MailDeliveryAdminAggregate[];
+  }
+
+  async deleteAnonymizedBefore(before: Date, limit: number) {
+    const result = await this.db.$executeRaw`
+      DELETE FROM mail_deliveries
+      WHERE id IN (
+        SELECT id
+        FROM mail_deliveries
+        WHERE retention_state = 'anonymized'
+          AND settled_at IS NOT NULL
+          AND settled_at < ${before}
+        ORDER BY settled_at
+        LIMIT ${limit}
+      )
+    `;
+    return Number(result);
+  }
+
+  async metricsSnapshot(): Promise<MailDeliveryMetricsSnapshot> {
+    const rows = await this.db.$queryRaw<
+      Array<{
+        retryWait: number;
+        failedRecent: number;
+        expiredLeases: number;
+        readyDelayMs: number;
+      }>
+    >`
+      SELECT
+        COUNT(*) FILTER (WHERE status = 'retry_wait')::int AS "retryWait",
+        COUNT(*) FILTER (
+          WHERE status = 'failed' AND failed_at >= now() - interval '1 hour'
+        )::int AS "failedRecent",
+        COUNT(*) FILTER (
+          WHERE status = 'sending' AND locked_until < now()
+        )::int AS "expiredLeases",
+        COALESCE(MAX(EXTRACT(EPOCH FROM now() - send_after) * 1000) FILTER (
+          WHERE status IN ('queued', 'retry_wait') AND send_after <= now()
+        ), 0)::int AS "readyDelayMs"
+      FROM mail_deliveries
+    `;
+    return (
+      rows[0] ?? {
+        retryWait: 0,
+        failedRecent: 0,
+        expiredLeases: 0,
+        readyDelayMs: 0,
+      }
+    );
+  }
+
+  private async markTerminal(
+    id: string,
+    workerId: string,
+    status: Extract<
+      MailDeliveryStatus,
+      'sent' | 'skipped' | 'failed' | 'canceled'
+    >,
+    input: {
+      providerMessageId?: string | null;
+      providerResponse?: string | null;
+      lastErrorCode?: string | null;
+      lastError?: string | null;
+    }
+  ) {
+    const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
+      UPDATE mail_deliveries
+      SET status = ${status},
+          sent_at = CASE WHEN ${status} = 'sent' THEN now() ELSE sent_at END,
+          failed_at = CASE WHEN ${status} = 'failed' THEN now() ELSE failed_at END,
+          canceled_at = CASE WHEN ${status} = 'canceled' THEN now() ELSE canceled_at END,
+          settled_at = now(),
+          locked_by = NULL,
+          locked_until = NULL,
+          recipient_email = NULL,
+          payload = NULL,
+          retention_state = 'anonymized',
+          anonymized_at = now(),
+          provider_message_id = ${input.providerMessageId ?? null},
+          provider_response = ${input.providerResponse ?? null},
+          last_error_code = ${input.lastErrorCode ?? null},
+          last_error = ${input.lastError ?? null},
+          updated_at = now()
+      WHERE id = ${id}::uuid
+        AND status = 'sending'
+        AND locked_by = ${workerId}
+      RETURNING ${SELECT_FIELDS}
+    `;
+    return rows[0] ? mapRow(rows[0]) : null;
+  }
+
+  private async cancelWhere(where: Prisma.Sql, reason: string) {
+    const result = await this.db.$executeRaw`
+      UPDATE mail_deliveries
+      SET status = 'canceled',
+          canceled_at = now(),
+          settled_at = now(),
+          locked_by = NULL,
+          locked_until = NULL,
+          recipient_email = NULL,
+          payload = NULL,
+          retention_state = 'anonymized',
+          anonymized_at = now(),
+          last_error_code = ${reason},
+          updated_at = now()
+      WHERE status IN ('queued', 'retry_wait', 'sending')
+        AND ${where}
+    `;
+    return Number(result);
+  }
+}

packages/backend/server/src/models/permission-write.ts

@@ -358,6 +358,32 @@ export class WorkspaceInvitationModel extends BaseModel {
     });
   }
 
+  @Transactional()
+  async cancelPendingByActor(actorUserId: string) {
+    await this.models.permissionProjection.markNewWriteOrigin();
+    return await this.db.workspaceInvitation.deleteMany({
+      where: {
+        inviterUserId: actorUserId,
+        status: {
+          in: ['pending', 'waiting_review', 'waiting_seat'],
+        },
+      },
+    });
+  }
+
+  @Transactional()
+  async cancelPendingByWorkspace(workspaceId: string) {
+    await this.models.permissionProjection.markNewWriteOrigin();
+    return await this.db.workspaceInvitation.deleteMany({
+      where: {
+        workspaceId,
+        status: {
+          in: ['pending', 'waiting_review', 'waiting_seat'],
+        },
+      },
+    });
+  }
+
   private async supportsCurrentInvitationColumns() {
     this.hasCurrentColumns ??= this.db.$queryRaw<Array<{ exists: boolean }>>`
         SELECT EXISTS (

packages/backend/server/src/models/verification-token.ts

@@ -28,17 +28,31 @@ export class VerificationTokenModel extends BaseModel {
     type: TokenType,
     credential?: string,
     ttlInSec: number = 30 * 60
+  ) {
+    const { token } = await this.createWithExpiresAt(
+      type,
+      credential,
+      ttlInSec
+    );
+    return token;
+  }
+
+  async createWithExpiresAt(
+    type: TokenType,
+    credential?: string,
+    ttlInSec: number = 30 * 60
   ) {
     const plaintextToken = randomUUID();
+    const expiresAt = new Date(Date.now() + ttlInSec * 1000);
     const { token } = await this.db.verificationToken.create({
       data: {
         type,
         token: plaintextToken,
         credential,
-        expiresAt: new Date(Date.now() + ttlInSec * 1000),
+        expiresAt,
       },
     });
-    return this.crypto.encrypt(token);
+    return { token: this.crypto.encrypt(token), expiresAt };
   }
 
   /**

packages/backend/server/src/native.ts

@@ -11,6 +11,8 @@ import serverNativeModule, {
   type CapabilityAttachmentContract,
   type CapabilityModelCapability,
   type CommandResponse,
+  type ContentPolicyScanInput,
+  type ContentPolicyScanResult,
   type ImageInspection,
   type ImageInspectionOptions,
   type LicenseError,
@@ -78,6 +80,8 @@ export type {
   CapabilityAttachmentContract,
   CapabilityModelCapability,
   CommandResponse,
+  ContentPolicyScanInput,
+  ContentPolicyScanResult,
   ImageInspection,
   ImageInspectionOptions,
   LicenseError,
@@ -255,6 +259,7 @@ export const inspectImageForProxy = serverNativeModule.inspectImageForProxy;
 export const fetchRemoteAttachment = serverNativeModule.fetchRemoteAttachment;
 export const inferRemoteMimeType = serverNativeModule.inferRemoteMimeType;
 export const assertSafeUrl = serverNativeModule.assertSafeUrl;
+export const scanContentPolicyV1 = serverNativeModule.scanContentPolicyV1;
 export const safeFetch = serverNativeModule.safeFetch;
 export const activateLicense = serverNativeModule.activateLicense;
 export const checkLicenseHealth = serverNativeModule.checkLicenseHealth;

packages/backend/server/src/plugins/payment/manager/selfhost.ts

@@ -148,6 +148,10 @@ export class SelfhostTeamSubscriptionManager extends SubscriptionManager {
         name: 'TeamLicense',
         to: userEmail,
         props: { license: key },
+        metadata: {
+          dedupeKey: `selfhost-license:${key}`,
+          source: { trusted: false },
+        },
       });
 
       await this.upsertStripeProviderSubscription(

packages/backend/server/src/schema.gql

@@ -105,6 +105,42 @@ type AdminLicensePreview {
   workspaceId: String!
 }
 
+input AdminMailDeliveriesInput {
+  hours: Int! = 24
+}
+
+type AdminMailDeliveryAnalytics {
+  byOutcome: [AdminMailDeliverySeries!]!
+  byStatus: [AdminMailDeliverySeries!]!
+  byType: [AdminMailDeliverySeries!]!
+  summary: AdminMailDeliverySummary!
+  window: TimeWindow!
+}
+
+type AdminMailDeliveryPoint {
+  bucket: DateTime!
+  count: Int!
+}
+
+type AdminMailDeliverySeries {
+  key: String!
+  label: String!
+  points: [AdminMailDeliveryPoint!]!
+  total: Int!
+}
+
+type AdminMailDeliverySummary {
+  canceled: Int!
+  failed: Int!
+  queued: Int!
+  retryWait: Int!
+  sending: Int!
+  sent: Int!
+  skipped: Int!
+  successRate: Float!
+  total: Int!
+}
+
 type AdminSharedLinkTopItem {
   docId: String!
   guestViews: SafeInt!
@@ -1952,6 +1988,9 @@ type Query {
   """Get aggregated dashboard metrics for admin panel"""
   adminDashboard(input: AdminDashboardInput): AdminDashboard!
 
+  """Aggregate mail delivery timeline facts for admin panel"""
+  adminMailDeliveries(input: AdminMailDeliveriesInput): AdminMailDeliveryAnalytics!
+
   """Get workspace detail for admin"""
   adminWorkspace(id: String!): AdminWorkspace
 
@@ -2420,6 +2459,7 @@ type TestWorkspaceByokConfigResultType {
 
 enum TimeBucket {
   Day
+  Hour
   Minute
 }