diff --git a/packages/backend/server/src/__tests__/__snapshots__/mails.spec.ts.md b/packages/backend/server/src/__tests__/__snapshots__/mails.spec.ts.md index dc531a626b..d67f5b0cd8 100644 --- a/packages/backend/server/src/__tests__/__snapshots__/mails.spec.ts.md +++ b/packages/backend/server/src/__tests__/__snapshots__/mails.spec.ts.md @@ -858,7 +858,7 @@ Generated by [AVA](https://avajs.dev). ␊ ` -> test@test.com invited you to join Test Workspace +> You were invited to join a workspace on AFFiNE `␊ ␊ ` -> test@test.com accepted your invitation +> Your workspace invitation was accepted `␊ ␊ ` -> test@test.com left Test Workspace +> A workspace member left `␊ ␊ ` -> New request to join Test Workspace +> New request to join a workspace `␊ ␊ ` -> Your request to join Test Workspace has been approved +> Your request to join a workspace has been approved `␊ ␊ ` -> Your request to join Test Workspace was declined +> Your request to join a workspace was declined `␊ ␊ ` -> You have been removed from Test Workspace +> You have been removed from a workspace `␊ ␊ ` -> Your ownership of Test Workspace has been transferred +> Your workspace ownership has been transferred `␊ ␊ ` -> You are now the owner of Test Workspace +> You are now the owner of a workspace `␊ ␊ ` -> test@test.com mentioned you in Test Doc +> You were mentioned in AFFiNE `␊ ␊ @@ -1601,7 +1601,7 @@ Generated by [AVA](https://avajs.dev). ␊ ` -> test@test.com commented on Test Doc +> New comment in AFFiNE `␊ ␊ @@ -1695,7 +1695,7 @@ Generated by [AVA](https://avajs.dev). ␊ ` -> test@test.com mentioned you in a comment on Test Doc +> You were mentioned in a comment `␊ ␊ @@ -1894,7 +1894,7 @@ Generated by [AVA](https://avajs.dev). ␊ ` -> You are now an admin of Test Workspace +> You are now a workspace admin `␊ ␊ ` -> Your role has been changed in Test Workspace +> Your workspace role has been changed `␊ ␊ ` -> [Action Required] Final warning: Your workspace Test Workspace will be deleted in 24 hours +> [Action Required] Final warning: Your workspace will be deleted in 24 hours `␊ ␊ ` -> [Action Required] Important: Your workspace Test Workspace will be deleted soon +> [Action Required] Important: Your workspace will be deleted soon `␊ ␊ ` -> Your workspace Test Workspace has been deleted +> Your workspace has been deleted `␊ ␊ ` -> [Action Required] Your Test Workspace team workspace will expire soon +> [Action Required] Your team workspace will expire soon `␊ ␊ ` -> Your Test Workspace team workspace has expired +> Your team workspace has expired `␊ test@test.com mentioned you in +> You were mentioned in AFFiNE `␊ ␊ diff --git a/packages/backend/server/src/__tests__/__snapshots__/mails.spec.ts.snap b/packages/backend/server/src/__tests__/__snapshots__/mails.spec.ts.snap index a8e784b77e..0b48451d26 100644 Binary files a/packages/backend/server/src/__tests__/__snapshots__/mails.spec.ts.snap and b/packages/backend/server/src/__tests__/__snapshots__/mails.spec.ts.snap differ diff --git a/packages/backend/server/src/__tests__/mails.spec.ts b/packages/backend/server/src/__tests__/mails.spec.ts index c0ee374769..904c9e77af 100644 --- a/packages/backend/server/src/__tests__/mails.spec.ts +++ b/packages/backend/server/src/__tests__/mails.spec.ts @@ -31,6 +31,7 @@ test('should normalize valid SMTP HELO hostnames', t => { }); test('should reject invalid SMTP HELO hostnames', t => { + t.is(normalizeSMTPHeloHostname(), undefined); t.is(normalizeSMTPHeloHostname(''), undefined); t.is(normalizeSMTPHeloHostname(' '), undefined); t.is(normalizeSMTPHeloHostname('AFFiNE Server'), undefined); diff --git a/packages/backend/server/src/__tests__/mocks/queue.mock.ts b/packages/backend/server/src/__tests__/mocks/queue.mock.ts index d44a9b6321..a2ec86b33b 100644 --- a/packages/backend/server/src/__tests__/mocks/queue.mock.ts +++ b/packages/backend/server/src/__tests__/mocks/queue.mock.ts @@ -6,6 +6,7 @@ import { JobQueue } from '../../base'; export class MockJobQueue { add = Sinon.createStubInstance(JobQueue).add.resolves(); remove = Sinon.createStubInstance(JobQueue).remove.resolves(); + removeWhere = Sinon.createStubInstance(JobQueue).removeWhere.resolves([]); last(name: Job): { name: Job; payload: Jobs[Job] } { const addJobName = this.add.lastCall?.args[0]; diff --git a/packages/backend/server/src/__tests__/mocks/user.mock.ts b/packages/backend/server/src/__tests__/mocks/user.mock.ts index 195dcc62ac..4868875b78 100644 --- a/packages/backend/server/src/__tests__/mocks/user.mock.ts +++ b/packages/backend/server/src/__tests__/mocks/user.mock.ts @@ -19,6 +19,7 @@ export class MockUser extends Mocker { const password = input?.password ?? faker.internet.password(); const user = await this.db.user.create({ data: { + createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000), email: faker.internet.email(), name: faker.person.fullName(), password: password ? hashSync(password) : undefined, diff --git a/packages/backend/server/src/__tests__/payment/service.spec.ts b/packages/backend/server/src/__tests__/payment/service.spec.ts index 7fea956f2f..db798bbffb 100644 --- a/packages/backend/server/src/__tests__/payment/service.spec.ts +++ b/packages/backend/server/src/__tests__/payment/service.spec.ts @@ -438,6 +438,37 @@ test('should throw if user has subscription already', async t => { ); }); +test('should allow checkout after local subscription period ended', async t => { + const { service, u1, db, stripe } = t.context; + + await db.subscription.create({ + data: { + targetId: u1.id, + stripeSubscriptionId: 'sub_expired_ai', + plan: SubscriptionPlan.AI, + recurring: SubscriptionRecurring.Yearly, + status: SubscriptionStatus.Active, + start: new Date('2026-05-04T13:11:45.000Z'), + end: new Date('2026-05-11T13:11:45.000Z'), + }, + }); + + await service.checkout( + { + plan: SubscriptionPlan.AI, + recurring: SubscriptionRecurring.Yearly, + successCallbackLink: '', + }, + { user: u1 } + ); + + t.true(stripe.checkout.sessions.create.calledOnce); + t.deepEqual(getLastCheckoutPrice(stripe.checkout.sessions.create), { + price: AI_YEARLY, + coupon: undefined, + }); +}); + test('should get correct pro plan price for checking out', async t => { const { app, service, u1, stripe, feature } = t.context; // non-ea user diff --git a/packages/backend/server/src/__tests__/utils/testing-app.ts b/packages/backend/server/src/__tests__/utils/testing-app.ts index 97f744e5a5..d6c88cb0e4 100644 --- a/packages/backend/server/src/__tests__/utils/testing-app.ts +++ b/packages/backend/server/src/__tests__/utils/testing-app.ts @@ -280,6 +280,7 @@ export class TestingApp extends ApplyType() { password: '1', name: email, emailVerifiedAt: new Date(), + createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000), ...override, }); diff --git a/packages/backend/server/src/__tests__/utils/utils.ts b/packages/backend/server/src/__tests__/utils/utils.ts index adbe1353f0..a797e4a197 100644 --- a/packages/backend/server/src/__tests__/utils/utils.ts +++ b/packages/backend/server/src/__tests__/utils/utils.ts @@ -11,13 +11,39 @@ async function flushDB(client: PrismaClient) { FROM pg_catalog.pg_tables WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema'`; + const query = `TRUNCATE TABLE ${result + .map(({ tablename }) => tablename) + .filter(name => !name.includes('migrations')) + .join(', ')}`; - // remove all table data - await client.$executeRawUnsafe( - `TRUNCATE TABLE ${result - .map(({ tablename }) => tablename) - .filter(name => !name.includes('migrations')) - .join(', ')}` + for (let attempt = 0; attempt < 3; attempt++) { + try { + // remove all table data + await client.$executeRawUnsafe(query); + return; + } catch (error) { + if (!isDeadlockError(error) || attempt === 2) { + throw error; + } + await sleep((attempt + 1) * 50); + } + } +} + +function isDeadlockError(error: unknown) { + if (typeof error !== 'object' || error === null || !('code' in error)) { + return false; + } + + const prismaError = error as { + code?: string; + meta?: { code?: string; message?: string }; + }; + + return ( + prismaError.code === 'P2010' && + (prismaError.meta?.code === '40P01' || + /deadlock detected/i.test(prismaError.meta?.message ?? '')) ); } diff --git a/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts b/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts index 1d770fac14..e4ba4cf19e 100644 --- a/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts +++ b/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts @@ -117,6 +117,24 @@ test('should remove job from queue', async t => { t.is(nullData, undefined); t.is(nullJob, undefined); }); + +test('should remove jobs by payload predicate', async t => { + const keep = await queue.add('nightly.__test__job', { name: 'keep' }); + const remove = await queue.add('nightly.__test__job', { name: 'remove' }); + const other = await queue.add('nightly.__test__job2', { name: 'remove' }); + const getJobs = Sinon.spy(bullmq, 'getJobs'); + + const removed = await queue.removeWhere( + 'nightly.__test__job', + job => job.name === 'remove' + ); + + t.deepEqual(getJobs.firstCall.args.slice(0, 3), [['waiting'], 0, 99]); + t.deepEqual(removed, [{ name: 'remove' }]); + t.truthy(await queue.get(keep.id!, 'nightly.__test__job')); + t.is(await queue.get(remove.id!, 'nightly.__test__job'), undefined); + t.truthy(await queue.get(other.id!, 'nightly.__test__job2')); +}); // #endregion // #region executor diff --git a/packages/backend/server/src/base/job/queue/queue.ts b/packages/backend/server/src/base/job/queue/queue.ts index 73c6e64725..c5729d968d 100644 --- a/packages/backend/server/src/base/job/queue/queue.ts +++ b/packages/backend/server/src/base/job/queue/queue.ts @@ -12,6 +12,15 @@ interface JobData { payload: Jobs[T]; } +const removableJobStates = [ + 'waiting', + 'delayed', + 'prioritized', + 'paused', + 'waiting-children', +] as const; +const removeWhereBatchSize = 100; + @Injectable() export class JobQueue { private readonly logger = new Logger(JobQueue.name); @@ -55,6 +64,59 @@ export class JobQueue { return undefined; } + async removeWhere( + jobName: T, + predicate: (payload: Jobs[T]) => boolean | Promise + ): Promise { + const ns = namespace(jobName); + const queue = this.getQueue(ns); + const removed: Jobs[T][] = []; + + for (const state of removableJobStates) { + let start = 0; + let removedFromBatch = false; + let hasMoreJobs = true; + + while (hasMoreJobs) { + removedFromBatch = false; + const jobs = (await queue.getJobs( + [state], + start, + start + removeWhereBatchSize - 1 + )) as Job>[]; + + if (!jobs.length) { + hasMoreJobs = false; + break; + } + + for (const job of jobs) { + if (job.name !== jobName) { + continue; + } + + const payload = job.data.payload; + if (!(await predicate(payload))) { + continue; + } + + await job.remove(); + this.logger.log( + `Job ${jobName}(id=${job.id}) removed from queue ${ns}` + ); + removed.push(payload); + removedFromBatch = true; + } + + if (!removedFromBatch) { + start += removeWhereBatchSize; + } + } + } + + return removed; + } + async get(jobId: string, jobName: T) { const ns = namespace(jobName); const queue = this.getQueue(ns); diff --git a/packages/backend/server/src/core/doc/reader.ts b/packages/backend/server/src/core/doc/reader.ts index c671f33171..d64912c137 100644 --- a/packages/backend/server/src/core/doc/reader.ts +++ b/packages/backend/server/src/core/doc/reader.ts @@ -16,6 +16,7 @@ import { parseDocToMarkdownFromDocSnapshot, parsePageDoc, parseWorkspaceDoc, + type WorkspaceDocContent, } from '../utils/blocksuite'; import { PgWorkspaceDocStorageAdapter } from './adapters/workspace'; import { type DocDiff, type DocRecord } from './storage'; @@ -242,10 +243,17 @@ export class DatabaseDocReader extends DocReader { if (!docRecord) { return null; } - const content = this.parseWorkspaceContent(docRecord.bin); - if (!content) { + let content: WorkspaceDocContent | null; + try { + content = this.parseWorkspaceContent(docRecord.bin); + } catch (error) { + this.logger.warn( + `Failed to parse workspace ${workspaceId} content`, + error as Error + ); return null; } + if (!content) return null; let avatarUrl: string | undefined; if (content.avatarKey) { avatarUrl = this.blobStorage.getAvatarUrl(workspaceId, content.avatarKey); diff --git a/packages/backend/server/src/core/mail/__tests__/job.spec.ts b/packages/backend/server/src/core/mail/__tests__/job.spec.ts new file mode 100644 index 0000000000..9e2283f1b7 --- /dev/null +++ b/packages/backend/server/src/core/mail/__tests__/job.spec.ts @@ -0,0 +1,243 @@ +import test from 'ava'; +import Sinon from 'sinon'; + +import { Mockers } from '../../../__tests__/mocks'; +import { createTestingModule } from '../../../__tests__/utils'; +import { Cache } from '../../../base'; +import { Models } from '../../../models'; +import { MailJob } from '../job'; +import { MailSender } from '../sender'; + +let module: Awaited>; +let cache: Cache; +let mailJob: MailJob; +let sender: MailSender; +let models: Models; + +test.before(async () => { + module = await createTestingModule(); + cache = module.get(Cache); + mailJob = module.get(MailJob); + sender = module.get(MailSender); + models = module.get(Models); +}); + +test.after.always(async () => { + await module.close(); +}); + +test.afterEach(() => { + Sinon.restore(); +}); + +test('should clear pending mail records when user is deleted', async t => { + const user = await module.create(Mockers.User); + const another = await module.create(Mockers.User); + const sendMailKey = 'mailjob:sendMail'; + const retryMailKey = 'mailjob:sendMail:retry'; + const userKey = `${sendMailKey}:SignIn:${user.email}`; + const userRetryKey = `${sendMailKey}:VerifyEmail:${user.email}`; + const anotherKey = `${sendMailKey}:SignIn:${another.email}`; + + await cache.mapSet(sendMailKey, userKey, 1); + await cache.mapSet(sendMailKey, anotherKey, 1); + await cache.mapSet( + retryMailKey, + userRetryKey, + JSON.stringify({ + startTime: Date.now(), + name: 'VerifyEmail', + to: user.email, + props: { url: 'https://affine.pro/verify' }, + }) + ); + + await mailJob.onUserDeleted({ ...user, ownedWorkspaces: [] }); + + t.true(module.queue.removeWhere.calledOnce); + t.is(module.queue.removeWhere.firstCall.args[0], 'notification.sendMail'); + const shouldRemove = module.queue.removeWhere.firstCall.args[1]; + t.true( + await shouldRemove({ + to: user.email, + } as Jobs['notification.sendMail']) + ); + t.false( + await shouldRemove({ + to: another.email, + } as Jobs['notification.sendMail']) + ); + t.is(await cache.mapGet(sendMailKey, userKey), undefined); + t.is(await cache.mapGet(retryMailKey, userRetryKey), undefined); + t.is(await cache.mapGet(sendMailKey, anotherKey), 1); +}); + +test('should skip queued mail for disabled recipient', async t => { + const user = await module.create(Mockers.User, { disabled: true }); + const send = Sinon.stub(sender, 'send').resolves(true); + + await mailJob.sendMail({ + startTime: Date.now(), + name: 'SignIn', + to: user.email, + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + }); + + t.false(send.called); + t.truthy(await models.user.get(user.id, { withDisabled: true })); +}); + +test('should drop expired mail retry', async t => { + const send = Sinon.stub(sender, 'send').resolves(true); + + await mailJob.sendMail({ + startTime: Date.now() - 25 * 60 * 60 * 1000, + name: 'SignIn', + to: 'expired-retry@example.com', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + }); + + t.false(send.called); +}); + +test('should drop time-sensitive mail after its business expiration', async t => { + const send = Sinon.stub(sender, 'send').resolves(true); + + await mailJob.sendMail({ + startTime: Date.now() - 31 * 60 * 1000, + name: 'SignIn', + to: 'expired-sign-in@example.com', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + }); + + t.false(send.called); +}); + +test('should use explicit mail expiration when provided', async t => { + const send = Sinon.stub(sender, 'send').resolves(true); + + await mailJob.sendMail({ + startTime: Date.now(), + expiresAt: Date.now() - 1, + name: 'MemberInvitation', + to: 'expired-invitation@example.com', + props: { + user: { + $$userId: 'owner-id', + }, + workspace: { + $$workspaceId: 'workspace-id', + }, + url: 'https://affine.pro/invite/test', + }, + }); + + t.false(send.called); +}); + +test('should drop mail retry after max attempts', async t => { + const send = Sinon.stub(sender, 'send').resolves(true); + + await mailJob.sendMail({ + startTime: Date.now(), + retryCount: 12, + name: 'SignIn', + to: 'max-retry@example.com', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + }); + + t.false(send.called); +}); + +test('should requeue legacy stringified retry mail', async t => { + const retryMailKey = 'mailjob:sendMail:retry'; + const job: Jobs['notification.sendMail'] = { + startTime: Date.now(), + name: 'SignIn', + to: 'legacy-retry@example.com', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + }; + const cacheKey = `${retryMailKey}:SignIn:${job.to}`; + + Sinon.stub(cache, 'mapRandomKey') + .onFirstCall() + .resolves(cacheKey) + .onSecondCall() + .resolves(undefined); + await cache.mapSet(retryMailKey, cacheKey, JSON.stringify(job)); + await mailJob.sendRetryMails(); + + t.true(module.queue.add.calledWith('notification.sendMail', job)); + t.is(await cache.mapGet(retryMailKey, cacheKey), undefined); +}); + +test('should skip member invitation mail when rendered workspace name contains domain', async t => { + const owner = await module.create(Mockers.User); + const member = await module.create(Mockers.User); + const workspace = await module.create(Mockers.Workspace, { + owner: { id: owner.id }, + name: 'BTC https://spam.example', + }); + const send = Sinon.stub(sender, 'send').resolves(true); + + await mailJob.sendMail({ + startTime: Date.now(), + name: 'MemberInvitation', + to: member.email, + props: { + user: { + $$userId: owner.id, + }, + workspace: { + $$workspaceId: workspace.id, + }, + url: 'https://affine.pro/invite/test', + }, + }); + + t.false(send.called); +}); + +test('should keep dynamic mail props untouched for retry', async t => { + const owner = await module.create(Mockers.User); + const member = await module.create(Mockers.User); + const workspace = await module.create(Mockers.Workspace, { + owner: { id: owner.id }, + name: 'Safe Workspace', + }); + Sinon.stub(sender, 'send').resolves(false); + const job: Jobs['notification.sendMail'] = { + startTime: Date.now(), + name: 'MemberInvitation', + to: member.email, + props: { + user: { + $$userId: owner.id, + }, + workspace: { + $$workspaceId: workspace.id, + }, + url: 'https://affine.pro/invite/test', + }, + }; + + await mailJob.sendMail(job); + + t.deepEqual(job.props.user, { $$userId: owner.id }); + t.deepEqual(job.props.workspace, { $$workspaceId: workspace.id }); +}); diff --git a/packages/backend/server/src/core/mail/job.ts b/packages/backend/server/src/core/mail/job.ts index fe427a219b..92dd2e5377 100644 --- a/packages/backend/server/src/core/mail/job.ts +++ b/packages/backend/server/src/core/mail/job.ts @@ -2,12 +2,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { getStreamAsBuffer } from 'get-stream'; -import { Cache, JOB_SIGNAL, JobQueue, OnJob, sleep } from '../../base'; +import { Cache, JOB_SIGNAL, JobQueue, OnEvent, OnJob, sleep } from '../../base'; import { type MailName, MailProps, Renderers } from '../../mails'; import { UserProps, WorkspaceProps } from '../../mails/components'; import { Models } from '../../models'; import { DocReader } from '../doc/reader'; import { WorkspaceBlobStorage } from '../storage'; +import { containsUrlOrDomain } from '../workspaces/abuse'; import { MailSender, SendOptions } from './sender'; type DynamicallyFetchedProps = { @@ -35,7 +36,11 @@ type SendMailJob> = { declare global { interface Jobs { - 'notification.sendMail': { startTime: number } & { + 'notification.sendMail': { + startTime: number; + retryCount?: number; + expiresAt?: number; + } & { [K in MailName]: SendMailJob; }[MailName]; } @@ -47,6 +52,19 @@ const sendMailCacheKey = (name: string, to: string) => `${sendMailKey}:${name}:${to}`; const retryMaxPerTick = 20; const retryFirstTime = 3; +const retryMaxAttempts = 12; +const retryMaxAge = 24 * 60 * 60 * 1000; +const magicLinkExpiresIn = 30 * 60 * 1000; + +const mailExpiresIn: Partial> = { + SignIn: magicLinkExpiresIn, + SignUp: magicLinkExpiresIn, + SetPassword: magicLinkExpiresIn, + ChangePassword: magicLinkExpiresIn, + VerifyEmail: magicLinkExpiresIn, + ChangeEmail: magicLinkExpiresIn, + VerifyChangeEmail: magicLinkExpiresIn, +}; @Injectable() export class MailJob { @@ -66,17 +84,65 @@ export class MailJob { return Math.min(30 * 1000, Math.round(elapsed / 2000) * 1000); } + private getRetryExhaustedReason({ + startTime, + retryCount, + expiresAt, + name, + }: Jobs['notification.sendMail']) { + const expiredAt = + expiresAt ?? startTime + (mailExpiresIn[name] ?? retryMaxAge); + if (Date.now() > expiredAt) { + return 'expired'; + } + + if ((retryCount ?? 0) > retryMaxAttempts) { + return 'max attempts reached'; + } + + return; + } + + private async shouldSkipRecipient(to: string) { + const user = await this.models.user.getUserByEmail(to, { + withDisabled: true, + }); + + return user?.disabled === true; + } + + private async deleteRecipientMailCache(to: string) { + const suffix = `:${to}`; + + await Promise.all( + [sendMailKey, retryMailKey].map(async map => { + const keys = await this.cache.mapKeys(map); + await Promise.all( + keys + .filter(key => key.endsWith(suffix)) + .map(key => this.cache.mapDelete(map, key)) + ); + }) + ); + } + private async sendMailInternal({ startTime, name, to, props, }: Jobs['notification.sendMail']) { - let options: Partial = {}; + if (await this.shouldSkipRecipient(to)) { + this.logger.debug(`Skip mail [${name}] to disabled user [${to}]`); + return; + } - for (const key in props) { + let options: Partial = {}; + const renderedProps = { ...props }; + + for (const key in renderedProps) { // @ts-expect-error allow - const val = props[key]; + const val = renderedProps[key]; if (val && typeof val === 'object') { if ('$$workspaceId' in val) { const workspaceProps = await this.fetchWorkspaceProps( @@ -87,6 +153,16 @@ export class MailJob { return; } + if ( + name === 'MemberInvitation' && + containsUrlOrDomain(workspaceProps.name) + ) { + this.logger.warn( + `Skip mail [${name}] to [${to}], reason=workspace name contains url or domain` + ); + return; + } + if (workspaceProps.avatar) { options.attachments = [ { @@ -99,7 +175,7 @@ export class MailJob { workspaceProps.avatar = 'cid:workspaceAvatar'; } // @ts-expect-error replacement - props[key] = workspaceProps; + renderedProps[key] = workspaceProps; } else if ('$$userId' in val) { const userProps = await this.fetchUserProps(val.$$userId); @@ -108,17 +184,30 @@ export class MailJob { } // @ts-expect-error replacement - props[key] = userProps; + renderedProps[key] = userProps; } } } + if ( + name === 'MemberInvitation' && + 'workspace' in renderedProps && + containsUrlOrDomain( + (renderedProps.workspace as WorkspaceProps | undefined)?.name + ) + ) { + this.logger.warn( + `Skip mail [${name}] to [${to}], reason=workspace name contains url or domain` + ); + return; + } + try { const result = await this.sender.send(name, { to, ...(await Renderers[name]( // @ts-expect-error the job trigger part has been typechecked - props + renderedProps )), ...options, }); @@ -177,17 +266,41 @@ export class MailJob { @OnJob('notification.sendMail') async sendMail(job: Jobs['notification.sendMail']) { const cacheKey = sendMailCacheKey(job.name, job.to); + job.retryCount = (job.retryCount ?? 0) + 1; + const exhaustedReason = this.getRetryExhaustedReason(job); + if (exhaustedReason) { + this.logger.warn( + `Drop mail [${job.name}] to [${job.to}], reason=${exhaustedReason}` + ); + await Promise.all([ + this.cache.mapDelete(sendMailKey, cacheKey), + this.cache.mapDelete(retryMailKey, cacheKey), + ]); + return; + } + const retried = await this.cache.mapIncrease(sendMailKey, cacheKey, 1); if (retried <= retryFirstTime) { const ret = await this.sendMailInternal(job); if (!ret) await this.cache.mapDelete(sendMailKey, cacheKey); return ret; } - await this.cache.mapSet(retryMailKey, cacheKey, JSON.stringify(job)); + await this.cache.mapSet(retryMailKey, cacheKey, job); await this.cache.mapDelete(sendMailKey, cacheKey); return undefined; } + @OnEvent('user.deleted') + async onUserDeleted(user: Events['user.deleted']) { + await Promise.all([ + this.deleteRecipientMailCache(user.email), + this.queue.removeWhere( + 'notification.sendMail', + job => job.to === user.email + ), + ]); + } + @Cron(CronExpression.EVERY_MINUTE) async sendRetryMails() { // pick random one from the retry map @@ -195,9 +308,14 @@ export class MailJob { let key = await this.cache.mapRandomKey(retryMailKey); while (key && processed < retryMaxPerTick) { try { - const job = await this.cache.mapGet(retryMailKey, key); + const job = await this.cache.mapGet< + Jobs['notification.sendMail'] | string + >(retryMailKey, key); if (job) { - const jobData = JSON.parse(job) as Jobs['notification.sendMail']; + const jobData = + typeof job === 'string' + ? (JSON.parse(job) as Jobs['notification.sendMail']) + : job; await this.queue.add('notification.sendMail', jobData); // wait for a while before retrying const retryDelay = this.calculateRetryDelay(jobData.startTime); diff --git a/packages/backend/server/src/core/mail/utils.ts b/packages/backend/server/src/core/mail/utils.ts index 33adf329a5..572406b751 100644 --- a/packages/backend/server/src/core/mail/utils.ts +++ b/packages/backend/server/src/core/mail/utils.ts @@ -17,7 +17,9 @@ function isValidSMTPAddressLiteral(hostname: string) { return false; } -export function normalizeSMTPHeloHostname(hostname: string) { +export function normalizeSMTPHeloHostname(hostname?: string) { + if (!hostname) return undefined; + const normalized = hostname.trim().replace(/\.$/, ''); if (!normalized) return undefined; if (isValidSMTPAddressLiteral(normalized)) return normalized; diff --git a/packages/backend/server/src/core/notification/__tests__/service.spec.ts b/packages/backend/server/src/core/notification/__tests__/service.spec.ts index dd1f4fd540..ba82af220b 100644 --- a/packages/backend/server/src/core/notification/__tests__/service.spec.ts +++ b/packages/backend/server/src/core/notification/__tests__/service.spec.ts @@ -87,6 +87,29 @@ test('should create invitation notification and email', async t => { t.is(invitationMail.payload.name, 'MemberInvitation'); }); +test('should not send invitation email when workspace name contains domain', async t => { + const spamWorkspace = await module.create(Mockers.Workspace, { + owner: { + id: owner.id, + }, + name: 'BTC https://spam.example', + }); + const inviteId = randomUUID(); + const invitationMailCount = module.queue.count('notification.sendMail'); + + const notification = await notificationService.createInvitation({ + userId: member.id, + body: { + workspaceId: spamWorkspace.id, + createdByUserId: owner.id, + inviteId, + }, + }); + + t.truthy(notification); + t.is(module.queue.count('notification.sendMail'), invitationMailCount); +}); + test('should not send invitation email if user setting is not to receive invitation email', async t => { const inviteId = randomUUID(); await module.create(Mockers.UserSettings, { diff --git a/packages/backend/server/src/core/notification/service.ts b/packages/backend/server/src/core/notification/service.ts index fa50273c7d..65e2b01715 100644 --- a/packages/backend/server/src/core/notification/service.ts +++ b/packages/backend/server/src/core/notification/service.ts @@ -23,6 +23,7 @@ import { generateWorkspaceSettingsPath, WorkspaceSettingsTab, } from '../utils/workspace'; +import { containsUrlOrDomain } from '../workspaces/abuse'; @Injectable() export class NotificationService { @@ -166,6 +167,16 @@ export class NotificationService { } private async sendInvitationEmail(input: InvitationNotificationCreate) { + const workspace = await this.docReader.getWorkspaceContent( + input.body.workspaceId + ); + if (containsUrlOrDomain(workspace?.name)) { + this.logger.warn( + `Skip invitation email for workspace ${input.body.workspaceId}, reason=workspace name contains url or domain` + ); + return; + } + const inviteUrl = this.url.link(`/invite/${input.body.inviteId}`); if (env.dev) { // make it easier to test in dev mode diff --git a/packages/backend/server/src/core/workspaces/__tests__/abuse.spec.ts b/packages/backend/server/src/core/workspaces/__tests__/abuse.spec.ts new file mode 100644 index 0000000000..26e6b4bc89 --- /dev/null +++ b/packages/backend/server/src/core/workspaces/__tests__/abuse.spec.ts @@ -0,0 +1,29 @@ +import test from 'ava'; + +import { + containsUrlOrDomain, + isUserOldEnoughForShareActions, + SHARE_ACTION_ACCOUNT_AGE_MS, +} from '../abuse'; + +test('should detect links and bare domains in workspace names', t => { + t.true(containsUrlOrDomain('BTC https://spam.example')); + t.true(containsUrlOrDomain('Join spam.example now')); + t.true(containsUrlOrDomain('Join spam.example, ltd')); + t.true(containsUrlOrDomain('Join spam.example。')); + t.true(containsUrlOrDomain('www.spam.example')); +}); + +test('should not detect email addresses or partial domain words', t => { + t.false(containsUrlOrDomain('Contact user@spam.example')); + t.false(containsUrlOrDomain('spam.example_btc')); +}); + +test('should check account age for share actions', t => { + t.false(isUserOldEnoughForShareActions({ createdAt: new Date() })); + t.true( + isUserOldEnoughForShareActions({ + createdAt: new Date(Date.now() - SHARE_ACTION_ACCOUNT_AGE_MS - 1), + }) + ); +}); diff --git a/packages/backend/server/src/core/workspaces/abuse.ts b/packages/backend/server/src/core/workspaces/abuse.ts new file mode 100644 index 0000000000..190899d9e9 --- /dev/null +++ b/packages/backend/server/src/core/workspaces/abuse.ts @@ -0,0 +1,12 @@ +export const SHARE_ACTION_ACCOUNT_AGE_MS = 24 * 60 * 60 * 1000; + +const URL_OR_DOMAIN_PATTERN = + /(?:https?:\/\/|www\.|(?= SHARE_ACTION_ACCOUNT_AGE_MS; +} diff --git a/packages/backend/server/src/core/workspaces/resolvers/doc.ts b/packages/backend/server/src/core/workspaces/resolvers/doc.ts index 4ba9aa6036..353c8255bf 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/doc.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/doc.ts @@ -15,6 +15,7 @@ import { Prisma, PrismaClient } from '@prisma/client'; import { SafeIntResolver } from 'graphql-scalars'; import { + ActionForbidden, Cache, DocActionDenied, DocDefaultRoleCanNotBeOwner, @@ -44,6 +45,7 @@ import { PermissionService, } from '../../permission'; import { PublicUserType, WorkspaceUserType } from '../../user'; +import { isUserOldEnoughForShareActions } from '../abuse'; import { DocGrantsService } from '../doc-grants'; import { WorkspaceType } from '../types'; import { TimeBucket, TimeWindow } from './analytics-types'; @@ -302,6 +304,15 @@ export class WorkspaceDocResolver { private readonly event: EventBus ) {} + private async assertCanShare(userId: string) { + const user = await this.models.user.get(userId); + if (!user || !isUserOldEnoughForShareActions(user)) { + throw new ActionForbidden( + 'Sharing links is unavailable during the first 24 hours after signup.' + ); + } + } + @ResolveField(() => WorkspaceDocMeta, { description: 'Cloud page metadata of workspace', complexity: 2, @@ -441,6 +452,7 @@ export class WorkspaceDocResolver { } await this.ac.user(user.id).doc(workspaceId, docId).assert('Doc.Publish'); + await this.assertCanShare(user.id); const doc = await this.models.doc.publish(workspaceId, docId, mode); this.event.emit('doc.public_state.changed', { workspaceId, docId }); diff --git a/packages/backend/server/src/core/workspaces/resolvers/member.ts b/packages/backend/server/src/core/workspaces/resolvers/member.ts index aa4eac875a..da43be2ddd 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/member.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/member.ts @@ -15,6 +15,7 @@ import { import { nanoid } from 'nanoid'; import { + ActionForbidden, ActionForbiddenOnNonTeamWorkspace, AlreadyInSpace, AuthenticationRequired, @@ -45,6 +46,7 @@ import { import { QuotaService } from '../../quota'; import { UserType } from '../../user'; import { validators } from '../../utils/validators'; +import { containsUrlOrDomain, isUserOldEnoughForShareActions } from '../abuse'; import { WorkspaceService } from '../service'; import { InvitationType, @@ -74,6 +76,24 @@ export class WorkspaceMemberResolver { private readonly quota: QuotaService ) {} + private async assertCanInviteOrShare(userId: string) { + const user = await this.models.user.get(userId); + if (!user || !isUserOldEnoughForShareActions(user)) { + throw new ActionForbidden( + 'Inviting members and creating share links are unavailable during the first 24 hours after signup.' + ); + } + } + + private async assertWorkspaceNameCanInvite(workspaceId: string) { + const workspace = await this.workspaceService.getWorkspaceInfo(workspaceId); + if (containsUrlOrDomain(workspace.name)) { + throw new ActionForbidden( + 'Workspace names containing links or domains cannot be used to invite members.' + ); + } + } + @ResolveField(() => UserType, { description: 'Owner of workspace', complexity: 2, @@ -149,6 +169,8 @@ export class WorkspaceMemberResolver { .user(me.id) .workspace(workspaceId) .assert('Workspace.Users.Manage'); + await this.assertCanInviteOrShare(me.id); + await this.assertWorkspaceNameCanInvite(workspaceId); if (emails.length > 512) { throw new TooManyRequest(); @@ -280,6 +302,8 @@ export class WorkspaceMemberResolver { .user(user.id) .workspace(workspaceId) .assert('Workspace.Users.Manage'); + await this.assertCanInviteOrShare(user.id); + await this.assertWorkspaceNameCanInvite(workspaceId); const cacheWorkspaceId = `workspace:inviteLink:${workspaceId}`; const invite = await this.cache.get<{ inviteId: string }>(cacheWorkspaceId); diff --git a/packages/backend/server/src/mails/index.tsx b/packages/backend/server/src/mails/index.tsx index be72c08aef..e0f597679d 100644 --- a/packages/backend/server/src/mails/index.tsx +++ b/packages/backend/server/src/mails/index.tsx @@ -83,95 +83,70 @@ export const Renderers = { //#region Workspace MemberInvitation: make( Invitation, - props => `${props.user.email} invited you to join ${props.workspace.name}` + 'You were invited to join a workspace on AFFiNE' ), MemberAccepted: make( InvitationAccepted, - props => `${props.user.email} accepted your invitation` - ), - MemberLeave: make( - MemberLeave, - props => `${props.user.email} left ${props.workspace.name}` + 'Your workspace invitation was accepted' ), + MemberLeave: make(MemberLeave, 'A workspace member left'), LinkInvitationReviewRequest: make( LinkInvitationReviewRequest, - props => `New request to join ${props.workspace.name}` + 'New request to join a workspace' ), LinkInvitationApprove: make( LinkInvitationApproved, - props => `Your request to join ${props.workspace.name} has been approved` + 'Your request to join a workspace has been approved' ), LinkInvitationDecline: make( LinkInvitationReviewDeclined, - props => `Your request to join ${props.workspace.name} was declined` - ), - MemberRemoved: make( - MemberRemoved, - props => `You have been removed from ${props.workspace.name}` + 'Your request to join a workspace was declined' ), + MemberRemoved: make(MemberRemoved, 'You have been removed from a workspace'), OwnershipTransferred: make( OwnershipTransferred, - props => `Your ownership of ${props.workspace.name} has been transferred` + 'Your workspace ownership has been transferred' ), OwnershipReceived: make( OwnershipReceived, - props => `You are now the owner of ${props.workspace.name}` + 'You are now the owner of a workspace' ), //#endregion //#region Doc - Mention: make( - Mention, - props => `${props.user.email} mentioned you in ${props.doc.title}` - ), - Comment: make( - Comment, - props => `${props.user.email} commented on ${props.doc.title}` - ), - CommentMention: make( - CommentMention, - props => - `${props.user.email} mentioned you in a comment on ${props.doc.title}` - ), + Mention: make(Mention, 'You were mentioned in AFFiNE'), + Comment: make(Comment, 'New comment in AFFiNE'), + CommentMention: make(CommentMention, 'You were mentioned in a comment'), //#endregion //#region Team TeamWorkspaceUpgraded: make(TeamWorkspaceUpgraded, props => props.isOwner ? 'Your workspace has been upgraded to team workspace! 🎉' - : `${props.workspace.name} has been upgraded to team workspace! 🎉` - ), - TeamBecomeAdmin: make( - TeamBecomeAdmin, - props => `You are now an admin of ${props.workspace.name}` + : 'A workspace has been upgraded to team workspace! 🎉' ), + TeamBecomeAdmin: make(TeamBecomeAdmin, 'You are now a workspace admin'), TeamBecomeCollaborator: make( TeamBecomeCollaborator, - props => `Your role has been changed in ${props.workspace.name}` + 'Your workspace role has been changed' ), TeamDeleteIn24Hours: make( TeamDeleteIn24Hours, - props => - `[Action Required] Final warning: Your workspace ${props.workspace.name} will be deleted in 24 hours` + '[Action Required] Final warning: Your workspace will be deleted in 24 hours' ), TeamDeleteInOneMonth: make( TeamDeleteInOneMonth, - props => - `[Action Required] Important: Your workspace ${props.workspace.name} will be deleted soon` + '[Action Required] Important: Your workspace will be deleted soon' ), TeamWorkspaceDeleted: make( TeamWorkspaceDeleted, - props => `Your workspace ${props.workspace.name} has been deleted` + 'Your workspace has been deleted' ), TeamWorkspaceExpireSoon: make( TeamExpireSoon, - props => - `[Action Required] Your ${props.workspace.name} team workspace will expire soon` - ), - TeamWorkspaceExpired: make( - TeamExpired, - props => `Your ${props.workspace.name} team workspace has expired` + '[Action Required] Your team workspace will expire soon' ), + TeamWorkspaceExpired: make(TeamExpired, 'Your team workspace has expired'), //#endregion //#region License diff --git a/packages/backend/server/src/plugins/payment/manager/user.ts b/packages/backend/server/src/plugins/payment/manager/user.ts index 0ce760dcbc..4aa50ce8f3 100644 --- a/packages/backend/server/src/plugins/payment/manager/user.ts +++ b/packages/backend/server/src/plugins/payment/manager/user.ts @@ -121,22 +121,17 @@ export class UserSubscriptionManager extends SubscriptionManager { throw new ManagedByAppStoreOrPlay(); } - const subscription = await this.getSubscription({ - plan: lookupKey.plan, - userId: user.id, - }); - if ( - subscription && + active && // do not allow to re-subscribe unless !( /* current subscription is a onetime subscription and so as the one that's checking out */ ( - (subscription.variant === SubscriptionVariant.Onetime && + (active.variant === SubscriptionVariant.Onetime && lookupKey.variant === SubscriptionVariant.Onetime) || /* current subscription is normal subscription and is checking-out a lifetime subscription */ - (subscription.recurring !== SubscriptionRecurring.Lifetime && - subscription.variant !== SubscriptionVariant.Onetime && + (active.recurring !== SubscriptionRecurring.Lifetime && + active.variant !== SubscriptionVariant.Onetime && lookupKey.recurring === SubscriptionRecurring.Lifetime) ) ) diff --git a/packages/common/graphql/src/graphql/admin/send-test-email.gql b/packages/common/graphql/src/graphql/admin/send-test-email.gql index 2af841df26..8e861e500a 100644 --- a/packages/common/graphql/src/graphql/admin/send-test-email.gql +++ b/packages/common/graphql/src/graphql/admin/send-test-email.gql @@ -1,4 +1,5 @@ mutation sendTestEmail( + $name: String! $host: String! $port: Int! $sender: String! @@ -8,6 +9,7 @@ mutation sendTestEmail( ) { sendTestEmail( config: { + name: $name host: $host port: $port sender: $sender diff --git a/packages/common/graphql/src/graphql/index.ts b/packages/common/graphql/src/graphql/index.ts index 2235f10d3c..b587b45d92 100644 --- a/packages/common/graphql/src/graphql/index.ts +++ b/packages/common/graphql/src/graphql/index.ts @@ -448,9 +448,9 @@ export const listUsersQuery = { export const sendTestEmailMutation = { id: 'sendTestEmailMutation' as const, op: 'sendTestEmail', - query: `mutation sendTestEmail($host: String!, $port: Int!, $sender: String!, $username: String!, $password: String!, $ignoreTLS: Boolean!) { + query: `mutation sendTestEmail($name: String!, $host: String!, $port: Int!, $sender: String!, $username: String!, $password: String!, $ignoreTLS: Boolean!) { sendTestEmail( - config: {host: $host, port: $port, sender: $sender, username: $username, password: $password, ignoreTLS: $ignoreTLS} + config: {name: $name, host: $host, port: $port, sender: $sender, username: $username, password: $password, ignoreTLS: $ignoreTLS} ) }`, }; diff --git a/packages/common/graphql/src/schema.ts b/packages/common/graphql/src/schema.ts index b19a492081..6f626a4cda 100644 --- a/packages/common/graphql/src/schema.ts +++ b/packages/common/graphql/src/schema.ts @@ -4175,6 +4175,7 @@ export type ListUsersQuery = { }; export type SendTestEmailMutationVariables = Exact<{ + name: Scalars['String']['input']; host: Scalars['String']['input']; port: Scalars['Int']['input']; sender: Scalars['String']['input']; diff --git a/tests/affine-local/e2e/blocksuite/paragraph.spec.ts b/tests/affine-local/e2e/blocksuite/paragraph.spec.ts index cb0ecf2895..5c94ee4669 100644 --- a/tests/affine-local/e2e/blocksuite/paragraph.spec.ts +++ b/tests/affine-local/e2e/blocksuite/paragraph.spec.ts @@ -328,25 +328,9 @@ test('also move children when dedent collapsed heading', async ({ page }) => { const paragraph = page.locator('affine-note affine-paragraph'); const subParagraph = paragraph.nth(0).locator('affine-paragraph'); - expect(await subParagraph.count()).toBe(2); - expect( - await subParagraph - .nth(0) - .evaluate( - (block: ParagraphBlockComponent) => - block.model.props.type === 'h1' && - block.model.props.text.toString() === 'bbb' - ) - ).toBeTruthy(); - expect( - await subParagraph - .nth(1) - .evaluate( - (block: ParagraphBlockComponent) => - block.model.props.type === 'text' && - block.model.props.text.toString() === 'ccc' - ) - ).toBeTruthy(); + await expect.poll(() => subParagraph.count()).toBe(2); + await expectParagraphState(subParagraph, 0, 'h1', 'bbb'); + await expectParagraphState(subParagraph, 1, 'text', 'ccc'); expect(await subParagraph.nth(1).isVisible()).toBeTruthy(); await subParagraph @@ -442,7 +426,12 @@ test('unfold collapsed heading when its other blocks indented to be its sibling' await type(page, '# bbb\nddd'); await page.keyboard.press('ArrowUp'); await pressTab(page); - await page.keyboard.press('ArrowRight'); + const paragraph = page.locator('affine-note affine-paragraph'); + const subParagraph = paragraph.nth(0).locator('affine-paragraph'); + await expect.poll(() => subParagraph.count()).toBe(1); + await expectParagraphState(subParagraph, 0, 'h1', 'bbb'); + await subParagraph.nth(0).click(); + await page.keyboard.press('End'); await pressEnter(page); await type(page, 'ccc'); @@ -453,7 +442,6 @@ test('unfold collapsed heading when its other blocks indented to be its sibling' * ddd */ - const paragraph = page.locator('affine-note affine-paragraph'); await expectParagraphVisibility(paragraph, 2, true); await expectParagraphState(paragraph, 2, 'text', 'ccc'); await paragraph.locator('blocksuite-toggle-button .toggle-icon').click(); diff --git a/tests/kit/src/utils/cloud.ts b/tests/kit/src/utils/cloud.ts index 8bfcd2510b..74c34de157 100644 --- a/tests/kit/src/utils/cloud.ts +++ b/tests/kit/src/utils/cloud.ts @@ -116,6 +116,7 @@ export async function createRandomUser(): Promise<{ data: { ...user, emailVerifiedAt: new Date(), + createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000), password: await hash(user.password), features: { create: {