From 998b255afd8f8094ca2a8380fbc967eb6b4142a4 Mon Sep 17 00:00:00 2001 From: DarkSky <25152247+darkskygit@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:38:16 +0800 Subject: [PATCH] feat(server): refactor mail queue (#15204) --- .docker/selfhost/schema.json | 42 ++ Cargo.lock | 26 +- packages/backend/native/Cargo.toml | 2 + packages/backend/native/index.d.ts | 131 ++++ packages/backend/native/src/content_policy.rs | 333 +++++++++ packages/backend/native/src/lib.rs | 1 + .../native/src/runtime/backend_runtime/mod.rs | 6 +- .../rolling_quota/invite_abuse_actions.rs | 261 +++++++ .../rolling_quota/mail_delivery.rs | 172 +++++ .../backend_runtime/rolling_quota/mod.rs | 175 +++++ .../rolling_quota/reservation.rs | 344 +++++++++ .../rolling_quota/workspace_invite.rs | 452 ++++++++++++ .../rolling_quota/workspace_invite_policy.rs | 654 +++++++++++++++++ .../src/runtime/backend_runtime/tests.rs | 390 +++++++++- packages/backend/native/src/runtime/config.rs | 94 ++- packages/backend/native/src/runtime/mod.rs | 2 +- .../src/runtime/sql/runtime_migrations.sql | 87 +++ packages/backend/native/src/runtime/types.rs | 101 +++ .../migration.sql | 132 ++++ packages/backend/server/schema.prisma | 52 ++ .../server/src/__tests__/auth/auth.e2e.ts | 2 +- .../src/__tests__/e2e/user/account.spec.ts | 27 + .../server/src/__tests__/mocks/mailer.mock.ts | 22 +- .../server/src/base/job/queue/config.ts | 8 + .../backend/server/src/base/job/queue/def.ts | 1 + .../backend/server/src/core/auth/config.ts | 18 + .../server/src/core/auth/controller.ts | 11 +- .../server/src/core/auth/magic-link.ts | 34 +- .../backend/server/src/core/auth/resolver.ts | 109 ++- .../backend/server/src/core/auth/service.ts | 40 +- .../backend-runtime/__tests__/job.spec.ts | 5 + .../server/src/core/backend-runtime/index.ts | 13 +- .../server/src/core/backend-runtime/job.ts | 5 +- .../src/core/backend-runtime/provider.ts | 281 ++++++++ .../backend/server/src/core/content-policy.ts | 14 + .../src/core/mail/__tests__/job.spec.ts | 431 +++++++----- .../src/core/mail/__tests__/mailer.spec.ts | 316 +++++++++ .../src/core/mail/__tests__/resolver.spec.ts | 257 +++++++ .../backend/server/src/core/mail/config.ts | 20 + .../backend/server/src/core/mail/index.ts | 3 +- packages/backend/server/src/core/mail/job.ts | 421 +++++------ .../backend/server/src/core/mail/mailer.ts | 176 ++++- .../backend/server/src/core/mail/resolver.ts | 306 +++++++- .../backend/server/src/core/mail/sender.ts | 67 +- .../backend/server/src/core/mail/types.ts | 45 ++ .../notification/__tests__/service.spec.ts | 109 +-- .../server/src/core/notification/service.ts | 305 +++++--- .../core/workspaces/__tests__/abuse.spec.ts | 400 ++++++++++- .../server/src/core/workspaces/abuse.ts | 382 +++++++++- .../server/src/core/workspaces/event.ts | 5 + .../server/src/core/workspaces/index.ts | 11 + .../workspaces/resolvers/analytics-types.ts | 1 + .../src/core/workspaces/resolvers/doc.ts | 24 +- .../src/core/workspaces/resolvers/member.ts | 227 ++++-- .../server/src/core/workspaces/service.ts | 79 ++- packages/backend/server/src/models/index.ts | 3 + .../server/src/models/magic-link-otp.ts | 1 + .../server/src/models/mail-delivery.ts | 665 ++++++++++++++++++ .../server/src/models/permission-write.ts | 26 + .../server/src/models/verification-token.ts | 18 +- packages/backend/server/src/native.ts | 5 + .../src/plugins/payment/manager/selfhost.ts | 4 + packages/backend/server/src/schema.gql | 40 ++ .../graphql/admin/admin-mail-deliveries.gql | 50 ++ packages/common/graphql/src/graphql/index.ts | 55 ++ packages/common/graphql/src/schema.ts | 117 +++ packages/frontend/admin/src/config.json | 28 + .../src/modules/dashboard/index.spec.tsx | 91 ++- .../admin/src/modules/dashboard/index.tsx | 400 ++++++++++- .../i18n/src/i18n-completenesses.json | 4 +- packages/frontend/i18n/src/i18n.gen.ts | 8 - packages/frontend/i18n/src/resources/en.json | 2 - 72 files changed, 8386 insertions(+), 763 deletions(-) create mode 100644 packages/backend/native/src/content_policy.rs create mode 100644 packages/backend/native/src/runtime/backend_runtime/rolling_quota/invite_abuse_actions.rs create mode 100644 packages/backend/native/src/runtime/backend_runtime/rolling_quota/mail_delivery.rs create mode 100644 packages/backend/native/src/runtime/backend_runtime/rolling_quota/mod.rs create mode 100644 packages/backend/native/src/runtime/backend_runtime/rolling_quota/reservation.rs create mode 100644 packages/backend/native/src/runtime/backend_runtime/rolling_quota/workspace_invite.rs create mode 100644 packages/backend/native/src/runtime/backend_runtime/rolling_quota/workspace_invite_policy.rs create mode 100644 packages/backend/server/migrations/20260706010719_mail_deliveries_and_invite_quota_indexes/migration.sql create mode 100644 packages/backend/server/src/core/content-policy.ts create mode 100644 packages/backend/server/src/core/mail/__tests__/mailer.spec.ts create mode 100644 packages/backend/server/src/core/mail/__tests__/resolver.spec.ts create mode 100644 packages/backend/server/src/core/mail/types.ts create mode 100644 packages/backend/server/src/models/mail-delivery.ts create mode 100644 packages/common/graphql/src/graphql/admin/admin-mail-deliveries.gql diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index 46ae6a40c5..1207337240 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -133,6 +133,18 @@ "default": { "concurrency": 1 } + }, + "queues.inviteAbuse": { + "type": "object", + "description": "The config for invite abuse disposition job queue\n@default {\"concurrency\":1}", + "properties": { + "concurrency": { + "type": "number" + } + }, + "default": { + "concurrency": 1 + } } } }, @@ -192,6 +204,21 @@ "description": "Minimum account age in seconds before new accounts can invite members or create share links.\n@default 86400", "default": 86400 }, + "trustedCloudflareHeaders": { + "type": "boolean", + "description": "Whether request abuse source facts should trust Cloudflare headers from the origin edge.\n@default false", + "default": false + }, + "inviteQuotaShadowMode": { + "type": "boolean", + "description": "Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions.\n@default false", + "default": false + }, + "inviteQuotaFailOpenOnRuntimeError": { + "type": "boolean", + "description": "Whether workspace invite quota should fail open when native runtime admission is unavailable. Keep disabled for production.\n@default false", + "default": false + }, "passwordRequirements": { "type": "object", "description": "The password strength requirements when set new password.\n@default {\"min\":8,\"max\":32}", @@ -264,6 +291,21 @@ "description": "The emails from these domains are always sent using the fallback SMTP server.\n@default []", "default": [] }, + "deliveryWorker.batchSize": { + "type": "number", + "description": "Number of mail delivery rows claimed by each worker tick.\n@default 50", + "default": 50 + }, + "deliveryWorker.leaseMs": { + "type": "number", + "description": "Mail delivery worker lease duration in milliseconds.\n@default 120000", + "default": 120000 + }, + "deliveryWorker.retentionDays": { + "type": "number", + "description": "Days to retain anonymized terminal mail delivery ledger rows.\n@default 30", + "default": 30 + }, "fallbackSMTP.name": { "type": "string", "description": "Hostname used for fallback SMTP HELO/EHLO (e.g. mail.example.com). Leave empty to use the system hostname.\n@default \"\"", diff --git a/Cargo.lock b/Cargo.lock index ffa99c5394..07f768b81e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -270,6 +270,8 @@ dependencies = [ "thiserror 2.0.18", "tiktoken-rs 0.7.0", "tokio", + "unicode-normalization", + "unicode_skeleton", "url", "uuid", "v_htmlescape", @@ -2225,7 +2227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3785,7 +3787,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5870,7 +5872,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6312,7 +6314,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6371,7 +6373,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7132,7 +7134,6 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -7583,7 +7584,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8668,6 +8669,15 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_skeleton" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd74db2c088d393d1fbf83db2cd5663137640f072d128287dd53c882a0f412" +dependencies = [ + "unicode-normalization", +] + [[package]] name = "uniffi" version = "0.29.5" @@ -9300,7 +9310,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/packages/backend/native/Cargo.toml b/packages/backend/native/Cargo.toml index 0e8e087452..e88be067d7 100644 --- a/packages/backend/native/Cargo.toml +++ b/packages/backend/native/Cargo.toml @@ -62,6 +62,8 @@ sqlx = { workspace = true, default-features = false, features = [ thiserror.workspace = true tiktoken-rs = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] } +unicode-normalization = "0.1" +unicode_skeleton = "0.1.1" url = { workspace = true } uuid = { workspace = true, features = ["v4"] } v_htmlescape = { workspace = true } diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index 8f9e4d77df..a89d127825 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -27,6 +27,18 @@ export declare class BackendRuntime { cleanupExpiredRuntimeGates(limit: number): Promise cleanupExpiredUserSessions(limit: number): Promise cleanupExpiredSnapshotHistories(limit: number): Promise + isInviteAbuseUserQuarantinedOrBanned(userId: string): Promise + isInviteAbuseWorkspaceQuarantined(workspaceId: string): Promise + claimInviteAbuseAction(actionId: string, workerId: string): Promise + claimRetryableInviteAbuseActions(workerId: string, limit: number): Promise> + markInviteAbuseAction(actionId: string, workerId: string, status: string, error?: string | undefined | null): Promise + assertWorkspaceInviteQuotaV1(input: RuntimeWorkspaceInviteQuotaInput): Promise + commitWorkspaceInviteQuotaV1(reservationId: string, usage: RuntimeWorkspaceInviteQuotaUsage): Promise + releaseWorkspaceInviteQuotaV1(reservationId: string): Promise + assertMailDeliveryQuotaV1(input: RuntimeMailDeliveryQuotaInput): Promise + commitMailDeliveryQuotaV1(reservationId: string): Promise + releaseMailDeliveryQuotaV1(reservationId: string): Promise + cleanupExpiredRollingQuota(limit: number): Promise createAuthChallenge(purpose: string, token: string, payload: any, ttlMs: number): Promise getAuthChallenge(purpose: string, token: string): Promise consumeAuthChallenge(purpose: string, token: string): Promise @@ -254,6 +266,33 @@ export interface CommandResponse { error?: LicenseError } +export interface ContentPolicyMatch { + type: string + reason: string + value?: string + span?: ContentPolicyMatchSpan +} + +export interface ContentPolicyMatchSpan { + start: number + end: number +} + +export interface ContentPolicyScanInput { + value: string + checks?: Array +} + +export interface ContentPolicyScanResult { + version: number + original: string + normalized: string + skeleton: string + matched: boolean + matches: Array + flags: Array +} + export interface CoordinationLeaseGrant { key: string owner: string @@ -914,12 +953,62 @@ export interface RuntimeDocHistoryInput { historyMaxAgeMs: number } +export interface RuntimeInviteAbuseActionRequired { + action: string + subjectKey: string + evidenceId: string + actionId: string +} + +export interface RuntimeInviteAbuseClaimedAction { + action: string + subjectKey: string + evidenceId: string + actionId: string + actorUserId: string + workspaceId: string +} + export interface RuntimeMagicLinkOtpConsumeResult { ok: boolean token?: string reason?: string } +export interface RuntimeMailDeliveryQuotaDecision { + allowed: boolean + reservationId?: string + mailClass: string + retryAfterSeconds?: number + reason?: string + scopeKey?: string + windowSeconds?: number + limit?: number + current?: number + requested?: number +} + +export interface RuntimeMailDeliveryQuotaInput { + requestId?: string + mailName: string + recipient: RuntimeMailDeliveryQuotaRecipientInput + metadata: RuntimeMailDeliveryQuotaMetadataInput + source?: RuntimeQuotaSourceInput +} + +export interface RuntimeMailDeliveryQuotaMetadataInput { + actorUserId?: string + workspaceId?: string + notificationId?: string + abuseSubjectKey?: string +} + +export interface RuntimeMailDeliveryQuotaRecipientInput { + email: string + domain: string + userId?: string +} + export interface RuntimeMultipartUploadInit { uploadId: string expiresAtMs: number @@ -960,6 +1049,19 @@ export interface RuntimePresignedObjectRequest { expiresAtMs: number } +export interface RuntimeQuotaSourceInput { + trusted: boolean + ip?: string + country?: string + asn?: number + rayId?: string +} + +export interface RuntimeQuotaTargetDomainInput { + domain: string + count: number +} + export interface RuntimeVerificationTokenRecord { tokenType: number token: string @@ -974,6 +1076,33 @@ export interface RuntimeWorkspaceInviteLinkRecord { expiresAtMs: number } +export interface RuntimeWorkspaceInviteQuotaDecision { + allowed: boolean + reservationId?: string + retryAfterSeconds?: number + reason?: string + scopeKey?: string + windowSeconds?: number + limit?: number + current?: number + requested?: number + actionRequired?: RuntimeInviteAbuseActionRequired +} + +export interface RuntimeWorkspaceInviteQuotaInput { + actorUserId: string + workspaceId: string + requestId?: string + targetCount: number + targetDomains: Array + source?: RuntimeQuotaSourceInput +} + +export interface RuntimeWorkspaceInviteQuotaUsage { + targetCount: number + targetDomains: Array +} + export interface RuntimeWorkspaceStatsDailyRecalibrationResult { processed: number lastSid: number @@ -1030,6 +1159,8 @@ export interface SafeFetchResponse { body: Buffer } +export declare function scanContentPolicyV1(input: ContentPolicyScanInput): ContentPolicyScanResult + export interface StorageProviderCapabilities { put: boolean get: boolean diff --git a/packages/backend/native/src/content_policy.rs b/packages/backend/native/src/content_policy.rs new file mode 100644 index 0000000000..c9a72e4744 --- /dev/null +++ b/packages/backend/native/src/content_policy.rs @@ -0,0 +1,333 @@ +use napi_derive::napi; +use unicode_normalization::UnicodeNormalization; +use unicode_skeleton::UnicodeSkeleton; + +const VERSION: u32 = 1; + +#[napi(object)] +pub struct ContentPolicyScanInput { + pub value: String, + pub checks: Option>, +} + +#[napi(object)] +pub struct ContentPolicyMatchSpan { + pub start: u32, + pub end: u32, +} + +#[napi(object)] +pub struct ContentPolicyMatch { + #[napi(js_name = "type")] + pub match_type: String, + pub reason: String, + pub value: Option, + pub span: Option, +} + +#[napi(object)] +pub struct ContentPolicyScanResult { + pub version: u32, + pub original: String, + pub normalized: String, + pub skeleton: String, + pub matched: bool, + pub matches: Vec, + pub flags: Vec, +} + +#[napi] +pub fn scan_content_policy_v1(input: ContentPolicyScanInput) -> ContentPolicyScanResult { + let original = input.value; + let normalized = normalize_content(&original); + let skeleton = skeleton_content(&normalized); + let mut flags = Vec::new(); + if normalized != original { + flags.push("unicode_normalized".to_string()); + } + if skeleton != normalized { + flags.push("confusable_skeleton_changed".to_string()); + } + + let mut matches = Vec::new(); + if should_run_check(input.checks.as_deref(), "url_or_domain") { + matches.extend(scan_url_or_domain(&skeleton)); + } + + ContentPolicyScanResult { + version: VERSION, + original, + normalized, + skeleton, + matched: !matches.is_empty(), + matches, + flags, + } +} + +fn should_run_check(checks: Option<&[String]>, check: &str) -> bool { + checks + .map(|checks| checks.iter().any(|configured| configured == check)) + .unwrap_or(true) +} + +fn normalize_content(value: &str) -> String { + value.nfkc().collect::() +} + +fn skeleton_content(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + for ch in value.chars() { + push_skeleton_char(&mut output, ch); + } + output +} + +fn push_skeleton_char(output: &mut String, ch: char) { + if is_default_ignorable(ch) { + return; + } + if let Some(replacement) = url_punctuation_ascii(ch).or_else(|| enclosed_ascii(ch)) { + output.push_str(replacement); + return; + } + if ch.is_ascii() { + output.extend(ch.to_lowercase()); + return; + } + + let source = ch.to_string(); + let skeleton = source.skeleton_chars().collect::(); + if skeleton == source { + output.extend(ch.to_lowercase()); + return; + } + + for skeleton_ch in skeleton.chars() { + push_skeleton_char(output, skeleton_ch); + } +} + +fn is_default_ignorable(ch: char) -> bool { + matches!( + ch, + '\u{00ad}' + | '\u{034f}' + | '\u{061c}' + | '\u{180e}' + | '\u{200b}'..='\u{200f}' + | '\u{202a}'..='\u{202e}' + | '\u{2060}'..='\u{206f}' + | '\u{fe00}'..='\u{fe0f}' + | '\u{feff}' + | '\u{e0100}'..='\u{e01ef}' + ) +} + +fn enclosed_ascii(ch: char) -> Option<&'static str> { + let code = ch as u32; + if (0x1f150..=0x1f169).contains(&code) { + return ASCII_LOWERCASE.get((code - 0x1f150) as usize).copied(); + } + if ch == '⓿' || ch == '🄌' { + return Some("0"); + } + for (start, digits) in [ + (0x2776, ASCII_DIGITS_1_TO_9), + (0x2780, ASCII_DIGITS_1_TO_9), + (0x278a, ASCII_DIGITS_1_TO_9), + ] { + if (start..=start + 8).contains(&code) { + return digits.get((code - start) as usize).copied(); + } + } + None +} + +const ASCII_LOWERCASE: [&str; 26] = [ + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", + "x", "y", "z", +]; + +const ASCII_DIGITS_1_TO_9: [&str; 9] = ["1", "2", "3", "4", "5", "6", "7", "8", "9"]; + +fn url_punctuation_ascii(ch: char) -> Option<&'static str> { + match ch { + // URL punctuation variants that are not always useful in generic skeletons. + '。' | '．' | '｡' | '․' | '‧' | '∙' | '●' | '•' => Some("."), + '⁄' | '∕' | '╱' | '／' => Some("/"), + '：' | '꞉' | 'ː' | '։' => Some(":"), + '‐' | '‑' | '‒' | '–' | '—' | '―' | '−' | '﹘' | '－' => Some("-"), + '_' | '＿' => Some("_"), + _ => None, + } +} + +fn scan_url_or_domain(value: &str) -> Vec { + let mut matches = Vec::new(); + let chars = value.char_indices().collect::>(); + let mut index = 0; + while index < chars.len() { + let start = chars[index].0; + let remaining = &value[start..]; + if remaining.starts_with("http://") || remaining.starts_with("https://") || remaining.starts_with("www.") { + let end = consume_non_space(value, start); + matches.push(match_result(value, start, end)); + index = char_index_after(&chars, end); + continue; + } + + if is_domain_boundary_before(value, start) { + let end = consume_domain(value, start); + if end > start && is_domain_candidate(&value[start..end]) && is_domain_boundary_after(value, end) { + matches.push(match_result(value, start, end)); + index = char_index_after(&chars, end); + continue; + } + } + index += 1; + } + matches +} + +fn match_result(value: &str, start: usize, end: usize) -> ContentPolicyMatch { + ContentPolicyMatch { + match_type: "url_or_domain".to_string(), + reason: "contains_url_or_domain".to_string(), + value: Some(value[start..end].to_string()), + span: Some(ContentPolicyMatchSpan { + start: start as u32, + end: end as u32, + }), + } +} + +fn consume_non_space(value: &str, start: usize) -> usize { + value[start..] + .char_indices() + .find_map(|(offset, ch)| ch.is_whitespace().then_some(start + offset)) + .unwrap_or(value.len()) +} + +fn consume_domain(value: &str, start: usize) -> usize { + value[start..] + .char_indices() + .find_map(|(offset, ch)| (!is_domain_char(ch)).then_some(start + offset)) + .unwrap_or(value.len()) +} + +fn is_domain_candidate(candidate: &str) -> bool { + let labels = candidate.split('.').collect::>(); + if labels.len() < 2 { + return false; + } + let Some(tld) = labels.last() else { + return false; + }; + if !(2..=63).contains(&tld.len()) || !tld.chars().all(|ch| ch.is_ascii_alphabetic()) { + return false; + } + labels.iter().all(|label| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '-') + }) +} + +fn is_domain_char(ch: char) -> bool { + ch.is_ascii_alphanumeric() || ch == '-' || ch == '.' +} + +fn is_domain_boundary_before(value: &str, start: usize) -> bool { + if start == 0 { + return true; + } + let Some(previous) = value[..start].chars().next_back() else { + return true; + }; + !(previous.is_ascii_alphanumeric() || previous == '-' || previous == '_' || previous == '@') +} + +fn is_domain_boundary_after(value: &str, end: usize) -> bool { + if end >= value.len() { + return true; + } + let Some(next) = value[end..].chars().next() else { + return true; + }; + !(next.is_ascii_alphanumeric() || next == '-' || next == '_' || next == '.') +} + +fn char_index_after(chars: &[(usize, char)], byte_index: usize) -> usize { + chars + .iter() + .position(|(offset, _)| *offset >= byte_index) + .unwrap_or(chars.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scan(value: &str) -> ContentPolicyScanResult { + scan_content_policy_v1(ContentPolicyScanInput { + value: value.to_string(), + checks: None, + }) + } + + #[test] + fn scans_url_or_domain_cases() { + struct Case { + input: &'static str, + matched: bool, + skeleton: &'static str, + value: Option<&'static str>, + } + + for case in [ + Case { + input: "https://ｅxample．com", + matched: true, + skeleton: "https://example.com", + value: Some("https://example.com"), + }, + Case { + input: "join 🅠⓿❶.example", + matched: true, + skeleton: "join q01.example", + value: Some("q01.example"), + }, + Case { + input: "exa\u{200b}mple.com", + matched: true, + skeleton: "example.com", + value: Some("example.com"), + }, + Case { + input: "hello workspace", + matched: false, + skeleton: "hello workspace", + value: None, + }, + Case { + input: "user@example.com", + matched: false, + skeleton: "user@example.com", + value: None, + }, + ] { + let result = scan(case.input); + assert_eq!(result.matched, case.matched, "{}", case.input); + assert_eq!(result.skeleton, case.skeleton, "{}", case.input); + assert_eq!( + result.matches.first().and_then(|matched| matched.value.as_deref()), + case.value, + "{}", + case.input + ); + } + } +} diff --git a/packages/backend/native/src/lib.rs b/packages/backend/native/src/lib.rs index c17852971f..6005e49c97 100644 --- a/packages/backend/native/src/lib.rs +++ b/packages/backend/native/src/lib.rs @@ -2,6 +2,7 @@ mod utils; +pub mod content_policy; pub mod doc; pub mod doc_loader; pub mod entitlement; diff --git a/packages/backend/native/src/runtime/backend_runtime/mod.rs b/packages/backend/native/src/runtime/backend_runtime/mod.rs index 7068b21db8..75bd1f1fb4 100644 --- a/packages/backend/native/src/runtime/backend_runtime/mod.rs +++ b/packages/backend/native/src/runtime/backend_runtime/mod.rs @@ -4,6 +4,7 @@ mod doc_compactor; mod doc_storage; mod gate; mod housekeeping; +mod rolling_quota; mod runtime_state; #[cfg(test)] mod tests; @@ -17,8 +18,9 @@ use tokio::sync::Mutex; use self::types::BackendRuntimeHealth; pub(crate) use super::types; -use super::{ - BackendRuntimeConfig, RuntimeError, RuntimeResult, migrations::migrate_runtime_tables, napi_error, to_napi_error, +pub(super) use super::{ + BackendRuntimeConfig, InviteQuotaConfig, RuntimeError, RuntimeResult, migrations::migrate_runtime_tables, napi_error, + to_napi_error, }; pub(super) fn token_hash(token: &str) -> String { diff --git a/packages/backend/native/src/runtime/backend_runtime/rolling_quota/invite_abuse_actions.rs b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/invite_abuse_actions.rs new file mode 100644 index 0000000000..175ca87369 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/invite_abuse_actions.rs @@ -0,0 +1,261 @@ +use napi::Result; +use sqlx::{PgPool, Row}; + +use super::{ + BackendRuntime, RuntimeError, RuntimeInviteAbuseClaimedAction, RuntimeResult, napi_error, workspace_subject_key, +}; + +async fn invite_abuse_user_quarantined_or_banned(pool: &PgPool, user_id: &str) -> RuntimeResult { + let row: Option = sqlx::query_scalar( + r#" + SELECT 1 + FROM runtime_invite_abuse_subjects + WHERE user_id = $1 + AND status IN ('quarantined', 'banned') + LIMIT 1 + "#, + ) + .bind(user_id) + .fetch_optional(pool) + .await + .map_err(|err| RuntimeError::database("failed to load invite abuse subject by user", err))?; + Ok(row.is_some()) +} + +async fn invite_abuse_workspace_quarantined(pool: &PgPool, workspace_id: &str) -> RuntimeResult { + let row: Option = sqlx::query_scalar( + r#" + SELECT 1 + FROM runtime_invite_abuse_subjects + WHERE subject_key = $1 + AND status = 'quarantined' + LIMIT 1 + "#, + ) + .bind(workspace_subject_key(workspace_id)) + .fetch_optional(pool) + .await + .map_err(|err| RuntimeError::database("failed to load invite abuse workspace subject", err))?; + Ok(row.is_some()) +} + +async fn claim_invite_abuse_action(pool: &PgPool, action_id: &str, worker_id: &str) -> RuntimeResult { + let result = sqlx::query( + r#" + UPDATE runtime_invite_abuse_actions action + SET status = 'running', + attempts = attempts + 1, + locked_by = $2, + locked_until = now() + interval '5 minutes', + last_error = NULL, + updated_at = now() + WHERE action.id = $1::bigint + AND action.action IN ('ban_actor', 'quarantine_actor', 'quarantine_workspace', 'quarantine_source_cohort') + AND ( + ( + action.status IN ('pending', 'retry_wait') + AND (action.next_attempt_at IS NULL OR action.next_attempt_at <= now()) + ) + OR ( + action.status = 'running' + AND action.locked_until IS NOT NULL + AND action.locked_until <= now() + ) + ) + "#, + ) + .bind(action_id) + .bind(worker_id) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("failed to claim invite abuse action", err))?; + Ok(result.rows_affected() > 0) +} + +async fn claim_retryable_invite_abuse_actions( + pool: &PgPool, + worker_id: &str, + limit: i64, +) -> RuntimeResult> { + let rows = sqlx::query( + r#" + WITH candidates AS ( + SELECT action.id + FROM runtime_invite_abuse_actions action + JOIN runtime_invite_abuse_evidence evidence + ON evidence.id = action.evidence_id + WHERE ( + ( + action.status IN ('pending', 'retry_wait') + AND (action.next_attempt_at IS NULL OR action.next_attempt_at <= now()) + ) + OR ( + action.status = 'running' + AND action.locked_until IS NOT NULL + AND action.locked_until <= now() + ) + ) + AND evidence.user_id IS NOT NULL + AND evidence.workspace_id IS NOT NULL + AND action.action IN ('ban_actor', 'quarantine_actor', 'quarantine_workspace', 'quarantine_source_cohort') + ORDER BY COALESCE(action.next_attempt_at, action.created_at), action.id + LIMIT $2 + FOR UPDATE SKIP LOCKED + ), + claimed AS ( + UPDATE runtime_invite_abuse_actions action + SET status = 'running', + attempts = attempts + 1, + locked_by = $1, + locked_until = now() + interval '5 minutes', + last_error = NULL, + updated_at = now() + FROM candidates + WHERE action.id = candidates.id + RETURNING + action.id, + action.subject_key, + action.evidence_id, + action.action + ) + SELECT + claimed.action, + claimed.subject_key, + claimed.evidence_id::text AS evidence_id, + claimed.id::text AS action_id, + evidence.user_id AS actor_user_id, + evidence.workspace_id + FROM claimed + JOIN runtime_invite_abuse_evidence evidence + ON evidence.id = claimed.evidence_id + "#, + ) + .bind(worker_id) + .bind(limit) + .fetch_all(pool) + .await + .map_err(|err| RuntimeError::database("failed to claim retryable invite abuse actions", err))?; + + Ok( + rows + .into_iter() + .map(|row| RuntimeInviteAbuseClaimedAction { + action: row.get("action"), + subject_key: row.get("subject_key"), + evidence_id: row.get("evidence_id"), + action_id: row.get("action_id"), + actor_user_id: row.get("actor_user_id"), + workspace_id: row.get("workspace_id"), + }) + .collect(), + ) +} + +async fn mark_invite_abuse_action( + pool: &PgPool, + action_id: &str, + worker_id: &str, + status: &str, + error: Option, +) -> RuntimeResult { + let result = match status { + "succeeded" => sqlx::query( + r#" + UPDATE runtime_invite_abuse_actions + SET status = 'succeeded', + next_attempt_at = NULL, + locked_by = NULL, + locked_until = NULL, + last_error = $3, + updated_at = now() + WHERE id = $1::bigint + AND status = 'running' + AND locked_by = $2 + "#, + ) + .bind(action_id) + .bind(worker_id) + .bind(error) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("failed to mark invite abuse action succeeded", err))?, + "failed" => sqlx::query( + r#" + UPDATE runtime_invite_abuse_actions + SET status = CASE WHEN attempts >= 5 THEN 'failed' ELSE 'retry_wait' END, + next_attempt_at = CASE WHEN attempts >= 5 THEN next_attempt_at ELSE now() + interval '5 minutes' END, + locked_by = NULL, + locked_until = NULL, + last_error = $3, + updated_at = now() + WHERE id = $1::bigint + AND status = 'running' + AND locked_by = $2 + "#, + ) + .bind(action_id) + .bind(worker_id) + .bind(error) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("failed to mark invite abuse action failed", err))?, + _ => return Err(RuntimeError::invalid_input("invalid invite abuse action status")), + }; + Ok(result.rows_affected() > 0) +} + +#[napi_derive::napi] +impl BackendRuntime { + #[napi] + pub async fn is_invite_abuse_user_quarantined_or_banned(&self, user_id: String) -> Result { + let pool = self.pool().await?; + invite_abuse_user_quarantined_or_banned(&pool, &user_id) + .await + .map_err(Into::into) + } + + #[napi] + pub async fn is_invite_abuse_workspace_quarantined(&self, workspace_id: String) -> Result { + let pool = self.pool().await?; + invite_abuse_workspace_quarantined(&pool, &workspace_id) + .await + .map_err(Into::into) + } + + #[napi] + pub async fn claim_invite_abuse_action(&self, action_id: String, worker_id: String) -> Result { + let pool = self.pool().await?; + claim_invite_abuse_action(&pool, &action_id, &worker_id) + .await + .map_err(Into::into) + } + + #[napi] + pub async fn claim_retryable_invite_abuse_actions( + &self, + worker_id: String, + limit: i64, + ) -> Result> { + if limit <= 0 { + return Err(napi_error("invite abuse action claim limit must be positive")); + } + let pool = self.pool().await?; + claim_retryable_invite_abuse_actions(&pool, &worker_id, limit) + .await + .map_err(Into::into) + } + + #[napi] + pub async fn mark_invite_abuse_action( + &self, + action_id: String, + worker_id: String, + status: String, + error: Option, + ) -> Result { + let pool = self.pool().await?; + mark_invite_abuse_action(&pool, &action_id, &worker_id, &status, error) + .await + .map_err(Into::into) + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/rolling_quota/mail_delivery.rs b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/mail_delivery.rs new file mode 100644 index 0000000000..b49486e56d --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/mail_delivery.rs @@ -0,0 +1,172 @@ +use super::{ + InviteQuotaConfig, QuotaViolation, RuntimeMailDeliveryQuotaDecision, RuntimeMailDeliveryQuotaInput, ScopeLimit, + high_risk_domain, normalize_domain, scope, short_hash, source_prefix, +}; +#[cfg(test)] +use super::{RuntimeMailDeliveryQuotaMetadataInput, RuntimeMailDeliveryQuotaRecipientInput, RuntimeQuotaSourceInput}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum MailClass { + Auth, + WorkspaceInvitation, + CollaborationNotice, + WorkspaceLifecycle, + BillingLicense, +} + +impl MailClass { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Auth => "auth", + Self::WorkspaceInvitation => "workspace_invitation", + Self::CollaborationNotice => "collaboration_notice", + Self::WorkspaceLifecycle => "workspace_lifecycle", + Self::BillingLicense => "billing_license", + } + } +} + +pub(super) fn mail_class(mail_name: &str, config: &InviteQuotaConfig) -> Option { + match config.mail_class_mapping.get(mail_name).map(String::as_str) { + Some("auth") => Some(MailClass::Auth), + Some("workspace_invitation") => Some(MailClass::WorkspaceInvitation), + Some("collaboration_notice") => Some(MailClass::CollaborationNotice), + Some("workspace_lifecycle") => Some(MailClass::WorkspaceLifecycle), + Some("billing_license") => Some(MailClass::BillingLicense), + _ => None, + } +} + +pub(super) fn build_mail_scopes( + input: &RuntimeMailDeliveryQuotaInput, + class: MailClass, + config: &InviteQuotaConfig, +) -> Vec { + let recipient_hash = short_hash(&input.recipient.email.trim().to_ascii_lowercase()); + let domain = normalize_domain(&input.recipient.domain); + let class_name = class.as_str(); + let mut scopes = vec![ + scope( + format!("mail:recipient:{recipient_hash}:class:{class_name}"), + 3600, + 20, + 1, + ), + scope( + format!("mail:recipient_domain:{domain}:class:{class_name}"), + 3600, + 250, + 1, + ), + scope("mail:provider_global:default".to_string(), 60, 500, 1), + ]; + + match class { + MailClass::Auth => { + if let Some(prefix) = source_prefix(input.source.as_ref()) { + scopes.push(scope(format!("mail:source_prefix:{prefix}:class:auth"), 3600, 50, 1)); + } + } + MailClass::WorkspaceInvitation => { + if let Some(prefix) = source_prefix(input.source.as_ref()) { + scopes.push(scope( + format!("mail:source_prefix_domain:{prefix}:{domain}:class:{class_name}"), + 3600, + if high_risk_domain(&domain, config) { 10 } else { 50 }, + 1, + )); + } + if let Some(subject) = input.metadata.abuse_subject_key.as_deref() { + scopes.push(scope( + format!("mail:abuse_subject:{subject}:class:{class_name}"), + 86_400, + 0, + 1, + )); + } + } + MailClass::CollaborationNotice | MailClass::WorkspaceLifecycle => { + if let Some(actor) = input.metadata.actor_user_id.as_deref() { + scopes.push(scope(format!("mail:actor:{actor}:class:{class_name}"), 3600, 200, 1)); + } + if let Some(workspace) = input.metadata.workspace_id.as_deref() { + scopes.push(scope( + format!("mail:workspace:{workspace}:class:{class_name}"), + 3600, + 1000, + 1, + )); + } + } + MailClass::BillingLicense => {} + } + + scopes +} + +pub(super) fn decision_from_violation( + violation: QuotaViolation, + class: MailClass, + reason: &str, +) -> RuntimeMailDeliveryQuotaDecision { + RuntimeMailDeliveryQuotaDecision { + allowed: false, + reservation_id: None, + mail_class: class.as_str().to_string(), + retry_after_seconds: Some(60), + reason: Some(reason.to_string()), + scope_key: Some(violation.scope_key), + window_seconds: Some(violation.window_seconds), + limit: Some(violation.limit), + current: Some(violation.current), + requested: Some(violation.requested), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn invite_config() -> InviteQuotaConfig { + InviteQuotaConfig::default() + } + + #[test] + fn mail_class_mapping_covers_known_renderers_and_fails_closed() { + let config = invite_config(); + for (name, class) in [ + ("SignIn", MailClass::Auth), + ("MemberInvitation", MailClass::WorkspaceInvitation), + ("CommentMention", MailClass::CollaborationNotice), + ("TeamWorkspaceExpired", MailClass::WorkspaceLifecycle), + ("TeamLicense", MailClass::BillingLicense), + ] { + assert_eq!(mail_class(name, &config), Some(class), "{name}"); + } + assert_eq!(mail_class("TestMail", &config), None); + assert_eq!(mail_class("UnexpectedMail", &config), None); + } + + #[test] + fn auth_mail_does_not_expand_actor_or_workspace_scopes() { + let input = RuntimeMailDeliveryQuotaInput { + request_id: None, + mail_name: "SignIn".to_string(), + recipient: RuntimeMailDeliveryQuotaRecipientInput { + email: "user@example.com".to_string(), + domain: "example.com".to_string(), + user_id: Some("u1".to_string()), + }, + metadata: RuntimeMailDeliveryQuotaMetadataInput { + actor_user_id: Some("actor".to_string()), + workspace_id: Some("workspace".to_string()), + notification_id: None, + abuse_subject_key: None, + }, + source: None::, + }; + let scopes = build_mail_scopes(&input, MailClass::Auth, &invite_config()); + assert!(scopes.iter().all(|scope| !scope.scope_key.contains("actor"))); + assert!(scopes.iter().all(|scope| !scope.scope_key.contains("workspace"))); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/rolling_quota/mod.rs b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/mod.rs new file mode 100644 index 0000000000..912d232859 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/mod.rs @@ -0,0 +1,175 @@ +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +mod invite_abuse_actions; +mod mail_delivery; +mod reservation; +mod workspace_invite; +mod workspace_invite_policy; + +use mail_delivery::{build_mail_scopes, decision_from_violation as mail_decision_from_violation, mail_class}; +use napi::Result; +use reservation::{ + QuotaViolation, ScopeLimit, bucket_seconds, cleanup_expired, commit_reservation, release_reservation, reserve_scopes, + scope, +}; +use sha2::{Digest, Sha256}; +use workspace_invite_policy::{ + ActorFacts, InviteAbuseDecision, InviteActivityFacts, QuotaFacts, WorkspaceFacts, build_invite_scopes, + evaluate_projection, high_confidence_invite_abuse, invite_commit_usage_for_scope, source_cohort_subject_key, + subject_hash, sum_domains, +}; + +#[cfg(test)] +pub(super) use super::types::{RuntimeMailDeliveryQuotaMetadataInput, RuntimeMailDeliveryQuotaRecipientInput}; +pub(super) use super::{ + BackendRuntime, InviteQuotaConfig, RuntimeError, RuntimeResult, napi_error, + types::{ + RuntimeInviteAbuseActionRequired, RuntimeInviteAbuseClaimedAction, RuntimeMailDeliveryQuotaDecision, + RuntimeMailDeliveryQuotaInput, RuntimeQuotaSourceInput, RuntimeQuotaTargetDomainInput, + RuntimeWorkspaceInviteQuotaDecision, RuntimeWorkspaceInviteQuotaInput, RuntimeWorkspaceInviteQuotaUsage, + }, +}; + +pub(super) fn normalize_domain(domain: &str) -> String { + domain.trim().trim_end_matches('.').to_ascii_lowercase() +} + +pub(super) fn high_risk_domain(domain: &str, config: &InviteQuotaConfig) -> bool { + let domain = normalize_domain(domain); + config + .high_risk_target_domains + .iter() + .any(|configured| normalize_domain(configured) == domain) +} + +pub(super) fn workspace_subject_key(workspace_id: &str) -> String { + format!("workspace:v1:{}", short_hash(workspace_id)) +} + +pub(super) fn short_hash(value: &str) -> String { + hex::encode(Sha256::digest(value.as_bytes()))[..24].to_string() +} + +pub(super) fn source_prefix(source: Option<&RuntimeQuotaSourceInput>) -> Option { + let source = source?; + if !source.trusted { + return None; + } + let ip = source.ip.as_ref()?.parse::().ok()?; + match ip { + IpAddr::V4(ip) => { + let octets = ip.octets(); + Some(Ipv4Addr::new(octets[0], octets[1], octets[2], 0).to_string() + "/24") + } + IpAddr::V6(ip) => { + let segments = ip.segments(); + Some(Ipv6Addr::new(segments[0], segments[1], segments[2], 0, 0, 0, 0, 0).to_string() + "/48") + } + } +} + +#[napi_derive::napi] +impl BackendRuntime { + #[napi] + pub async fn assert_mail_delivery_quota_v1( + &self, + input: RuntimeMailDeliveryQuotaInput, + ) -> Result { + let config = self.config()?.invite_quota; + let Some(class) = mail_class(&input.mail_name, &config) else { + return Ok(RuntimeMailDeliveryQuotaDecision { + allowed: false, + reservation_id: None, + mail_class: "unmapped".to_string(), + retry_after_seconds: None, + reason: Some("unmapped_mail_name".to_string()), + scope_key: Some(format!("mail:name:{}", input.mail_name)), + window_seconds: None, + limit: None, + current: None, + requested: Some(1), + }); + }; + let pool = self.pool().await?; + let scopes = build_mail_scopes(&input, class, &config); + match reserve_scopes(&pool, "mail_delivery", input.request_id.as_deref(), scopes).await? { + Ok(reservation) => Ok(RuntimeMailDeliveryQuotaDecision { + allowed: true, + reservation_id: Some(reservation.reservation_id), + mail_class: class.as_str().to_string(), + retry_after_seconds: None, + reason: None, + scope_key: None, + window_seconds: None, + limit: None, + current: None, + requested: Some(1), + }), + Err(violation) => Ok(mail_decision_from_violation(violation, class, "mail_class")), + } + } + + #[napi] + pub async fn commit_mail_delivery_quota_v1(&self, reservation_id: String) -> Result { + let pool = self.pool().await?; + commit_reservation(&pool, &reservation_id, 1, |_, reserved_count| reserved_count.min(1)) + .await + .map_err(Into::into) + } + + #[napi] + pub async fn release_mail_delivery_quota_v1(&self, reservation_id: String) -> Result { + let pool = self.pool().await?; + release_reservation(&pool, &reservation_id).await.map_err(Into::into) + } + + #[napi] + pub async fn cleanup_expired_rolling_quota(&self, limit: i64) -> Result { + if limit <= 0 { + return Err(napi_error("rolling quota cleanup limit must be positive")); + } + let pool = self.pool().await?; + cleanup_expired(&pool, limit).await.map_err(Into::into) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn derives_source_prefix_only_from_trusted_raw_ip() { + assert_eq!( + source_prefix(Some(&RuntimeQuotaSourceInput { + trusted: true, + ip: Some("192.168.12.34".to_string()), + country: None, + asn: None, + ray_id: None, + })) + .as_deref(), + Some("192.168.12.0/24") + ); + assert_eq!( + source_prefix(Some(&RuntimeQuotaSourceInput { + trusted: true, + ip: Some("2001:db8:abcd:1234::1".to_string()), + country: None, + asn: None, + ray_id: None, + })) + .as_deref(), + Some("2001:db8:abcd::/48") + ); + assert!( + source_prefix(Some(&RuntimeQuotaSourceInput { + trusted: false, + ip: Some("192.168.12.34".to_string()), + country: None, + asn: None, + ray_id: None, + })) + .is_none() + ); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/rolling_quota/reservation.rs b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/reservation.rs new file mode 100644 index 0000000000..beb73667d0 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/reservation.rs @@ -0,0 +1,344 @@ +use chrono::{DateTime, Duration, TimeZone, Utc}; +use sha2::{Digest, Sha256}; +use sqlx::{PgPool, Postgres, Row, Transaction}; +use uuid::Uuid; + +use super::{RuntimeError, RuntimeResult}; + +const RESERVATION_TTL_SECONDS: i64 = 120; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct ScopeLimit { + pub(super) scope_key: String, + pub(super) window_seconds: i32, + pub(super) bucket_seconds: i64, + pub(super) limit: i32, + pub(super) requested: i32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct QuotaViolation { + pub(super) scope_key: String, + pub(super) window_seconds: i32, + pub(super) limit: i32, + pub(super) current: i32, + pub(super) requested: i32, +} + +#[derive(Clone, Debug)] +pub(super) struct ReservationDecision { + pub(super) reservation_id: String, +} + +pub(super) fn bucket_seconds(window_seconds: i32) -> i64 { + match window_seconds { + 60 => 10, + 3600 => 5 * 60, + 86_400 => 60 * 60, + 604_800 => 6 * 60 * 60, + _ => 60, + } +} + +fn bucket_start(now: DateTime, bucket_seconds: i64) -> DateTime { + let timestamp = now.timestamp(); + Utc + .timestamp_opt(timestamp - timestamp.rem_euclid(bucket_seconds), 0) + .single() + .expect("valid unix timestamp bucket") +} + +pub(super) fn scope(scope_key: String, window_seconds: i32, limit: i32, requested: i32) -> ScopeLimit { + ScopeLimit { + scope_key, + window_seconds, + bucket_seconds: bucket_seconds(window_seconds), + limit, + requested, + } +} + +pub(super) async fn reserve_scopes( + pool: &PgPool, + purpose: &str, + request_id: Option<&str>, + scopes: Vec, +) -> RuntimeResult> { + let now: DateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(pool) + .await + .map_err(|err| RuntimeError::database("failed to read database clock", err))?; + let mut tx = pool + .begin() + .await + .map_err(|err| RuntimeError::database("failed to begin quota reservation transaction", err))?; + + let mut sorted_scope_keys = std::collections::BTreeSet::new(); + for scope in &scopes { + sorted_scope_keys.insert(scope.scope_key.clone()); + } + for scope_key in sorted_scope_keys { + advisory_lock(&mut tx, &scope_key).await?; + } + + let reservation_id = Uuid::new_v4(); + for scope in &scopes { + let bucket_start = bucket_start(now, scope.bucket_seconds); + let window_start = now - Duration::seconds(scope.window_seconds as i64); + let committed: i64 = sqlx::query_scalar( + r#" + SELECT COALESCE(SUM(count), 0)::bigint + FROM runtime_rolling_quota_counters + WHERE scope_key = $1 + AND window_seconds = $2 + AND bucket_start >= $3 + "#, + ) + .bind(&scope.scope_key) + .bind(scope.window_seconds) + .bind(window_start) + .fetch_one(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to read rolling quota counter", err))?; + let reserved: i64 = sqlx::query_scalar( + r#" + SELECT COALESCE(SUM(count), 0)::bigint + FROM runtime_rolling_quota_reservations + WHERE scope_key = $1 + AND window_seconds = $2 + AND bucket_start >= $3 + AND status = 'reserved' + AND expires_at > $4 + "#, + ) + .bind(&scope.scope_key) + .bind(scope.window_seconds) + .bind(window_start) + .bind(now) + .fetch_one(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to read rolling quota reservations", err))?; + let current = committed.saturating_add(reserved) as i32; + if current.saturating_add(scope.requested) > scope.limit { + tx.rollback() + .await + .map_err(|err| RuntimeError::database("failed to rollback rejected quota reservation", err))?; + return Ok(Err(QuotaViolation { + scope_key: scope.scope_key.clone(), + window_seconds: scope.window_seconds, + limit: scope.limit, + current, + requested: scope.requested, + })); + } + sqlx::query( + r#" + INSERT INTO runtime_rolling_quota_reservations ( + id, scope_key, window_seconds, bucket_start, count, status, purpose, request_id, expires_at + ) + VALUES ($1::uuid, $2, $3, $4, $5, 'reserved', $6, $7, $8) + "#, + ) + .bind(reservation_id.to_string()) + .bind(&scope.scope_key) + .bind(scope.window_seconds) + .bind(bucket_start) + .bind(scope.requested) + .bind(purpose) + .bind(request_id) + .bind(now + Duration::seconds(RESERVATION_TTL_SECONDS)) + .execute(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to create rolling quota reservation", err))?; + } + + tx.commit() + .await + .map_err(|err| RuntimeError::database("failed to commit quota reservation", err))?; + Ok(Ok(ReservationDecision { + reservation_id: reservation_id.to_string(), + })) +} + +async fn advisory_lock(tx: &mut Transaction<'_, Postgres>, scope_key: &str) -> RuntimeResult<()> { + let hash = Sha256::digest(scope_key.as_bytes()); + let mut bytes = [0_u8; 8]; + bytes.copy_from_slice(&hash[..8]); + let lock_key = i64::from_be_bytes(bytes); + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(lock_key) + .execute(&mut **tx) + .await + .map_err(|err| RuntimeError::database("failed to acquire rolling quota advisory lock", err))?; + Ok(()) +} + +pub(super) async fn commit_reservation( + pool: &PgPool, + reservation_id: &str, + settle_usage: i32, + actual_usage_for_scope: F, +) -> RuntimeResult +where + F: Fn(&str, i32) -> i32, +{ + let reservation_id = + Uuid::parse_str(reservation_id).map_err(|_| RuntimeError::invalid_input("invalid reservation id"))?; + let reservation_id = reservation_id.to_string(); + let now: DateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(pool) + .await + .map_err(|err| RuntimeError::database("failed to read database clock", err))?; + let mut tx = pool + .begin() + .await + .map_err(|err| RuntimeError::database("failed to begin quota commit transaction", err))?; + let rows = sqlx::query( + r#" + SELECT scope_key, window_seconds, bucket_start, count + FROM runtime_rolling_quota_reservations + WHERE id = $1::uuid AND status = 'reserved' + FOR UPDATE + "#, + ) + .bind(&reservation_id) + .fetch_all(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to load quota reservation", err))?; + if rows.is_empty() { + tx.rollback() + .await + .map_err(|err| RuntimeError::database("failed to rollback empty quota commit", err))?; + return Ok(false); + } + for row in &rows { + advisory_lock(&mut tx, row.get("scope_key")).await?; + } + for row in &rows { + let reserved_count: i32 = row.get("count"); + let scope_key: String = row.get("scope_key"); + let count = actual_usage_for_scope(&scope_key, reserved_count) + .min(reserved_count) + .max(0); + if count > 0 { + let window_seconds: i32 = row.get("window_seconds"); + sqlx::query( + r#" + INSERT INTO runtime_rolling_quota_counters (scope_key, window_seconds, bucket_start, count, expires_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (scope_key, window_seconds, bucket_start) + DO UPDATE SET count = runtime_rolling_quota_counters.count + EXCLUDED.count, + expires_at = GREATEST(runtime_rolling_quota_counters.expires_at, EXCLUDED.expires_at), + updated_at = EXCLUDED.updated_at + "#, + ) + .bind(scope_key) + .bind(window_seconds) + .bind(row.get::, _>("bucket_start")) + .bind(count) + .bind(now + Duration::seconds(window_seconds as i64 * 2)) + .bind(now) + .execute(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to commit rolling quota counter", err))?; + } + } + sqlx::query( + r#" + UPDATE runtime_rolling_quota_reservations + SET status = CASE WHEN $2 > 0 THEN 'committed' ELSE 'released' END, + committed_at = CASE WHEN $2 > 0 THEN $3 ELSE committed_at END, + released_at = CASE WHEN $2 <= 0 THEN $3 ELSE released_at END, + updated_at = $3 + WHERE id = $1::uuid AND status = 'reserved' + "#, + ) + .bind(&reservation_id) + .bind(settle_usage) + .bind(now) + .execute(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to settle quota reservation", err))?; + tx.commit() + .await + .map_err(|err| RuntimeError::database("failed to commit quota commit transaction", err))?; + Ok(true) +} + +pub(super) async fn release_reservation(pool: &PgPool, reservation_id: &str) -> RuntimeResult { + let reservation_id = + Uuid::parse_str(reservation_id).map_err(|_| RuntimeError::invalid_input("invalid reservation id"))?; + let reservation_id = reservation_id.to_string(); + let result = sqlx::query( + r#" + UPDATE runtime_rolling_quota_reservations + SET status = 'released', + released_at = clock_timestamp(), + updated_at = clock_timestamp() + WHERE id = $1::uuid AND status = 'reserved' + "#, + ) + .bind(reservation_id) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("failed to release quota reservation", err))?; + Ok(result.rows_affected() > 0) +} + +pub(super) async fn cleanup_expired(pool: &PgPool, limit: i64) -> RuntimeResult { + let released = sqlx::query( + r#" + UPDATE runtime_rolling_quota_reservations + SET status = 'expired', + updated_at = clock_timestamp() + WHERE (id, scope_key, window_seconds, bucket_start) IN ( + SELECT id, scope_key, window_seconds, bucket_start + FROM runtime_rolling_quota_reservations + WHERE status = 'reserved' AND expires_at <= clock_timestamp() + LIMIT $1 + ) + "#, + ) + .bind(limit) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("failed to expire rolling quota reservations", err))? + .rows_affected() as i64; + + let counters = sqlx::query( + r#" + DELETE FROM runtime_rolling_quota_counters + WHERE (scope_key, window_seconds, bucket_start) IN ( + SELECT scope_key, window_seconds, bucket_start + FROM runtime_rolling_quota_counters + WHERE expires_at <= clock_timestamp() + LIMIT $1 + ) + "#, + ) + .bind(limit) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("failed to delete expired rolling quota counters", err))? + .rows_affected() as i64; + + Ok(released + counters) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bucket_boundaries_match_policy_windows() { + let now = Utc.with_ymd_and_hms(2026, 7, 6, 1, 2, 3).single().unwrap(); + assert_eq!(bucket_seconds(60), 10); + assert_eq!(bucket_seconds(3600), 300); + assert_eq!(bucket_seconds(86_400), 3600); + assert_eq!(bucket_seconds(604_800), 21_600); + assert_eq!( + bucket_start(now, 300), + Utc.with_ymd_and_hms(2026, 7, 6, 1, 0, 0).single().unwrap() + ); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/rolling_quota/workspace_invite.rs b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/workspace_invite.rs new file mode 100644 index 0000000000..bfde78f11c --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/workspace_invite.rs @@ -0,0 +1,452 @@ +use chrono::{DateTime, Utc}; +use napi::Result; +use serde_json::json; +use sqlx::{PgPool, Row}; + +use super::{ + ActorFacts, BackendRuntime, InviteAbuseDecision, InviteActivityFacts, InviteQuotaConfig, QuotaFacts, QuotaViolation, + RuntimeError, RuntimeInviteAbuseActionRequired, RuntimeResult, RuntimeWorkspaceInviteQuotaDecision, + RuntimeWorkspaceInviteQuotaInput, RuntimeWorkspaceInviteQuotaUsage, WorkspaceFacts, build_invite_scopes, + commit_reservation, evaluate_projection, high_confidence_invite_abuse, invite_commit_usage_for_scope, napi_error, + normalize_domain, release_reservation, reserve_scopes, short_hash, source_cohort_subject_key, source_prefix, + subject_hash, sum_domains, workspace_subject_key, +}; + +async fn load_actor(pool: &PgPool, user_id: &str) -> RuntimeResult { + let row = sqlx::query( + r#" + SELECT email, created_at, registered, email_verified IS NOT NULL AS email_verified, disabled + FROM users + WHERE id = $1 + "#, + ) + .bind(user_id) + .fetch_optional(pool) + .await + .map_err(|err| RuntimeError::database("failed to load invite actor", err))? + .ok_or_else(|| RuntimeError::invalid_input("invite actor not found"))?; + + Ok(ActorFacts { + email: row.get("email"), + created_at: row.get("created_at"), + registered: row.get("registered"), + email_verified: row.get("email_verified"), + disabled: row.get("disabled"), + }) +} + +async fn load_workspace(pool: &PgPool, workspace_id: &str) -> RuntimeResult { + let row = sqlx::query("SELECT created_at FROM workspaces WHERE id = $1") + .bind(workspace_id) + .fetch_optional(pool) + .await + .map_err(|err| RuntimeError::database("failed to load workspace", err))? + .ok_or_else(|| RuntimeError::invalid_input("workspace not found"))?; + + Ok(WorkspaceFacts { + created_at: row.get("created_at"), + }) +} + +async fn load_invite_activity( + pool: &PgPool, + actor_user_id: &str, + workspace_id: &str, +) -> RuntimeResult { + let row = sqlx::query( + r#" + SELECT + COUNT(*) FILTER ( + WHERE inviter_user_id = $1 AND created_at >= clock_timestamp() - interval '7 days' + )::int AS actor_created_7d, + COUNT(*) FILTER ( + WHERE inviter_user_id = $1 AND accepted_at >= clock_timestamp() - interval '7 days' + )::int AS actor_accepted_7d, + COUNT(*) FILTER ( + WHERE workspace_id = $2 AND status IN ('pending', 'waiting_review', 'waiting_seat') + )::int AS workspace_pending, + COUNT(*) FILTER ( + WHERE workspace_id = $2 AND created_at >= clock_timestamp() - interval '7 days' + )::int AS workspace_created_7d, + COUNT(*) FILTER ( + WHERE workspace_id = $2 AND accepted_at >= clock_timestamp() - interval '7 days' + )::int AS workspace_accepted_7d + FROM workspace_invitations + WHERE inviter_user_id = $1 OR workspace_id = $2 + "#, + ) + .bind(actor_user_id) + .bind(workspace_id) + .fetch_one(pool) + .await + .map_err(|err| RuntimeError::database("failed to load invite activity facts", err))?; + + Ok(InviteActivityFacts { + actor_created_7d: row.get("actor_created_7d"), + actor_accepted_7d: row.get("actor_accepted_7d"), + workspace_pending: row.get("workspace_pending"), + workspace_created_7d: row.get("workspace_created_7d"), + workspace_accepted_7d: row.get("workspace_accepted_7d"), + }) +} + +async fn load_quota(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { + let row = sqlx::query( + r#" + SELECT plan, owner_user_id, uses_owner_quota, seat_limit, member_count, known, stale, stale_after + FROM effective_workspace_quota_states + WHERE workspace_id = $1 + "#, + ) + .bind(workspace_id) + .fetch_optional(pool) + .await + .map_err(|err| RuntimeError::database("failed to load workspace quota state", err))?; + + Ok(row.map(|row| QuotaFacts { + plan: row.get("plan"), + owner_user_id: row.get("owner_user_id"), + uses_owner_quota: row.get("uses_owner_quota"), + seat_limit: row.get("seat_limit"), + member_count: row.get("member_count"), + known: row.get("known"), + stale: row.get("stale"), + stale_after: row.get("stale_after"), + })) +} + +async fn active_subject_status(pool: &PgPool, subject_key: &str) -> RuntimeResult> { + let row = sqlx::query("SELECT status FROM runtime_invite_abuse_subjects WHERE subject_key = $1") + .bind(subject_key) + .fetch_optional(pool) + .await + .map_err(|err| RuntimeError::database("failed to load invite abuse subject", err))?; + Ok(row.map(|row| row.get("status"))) +} + +async fn record_invite_abuse_action( + pool: &PgPool, + input: &RuntimeWorkspaceInviteQuotaInput, + actor: &ActorFacts, + decision: InviteAbuseDecision, + config: &InviteQuotaConfig, +) -> RuntimeResult { + let action = decision.action; + let status = if action == "ban_actor" { "banned" } else { "quarantined" }; + let source_prefix_hash = source_prefix(input.source.as_ref()).map(|prefix| short_hash(&prefix)); + let target_domains = json!( + input + .target_domains + .iter() + .map(|target| json!({ + "domain": normalize_domain(&target.domain), + "count": target.count, + })) + .collect::>() + ); + let counters = json!({ + "requested": input.target_count, + }); + let actor_email_hash = subject_hash(&actor.email, config); + let actor_domain = actor.email.split('@').next_back().map(normalize_domain); + let source_asn = input.source.as_ref().and_then(|source| source.asn).map(i64::from); + let subject_user_id = if decision.subject_kind == "actor_email" { + Some(input.actor_user_id.as_str()) + } else { + None + }; + + let mut tx = pool + .begin() + .await + .map_err(|err| RuntimeError::database("failed to start invite abuse transaction", err))?; + + sqlx::query( + r#" + INSERT INTO runtime_invite_abuse_subjects ( + subject_key, + kind, + user_id, + actor_email_hash, + email_domain, + first_seen_at, + last_seen_at, + status, + action, + action_reason, + action_at + ) + VALUES ($1, $2, $3, $4, $5, now(), now(), $6, $7, $8, now()) + ON CONFLICT (subject_key) + DO UPDATE SET + user_id = EXCLUDED.user_id, + actor_email_hash = EXCLUDED.actor_email_hash, + email_domain = EXCLUDED.email_domain, + last_seen_at = now(), + status = EXCLUDED.status, + action = EXCLUDED.action, + action_reason = EXCLUDED.action_reason, + action_at = now(), + updated_at = now() + "#, + ) + .bind(&decision.subject_key) + .bind(decision.subject_kind) + .bind(subject_user_id) + .bind(actor_email_hash) + .bind(actor_domain) + .bind(status) + .bind(action) + .bind(decision.reason) + .execute(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to upsert invite abuse subject", err))?; + + let evidence_id: i64 = sqlx::query_scalar( + r#" + INSERT INTO runtime_invite_abuse_evidence ( + subject_key, + request_id, + workspace_id, + user_id, + actor_email_hash, + source_prefix_hash, + source_asn, + target_domains, + counters, + decision, + reason + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + RETURNING id + "#, + ) + .bind(&decision.subject_key) + .bind(&input.request_id) + .bind(&input.workspace_id) + .bind(&input.actor_user_id) + .bind(subject_hash(&actor.email, config)) + .bind(source_prefix_hash) + .bind(source_asn) + .bind(target_domains) + .bind(counters) + .bind(action) + .bind(decision.reason) + .fetch_one(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to insert invite abuse evidence", err))?; + + let action_id: i64 = sqlx::query_scalar( + r#" + INSERT INTO runtime_invite_abuse_actions ( + subject_key, + evidence_id, + action, + status, + next_attempt_at + ) + VALUES ($1, $2, $3, 'pending', now()) + RETURNING id + "#, + ) + .bind(&decision.subject_key) + .bind(evidence_id) + .bind(action) + .fetch_one(&mut *tx) + .await + .map_err(|err| RuntimeError::database("failed to insert invite abuse action", err))?; + + tx.commit() + .await + .map_err(|err| RuntimeError::database("failed to commit invite abuse transaction", err))?; + + Ok(RuntimeInviteAbuseActionRequired { + action: action.to_string(), + subject_key: decision.subject_key, + evidence_id: evidence_id.to_string(), + action_id: action_id.to_string(), + }) +} + +fn decision_from_violation(violation: QuotaViolation, reason: &str) -> RuntimeWorkspaceInviteQuotaDecision { + RuntimeWorkspaceInviteQuotaDecision { + allowed: false, + reservation_id: None, + retry_after_seconds: Some(60), + reason: Some(reason.to_string()), + scope_key: Some(violation.scope_key), + window_seconds: Some(violation.window_seconds), + limit: Some(violation.limit), + current: Some(violation.current), + requested: Some(violation.requested), + action_required: None, + } +} + +#[napi_derive::napi] +impl BackendRuntime { + #[napi] + pub async fn assert_workspace_invite_quota_v1( + &self, + input: RuntimeWorkspaceInviteQuotaInput, + ) -> Result { + if input.target_count <= 0 { + return Err(napi_error("target_count must be positive")); + } + let config = self.config()?.invite_quota; + let pool = self.pool().await?; + let now: DateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&pool) + .await + .map_err(|err| RuntimeError::database("failed to read database clock", err))?; + let actor = load_actor(&pool, &input.actor_user_id).await?; + let actor_subject = subject_hash(&actor.email, &config); + if let Some(status) = active_subject_status(&pool, &actor_subject).await? + && matches!(status.as_str(), "banned" | "quarantined") + { + return Ok(RuntimeWorkspaceInviteQuotaDecision { + allowed: false, + reservation_id: None, + retry_after_seconds: None, + reason: Some("abuse_subject".to_string()), + scope_key: Some(format!("invite:actor_subject:{actor_subject}")), + window_seconds: None, + limit: None, + current: None, + requested: Some(input.target_count), + action_required: None, + }); + } + let workspace_subject = workspace_subject_key(&input.workspace_id); + if let Some(status) = active_subject_status(&pool, &workspace_subject).await? + && status == "quarantined" + { + return Ok(RuntimeWorkspaceInviteQuotaDecision { + allowed: false, + reservation_id: None, + retry_after_seconds: None, + reason: Some("abuse_workspace".to_string()), + scope_key: Some(format!("invite:workspace_subject:{workspace_subject}")), + window_seconds: None, + limit: None, + current: None, + requested: Some(input.target_count), + action_required: None, + }); + } + if let Some(prefix) = source_prefix(input.source.as_ref()) { + for target in &input.target_domains { + let source_subject = source_cohort_subject_key(&prefix, &target.domain); + if let Some(status) = active_subject_status(&pool, &source_subject).await? + && status == "quarantined" + { + return Ok(RuntimeWorkspaceInviteQuotaDecision { + allowed: false, + reservation_id: None, + retry_after_seconds: None, + reason: Some("abuse_source_cohort".to_string()), + scope_key: Some(format!("invite:source_cohort_subject:{source_subject}")), + window_seconds: None, + limit: None, + current: None, + requested: Some(input.target_count), + action_required: None, + }); + } + } + } + + let quota = match load_quota(&pool, &input.workspace_id).await? { + Some(quota) => quota, + None => { + return Ok(RuntimeWorkspaceInviteQuotaDecision { + allowed: false, + reservation_id: None, + retry_after_seconds: None, + reason: Some("quota_state_unavailable".to_string()), + scope_key: None, + window_seconds: None, + limit: None, + current: None, + requested: Some(input.target_count), + action_required: None, + }); + } + }; + if let Some(reason) = evaluate_projection("a, now) { + return Ok(RuntimeWorkspaceInviteQuotaDecision { + allowed: false, + reservation_id: None, + retry_after_seconds: None, + reason: Some(reason.to_string()), + scope_key: None, + window_seconds: None, + limit: None, + current: None, + requested: Some(input.target_count), + action_required: None, + }); + } + if let Some(abuse_decision) = high_confidence_invite_abuse(&input, &actor, &config) { + let reason = abuse_decision.reason; + let scope_key = match abuse_decision.subject_kind { + "workspace" => format!("invite:workspace_subject:{}", abuse_decision.subject_key), + "source_prefix_domain" => format!("invite:source_cohort_subject:{}", abuse_decision.subject_key), + _ => format!("invite:actor_subject:{}", abuse_decision.subject_key), + }; + let action_required = record_invite_abuse_action(&pool, &input, &actor, abuse_decision, &config).await?; + return Ok(RuntimeWorkspaceInviteQuotaDecision { + allowed: false, + reservation_id: None, + retry_after_seconds: None, + reason: Some(reason.to_string()), + scope_key: Some(scope_key), + window_seconds: None, + limit: None, + current: None, + requested: Some(input.target_count), + action_required: Some(action_required), + }); + } + + let workspace = load_workspace(&pool, &input.workspace_id).await?; + let activity = load_invite_activity(&pool, &input.actor_user_id, &input.workspace_id).await?; + let scopes = build_invite_scopes(&input, &actor, &workspace, "a, &activity, &config, now)?; + match reserve_scopes(&pool, "workspace_invite", input.request_id.as_deref(), scopes).await? { + Ok(reservation) => Ok(RuntimeWorkspaceInviteQuotaDecision { + allowed: true, + reservation_id: Some(reservation.reservation_id), + retry_after_seconds: None, + reason: None, + scope_key: None, + window_seconds: None, + limit: None, + current: None, + requested: Some(input.target_count), + action_required: None, + }), + Err(violation) => Ok(decision_from_violation(violation, "quota_subject")), + } + } + + #[napi] + pub async fn commit_workspace_invite_quota_v1( + &self, + reservation_id: String, + usage: RuntimeWorkspaceInviteQuotaUsage, + ) -> Result { + let domain_usage = sum_domains(&usage.target_domains); + let pool = self.pool().await?; + commit_reservation(&pool, &reservation_id, usage.target_count, |scope_key, _| { + invite_commit_usage_for_scope(scope_key, usage.target_count, &domain_usage) + }) + .await + .map_err(Into::into) + } + + #[napi] + pub async fn release_workspace_invite_quota_v1(&self, reservation_id: String) -> Result { + let pool = self.pool().await?; + release_reservation(&pool, &reservation_id).await.map_err(Into::into) + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/rolling_quota/workspace_invite_policy.rs b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/workspace_invite_policy.rs new file mode 100644 index 0000000000..56b3b539e3 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/rolling_quota/workspace_invite_policy.rs @@ -0,0 +1,654 @@ +use std::collections::BTreeMap; + +use chrono::{DateTime, Duration, Utc}; +use napi::Result; +use sha2::{Digest, Sha256}; + +#[cfg(test)] +use super::RuntimeQuotaSourceInput; +use super::{ + InviteQuotaConfig, RuntimeQuotaTargetDomainInput, RuntimeWorkspaceInviteQuotaInput, ScopeLimit, bucket_seconds, + high_risk_domain, napi_error, normalize_domain, scope, short_hash, source_prefix, workspace_subject_key, +}; + +#[derive(Clone, Debug)] +pub(super) struct ActorFacts { + pub(super) email: String, + pub(super) created_at: DateTime, + pub(super) registered: bool, + pub(super) email_verified: bool, + pub(super) disabled: bool, +} + +#[derive(Clone, Debug)] +pub(super) struct QuotaFacts { + pub(super) plan: String, + pub(super) owner_user_id: Option, + pub(super) uses_owner_quota: bool, + pub(super) seat_limit: i32, + pub(super) member_count: i32, + pub(super) known: bool, + pub(super) stale: bool, + pub(super) stale_after: Option>, +} + +#[derive(Clone, Debug)] +pub(super) struct WorkspaceFacts { + pub(super) created_at: DateTime, +} + +#[derive(Clone, Debug, Default)] +pub(super) struct InviteActivityFacts { + pub(super) actor_created_7d: i32, + pub(super) actor_accepted_7d: i32, + pub(super) workspace_pending: i32, + pub(super) workspace_created_7d: i32, + pub(super) workspace_accepted_7d: i32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct InviteAbuseDecision { + pub(super) reason: &'static str, + pub(super) action: &'static str, + pub(super) subject_kind: &'static str, + pub(super) subject_key: String, +} + +pub(super) fn subject_hash(email: &str, config: &InviteQuotaConfig) -> String { + let normalized = email.trim().to_ascii_lowercase(); + let hash = Sha256::digest(format!("{}:{normalized}", config.subject_hash_salt).as_bytes()); + format!("actor_email_sha256:v1:{}", hex::encode(hash)) +} + +pub(super) fn source_cohort_subject_key(prefix: &str, domain: &str) -> String { + format!( + "source_prefix_domain_sha256:v1:{}", + short_hash(&format!("{}:{}", prefix, normalize_domain(domain))) + ) +} + +fn quota_subject(workspace_id: &str, quota: &QuotaFacts) -> (String, i32) { + if quota.uses_owner_quota || !quota.plan.to_ascii_lowercase().contains("team") { + ( + format!("owner:{}", quota.owner_user_id.as_deref().unwrap_or(workspace_id)), + quota.seat_limit, + ) + } else { + (format!("workspace:{workspace_id}"), quota.seat_limit) + } +} + +fn plan_ceiling_7d(plan: &str, seat_limit: i32) -> i32 { + let normalized = plan.to_ascii_lowercase(); + if normalized.contains("enterprise") || normalized.contains("team") && !normalized.contains("trial") { + seat_limit.saturating_mul(2) + } else if normalized.contains("trial") { + 50 + } else if normalized.contains("pro") { + 20 + } else { + 10 + } +} + +fn base_invite_limits( + now: DateTime, + actor: &ActorFacts, + workspace: &WorkspaceFacts, + quota: &QuotaFacts, + activity: &InviteActivityFacts, +) -> (i32, i32, i32, i32) { + if actor.disabled || !actor.registered { + return (0, 0, 0, 0); + } + let account_age = now - actor.created_at; + let workspace_age = now - workspace.created_at; + let mut single = 5; + let mut per_hour = 5; + let mut per_day = 15; + let mut per_week = 30; + + if account_age < Duration::hours(24) { + return (0, 0, 0, 0); + } + if !actor.email_verified { + single = 1; + per_hour = 1; + per_day = 3; + per_week = 5; + } else if account_age < Duration::days(7) { + single = 3; + per_hour = 3; + per_day = 5; + per_week = 10; + } else if account_age < Duration::days(30) { + single = 5; + per_hour = 5; + per_day = 15; + per_week = 30; + } else { + let plan = quota.plan.to_ascii_lowercase(); + if plan.contains("team") && !plan.contains("trial") { + single = quota.seat_limit.min(20); + per_hour = 20; + per_day = 50; + } else if plan.contains("trial") { + single = 10; + per_hour = 10; + per_day = 30; + per_week = 80; + } else if plan.contains("pro") { + single = 8; + per_hour = 8; + per_day = 20; + per_week = 50; + } + } + + if workspace_age < Duration::hours(24) || quota.member_count <= 1 { + single = single.min(3); + per_day = per_day.min(10); + } else if workspace_age < Duration::days(7) || quota.member_count <= 5 { + single = single.min((single as f32 * 0.5).ceil() as i32).max(1); + per_hour = per_hour.min((per_hour as f32 * 0.5).ceil() as i32).max(1); + per_day = per_day.min((per_day as f32 * 0.5).ceil() as i32).max(1); + } + + if activity.workspace_pending > quota.seat_limit.saturating_mul(2).max(6) { + single = single.min(2); + per_day = per_day.min(5); + } + if activity.actor_created_7d >= 5 && activity.actor_accepted_7d.saturating_mul(10) < activity.actor_created_7d { + single = single.min(2); + per_hour = per_hour.min(2); + per_day = per_day.min(5); + per_week = per_week.min(10); + } + if activity.workspace_created_7d >= 10 + && activity.workspace_accepted_7d.saturating_mul(10) < activity.workspace_created_7d + { + per_day = per_day.min(10); + per_week = per_week.min(20); + } + + let seat_7d = plan_ceiling_7d("a.plan, quota.seat_limit).min(quota.seat_limit.saturating_mul(2)); + (single, per_hour, per_day, per_week.min(seat_7d)) +} + +pub(super) fn evaluate_projection(quota: &QuotaFacts, now: DateTime) -> Option<&'static str> { + if !quota.known { + return Some("quota_state_unavailable"); + } + if quota.stale || quota.stale_after.is_some_and(|stale_after| stale_after <= now) { + return Some("quota_projection_stale"); + } + None +} + +pub(super) fn build_invite_scopes( + input: &RuntimeWorkspaceInviteQuotaInput, + actor: &ActorFacts, + workspace: &WorkspaceFacts, + quota: &QuotaFacts, + activity: &InviteActivityFacts, + config: &InviteQuotaConfig, + now: DateTime, +) -> Result> { + if input.target_count <= 0 { + return Err(napi_error("target_count must be positive")); + } + let (single, per_hour, per_day, per_week) = base_invite_limits(now, actor, workspace, quota, activity); + if input.target_count > single { + return Ok(vec![ScopeLimit { + scope_key: format!("invite:single_request:{}", input.actor_user_id), + window_seconds: 60, + bucket_seconds: bucket_seconds(60), + limit: single, + requested: input.target_count, + }]); + } + + let actor_subject = subject_hash(&actor.email, config); + let (quota_subject, seat_limit) = quota_subject(&input.workspace_id, quota); + let quota_subject_7d = plan_ceiling_7d("a.plan, seat_limit).min(seat_limit.saturating_mul(2)); + let mut scopes = vec![ + scope( + format!("invite:user:{}", input.actor_user_id), + 3600, + per_hour, + input.target_count, + ), + scope( + format!("invite:user:{}", input.actor_user_id), + 86_400, + per_day, + input.target_count, + ), + scope( + format!("invite:user:{}", input.actor_user_id), + 604_800, + per_week, + input.target_count, + ), + scope( + format!("invite:actor_subject:{actor_subject}"), + 604_800, + per_week, + input.target_count, + ), + scope( + format!("invite:workspace:{}", input.workspace_id), + 3600, + per_hour.max(5), + input.target_count, + ), + scope( + format!("invite:workspace:{}", input.workspace_id), + 86_400, + per_day.max(10), + input.target_count, + ), + scope( + format!("invite:quota_subject:{quota_subject}"), + 604_800, + quota_subject_7d, + input.target_count, + ), + ]; + + for target in &input.target_domains { + let domain = normalize_domain(&target.domain); + let limit_7d = if high_risk_domain(&domain, config) { + quota_subject_7d.min((seat_limit / 2).max(2)) + } else { + quota_subject_7d + }; + scopes.push(scope( + format!("invite:user_domain:{}:{domain}", input.actor_user_id), + 86_400, + per_day.min(limit_7d).max(1), + target.count, + )); + scopes.push(scope( + format!("invite:workspace_domain:{}:{domain}", input.workspace_id), + 86_400, + per_day.max(10).min(limit_7d).max(1), + target.count, + )); + scopes.push(scope( + format!("invite:quota_subject_domain:{quota_subject}:{domain}"), + 604_800, + limit_7d, + target.count, + )); + scopes.push(scope( + format!("invite:target_domain_global:{domain}"), + 60, + if high_risk_domain(&domain, config) { 30 } else { 100 }, + target.count, + )); + scopes.push(scope( + format!("invite:target_domain_global:{domain}"), + 86_400, + if high_risk_domain(&domain, config) { 500 } else { 2000 }, + target.count, + )); + } + + if let Some(prefix) = source_prefix(input.source.as_ref()) { + scopes.push(scope( + format!("invite:source_prefix:{prefix}"), + 3600, + 30, + input.target_count, + )); + scopes.push(scope( + format!("invite:source_prefix:{prefix}"), + 86_400, + 100, + input.target_count, + )); + for target in &input.target_domains { + let domain = normalize_domain(&target.domain); + scopes.push(scope( + format!("invite:source_prefix_domain:{prefix}:{domain}"), + 3600, + if high_risk_domain(&domain, config) { 5 } else { 15 }, + target.count, + )); + scopes.push(scope( + format!("invite:source_prefix_domain:{prefix}:{domain}"), + 86_400, + if high_risk_domain(&domain, config) { 15 } else { 50 }, + target.count, + )); + } + } + + if input.source.as_ref().is_some_and(|source| source.trusted) + && let Some(asn) = input.source.as_ref().and_then(|source| source.asn) + { + for target in &input.target_domains { + let domain = normalize_domain(&target.domain); + scopes.push(scope( + format!("invite:source_asn_domain:{asn}:{domain}"), + 3600, + if high_risk_domain(&domain, config) { 50 } else { 150 }, + target.count, + )); + scopes.push(scope( + format!("invite:source_asn_domain:{asn}:{domain}"), + 86_400, + if high_risk_domain(&domain, config) { 150 } else { 500 }, + target.count, + )); + } + } + + Ok(scopes) +} + +pub(super) fn high_confidence_invite_abuse( + input: &RuntimeWorkspaceInviteQuotaInput, + actor: &ActorFacts, + config: &InviteQuotaConfig, +) -> Option { + let high_risk_domain_counts: Vec<(String, i32)> = input + .target_domains + .iter() + .filter(|target| high_risk_domain(&target.domain, config)) + .map(|target| (normalize_domain(&target.domain), target.count.max(0))) + .collect(); + let high_risk_targets: i32 = high_risk_domain_counts.iter().map(|(_, count)| *count).sum(); + + if !actor.email_verified && high_risk_targets >= 3 { + return Some(InviteAbuseDecision { + reason: "unverified_high_risk_domain_burst", + action: "quarantine_actor", + subject_kind: "actor_email", + subject_key: subject_hash(&actor.email, config), + }); + } + + if input.target_count >= 30 && high_risk_targets * 2 >= input.target_count { + return Some(InviteAbuseDecision { + reason: "workspace_high_risk_domain_burst", + action: "quarantine_workspace", + subject_kind: "workspace", + subject_key: workspace_subject_key(&input.workspace_id), + }); + } + + if let Some(prefix) = source_prefix(input.source.as_ref()) + && high_risk_targets >= 10 + && high_risk_targets * 2 >= input.target_count + && let Some((domain, _)) = high_risk_domain_counts.iter().max_by_key(|(_, count)| *count) + { + return Some(InviteAbuseDecision { + reason: "source_high_risk_domain_burst", + action: "quarantine_source_cohort", + subject_kind: "source_prefix_domain", + subject_key: source_cohort_subject_key(&prefix, domain), + }); + } + + if input.target_count >= 10 && high_risk_targets * 2 >= input.target_count { + return Some(InviteAbuseDecision { + reason: "high_risk_domain_burst", + action: if input.target_count >= 20 { + "ban_actor" + } else { + "quarantine_actor" + }, + subject_kind: "actor_email", + subject_key: subject_hash(&actor.email, config), + }); + } + + None +} + +pub(super) fn sum_domains(target_domains: &[RuntimeQuotaTargetDomainInput]) -> BTreeMap { + let mut domains = BTreeMap::new(); + for domain in target_domains { + *domains.entry(normalize_domain(&domain.domain)).or_insert(0) += domain.count.max(0); + } + domains +} + +pub(super) fn invite_commit_usage_for_scope( + scope_key: &str, + target_count: i32, + domain_usage: &BTreeMap, +) -> i32 { + for (domain, count) in domain_usage { + if scope_key.ends_with(&format!(":{domain}")) { + return *count; + } + } + target_count +} + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + + use super::*; + + fn user(created_at: DateTime) -> ActorFacts { + ActorFacts { + email: "actor@example.com".to_string(), + created_at, + registered: true, + email_verified: true, + disabled: false, + } + } + + fn workspace(created_at: DateTime) -> WorkspaceFacts { + WorkspaceFacts { created_at } + } + + fn quota(plan: &str, seat_limit: i32) -> QuotaFacts { + QuotaFacts { + plan: plan.to_string(), + owner_user_id: Some("owner-1".to_string()), + uses_owner_quota: false, + seat_limit, + member_count: 10, + known: true, + stale: false, + stale_after: None, + } + } + + fn invite_config() -> InviteQuotaConfig { + InviteQuotaConfig::default() + } + + #[test] + fn seat_based_weekly_limit_binds_paid_team_and_high_risk_domain() { + let now = Utc.with_ymd_and_hms(2026, 7, 6, 0, 0, 0).single().unwrap(); + let input = RuntimeWorkspaceInviteQuotaInput { + actor_user_id: "u1".to_string(), + workspace_id: "w1".to_string(), + request_id: None, + target_count: 5, + target_domains: vec![RuntimeQuotaTargetDomainInput { + domain: "qq.com".to_string(), + count: 5, + }], + source: None, + }; + let scopes = build_invite_scopes( + &input, + &user(now - Duration::days(60)), + &workspace(now - Duration::days(60)), + "a("paid_team", 10), + &InviteActivityFacts::default(), + &invite_config(), + now, + ) + .unwrap(); + let quota_subject = scopes + .iter() + .find(|scope| scope.scope_key == "invite:quota_subject:workspace:w1" && scope.window_seconds == 604_800) + .unwrap(); + assert_eq!(quota_subject.limit, 20); + let high_risk = scopes + .iter() + .find(|scope| scope.scope_key == "invite:quota_subject_domain:workspace:w1:qq.com") + .unwrap(); + assert_eq!(high_risk.limit, 5); + } + + #[test] + fn owner_quota_subject_does_not_scale_by_workspace_count() { + let facts = QuotaFacts { + uses_owner_quota: true, + owner_user_id: Some("owner-a".to_string()), + ..quota("free", 3) + }; + assert_eq!(quota_subject("w1", &facts), ("owner:owner-a".to_string(), 3)); + assert_eq!(plan_ceiling_7d("free", 3).min(3 * 2), 6); + } + + #[test] + fn low_acceptance_activity_reduces_invite_limits() { + let now = Utc.with_ymd_and_hms(2026, 7, 6, 0, 0, 0).single().unwrap(); + let input = RuntimeWorkspaceInviteQuotaInput { + actor_user_id: "u1".to_string(), + workspace_id: "w1".to_string(), + request_id: None, + target_count: 2, + target_domains: vec![RuntimeQuotaTargetDomainInput { + domain: "example.com".to_string(), + count: 2, + }], + source: None, + }; + let activity = InviteActivityFacts { + actor_created_7d: 10, + actor_accepted_7d: 0, + ..Default::default() + }; + + let scopes = build_invite_scopes( + &input, + &user(now - Duration::days(60)), + &workspace(now - Duration::days(60)), + "a("paid_team", 10), + &activity, + &invite_config(), + now, + ) + .unwrap(); + + let day = scopes + .iter() + .find(|scope| scope.scope_key == "invite:user:u1" && scope.window_seconds == 86_400) + .unwrap(); + assert_eq!(day.limit, 5); + } + + #[test] + fn invite_commit_usage_uses_domain_counts_for_domain_scopes() { + let domain_usage = sum_domains(&[ + RuntimeQuotaTargetDomainInput { + domain: "Example.com".to_string(), + count: 2, + }, + RuntimeQuotaTargetDomainInput { + domain: "qq.com".to_string(), + count: 1, + }, + ]); + + assert_eq!( + invite_commit_usage_for_scope("invite:user_domain:u1:example.com", 3, &domain_usage), + 2 + ); + assert_eq!( + invite_commit_usage_for_scope("invite:quota_subject:workspace:w1", 3, &domain_usage), + 3 + ); + } + + #[test] + fn high_confidence_abuse_selects_workspace_and_source_subjects() { + let config = invite_config(); + let mut input = RuntimeWorkspaceInviteQuotaInput { + actor_user_id: "u1".to_string(), + workspace_id: "w1".to_string(), + request_id: None, + target_count: 30, + target_domains: vec![RuntimeQuotaTargetDomainInput { + domain: "qq.com".to_string(), + count: 30, + }], + source: Some(RuntimeQuotaSourceInput { + trusted: true, + ip: Some("192.168.12.34".to_string()), + country: Some("US".to_string()), + asn: Some(13335), + ray_id: Some("ray".to_string()), + }), + }; + + let workspace_decision = high_confidence_invite_abuse( + &input, + &user(Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).single().unwrap()), + &config, + ) + .unwrap(); + assert_eq!(workspace_decision.action, "quarantine_workspace"); + assert_eq!(workspace_decision.subject_kind, "workspace"); + + input.target_count = 12; + input.target_domains[0].count = 12; + let source_decision = high_confidence_invite_abuse( + &input, + &user(Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).single().unwrap()), + &config, + ) + .unwrap(); + assert_eq!(source_decision.action, "quarantine_source_cohort"); + assert_eq!(source_decision.subject_kind, "source_prefix_domain"); + assert!( + source_decision + .subject_key + .starts_with("source_prefix_domain_sha256:v1:") + ); + } + + #[test] + fn source_cohort_action_requires_trusted_source() { + let config = invite_config(); + let input = RuntimeWorkspaceInviteQuotaInput { + actor_user_id: "u1".to_string(), + workspace_id: "w1".to_string(), + request_id: None, + target_count: 12, + target_domains: vec![RuntimeQuotaTargetDomainInput { + domain: "qq.com".to_string(), + count: 12, + }], + source: Some(RuntimeQuotaSourceInput { + trusted: false, + ip: Some("192.168.12.34".to_string()), + country: Some("US".to_string()), + asn: Some(13335), + ray_id: Some("ray".to_string()), + }), + }; + + let decision = high_confidence_invite_abuse( + &input, + &user(Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).single().unwrap()), + &config, + ) + .unwrap(); + assert_eq!(decision.action, "quarantine_actor"); + assert_eq!(decision.subject_kind, "actor_email"); + } +} diff --git a/packages/backend/native/src/runtime/backend_runtime/tests.rs b/packages/backend/native/src/runtime/backend_runtime/tests.rs index 63bb4a7e8e..4a06d18734 100644 --- a/packages/backend/native/src/runtime/backend_runtime/tests.rs +++ b/packages/backend/native/src/runtime/backend_runtime/tests.rs @@ -72,13 +72,129 @@ async fn runtime_from_database_url() -> AnyResult> { .execute(&pool) .await .context("cleanup runtime_leases for backend runtime tests")?; + sqlx::query("DELETE FROM runtime_rolling_quota_reservations WHERE request_id LIKE 'rust-test:%'") + .execute(&pool) + .await + .context("cleanup rolling quota reservations for backend runtime tests")?; + sqlx::query("DELETE FROM runtime_rolling_quota_counters WHERE scope_key LIKE 'invite:%rust-test%'") + .execute(&pool) + .await + .context("cleanup rolling quota counters for backend runtime tests")?; + sqlx::query("DELETE FROM runtime_invite_abuse_actions WHERE subject_key LIKE 'rust-test:%'") + .execute(&pool) + .await + .context("cleanup invite abuse actions for backend runtime tests")?; + sqlx::query("DELETE FROM runtime_invite_abuse_evidence WHERE subject_key LIKE 'rust-test:%'") + .execute(&pool) + .await + .context("cleanup invite abuse evidence for backend runtime tests")?; + sqlx::query( + "DELETE FROM runtime_invite_abuse_subjects WHERE subject_key LIKE 'rust-test:%' OR user_id LIKE 'rust-test:%'", + ) + .execute(&pool) + .await + .context("cleanup invite abuse subjects for backend runtime tests")?; Ok(Some(BackendRuntime { - config: std::sync::RwLock::new(BackendRuntimeConfig { database_url }), + config: std::sync::RwLock::new(BackendRuntimeConfig { + database_url, + invite_quota: Default::default(), + }), pool: Mutex::new(Some(pool)), })) } +async fn insert_invite_quota_fixture( + runtime: &BackendRuntime, + suffix: &str, + stale: bool, +) -> AnyResult<(String, String)> { + let pool = runtime.pool().await.map_err(|err| anyhow!(err.to_string()))?; + let user_id = format!("rust-test:quota:user:{suffix}"); + let workspace_id = format!("rust-test:quota:workspace:{suffix}"); + let email = format!("rust-test-quota-{suffix}@example.com"); + + sqlx::query("DELETE FROM effective_workspace_quota_states WHERE workspace_id = $1") + .bind(&workspace_id) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM workspaces WHERE id = $1") + .bind(&workspace_id) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM users WHERE id = $1") + .bind(&user_id) + .execute(&pool) + .await?; + sqlx::query( + r#" + INSERT INTO users (id, name, email, registered, email_verified, disabled, created_at) + VALUES ($1, 'Rust Quota Actor', $2, true, clock_timestamp(), false, clock_timestamp() - interval '60 days') + "#, + ) + .bind(&user_id) + .bind(email) + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO workspaces (id, public, created_at) VALUES ($1, false, clock_timestamp() - interval '60 days')", + ) + .bind(&workspace_id) + .execute(&pool) + .await?; + sqlx::query( + r#" + INSERT INTO effective_workspace_quota_states ( + workspace_id, + plan, + owner_user_id, + uses_owner_quota, + seat_limit, + member_count, + overcapacity_member_count, + blob_limit, + storage_quota, + used_storage_quota, + history_period_seconds, + readonly, + readonly_reasons, + flags, + known, + stale, + last_reconciled_at, + stale_after + ) + VALUES ($1, 'paid_team', $2, false, 10, 3, 0, 1, 1, 0, 1, false, ARRAY[]::TEXT[], '{}'::jsonb, true, $3, clock_timestamp(), clock_timestamp() + interval '1 day') + "#, + ) + .bind(&workspace_id) + .bind(&user_id) + .bind(stale) + .execute(&pool) + .await?; + + Ok((user_id, workspace_id)) +} + +fn invite_quota_input( + user_id: &str, + workspace_id: &str, + request_id: &str, + count: i32, +) -> types::RuntimeWorkspaceInviteQuotaInput { + types::RuntimeWorkspaceInviteQuotaInput { + actor_user_id: user_id.to_string(), + workspace_id: workspace_id.to_string(), + request_id: Some(request_id.to_string()), + target_count: count, + target_domains: vec![types::RuntimeQuotaTargetDomainInput { + domain: "example.com".to_string(), + count, + }], + source: None, + } +} + #[tokio::test] async fn runtime_gate_sql_semantics_are_atomic_and_ttl_bound() { let _guard = pg_test_lock().lock().await; @@ -160,6 +276,278 @@ async fn runtime_gate_sql_semantics_are_atomic_and_ttl_bound() { assert_eq!(runtime.cleanup_expired_runtime_gates(100).await.unwrap(), 0); } +#[tokio::test] +async fn rolling_quota_sql_state_machine_commits_releases_and_expires() { + let _guard = pg_test_lock().lock().await; + let Some(runtime) = runtime_from_database_url().await.unwrap() else { + eprintln!("skipping postgres integration test: DATABASE_URL is not set"); + return; + }; + let (user_id, workspace_id) = insert_invite_quota_fixture(&runtime, "state-machine", false) + .await + .unwrap(); + let pool = runtime.pool().await.unwrap(); + + let decision = runtime + .assert_workspace_invite_quota_v1(invite_quota_input(&user_id, &workspace_id, "rust-test:quota:commit", 2)) + .await + .unwrap(); + assert!(decision.allowed); + let reservation_id = decision.reservation_id.unwrap(); + assert!( + runtime + .commit_workspace_invite_quota_v1( + reservation_id, + types::RuntimeWorkspaceInviteQuotaUsage { + target_count: 1, + target_domains: vec![types::RuntimeQuotaTargetDomainInput { + domain: "example.com".to_string(), + count: 1, + }], + }, + ) + .await + .unwrap() + ); + let committed: i64 = sqlx::query_scalar( + r#" + SELECT COALESCE(SUM(count), 0)::bigint + FROM runtime_rolling_quota_counters + WHERE scope_key = $1 + "#, + ) + .bind(format!("invite:user_domain:{user_id}:example.com")) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(committed, 1); + + let decision = runtime + .assert_workspace_invite_quota_v1(invite_quota_input( + &user_id, + &workspace_id, + "rust-test:quota:release", + 1, + )) + .await + .unwrap(); + let reservation_id = decision.reservation_id.unwrap(); + assert!( + runtime + .release_workspace_invite_quota_v1(reservation_id.clone()) + .await + .unwrap() + ); + let released: String = + sqlx::query_scalar("SELECT status FROM runtime_rolling_quota_reservations WHERE id = $1::uuid LIMIT 1") + .bind(&reservation_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(released, "released"); + + let decision = runtime + .assert_workspace_invite_quota_v1(invite_quota_input(&user_id, &workspace_id, "rust-test:quota:expire", 1)) + .await + .unwrap(); + let reservation_id = decision.reservation_id.unwrap(); + sqlx::query( + "UPDATE runtime_rolling_quota_reservations SET expires_at = clock_timestamp() - interval '1 second' WHERE id = \ + $1::uuid", + ) + .bind(&reservation_id) + .execute(&pool) + .await + .unwrap(); + assert!(runtime.cleanup_expired_rolling_quota(100).await.unwrap() > 0); + let expired: String = + sqlx::query_scalar("SELECT status FROM runtime_rolling_quota_reservations WHERE id = $1::uuid LIMIT 1") + .bind(&reservation_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(expired, "expired"); +} + +#[tokio::test] +async fn rolling_quota_projection_stale_fails_closed() { + let _guard = pg_test_lock().lock().await; + let Some(runtime) = runtime_from_database_url().await.unwrap() else { + eprintln!("skipping postgres integration test: DATABASE_URL is not set"); + return; + }; + let (user_id, workspace_id) = insert_invite_quota_fixture(&runtime, "stale-projection", true) + .await + .unwrap(); + + let decision = runtime + .assert_workspace_invite_quota_v1(invite_quota_input(&user_id, &workspace_id, "rust-test:quota:stale", 1)) + .await + .unwrap(); + + assert!(!decision.allowed); + assert_eq!(decision.reason.as_deref(), Some("quota_projection_stale")); + assert!(decision.reservation_id.is_none()); +} + +#[tokio::test] +async fn invite_abuse_action_sql_state_machine_retries_and_fences_workers() { + let _guard = pg_test_lock().lock().await; + let Some(runtime) = runtime_from_database_url().await.unwrap() else { + eprintln!("skipping postgres integration test: DATABASE_URL is not set"); + return; + }; + let pool = runtime.pool().await.unwrap(); + let actor_id = "rust-test:invite-abuse-action:user"; + let workspace_id = "rust-test:invite-abuse-action:workspace"; + let subject_key = "rust-test:invite-abuse-action:subject"; + let action_id: i64 = sqlx::query_scalar( + r#" + WITH subject AS ( + INSERT INTO runtime_invite_abuse_subjects ( + subject_key, + kind, + user_id, + actor_email_hash, + status, + first_seen_at, + last_seen_at + ) + VALUES ($1, 'actor_email', $2, 'hash', 'quarantined', now(), now()) + RETURNING subject_key + ), + evidence AS ( + INSERT INTO runtime_invite_abuse_evidence ( + subject_key, + workspace_id, + user_id, + actor_email_hash, + decision, + reason + ) + VALUES ($1, $3, $2, 'hash', 'quarantine_actor', 'test') + RETURNING id + ) + INSERT INTO runtime_invite_abuse_actions ( + subject_key, + evidence_id, + action, + status + ) + SELECT $1, evidence.id, 'quarantine_actor', 'pending' + FROM evidence + RETURNING id + "#, + ) + .bind(subject_key) + .bind(actor_id) + .bind(workspace_id) + .fetch_one(&pool) + .await + .unwrap(); + + assert!( + runtime + .claim_invite_abuse_action(action_id.to_string(), "rust-test:inline-worker".to_string()) + .await + .unwrap() + ); + assert!( + runtime + .mark_invite_abuse_action( + action_id.to_string(), + "rust-test:inline-worker".to_string(), + "failed".to_string(), + Some("transient cleanup failure".to_string()) + ) + .await + .unwrap() + ); + + let waiting = sqlx::query( + r#" + SELECT status, attempts, last_error + FROM runtime_invite_abuse_actions + WHERE id = $1 + "#, + ) + .bind(action_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(waiting.get::("status"), "retry_wait"); + assert_eq!(waiting.get::("attempts"), 1); + assert_eq!( + waiting.get::, _>("last_error").as_deref(), + Some("transient cleanup failure") + ); + + sqlx::query( + r#" + UPDATE runtime_invite_abuse_actions + SET next_attempt_at = now() - interval '1 second' + WHERE id = $1 + "#, + ) + .bind(action_id) + .execute(&pool) + .await + .unwrap(); + + let claimed = runtime + .claim_retryable_invite_abuse_actions("rust-test:worker".to_string(), 10) + .await + .unwrap(); + let current = claimed + .iter() + .find(|action| action.action_id == action_id.to_string()) + .expect("retryable action should be claimed"); + assert_eq!(current.action, "quarantine_actor"); + assert_eq!(current.subject_key, subject_key); + assert_eq!(current.actor_user_id, actor_id); + assert_eq!(current.workspace_id, workspace_id); + + assert!( + !runtime + .mark_invite_abuse_action( + action_id.to_string(), + "rust-test:inline-worker".to_string(), + "succeeded".to_string(), + None + ) + .await + .unwrap() + ); + let still_claimed = sqlx::query( + r#" + SELECT status, locked_by + FROM runtime_invite_abuse_actions + WHERE id = $1 + "#, + ) + .bind(action_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(still_claimed.get::("status"), "running"); + assert_eq!( + still_claimed.get::, _>("locked_by").as_deref(), + Some("rust-test:worker") + ); + + assert!( + runtime + .mark_invite_abuse_action( + action_id.to_string(), + "rust-test:worker".to_string(), + "succeeded".to_string(), + None + ) + .await + .unwrap() + ); +} + #[tokio::test] async fn coordination_lease_sql_semantics_are_fenced_and_ttl_bound() { let _guard = pg_test_lock().lock().await; diff --git a/packages/backend/native/src/runtime/config.rs b/packages/backend/native/src/runtime/config.rs index 047ff3f1a7..529dfdbe49 100644 --- a/packages/backend/native/src/runtime/config.rs +++ b/packages/backend/native/src/runtime/config.rs @@ -1,4 +1,5 @@ use std::{ + collections::BTreeMap, env, fs, path::{Path, PathBuf}, }; @@ -12,6 +13,35 @@ use super::{RuntimeError, RuntimeResult}; #[derive(Clone, Debug)] pub(crate) struct BackendRuntimeConfig { pub(crate) database_url: String, + pub(crate) invite_quota: InviteQuotaConfig, +} + +#[derive(Clone, Debug)] +pub(crate) struct InviteQuotaConfig { + pub(crate) high_risk_target_domains: Vec, + pub(crate) subject_hash_salt: String, + pub(crate) mail_class_mapping: BTreeMap, +} + +impl Default for InviteQuotaConfig { + fn default() -> Self { + Self { + high_risk_target_domains: [ + "qq.com", + "proton.me", + "protonmail.com", + "163.com", + "126.com", + "outlook.com", + "hotmail.com", + ] + .into_iter() + .map(str::to_string) + .collect(), + subject_hash_salt: "affine-runtime-invite-quota-v1-local".to_string(), + mail_class_mapping: default_mail_class_mapping(), + } + } } impl BackendRuntimeConfig { @@ -20,7 +50,10 @@ impl BackendRuntimeConfig { let database_url = database_url_from_env() .or(app_config.database_url()) .unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string()); - Ok(Self { database_url }) + Ok(Self { + database_url, + invite_quota: app_config.invite_quota_config(), + }) } pub(crate) async fn with_db_overrides(&self, pool: &PgPool) -> RuntimeResult { @@ -30,6 +63,7 @@ impl BackendRuntimeConfig { // The DB override is loaded after this connection already exists, so it // must not rewrite the active datasource URL. database_url: self.database_url.clone(), + invite_quota: app_config.invite_quota_config(), }) } } @@ -53,6 +87,10 @@ impl AppConfigFile { .and_then(|db| db.datasource_url.clone()) .and_then(non_empty_string) } + + fn invite_quota_config(&self) -> InviteQuotaConfig { + InviteQuotaConfig::default() + } } fn database_url_from_env() -> Option { @@ -86,6 +124,43 @@ impl AppConfigFile { } } +fn default_mail_class_mapping() -> BTreeMap { + [ + ("SignIn", "auth"), + ("SignUp", "auth"), + ("SetPassword", "auth"), + ("ChangePassword", "auth"), + ("VerifyEmail", "auth"), + ("ChangeEmail", "auth"), + ("VerifyChangeEmail", "auth"), + ("EmailChanged", "auth"), + ("MemberInvitation", "workspace_invitation"), + ("Mention", "collaboration_notice"), + ("Comment", "collaboration_notice"), + ("CommentMention", "collaboration_notice"), + ("MemberAccepted", "collaboration_notice"), + ("LinkInvitationReviewRequest", "collaboration_notice"), + ("LinkInvitationApprove", "collaboration_notice"), + ("LinkInvitationDecline", "collaboration_notice"), + ("MemberLeave", "workspace_lifecycle"), + ("MemberRemoved", "workspace_lifecycle"), + ("OwnershipTransferred", "workspace_lifecycle"), + ("OwnershipReceived", "workspace_lifecycle"), + ("TeamWorkspaceUpgraded", "workspace_lifecycle"), + ("TeamBecomeAdmin", "workspace_lifecycle"), + ("TeamBecomeCollaborator", "workspace_lifecycle"), + ("TeamDeleteIn24Hours", "workspace_lifecycle"), + ("TeamDeleteInOneMonth", "workspace_lifecycle"), + ("TeamWorkspaceDeleted", "workspace_lifecycle"), + ("TeamWorkspaceExpireSoon", "workspace_lifecycle"), + ("TeamWorkspaceExpired", "workspace_lifecycle"), + ("TeamLicense", "billing_license"), + ] + .into_iter() + .map(|(mail_name, class)| (mail_name.to_string(), class.to_string())) + .collect() +} + async fn load_app_config_overrides_from_db(pool: &PgPool) -> RuntimeResult { let rows = match sqlx::query("SELECT id, value FROM app_configs").fetch_all(pool).await { Ok(rows) => rows, @@ -197,4 +272,21 @@ mod tests { Some("postgresql://example/runtime") ); } + + #[test] + fn invite_quota_policy_is_internal_not_app_configurable() { + let app_config = app_config_from_flat_overrides([ + ("auth.untrustedPolicyOverride", serde_json::json!("runtime-salt-v2")), + ("auth.untrustedDomainList", serde_json::json!(["Example.COM."])), + ]) + .unwrap(); + + let config = app_config.invite_quota_config(); + assert!(!config.high_risk_target_domains.contains(&"example.com".to_string())); + assert_ne!(config.subject_hash_salt, "runtime-salt-v2"); + assert_eq!( + config.mail_class_mapping.get("MemberInvitation").map(String::as_str), + Some("workspace_invitation") + ); + } } diff --git a/packages/backend/native/src/runtime/mod.rs b/packages/backend/native/src/runtime/mod.rs index 73a9fd8132..6449b56fb6 100644 --- a/packages/backend/native/src/runtime/mod.rs +++ b/packages/backend/native/src/runtime/mod.rs @@ -6,5 +6,5 @@ pub(crate) mod error; pub(crate) mod migrations; pub(crate) mod types; -pub(crate) use config::BackendRuntimeConfig; +pub(crate) use config::{BackendRuntimeConfig, InviteQuotaConfig}; pub(crate) use error::{RuntimeError, RuntimeResult, napi_error, to_napi_error}; diff --git a/packages/backend/native/src/runtime/sql/runtime_migrations.sql b/packages/backend/native/src/runtime/sql/runtime_migrations.sql index 859ff753d9..98b523db30 100644 --- a/packages/backend/native/src/runtime/sql/runtime_migrations.sql +++ b/packages/backend/native/src/runtime/sql/runtime_migrations.sql @@ -110,3 +110,90 @@ CREATE TABLE IF NOT EXISTS blob_cleanup_candidates ( CREATE INDEX IF NOT EXISTS blob_cleanup_candidates_run_idx ON blob_cleanup_candidates (run_id, status); + +CREATE TABLE IF NOT EXISTS runtime_rolling_quota_counters ( + scope_key TEXT NOT NULL, + window_seconds INTEGER NOT NULL, + bucket_start TIMESTAMPTZ(3) NOT NULL, + count INTEGER NOT NULL CHECK (count >= 0), + expires_at TIMESTAMPTZ(3) NOT NULL, + updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (scope_key, window_seconds, bucket_start) +); + +CREATE INDEX IF NOT EXISTS runtime_rolling_quota_counters_expires_at_idx + ON runtime_rolling_quota_counters (expires_at); + +CREATE TABLE IF NOT EXISTS runtime_rolling_quota_reservations ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + scope_key TEXT NOT NULL, + window_seconds INTEGER NOT NULL, + bucket_start TIMESTAMPTZ(3) NOT NULL, + count INTEGER NOT NULL CHECK (count >= 0), + status TEXT NOT NULL CHECK (status IN ('reserved', 'committed', 'released', 'expired')), + purpose TEXT NOT NULL, + request_id TEXT, + expires_at TIMESTAMPTZ(3) NOT NULL, + committed_at TIMESTAMPTZ(3), + released_at TIMESTAMPTZ(3), + created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id, scope_key, window_seconds, bucket_start) +); + +CREATE INDEX IF NOT EXISTS runtime_rolling_quota_reservations_status_expires_at_idx + ON runtime_rolling_quota_reservations (status, expires_at); + +CREATE INDEX IF NOT EXISTS runtime_rolling_quota_reservations_scope_idx + ON runtime_rolling_quota_reservations (scope_key, window_seconds, bucket_start) + WHERE status = 'reserved'; + +CREATE TABLE IF NOT EXISTS runtime_invite_abuse_subjects ( + subject_key TEXT PRIMARY KEY, + kind TEXT NOT NULL, + user_id TEXT, + actor_email_hash TEXT, + email_domain TEXT, + first_seen_at TIMESTAMPTZ(3) NOT NULL, + last_seen_at TIMESTAMPTZ(3) NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + action TEXT, + action_reason TEXT, + action_at TIMESTAMPTZ(3), + created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS runtime_invite_abuse_evidence ( + id BIGSERIAL PRIMARY KEY, + subject_key TEXT NOT NULL REFERENCES runtime_invite_abuse_subjects(subject_key), + request_id TEXT, + workspace_id TEXT, + user_id TEXT, + actor_email_hash TEXT, + source_prefix_hash TEXT, + source_asn BIGINT, + target_domains JSONB NOT NULL DEFAULT '[]'::jsonb, + counters JSONB NOT NULL DEFAULT '{}'::jsonb, + decision TEXT NOT NULL, + reason TEXT NOT NULL, + created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS runtime_invite_abuse_actions ( + id BIGSERIAL PRIMARY KEY, + subject_key TEXT NOT NULL REFERENCES runtime_invite_abuse_subjects(subject_key), + evidence_id BIGINT NOT NULL REFERENCES runtime_invite_abuse_evidence(id), + action TEXT NOT NULL CHECK (action IN ('ban_actor', 'quarantine_actor', 'quarantine_workspace', 'quarantine_source_cohort')), + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'succeeded', 'retry_wait', 'failed')), + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + next_attempt_at TIMESTAMPTZ(3), + locked_by TEXT, + locked_until TIMESTAMPTZ(3), + last_error TEXT, + created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS runtime_invite_abuse_actions_status_next_attempt_idx + ON runtime_invite_abuse_actions (status, next_attempt_at); diff --git a/packages/backend/native/src/runtime/types.rs b/packages/backend/native/src/runtime/types.rs index 23e73f7c59..295d6fe1d9 100644 --- a/packages/backend/native/src/runtime/types.rs +++ b/packages/backend/native/src/runtime/types.rs @@ -14,6 +14,107 @@ pub struct BackendRuntimeHealth { pub database_connected: bool, } +#[napi_derive::napi(object)] +pub struct RuntimeQuotaTargetDomainInput { + pub domain: String, + pub count: i32, +} + +#[napi_derive::napi(object)] +pub struct RuntimeQuotaSourceInput { + pub trusted: bool, + pub ip: Option, + pub country: Option, + pub asn: Option, + pub ray_id: Option, +} + +#[napi_derive::napi(object)] +pub struct RuntimeWorkspaceInviteQuotaInput { + pub actor_user_id: String, + pub workspace_id: String, + pub request_id: Option, + pub target_count: i32, + pub target_domains: Vec, + pub source: Option, +} + +#[napi_derive::napi(object)] +pub struct RuntimeWorkspaceInviteQuotaUsage { + pub target_count: i32, + pub target_domains: Vec, +} + +#[napi_derive::napi(object)] +pub struct RuntimeInviteAbuseActionRequired { + pub action: String, + pub subject_key: String, + pub evidence_id: String, + pub action_id: String, +} + +#[napi_derive::napi(object)] +pub struct RuntimeInviteAbuseClaimedAction { + pub action: String, + pub subject_key: String, + pub evidence_id: String, + pub action_id: String, + pub actor_user_id: String, + pub workspace_id: String, +} + +#[napi_derive::napi(object)] +pub struct RuntimeWorkspaceInviteQuotaDecision { + pub allowed: bool, + pub reservation_id: Option, + pub retry_after_seconds: Option, + pub reason: Option, + pub scope_key: Option, + pub window_seconds: Option, + pub limit: Option, + pub current: Option, + pub requested: Option, + pub action_required: Option, +} + +#[napi_derive::napi(object)] +pub struct RuntimeMailDeliveryQuotaMetadataInput { + pub actor_user_id: Option, + pub workspace_id: Option, + pub notification_id: Option, + pub abuse_subject_key: Option, +} + +#[napi_derive::napi(object)] +pub struct RuntimeMailDeliveryQuotaRecipientInput { + pub email: String, + pub domain: String, + pub user_id: Option, +} + +#[napi_derive::napi(object)] +pub struct RuntimeMailDeliveryQuotaInput { + pub request_id: Option, + pub mail_name: String, + pub recipient: RuntimeMailDeliveryQuotaRecipientInput, + pub metadata: RuntimeMailDeliveryQuotaMetadataInput, + pub source: Option, +} + +#[napi_derive::napi(object)] +pub struct RuntimeMailDeliveryQuotaDecision { + pub allowed: bool, + pub reservation_id: Option, + pub mail_class: String, + pub retry_after_seconds: Option, + pub reason: Option, + pub scope_key: Option, + pub window_seconds: Option, + pub limit: Option, + pub current: Option, + pub requested: Option, +} + #[napi_derive::napi(object)] pub struct CoordinationLeaseGrant { pub key: String, diff --git a/packages/backend/server/migrations/20260706010719_mail_deliveries_and_invite_quota_indexes/migration.sql b/packages/backend/server/migrations/20260706010719_mail_deliveries_and_invite_quota_indexes/migration.sql new file mode 100644 index 0000000000..03c507bf3b --- /dev/null +++ b/packages/backend/server/migrations/20260706010719_mail_deliveries_and_invite_quota_indexes/migration.sql @@ -0,0 +1,132 @@ +-- CreateTable +CREATE TABLE "mail_deliveries" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "mail_name" TEXT NOT NULL, + "mail_class" TEXT NOT NULL, + "priority" TEXT NOT NULL DEFAULT 'normal', + "status" TEXT NOT NULL, + "dedupe_key" TEXT, + "recipient_email" TEXT, + "recipient_hash" TEXT NOT NULL, + "recipient_domain" TEXT NOT NULL, + "recipient_user_id" VARCHAR, + "actor_user_id" VARCHAR, + "workspace_id" VARCHAR, + "notification_id" VARCHAR, + "abuse_subject_key" TEXT, + "quota_reservation_id" UUID, + "quota_decision" JSONB, + "payload" JSONB, + "send_after" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "expires_at" TIMESTAMPTZ(3), + "attempt_count" INTEGER NOT NULL DEFAULT 0, + "max_attempts" INTEGER NOT NULL DEFAULT 3, + "locked_by" TEXT, + "locked_until" TIMESTAMPTZ(3), + "first_attempt_at" TIMESTAMPTZ(3), + "last_attempt_at" TIMESTAMPTZ(3), + "sent_at" TIMESTAMPTZ(3), + "settled_at" TIMESTAMPTZ(3), + "canceled_at" TIMESTAMPTZ(3), + "failed_at" TIMESTAMPTZ(3), + "provider_message_id" TEXT, + "provider_response" TEXT, + "last_error_code" TEXT, + "last_error" TEXT, + "retention_state" TEXT NOT NULL DEFAULT 'full', + "anonymized_at" TIMESTAMPTZ(3), + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "mail_deliveries_pkey" PRIMARY KEY ("id"), + CONSTRAINT "mail_deliveries_priority_check" CHECK ("priority" IN ('critical', 'high', 'normal', 'low')), + CONSTRAINT "mail_deliveries_status_check" CHECK ("status" IN ('queued', 'sending', 'retry_wait', 'sent', 'skipped', 'failed', 'canceled')), + CONSTRAINT "mail_deliveries_attempt_count_check" CHECK ("attempt_count" >= 0), + CONSTRAINT "mail_deliveries_max_attempts_check" CHECK ("max_attempts" >= 0), + CONSTRAINT "mail_deliveries_retention_state_check" CHECK ("retention_state" IN ('full', 'anonymized')), + CONSTRAINT "mail_deliveries_payload_retention_check" CHECK ( + ( + "status" IN ('queued', 'sending', 'retry_wait') + AND "retention_state" = 'full' + AND "recipient_email" IS NOT NULL + AND "payload" IS NOT NULL + ) + OR ( + "status" IN ('sent', 'skipped', 'failed', 'canceled') + AND "retention_state" = 'anonymized' + AND "recipient_email" IS NULL + AND "payload" IS NULL + AND "anonymized_at" IS NOT NULL + ) + ) +); + +-- CreateIndex +CREATE UNIQUE INDEX "mail_deliveries_dedupe_key_idx" + ON "mail_deliveries"("dedupe_key") + WHERE "dedupe_key" IS NOT NULL; + +-- CreateIndex +CREATE INDEX "mail_deliveries_ready_idx" + ON "mail_deliveries"("status", "send_after", "locked_until"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_pending_expires_at_idx" + ON "mail_deliveries"("expires_at") + WHERE "status" IN ('queued', 'sending', 'retry_wait') AND "expires_at" IS NOT NULL; + +-- CreateIndex +CREATE INDEX "mail_deliveries_created_at_idx" ON "mail_deliveries"("created_at"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_mail_name_status_created_at_idx" ON "mail_deliveries"("mail_name", "status", "created_at"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_mail_class_status_created_at_idx" ON "mail_deliveries"("mail_class", "status", "created_at"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_status_created_at_idx" ON "mail_deliveries"("status", "created_at"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_settled_at_idx" ON "mail_deliveries"("settled_at"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_settled_name_status_idx" + ON "mail_deliveries"("settled_at", "mail_name", "status") + WHERE "settled_at" IS NOT NULL; + +-- CreateIndex +CREATE INDEX "mail_deliveries_recipient_domain_created_at_idx" ON "mail_deliveries"("recipient_domain", "created_at"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_workspace_id_created_at_idx" ON "mail_deliveries"("workspace_id", "created_at"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_actor_user_id_created_at_idx" ON "mail_deliveries"("actor_user_id", "created_at"); + +-- CreateIndex +CREATE INDEX "mail_deliveries_abuse_subject_key_created_at_idx" ON "mail_deliveries"("abuse_subject_key", "created_at"); + +-- CreateIndex +CREATE INDEX "workspace_invitations_inviter_created_at_idx" + ON "workspace_invitations"("inviter_user_id", "created_at"); + +-- CreateIndex +CREATE INDEX "workspace_invitations_inviter_status_created_at_idx" + ON "workspace_invitations"("inviter_user_id", "status", "created_at"); + +-- CreateIndex +CREATE INDEX "workspace_invitations_workspace_inviter_created_at_idx" + ON "workspace_invitations"("workspace_id", "inviter_user_id", "created_at"); + +-- CreateIndex +CREATE INDEX "workspace_invitations_workspace_status_created_at_idx" + ON "workspace_invitations"("workspace_id", "status", "created_at"); + +-- CreateIndex +CREATE INDEX "workspace_invitations_workspace_accepted_at_idx" + ON "workspace_invitations"("workspace_id", "accepted_at"); + +-- CreateIndex +CREATE INDEX "workspace_members_workspace_state_created_at_idx" + ON "workspace_members"("workspace_id", "state", "created_at"); diff --git a/packages/backend/server/schema.prisma b/packages/backend/server/schema.prisma index 180a1f1188..4d0c59eecd 100644 --- a/packages/backend/server/schema.prisma +++ b/packages/backend/server/schema.prisma @@ -360,6 +360,58 @@ model DocAccessPolicy { @@map("doc_access_policies") } +model MailDelivery { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + mailName String @map("mail_name") @db.Text + mailClass String @map("mail_class") @db.Text + priority String @default("normal") @db.Text + status String @db.Text + dedupeKey String? @map("dedupe_key") @db.Text + recipientEmail String? @map("recipient_email") @db.Text + recipientHash String @map("recipient_hash") @db.Text + recipientDomain String @map("recipient_domain") @db.Text + recipientUserId String? @map("recipient_user_id") @db.VarChar + actorUserId String? @map("actor_user_id") @db.VarChar + workspaceId String? @map("workspace_id") @db.VarChar + notificationId String? @map("notification_id") @db.VarChar + abuseSubjectKey String? @map("abuse_subject_key") @db.Text + quotaReservationId String? @map("quota_reservation_id") @db.Uuid + quotaDecision Json? @map("quota_decision") @db.JsonB + payload Json? @db.JsonB + sendAfter DateTime @default(now()) @map("send_after") @db.Timestamptz(3) + expiresAt DateTime? @map("expires_at") @db.Timestamptz(3) + attemptCount Int @default(0) @map("attempt_count") @db.Integer + maxAttempts Int @default(3) @map("max_attempts") @db.Integer + lockedBy String? @map("locked_by") @db.Text + lockedUntil DateTime? @map("locked_until") @db.Timestamptz(3) + firstAttemptAt DateTime? @map("first_attempt_at") @db.Timestamptz(3) + lastAttemptAt DateTime? @map("last_attempt_at") @db.Timestamptz(3) + sentAt DateTime? @map("sent_at") @db.Timestamptz(3) + settledAt DateTime? @map("settled_at") @db.Timestamptz(3) + canceledAt DateTime? @map("canceled_at") @db.Timestamptz(3) + failedAt DateTime? @map("failed_at") @db.Timestamptz(3) + providerMessageId String? @map("provider_message_id") @db.Text + providerResponse String? @map("provider_response") @db.Text + lastErrorCode String? @map("last_error_code") @db.Text + lastError String? @map("last_error") @db.Text + retentionState String @default("full") @map("retention_state") @db.Text + anonymizedAt DateTime? @map("anonymized_at") @db.Timestamptz(3) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3) + + @@index([createdAt]) + @@index([mailName, status, createdAt]) + @@index([mailClass, status, createdAt]) + @@index([status, createdAt]) + @@index([status, sendAfter, lockedUntil], map: "mail_deliveries_ready_idx") + @@index([settledAt]) + @@index([recipientDomain, createdAt]) + @@index([workspaceId, createdAt]) + @@index([actorUserId, createdAt]) + @@index([abuseSubjectKey, createdAt]) + @@map("mail_deliveries") +} + model DocGrant { workspaceId String @map("workspace_id") @db.VarChar docId String @map("doc_id") @db.VarChar diff --git a/packages/backend/server/src/__tests__/auth/auth.e2e.ts b/packages/backend/server/src/__tests__/auth/auth.e2e.ts index d05207cf17..46b217b994 100644 --- a/packages/backend/server/src/__tests__/auth/auth.e2e.ts +++ b/packages/backend/server/src/__tests__/auth/auth.e2e.ts @@ -110,7 +110,7 @@ test('set and change password', async t => { const u1 = await app.signupV1(u1Email); await sendSetPasswordEmail(app, u1Email, '/password-change'); - const setPasswordMail = app.mails.last('ChangePassword'); + const setPasswordMail = app.mails.last('SetPassword'); const link = new URL(setPasswordMail.props.url); const setPasswordToken = link.searchParams.get('token'); diff --git a/packages/backend/server/src/__tests__/e2e/user/account.spec.ts b/packages/backend/server/src/__tests__/e2e/user/account.spec.ts index 03d25f3db4..1b7fe84184 100644 --- a/packages/backend/server/src/__tests__/e2e/user/account.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/user/account.spec.ts @@ -4,7 +4,9 @@ import { getCurrentUserQuery, getWorkspaceQuery, } from '@affine/graphql'; +import { WorkspaceMemberStatus } from '@prisma/client'; +import { WorkspaceRole } from '../../../models'; import { app, e2e, Mockers } from '../test'; const admin = await app.create(Mockers.User, { @@ -107,6 +109,31 @@ e2e('should ban account', async t => { t.is(banUser.disabled, true); }); +e2e('should ban account with pending workspace invitation', async t => { + const owner = await app.create(Mockers.User); + const user = await app.create(Mockers.User); + const workspace = await app.create(Mockers.Workspace, { owner }); + + await app.models.workspaceInvitation.set( + workspace.id, + user.id, + WorkspaceRole.Collaborator, + WorkspaceMemberStatus.Pending, + { inviterId: owner.id } + ); + + await app.login(admin); + + const { banUser } = await app.gql({ + query: disableUserMutation, + variables: { + id: user.id, + }, + }); + + t.is(banUser.disabled, true); +}); + e2e('should not login banned account', async t => { const user = await app.create(Mockers.User); diff --git a/packages/backend/server/src/__tests__/mocks/mailer.mock.ts b/packages/backend/server/src/__tests__/mocks/mailer.mock.ts index 49632f52fa..59cc75de6b 100644 --- a/packages/backend/server/src/__tests__/mocks/mailer.mock.ts +++ b/packages/backend/server/src/__tests__/mocks/mailer.mock.ts @@ -2,17 +2,17 @@ import { interval, map, take, takeUntil } from 'rxjs'; import Sinon from 'sinon'; import { Mailer } from '../../core/mail'; +import type { SendMailCommand } from '../../core/mail/types'; import { MailName } from '../../mails'; export class MockMailer { send = Sinon.createStubInstance(Mailer).send.resolves(true); - trySend(command: Jobs['notification.sendMail']) { + skip = Sinon.createStubInstance(Mailer).skip.resolves(false); + trySend(command: SendMailCommand) { return this.send(command, true); } - last( - name: Mail - ): Extract { + last(name: Mail): SendMailCommand & { name: Mail } { const last = this.send.lastCall.args[0]; if (!last) { @@ -23,22 +23,26 @@ export class MockMailer { throw new Error(`Mail name mismatch: ${last.name} !== ${name}`); } - return last as any; + return last as SendMailCommand & { name: Mail }; } waitFor( name: Mail, timeout: number = 1000 - ): Promise> { - const { promise, reject, resolve } = Promise.withResolvers(); + ): Promise { + const { promise, reject, resolve } = Promise.withResolvers< + SendMailCommand & { name: Mail } + >(); interval(10) .pipe( take(Math.floor(timeout / 10)), takeUntil(promise), map(() => { - const last = this.send.lastCall.args[0]; - return last.name === name ? last : undefined; + const last = this.send.lastCall?.args[0]; + return last?.name === name + ? (last as SendMailCommand & { name: Mail }) + : undefined; }) ) .subscribe({ diff --git a/packages/backend/server/src/base/job/queue/config.ts b/packages/backend/server/src/base/job/queue/config.ts index 2752bd2b38..8c8c85425b 100644 --- a/packages/backend/server/src/base/job/queue/config.ts +++ b/packages/backend/server/src/base/job/queue/config.ts @@ -102,4 +102,12 @@ defineModuleConfig('job', { }, schema, }, + + 'queues.inviteAbuse': { + desc: 'The config for invite abuse disposition job queue', + default: { + concurrency: 1, + }, + schema, + }, }); diff --git a/packages/backend/server/src/base/job/queue/def.ts b/packages/backend/server/src/base/job/queue/def.ts index e26c50b8fb..a57b902719 100644 --- a/packages/backend/server/src/base/job/queue/def.ts +++ b/packages/backend/server/src/base/job/queue/def.ts @@ -30,6 +30,7 @@ export enum Queue { INDEXER = 'indexer', CALENDAR = 'calendar', BACKENDRUNTIME = 'backendRuntime', + INVITE_ABUSE = 'inviteAbuse', } export const QUEUES = Object.values(Queue); diff --git a/packages/backend/server/src/core/auth/config.ts b/packages/backend/server/src/core/auth/config.ts index 6e47bf7995..b3181903ee 100644 --- a/packages/backend/server/src/core/auth/config.ts +++ b/packages/backend/server/src/core/auth/config.ts @@ -12,6 +12,9 @@ export interface AuthConfig { requireEmailDomainVerification: boolean; requireEmailVerification: boolean; newAccountShareActionDelay: number; + trustedCloudflareHeaders: boolean; + inviteQuotaShadowMode: boolean; + inviteQuotaFailOpenOnRuntimeError: boolean; passwordRequirements: ConfigItem<{ min: number; max: number; @@ -46,6 +49,21 @@ defineModuleConfig('auth', { default: 24 * 60 * 60, shape: z.number().int().min(0), }, + trustedCloudflareHeaders: { + desc: 'Whether request abuse source facts should trust Cloudflare headers from the origin edge.', + default: false, + shape: z.boolean(), + }, + inviteQuotaShadowMode: { + desc: 'Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions.', + default: false, + shape: z.boolean(), + }, + inviteQuotaFailOpenOnRuntimeError: { + desc: 'Whether workspace invite quota should fail open when native runtime admission is unavailable. Keep disabled for production.', + default: false, + shape: z.boolean(), + }, passwordRequirements: { desc: 'The password strength requirements when set new password.', default: { diff --git a/packages/backend/server/src/core/auth/controller.ts b/packages/backend/server/src/core/auth/controller.ts index 1d72e8719c..3dbcb8fd8a 100644 --- a/packages/backend/server/src/core/auth/controller.ts +++ b/packages/backend/server/src/core/auth/controller.ts @@ -15,6 +15,7 @@ import type { Request, Response } from 'express'; import { ActionForbidden, + Config, EmailTokenNotFound, getRequestCookie, InvalidAuthState, @@ -25,6 +26,7 @@ import { } from '../../base'; import { Models } from '../../models'; import { validators } from '../utils/validators'; +import { getAbuseRequestSource } from '../workspaces/abuse'; import { Public } from './guard'; import { MagicLinkAuthService } from './magic-link'; import { AuthMethodsService } from './methods'; @@ -79,7 +81,8 @@ export class AuthController { private readonly openApp: OpenAppAuthService, private readonly authMethods: AuthMethodsService, private readonly sessionExchange: SessionExchangeService, - private readonly models: Models + private readonly models: Models, + private readonly config: Config ) { if (env.dev) { // set DNS servers in dev mode @@ -135,6 +138,7 @@ export class AuthController { ); } else { await this.sendMagicLink( + req, res, credential.email, credential.callbackUrl, @@ -163,12 +167,15 @@ export class AuthController { } async sendMagicLink( + req: Request, res: Response, email: string, callbackUrl = '/magic-link', clientNonce?: string ) { - const payload = await this.magicLink.send(email, callbackUrl, clientNonce); + const payload = await this.magicLink.send(email, callbackUrl, clientNonce, { + source: getAbuseRequestSource(req, this.config), + }); res.status(HttpStatus.OK).send(payload); } diff --git a/packages/backend/server/src/core/auth/magic-link.ts b/packages/backend/server/src/core/auth/magic-link.ts index 2da2ec767c..5c607e18cb 100644 --- a/packages/backend/server/src/core/auth/magic-link.ts +++ b/packages/backend/server/src/core/auth/magic-link.ts @@ -12,6 +12,7 @@ import { WrongSignInCredentials, } from '../../base'; import { Models, TokenType } from '../../models'; +import type { MailDeliveryMetadata } from '../mail/types'; import { validators } from '../utils/validators'; import { verifyEmailDomainRecords } from './email-domain'; import type { VerifiedIdentity } from './identity'; @@ -29,7 +30,12 @@ export class MagicLinkAuthService { private readonly crypto: CryptoHelper ) {} - async send(email: string, callbackUrl = '/magic-link', clientNonce?: string) { + async send( + email: string, + callbackUrl = '/magic-link', + clientNonce?: string, + metadata?: Pick + ) { validators.assertValidEmail(email); if (!this.url.isAllowedCallbackUrl(callbackUrl)) { @@ -57,21 +63,33 @@ export class MagicLinkAuthService { } const ttlInSec = 30 * 60; - const token = await this.models.verificationToken.create( - TokenType.SignIn, - email, - ttlInSec - ); + const { token, expiresAt: tokenExpiresAt } = + await this.models.verificationToken.createWithExpiresAt( + TokenType.SignIn, + email, + ttlInSec + ); const otp = this.crypto.otp(); - await this.models.magicLinkOtp.upsert(email, otp, token, clientNonce); + const { expiresAt: otpExpiresAt } = await this.models.magicLinkOtp.upsert( + email, + otp, + token, + clientNonce + ); const magicLink = this.url.link(callbackUrl, { token: otp, email }); if (env.dev) { this.logger.debug(`Magic link: ${magicLink}`); } - await this.auth.sendSignInEmail(email, magicLink, otp, !user); + await this.auth.sendSignInEmail(email, magicLink, otp, !user, { + ...metadata, + expiresAt: + tokenExpiresAt.getTime() < otpExpiresAt.getTime() + ? tokenExpiresAt + : otpExpiresAt, + }); return { email }; } diff --git a/packages/backend/server/src/core/auth/resolver.ts b/packages/backend/server/src/core/auth/resolver.ts index 68b0e07d13..eb75404839 100644 --- a/packages/backend/server/src/core/auth/resolver.ts +++ b/packages/backend/server/src/core/auth/resolver.ts @@ -1,5 +1,6 @@ import { Args, + Context, Field, Mutation, ObjectType, @@ -11,6 +12,7 @@ import { import { ActionForbidden, + Config, EmailAlreadyUsed, EmailTokenNotFound, EmailVerificationRequired, @@ -21,10 +23,13 @@ import { Throttle, URLHelper, } from '../../base'; +import type { GraphqlContext } from '../../base/graphql'; import { Models, TokenType } from '../../models'; import { Admin } from '../common'; +import type { MailDeliveryMetadata } from '../mail/types'; import { UserType } from '../user/types'; import { validators } from '../utils/validators'; +import { getAbuseRequestSource } from '../workspaces/abuse'; import { Public } from './guard'; import { AuthService } from './service'; import { CurrentUser } from './session'; @@ -47,9 +52,20 @@ export class AuthResolver { constructor( private readonly url: URLHelper, private readonly auth: AuthService, - private readonly models: Models + private readonly models: Models, + private readonly config: Config ) {} + private mailMetadata( + context: GraphqlContext, + expiresAt?: Date + ): MailDeliveryMetadata { + return { + source: getAbuseRequestSource(context.req, this.config), + expiresAt, + }; + } + @SkipThrottle() @Public() @Query(() => UserType, { @@ -145,23 +161,30 @@ export class AuthResolver { @CurrentUser() user: CurrentUser, @Args('callbackUrl') callbackUrl: string, @Args('email', { + type: () => String, nullable: true, deprecationReason: 'fetched from signed in user', }) - _email?: string + _email: string | undefined, + @Context() context: GraphqlContext ) { if (!user.emailVerified) { throw new EmailVerificationRequired(); } - const token = await this.models.verificationToken.create( - TokenType.ChangePassword, - user.id - ); + const { token, expiresAt } = + await this.models.verificationToken.createWithExpiresAt( + TokenType.ChangePassword, + user.id + ); const url = this.url.safeLink(callbackUrl, { userId: user.id, token }); - return await this.auth.sendChangePasswordEmail(user.email, url); + return await this.auth.sendChangePasswordEmail( + user.email, + url, + this.mailMetadata(context, expiresAt) + ); } @Mutation(() => Boolean) @@ -169,12 +192,26 @@ export class AuthResolver { @CurrentUser() user: CurrentUser, @Args('callbackUrl') callbackUrl: string, @Args('email', { + type: () => String, nullable: true, deprecationReason: 'fetched from signed in user', }) - _email?: string + _email: string | undefined, + @Context() context: GraphqlContext ) { - return this.sendChangePasswordEmail(user, callbackUrl); + const { token, expiresAt } = + await this.models.verificationToken.createWithExpiresAt( + TokenType.ChangePassword, + user.id + ); + + const url = this.url.safeLink(callbackUrl, { userId: user.id, token }); + + return await this.auth.sendSetPasswordEmail( + user.email, + url, + this.mailMetadata(context, expiresAt) + ); } // The change email step is: @@ -187,20 +224,26 @@ export class AuthResolver { @Mutation(() => Boolean) async sendChangeEmail( @CurrentUser() user: CurrentUser, - @Args('callbackUrl') callbackUrl: string + @Args('callbackUrl') callbackUrl: string, + @Context() context: GraphqlContext ) { if (!user.emailVerified) { throw new EmailVerificationRequired(); } - const token = await this.models.verificationToken.create( - TokenType.ChangeEmail, - user.id - ); + const { token, expiresAt } = + await this.models.verificationToken.createWithExpiresAt( + TokenType.ChangeEmail, + user.id + ); const url = this.url.safeLink(callbackUrl, { token }); - return await this.auth.sendChangeEmail(user.email, url); + return await this.auth.sendChangeEmail( + user.email, + url, + this.mailMetadata(context, expiresAt) + ); } @Mutation(() => Boolean) @@ -208,7 +251,8 @@ export class AuthResolver { @CurrentUser() user: CurrentUser, @Args('token') token: string, @Args('email') email: string, - @Args('callbackUrl') callbackUrl: string + @Args('callbackUrl') callbackUrl: string, + @Context() context: GraphqlContext ) { if (!token) { throw new EmailTokenNotFound(); @@ -237,31 +281,42 @@ export class AuthResolver { } } - const verifyEmailToken = await this.models.verificationToken.create( - TokenType.VerifyEmail, - user.id - ); + const { token: verifyEmailToken, expiresAt } = + await this.models.verificationToken.createWithExpiresAt( + TokenType.VerifyEmail, + user.id + ); const url = this.url.safeLink(callbackUrl, { token: verifyEmailToken, email, }); - return await this.auth.sendVerifyChangeEmail(email, url); + return await this.auth.sendVerifyChangeEmail( + email, + url, + this.mailMetadata(context, expiresAt) + ); } @Mutation(() => Boolean) async sendVerifyEmail( @CurrentUser() user: CurrentUser, - @Args('callbackUrl') callbackUrl: string + @Args('callbackUrl') callbackUrl: string, + @Context() context: GraphqlContext ) { - const token = await this.models.verificationToken.create( - TokenType.VerifyEmail, - user.id - ); + const { token, expiresAt } = + await this.models.verificationToken.createWithExpiresAt( + TokenType.VerifyEmail, + user.id + ); const url = this.url.safeLink(callbackUrl, { token }); - return await this.auth.sendVerifyEmail(user.email, url); + return await this.auth.sendVerifyEmail( + user.email, + url, + this.mailMetadata(context, expiresAt) + ); } @Mutation(() => Boolean) diff --git a/packages/backend/server/src/core/auth/service.ts b/packages/backend/server/src/core/auth/service.ts index 90e18baac0..d21b78c98f 100644 --- a/packages/backend/server/src/core/auth/service.ts +++ b/packages/backend/server/src/core/auth/service.ts @@ -7,6 +7,7 @@ import { assign, pick } from 'lodash-es'; import { Config, SignUpForbidden } from '../../base'; import { Models, type User, type UserSession } from '../../models'; import { Mailer } from '../mail/mailer'; +import type { MailDeliveryMetadata } from '../mail/types'; import { createDevUsers } from './dev'; import type { VerifiedIdentity } from './identity'; import { @@ -312,49 +313,74 @@ export class AuthService implements OnApplicationBootstrap { }); } - async sendChangePasswordEmail(email: string, callbackUrl: string) { + async sendChangePasswordEmail( + email: string, + callbackUrl: string, + metadata?: MailDeliveryMetadata + ) { return await this.mailer.send({ name: 'ChangePassword', to: email, props: { url: callbackUrl, }, + metadata, }); } - async sendSetPasswordEmail(email: string, callbackUrl: string) { + async sendSetPasswordEmail( + email: string, + callbackUrl: string, + metadata?: MailDeliveryMetadata + ) { return await this.mailer.send({ name: 'SetPassword', to: email, props: { url: callbackUrl, }, + metadata, }); } - async sendChangeEmail(email: string, callbackUrl: string) { + async sendChangeEmail( + email: string, + callbackUrl: string, + metadata?: MailDeliveryMetadata + ) { return await this.mailer.send({ name: 'ChangeEmail', to: email, props: { url: callbackUrl, }, + metadata, }); } - async sendVerifyChangeEmail(email: string, callbackUrl: string) { + async sendVerifyChangeEmail( + email: string, + callbackUrl: string, + metadata?: MailDeliveryMetadata + ) { return await this.mailer.send({ name: 'VerifyChangeEmail', to: email, props: { url: callbackUrl, }, + metadata, }); } - async sendVerifyEmail(email: string, callbackUrl: string) { + async sendVerifyEmail( + email: string, + callbackUrl: string, + metadata?: MailDeliveryMetadata + ) { return await this.mailer.send({ name: 'VerifyEmail', to: email, props: { url: callbackUrl, }, + metadata, }); } async sendNotificationChangeEmail(email: string) { @@ -371,7 +397,8 @@ export class AuthService implements OnApplicationBootstrap { email: string, link: string, otp: string, - signUp: boolean + signUp: boolean, + metadata?: MailDeliveryMetadata ) { return await this.mailer.send({ name: signUp ? 'SignUp' : 'SignIn', @@ -381,6 +408,7 @@ export class AuthService implements OnApplicationBootstrap { otp, serverName: this.getServerName(), }, + metadata, }); } } diff --git a/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts b/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts index f3fcf28d22..77e684d798 100644 --- a/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts +++ b/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts @@ -15,6 +15,7 @@ interface Context { runtime: { cleanupExpiredRuntimeStates: Sinon.SinonStub; cleanupExpiredRuntimeGates: Sinon.SinonStub; + cleanupExpiredRollingQuota: Sinon.SinonStub; }; } @@ -24,6 +25,7 @@ test.before(async t => { t.context.runtime = { cleanupExpiredRuntimeStates: Sinon.stub(), cleanupExpiredRuntimeGates: Sinon.stub(), + cleanupExpiredRollingQuota: Sinon.stub(), }; t.context.module = await createTestingModule({ imports: [ScheduleModule.forRoot(), BackendRuntimeModule], @@ -39,6 +41,7 @@ test.before(async t => { test.beforeEach(t => { t.context.runtime.cleanupExpiredRuntimeStates.reset(); t.context.runtime.cleanupExpiredRuntimeGates.reset(); + t.context.runtime.cleanupExpiredRollingQuota.reset(); }); test.after.always(async t => { @@ -49,9 +52,11 @@ test('backend-runtime housekeeping cleans runtime state and gate batches', async t.context.runtime.cleanupExpiredRuntimeStates.onCall(0).resolves(1000); t.context.runtime.cleanupExpiredRuntimeStates.onCall(1).resolves(2); t.context.runtime.cleanupExpiredRuntimeGates.resolves(1); + t.context.runtime.cleanupExpiredRollingQuota.resolves(1); await t.context.job.cleanExpiredRuntimeHousekeeping(); t.is(t.context.runtime.cleanupExpiredRuntimeStates.callCount, 2); t.is(t.context.runtime.cleanupExpiredRuntimeGates.callCount, 1); + t.is(t.context.runtime.cleanupExpiredRollingQuota.callCount, 1); }); diff --git a/packages/backend/server/src/core/backend-runtime/index.ts b/packages/backend/server/src/core/backend-runtime/index.ts index 19ecf7a3e5..d0d12ee2ba 100644 --- a/packages/backend/server/src/core/backend-runtime/index.ts +++ b/packages/backend/server/src/core/backend-runtime/index.ts @@ -10,4 +10,15 @@ import { BackendRuntimeProvider } from './provider'; }) export class BackendRuntimeModule {} -export { BackendRuntimeProvider } from './provider'; +export { + BackendRuntimeProvider, + type RuntimeInviteAbuseAction, + type RuntimeInviteAbuseClaimedAction, + type RuntimeMailDeliveryQuotaDecision, + type RuntimeMailDeliveryQuotaInput, + type RuntimeQuotaSourceInput, + type RuntimeQuotaTargetDomainInput, + type RuntimeWorkspaceInviteQuotaDecision, + type RuntimeWorkspaceInviteQuotaInput, + type RuntimeWorkspaceInviteQuotaUsage, +} from './provider'; diff --git a/packages/backend/server/src/core/backend-runtime/job.ts b/packages/backend/server/src/core/backend-runtime/job.ts index fce8da818a..1ff8567066 100644 --- a/packages/backend/server/src/core/backend-runtime/job.ts +++ b/packages/backend/server/src/core/backend-runtime/job.ts @@ -38,9 +38,12 @@ export class BackendRuntimeHousekeepingJob { const gates = await this.cleanBatches(() => this.rt.cleanupExpiredRuntimeGates(1000) ); + const rollingQuota = await this.cleanBatches(() => + this.rt.cleanupExpiredRollingQuota(1000) + ); this.logger.log( - `cleaned runtime housekeeping states=${states} gates=${gates}` + `cleaned runtime housekeeping states=${states} gates=${gates} rollingQuota=${rollingQuota}` ); } diff --git a/packages/backend/server/src/core/backend-runtime/provider.ts b/packages/backend/server/src/core/backend-runtime/provider.ts index 91c6f33235..92b5e71e2a 100644 --- a/packages/backend/server/src/core/backend-runtime/provider.ts +++ b/packages/backend/server/src/core/backend-runtime/provider.ts @@ -10,6 +10,187 @@ import { BackendRuntime, type BackendRuntimeHealth } from '../../native'; type RuntimeInstance = InstanceType; +export type RuntimeQuotaTargetDomainInput = { + domain: string; + count: number; +}; + +export type RuntimeQuotaSourceInput = { + trusted: boolean; + ip?: string; + country?: string; + asn?: number; + rayId?: string; +}; + +export type RuntimeWorkspaceInviteQuotaInput = { + actorUserId: string; + workspaceId: string; + requestId?: string; + targetCount: number; + targetDomains: RuntimeQuotaTargetDomainInput[]; + source?: RuntimeQuotaSourceInput; +}; + +export type RuntimeWorkspaceInviteQuotaUsage = { + targetCount: number; + targetDomains: RuntimeQuotaTargetDomainInput[]; +}; + +export type RuntimeInviteAbuseAction = + | 'ban_actor' + | 'quarantine_actor' + | 'quarantine_workspace' + | 'quarantine_source_cohort'; + +const RUNTIME_INVITE_ABUSE_ACTIONS = new Set([ + 'ban_actor', + 'quarantine_actor', + 'quarantine_workspace', + 'quarantine_source_cohort', +]); + +export type RuntimeInviteAbuseClaimedAction = { + action: RuntimeInviteAbuseAction; + subjectKey: string; + evidenceId: string; + actionId: string; + actorUserId: string; + workspaceId: string; +}; + +type NativeRuntimeInviteAbuseClaimedAction = Omit< + RuntimeInviteAbuseClaimedAction, + 'action' +> & { + action: string; +}; + +export type RuntimeWorkspaceInviteQuotaDecision = { + allowed: boolean; + reservationId?: string; + retryAfterSeconds?: number; + reason?: string; + scopeKey?: string; + windowSeconds?: number; + limit?: number; + current?: number; + requested?: number; + actionRequired?: { + action: RuntimeInviteAbuseAction; + subjectKey: string; + evidenceId: string; + actionId: string; + }; +}; + +type NativeRuntimeInviteAbuseActionRequired = Omit< + NonNullable, + 'action' +> & { + action: string; +}; + +type NativeRuntimeWorkspaceInviteQuotaDecision = Omit< + RuntimeWorkspaceInviteQuotaDecision, + 'actionRequired' +> & { + actionRequired?: NativeRuntimeInviteAbuseActionRequired; +}; + +export type RuntimeMailDeliveryQuotaInput = { + requestId?: string; + mailName: string; + recipient: { + email: string; + domain: string; + userId?: string; + }; + metadata: { + actorUserId?: string; + workspaceId?: string; + notificationId?: string; + abuseSubjectKey?: string; + }; + source?: RuntimeQuotaSourceInput; +}; + +export type RuntimeMailDeliveryQuotaDecision = { + allowed: boolean; + reservationId?: string; + mailClass: string; + retryAfterSeconds?: number; + reason?: string; + scopeKey?: string; + windowSeconds?: number; + limit?: number; + current?: number; + requested?: number; +}; + +type RuntimeQuotaMethods = RuntimeInstance & { + assertWorkspaceInviteQuotaV1( + input: RuntimeWorkspaceInviteQuotaInput + ): Promise; + commitWorkspaceInviteQuotaV1( + reservationId: string, + usage: RuntimeWorkspaceInviteQuotaUsage + ): Promise; + releaseWorkspaceInviteQuotaV1(reservationId: string): Promise; + assertMailDeliveryQuotaV1( + input: RuntimeMailDeliveryQuotaInput + ): Promise; + commitMailDeliveryQuotaV1(reservationId: string): Promise; + releaseMailDeliveryQuotaV1(reservationId: string): Promise; + cleanupExpiredRollingQuota(limit: number): Promise; + isInviteAbuseUserQuarantinedOrBanned(userId: string): Promise; + isInviteAbuseWorkspaceQuarantined(workspaceId: string): Promise; + claimInviteAbuseAction(actionId: string, workerId: string): Promise; + claimRetryableInviteAbuseActions( + workerId: string, + limit: number + ): Promise; + markInviteAbuseAction( + actionId: string, + workerId: string, + status: 'succeeded' | 'failed', + error?: string | null + ): Promise; +}; + +function normalizeInviteAbuseAction(action: string): RuntimeInviteAbuseAction { + if (RUNTIME_INVITE_ABUSE_ACTIONS.has(action as RuntimeInviteAbuseAction)) { + return action as RuntimeInviteAbuseAction; + } + throw new Error(`Unknown invite abuse action: ${action}`); +} + +function normalizeWorkspaceInviteQuotaDecision( + decision: NativeRuntimeWorkspaceInviteQuotaDecision +): RuntimeWorkspaceInviteQuotaDecision { + const { actionRequired, ...rest } = decision; + if (!actionRequired) { + return rest; + } + + return { + ...rest, + actionRequired: { + ...actionRequired, + action: normalizeInviteAbuseAction(actionRequired.action), + }, + }; +} + +function normalizeClaimedInviteAbuseAction( + action: NativeRuntimeInviteAbuseClaimedAction +): RuntimeInviteAbuseClaimedAction { + return { + ...action, + action: normalizeInviteAbuseAction(action.action), + }; +} + @Injectable() export class BackendRuntimeProvider implements OnApplicationBootstrap, OnApplicationShutdown @@ -66,6 +247,102 @@ export class BackendRuntimeProvider ); } + async assertWorkspaceInviteQuotaV1( + input: RuntimeWorkspaceInviteQuotaInput + ): Promise { + return normalizeWorkspaceInviteQuotaDecision( + await this.measured('assertWorkspaceInviteQuotaV1', rt => + this.quotaRuntime(rt).assertWorkspaceInviteQuotaV1(input) + ) + ); + } + + async commitWorkspaceInviteQuotaV1( + reservationId: string, + usage: RuntimeWorkspaceInviteQuotaUsage + ): Promise { + return await this.measured('commitWorkspaceInviteQuotaV1', rt => + this.quotaRuntime(rt).commitWorkspaceInviteQuotaV1(reservationId, usage) + ); + } + + async releaseWorkspaceInviteQuotaV1(reservationId: string): Promise { + return await this.measured('releaseWorkspaceInviteQuotaV1', rt => + this.quotaRuntime(rt).releaseWorkspaceInviteQuotaV1(reservationId) + ); + } + + async assertMailDeliveryQuotaV1( + input: RuntimeMailDeliveryQuotaInput + ): Promise { + return await this.measured('assertMailDeliveryQuotaV1', rt => + this.quotaRuntime(rt).assertMailDeliveryQuotaV1(input) + ); + } + + async commitMailDeliveryQuotaV1(reservationId: string): Promise { + return await this.measured('commitMailDeliveryQuotaV1', rt => + this.quotaRuntime(rt).commitMailDeliveryQuotaV1(reservationId) + ); + } + + async releaseMailDeliveryQuotaV1(reservationId: string): Promise { + return await this.measured('releaseMailDeliveryQuotaV1', rt => + this.quotaRuntime(rt).releaseMailDeliveryQuotaV1(reservationId) + ); + } + + async cleanupExpiredRollingQuota(limit: number) { + return await this.measured('cleanupExpiredRollingQuota', rt => + this.quotaRuntime(rt).cleanupExpiredRollingQuota(limit) + ); + } + + async isInviteAbuseUserQuarantinedOrBanned(userId: string) { + return await this.measured('isInviteAbuseUserQuarantinedOrBanned', rt => + this.quotaRuntime(rt).isInviteAbuseUserQuarantinedOrBanned(userId) + ); + } + + async isInviteAbuseWorkspaceQuarantined(workspaceId: string) { + return await this.measured('isInviteAbuseWorkspaceQuarantined', rt => + this.quotaRuntime(rt).isInviteAbuseWorkspaceQuarantined(workspaceId) + ); + } + + async claimInviteAbuseAction(actionId: string, workerId: string) { + return await this.measured('claimInviteAbuseAction', rt => + this.quotaRuntime(rt).claimInviteAbuseAction(actionId, workerId) + ); + } + + async claimRetryableInviteAbuseActions( + workerId: string, + limit: number + ): Promise { + return ( + await this.measured('claimRetryableInviteAbuseActions', rt => + this.quotaRuntime(rt).claimRetryableInviteAbuseActions(workerId, limit) + ) + ).map(normalizeClaimedInviteAbuseAction); + } + + async markInviteAbuseAction( + actionId: string, + workerId: string, + status: 'succeeded' | 'failed', + error?: string | null + ) { + return await this.measured('markInviteAbuseAction', rt => + this.quotaRuntime(rt).markInviteAbuseAction( + actionId, + workerId, + status, + error + ) + ); + } + private async measured( method: string, fn: (runtime: RuntimeInstance) => Promise @@ -78,6 +355,10 @@ export class BackendRuntimeProvider )(); } + private quotaRuntime(runtime: RuntimeInstance): RuntimeQuotaMethods { + return runtime as unknown as RuntimeQuotaMethods; + } + private async runMigrationsOnce() { if (this.migrationsStarted) { return; diff --git a/packages/backend/server/src/core/content-policy.ts b/packages/backend/server/src/core/content-policy.ts new file mode 100644 index 0000000000..8de91572d2 --- /dev/null +++ b/packages/backend/server/src/core/content-policy.ts @@ -0,0 +1,14 @@ +import { scanContentPolicyV1 } from '../native'; + +export function scanContentPolicy(value: string | null | undefined) { + return scanContentPolicyV1({ + value: value ?? '', + checks: ['url_or_domain'], + }); +} + +export function containsUrlOrDomain(value: string | null | undefined) { + return scanContentPolicy(value).matches.some( + match => match.type === 'url_or_domain' + ); +} diff --git a/packages/backend/server/src/core/mail/__tests__/job.spec.ts b/packages/backend/server/src/core/mail/__tests__/job.spec.ts index 5bad0da05d..eb413847c2 100644 --- a/packages/backend/server/src/core/mail/__tests__/job.spec.ts +++ b/packages/backend/server/src/core/mail/__tests__/job.spec.ts @@ -1,62 +1,70 @@ +import { Prisma, PrismaClient } from '@prisma/client'; import test from 'ava'; import Sinon from 'sinon'; import { Mockers } from '../../../__tests__/mocks'; import { createTestingModule } from '../../../__tests__/utils'; -import { Cache } from '../../../base'; import { Models } from '../../../models'; import { MailJob } from '../job'; import { MailSender } from '../sender'; let module: Awaited>; -let cache: Cache; let mailJob: MailJob; let sender: MailSender; let models: Models; +let db: PrismaClient; test.before(async () => { module = await createTestingModule(); - cache = module.get(Cache); mailJob = module.get(MailJob); sender = module.get(MailSender); models = module.get(Models); + db = module.get(PrismaClient); }); test.after.always(async () => { await module.close(); }); -test.afterEach(() => { +test.afterEach.always(async () => { Sinon.restore(); + await db.mailDelivery.deleteMany(); }); -test('should clear pending mail records when user is deleted', async t => { +async function createDelivery( + input: { + name: 'SignIn' | 'VerifyEmail' | 'MemberInvitation'; + to: string; + props: Record; + }, + overrides: Partial[0]> = {} +) { + const mailClass = + input.name === 'MemberInvitation' ? 'workspace_invitation' : 'auth'; + return await models.mailDelivery.create({ + mailName: input.name, + mailClass, + priority: mailClass === 'auth' ? 'critical' : 'normal', + recipientEmail: input.to, + payload: input as Prisma.JsonObject, + ...overrides, + }); +} + +async function delivery(id: string) { + return await db.mailDelivery.findUniqueOrThrow({ where: { id } }); +} + +test('should cancel pending mail deliveries when user is deleted', async t => { const user = await module.create(Mockers.User); const another = await module.create(Mockers.User); - const sendMailKey = 'mailjob:sendMail'; - const retryMailKey = 'mailjob:sendMail:retry'; - const userKey = `${sendMailKey}:SignIn:${user.email}`; - const userRetryKey = `${sendMailKey}:VerifyEmail:${user.email}`; - const anotherKey = `${sendMailKey}:SignIn:${another.email}`; - const senderRetryKey = `${retryMailKey}:MemberInvitation:invited@affine.pro`; - - await cache.mapSet(sendMailKey, userKey, 1); - await cache.mapSet(sendMailKey, anotherKey, 1); - await cache.mapSet( - retryMailKey, - userRetryKey, - JSON.stringify({ - startTime: Date.now(), - name: 'VerifyEmail', - to: user.email, - props: { url: 'https://affine.pro/verify' }, - }) - ); - await cache.mapSet( - retryMailKey, - senderRetryKey, - JSON.stringify({ - startTime: Date.now(), + const recipientDelivery = await createDelivery({ + name: 'SignIn', + to: user.email, + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, + }); + const senderDelivery = await createDelivery( + { name: 'MemberInvitation', to: 'invited@affine.pro', props: { @@ -64,153 +72,126 @@ test('should clear pending mail records when user is deleted', async t => { workspace: { $$workspaceId: 'workspace-id' }, url: 'https://affine.pro/invite', }, - }) + }, + { actorUserId: user.id } ); + const anotherDelivery = await createDelivery({ + name: 'SignIn', + to: another.email, + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, + }); await mailJob.onUserDeleted({ ...user, ownedWorkspaces: [] }); - t.true(module.queue.removeWhere.calledOnce); - t.is(module.queue.removeWhere.firstCall.args[0], 'notification.sendMail'); - const shouldRemove = module.queue.removeWhere.firstCall.args[1]; - t.true( - await shouldRemove({ - to: user.email, - } as Jobs['notification.sendMail']) - ); - t.true( - await shouldRemove({ - name: 'MemberInvitation', - to: 'invited@affine.pro', - props: { - user: { $$userId: user.id }, - workspace: { $$workspaceId: 'workspace-id' }, - url: 'https://affine.pro/invite', - }, - } as Jobs['notification.sendMail']) - ); - t.false( - await shouldRemove({ - to: another.email, - } as Jobs['notification.sendMail']) - ); - t.is(await cache.mapGet(sendMailKey, userKey), undefined); - t.is(await cache.mapGet(retryMailKey, userRetryKey), undefined); - t.is(await cache.mapGet(retryMailKey, senderRetryKey), undefined); - t.is(await cache.mapGet(sendMailKey, anotherKey), 1); + t.is((await delivery(recipientDelivery.id)).status, 'canceled'); + t.is((await delivery(senderDelivery.id)).status, 'canceled'); + t.is((await delivery(anotherDelivery.id)).status, 'queued'); + t.is((await delivery(recipientDelivery.id)).recipientEmail, null); + t.is((await delivery(senderDelivery.id)).payload, null); }); test('should skip queued mail for disabled recipient', async t => { const user = await module.create(Mockers.User, { disabled: true }); - const send = Sinon.stub(sender, 'send').resolves(true); - - await mailJob.sendMail({ - startTime: Date.now(), + const send = Sinon.stub(sender, 'send').resolves({ + status: 'accepted', + retryable: false, + }); + const row = await createDelivery({ name: 'SignIn', to: user.email, - props: { - url: 'https://affine.pro/sign-in', - otp: '123456', - }, + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, }); + await mailJob.processReadyDeliveries(); + + const updated = await delivery(row.id); t.false(send.called); - t.truthy(await models.user.get(user.id, { withDisabled: true })); + t.is(updated.status, 'skipped'); + t.is(updated.lastErrorCode, 'disabled_recipient'); + t.is(updated.recipientEmail, null); + t.is(updated.payload, null); }); -test('should drop expired mail retry', async t => { - const send = Sinon.stub(sender, 'send').resolves(true); - - await mailJob.sendMail({ - startTime: Date.now() - 25 * 60 * 60 * 1000, - name: 'SignIn', - to: 'expired-retry@example.com', - props: { - url: 'https://affine.pro/sign-in', - otp: '123456', - }, +test('should not create sendable row for expired mail', async t => { + const send = Sinon.stub(sender, 'send').resolves({ + status: 'accepted', + retryable: false, }); - - t.false(send.called); -}); - -test('should drop time-sensitive mail after its business expiration', async t => { - const send = Sinon.stub(sender, 'send').resolves(true); - - await mailJob.sendMail({ - startTime: Date.now() - 31 * 60 * 1000, - name: 'SignIn', - to: 'expired-sign-in@example.com', - props: { - url: 'https://affine.pro/sign-in', - otp: '123456', + const row = await createDelivery( + { + name: 'SignIn', + to: 'expired-retry@example.com', + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, }, - }); + { expiresAt: new Date(Date.now() - 1) } + ); + await mailJob.processReadyDeliveries(); + + const updated = await delivery(row.id); t.false(send.called); + t.is(updated.status, 'failed'); + t.is(updated.lastErrorCode, 'expired'); + t.is(updated.retentionState, 'anonymized'); }); -test('should use explicit mail expiration when provided', async t => { - const send = Sinon.stub(sender, 'send').resolves(true); +test('should not claim delivery when max attempts is zero', async t => { + const send = Sinon.stub(sender, 'send').resolves({ + status: 'accepted', + retryable: false, + }); + const row = await createDelivery( + { + name: 'SignIn', + to: 'max-attempts@example.com', + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, + }, + { maxAttempts: 0 } + ); - await mailJob.sendMail({ - startTime: Date.now(), - expiresAt: Date.now() - 1, - name: 'MemberInvitation', - to: 'expired-invitation@example.com', - props: { - user: { - $$userId: 'owner-id', + await mailJob.processReadyDeliveries(); + + t.false(send.called); + t.is((await delivery(row.id)).status, 'queued'); +}); + +test('should retry retryable send failures without mutating stored dynamic props', async t => { + const owner = await module.create(Mockers.User); + const member = await module.create(Mockers.User); + const workspace = await module.create(Mockers.Workspace, { + owner: { id: owner.id }, + name: 'Safe Workspace', + }); + Sinon.stub(sender, 'send').resolves({ + status: 'failed', + retryable: true, + errorCode: 'transport_failed', + error: 'temporary failure', + }); + const row = await createDelivery( + { + name: 'MemberInvitation', + to: member.email, + props: { + user: { $$userId: owner.id }, + workspace: { $$workspaceId: workspace.id }, + url: 'https://affine.pro/invite/test', }, - workspace: { - $$workspaceId: 'workspace-id', - }, - url: 'https://affine.pro/invite/test', + }, + { actorUserId: owner.id, workspaceId: workspace.id } + ); + + await mailJob.processReadyDeliveries(); + + const updated = await delivery(row.id); + t.is(updated.status, 'retry_wait'); + t.is(updated.attemptCount, 1); + t.like(updated.payload as object, { + props: { + user: { $$userId: owner.id }, + workspace: { $$workspaceId: workspace.id }, }, }); - - t.false(send.called); -}); - -test('should drop mail retry after max attempts', async t => { - const send = Sinon.stub(sender, 'send').resolves(true); - - await mailJob.sendMail({ - startTime: Date.now(), - retryCount: 12, - name: 'SignIn', - to: 'max-retry@example.com', - props: { - url: 'https://affine.pro/sign-in', - otp: '123456', - }, - }); - - t.false(send.called); -}); - -test('should requeue legacy stringified retry mail', async t => { - const retryMailKey = 'mailjob:sendMail:retry'; - const job: Jobs['notification.sendMail'] = { - startTime: Date.now(), - name: 'SignIn', - to: 'legacy-retry@example.com', - props: { - url: 'https://affine.pro/sign-in', - otp: '123456', - }, - }; - const cacheKey = `${retryMailKey}:SignIn:${job.to}`; - - Sinon.stub(cache, 'mapRandomKey') - .onFirstCall() - .resolves(cacheKey) - .onSecondCall() - .resolves(undefined); - await cache.mapSet(retryMailKey, cacheKey, JSON.stringify(job)); - await mailJob.sendRetryMails(); - - t.true(module.queue.add.calledWith('notification.sendMail', job)); - t.is(await cache.mapGet(retryMailKey, cacheKey), undefined); }); test('should skip member invitation mail when rendered workspace name contains domain', async t => { @@ -218,53 +199,123 @@ test('should skip member invitation mail when rendered workspace name contains d const member = await module.create(Mockers.User); const workspace = await module.create(Mockers.Workspace, { owner: { id: owner.id }, - name: 'BTC https://spam.example', + name: 'BTC example.com', }); - const send = Sinon.stub(sender, 'send').resolves(true); - - await mailJob.sendMail({ - startTime: Date.now(), - name: 'MemberInvitation', - to: member.email, - props: { - user: { - $$userId: owner.id, + const send = Sinon.stub(sender, 'send').resolves({ + status: 'accepted', + retryable: false, + }); + const row = await createDelivery( + { + name: 'MemberInvitation', + to: member.email, + props: { + user: { $$userId: owner.id }, + workspace: { $$workspaceId: workspace.id }, + url: 'https://affine.pro/invite/test', }, - workspace: { - $$workspaceId: workspace.id, - }, - url: 'https://affine.pro/invite/test', }, - }); + { actorUserId: owner.id, workspaceId: workspace.id } + ); + await mailJob.processReadyDeliveries(); + + const updated = await delivery(row.id); t.false(send.called); + t.is(updated.status, 'skipped'); + t.is(updated.lastErrorCode, 'dynamic_props_missing'); }); -test('should keep dynamic mail props untouched for retry', async t => { - const owner = await module.create(Mockers.User); - const member = await module.create(Mockers.User); - const workspace = await module.create(Mockers.Workspace, { - owner: { id: owner.id }, - name: 'Safe Workspace', +test('should mark accepted mail as sent and anonymize sendable payload', async t => { + const user = await module.create(Mockers.User); + Sinon.stub(sender, 'send').resolves({ + status: 'accepted', + retryable: false, + providerMessageId: 'message-id', + providerResponse: '250 ok', + }); + const row = await createDelivery({ + name: 'SignIn', + to: user.email, + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, }); - Sinon.stub(sender, 'send').resolves(false); - const job: Jobs['notification.sendMail'] = { - startTime: Date.now(), - name: 'MemberInvitation', - to: member.email, - props: { - user: { - $$userId: owner.id, - }, - workspace: { - $$workspaceId: workspace.id, - }, - url: 'https://affine.pro/invite/test', - }, - }; - await mailJob.sendMail(job); + await mailJob.processReadyDeliveries(); - t.deepEqual(job.props.user, { $$userId: owner.id }); - t.deepEqual(job.props.workspace, { $$workspaceId: workspace.id }); + const updated = await delivery(row.id); + t.is(updated.status, 'sent'); + t.is(updated.providerMessageId, 'message-id'); + t.is(updated.recipientEmail, null); + t.is(updated.payload, null); + t.is(updated.retentionState, 'anonymized'); +}); + +test('should claim critical priority before lower priority rows', async t => { + Sinon.stub(sender, 'send').resolves({ + status: 'accepted', + retryable: false, + }); + const low = await createDelivery( + { + name: 'SignIn', + to: 'low-priority@example.com', + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, + }, + { priority: 'low' } + ); + const critical = await createDelivery({ + name: 'SignIn', + to: 'critical-priority@example.com', + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, + }); + + await mailJob.processReadyDeliveries(1); + + t.is((await delivery(critical.id)).status, 'sent'); + t.is((await delivery(low.id)).status, 'queued'); +}); + +test('should reclaim expired sending lease', async t => { + Sinon.stub(sender, 'send').resolves({ + status: 'accepted', + retryable: false, + }); + const row = await createDelivery({ + name: 'SignIn', + to: 'reclaim@example.com', + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, + }); + await db.mailDelivery.update({ + where: { id: row.id }, + data: { + status: 'sending', + lockedBy: 'dead-worker', + lockedUntil: new Date(Date.now() - 1000), + }, + }); + + await mailJob.processReadyDeliveries(); + + t.is((await delivery(row.id)).status, 'sent'); +}); + +test('should delete retained anonymized terminal rows on worker tick', async t => { + const row = await createDelivery( + { + name: 'SignIn', + to: 'retention@example.com', + props: { url: 'https://affine.pro/sign-in', otp: '123456' }, + }, + { status: 'skipped' } + ); + await db.mailDelivery.update({ + where: { id: row.id }, + data: { + settledAt: new Date(Date.now() - 31 * 24 * 60 * 60 * 1000), + }, + }); + + await mailJob.sendPendingMails(); + + t.is(await db.mailDelivery.count({ where: { id: row.id } }), 0); }); diff --git a/packages/backend/server/src/core/mail/__tests__/mailer.spec.ts b/packages/backend/server/src/core/mail/__tests__/mailer.spec.ts new file mode 100644 index 0000000000..1f54301077 --- /dev/null +++ b/packages/backend/server/src/core/mail/__tests__/mailer.spec.ts @@ -0,0 +1,316 @@ +import { createHash, createHmac } from 'node:crypto'; + +import { PrismaClient } from '@prisma/client'; +import ava, { TestFn } from 'ava'; +import Sinon from 'sinon'; + +import { + createTestingModule, + type TestingModule, +} from '../../../__tests__/utils'; +import { CryptoHelper } from '../../../base'; +import { Models } from '../../../models'; +import { BackendRuntimeProvider } from '../../backend-runtime'; +import { Mailer } from '../mailer'; +import { MailSender } from '../sender'; + +interface Context { + module: TestingModule; + mailer: Mailer; + models: Models; + db: PrismaClient; + crypto: CryptoHelper; + runtime: { + assertMailDeliveryQuotaV1: Sinon.SinonStub; + commitMailDeliveryQuotaV1: Sinon.SinonStub; + releaseMailDeliveryQuotaV1: Sinon.SinonStub; + }; +} + +const test = ava as TestFn; + +test.before(async t => { + t.context.runtime = { + assertMailDeliveryQuotaV1: Sinon.stub(), + commitMailDeliveryQuotaV1: Sinon.stub(), + releaseMailDeliveryQuotaV1: Sinon.stub(), + }; + t.context.module = await createTestingModule({ + tapModule: builder => { + builder + .overrideProvider(Mailer) + .useClass(Mailer) + .overrideProvider(MailSender) + .useValue({ configured: true }) + .overrideProvider(BackendRuntimeProvider) + .useValue(t.context.runtime); + }, + }); + t.context.mailer = t.context.module.get(Mailer); + t.context.models = t.context.module.get(Models); + t.context.db = t.context.module.get(PrismaClient); + t.context.crypto = t.context.module.get(CryptoHelper); +}); + +test.beforeEach(t => { + t.context.runtime.assertMailDeliveryQuotaV1.reset(); + t.context.runtime.commitMailDeliveryQuotaV1.reset(); + t.context.runtime.releaseMailDeliveryQuotaV1.reset(); + t.context.runtime.assertMailDeliveryQuotaV1.resolves({ + allowed: true, + reservationId: '00000000-0000-0000-0000-000000000001', + mailClass: 'auth', + }); + t.context.runtime.commitMailDeliveryQuotaV1.resolves(true); + t.context.runtime.releaseMailDeliveryQuotaV1.resolves(true); +}); + +test.afterEach.always(async t => { + Sinon.restore(); + await t.context.db.mailDelivery.deleteMany(); +}); + +test.after.always(async t => { + await t.context.module.close(); +}); + +test('trySend creates a delivery row and commits quota reservation', async t => { + const sent = await t.context.mailer.trySend({ + name: 'SignIn', + to: 'auth-user@example.com', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + metadata: { + dedupeKey: 'signin:auth-user@example.com:1', + recipientUserId: 'user-1', + source: { trusted: false }, + }, + }); + + const row = await t.context.db.mailDelivery.findFirstOrThrow({ + where: { dedupeKey: 'signin:auth-user@example.com:1' }, + }); + + t.true(sent); + t.is(row.status, 'queued'); + t.is(row.recipientUserId, 'user-1'); + t.is(row.mailClass, 'auth'); + t.is( + row.recipientHash, + createHmac('sha256', t.context.crypto.keyPair.sha256.privateKey) + .update('auth-user@example.com') + .digest('hex') + ); + t.not( + row.recipientHash, + createHash('sha256').update('auth-user@example.com').digest('hex') + ); + t.true( + t.context.runtime.commitMailDeliveryQuotaV1.calledOnceWithExactly( + '00000000-0000-0000-0000-000000000001' + ) + ); +}); + +test('cancelByRecipient matches the keyed recipient hash', async t => { + await t.context.mailer.trySend({ + name: 'SignIn', + to: ' Delete-Me@Example.COM ', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + metadata: { + dedupeKey: 'signin:delete-me@example.com:1', + source: { trusted: false }, + }, + }); + + await t.context.models.mailDelivery.cancelByRecipient( + 'delete-me@example.com' + ); + + const row = await t.context.db.mailDelivery.findFirstOrThrow({ + where: { dedupeKey: 'signin:delete-me@example.com:1' }, + }); + t.is( + row.recipientHash, + createHmac('sha256', t.context.crypto.keyPair.sha256.privateKey) + .update('delete-me@example.com') + .digest('hex') + ); + t.is(row.status, 'canceled'); + t.is(row.lastErrorCode, 'recipient_deleted'); +}); + +test('dedupe replay does not consume another mail quota reservation', async t => { + const command = { + name: 'SignIn' as const, + to: 'dedupe@example.com', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + metadata: { + dedupeKey: 'signin:dedupe@example.com:1', + source: { trusted: false }, + }, + }; + + t.true(await t.context.mailer.trySend(command)); + t.true(await t.context.mailer.trySend(command)); + + t.is(t.context.runtime.assertMailDeliveryQuotaV1.callCount, 1); + t.is(t.context.runtime.commitMailDeliveryQuotaV1.callCount, 1); + t.is( + await t.context.db.mailDelivery.count({ + where: { dedupeKey: command.metadata.dedupeKey }, + }), + 1 + ); +}); + +test('quota denial records skipped deliveries without committing quota', async t => { + for (const quota of [ + { + email: 'limited@example.com', + mailClass: 'auth', + reason: 'recipient_rate_limited', + }, + { + email: 'unmapped@example.com', + mailClass: 'unknown', + reason: 'unmapped_mail_name', + }, + ] as const) { + t.context.runtime.assertMailDeliveryQuotaV1.resolves({ + allowed: false, + mailClass: quota.mailClass, + reason: quota.reason, + }); + + const dedupeKey = `signin:${quota.email}:1`; + const sent = await t.context.mailer.trySend({ + name: 'SignIn', + to: quota.email, + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + metadata: { + dedupeKey, + source: { trusted: false }, + }, + }); + + const row = await t.context.db.mailDelivery.findFirstOrThrow({ + where: { dedupeKey }, + }); + + t.false(sent, quota.reason); + t.is(row.status, 'skipped', quota.reason); + t.is(row.mailClass, quota.mailClass, quota.reason); + t.is(row.lastErrorCode, quota.reason, quota.reason); + t.is(row.recipientEmail, null, quota.reason); + t.false(t.context.runtime.commitMailDeliveryQuotaV1.called, quota.reason); + t.context.runtime.assertMailDeliveryQuotaV1.resetHistory(); + } +}); + +test('skip records terminal delivery without quota admission', async t => { + const sent = await t.context.mailer.skip( + { + name: 'MemberInvitation', + to: 'skip@example.com', + props: { + url: 'https://affine.pro/invite', + user: { $$userId: 'actor-1' }, + workspace: { $$workspaceId: 'workspace-1' }, + }, + metadata: { + dedupeKey: 'invite:skip@example.com:1', + recipientUserId: 'user-1', + actorUserId: 'actor-1', + workspaceId: 'workspace-1', + source: { trusted: false }, + }, + }, + { + mailClass: 'workspace_invitation', + reason: 'workspace_name_contains_domain', + } + ); + + const row = await t.context.db.mailDelivery.findFirstOrThrow({ + where: { dedupeKey: 'invite:skip@example.com:1' }, + }); + + t.false(sent); + t.is(row.status, 'skipped'); + t.is(row.mailClass, 'workspace_invitation'); + t.is(row.lastErrorCode, 'workspace_name_contains_domain'); + t.is(row.maxAttempts, 0); + t.false(t.context.runtime.assertMailDeliveryQuotaV1.called); +}); + +test('send releases quota reservation and rethrows when ledger write fails', async t => { + Sinon.stub(t.context.models.mailDelivery, 'create').rejects( + new Error('ledger failed') + ); + + await t.throwsAsync( + t.context.mailer.send({ + name: 'SignIn', + to: 'throw@example.com', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + metadata: { + source: { trusted: false }, + }, + }), + { message: 'ledger failed' } + ); + + t.true( + t.context.runtime.releaseMailDeliveryQuotaV1.calledOnceWithExactly( + '00000000-0000-0000-0000-000000000001' + ) + ); +}); + +test('send cancels queued delivery when quota commit fails', async t => { + t.context.runtime.commitMailDeliveryQuotaV1.rejects( + new Error('commit failed') + ); + + await t.throwsAsync( + t.context.mailer.send({ + name: 'SignIn', + to: 'commit-failed@example.com', + props: { + url: 'https://affine.pro/sign-in', + otp: '123456', + }, + metadata: { + dedupeKey: 'signin:commit-failed@example.com:1', + source: { trusted: false }, + }, + }), + { message: 'commit failed' } + ); + + const row = await t.context.db.mailDelivery.findFirstOrThrow({ + where: { dedupeKey: 'signin:commit-failed@example.com:1' }, + }); + t.is(row.status, 'canceled'); + t.is(row.lastErrorCode, 'quota_commit_failed'); + t.true( + t.context.runtime.releaseMailDeliveryQuotaV1.calledOnceWithExactly( + '00000000-0000-0000-0000-000000000001' + ) + ); +}); diff --git a/packages/backend/server/src/core/mail/__tests__/resolver.spec.ts b/packages/backend/server/src/core/mail/__tests__/resolver.spec.ts new file mode 100644 index 0000000000..d4abd7aff9 --- /dev/null +++ b/packages/backend/server/src/core/mail/__tests__/resolver.spec.ts @@ -0,0 +1,257 @@ +import { PrismaClient } from '@prisma/client'; +import test from 'ava'; + +import { createApp, type TestingApp } from '../../../__tests__/e2e/test'; +import { Mockers } from '../../../__tests__/mocks'; + +let app: TestingApp; + +function startOfUtcHour(value: Date) { + return new Date( + Date.UTC( + value.getUTCFullYear(), + value.getUTCMonth(), + value.getUTCDate(), + value.getUTCHours() + ) + ); +} + +test.before(async () => { + app = await createApp(); +}); + +test.beforeEach(async () => { + await app.get(PrismaClient).mailDelivery.deleteMany(); +}); + +test.after.always(async () => { + await app.close(); +}); + +test.afterEach.always(async () => { + await app.get(PrismaClient).mailDelivery.deleteMany(); +}); + +test('sendTestEmail rejects non-admin users before SMTP or ledger', async t => { + const user = await app.create(Mockers.User); + + await app.login(user); + + const response = await app.request('post', '/graphql').send({ + query: /* GraphQL */ ` + mutation SendTestEmail($config: JSONObject!) { + sendTestEmail(config: $config) + } + `, + variables: { + config: { + name: '', + host: 'smtp.example.com', + port: 587, + username: 'user', + password: 'password', + ignoreTLS: false, + sender: 'AFFiNE ', + }, + }, + }); + + t.is(response.status, 200); + t.truthy(response.body.errors); + t.is(await app.get(PrismaClient).mailDelivery.count(), 0); +}); + +test('adminMailDeliveries returns timeline series for status type and outcome', async t => { + const db = app.get(PrismaClient); + const admin = await app.create(Mockers.User, { + feature: 'administrator', + }); + const now = new Date(); + const createdAt = new Date(startOfUtcHour(now).getTime() - 60 * 60 * 1000); + const base = { + priority: 'normal', + recipientHash: 'hash', + recipientDomain: 'example.com', + sendAfter: now, + retentionState: 'anonymized', + anonymizedAt: now, + createdAt, + updatedAt: now, + }; + + await db.mailDelivery.createMany({ + data: [ + { + ...base, + mailName: 'SignIn', + mailClass: 'auth', + status: 'sent', + settledAt: now, + sentAt: now, + }, + { + ...base, + mailName: 'MemberInvitation', + mailClass: 'workspace_invitation', + status: 'failed', + settledAt: now, + failedAt: now, + lastErrorCode: 'transport_failed', + }, + { + ...base, + mailName: 'Mention', + mailClass: 'notification', + status: 'queued', + recipientEmail: 'queued@example.com', + payload: {}, + retentionState: 'full', + anonymizedAt: null, + }, + ], + }); + + await app.login(admin); + const response = await app.request('post', '/graphql').send({ + query: /* GraphQL */ ` + query AdminMailDeliveries($input: AdminMailDeliveriesInput) { + adminMailDeliveries(input: $input) { + window { + bucket + effectiveSize + } + summary { + total + sent + failed + queued + successRate + } + byStatus { + key + total + points { + bucket + count + } + } + byType { + key + total + } + byOutcome { + key + total + } + } + } + `, + variables: { + input: { + hours: 24, + }, + }, + }); + + t.is(response.status, 200); + t.falsy(response.body.errors); + const analytics = response.body.data.adminMailDeliveries; + t.is(analytics.window.bucket, 'Hour'); + t.is(analytics.window.effectiveSize, 24); + t.like(analytics.summary, { + total: 3, + sent: 1, + failed: 1, + queued: 1, + successRate: 0.5, + }); + t.true( + analytics.byStatus.some((series: { key: string; total: number }) => { + return series.key === 'sent' && series.total === 1; + }) + ); + const sent = analytics.byStatus.find( + (series: { key: string; points: { bucket: string; count: number }[] }) => + series.key === 'sent' + ); + t.deepEqual( + sent?.points.filter((point: { count: number }) => point.count > 0), + [{ bucket: createdAt.toISOString(), count: 1 }] + ); + t.true( + analytics.byType.some( + (series: { key: string; total: number }) => + series.key === 'workspace_invitation' && series.total === 1 + ) + ); + t.true( + analytics.byOutcome.some( + (series: { key: string; total: number }) => + series.key === 'pending' && series.total === 1 + ) + ); +}); + +test('adminMailDeliveries uses day buckets for seven day window', async t => { + const admin = await app.create(Mockers.User, { + feature: 'administrator', + }); + + await app.login(admin); + const response = await app.request('post', '/graphql').send({ + query: /* GraphQL */ ` + query AdminMailDeliveries($input: AdminMailDeliveriesInput) { + adminMailDeliveries(input: $input) { + window { + bucket + effectiveSize + } + } + } + `, + variables: { + input: { + hours: 168, + }, + }, + }); + + t.is(response.status, 200); + t.falsy(response.body.errors); + t.like(response.body.data.adminMailDeliveries.window, { + bucket: 'Day', + effectiveSize: 7, + }); +}); + +test('adminMailDeliveries rejects unsupported window sizes', async t => { + const admin = await app.create(Mockers.User, { + feature: 'administrator', + }); + + await app.login(admin); + const response = await app.request('post', '/graphql').send({ + query: /* GraphQL */ ` + query AdminMailDeliveries($input: AdminMailDeliveriesInput) { + adminMailDeliveries(input: $input) { + window { + requestedSize + } + } + } + `, + variables: { + input: { + hours: 48, + }, + }, + }); + + t.is(response.status, 200); + t.truthy(response.body.errors); + t.regex( + response.body.errors[0].message, + /Mail delivery analytics window must be 24 or 168 hours/ + ); +}); diff --git a/packages/backend/server/src/core/mail/config.ts b/packages/backend/server/src/core/mail/config.ts index fdac7b4eb1..025c6f25ee 100644 --- a/packages/backend/server/src/core/mail/config.ts +++ b/packages/backend/server/src/core/mail/config.ts @@ -16,6 +16,11 @@ declare global { }; fallbackDomains: ConfigItem; + deliveryWorker: { + batchSize: number; + leaseMs: number; + retentionDays: number; + }; fallbackSMTP: { name: string; host: string; @@ -71,6 +76,21 @@ defineModuleConfig('mailer', { default: [], shape: z.array(z.string()), }, + 'deliveryWorker.batchSize': { + desc: 'Number of mail delivery rows claimed by each worker tick.', + default: 50, + shape: z.number().int().min(1).max(1000), + }, + 'deliveryWorker.leaseMs': { + desc: 'Mail delivery worker lease duration in milliseconds.', + default: 2 * 60 * 1000, + shape: z.number().int().min(1000), + }, + 'deliveryWorker.retentionDays': { + desc: 'Days to retain anonymized terminal mail delivery ledger rows.', + default: 30, + shape: z.number().int().min(1), + }, 'fallbackSMTP.name': { desc: 'Hostname used for fallback SMTP HELO/EHLO (e.g. mail.example.com). Leave empty to use the system hostname.', default: '', diff --git a/packages/backend/server/src/core/mail/index.ts b/packages/backend/server/src/core/mail/index.ts index ffa2429899..7090bbfdae 100644 --- a/packages/backend/server/src/core/mail/index.ts +++ b/packages/backend/server/src/core/mail/index.ts @@ -2,6 +2,7 @@ import './config'; import { Module } from '@nestjs/common'; +import { BackendRuntimeModule } from '../backend-runtime'; import { DocStorageModule } from '../doc'; import { StorageModule } from '../storage'; import { MailJob } from './job'; @@ -10,7 +11,7 @@ import { MailResolver } from './resolver'; import { MailSender } from './sender'; @Module({ - imports: [DocStorageModule, StorageModule], + imports: [BackendRuntimeModule, DocStorageModule, StorageModule], providers: [MailSender, Mailer, MailJob, MailResolver], exports: [Mailer], }) diff --git a/packages/backend/server/src/core/mail/job.ts b/packages/backend/server/src/core/mail/job.ts index 4b21d00f75..b55d65ccbe 100644 --- a/packages/backend/server/src/core/mail/job.ts +++ b/packages/backend/server/src/core/mail/job.ts @@ -1,106 +1,184 @@ +import { randomUUID } from 'node:crypto'; + import { Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { getStreamAsBuffer } from 'get-stream'; -import { Cache, JOB_SIGNAL, JobQueue, OnEvent, OnJob, sleep } from '../../base'; -import { type MailName, MailProps, Renderers } from '../../mails'; +import { Config, metrics, OnEvent } from '../../base'; +import { type MailName, Renderers } from '../../mails'; import { UserProps, WorkspaceProps } from '../../mails/components'; -import { Models } from '../../models'; +import { MailDeliveryRow, Models } from '../../models'; +import { containsUrlOrDomain } from '../content-policy'; import { DocReader } from '../doc/reader'; import { WorkspaceBlobStorage } from '../storage'; -import { containsUrlOrDomain } from '../workspaces/abuse'; import { MailSender, SendOptions } from './sender'; +import { SendMailPayload } from './types'; -type DynamicallyFetchedProps = { - [Key in keyof Props]: Props[Key] extends infer Prop - ? Prop extends UserProps - ? { - $$userId: string; - } & Omit - : Prop extends WorkspaceProps - ? { - $$workspaceId: string; - } & Omit - : Prop - : never; -}; - -type SendMailJob> = { - name: Mail; - to: string; - // NOTE(@forehalo): - // workspace avatar currently send as base64 img instead of a avatar url, - // so the content might be too large to be put in job payload. - props: DynamicallyFetchedProps; -}; - -declare global { - interface Jobs { - 'notification.sendMail': { - startTime: number; - retryCount?: number; - expiresAt?: number; - } & { - [K in MailName]: SendMailJob; - }[MailName]; - } -} - -const sendMailKey = 'mailjob:sendMail'; -const retryMailKey = 'mailjob:sendMail:retry'; -const sendMailCacheKey = (name: string, to: string) => - `${sendMailKey}:${name}:${to}`; -const retryMaxPerTick = 20; -const retryFirstTime = 3; -const retryMaxAttempts = 12; -const retryMaxAge = 24 * 60 * 60 * 1000; -const magicLinkExpiresIn = 30 * 60 * 1000; - -const mailExpiresIn: Partial> = { - SignIn: magicLinkExpiresIn, - SignUp: magicLinkExpiresIn, - SetPassword: magicLinkExpiresIn, - ChangePassword: magicLinkExpiresIn, - VerifyEmail: magicLinkExpiresIn, - ChangeEmail: magicLinkExpiresIn, - VerifyChangeEmail: magicLinkExpiresIn, +type DynamicProp = Record & { + $$workspaceId?: string; + $$userId?: string; }; @Injectable() export class MailJob { - private readonly logger = new Logger('MailJob'); + private readonly logger = new Logger('MailDeliveryWorker'); + private readonly workerId = `mail-delivery-${process.pid}-${randomUUID()}`; constructor( - private readonly cache: Cache, - private readonly queue: JobQueue, private readonly sender: MailSender, private readonly doc: DocReader, private readonly workspaceBlob: WorkspaceBlobStorage, - private readonly models: Models + private readonly models: Models, + private readonly config: Config ) {} - private calculateRetryDelay(startTime: number) { - const elapsed = Date.now() - startTime; - return Math.min(30 * 1000, Math.round(elapsed / 2000) * 1000); + @OnEvent('user.deleted') + async onUserDeleted(user: Events['user.deleted']) { + await Promise.all([ + this.models.mailDelivery.cancelByRecipient(user.email), + this.models.mailDelivery.cancelMemberInvitationByActor(user.id), + ]); } - private getRetryExhaustedReason({ - startTime, - retryCount, - expiresAt, - name, - }: Jobs['notification.sendMail']) { - const expiredAt = - expiresAt ?? startTime + (mailExpiresIn[name] ?? retryMaxAge); - if (Date.now() > expiredAt) { - return 'expired'; + @Cron(CronExpression.EVERY_MINUTE) + async sendPendingMails() { + await this.processReadyDeliveries(); + await this.cleanupRetainedDeliveries(); + await this.recordDeliveryMetrics(); + } + + async processReadyDeliveries( + batchSize = this.config.mailer.deliveryWorker.batchSize + ) { + const rows = await this.models.mailDelivery.claimReady(this.workerId, { + batchSize, + leaseMs: this.config.mailer.deliveryWorker.leaseMs, + }); + + for (const row of rows) { + await this.processDelivery(row); + } + return rows.length; + } + + private async cleanupRetainedDeliveries() { + const retentionMs = + this.config.mailer.deliveryWorker.retentionDays * 24 * 60 * 60 * 1000; + const before = new Date(Date.now() - retentionMs); + await this.models.mailDelivery.deleteAnonymizedBefore( + before, + this.config.mailer.deliveryWorker.batchSize + ); + } + + private async recordDeliveryMetrics() { + const snapshot = await this.models.mailDelivery.metricsSnapshot(); + metrics.mail.gauge('retry_wait_backlog').record(snapshot.retryWait); + metrics.mail.gauge('failed_recent').record(snapshot.failedRecent); + metrics.mail.gauge('expired_lease_backlog').record(snapshot.expiredLeases); + metrics.mail.histogram('ready_delay_ms').record(snapshot.readyDelayMs); + } + + private async processDelivery(row: MailDeliveryRow) { + if (!row.recipientEmail || !row.payload) { + await this.models.mailDelivery.markSkipped( + row.id, + this.workerId, + 'missing_payload' + ); + return; + } + if (row.expiresAt && row.expiresAt.getTime() <= Date.now()) { + await this.models.mailDelivery.markFailed( + row.id, + this.workerId, + 'expired' + ); + return; + } + if (await this.shouldSkipRecipient(row.recipientEmail)) { + await this.models.mailDelivery.markSkipped( + row.id, + this.workerId, + 'disabled_recipient' + ); + return; } - if ((retryCount ?? 0) > retryMaxAttempts) { - return 'max attempts reached'; + const payload = row.payload as SendMailPayload; + const rendered = await this.renderPayload(payload); + if (!rendered) { + await this.models.mailDelivery.markSkipped( + row.id, + this.workerId, + 'dynamic_props_missing' + ); + return; + } + if (row.expiresAt && row.expiresAt.getTime() <= Date.now()) { + await this.models.mailDelivery.markFailed( + row.id, + this.workerId, + 'expired' + ); + return; } - return; + const attempt = await this.models.mailDelivery.markAttemptStarted( + row.id, + this.workerId + ); + if (!attempt) { + return; + } + + const result = await this.sender.send(payload.name, { + to: row.recipientEmail, + ...rendered.options, + ...rendered.content, + }); + + if (result.status === 'accepted') { + await this.models.mailDelivery.markSent(row.id, this.workerId, { + providerMessageId: result.providerMessageId, + providerResponse: result.providerResponse, + }); + return; + } + + const nextSendAfter = this.nextSendAfter(attempt.attemptCount); + const canRetry = + result.retryable && + attempt.attemptCount < attempt.maxAttempts && + (!attempt.expiresAt || nextSendAfter < attempt.expiresAt); + if (canRetry) { + const retry = await this.models.mailDelivery.markRetry( + row.id, + this.workerId, + { + sendAfter: nextSendAfter, + errorCode: result.errorCode, + error: result.error, + } + ); + if (retry) { + return; + } + } + + await this.models.mailDelivery.markFailed( + row.id, + this.workerId, + attempt.expiresAt && nextSendAfter >= attempt.expiresAt + ? 'expired' + : (result.errorCode ?? result.status), + result.error ?? result.providerResponse ?? undefined + ); + } + + private nextSendAfter(attemptCount: number) { + const delayMs = Math.min(30 * 60 * 1000, 2 ** attemptCount * 30 * 1000); + return new Date(Date.now() + delayMs); } private async shouldSkipRecipient(to: string) { @@ -111,72 +189,16 @@ export class MailJob { return user?.disabled === true; } - private async deleteRecipientMailCache(to: string) { - const suffix = `:${to}`; - - await Promise.all( - [sendMailKey, retryMailKey].map(async map => { - const keys = await this.cache.mapKeys(map); - await Promise.all( - keys - .filter(key => key.endsWith(suffix)) - .map(key => this.cache.mapDelete(map, key)) - ); - }) - ); - } - - private isInvitationMailSentByUser( - job: Jobs['notification.sendMail'], - userId: string - ) { - return ( - job.name === 'MemberInvitation' && - 'user' in job.props && - typeof job.props.user === 'object' && - job.props.user !== null && - '$$userId' in job.props.user && - job.props.user.$$userId === userId - ); - } - - private async deleteInvitationMailCacheBySender(userId: string) { - const keys = await this.cache.mapKeys(retryMailKey); - await Promise.all( - keys.map(async key => { - const job = await this.cache.mapGet< - Jobs['notification.sendMail'] | string - >(retryMailKey, key); - const jobData = - typeof job === 'string' - ? (JSON.parse(job) as Jobs['notification.sendMail']) - : job; - if (jobData && this.isInvitationMailSentByUser(jobData, userId)) { - await this.cache.mapDelete(retryMailKey, key); - } - }) - ); - } - - private async sendMailInternal({ - startTime, - name, - to, - props, - }: Jobs['notification.sendMail']) { - if (await this.shouldSkipRecipient(to)) { - this.logger.debug(`Skip mail [${name}] to disabled user [${to}]`); - return; - } - + private async renderPayload(payload: SendMailPayload) { let options: Partial = {}; - const renderedProps = { ...props }; + const renderedProps = { ...payload.props }; for (const key in renderedProps) { - // @ts-expect-error allow - const val = renderedProps[key]; + const val = renderedProps[key as keyof typeof renderedProps] as + | DynamicProp + | undefined; if (val && typeof val === 'object') { - if ('$$workspaceId' in val) { + if (typeof val.$$workspaceId === 'string') { const workspaceProps = await this.fetchWorkspaceProps( val.$$workspaceId ); @@ -186,11 +208,11 @@ export class MailJob { } if ( - name === 'MemberInvitation' && + payload.name === 'MemberInvitation' && containsUrlOrDomain(workspaceProps.name) ) { this.logger.warn( - `Skip mail [${name}] to [${to}], reason=workspace name contains url or domain` + `Skip mail [${payload.name}] to [${payload.to}], reason=workspace name contains url or domain` ); return; } @@ -206,57 +228,42 @@ export class MailJob { ]; workspaceProps.avatar = 'cid:workspaceAvatar'; } - // @ts-expect-error replacement - renderedProps[key] = workspaceProps; - } else if ('$$userId' in val) { + Object.assign(val, workspaceProps); + delete val.$$workspaceId; + } else if (typeof val.$$userId === 'string') { const userProps = await this.fetchUserProps(val.$$userId); if (!userProps) { return; } - // @ts-expect-error replacement - renderedProps[key] = userProps; + Object.assign(val, userProps); + delete val.$$userId; } } } if ( - name === 'MemberInvitation' && + payload.name === 'MemberInvitation' && 'workspace' in renderedProps && containsUrlOrDomain( (renderedProps.workspace as WorkspaceProps | undefined)?.name ) ) { this.logger.warn( - `Skip mail [${name}] to [${to}], reason=workspace name contains url or domain` + `Skip mail [${payload.name}] to [${payload.to}], reason=workspace name contains url or domain` ); return; } - try { - const result = await this.sender.send(name, { - to, - ...(await Renderers[name]( - // @ts-expect-error the job trigger part has been typechecked - renderedProps - )), - ...options, - }); - if (!result) { - // wait for a while before retrying - const retryDelay = this.calculateRetryDelay(startTime); - await sleep(retryDelay); - return JOB_SIGNAL.Retry; - } - return undefined; - } catch (e) { - this.logger.error(`Failed to send mail [${name}] to [${to}]`, e); - // wait for a while before retrying - const retryDelay = this.calculateRetryDelay(startTime); - await sleep(retryDelay); - return JOB_SIGNAL.Retry; - } + return { + options, + content: await ( + Renderers[payload.name] as ( + props: unknown + ) => ReturnType<(typeof Renderers)[MailName]> + )(renderedProps), + }; } private async fetchWorkspaceProps(workspaceId: string) { @@ -294,76 +301,4 @@ export class MailJob { return { email: user.email } satisfies UserProps; } - - @OnJob('notification.sendMail') - async sendMail(job: Jobs['notification.sendMail']) { - const cacheKey = sendMailCacheKey(job.name, job.to); - job.retryCount = (job.retryCount ?? 0) + 1; - const exhaustedReason = this.getRetryExhaustedReason(job); - if (exhaustedReason) { - this.logger.warn( - `Drop mail [${job.name}] to [${job.to}], reason=${exhaustedReason}` - ); - await Promise.all([ - this.cache.mapDelete(sendMailKey, cacheKey), - this.cache.mapDelete(retryMailKey, cacheKey), - ]); - return; - } - - const retried = await this.cache.mapIncrease(sendMailKey, cacheKey, 1); - if (retried <= retryFirstTime) { - const ret = await this.sendMailInternal(job); - if (!ret) await this.cache.mapDelete(sendMailKey, cacheKey); - return ret; - } - await this.cache.mapSet(retryMailKey, cacheKey, job); - await this.cache.mapDelete(sendMailKey, cacheKey); - return undefined; - } - - @OnEvent('user.deleted') - async onUserDeleted(user: Events['user.deleted']) { - await Promise.all([ - this.deleteRecipientMailCache(user.email), - this.deleteInvitationMailCacheBySender(user.id), - this.queue.removeWhere( - 'notification.sendMail', - job => - job.to === user.email || this.isInvitationMailSentByUser(job, user.id) - ), - ]); - } - - @Cron(CronExpression.EVERY_MINUTE) - async sendRetryMails() { - // pick random one from the retry map - let processed = 0; - let key = await this.cache.mapRandomKey(retryMailKey); - while (key && processed < retryMaxPerTick) { - try { - const job = await this.cache.mapGet< - Jobs['notification.sendMail'] | string - >(retryMailKey, key); - if (job) { - const jobData = - typeof job === 'string' - ? (JSON.parse(job) as Jobs['notification.sendMail']) - : job; - await this.queue.add('notification.sendMail', jobData); - // wait for a while before retrying - const retryDelay = this.calculateRetryDelay(jobData.startTime); - await sleep(retryDelay); - } - await this.cache.mapDelete(retryMailKey, key); - } catch (e) { - this.logger.error( - `Failed to re-queue retry mail job for key [${key}]`, - e - ); - } - key = await this.cache.mapRandomKey(retryMailKey); - processed++; - } - } } diff --git a/packages/backend/server/src/core/mail/mailer.ts b/packages/backend/server/src/core/mail/mailer.ts index 4ad492e23c..6853a6aea1 100644 --- a/packages/backend/server/src/core/mail/mailer.ts +++ b/packages/backend/server/src/core/mail/mailer.ts @@ -1,13 +1,50 @@ import { Injectable } from '@nestjs/common'; -import { EmailServiceNotConfigured, JobQueue } from '../../base'; +import { EmailServiceNotConfigured } from '../../base'; +import type { MailName } from '../../mails'; +import { Models } from '../../models'; +import { BackendRuntimeProvider } from '../backend-runtime'; import { MailSender } from './sender'; +import type { SendMailCommand } from './types'; + +type MailCommand = SendMailCommand; +type SkipMailOptions = { + mailClass: string; + reason: string; + priority?: 'critical' | 'high' | 'normal' | 'low'; +}; + +function recipientDomain(email: string) { + const parts = email.trim().toLowerCase().split('@'); + return parts.length === 2 ? parts[1] : ''; +} + +function defaultPriority(mailClass: string): 'critical' | 'high' | 'normal' { + if (mailClass === 'auth') { + return 'critical'; + } + if (mailClass === 'billing_license') { + return 'high'; + } + return 'normal'; +} + +function serializePayload(command: MailCommand) { + return JSON.parse( + JSON.stringify({ + name: command.name, + to: command.to, + props: command.props, + }) + ); +} @Injectable() export class Mailer { constructor( - private readonly queue: JobQueue, - private readonly sender: MailSender + private readonly sender: MailSender, + private readonly models: Models, + private readonly runtime: BackendRuntimeProvider ) {} /** @@ -15,14 +52,43 @@ export class Mailer { * * @note never throw */ - async trySend(command: Omit) { + async trySend(command: MailCommand) { return this.send(command, true); } - async send( - command: Omit, - suppressError = false - ) { + async skip(command: MailCommand, options: SkipMailOptions) { + const metadata = command.metadata ?? {}; + const deduped = metadata.dedupeKey + ? await this.models.mailDelivery.findByDedupeKey(metadata.dedupeKey) + : null; + if (deduped) { + return false; + } + + await this.models.mailDelivery.create({ + mailName: command.name, + mailClass: options.mailClass, + priority: + options.priority ?? + metadata.priority ?? + defaultPriority(options.mailClass), + status: 'skipped', + dedupeKey: metadata.dedupeKey, + recipientEmail: command.to, + recipientUserId: metadata.recipientUserId, + actorUserId: metadata.actorUserId, + workspaceId: metadata.workspaceId, + notificationId: metadata.notificationId, + abuseSubjectKey: metadata.abuseSubjectKey, + payload: serializePayload(command), + expiresAt: metadata.expiresAt, + maxAttempts: 0, + lastErrorCode: options.reason, + }); + return false; + } + + async send(command: MailCommand, suppressError = false) { if (!this.sender.configured) { if (suppressError) { return false; @@ -30,15 +96,95 @@ export class Mailer { throw new EmailServiceNotConfigured(); } + let reservationId: string | undefined; + let deliveryId: string | undefined; try { - await this.queue.add( - 'notification.sendMail', - Object.assign({}, command, { - startTime: Date.now(), - }) as Jobs['notification.sendMail'] - ); + const metadata = command.metadata ?? {}; + const deduped = metadata.dedupeKey + ? await this.models.mailDelivery.findByDedupeKey(metadata.dedupeKey) + : null; + if (deduped) { + return !['failed', 'canceled', 'skipped'].includes(deduped.status); + } + + const decision = await this.runtime.assertMailDeliveryQuotaV1({ + mailName: command.name as MailName, + recipient: { + email: command.to, + domain: recipientDomain(command.to), + userId: metadata.recipientUserId, + }, + metadata: { + actorUserId: metadata.actorUserId, + workspaceId: metadata.workspaceId, + notificationId: metadata.notificationId, + abuseSubjectKey: metadata.abuseSubjectKey, + }, + source: metadata.source, + }); + reservationId = decision.reservationId; + + if (!decision.allowed) { + await this.models.mailDelivery.create({ + mailName: command.name, + mailClass: decision.mailClass, + priority: metadata.priority ?? defaultPriority(decision.mailClass), + status: 'skipped', + dedupeKey: metadata.dedupeKey, + recipientEmail: command.to, + recipientUserId: metadata.recipientUserId, + actorUserId: metadata.actorUserId, + workspaceId: metadata.workspaceId, + notificationId: metadata.notificationId, + abuseSubjectKey: metadata.abuseSubjectKey, + quotaDecision: decision, + payload: serializePayload(command), + expiresAt: metadata.expiresAt, + maxAttempts: metadata.maxAttempts, + lastErrorCode: decision.reason ?? 'quota_denied', + }); + return false; + } + + const delivery = await this.models.mailDelivery.create({ + mailName: command.name, + mailClass: decision.mailClass, + priority: metadata.priority ?? defaultPriority(decision.mailClass), + dedupeKey: metadata.dedupeKey, + recipientEmail: command.to, + recipientUserId: metadata.recipientUserId, + actorUserId: metadata.actorUserId, + workspaceId: metadata.workspaceId, + notificationId: metadata.notificationId, + abuseSubjectKey: metadata.abuseSubjectKey, + quotaReservationId: decision.reservationId, + quotaDecision: decision, + payload: serializePayload(command), + expiresAt: metadata.expiresAt, + maxAttempts: metadata.maxAttempts, + }); + deliveryId = delivery.id; + if (decision.reservationId) { + if (delivery.quotaReservationId === decision.reservationId) { + await this.runtime.commitMailDeliveryQuotaV1(decision.reservationId); + } else { + await this.runtime.releaseMailDeliveryQuotaV1(decision.reservationId); + } + } return true; - } catch { + } catch (error) { + if (deliveryId) { + await this.models.mailDelivery.cancelById( + deliveryId, + 'quota_commit_failed' + ); + } + if (reservationId) { + await this.runtime.releaseMailDeliveryQuotaV1(reservationId); + } + if (!suppressError) { + throw error; + } return false; } } diff --git a/packages/backend/server/src/core/mail/resolver.ts b/packages/backend/server/src/core/mail/resolver.ts index 78fe19147c..220fd438e1 100644 --- a/packages/backend/server/src/core/mail/resolver.ts +++ b/packages/backend/server/src/core/mail/resolver.ts @@ -1,15 +1,212 @@ -import { Args, Mutation, Resolver } from '@nestjs/graphql'; +import { + Args, + Field, + Float, + InputType, + Int, + Mutation, + ObjectType, + Query, + Resolver, +} from '@nestjs/graphql'; import { GraphQLJSONObject } from 'graphql-scalars'; import { BadRequest } from '../../base'; import { Renderers } from '../../mails'; +import { Models } from '../../models'; import { CurrentUser } from '../auth/session'; import { Admin } from '../common'; +import { + TimeBucket, + TimeWindow, +} from '../workspaces/resolvers/analytics-types'; import { MailSender } from './sender'; +@InputType() +class AdminMailDeliveriesInput { + @Field(() => Int, { defaultValue: 24 }) + hours!: number; +} + +@ObjectType() +class AdminMailDeliveryPoint { + @Field(() => Date) + bucket!: Date; + + @Field(() => Int) + count!: number; +} + +@ObjectType() +class AdminMailDeliverySeries { + @Field() + key!: string; + + @Field() + label!: string; + + @Field(() => Int) + total!: number; + + @Field(() => [AdminMailDeliveryPoint]) + points!: AdminMailDeliveryPoint[]; +} + +@ObjectType() +class AdminMailDeliverySummary { + @Field(() => Int) + total!: number; + + @Field(() => Int) + sent!: number; + + @Field(() => Int) + failed!: number; + + @Field(() => Int) + skipped!: number; + + @Field(() => Int) + canceled!: number; + + @Field(() => Int) + queued!: number; + + @Field(() => Int) + sending!: number; + + @Field(() => Int) + retryWait!: number; + + @Field(() => Float) + successRate!: number; +} + +@ObjectType() +class AdminMailDeliveryAnalytics { + @Field(() => TimeWindow) + window!: TimeWindow; + + @Field(() => AdminMailDeliverySummary) + summary!: AdminMailDeliverySummary; + + @Field(() => [AdminMailDeliverySeries]) + byStatus!: AdminMailDeliverySeries[]; + + @Field(() => [AdminMailDeliverySeries]) + byType!: AdminMailDeliverySeries[]; + + @Field(() => [AdminMailDeliverySeries]) + byOutcome!: AdminMailDeliverySeries[]; +} + +const MAIL_STATUSES = [ + 'sent', + 'failed', + 'skipped', + 'canceled', + 'queued', + 'sending', + 'retry_wait', +] as const; + +const STATUS_LABELS = { + sent: 'Sent', + failed: 'Failed', + skipped: 'Skipped', + canceled: 'Canceled', + queued: 'Queued', + sending: 'Sending', + retry_wait: 'Retry wait', +} satisfies Record<(typeof MAIL_STATUSES)[number], string>; + +function startOfUtcHour(value: Date) { + return new Date( + Date.UTC( + value.getUTCFullYear(), + value.getUTCMonth(), + value.getUTCDate(), + value.getUTCHours() + ) + ); +} + +function startOfUtcDay(value: Date) { + return new Date( + Date.UTC(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate()) + ); +} + +function addUtcHours(value: Date, hours: number) { + return new Date(value.getTime() + hours * 60 * 60 * 1000); +} + +function addUtcDays(value: Date, days: number) { + return new Date(value.getTime() + days * 24 * 60 * 60 * 1000); +} + +function normalizeMailWindow(hours: number | undefined) { + if (hours !== undefined && hours !== 24 && hours !== 24 * 7) { + throw new BadRequest( + 'Mail delivery analytics window must be 24 or 168 hours.' + ); + } + const requestedHours = hours ?? 24; + const bucket = requestedHours > 24 ? ('day' as const) : ('hour' as const); + const now = new Date(); + const to = + bucket === 'hour' + ? addUtcHours(startOfUtcHour(now), 1) + : addUtcDays(startOfUtcDay(now), 1); + const effectiveSize = + bucket === 'hour' ? requestedHours : requestedHours / 24; + const from = + bucket === 'hour' + ? addUtcHours(to, -effectiveSize) + : addUtcDays(to, -effectiveSize); + + return { + from, + to, + bucket, + requestedHours, + effectiveSize, + }; +} + +function buildBuckets(input: ReturnType) { + return Array.from({ length: input.effectiveSize }, (_, index) => + input.bucket === 'hour' + ? addUtcHours(input.from, index) + : addUtcDays(input.from, index) + ); +} + +function bucketKey(value: Date) { + return value.toISOString(); +} + +function buildSeries( + keys: { key: string; label: string }[], + buckets: Date[], + counts: Map +): AdminMailDeliverySeries[] { + return keys.map(({ key, label }) => { + let total = 0; + const points = buckets.map(bucket => { + const count = counts.get(`${key}:${bucketKey(bucket)}`) ?? 0; + total += count; + return { bucket, count }; + }); + return { key, label, total, points }; + }); +} + @Admin() @Resolver(() => Boolean) export class MailResolver { + constructor(private readonly models: Models) {} + @Mutation(() => Boolean) async sendTestEmail( @CurrentUser() user: CurrentUser, @@ -46,4 +243,111 @@ export class MailResolver { return true; } + + @Query(() => AdminMailDeliveryAnalytics, { + description: 'Aggregate mail delivery timeline facts for admin panel', + }) + async adminMailDeliveries( + @Args('input', { nullable: true, type: () => AdminMailDeliveriesInput }) + input?: AdminMailDeliveriesInput + ) { + const window = normalizeMailWindow(input?.hours); + const buckets = buildBuckets(window); + const rows = await this.models.mailDelivery.adminAggregate({ + from: window.from, + to: window.to, + bucket: window.bucket, + }); + + const statusCounts = new Map(); + const classCounts = new Map(); + const outcomeCounts = new Map(); + const classTotals = new Map(); + const summary = { + total: 0, + sent: 0, + failed: 0, + skipped: 0, + canceled: 0, + queued: 0, + sending: 0, + retryWait: 0, + successRate: 0, + }; + + for (const row of rows) { + const bucket = bucketKey(row.bucket); + summary.total += row.count; + if (row.status === 'retry_wait') { + summary.retryWait += row.count; + } else { + summary[row.status] += row.count; + } + + statusCounts.set( + `${row.status}:${bucket}`, + (statusCounts.get(`${row.status}:${bucket}`) ?? 0) + row.count + ); + classCounts.set( + `${row.mailClass}:${bucket}`, + (classCounts.get(`${row.mailClass}:${bucket}`) ?? 0) + row.count + ); + classTotals.set( + row.mailClass, + (classTotals.get(row.mailClass) ?? 0) + row.count + ); + + const outcome = + row.status === 'sent' + ? 'successful' + : row.status === 'queued' || + row.status === 'sending' || + row.status === 'retry_wait' + ? 'pending' + : 'unsuccessful'; + outcomeCounts.set( + `${outcome}:${bucket}`, + (outcomeCounts.get(`${outcome}:${bucket}`) ?? 0) + row.count + ); + } + + const terminal = + summary.sent + summary.failed + summary.skipped + summary.canceled; + summary.successRate = terminal ? summary.sent / terminal : 0; + + const topClasses = [...classTotals.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, 6) + .map(([key]) => ({ key, label: key })); + + return { + window: { + from: window.from, + to: window.to, + timezone: 'UTC', + bucket: window.bucket === 'hour' ? TimeBucket.Hour : TimeBucket.Day, + requestedSize: window.requestedHours, + effectiveSize: window.effectiveSize, + }, + summary, + byStatus: buildSeries( + MAIL_STATUSES.map(status => ({ + key: status, + label: STATUS_LABELS[status], + })), + buckets, + statusCounts + ), + byType: buildSeries(topClasses, buckets, classCounts), + byOutcome: buildSeries( + [ + { key: 'successful', label: 'Successful' }, + { key: 'unsuccessful', label: 'Unsuccessful' }, + { key: 'pending', label: 'Pending' }, + ], + buckets, + outcomeCounts + ), + }; + } } diff --git a/packages/backend/server/src/core/mail/sender.ts b/packages/backend/server/src/core/mail/sender.ts index aadcaa976d..79ba8cf5da 100644 --- a/packages/backend/server/src/core/mail/sender.ts +++ b/packages/backend/server/src/core/mail/sender.ts @@ -17,8 +17,24 @@ export type SendOptions = Omit & { html: string; }; +export type MailSendResult = + | { + status: 'accepted'; + providerMessageId?: string; + providerResponse?: string; + retryable: false; + } + | { + status: 'rejected' | 'failed'; + providerResponse?: string; + retryable: boolean; + errorCode?: string; + error?: string; + }; + function configToSMTPOptions( - config: AppConfig['mailer']['SMTP'] + config: AppConfig['mailer']['SMTP'], + timeoutMs = 30 * 1000 ): SMTPTransport.Options { const name = resolveSMTPHeloHostname(config.name); @@ -33,6 +49,9 @@ function configToSMTPOptions( user: config.username, pass: config.password, }, + connectionTimeout: timeoutMs, + greetingTimeout: timeoutMs, + socketTimeout: timeoutMs, }; } @@ -68,7 +87,11 @@ export class MailSender { private setup() { const { SMTP, fallbackDomains, fallbackSMTP } = this.config.mailer; - const opts = configToSMTPOptions(SMTP); + const timeoutMs = Math.max( + 1000, + Math.min(30 * 1000, this.config.mailer.deliveryWorker.leaseMs - 1000) + ); + const opts = configToSMTPOptions(SMTP, timeoutMs); if (SMTP.host) { this.smtp = createTransport(opts); @@ -76,7 +99,9 @@ export class MailSender { this.logger.warn( `Fallback SMTP is configured for domains: ${fallbackDomains.join(', ')}` ); - this.fallbackSMTP = createTransport(configToSMTPOptions(fallbackSMTP)); + this.fallbackSMTP = createTransport( + configToSMTPOptions(fallbackSMTP, timeoutMs) + ); } } else if (env.dev) { createTestAccount((err, account) => { @@ -107,17 +132,25 @@ export class MailSender { return [this.smtp, SMTP.sender] as const; } - async send(name: string, options: SendOptions) { + async send(name: string, options: SendOptions): Promise { const [, domain, ...rest] = options.to.split('@'); if (rest.length || !domain) { this.logger.error(`Invalid email address: ${options.to}`); - return null; + return { + status: 'rejected', + retryable: false, + errorCode: 'invalid_recipient', + }; } const [smtpClient, from] = this.getSender(domain); if (!smtpClient) { this.logger.warn(`Mailer SMTP transport is not configured to send mail.`); - return null; + return { + status: 'failed', + retryable: true, + errorCode: 'smtp_not_configured', + }; } metrics.mail.counter('send_total').add(1, { name }); @@ -129,7 +162,12 @@ export class MailSender { this.logger.error( `Mail [${name}] rejected with response: ${result.response}` ); - return false; + return { + status: 'rejected', + providerResponse: result.response, + retryable: false, + errorCode: 'provider_rejected', + }; } metrics.mail.counter('accepted_total').add(1, { name }); @@ -140,11 +178,22 @@ export class MailSender { ); } - return true; + return { + status: 'accepted', + providerMessageId: + typeof result.messageId === 'string' ? result.messageId : undefined, + providerResponse: result.response, + retryable: false, + }; } catch (e) { metrics.mail.counter('failed_total').add(1, { name }); this.logger.error(`Failed to send mail [${name}].`, e); - return false; + return { + status: 'failed', + retryable: true, + errorCode: 'transport_failed', + error: e instanceof Error ? e.message : String(e), + }; } } } diff --git a/packages/backend/server/src/core/mail/types.ts b/packages/backend/server/src/core/mail/types.ts new file mode 100644 index 0000000000..0db20bfe47 --- /dev/null +++ b/packages/backend/server/src/core/mail/types.ts @@ -0,0 +1,45 @@ +import type { MailName, MailProps } from '../../mails'; +import type { UserProps, WorkspaceProps } from '../../mails/components'; +import type { RuntimeQuotaSourceInput } from '../backend-runtime'; + +export type MailDeliveryMetadata = { + dedupeKey?: string; + recipientUserId?: string; + actorUserId?: string; + workspaceId?: string; + notificationId?: string; + abuseSubjectKey?: string; + source?: RuntimeQuotaSourceInput; + expiresAt?: Date; + maxAttempts?: number; + priority?: 'critical' | 'high' | 'normal' | 'low'; +}; + +export type DynamicallyFetchedProps = { + [Key in keyof Props]: Props[Key] extends infer Prop + ? Prop extends UserProps + ? { + $$userId: string; + } & Omit + : Prop extends WorkspaceProps + ? { + $$workspaceId: string; + } & Omit + : Prop + : never; +}; + +export type SendMailPayload< + Mail extends MailName = MailName, + Props = MailProps, +> = { + name: Mail; + to: string; + props: DynamicallyFetchedProps; +}; + +export type SendMailCommand = { + [K in MailName]: SendMailPayload; +}[MailName] & { + metadata?: MailDeliveryMetadata; +}; diff --git a/packages/backend/server/src/core/notification/__tests__/service.spec.ts b/packages/backend/server/src/core/notification/__tests__/service.spec.ts index ba82af220b..bbfe23dbd7 100644 --- a/packages/backend/server/src/core/notification/__tests__/service.spec.ts +++ b/packages/backend/server/src/core/notification/__tests__/service.spec.ts @@ -4,7 +4,7 @@ import { mock } from 'node:test'; import test from 'ava'; import { createModule } from '../../../__tests__/create-module'; -import { Mockers } from '../../../__tests__/mocks'; +import { Mockers, MockMailer } from '../../../__tests__/mocks'; import { Due, NotificationNotFound } from '../../../base'; import { DocMode, @@ -18,6 +18,7 @@ import { import { DocStorageModule } from '../../doc'; import { FeatureModule } from '../../features'; import { MailModule } from '../../mail'; +import { Mailer } from '../../mail/mailer'; import { PermissionModule } from '../../permission'; import { StorageModule } from '../../storage'; import { NotificationModule } from '../index'; @@ -33,9 +34,13 @@ const module = await createModule({ NotificationModule, ], providers: [NotificationService], + tapModule: builder => { + builder.overrideProvider(Mailer).useValue(new MockMailer()); + }, }); const notificationService = module.get(NotificationService); const models = module.get(Models); +const mailer = module.get(Mailer) as unknown as MockMailer; let owner: User; let member: User; @@ -82,9 +87,9 @@ test('should create invitation notification and email', async t => { t.is(notification!.body.inviteId, inviteId); // should send invitation email - const invitationMail = module.queue.last('notification.sendMail'); - t.is(invitationMail.payload.to, member.email); - t.is(invitationMail.payload.name, 'MemberInvitation'); + const invitationMail = mailer.last('MemberInvitation'); + t.is(invitationMail.to, member.email); + t.is(invitationMail.name, 'MemberInvitation'); }); test('should not send invitation email when workspace name contains domain', async t => { @@ -92,10 +97,11 @@ test('should not send invitation email when workspace name contains domain', asy owner: { id: owner.id, }, - name: 'BTC https://spam.example', + name: 'BTC example.com', }); const inviteId = randomUUID(); - const invitationMailCount = module.queue.count('notification.sendMail'); + const invitationMailCount = mailer.send.callCount; + const skippedMailCount = mailer.skip.callCount; const notification = await notificationService.createInvitation({ userId: member.id, @@ -107,7 +113,9 @@ test('should not send invitation email when workspace name contains domain', asy }); t.truthy(notification); - t.is(module.queue.count('notification.sendMail'), invitationMailCount); + t.is(mailer.send.callCount, invitationMailCount); + t.is(mailer.skip.callCount, skippedMailCount + 1); + t.is(mailer.skip.lastCall.args[1].reason, 'workspace_name_contains_domain'); }); test('should not send invitation email if user setting is not to receive invitation email', async t => { @@ -116,7 +124,8 @@ test('should not send invitation email if user setting is not to receive invitat userId: member.id, receiveInvitationEmail: false, }); - const invitationMailCount = module.queue.count('notification.sendMail'); + const invitationMailCount = mailer.send.callCount; + const skippedMailCount = mailer.skip.callCount; const notification = await notificationService.createInvitation({ userId: member.id, @@ -130,7 +139,9 @@ test('should not send invitation email if user setting is not to receive invitat t.truthy(notification); // no new invitation email should be sent - t.is(module.queue.count('notification.sendMail'), invitationMailCount); + t.is(mailer.send.callCount, invitationMailCount); + t.is(mailer.skip.callCount, skippedMailCount + 1); + t.is(mailer.skip.lastCall.args[1].reason, 'recipient_notification_disabled'); }); test('should not create invitation notification if user is already a member', async t => { @@ -174,9 +185,9 @@ test('should create invitation accepted notification and email', async t => { t.is(notification!.body.inviteId, inviteId); // should send email - const invitationAcceptedMail = module.queue.last('notification.sendMail'); - t.is(invitationAcceptedMail.payload.to, owner.email); - t.is(invitationAcceptedMail.payload.name, 'MemberAccepted'); + const invitationAcceptedMail = mailer.last('MemberAccepted'); + t.is(invitationAcceptedMail.to, owner.email); + t.is(invitationAcceptedMail.name, 'MemberAccepted'); }); test('should not send invitation accepted email if user settings is not receive invitation email', async t => { @@ -189,9 +200,7 @@ test('should not send invitation accepted email if user settings is not receive userId: owner.id, receiveInvitationEmail: false, }); - const invitationAcceptedMailCount = module.queue.count( - 'notification.sendMail' - ); + const invitationAcceptedMailCount = mailer.send.callCount; const notification = await notificationService.createInvitationAccepted({ userId: owner.id, @@ -205,10 +214,7 @@ test('should not send invitation accepted email if user settings is not receive t.truthy(notification); // no new invitation accepted email should be sent - t.is( - module.queue.count('notification.sendMail'), - invitationAcceptedMailCount - ); + t.is(mailer.send.callCount, invitationAcceptedMailCount); }); test('should not create invitation accepted notification if user is not an active member', async t => { @@ -286,11 +292,11 @@ test('should create invitation review request notification if user is not an act t.is(notification!.body.inviteId, inviteId); // should send email - const invitationReviewRequestMail = module.queue.last( - 'notification.sendMail' + const invitationReviewRequestMail = mailer.last( + 'LinkInvitationReviewRequest' ); - t.is(invitationReviewRequestMail.payload.to, owner.email); - t.is(invitationReviewRequestMail.payload.name, 'LinkInvitationReviewRequest'); + t.is(invitationReviewRequestMail.to, owner.email); + t.is(invitationReviewRequestMail.name, 'LinkInvitationReviewRequest'); }); test('should not create invitation review request notification if user is an active member', async t => { @@ -336,11 +342,9 @@ test('should create invitation review approved notification if user is an active t.is(notification!.body.inviteId, inviteId); // should send email - const invitationReviewApprovedMail = module.queue.last( - 'notification.sendMail' - ); - t.is(invitationReviewApprovedMail.payload.to, member.email); - t.is(invitationReviewApprovedMail.payload.name, 'LinkInvitationApprove'); + const invitationReviewApprovedMail = mailer.last('LinkInvitationApprove'); + t.is(invitationReviewApprovedMail.to, member.email); + t.is(invitationReviewApprovedMail.name, 'LinkInvitationApprove'); }); test('should not create invitation review approved notification if user is not an active member', async t => { @@ -382,11 +386,9 @@ test('should create invitation review declined notification if user is not an ac t.is(notification!.body.createdByUserId, owner.id); // should send email - const invitationReviewDeclinedMail = module.queue.last( - 'notification.sendMail' - ); - t.is(invitationReviewDeclinedMail.payload.to, member.email); - t.is(invitationReviewDeclinedMail.payload.name, 'LinkInvitationDecline'); + const invitationReviewDeclinedMail = mailer.last('LinkInvitationDecline'); + t.is(invitationReviewDeclinedMail.to, member.email); + t.is(invitationReviewDeclinedMail.name, 'LinkInvitationDecline'); }); test('should not create invitation review declined notification if user is an active member', async t => { @@ -618,12 +620,12 @@ test('should send mention email by user setting', async t => { t.truthy(notification); // should send mention email - const mentionMail = module.queue.last('notification.sendMail'); - t.is(mentionMail.payload.to, member.email); - t.is(mentionMail.payload.name, 'Mention'); + const mentionMail = mailer.last('Mention'); + t.is(mentionMail.to, member.email); + t.is(mentionMail.name, 'Mention'); // update user setting to not receive mention email - const mentionMailCount = module.queue.count('notification.sendMail'); + const mentionMailCount = mailer.send.callCount; await module.create(Mockers.UserSettings, { userId: member.id, receiveMentionEmail: false, @@ -644,7 +646,7 @@ test('should send mention email by user setting', async t => { }); // should not send mention email - t.is(module.queue.count('notification.sendMail'), mentionMailCount); + t.is(mailer.send.callCount, mentionMailCount); }); test('should send mention email with use client doc title if server doc title is empty', async t => { @@ -672,11 +674,10 @@ test('should send mention email with use client doc title if server doc title is t.truthy(notification); - const mentionMail = module.queue.last('notification.sendMail'); - t.is(mentionMail.payload.to, member.email); - t.is(mentionMail.payload.name, 'Mention'); - // @ts-expect-error - payload is not typed - t.is(mentionMail.payload.props.doc.title, 'doc-title-1'); + const mentionMail = mailer.last('Mention'); + t.is(mentionMail.to, member.email); + t.is(mentionMail.name, 'Mention'); + t.is(mentionMail.props.doc.title, 'doc-title-1'); }); test('should send comment notification and email', async t => { @@ -699,9 +700,9 @@ test('should send comment notification and email', async t => { t.truthy(notification); - const commentMail = module.queue.last('notification.sendMail'); - t.is(commentMail.payload.to, member.email); - t.is(commentMail.payload.name, 'Comment'); + const commentMail = mailer.last('Comment'); + t.is(commentMail.to, member.email); + t.is(commentMail.name, 'Comment'); }); test('should send comment mention notification and email', async t => { @@ -729,9 +730,9 @@ test('should send comment mention notification and email', async t => { t.truthy(notification); - const commentMentionMail = module.queue.last('notification.sendMail'); - t.is(commentMentionMail.payload.to, member.email); - t.is(commentMentionMail.payload.name, 'CommentMention'); + const commentMentionMail = mailer.last('CommentMention'); + t.is(commentMentionMail.to, member.email); + t.is(commentMentionMail.name, 'CommentMention'); }); test('should send comment email by user setting', async t => { @@ -753,12 +754,12 @@ test('should send comment email by user setting', async t => { t.truthy(notification); - const commentMail = module.queue.last('notification.sendMail'); - t.is(commentMail.payload.to, member.email); - t.is(commentMail.payload.name, 'Comment'); + const commentMail = mailer.last('Comment'); + t.is(commentMail.to, member.email); + t.is(commentMail.name, 'Comment'); // update user setting to not receive comment email - const commentMailCount = module.queue.count('notification.sendMail'); + const commentMailCount = mailer.send.callCount; await module.create(Mockers.UserSettings, { userId: member.id, receiveCommentEmail: false, @@ -779,5 +780,5 @@ test('should send comment email by user setting', async t => { }); // should not send comment email - t.is(module.queue.count('notification.sendMail'), commentMailCount); + t.is(mailer.send.callCount, commentMailCount); }); diff --git a/packages/backend/server/src/core/notification/service.ts b/packages/backend/server/src/core/notification/service.ts index 65e2b01715..92d029c2de 100644 --- a/packages/backend/server/src/core/notification/service.ts +++ b/packages/backend/server/src/core/notification/service.ts @@ -15,15 +15,17 @@ import { UnionNotificationBody, Workspace, } from '../../models'; +import { BackendRuntimeProvider } from '../backend-runtime'; +import { containsUrlOrDomain } from '../content-policy'; import { DocReader } from '../doc'; import { Mailer } from '../mail'; +import type { SendMailCommand } from '../mail/types'; import { realtimeNotificationRoom, RealtimePublisher } from '../realtime'; import { generateDocPath } from '../utils/doc'; import { generateWorkspaceSettingsPath, WorkspaceSettingsTab, } from '../utils/workspace'; -import { containsUrlOrDomain } from '../workspaces/abuse'; @Injectable() export class NotificationService { @@ -34,7 +36,8 @@ export class NotificationService { private readonly docReader: DocReader, private readonly mailer: Mailer, private readonly url: URLHelper, - private readonly realtime: RealtimePublisher + private readonly realtime: RealtimePublisher, + private readonly runtime: BackendRuntimeProvider ) {} async cleanExpiredNotifications() { @@ -55,23 +58,21 @@ export class NotificationService { const notification = isMention ? await this.models.notification.createCommentMention(input) : await this.models.notification.createComment(input); - await this.sendCommentEmail(input, isMention); + await this.sendCommentEmail(input, isMention, notification.id); await this.publishCountChanged(input.userId, 'created'); return notification; } private async sendCommentEmail( input: CommentNotificationCreate, - isMention?: boolean + isMention?: boolean, + notificationId?: string ) { - const userSetting = await this.models.userSettings.get(input.userId); - if (!userSetting.receiveCommentEmail) { - return; - } const receiver = await this.models.user.getWorkspaceUser(input.userId); if (!receiver) { return; } + const userSetting = await this.models.userSettings.get(input.userId); const doc = await this.models.doc.getMeta( input.body.workspaceId, input.body.doc.id @@ -88,7 +89,7 @@ export class NotificationService { replyId: input.body.replyId, }) ); - await this.mailer.trySend({ + const command: SendMailCommand = { name: isMention ? 'CommentMention' : 'Comment', to: receiver.email, props: { @@ -100,26 +101,44 @@ export class NotificationService { url, }, }, - }); + metadata: { + dedupeKey: notificationId + ? `notification:${notificationId}:mail:${isMention ? 'CommentMention' : 'Comment'}` + : undefined, + recipientUserId: receiver.id, + actorUserId: input.body.createdByUserId, + workspaceId: input.body.workspaceId, + notificationId, + source: { trusted: false }, + }, + }; + if (!userSetting.receiveCommentEmail) { + await this.mailer.skip(command, { + mailClass: 'collaboration_notice', + reason: 'recipient_notification_disabled', + }); + return; + } + await this.trySendMail(command, 'collaboration_notice'); this.logger.debug(`Comment email sent to user ${receiver.id}`); } async createMention(input: MentionNotificationCreate) { const notification = await this.models.notification.createMention(input); - await this.sendMentionEmail(input); + await this.sendMentionEmail(input, notification.id); await this.publishCountChanged(input.userId, 'created'); return notification; } - private async sendMentionEmail(input: MentionNotificationCreate) { - const userSetting = await this.models.userSettings.get(input.userId); - if (!userSetting.receiveMentionEmail) { - return; - } + private async sendMentionEmail( + input: MentionNotificationCreate, + notificationId?: string + ) { const receiver = await this.models.user.getWorkspaceUser(input.userId); if (!receiver) { return; } + const userSetting = await this.models.userSettings.get(input.userId); const doc = await this.models.doc.getMeta( input.body.workspaceId, input.body.doc.id @@ -134,7 +153,7 @@ export class NotificationService { elementId: input.body.doc.elementId, }) ); - await this.mailer.trySend({ + const command: SendMailCommand = { name: 'Mention', to: receiver.email, props: { @@ -146,7 +165,25 @@ export class NotificationService { url, }, }, - }); + metadata: { + dedupeKey: notificationId + ? `notification:${notificationId}:mail:Mention` + : undefined, + recipientUserId: receiver.id, + actorUserId: input.body.createdByUserId, + workspaceId: input.body.workspaceId, + notificationId, + source: { trusted: false }, + }, + }; + if (!userSetting.receiveMentionEmail) { + await this.mailer.skip(command, { + mailClass: 'collaboration_notice', + reason: 'recipient_notification_disabled', + }); + return; + } + await this.trySendMail(command, 'collaboration_notice'); this.logger.debug(`Mention email sent to user ${receiver.id}`); } @@ -161,36 +198,25 @@ export class NotificationService { input, NotificationType.Invitation ); - await this.sendInvitationEmail(input); + await this.sendInvitationEmail(input, notification.id); await this.publishCountChanged(input.userId, 'created'); return notification; } - private async sendInvitationEmail(input: InvitationNotificationCreate) { - const workspace = await this.docReader.getWorkspaceContent( - input.body.workspaceId - ); - if (containsUrlOrDomain(workspace?.name)) { - this.logger.warn( - `Skip invitation email for workspace ${input.body.workspaceId}, reason=workspace name contains url or domain` - ); - return; - } - + private async sendInvitationEmail( + input: InvitationNotificationCreate, + notificationId?: string + ) { const inviteUrl = this.url.link(`/invite/${input.body.inviteId}`); if (env.dev) { // make it easier to test in dev mode this.logger.debug(`Invite link: ${inviteUrl}`); } - const userSetting = await this.models.userSettings.get(input.userId); - if (!userSetting.receiveInvitationEmail) { - return; - } const receiver = await this.models.user.getWorkspaceUser(input.userId); if (!receiver) { return; } - await this.mailer.trySend({ + const command: SendMailCommand = { name: 'MemberInvitation', to: receiver.email, props: { @@ -202,7 +228,40 @@ export class NotificationService { }, url: inviteUrl, }, - }); + metadata: { + dedupeKey: notificationId + ? `notification:${notificationId}:mail:MemberInvitation` + : `invite:${input.body.inviteId}:mail:MemberInvitation`, + recipientUserId: receiver.id, + actorUserId: input.body.createdByUserId, + workspaceId: input.body.workspaceId, + notificationId, + source: { trusted: false }, + }, + }; + const workspace = await this.docReader.getWorkspaceContent( + input.body.workspaceId + ); + if (containsUrlOrDomain(workspace?.name)) { + this.logger.warn( + `Skip invitation email for workspace ${input.body.workspaceId}, reason=workspace name contains url or domain` + ); + await this.mailer.skip(command, { + mailClass: 'workspace_invitation', + reason: 'workspace_name_contains_domain', + }); + return; + } + + const userSetting = await this.models.userSettings.get(input.userId); + if (!userSetting.receiveInvitationEmail) { + await this.mailer.skip(command, { + mailClass: 'workspace_invitation', + reason: 'recipient_notification_disabled', + }); + return; + } + await this.trySendMail(command, 'workspace_invitation'); this.logger.debug( `Invitation email sent to user ${receiver.id} for workspace ${input.body.workspaceId}` ); @@ -223,26 +282,23 @@ export class NotificationService { input, NotificationType.InvitationAccepted ); - await this.sendInvitationAcceptedEmail(input); + await this.sendInvitationAcceptedEmail(input, notification.id); await this.publishCountChanged(input.userId, 'created'); return notification; } private async sendInvitationAcceptedEmail( - input: InvitationNotificationCreate + input: InvitationNotificationCreate, + notificationId?: string ) { const inviterUserId = input.userId; const inviteeUserId = input.body.createdByUserId; const workspaceId = input.body.workspaceId; - const userSetting = await this.models.userSettings.get(inviterUserId); - if (!userSetting.receiveInvitationEmail) { - return; - } const inviter = await this.models.user.getWorkspaceUser(inviterUserId); if (!inviter) { return; } - await this.mailer.trySend({ + const command: SendMailCommand = { name: 'MemberAccepted', to: inviter.email, props: { @@ -259,7 +315,26 @@ export class NotificationService { }) ), }, - }); + metadata: { + dedupeKey: notificationId + ? `notification:${notificationId}:mail:MemberAccepted` + : undefined, + recipientUserId: inviter.id, + actorUserId: inviteeUserId, + workspaceId, + notificationId, + source: { trusted: false }, + }, + }; + const userSetting = await this.models.userSettings.get(inviterUserId); + if (!userSetting.receiveInvitationEmail) { + await this.mailer.skip(command, { + mailClass: 'workspace_lifecycle', + reason: 'recipient_notification_disabled', + }); + return; + } + await this.trySendMail(command, 'workspace_lifecycle'); this.logger.debug( `Invitation accepted email sent to user ${inviter.id} for workspace ${workspaceId}` ); @@ -297,13 +372,14 @@ export class NotificationService { input, NotificationType.InvitationReviewRequest ); - await this.sendInvitationReviewRequestEmail(input); + await this.sendInvitationReviewRequestEmail(input, notification.id); await this.publishCountChanged(input.userId, 'created'); return notification; } private async sendInvitationReviewRequestEmail( - input: InvitationNotificationCreate + input: InvitationNotificationCreate, + notificationId?: string ) { const inviteeUserId = input.body.createdByUserId; const reviewerUserId = input.userId; @@ -312,24 +388,37 @@ export class NotificationService { if (!reviewer) { return; } - await this.mailer.trySend({ - name: 'LinkInvitationReviewRequest', - to: reviewer.email, - props: { - user: { - $$userId: inviteeUserId, + await this.trySendMail( + { + name: 'LinkInvitationReviewRequest', + to: reviewer.email, + props: { + user: { + $$userId: inviteeUserId, + }, + workspace: { + $$workspaceId: workspaceId, + }, + url: this.url.link( + generateWorkspaceSettingsPath({ + workspaceId, + tab: WorkspaceSettingsTab.members, + }) + ), }, - workspace: { - $$workspaceId: workspaceId, + metadata: { + dedupeKey: notificationId + ? `notification:${notificationId}:mail:LinkInvitationReviewRequest` + : undefined, + recipientUserId: reviewer.id, + actorUserId: inviteeUserId, + workspaceId, + notificationId, + source: { trusted: false }, }, - url: this.url.link( - generateWorkspaceSettingsPath({ - workspaceId, - tab: WorkspaceSettingsTab.members, - }) - ), }, - }); + 'workspace_invitation' + ); this.logger.debug( `Invitation review request email sent to user ${reviewer.id} for workspace ${workspaceId}` ); @@ -346,13 +435,14 @@ export class NotificationService { input, NotificationType.InvitationReviewApproved ); - await this.sendInvitationReviewApprovedEmail(input); + await this.sendInvitationReviewApprovedEmail(input, notification.id); await this.publishCountChanged(input.userId, 'created'); return notification; } private async sendInvitationReviewApprovedEmail( - input: InvitationNotificationCreate + input: InvitationNotificationCreate, + notificationId?: string ) { const workspaceId = input.body.workspaceId; const receiverUserId = input.userId; @@ -360,16 +450,29 @@ export class NotificationService { if (!receiver) { return; } - await this.mailer.trySend({ - name: 'LinkInvitationApprove', - to: receiver.email, - props: { - workspace: { - $$workspaceId: workspaceId, + await this.trySendMail( + { + name: 'LinkInvitationApprove', + to: receiver.email, + props: { + workspace: { + $$workspaceId: workspaceId, + }, + url: this.url.link(`/workspace/${workspaceId}`), + }, + metadata: { + dedupeKey: notificationId + ? `notification:${notificationId}:mail:LinkInvitationApprove` + : undefined, + recipientUserId: receiver.id, + actorUserId: input.body.createdByUserId, + workspaceId, + notificationId, + source: { trusted: false }, }, - url: this.url.link(`/workspace/${workspaceId}`), }, - }); + 'workspace_invitation' + ); this.logger.debug( `Invitation review approved email sent to user ${receiver.id} for workspace ${workspaceId}` ); @@ -386,13 +489,14 @@ export class NotificationService { await this.ensureWorkspaceContentExists(workspaceId); const notification = await this.models.notification.createInvitationReviewDeclined(input); - await this.sendInvitationReviewDeclinedEmail(input); + await this.sendInvitationReviewDeclinedEmail(input, notification.id); await this.publishCountChanged(input.userId, 'created'); return notification; } private async sendInvitationReviewDeclinedEmail( - input: InvitationReviewDeclinedNotificationCreate + input: InvitationReviewDeclinedNotificationCreate, + notificationId?: string ) { const workspaceId = input.body.workspaceId; const receiverUserId = input.userId; @@ -400,15 +504,28 @@ export class NotificationService { if (!receiver) { return; } - await this.mailer.trySend({ - name: 'LinkInvitationDecline', - to: receiver.email, - props: { - workspace: { - $$workspaceId: workspaceId, + await this.trySendMail( + { + name: 'LinkInvitationDecline', + to: receiver.email, + props: { + workspace: { + $$workspaceId: workspaceId, + }, + }, + metadata: { + dedupeKey: notificationId + ? `notification:${notificationId}:mail:LinkInvitationDecline` + : undefined, + recipientUserId: receiver.id, + actorUserId: input.body.createdByUserId, + workspaceId, + notificationId, + source: { trusted: false }, }, }, - }); + 'workspace_invitation' + ); this.logger.debug( `Invitation review declined email sent to user ${receiver.id} for workspace ${workspaceId}` ); @@ -539,4 +656,32 @@ export class NotificationService { ); return !!isActive; } + + private async trySendMail(command: SendMailCommand, mailClass: string) { + const actorUserId = command.metadata?.actorUserId; + if ( + actorUserId && + (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(actorUserId)) + ) { + await this.mailer.skip(command, { + mailClass, + reason: 'actor_quarantined', + }); + return false; + } + + const workspaceId = command.metadata?.workspaceId; + if ( + workspaceId && + (await this.runtime.isInviteAbuseWorkspaceQuarantined(workspaceId)) + ) { + await this.mailer.skip(command, { + mailClass, + reason: 'workspace_quarantined', + }); + return false; + } + + return await this.mailer.trySend(command); + } } diff --git a/packages/backend/server/src/core/workspaces/__tests__/abuse.spec.ts b/packages/backend/server/src/core/workspaces/__tests__/abuse.spec.ts index ade7fd49fc..fd90008374 100644 --- a/packages/backend/server/src/core/workspaces/__tests__/abuse.spec.ts +++ b/packages/backend/server/src/core/workspaces/__tests__/abuse.spec.ts @@ -1,36 +1,390 @@ +import { createHash } from 'node:crypto'; + +import { + createInviteLinkMutation, + inviteByEmailsMutation, + WorkspaceInviteLinkExpireTime, +} from '@affine/graphql'; +import { PrismaClient, WorkspaceMemberStatus } from '@prisma/client'; import test from 'ava'; +import type { Request } from 'express'; +import Sinon from 'sinon'; -import { canUserExecuteLimitedActions, containsUrlOrDomain } from '../abuse'; +import { createApp, type TestingApp } from '../../../__tests__/e2e/test'; +import { Mockers } from '../../../__tests__/mocks'; +import { Config } from '../../../base'; +import { ActionForbidden, TooManyRequest } from '../../../base/error'; +import { Models, WorkspaceRole } from '../../../models'; +import { + getAbuseRequestSource, + InviteAbuseDispositionService, + InviteQuotaAssertService, +} from '../abuse'; -test('should detect links and bare domains in workspace names', t => { - t.true(containsUrlOrDomain('BTC https://spam.example')); - t.true(containsUrlOrDomain('Join spam.example now')); - t.true(containsUrlOrDomain('Join spam.example, ltd')); - t.true(containsUrlOrDomain('Join spam.example。')); - t.true(containsUrlOrDomain('www.spam.example')); +let app: TestingApp; +const quota = { + assertWorkspaceInviteQuota: Sinon.stub(), + commitWorkspaceInviteQuota: Sinon.stub(), + releaseWorkspaceInviteQuota: Sinon.stub(), +}; + +function workspaceSubjectKey(workspaceId: string) { + return `workspace:v1:${createHash('sha256').update(workspaceId).digest('hex').slice(0, 24)}`; +} + +test.before(async () => { + app = await createApp({ + tapModule: builder => { + builder.overrideProvider(InviteQuotaAssertService).useValue(quota); + }, + }); }); -test('should not detect email addresses or partial domain words', t => { - t.false(containsUrlOrDomain('Contact user@spam.example')); - t.false(containsUrlOrDomain('spam.example_btc')); +test.beforeEach(() => { + quota.assertWorkspaceInviteQuota.reset(); + quota.commitWorkspaceInviteQuota.reset(); + quota.releaseWorkspaceInviteQuota.reset(); }); -test('should check account age for share actions', t => { - const minimumAccountAgeMs = 24 * 60 * 60 * 1000; +test.after.always(async () => { + await app.close(); +}); - t.false( - canUserExecuteLimitedActions({ createdAt: new Date() }, minimumAccountAgeMs) - ); - t.true( - canUserExecuteLimitedActions( - { - createdAt: new Date(Date.now() - minimumAccountAgeMs - 1), +test('invite quota rejection has no invite side effects', async t => { + quota.assertWorkspaceInviteQuota.rejects(new TooManyRequest()); + const models = app.get(Models); + const owner = await app.create(Mockers.User); + const workspace = await app.create(Mockers.Workspace, { + owner: { id: owner.id }, + }); + const targetEmail = `quota-${Date.now()}@example.com`; + + await app.login(owner); + await t.throwsAsync( + app.gql({ + query: inviteByEmailsMutation, + variables: { + workspaceId: workspace.id, + emails: [targetEmail], }, - minimumAccountAgeMs - ) + }) + ); + + t.is(await models.user.getUserByEmail(targetEmail), null); + t.is(await models.workspaceUser.count(workspace.id), 1); + t.is(app.mails.send.callCount, 0); + t.is(app.queue.count('notification.sendInvitation'), 0); + t.is(quota.commitWorkspaceInviteQuota.callCount, 0); + t.is(quota.releaseWorkspaceInviteQuota.callCount, 0); +}); + +test('abuse request source trusts Cloudflare facts only when configured', t => { + const config = app.get(Config); + const previousTrusted = config.auth.trustedCloudflareHeaders; + const req = { + ip: '10.0.0.1', + get(name: string) { + return ( + { + 'CF-Connecting-IP': '114.51.41.91', + 'CF-IPCountry': 'JP', + 'CF-Ray': 'ray-id', + 'x-affine-cf-asn': '4294967295', + 'X-Forwarded-For': '198.51.100.9', + } satisfies Record + )[name]; + }, + } as Request; + + try { + config.auth.trustedCloudflareHeaders = false; + t.deepEqual(getAbuseRequestSource(req, config), { trusted: false }); + + config.auth.trustedCloudflareHeaders = true; + t.deepEqual(getAbuseRequestSource(req, config), { + trusted: true, + ip: '114.51.41.91', + country: 'JP', + asn: 4294967295, + rayId: 'ray-id', + }); + } finally { + config.auth.trustedCloudflareHeaders = previousTrusted; + } +}); + +test('invite quota rejection keeps mapped response when disposition fails', async t => { + const service = new InviteQuotaAssertService( + app.get(Config), + { + getWorkspaceSeatQuota: Sinon.stub().resolves({ + memberLimit: 10, + memberCount: 1, + }), + } as any, + { + assertWorkspaceInviteQuotaV1: Sinon.stub().resolves({ + allowed: false, + reason: 'abuse_subject', + actionRequired: { + action: 'quarantine_actor', + actionId: '1', + subjectKey: 'subject', + evidenceId: '1', + }, + }), + } as any, + { + execute: Sinon.stub().rejects(new Error('disposition failed')), + } as any + ); + + await t.throwsAsync( + service.assertWorkspaceInviteQuota({ + actorUserId: 'actor', + workspaceId: 'workspace', + targetCount: 1, + targetDomains: [{ domain: 'example.com', count: 1 }], + }), + { instanceOf: ActionForbidden } ); }); -test('should skip account age check when share action delay is disabled', t => { - t.true(canUserExecuteLimitedActions({ createdAt: new Date() }, 0)); +test('abuse disposition applies action scope to invitation artifacts', async t => { + const models = app.get(Models); + const db = app.get(PrismaClient); + const disposition = app.get(InviteAbuseDispositionService); + + for (const scenario of [ + { + name: 'actor', + subjectKey: 'actor-subject', + subjectKind: 'actor_email', + action: 'quarantine_actor', + }, + { + name: 'workspace', + subjectKey: 'workspace-subject', + subjectKind: 'workspace', + action: 'quarantine_workspace', + }, + ] as const) { + const actor = await app.create(Mockers.User); + const invitee = await app.create(Mockers.User); + const workspace = await app.create(Mockers.Workspace, { owner: actor }); + const anotherWorkspace = await app.create(Mockers.Workspace, { + owner: actor, + }); + await models.workspaceInvitation.set( + workspace.id, + invitee.id, + WorkspaceRole.Collaborator, + WorkspaceMemberStatus.Pending, + { inviterId: actor.id } + ); + await models.workspaceInvitation.set( + anotherWorkspace.id, + invitee.id, + WorkspaceRole.Collaborator, + WorkspaceMemberStatus.Pending, + { inviterId: actor.id } + ); + const canceledDelivery = await models.mailDelivery.create({ + mailName: 'MemberInvitation', + mailClass: 'workspace_invitation', + priority: 'normal', + recipientEmail: invitee.email, + actorUserId: actor.id, + workspaceId: workspace.id, + abuseSubjectKey: scenario.subjectKey, + payload: { + name: 'MemberInvitation', + to: invitee.email, + props: { + url: 'https://affine.pro/invite', + user: { $$userId: actor.id }, + workspace: { $$workspaceId: workspace.id }, + }, + }, + }); + const otherDelivery = + scenario.name === 'workspace' + ? await models.mailDelivery.create({ + mailName: 'MemberInvitation', + mailClass: 'workspace_invitation', + priority: 'normal', + recipientEmail: invitee.email, + actorUserId: actor.id, + workspaceId: anotherWorkspace.id, + abuseSubjectKey: 'other-workspace-subject', + payload: { + name: 'MemberInvitation', + to: invitee.email, + props: { + url: 'https://affine.pro/invite', + user: { $$userId: actor.id }, + workspace: { $$workspaceId: anotherWorkspace.id }, + }, + }, + }) + : null; + const [{ id: actionId }] = await db.$queryRaw>` + WITH subject AS ( + INSERT INTO runtime_invite_abuse_subjects ( + subject_key, + kind, + user_id, + actor_email_hash, + status, + first_seen_at, + last_seen_at + ) + VALUES (${scenario.subjectKey}, ${scenario.subjectKind}, ${actor.id}, 'hash', 'quarantined', now(), now()) + ON CONFLICT (subject_key) DO NOTHING + RETURNING subject_key + ), + evidence AS ( + INSERT INTO runtime_invite_abuse_evidence ( + subject_key, + workspace_id, + user_id, + actor_email_hash, + decision, + reason + ) + VALUES (${scenario.subjectKey}, ${workspace.id}, ${actor.id}, 'hash', ${scenario.action}, 'test') + RETURNING id + ) + INSERT INTO runtime_invite_abuse_actions ( + subject_key, + evidence_id, + action, + status + ) + SELECT ${scenario.subjectKey}, evidence.id, ${scenario.action}, 'pending' + FROM evidence + RETURNING id + `; + + await disposition.execute({ + actorUserId: actor.id, + workspaceId: workspace.id, + actionRequired: { + action: scenario.action, + subjectKey: scenario.subjectKey, + evidenceId: '1', + actionId: actionId.toString(), + }, + }); + + if (scenario.name === 'actor') { + t.is( + await db.workspaceInvitation.count({ + where: { inviterUserId: actor.id }, + }), + 0, + scenario.action + ); + } else { + t.is( + await db.workspaceInvitation.count({ + where: { workspaceId: workspace.id }, + }), + 0, + scenario.action + ); + t.is( + await db.workspaceInvitation.count({ + where: { workspaceId: anotherWorkspace.id }, + }), + 1, + scenario.action + ); + } + t.is( + ( + await db.mailDelivery.findUniqueOrThrow({ + where: { id: canceledDelivery.id }, + }) + ).status, + 'canceled', + scenario.action + ); + if (otherDelivery) { + t.is( + ( + await db.mailDelivery.findUniqueOrThrow({ + where: { id: otherDelivery.id }, + }) + ).status, + 'queued', + scenario.action + ); + } + } +}); + +test('workspace quarantine blocks invite link creation', async t => { + const db = app.get(PrismaClient); + const config = app.get(Config); + const owner = await app.create(Mockers.User); + const workspace = await app.create(Mockers.Workspace, { owner }); + const subjectKey = workspaceSubjectKey(workspace.id); + await db.$executeRaw` + INSERT INTO runtime_invite_abuse_subjects ( + subject_key, + kind, + status, + first_seen_at, + last_seen_at + ) + VALUES (${subjectKey}, 'workspace', 'quarantined', now(), now()) + ON CONFLICT (subject_key) + DO UPDATE SET + status = 'quarantined', + updated_at = now() + `; + + const previousDelay = config.auth.newAccountShareActionDelay; + config.auth.newAccountShareActionDelay = 0; + try { + await app.login(owner); + await t.throwsAsync( + app.gql({ + query: createInviteLinkMutation, + variables: { + workspaceId: workspace.id, + expireTime: WorkspaceInviteLinkExpireTime.OneDay, + }, + }) + ); + } finally { + config.auth.newAccountShareActionDelay = previousDelay; + } +}); + +test('domain workspace name blocks invite link creation', async t => { + const config = app.get(Config); + const owner = await app.create(Mockers.User); + const workspace = await app.create(Mockers.Workspace, { + owner, + name: 'Join example.com', + }); + + const previousDelay = config.auth.newAccountShareActionDelay; + config.auth.newAccountShareActionDelay = 0; + try { + await app.login(owner); + await t.throwsAsync( + app.gql({ + query: createInviteLinkMutation, + variables: { + workspaceId: workspace.id, + expireTime: WorkspaceInviteLinkExpireTime.OneDay, + }, + }) + ); + } finally { + config.auth.newAccountShareActionDelay = previousDelay; + } }); diff --git a/packages/backend/server/src/core/workspaces/abuse.ts b/packages/backend/server/src/core/workspaces/abuse.ts index a6fc8c1336..1b99d40a2c 100644 --- a/packages/backend/server/src/core/workspaces/abuse.ts +++ b/packages/backend/server/src/core/workspaces/abuse.ts @@ -1,8 +1,41 @@ -const URL_OR_DOMAIN_PATTERN = - /(?:https?:\/\/|www\.|(?; + +export type InviteQuotaAdmission = { + reservationId?: string; + decision: RuntimeWorkspaceInviteQuotaDecision; +}; + +declare global { + interface Jobs { + 'inviteAbuse.executePendingActions': {}; + } } export function canUserExecuteLimitedActions( @@ -12,3 +45,344 @@ export function canUserExecuteLimitedActions( if (minimumAccountAgeMs <= 0) return true; return Date.now() - user.createdAt.getTime() >= minimumAccountAgeMs; } + +function parseAsn(value: string | undefined) { + if (!value) { + return; + } + const asn = Number(value); + return Number.isSafeInteger(asn) && asn > 0 && asn <= 0xffffffff + ? asn + : undefined; +} + +export function getAbuseRequestSource( + req: Request | undefined, + config: Config +): RuntimeQuotaSourceInput { + if (!req || !config.auth.trustedCloudflareHeaders) { + return { trusted: false }; + } + + return { + trusted: true, + ip: getRequestClientIp(req), + country: req.get('CF-IPCountry')?.trim() || undefined, + asn: parseAsn(req.get('x-affine-cf-asn')), + rayId: req.get('CF-Ray')?.trim() || undefined, + }; +} + +function hashLogValue(value: string | undefined) { + if (!value) { + return undefined; + } + return createHash('sha256').update(value).digest('hex').slice(0, 24); +} + +@Injectable() +export class InviteAbuseDispositionService { + private readonly logger = new Logger(InviteAbuseDispositionService.name); + private readonly workerOwnerId = randomUUID(); + private readonly inlineWorkerId = `node-inline:${this.workerOwnerId}`; + + constructor( + private readonly models: Models, + private readonly runtime: BackendRuntimeProvider, + private readonly queue: JobQueue + ) {} + + async execute(input: { + actionRequired?: ActionRequired; + actorUserId: string; + workspaceId: string; + alreadyClaimed?: boolean; + workerId?: string; + }) { + const { actionRequired } = input; + if (!actionRequired) { + return; + } + const workerId = input.workerId ?? this.inlineWorkerId; + + try { + if (!input.alreadyClaimed) { + const claimed = await this.runtime.claimInviteAbuseAction( + actionRequired.actionId, + workerId + ); + if (!claimed) { + return; + } + } + + switch (actionRequired.action) { + case 'ban_actor': + await this.cancelPendingActorArtifacts(input, actionRequired); + await this.models.session.deleteUserSessions(input.actorUserId); + await this.models.user.ban(input.actorUserId); + break; + case 'quarantine_actor': + await this.cancelPendingActorArtifacts(input, actionRequired); + await this.models.session.deleteUserSessions(input.actorUserId); + break; + case 'quarantine_workspace': + await this.cancelPendingWorkspaceArtifacts(input.workspaceId); + break; + case 'quarantine_source_cohort': + await this.models.mailDelivery.cancelByAbuseSubject( + actionRequired.subjectKey, + 'source_cohort_quarantined' + ); + break; + default: + throw new Error( + `Unknown invite abuse action: ${actionRequired.action}` + ); + } + + await this.runtime.markInviteAbuseAction( + actionRequired.actionId, + workerId, + 'succeeded' + ); + } catch (error) { + const message = + error instanceof Error ? error.message : 'Failed to execute action.'; + await this.runtime.markInviteAbuseAction( + actionRequired.actionId, + workerId, + 'failed', + message + ); + this.logger.error('Failed to execute invite abuse disposition', { + userId: input.actorUserId, + workspaceId: input.workspaceId, + action: actionRequired.action, + actionId: actionRequired.actionId, + evidenceId: actionRequired.evidenceId, + subjectKey: actionRequired.subjectKey, + error: message, + }); + throw error; + } + } + + @Cron(CronExpression.EVERY_MINUTE) + async enqueuePendingActions() { + await this.queue.add( + 'inviteAbuse.executePendingActions', + {}, + { jobId: 'invite-abuse-execute-pending-actions' } + ); + } + + @OnJob('inviteAbuse.executePendingActions') + async executePendingActions() { + const workerId = `node:${this.workerOwnerId}`; + const actions = await this.runtime.claimRetryableInviteAbuseActions( + workerId, + 20 + ); + for (const action of actions) { + await this.execute({ + actorUserId: action.actorUserId, + workspaceId: action.workspaceId, + alreadyClaimed: true, + workerId, + actionRequired: { + action: action.action, + subjectKey: action.subjectKey, + evidenceId: action.evidenceId, + actionId: action.actionId, + }, + }); + } + } + + private async cancelPendingActorArtifacts( + input: { + actorUserId: string; + workspaceId: string; + }, + action: ActionRequired + ) { + await this.models.mailDelivery.cancelByActor(input.actorUserId); + await this.models.mailDelivery.cancelByWorkspace(input.workspaceId); + await this.models.mailDelivery.cancelByAbuseSubject(action.subjectKey); + + await this.models.workspaceInvitation.cancelPendingByActor( + input.actorUserId + ); + } + + private async cancelPendingWorkspaceArtifacts(workspaceId: string) { + await this.models.mailDelivery.cancelByWorkspace(workspaceId); + await this.models.workspaceInvitation.cancelPendingByWorkspace(workspaceId); + } +} + +@Injectable() +export class InviteQuotaAssertService { + private readonly logger = new Logger(InviteQuotaAssertService.name); + + constructor( + private readonly config: Config, + private readonly quota: QuotaService, + private readonly runtime: BackendRuntimeProvider, + private readonly disposition: InviteAbuseDispositionService + ) {} + + async assertWorkspaceInviteQuota(input: { + actorUserId: string; + workspaceId: string; + requestId?: string; + targetCount: number; + targetDomains: RuntimeQuotaTargetDomainInput[]; + source?: RuntimeQuotaSourceInput; + }): Promise { + const start = Date.now(); + const seatQuota = await this.quota.getWorkspaceSeatQuota(input.workspaceId); + let decision: RuntimeWorkspaceInviteQuotaDecision; + try { + decision = await this.runtime.assertWorkspaceInviteQuotaV1(input); + } catch (error) { + const message = + error instanceof Error ? error.message : 'native assert failed'; + metrics.workspace.counter('invite_quota_runtime_fallback').add(1, { + mode: this.config.auth.inviteQuotaFailOpenOnRuntimeError + ? 'fail_open' + : 'fail_closed', + }); + this.logger.error('Workspace invite quota native assert failed', { + userId: input.actorUserId, + workspaceId: input.workspaceId, + targetCount: input.targetCount, + targetDomainsSummary: input.targetDomains, + sourceTrusted: input.source?.trusted ?? false, + country: input.source?.country, + asn: input.source?.asn, + requestId: input.requestId, + cfRay: input.source?.rayId, + error: message, + }); + if (this.config.auth.inviteQuotaFailOpenOnRuntimeError) { + return { decision: { allowed: true, requested: input.targetCount } }; + } + throw new TooManyRequest(); + } + metrics.workspace + .counter('invite_quota_requested_targets') + .add(input.targetCount); + metrics.workspace + .histogram('invite_quota_counter_latency_ms') + .record(Date.now() - start); + + if (!decision.allowed) { + metrics.workspace.counter('invite_quota_rejected').add(1, { + reason: decision.reason ?? 'unknown', + }); + metrics.workspace.counter('invite_quota_reject_by_reason').add(1, { + reason: decision.reason ?? 'unknown', + }); + if (this.config.auth.inviteQuotaShadowMode) { + this.logger.warn('Workspace invite quota shadow rejected', { + userId: input.actorUserId, + workspaceId: input.workspaceId, + targetCount: input.targetCount, + targetDomainsSummary: input.targetDomains, + sourceTrusted: input.source?.trusted ?? false, + country: input.source?.country, + asn: input.source?.asn, + reason: decision.reason, + scopeKeyHash: hashLogValue(decision.scopeKey), + limit: decision.limit, + current: decision.current, + requested: decision.requested, + memberLimit: seatQuota.memberLimit, + memberCount: seatQuota.memberCount, + retryAfter: decision.retryAfterSeconds, + requestId: input.requestId, + cfRay: input.source?.rayId, + wouldDispose: decision.actionRequired?.action, + }); + return { decision }; + } + + try { + await this.disposition.execute({ + actionRequired: decision.actionRequired, + actorUserId: input.actorUserId, + workspaceId: input.workspaceId, + }); + } catch (error) { + this.logger.error('Workspace invite quota disposition failed', { + userId: input.actorUserId, + workspaceId: input.workspaceId, + action: decision.actionRequired?.action, + actionId: decision.actionRequired?.actionId, + reason: decision.reason, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + this.logger.warn('Workspace invite quota rejected', { + userId: input.actorUserId, + workspaceId: input.workspaceId, + targetCount: input.targetCount, + targetDomainsSummary: input.targetDomains, + sourceTrusted: input.source?.trusted ?? false, + country: input.source?.country, + asn: input.source?.asn, + reason: decision.reason, + scopeKeyHash: hashLogValue(decision.scopeKey), + limit: decision.limit, + current: decision.current, + requested: decision.requested, + memberLimit: seatQuota.memberLimit, + memberCount: seatQuota.memberCount, + retryAfter: decision.retryAfterSeconds, + requestId: input.requestId, + cfRay: input.source?.rayId, + }); + throw this.mapDecision(decision); + } + + metrics.workspace.counter('invite_quota_allowed').add(1); + return { + reservationId: decision.reservationId, + decision, + }; + } + + async commitWorkspaceInviteQuota( + reservationId: string | undefined, + usage: { + targetCount: number; + targetDomains: RuntimeQuotaTargetDomainInput[]; + } + ) { + if (!reservationId) { + return false; + } + return await this.runtime.commitWorkspaceInviteQuotaV1( + reservationId, + usage + ); + } + + async releaseWorkspaceInviteQuota(reservationId: string | undefined) { + if (!reservationId) { + return false; + } + return await this.runtime.releaseWorkspaceInviteQuotaV1(reservationId); + } + + private mapDecision( + decision: RuntimeWorkspaceInviteQuotaDecision + ): UserFriendlyError { + if (decision.reason === 'abuse_subject' || decision.actionRequired) { + return new ActionForbidden('This feature is temporarily unavailable.'); + } + return new TooManyRequest(); + } +} diff --git a/packages/backend/server/src/core/workspaces/event.ts b/packages/backend/server/src/core/workspaces/event.ts index 2a9fbf9ef5..ec3924d1f0 100644 --- a/packages/backend/server/src/core/workspaces/event.ts +++ b/packages/backend/server/src/core/workspaces/event.ts @@ -79,6 +79,11 @@ export class WorkspaceEvents { $$workspaceId: workspaceId, }, }, + metadata: { + workspaceId, + recipientUserId: userId, + source: { trusted: false }, + }, }); } diff --git a/packages/backend/server/src/core/workspaces/index.ts b/packages/backend/server/src/core/workspaces/index.ts index a38deffdce..957e145671 100644 --- a/packages/backend/server/src/core/workspaces/index.ts +++ b/packages/backend/server/src/core/workspaces/index.ts @@ -9,6 +9,10 @@ import { PermissionModule } from '../permission'; import { QuotaModule } from '../quota'; import { StorageModule } from '../storage'; import { UserModule } from '../user'; +import { + InviteAbuseDispositionService, + InviteQuotaAssertService, +} from './abuse'; import { WorkspacesController } from './controller'; import { WorkspaceEvents } from './event'; import { WorkspaceRealtimeModule } from './realtime.module'; @@ -46,6 +50,8 @@ import { WorkspaceStatsJob } from './stats.job'; DocHistoryResolver, WorkspaceBlobResolver, WorkspaceService, + InviteAbuseDispositionService, + InviteQuotaAssertService, WorkspaceEvents, AdminWorkspaceResolver, WorkspaceStatsJob, @@ -54,6 +60,11 @@ import { WorkspaceStatsJob } from './stats.job'; }) export class WorkspaceModule {} +export { + getAbuseRequestSource, + InviteAbuseDispositionService, + InviteQuotaAssertService, +} from './abuse'; export { WorkspaceRealtimeModule } from './realtime.module'; export { WorkspaceService } from './service'; export { InvitationType, WorkspaceType } from './types'; diff --git a/packages/backend/server/src/core/workspaces/resolvers/analytics-types.ts b/packages/backend/server/src/core/workspaces/resolvers/analytics-types.ts index b1142afa4b..8c76264118 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/analytics-types.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/analytics-types.ts @@ -2,6 +2,7 @@ import { Field, Int, ObjectType, registerEnumType } from '@nestjs/graphql'; export enum TimeBucket { Minute = 'Minute', + Hour = 'Hour', Day = 'Day', } diff --git a/packages/backend/server/src/core/workspaces/resolvers/doc.ts b/packages/backend/server/src/core/workspaces/resolvers/doc.ts index 951e74d64c..764ecc5aa3 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/doc.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/doc.ts @@ -35,6 +35,7 @@ import { import { PageInfo } from '../../../base/graphql/pagination'; import { Models, PublicDocMode } from '../../../models'; import { CurrentUser } from '../../auth'; +import { BackendRuntimeProvider } from '../../backend-runtime'; import { Editor } from '../../doc'; import { DOC_ACTIONS, @@ -303,13 +304,34 @@ export class WorkspaceDocResolver { private readonly models: Models, private readonly cache: Cache, private readonly event: EventBus, - private readonly config: Config + private readonly config: Config, + private readonly runtime: BackendRuntimeProvider ) {} private async assertCanShare( userId: string, context: { workspaceId: string; docId: string; action: 'publishDoc' } ) { + if (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(userId)) { + this.logger.warn('Share action blocked for quarantined actor', { + userId, + ...context, + }); + throw new ActionForbidden( + 'This feature is temporarily unavailable for you.' + ); + } + if ( + await this.runtime.isInviteAbuseWorkspaceQuarantined(context.workspaceId) + ) { + this.logger.warn('Share action blocked for quarantined workspace', { + userId, + ...context, + }); + throw new ActionForbidden( + 'This feature is temporarily unavailable for you.' + ); + } const user = await this.models.user.get(userId); const newAccountAgeMs = this.config.auth.newAccountShareActionDelay * 1000; if (!user || !canUserExecuteLimitedActions(user, newAccountAgeMs)) { diff --git a/packages/backend/server/src/core/workspaces/resolvers/member.ts b/packages/backend/server/src/core/workspaces/resolvers/member.ts index 593536ba11..ad23e11a53 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/member.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/member.ts @@ -1,6 +1,7 @@ import { Logger } from '@nestjs/common'; import { Args, + Context, Int, Mutation, Parent, @@ -24,6 +25,7 @@ import { CanNotRevokeYourself, Config, EventBus, + getRequestTrackerId, InvalidInvitation, isValidCacheTtl, mapAnyError, @@ -38,8 +40,11 @@ import { URLHelper, UserNotFound, } from '../../../base'; +import type { GraphqlContext } from '../../../base/graphql'; import { Models } from '../../../models'; import { CurrentUser, Public } from '../../auth'; +import { BackendRuntimeProvider } from '../../backend-runtime'; +import { containsUrlOrDomain } from '../../content-policy'; import { PermissionAccess, WorkspacePolicyService, @@ -48,7 +53,11 @@ import { import { QuotaService } from '../../quota'; import { UserType } from '../../user'; import { validators } from '../../utils/validators'; -import { canUserExecuteLimitedActions, containsUrlOrDomain } from '../abuse'; +import { + canUserExecuteLimitedActions, + getAbuseRequestSource, + InviteQuotaAssertService, +} from '../abuse'; import { WorkspaceService } from '../service'; import { InvitationType, @@ -59,6 +68,27 @@ import { WorkspaceType, } from '../types'; +type InviteCandidate = { + index: number; + email: string; + normalizedEmail: string; + domain: string; + target?: { id: string }; +}; + +function emailDomain(email: string) { + const parts = email.split('@'); + return parts.length === 2 ? parts[1] : ''; +} + +function aggregateTargetDomains(candidates: InviteCandidate[]) { + const domains = new Map(); + for (const candidate of candidates) { + domains.set(candidate.domain, (domains.get(candidate.domain) ?? 0) + 1); + } + return Array.from(domains, ([domain, count]) => ({ domain, count })); +} + /** * Workspace team resolver * Public apis rate limit: 10 req/m @@ -78,16 +108,39 @@ export class WorkspaceMemberResolver { private readonly policy: WorkspacePolicyService, private readonly workspaceService: WorkspaceService, private readonly quota: QuotaService, - private readonly config: Config + private readonly config: Config, + private readonly inviteQuota: InviteQuotaAssertService, + private readonly runtime: BackendRuntimeProvider ) {} private async assertCanInviteOrShare( userId: string, context: { workspaceId: string; - action: 'inviteMembers' | 'createInviteLink'; + action: 'createInviteLink'; } ) { + if (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(userId)) { + this.logger.warn('Share action blocked for quarantined actor', { + userId, + ...context, + }); + throw new ActionForbidden( + 'This feature is temporarily unavailable for you.' + ); + } + if ( + await this.runtime.isInviteAbuseWorkspaceQuarantined(context.workspaceId) + ) { + this.logger.warn('Share action blocked for quarantined workspace', { + userId, + ...context, + }); + throw new ActionForbidden( + 'This feature is temporarily unavailable for you.' + ); + } + // Member invites are owned by native quota; this guard stays for invite links until share/link actions migrate. const user = await this.models.user.get(userId); const newAccountAgeMs = this.config.auth.newAccountShareActionDelay * 1000; if (!user || !canUserExecuteLimitedActions(user, newAccountAgeMs)) { @@ -182,6 +235,7 @@ export class WorkspaceMemberResolver { @Mutation(() => [InviteResult]) async inviteMembers( @CurrentUser() me: CurrentUser, + @Context() context: GraphqlContext, @Args('workspaceId') workspaceId: string, @Args({ name: 'emails', type: () => [String] }) emails: string[] ): Promise { @@ -189,16 +243,54 @@ export class WorkspaceMemberResolver { .user(me.id) .workspace(workspaceId) .assert('Workspace.Users.Manage'); - await this.assertCanInviteOrShare(me.id, { - workspaceId, - action: 'inviteMembers', - }); await this.assertWorkspaceNameCanInvite(workspaceId); if (emails.length > 512) { throw new TooManyRequest(); } + const results: InviteResult[] = emails.map(email => ({ email })); + const candidates: InviteCandidate[] = []; + const seen = new Set(); + for (const [index, email] of emails.entries()) { + try { + const normalizedEmail = email.trim().toLowerCase(); + validators.assertValidEmail(normalizedEmail); + if (seen.has(normalizedEmail)) { + throw new ActionForbidden('Duplicate invite email.'); + } + seen.add(normalizedEmail); + + const target = await this.models.user.getUserByEmail(normalizedEmail); + if (target) { + const originRecord = await this.models.workspaceUser.get( + workspaceId, + target.id + ); + if (originRecord) { + throw new AlreadyInSpace({ spaceId: workspaceId }); + } + } + + candidates.push({ + index, + email, + normalizedEmail, + domain: emailDomain(normalizedEmail), + target: target ? { id: target.id } : undefined, + }); + } catch (error) { + results[index] = { + email, + error: mapAnyError(error), + }; + } + } + + if (candidates.length === 0) { + return results; + } + // lock to prevent concurrent invite const lockFlag = `invite:${workspaceId}`; await using lock = await this.mutex.acquire(lockFlag); @@ -206,49 +298,68 @@ export class WorkspaceMemberResolver { throw new TooManyRequest(); } + const admission = await this.inviteQuota.assertWorkspaceInviteQuota({ + actorUserId: me.id, + workspaceId, + requestId: getRequestTrackerId(context.req), + targetCount: candidates.length, + targetDomains: aggregateTargetDomains(candidates), + source: getAbuseRequestSource(context.req, this.config), + }); const quota = await this.quota.getWorkspaceSeatQuota(workspaceId); const isTeam = await this.workspaceService.isTeamWorkspace(workspaceId); + const successfulCandidates: InviteCandidate[] = []; + let reservationSettled = false; - const results: InviteResult[] = []; + try { + for (const candidate of candidates) { + try { + let target = candidate.target; + if (!target) { + target = await this.models.user.create({ + email: candidate.normalizedEmail, + registered: false, + }); + } - for (const [idx, email] of emails.entries()) { - try { - validators.assertValidEmail(email); - let target = await this.models.user.getUserByEmail(email); - if (target) { - const originRecord = await this.models.workspaceUser.get( + const existingMember = await this.models.workspaceUser.get( workspaceId, target.id ); - // only invite if the user is not already in the workspace - if (originRecord) { + if (existingMember) { throw new AlreadyInSpace({ spaceId: workspaceId }); } - } else { - target = await this.models.user.create({ - email, - registered: false, - }); - } - // no need to check quota, directly go allocating seat path - if (isTeam) { - const role = await this.models.workspaceUser.set( - workspaceId, - target.id, - WorkspaceRole.Collaborator, - { - status: WorkspaceMemberStatus.AllocatingSeat, - source: WorkspaceMemberSource.Email, - inviterId: me.id, + if (!isTeam) { + const needMoreSeat = + quota.memberCount + successfulCandidates.length + 1 > + quota.memberLimit; + if (needMoreSeat) { + throw new NoMoreSeat({ spaceId: workspaceId }); } - ); - await this.allocateAvailableTeamSeats(workspaceId, quota.memberLimit); - results.push({ email, inviteId: role.id }); - } else { - const needMoreSeat = quota.memberCount + idx + 1 > quota.memberLimit; - if (needMoreSeat) { - throw new NoMoreSeat({ spaceId: workspaceId }); + } + + // no need to check quota, directly go allocating seat path + if (isTeam) { + const role = await this.models.workspaceUser.set( + workspaceId, + target.id, + WorkspaceRole.Collaborator, + { + status: WorkspaceMemberStatus.AllocatingSeat, + source: WorkspaceMemberSource.Email, + inviterId: me.id, + } + ); + await this.allocateAvailableTeamSeats( + workspaceId, + quota.memberLimit + ); + results[candidate.index] = { + email: candidate.email, + inviteId: role.id, + }; + successfulCandidates.push(candidate); } else { const role = await this.models.workspaceUser.set( workspaceId, @@ -264,17 +375,39 @@ export class WorkspaceMemberResolver { inviteId: role.id, inviterId: me.id, }); - results.push({ - email, + results[candidate.index] = { + email: candidate.email, inviteId: role.id, - }); + }; + successfulCandidates.push(candidate); } + } catch (error) { + results[candidate.index] = { + email: candidate.email, + error: mapAnyError(error), + }; } - } catch (error) { - results.push({ - email, - error: mapAnyError(error), - }); + } + + if (successfulCandidates.length > 0) { + await this.inviteQuota.commitWorkspaceInviteQuota( + admission.reservationId, + { + targetCount: successfulCandidates.length, + targetDomains: aggregateTargetDomains(successfulCandidates), + } + ); + } else { + await this.inviteQuota.releaseWorkspaceInviteQuota( + admission.reservationId + ); + } + reservationSettled = true; + } finally { + if (!reservationSettled) { + await this.inviteQuota.releaseWorkspaceInviteQuota( + admission.reservationId + ); } } diff --git a/packages/backend/server/src/core/workspaces/service.ts b/packages/backend/server/src/core/workspaces/service.ts index 3085668e82..8288bc0e89 100644 --- a/packages/backend/server/src/core/workspaces/service.ts +++ b/packages/backend/server/src/core/workspaces/service.ts @@ -7,8 +7,10 @@ import { DEFAULT_WORKSPACE_NAME, Models, } from '../../models'; +import { BackendRuntimeProvider } from '../backend-runtime'; import { DocReader } from '../doc'; import { Mailer } from '../mail'; +import type { SendMailCommand } from '../mail/types'; import { WorkspaceRole } from '../permission'; import { QuotaStateService } from '../quota/state'; import { WorkspaceBlobStorage } from '../storage'; @@ -32,7 +34,8 @@ export class WorkspaceService { private readonly blobStorage: WorkspaceBlobStorage, private readonly mailer: Mailer, private readonly queue: JobQueue, - private readonly quotaState: QuotaStateService + private readonly quotaState: QuotaStateService, + private readonly runtime: BackendRuntimeProvider ) {} async getInviteInfo(inviteId: string): Promise { @@ -111,7 +114,7 @@ export class WorkspaceService { const admins = await this.models.workspaceUser.getAdmins(workspaceId); const link = this.url.link(`/workspace/${workspaceId}`); - await this.mailer.trySend({ + await this.trySendWorkspaceMail({ name: 'TeamWorkspaceUpgraded', to: owner.email, props: { @@ -121,11 +124,16 @@ export class WorkspaceService { isOwner: true, url: link, }, + metadata: { + workspaceId, + recipientUserId: owner.id, + source: { trusted: false }, + }, }); await Promise.allSettled( admins.map(async user => { - await this.mailer.trySend({ + await this.trySendWorkspaceMail({ name: 'TeamWorkspaceUpgraded', to: user.email, props: { @@ -135,6 +143,11 @@ export class WorkspaceService { isOwner: false, url: link, }, + metadata: { + workspaceId, + recipientUserId: user.id, + source: { trusted: false }, + }, }); }) ); @@ -192,7 +205,7 @@ export class WorkspaceService { } if (ws.role === WorkspaceRole.Admin) { - await this.mailer.trySend({ + await this.trySendWorkspaceMail({ name: 'TeamBecomeAdmin', to: user.email, props: { @@ -201,9 +214,14 @@ export class WorkspaceService { }, url: this.url.link(`/workspace/${ws.id}`), }, + metadata: { + workspaceId: ws.id, + recipientUserId: user.id, + source: { trusted: false }, + }, }); } else { - await this.mailer.trySend({ + await this.trySendWorkspaceMail({ name: 'TeamBecomeCollaborator', to: user.email, props: { @@ -212,12 +230,17 @@ export class WorkspaceService { }, url: this.url.link(`/workspace/${ws.id}`), }, + metadata: { + workspaceId: ws.id, + recipientUserId: user.id, + source: { trusted: false }, + }, }); } } async sendOwnershipTransferredEmail(email: string, ws: { id: string }) { - await this.mailer.trySend({ + await this.trySendWorkspaceMail({ name: 'OwnershipTransferred', to: email, props: { @@ -225,11 +248,15 @@ export class WorkspaceService { $$workspaceId: ws.id, }, }, + metadata: { + workspaceId: ws.id, + source: { trusted: false }, + }, }); } async sendOwnershipReceivedEmail(email: string, ws: { id: string }) { - await this.mailer.trySend({ + await this.trySendWorkspaceMail({ name: 'OwnershipReceived', to: email, props: { @@ -237,12 +264,16 @@ export class WorkspaceService { $$workspaceId: ws.id, }, }, + metadata: { + workspaceId: ws.id, + source: { trusted: false }, + }, }); } async sendLeaveEmail(workspaceId: string, userId: string) { const owner = await this.models.workspaceUser.getOwner(workspaceId); - await this.mailer.trySend({ + await this.trySendWorkspaceMail({ name: 'MemberLeave', to: owner.email, props: { @@ -253,9 +284,41 @@ export class WorkspaceService { $$userId: userId, }, }, + metadata: { + workspaceId, + recipientUserId: owner.id, + actorUserId: userId, + source: { trusted: false }, + }, }); } + private async trySendWorkspaceMail(command: SendMailCommand) { + const actorUserId = command.metadata?.actorUserId; + if ( + actorUserId && + (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(actorUserId)) + ) { + await this.mailer.skip(command, { + mailClass: 'workspace_lifecycle', + reason: 'actor_quarantined', + }); + return false; + } + const workspaceId = command.metadata?.workspaceId; + if ( + workspaceId && + (await this.runtime.isInviteAbuseWorkspaceQuarantined(workspaceId)) + ) { + await this.mailer.skip(command, { + mailClass: 'workspace_lifecycle', + reason: 'workspace_quarantined', + }); + return false; + } + return await this.mailer.trySend(command); + } + async allocateSeats(workspaceId: string, quantity: number) { const pendings = await this.models.workspaceUser.allocateSeats( workspaceId, diff --git a/packages/backend/server/src/models/index.ts b/packages/backend/server/src/models/index.ts index 8935ac8001..005f8d2ce6 100644 --- a/packages/backend/server/src/models/index.ts +++ b/packages/backend/server/src/models/index.ts @@ -29,6 +29,7 @@ import { DocUserModel } from './doc-user'; import { FeatureModel } from './feature'; import { HistoryModel } from './history'; import { MagicLinkOtpModel } from './magic-link-otp'; +import { MailDeliveryModel } from './mail-delivery'; import { NotificationModel } from './notification'; import { PermissionProjectionModel } from './permission-projection'; import { @@ -57,6 +58,7 @@ const MODELS = { session: SessionModel, verificationToken: VerificationTokenModel, magicLinkOtp: MagicLinkOtpModel, + mailDelivery: MailDeliveryModel, feature: FeatureModel, workspace: WorkspaceModel, userFeature: UserFeatureModel, @@ -165,6 +167,7 @@ export * from './doc-user'; export * from './feature'; export * from './history'; export * from './magic-link-otp'; +export * from './mail-delivery'; export * from './notification'; export * from './permission-projection'; export * from './permission-write'; diff --git a/packages/backend/server/src/models/magic-link-otp.ts b/packages/backend/server/src/models/magic-link-otp.ts index 73a134e65e..3dc74d9e1d 100644 --- a/packages/backend/server/src/models/magic-link-otp.ts +++ b/packages/backend/server/src/models/magic-link-otp.ts @@ -36,6 +36,7 @@ export class MagicLinkOtpModel extends BaseModel { create: { email, otpHash, token, clientNonce, expiresAt, attempts: 0 }, update: { otpHash, token, clientNonce, expiresAt, attempts: 0 }, }); + return { expiresAt }; } @Transactional() diff --git a/packages/backend/server/src/models/mail-delivery.ts b/packages/backend/server/src/models/mail-delivery.ts new file mode 100644 index 0000000000..418a5a6d49 --- /dev/null +++ b/packages/backend/server/src/models/mail-delivery.ts @@ -0,0 +1,665 @@ +import { createHmac } from 'node:crypto'; + +import { Injectable } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; + +import { CryptoHelper } from '../base'; +import { BaseModel } from './base'; + +export type MailDeliveryStatus = + | 'queued' + | 'sending' + | 'retry_wait' + | 'sent' + | 'skipped' + | 'failed' + | 'canceled'; + +export type MailDeliveryPriority = 'critical' | 'high' | 'normal' | 'low'; + +export type MailDeliveryRow = { + id: string; + mailName: string; + mailClass: string; + priority: MailDeliveryPriority; + status: MailDeliveryStatus; + dedupeKey: string | null; + recipientEmail: string | null; + recipientHash: string; + recipientDomain: string; + recipientUserId: string | null; + actorUserId: string | null; + workspaceId: string | null; + notificationId: string | null; + abuseSubjectKey: string | null; + quotaReservationId: string | null; + quotaDecision: Prisma.JsonValue | null; + payload: Prisma.JsonValue | null; + sendAfter: Date; + expiresAt: Date | null; + attemptCount: number; + maxAttempts: number; + lockedBy: string | null; + lockedUntil: Date | null; + firstAttemptAt: Date | null; + lastAttemptAt: Date | null; + sentAt: Date | null; + settledAt: Date | null; + canceledAt: Date | null; + failedAt: Date | null; + providerMessageId: string | null; + providerResponse: string | null; + lastErrorCode: string | null; + lastError: string | null; + retentionState: 'full' | 'anonymized'; + anonymizedAt: Date | null; + createdAt: Date; + updatedAt: Date; +}; + +export type MailDeliveryCreateInput = { + mailName: string; + mailClass: string; + priority: MailDeliveryPriority; + status?: Extract; + dedupeKey?: string; + recipientEmail: string; + recipientUserId?: string; + actorUserId?: string; + workspaceId?: string; + notificationId?: string; + abuseSubjectKey?: string; + quotaReservationId?: string; + quotaDecision?: Prisma.JsonValue; + payload: Prisma.JsonValue; + sendAfter?: Date; + expiresAt?: Date; + maxAttempts?: number; + lastErrorCode?: string; + lastError?: string; +}; + +export type MailDeliveryAdminAggregate = { + bucket: Date; + mailName: string; + mailClass: string; + status: MailDeliveryStatus; + count: number; +}; + +export type MailDeliveryMetricsSnapshot = { + retryWait: number; + failedRecent: number; + expiredLeases: number; + readyDelayMs: number; +}; + +const TERMINAL_STATUSES = new Set([ + 'sent', + 'skipped', + 'failed', + 'canceled', +]); + +function recipientDomain(email: string) { + const parts = email.trim().toLowerCase().split('@'); + return parts.length === 2 ? parts[1] : ''; +} + +function redact(value: string | undefined | null) { + if (!value) { + return null; + } + return value + .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '[email]') + .replace(/\b\d{6,}\b/g, '[code]') + .replace(/https?:\/\/\S+/gi, '[url]') + .slice(0, 1000); +} + +function mapRow(row: Record): MailDeliveryRow { + return { + id: row.id as string, + mailName: row.mailName as string, + mailClass: row.mailClass as string, + priority: row.priority as MailDeliveryPriority, + status: row.status as MailDeliveryStatus, + dedupeKey: row.dedupeKey as string | null, + recipientEmail: row.recipientEmail as string | null, + recipientHash: row.recipientHash as string, + recipientDomain: row.recipientDomain as string, + recipientUserId: row.recipientUserId as string | null, + actorUserId: row.actorUserId as string | null, + workspaceId: row.workspaceId as string | null, + notificationId: row.notificationId as string | null, + abuseSubjectKey: row.abuseSubjectKey as string | null, + quotaReservationId: row.quotaReservationId as string | null, + quotaDecision: row.quotaDecision as Prisma.JsonValue | null, + payload: row.payload as Prisma.JsonValue | null, + sendAfter: row.sendAfter as Date, + expiresAt: row.expiresAt as Date | null, + attemptCount: row.attemptCount as number, + maxAttempts: row.maxAttempts as number, + lockedBy: row.lockedBy as string | null, + lockedUntil: row.lockedUntil as Date | null, + firstAttemptAt: row.firstAttemptAt as Date | null, + lastAttemptAt: row.lastAttemptAt as Date | null, + sentAt: row.sentAt as Date | null, + settledAt: row.settledAt as Date | null, + canceledAt: row.canceledAt as Date | null, + failedAt: row.failedAt as Date | null, + providerMessageId: row.providerMessageId as string | null, + providerResponse: row.providerResponse as string | null, + lastErrorCode: row.lastErrorCode as string | null, + lastError: row.lastError as string | null, + retentionState: row.retentionState as 'full' | 'anonymized', + anonymizedAt: row.anonymizedAt as Date | null, + createdAt: row.createdAt as Date, + updatedAt: row.updatedAt as Date, + }; +} + +const SELECT_FIELDS = Prisma.sql` + id::text AS "id", + mail_name AS "mailName", + mail_class AS "mailClass", + priority, + status, + dedupe_key AS "dedupeKey", + recipient_email AS "recipientEmail", + recipient_hash AS "recipientHash", + recipient_domain AS "recipientDomain", + recipient_user_id AS "recipientUserId", + actor_user_id AS "actorUserId", + workspace_id AS "workspaceId", + notification_id AS "notificationId", + abuse_subject_key AS "abuseSubjectKey", + quota_reservation_id::text AS "quotaReservationId", + quota_decision AS "quotaDecision", + payload, + send_after AS "sendAfter", + expires_at AS "expiresAt", + attempt_count AS "attemptCount", + max_attempts AS "maxAttempts", + locked_by AS "lockedBy", + locked_until AS "lockedUntil", + first_attempt_at AS "firstAttemptAt", + last_attempt_at AS "lastAttemptAt", + sent_at AS "sentAt", + settled_at AS "settledAt", + canceled_at AS "canceledAt", + failed_at AS "failedAt", + provider_message_id AS "providerMessageId", + provider_response AS "providerResponse", + last_error_code AS "lastErrorCode", + last_error AS "lastError", + retention_state AS "retentionState", + anonymized_at AS "anonymizedAt", + created_at AS "createdAt", + updated_at AS "updatedAt" +`; + +@Injectable() +export class MailDeliveryModel extends BaseModel { + constructor(private readonly crypto: CryptoHelper) { + super(); + } + + private recipientHash(email: string) { + return createHmac('sha256', this.crypto.keyPair.sha256.privateKey) + .update(email.trim().toLowerCase()) + .digest('hex'); + } + + async create(input: MailDeliveryCreateInput): Promise { + const now = new Date(); + const sendAfter = input.sendAfter ?? now; + const status = input.status ?? 'queued'; + const expired = + input.expiresAt && + (input.expiresAt.getTime() <= now.getTime() || + sendAfter.getTime() >= input.expiresAt.getTime()); + const finalStatus = expired ? 'failed' : status; + const terminal = TERMINAL_STATUSES.has(finalStatus); + const lastErrorCode = expired + ? 'expired' + : (input.lastErrorCode ?? (status === 'skipped' ? 'skipped' : null)); + const rows = await this.db.$queryRaw>>` + INSERT INTO mail_deliveries ( + mail_name, + mail_class, + priority, + status, + dedupe_key, + recipient_email, + recipient_hash, + recipient_domain, + recipient_user_id, + actor_user_id, + workspace_id, + notification_id, + abuse_subject_key, + quota_reservation_id, + quota_decision, + payload, + send_after, + expires_at, + max_attempts, + settled_at, + failed_at, + last_error_code, + last_error, + retention_state, + anonymized_at + ) + VALUES ( + ${input.mailName}, + ${input.mailClass}, + ${input.priority}, + ${finalStatus}, + ${input.dedupeKey ?? null}, + ${terminal ? null : input.recipientEmail}, + ${this.recipientHash(input.recipientEmail)}, + ${recipientDomain(input.recipientEmail)}, + ${input.recipientUserId ?? null}, + ${input.actorUserId ?? null}, + ${input.workspaceId ?? null}, + ${input.notificationId ?? null}, + ${input.abuseSubjectKey ?? null}, + ${input.quotaReservationId ?? null}::uuid, + ${input.quotaDecision ?? null}, + ${terminal ? null : input.payload}, + ${sendAfter}, + ${input.expiresAt ?? null}, + ${input.maxAttempts ?? 3}, + ${terminal ? now : null}, + ${finalStatus === 'failed' ? now : null}, + ${lastErrorCode}, + ${redact(input.lastError)}, + ${terminal ? 'anonymized' : 'full'}, + ${terminal ? now : null} + ) + ON CONFLICT (dedupe_key) WHERE dedupe_key IS NOT NULL + DO NOTHING + RETURNING ${SELECT_FIELDS} + `; + + if (rows[0]) { + return mapRow(rows[0]); + } + if (!input.dedupeKey) { + throw new Error('Failed to create mail delivery.'); + } + const existing = await this.findByDedupeKey(input.dedupeKey); + if (!existing) { + throw new Error('Failed to find deduped mail delivery.'); + } + return existing; + } + + async findByDedupeKey(dedupeKey: string) { + const rows = await this.db.$queryRaw>>` + SELECT ${SELECT_FIELDS} + FROM mail_deliveries + WHERE dedupe_key = ${dedupeKey} + LIMIT 1 + `; + return rows[0] ? mapRow(rows[0]) : null; + } + + async claimReady( + workerId: string, + options: { batchSize: number; leaseMs: number } + ) { + await this.markExpiredPending(options.batchSize); + const rows = await this.db.$queryRaw>>` + UPDATE mail_deliveries + SET status = 'sending', + locked_by = ${workerId}, + locked_until = now() + (${options.leaseMs}::text || ' milliseconds')::interval, + updated_at = now() + WHERE id IN ( + SELECT id + FROM mail_deliveries + WHERE ( + status IN ('queued', 'retry_wait') + OR (status = 'sending' AND locked_until < now()) + ) + AND send_after <= now() + AND (expires_at IS NULL OR expires_at > now()) + AND (locked_until IS NULL OR locked_until < now()) + AND max_attempts > attempt_count + ORDER BY + CASE priority + WHEN 'critical' THEN 0 + WHEN 'high' THEN 1 + WHEN 'normal' THEN 2 + ELSE 3 + END, + send_after, + created_at + FOR UPDATE SKIP LOCKED + LIMIT ${options.batchSize} + ) + RETURNING ${SELECT_FIELDS} + `; + return rows.map(mapRow); + } + + async markAttemptStarted(id: string, workerId: string) { + const rows = await this.db.$queryRaw>>` + UPDATE mail_deliveries + SET attempt_count = attempt_count + 1, + first_attempt_at = COALESCE(first_attempt_at, now()), + last_attempt_at = now(), + updated_at = now() + WHERE id = ${id}::uuid + AND status = 'sending' + AND locked_by = ${workerId} + AND (expires_at IS NULL OR expires_at > now()) + AND max_attempts > attempt_count + RETURNING ${SELECT_FIELDS} + `; + return rows[0] ? mapRow(rows[0]) : null; + } + + async markSent( + id: string, + workerId: string, + result: { + providerMessageId?: string | null; + providerResponse?: string | null; + } + ) { + return await this.markTerminal(id, workerId, 'sent', { + providerMessageId: result.providerMessageId, + providerResponse: redact(result.providerResponse), + }); + } + + async markRetry( + id: string, + workerId: string, + input: { + sendAfter: Date; + errorCode?: string | null; + error?: string | null; + } + ) { + const rows = await this.db.$queryRaw>>` + UPDATE mail_deliveries + SET status = 'retry_wait', + send_after = ${input.sendAfter}, + locked_by = NULL, + locked_until = NULL, + last_error_code = ${input.errorCode ?? null}, + last_error = ${redact(input.error)}, + updated_at = now() + WHERE id = ${id}::uuid + AND status = 'sending' + AND locked_by = ${workerId} + AND (expires_at IS NULL OR ${input.sendAfter} < expires_at) + AND attempt_count < max_attempts + RETURNING ${SELECT_FIELDS} + `; + return rows[0] ? mapRow(rows[0]) : null; + } + + async markSkipped( + id: string, + workerId: string, + reason: string, + detail?: string + ) { + return await this.markTerminal(id, workerId, 'skipped', { + lastErrorCode: reason, + lastError: redact(detail), + }); + } + + async markFailed( + id: string, + workerId: string, + errorCode: string, + error?: string + ) { + return await this.markTerminal(id, workerId, 'failed', { + lastErrorCode: errorCode, + lastError: redact(error), + }); + } + + async markCanceled(id: string, workerId: string, reason = 'canceled') { + return await this.markTerminal(id, workerId, 'canceled', { + lastErrorCode: reason, + }); + } + + async cancelByRecipient( + recipientEmail: string, + reason = 'recipient_deleted' + ) { + return await this.cancelWhere( + Prisma.sql`recipient_hash = ${this.recipientHash(recipientEmail)}`, + reason + ); + } + + async cancelById(id: string, reason = 'canceled') { + return await this.cancelWhere(Prisma.sql`id = ${id}::uuid`, reason); + } + + async cancelMemberInvitationByActor( + actorUserId: string, + reason = 'actor_deleted' + ) { + return await this.cancelWhere( + Prisma.sql`mail_name = 'MemberInvitation' AND actor_user_id = ${actorUserId}`, + reason + ); + } + + async cancelByActor(actorUserId: string, reason = 'actor_quarantined') { + return await this.cancelWhere( + Prisma.sql`actor_user_id = ${actorUserId}`, + reason + ); + } + + async cancelByWorkspace( + workspaceId: string, + reason = 'workspace_quarantined' + ) { + return await this.cancelWhere( + Prisma.sql`workspace_id = ${workspaceId}`, + reason + ); + } + + async cancelByAbuseSubject( + subjectKey: string, + reason = 'abuse_subject_quarantined' + ) { + return await this.cancelWhere( + Prisma.sql`abuse_subject_key = ${subjectKey}`, + reason + ); + } + + async markExpiredPending(limit: number) { + const result = await this.db.$executeRaw` + UPDATE mail_deliveries + SET status = 'failed', + failed_at = now(), + settled_at = now(), + locked_by = NULL, + locked_until = NULL, + recipient_email = NULL, + payload = NULL, + retention_state = 'anonymized', + anonymized_at = now(), + last_error_code = 'expired', + updated_at = now() + WHERE id IN ( + SELECT id + FROM mail_deliveries + WHERE ( + status IN ('queued', 'retry_wait') + OR (status = 'sending' AND locked_until < now()) + ) + AND expires_at IS NOT NULL + AND expires_at <= now() + LIMIT ${limit} + ) + `; + return Number(result); + } + + async adminAggregate(input: { + from: Date; + to: Date; + bucket: 'hour' | 'day'; + }) { + const rows = await this.db.$queryRaw>>` + WITH events AS ( + SELECT + date_trunc(${input.bucket}, created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AS bucket, + mail_name, + mail_class, + status, + COUNT(*)::int AS count + FROM mail_deliveries + WHERE created_at >= ${input.from} + AND created_at < ${input.to} + GROUP BY + bucket, + mail_name, + mail_class, + status + ) + SELECT + bucket AS "bucket", + mail_name AS "mailName", + mail_class AS "mailClass", + status, + count + FROM events + ORDER BY bucket ASC, count DESC, "mailName" ASC + `; + return rows.map(row => ({ + bucket: row.bucket as Date, + mailName: row.mailName as string, + mailClass: row.mailClass as string, + status: row.status as MailDeliveryStatus, + count: row.count as number, + })) satisfies MailDeliveryAdminAggregate[]; + } + + async deleteAnonymizedBefore(before: Date, limit: number) { + const result = await this.db.$executeRaw` + DELETE FROM mail_deliveries + WHERE id IN ( + SELECT id + FROM mail_deliveries + WHERE retention_state = 'anonymized' + AND settled_at IS NOT NULL + AND settled_at < ${before} + ORDER BY settled_at + LIMIT ${limit} + ) + `; + return Number(result); + } + + async metricsSnapshot(): Promise { + const rows = await this.db.$queryRaw< + Array<{ + retryWait: number; + failedRecent: number; + expiredLeases: number; + readyDelayMs: number; + }> + >` + SELECT + COUNT(*) FILTER (WHERE status = 'retry_wait')::int AS "retryWait", + COUNT(*) FILTER ( + WHERE status = 'failed' AND failed_at >= now() - interval '1 hour' + )::int AS "failedRecent", + COUNT(*) FILTER ( + WHERE status = 'sending' AND locked_until < now() + )::int AS "expiredLeases", + COALESCE(MAX(EXTRACT(EPOCH FROM now() - send_after) * 1000) FILTER ( + WHERE status IN ('queued', 'retry_wait') AND send_after <= now() + ), 0)::int AS "readyDelayMs" + FROM mail_deliveries + `; + return ( + rows[0] ?? { + retryWait: 0, + failedRecent: 0, + expiredLeases: 0, + readyDelayMs: 0, + } + ); + } + + private async markTerminal( + id: string, + workerId: string, + status: Extract< + MailDeliveryStatus, + 'sent' | 'skipped' | 'failed' | 'canceled' + >, + input: { + providerMessageId?: string | null; + providerResponse?: string | null; + lastErrorCode?: string | null; + lastError?: string | null; + } + ) { + const rows = await this.db.$queryRaw>>` + UPDATE mail_deliveries + SET status = ${status}, + sent_at = CASE WHEN ${status} = 'sent' THEN now() ELSE sent_at END, + failed_at = CASE WHEN ${status} = 'failed' THEN now() ELSE failed_at END, + canceled_at = CASE WHEN ${status} = 'canceled' THEN now() ELSE canceled_at END, + settled_at = now(), + locked_by = NULL, + locked_until = NULL, + recipient_email = NULL, + payload = NULL, + retention_state = 'anonymized', + anonymized_at = now(), + provider_message_id = ${input.providerMessageId ?? null}, + provider_response = ${input.providerResponse ?? null}, + last_error_code = ${input.lastErrorCode ?? null}, + last_error = ${input.lastError ?? null}, + updated_at = now() + WHERE id = ${id}::uuid + AND status = 'sending' + AND locked_by = ${workerId} + RETURNING ${SELECT_FIELDS} + `; + return rows[0] ? mapRow(rows[0]) : null; + } + + private async cancelWhere(where: Prisma.Sql, reason: string) { + const result = await this.db.$executeRaw` + UPDATE mail_deliveries + SET status = 'canceled', + canceled_at = now(), + settled_at = now(), + locked_by = NULL, + locked_until = NULL, + recipient_email = NULL, + payload = NULL, + retention_state = 'anonymized', + anonymized_at = now(), + last_error_code = ${reason}, + updated_at = now() + WHERE status IN ('queued', 'retry_wait', 'sending') + AND ${where} + `; + return Number(result); + } +} diff --git a/packages/backend/server/src/models/permission-write.ts b/packages/backend/server/src/models/permission-write.ts index 941509f030..82b7461536 100644 --- a/packages/backend/server/src/models/permission-write.ts +++ b/packages/backend/server/src/models/permission-write.ts @@ -358,6 +358,32 @@ export class WorkspaceInvitationModel extends BaseModel { }); } + @Transactional() + async cancelPendingByActor(actorUserId: string) { + await this.models.permissionProjection.markNewWriteOrigin(); + return await this.db.workspaceInvitation.deleteMany({ + where: { + inviterUserId: actorUserId, + status: { + in: ['pending', 'waiting_review', 'waiting_seat'], + }, + }, + }); + } + + @Transactional() + async cancelPendingByWorkspace(workspaceId: string) { + await this.models.permissionProjection.markNewWriteOrigin(); + return await this.db.workspaceInvitation.deleteMany({ + where: { + workspaceId, + status: { + in: ['pending', 'waiting_review', 'waiting_seat'], + }, + }, + }); + } + private async supportsCurrentInvitationColumns() { this.hasCurrentColumns ??= this.db.$queryRaw>` SELECT EXISTS ( diff --git a/packages/backend/server/src/models/verification-token.ts b/packages/backend/server/src/models/verification-token.ts index c049309db8..ece8f2852b 100644 --- a/packages/backend/server/src/models/verification-token.ts +++ b/packages/backend/server/src/models/verification-token.ts @@ -28,17 +28,31 @@ export class VerificationTokenModel extends BaseModel { type: TokenType, credential?: string, ttlInSec: number = 30 * 60 + ) { + const { token } = await this.createWithExpiresAt( + type, + credential, + ttlInSec + ); + return token; + } + + async createWithExpiresAt( + type: TokenType, + credential?: string, + ttlInSec: number = 30 * 60 ) { const plaintextToken = randomUUID(); + const expiresAt = new Date(Date.now() + ttlInSec * 1000); const { token } = await this.db.verificationToken.create({ data: { type, token: plaintextToken, credential, - expiresAt: new Date(Date.now() + ttlInSec * 1000), + expiresAt, }, }); - return this.crypto.encrypt(token); + return { token: this.crypto.encrypt(token), expiresAt }; } /** diff --git a/packages/backend/server/src/native.ts b/packages/backend/server/src/native.ts index 1cc5b8edaa..d821a67c1b 100644 --- a/packages/backend/server/src/native.ts +++ b/packages/backend/server/src/native.ts @@ -11,6 +11,8 @@ import serverNativeModule, { type CapabilityAttachmentContract, type CapabilityModelCapability, type CommandResponse, + type ContentPolicyScanInput, + type ContentPolicyScanResult, type ImageInspection, type ImageInspectionOptions, type LicenseError, @@ -78,6 +80,8 @@ export type { CapabilityAttachmentContract, CapabilityModelCapability, CommandResponse, + ContentPolicyScanInput, + ContentPolicyScanResult, ImageInspection, ImageInspectionOptions, LicenseError, @@ -255,6 +259,7 @@ export const inspectImageForProxy = serverNativeModule.inspectImageForProxy; export const fetchRemoteAttachment = serverNativeModule.fetchRemoteAttachment; export const inferRemoteMimeType = serverNativeModule.inferRemoteMimeType; export const assertSafeUrl = serverNativeModule.assertSafeUrl; +export const scanContentPolicyV1 = serverNativeModule.scanContentPolicyV1; export const safeFetch = serverNativeModule.safeFetch; export const activateLicense = serverNativeModule.activateLicense; export const checkLicenseHealth = serverNativeModule.checkLicenseHealth; diff --git a/packages/backend/server/src/plugins/payment/manager/selfhost.ts b/packages/backend/server/src/plugins/payment/manager/selfhost.ts index 481ff0e978..ba57eed1da 100644 --- a/packages/backend/server/src/plugins/payment/manager/selfhost.ts +++ b/packages/backend/server/src/plugins/payment/manager/selfhost.ts @@ -148,6 +148,10 @@ export class SelfhostTeamSubscriptionManager extends SubscriptionManager { name: 'TeamLicense', to: userEmail, props: { license: key }, + metadata: { + dedupeKey: `selfhost-license:${key}`, + source: { trusted: false }, + }, }); await this.upsertStripeProviderSubscription( diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index 01d3e34a9e..c825f88533 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -105,6 +105,42 @@ type AdminLicensePreview { workspaceId: String! } +input AdminMailDeliveriesInput { + hours: Int! = 24 +} + +type AdminMailDeliveryAnalytics { + byOutcome: [AdminMailDeliverySeries!]! + byStatus: [AdminMailDeliverySeries!]! + byType: [AdminMailDeliverySeries!]! + summary: AdminMailDeliverySummary! + window: TimeWindow! +} + +type AdminMailDeliveryPoint { + bucket: DateTime! + count: Int! +} + +type AdminMailDeliverySeries { + key: String! + label: String! + points: [AdminMailDeliveryPoint!]! + total: Int! +} + +type AdminMailDeliverySummary { + canceled: Int! + failed: Int! + queued: Int! + retryWait: Int! + sending: Int! + sent: Int! + skipped: Int! + successRate: Float! + total: Int! +} + type AdminSharedLinkTopItem { docId: String! guestViews: SafeInt! @@ -1952,6 +1988,9 @@ type Query { """Get aggregated dashboard metrics for admin panel""" adminDashboard(input: AdminDashboardInput): AdminDashboard! + """Aggregate mail delivery timeline facts for admin panel""" + adminMailDeliveries(input: AdminMailDeliveriesInput): AdminMailDeliveryAnalytics! + """Get workspace detail for admin""" adminWorkspace(id: String!): AdminWorkspace @@ -2420,6 +2459,7 @@ type TestWorkspaceByokConfigResultType { enum TimeBucket { Day + Hour Minute } diff --git a/packages/common/graphql/src/graphql/admin/admin-mail-deliveries.gql b/packages/common/graphql/src/graphql/admin/admin-mail-deliveries.gql new file mode 100644 index 0000000000..f83e3aef8f --- /dev/null +++ b/packages/common/graphql/src/graphql/admin/admin-mail-deliveries.gql @@ -0,0 +1,50 @@ +query adminMailDeliveries($input: AdminMailDeliveriesInput) { + adminMailDeliveries(input: $input) { + window { + from + to + timezone + bucket + requestedSize + effectiveSize + } + summary { + total + sent + failed + skipped + canceled + queued + sending + retryWait + successRate + } + byStatus { + key + label + total + points { + bucket + count + } + } + byType { + key + label + total + points { + bucket + count + } + } + byOutcome { + key + label + total + points { + bucket + count + } + } + } +} diff --git a/packages/common/graphql/src/graphql/index.ts b/packages/common/graphql/src/graphql/index.ts index a0e0a4e231..af43b28723 100644 --- a/packages/common/graphql/src/graphql/index.ts +++ b/packages/common/graphql/src/graphql/index.ts @@ -191,6 +191,61 @@ export const adminDashboardQuery = { }`, }; +export const adminMailDeliveriesQuery = { + id: 'adminMailDeliveriesQuery' as const, + op: 'adminMailDeliveries', + query: `query adminMailDeliveries($input: AdminMailDeliveriesInput) { + adminMailDeliveries(input: $input) { + window { + from + to + timezone + bucket + requestedSize + effectiveSize + } + summary { + total + sent + failed + skipped + canceled + queued + sending + retryWait + successRate + } + byStatus { + key + label + total + points { + bucket + count + } + } + byType { + key + label + total + points { + bucket + count + } + } + byOutcome { + key + label + total + points { + bucket + count + } + } + } +}`, +}; + export const adminServerConfigQuery = { id: 'adminServerConfigQuery' as const, op: 'adminServerConfig', diff --git a/packages/common/graphql/src/schema.ts b/packages/common/graphql/src/schema.ts index 3690cfd83d..c1863134af 100644 --- a/packages/common/graphql/src/schema.ts +++ b/packages/common/graphql/src/schema.ts @@ -147,6 +147,46 @@ export interface AdminLicensePreview { workspaceId: Scalars['String']['output']; } +export interface AdminMailDeliveriesInput { + hours?: Scalars['Int']['input']; +} + +export interface AdminMailDeliveryAnalytics { + __typename?: 'AdminMailDeliveryAnalytics'; + byOutcome: Array; + byStatus: Array; + byType: Array; + summary: AdminMailDeliverySummary; + window: TimeWindow; +} + +export interface AdminMailDeliveryPoint { + __typename?: 'AdminMailDeliveryPoint'; + bucket: Scalars['DateTime']['output']; + count: Scalars['Int']['output']; +} + +export interface AdminMailDeliverySeries { + __typename?: 'AdminMailDeliverySeries'; + key: Scalars['String']['output']; + label: Scalars['String']['output']; + points: Array; + total: Scalars['Int']['output']; +} + +export interface AdminMailDeliverySummary { + __typename?: 'AdminMailDeliverySummary'; + canceled: Scalars['Int']['output']; + failed: Scalars['Int']['output']; + queued: Scalars['Int']['output']; + retryWait: Scalars['Int']['output']; + sending: Scalars['Int']['output']; + sent: Scalars['Int']['output']; + skipped: Scalars['Int']['output']; + successRate: Scalars['Float']['output']; + total: Scalars['Int']['output']; +} + export interface AdminSharedLinkTopItem { __typename?: 'AdminSharedLinkTopItem'; docId: Scalars['String']['output']; @@ -2620,6 +2660,8 @@ export interface Query { adminAllSharedLinks: PaginatedAdminAllSharedLink; /** Get aggregated dashboard metrics for admin panel */ adminDashboard: AdminDashboard; + /** Aggregate mail delivery timeline facts for admin panel */ + adminMailDeliveries: AdminMailDeliveryAnalytics; /** Get workspace detail for admin */ adminWorkspace: Maybe; /** List workspaces for admin */ @@ -2677,6 +2719,10 @@ export interface QueryAdminDashboardArgs { input?: InputMaybe; } +export interface QueryAdminMailDeliveriesArgs { + input?: InputMaybe; +} + export interface QueryAdminWorkspaceArgs { id: Scalars['String']['input']; } @@ -3143,6 +3189,7 @@ export interface TestWorkspaceByokConfigResultType { export enum TimeBucket { Day = 'Day', + Hour = 'Hour', Minute = 'Minute', } @@ -3907,6 +3954,71 @@ export type AdminDashboardQuery = { }; }; +export type AdminMailDeliveriesQueryVariables = Exact<{ + input?: InputMaybe; +}>; + +export type AdminMailDeliveriesQuery = { + __typename?: 'Query'; + adminMailDeliveries: { + __typename?: 'AdminMailDeliveryAnalytics'; + window: { + __typename?: 'TimeWindow'; + from: string; + to: string; + timezone: string; + bucket: TimeBucket; + requestedSize: number; + effectiveSize: number; + }; + summary: { + __typename?: 'AdminMailDeliverySummary'; + total: number; + sent: number; + failed: number; + skipped: number; + canceled: number; + queued: number; + sending: number; + retryWait: number; + successRate: number; + }; + byStatus: Array<{ + __typename?: 'AdminMailDeliverySeries'; + key: string; + label: string; + total: number; + points: Array<{ + __typename?: 'AdminMailDeliveryPoint'; + bucket: string; + count: number; + }>; + }>; + byType: Array<{ + __typename?: 'AdminMailDeliverySeries'; + key: string; + label: string; + total: number; + points: Array<{ + __typename?: 'AdminMailDeliveryPoint'; + bucket: string; + count: number; + }>; + }>; + byOutcome: Array<{ + __typename?: 'AdminMailDeliverySeries'; + key: string; + label: string; + total: number; + points: Array<{ + __typename?: 'AdminMailDeliveryPoint'; + bucket: string; + count: number; + }>; + }>; + }; +}; + export type AdminServerConfigQueryVariables = Exact<{ [key: string]: never }>; export type AdminServerConfigQuery = { @@ -7718,6 +7830,11 @@ export type Queries = variables: AdminDashboardQueryVariables; response: AdminDashboardQuery; } + | { + name: 'adminMailDeliveriesQuery'; + variables: AdminMailDeliveriesQueryVariables; + response: AdminMailDeliveriesQuery; + } | { name: 'adminServerConfigQuery'; variables: AdminServerConfigQueryVariables; diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json index b24b005f62..aee04b2468 100644 --- a/packages/frontend/admin/src/config.json +++ b/packages/frontend/admin/src/config.json @@ -50,6 +50,10 @@ "queues.backendRuntime": { "type": "Object", "desc": "The config for backend runtime job queue" + }, + "queues.inviteAbuse": { + "type": "Object", + "desc": "The config for invite abuse disposition job queue" } }, "throttle": { @@ -87,6 +91,18 @@ "type": "Number", "desc": "Minimum account age in seconds before new accounts can invite members or create share links." }, + "trustedCloudflareHeaders": { + "type": "Boolean", + "desc": "Whether request abuse source facts should trust Cloudflare headers from the origin edge." + }, + "inviteQuotaShadowMode": { + "type": "Boolean", + "desc": "Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions." + }, + "inviteQuotaFailOpenOnRuntimeError": { + "type": "Boolean", + "desc": "Whether workspace invite quota should fail open when native runtime admission is unavailable. Keep disabled for production." + }, "passwordRequirements": { "type": "Object", "desc": "The password strength requirements when set new password." @@ -140,6 +156,18 @@ "type": "Array", "desc": "The emails from these domains are always sent using the fallback SMTP server." }, + "deliveryWorker.batchSize": { + "type": "Number", + "desc": "Number of mail delivery rows claimed by each worker tick." + }, + "deliveryWorker.leaseMs": { + "type": "Number", + "desc": "Mail delivery worker lease duration in milliseconds." + }, + "deliveryWorker.retentionDays": { + "type": "Number", + "desc": "Days to retain anonymized terminal mail delivery ledger rows." + }, "fallbackSMTP.name": { "type": "String", "desc": "Hostname used for fallback SMTP HELO/EHLO (e.g. mail.example.com). Leave empty to use the system hostname." diff --git a/packages/frontend/admin/src/modules/dashboard/index.spec.tsx b/packages/frontend/admin/src/modules/dashboard/index.spec.tsx index d54555c7c2..b7f21c8b05 100644 --- a/packages/frontend/admin/src/modules/dashboard/index.spec.tsx +++ b/packages/frontend/admin/src/modules/dashboard/index.spec.tsx @@ -87,16 +87,91 @@ const dashboardData = { }, }; +const mailDeliveryData = { + adminMailDeliveries: { + window: { + from: '2026-02-15T20:00:00.000Z', + to: '2026-02-16T20:00:00.000Z', + timezone: 'UTC', + bucket: 'Hour', + requestedSize: 24, + effectiveSize: 24, + }, + summary: { + total: 4, + sent: 2, + failed: 1, + skipped: 0, + canceled: 0, + queued: 1, + sending: 0, + retryWait: 0, + successRate: 2 / 3, + }, + byStatus: [ + { + key: 'sent', + label: 'Sent', + total: 2, + points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 2 }], + }, + { + key: 'failed', + label: 'Failed', + total: 1, + points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 1 }], + }, + { + key: 'queued', + label: 'Queued', + total: 1, + points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 1 }], + }, + ], + byType: [ + { + key: 'auth', + label: 'auth', + total: 2, + points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 2 }], + }, + ], + byOutcome: [ + { + key: 'successful', + label: 'Successful', + total: 2, + points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 2 }], + }, + { + key: 'unsuccessful', + label: 'Unsuccessful', + total: 1, + points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 1 }], + }, + { + key: 'pending', + label: 'Pending', + total: 1, + points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 1 }], + }, + ], + }, +}; + describe('DashboardPage', () => { beforeEach(() => { (globalThis as any).environment = { isSelfHosted: true, }; useQueryMock.mockReset(); - useQueryMock.mockReturnValue({ - data: dashboardData, + useQueryMock.mockImplementation(({ query }: { query: { id: string } }) => ({ + data: + query.id === 'adminMailDeliveriesQuery' + ? mailDeliveryData + : dashboardData, isValidating: false, - }); + })); mutateQueryResourceMock.mockReset(); }); @@ -125,4 +200,14 @@ describe('DashboardPage', () => { expect(styles).toContain('--color-secondary: var(--muted-foreground);'); expect(styles).not.toContain('hsl(var(--primary))'); }); + + test('renders mail delivery analytics controls and summary', () => { + const { getByText } = render(); + + expect(getByText('Email Delivery Trend')).toBeTruthy(); + expect(getByText('Mail Delivery')).toBeTruthy(); + expect(getByText('24 hours')).toBeTruthy(); + expect(getByText('Success rate')).toBeTruthy(); + expect(getByText('66.7%')).toBeTruthy(); + }); }); diff --git a/packages/frontend/admin/src/modules/dashboard/index.tsx b/packages/frontend/admin/src/modules/dashboard/index.tsx index 766499ea8b..50a7a2e111 100644 --- a/packages/frontend/admin/src/modules/dashboard/index.tsx +++ b/packages/frontend/admin/src/modules/dashboard/index.tsx @@ -46,12 +46,17 @@ import { } from '@affine/admin/components/ui/table'; import { useMutation } from '@affine/admin/use-mutation'; import { useQuery } from '@affine/admin/use-query'; -import { adminDashboardQuery, previewLicenseMutation } from '@affine/graphql'; +import { + adminDashboardQuery, + adminMailDeliveriesQuery, + previewLicenseMutation, +} from '@affine/graphql'; import { ROUTES } from '@affine/routes'; import { ChevronDownIcon, DatabaseIcon, FileSearchIcon, + MailIcon, MessageSquareTextIcon, RefreshCwIcon, UsersIcon, @@ -166,6 +171,7 @@ const utcDateFormatter = new Intl.DateTimeFormat('en-US', { const STORAGE_DAY_OPTIONS = [7, 14, 30, 60, 90] as const; const SYNC_HOUR_OPTIONS = [1, 6, 12, 24, 48, 72] as const; const SHARED_DAY_OPTIONS = [7, 14, 28, 60, 90] as const; +type MailChartMode = 'status' | 'type' | 'outcome'; type DualNumberPoint = { label: string; @@ -180,6 +186,18 @@ type TrendPoint = { secondary?: number; }; +type MultiTrendPoint = { + x: number; + label: string; +} & Record; + +type MultiTrendSeries = { + key: string; + label: string; + color: string; + total: number; +}; + type LicensePreview = { id: string; workspaceId: string; @@ -362,6 +380,110 @@ function TrendChart({ ); } +function MultiTrendChart({ + ariaLabel, + points, + series, + valueFormatter, +}: { + ariaLabel: string; + points: MultiTrendPoint[]; + series: MultiTrendSeries[]; + valueFormatter: (value: number) => string; +}) { + const visibleSeries = series.filter(item => item.total > 0).slice(0, 4); + + if (points.length === 0 || visibleSeries.length === 0) { + return ( +
+ No mail deliveries in this window +
+ ); + } + + const chartPoints = + points.length === 1 + ? [points[0], { ...points[0], x: Number(points[0].x) + 1 }] + : points; + const config: ChartConfig = Object.fromEntries( + visibleSeries.map(item => [ + item.key, + { + label: item.label, + color: item.color, + }, + ]) + ); + + return ( + + + + + { + if (max <= 0) { + return 1; + } + return Math.ceil(max * 1.1); + }, + ]} + /> + { + const item = payload?.[0]; + return item?.payload?.label ?? ''; + }} + valueFormatter={value => valueFormatter(value)} + /> + } + /> + {visibleSeries.map(item => ( + + ))} + + + ); +} + function PrimaryMetricCard({ value, description, @@ -458,6 +580,37 @@ function WindowSelect({ ); } +function MailWindowSelect({ + value, + onChange, +}: { + value: number; + onChange: (value: number) => void; +}) { + return ( +
+ + +
+ ); +} + function LicensePreviewDialog({ license, onOpenChange, @@ -830,10 +983,244 @@ function TopSharedLinksSection({ ); } +function mailWindowLabel(hours: number) { + return hours === 24 ? '24h' : '7d'; +} + +function mailBucketLabel(value: string, hours: number) { + return hours === 24 ? formatDateTime(value) : formatDate(value); +} + +function toMailChartPoints( + series: { + key: string; + points: { bucket: string; count: number }[]; + }[], + hours: number +) { + const first = series[0]?.points ?? []; + return first.map((point, index) => { + const row: MultiTrendPoint = { + x: index, + label: mailBucketLabel(point.bucket, hours), + }; + for (const item of series) { + row[item.key] = item.points[index]?.count ?? 0; + } + return row; + }); +} + +function seriesColor(key: string, index: number) { + const colors: Record = { + sent: 'var(--primary)', + failed: 'var(--destructive)', + retry_wait: 'var(--muted-foreground)', + queued: 'var(--foreground)', + pending_status: 'var(--muted-foreground)', + successful: 'var(--primary)', + unsuccessful: 'var(--destructive)', + pending: 'var(--muted-foreground)', + auth: 'var(--primary)', + notification: 'var(--muted-foreground)', + workspace_invitation: 'var(--foreground)', + }; + return ( + colors[key] ?? + ['var(--primary)', 'var(--muted-foreground)', 'var(--foreground)'][ + index % 3 + ] + ); +} + +function combineMailSeries( + key: string, + label: string, + series: { points: { bucket: string; count: number }[] }[] +) { + const first = series[0]; + return { + key, + label, + total: series.reduce( + (total, item) => + total + item.points.reduce((sum, point) => sum + point.count, 0), + 0 + ), + points: + first?.points.map((point, index) => ({ + bucket: point.bucket, + count: series.reduce( + (total, item) => total + (item.points[index]?.count ?? 0), + 0 + ), + })) ?? [], + }; +} + +function MailDeliverySection({ hours }: { hours: number }) { + const [mode, setMode] = useState('status'); + const { data } = useQuery( + { + query: adminMailDeliveriesQuery, + variables: { + input: { hours }, + }, + }, + { + keepPreviousData: true, + revalidateOnFocus: false, + revalidateIfStale: true, + revalidateOnReconnect: true, + } + ); + + const analytics = data.adminMailDeliveries; + const sourceSeries = + mode === 'type' + ? analytics.byType + : mode === 'outcome' + ? analytics.byOutcome + : [ + analytics.byStatus.find(series => series.key === 'sent'), + analytics.byStatus.find(series => series.key === 'failed'), + combineMailSeries( + 'pending_status', + 'Pending', + analytics.byStatus.filter(series => + ['queued', 'sending'].includes(series.key) + ) + ), + analytics.byStatus.find(series => series.key === 'retry_wait'), + ].filter(series => series !== undefined); + const chartSeries = sourceSeries.map((series, index) => ({ + key: series.key, + label: series.label, + total: series.total, + color: seriesColor(series.key, index), + })); + const chartPoints = toMailChartPoints(sourceSeries, hours); + const failedLike = + analytics.summary.failed + + analytics.summary.skipped + + analytics.summary.canceled; + const pending = + analytics.summary.queued + + analytics.summary.sending + + analytics.summary.retryWait; + + return ( + + +
+ + + + {mailWindowLabel(hours)} at{' '} + {analytics.window.bucket === 'Hour' ? 'hour' : 'day'} bucket in UTC + +
+ + + +
+
+
Sent
+
+ {compactFormatter.format(analytics.summary.sent)} +
+
+
+
Not delivered
+
+ {compactFormatter.format(failedLike)} +
+
+
+
Pending
+
+ {compactFormatter.format(pending)} +
+
+
+
Success rate
+
+ {(analytics.summary.successRate * 100).toFixed(1)}% +
+
+
+ + intFormatter.format(value)} + /> + +
+ {chartSeries + .filter(series => series.total > 0) + .slice(0, 4) + .map(series => ( +
+
+ ))} +
+ +
+ {mailBucketLabel(analytics.window.from, hours)} + {mailBucketLabel(analytics.window.to, hours)} +
+ + + ); +} + +function MailDeliveryCardSkeleton() { + return ( + + + + + + +
+ + + + +
+ + + + ); +} + function DashboardPageContent() { const [storageHistoryDays, setStorageHistoryDays] = useState(30); const [syncHistoryHours, setSyncHistoryHours] = useState(48); const [sharedLinkWindowDays, setSharedLinkWindowDays] = useState(28); + const [mailDeliveryHours, setMailDeliveryHours] = useState(24); const shouldShowTopSharedLinks = !environment.isSelfHosted; const revalidateQueryResource = useMutateQueryResource(); @@ -902,6 +1289,7 @@ function DashboardPageContent() { isValidating={isValidating} onRefresh={() => { revalidateQueryResource(adminDashboardQuery).catch(() => {}); + revalidateQueryResource(adminMailDeliveriesQuery).catch(() => {}); }} /> } @@ -916,7 +1304,7 @@ function DashboardPageContent() { automatically. - + + @@ -1035,6 +1427,10 @@ function DashboardPageContent() { + }> + + + {shouldShowTopSharedLinks ? ( }>