feat(server): refactor mail queue (#15204)

This commit is contained in:
DarkSky
2026-07-07 08:38:16 +08:00
committed by GitHub
parent 9581432d21
commit 998b255afd
72 changed files with 8386 additions and 763 deletions
+42
View File
@@ -133,6 +133,18 @@
"default": {
"concurrency": 1
}
},
"queues.inviteAbuse": {
"type": "object",
"description": "The config for invite abuse disposition job queue\n@default {\"concurrency\":1}",
"properties": {
"concurrency": {
"type": "number"
}
},
"default": {
"concurrency": 1
}
}
}
},
@@ -192,6 +204,21 @@
"description": "Minimum account age in seconds before new accounts can invite members or create share links.\n@default 86400",
"default": 86400
},
"trustedCloudflareHeaders": {
"type": "boolean",
"description": "Whether request abuse source facts should trust Cloudflare headers from the origin edge.\n@default false",
"default": false
},
"inviteQuotaShadowMode": {
"type": "boolean",
"description": "Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions.\n@default false",
"default": false
},
"inviteQuotaFailOpenOnRuntimeError": {
"type": "boolean",
"description": "Whether workspace invite quota should fail open when native runtime admission is unavailable. Keep disabled for production.\n@default false",
"default": false
},
"passwordRequirements": {
"type": "object",
"description": "The password strength requirements when set new password.\n@default {\"min\":8,\"max\":32}",
@@ -264,6 +291,21 @@
"description": "The emails from these domains are always sent using the fallback SMTP server.\n@default []",
"default": []
},
"deliveryWorker.batchSize": {
"type": "number",
"description": "Number of mail delivery rows claimed by each worker tick.\n@default 50",
"default": 50
},
"deliveryWorker.leaseMs": {
"type": "number",
"description": "Mail delivery worker lease duration in milliseconds.\n@default 120000",
"default": 120000
},
"deliveryWorker.retentionDays": {
"type": "number",
"description": "Days to retain anonymized terminal mail delivery ledger rows.\n@default 30",
"default": 30
},
"fallbackSMTP.name": {
"type": "string",
"description": "Hostname used for fallback SMTP HELO/EHLO (e.g. mail.example.com). Leave empty to use the system hostname.\n@default \"\"",
Generated
+18 -8
View File
@@ -270,6 +270,8 @@ dependencies = [
"thiserror 2.0.18",
"tiktoken-rs 0.7.0",
"tokio",
"unicode-normalization",
"unicode_skeleton",
"url",
"uuid",
"v_htmlescape",
@@ -2225,7 +2227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3785,7 +3787,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5870,7 +5872,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -6312,7 +6314,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6371,7 +6373,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -7132,7 +7134,6 @@ dependencies = [
"cfg-if",
"libc",
"psm",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
@@ -7583,7 +7584,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -8668,6 +8669,15 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "unicode_skeleton"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66bd74db2c088d393d1fbf83db2cd5663137640f072d128287dd53c882a0f412"
dependencies = [
"unicode-normalization",
]
[[package]]
name = "uniffi"
version = "0.29.5"
@@ -9300,7 +9310,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.61.2",
]
[[package]]
+2
View File
@@ -62,6 +62,8 @@ sqlx = { workspace = true, default-features = false, features = [
thiserror.workspace = true
tiktoken-rs = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] }
unicode-normalization = "0.1"
unicode_skeleton = "0.1.1"
url = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
v_htmlescape = { workspace = true }
+131
View File
@@ -27,6 +27,18 @@ export declare class BackendRuntime {
cleanupExpiredRuntimeGates(limit: number): Promise<number>
cleanupExpiredUserSessions(limit: number): Promise<number>
cleanupExpiredSnapshotHistories(limit: number): Promise<number>
isInviteAbuseUserQuarantinedOrBanned(userId: string): Promise<boolean>
isInviteAbuseWorkspaceQuarantined(workspaceId: string): Promise<boolean>
claimInviteAbuseAction(actionId: string, workerId: string): Promise<boolean>
claimRetryableInviteAbuseActions(workerId: string, limit: number): Promise<Array<RuntimeInviteAbuseClaimedAction>>
markInviteAbuseAction(actionId: string, workerId: string, status: string, error?: string | undefined | null): Promise<boolean>
assertWorkspaceInviteQuotaV1(input: RuntimeWorkspaceInviteQuotaInput): Promise<RuntimeWorkspaceInviteQuotaDecision>
commitWorkspaceInviteQuotaV1(reservationId: string, usage: RuntimeWorkspaceInviteQuotaUsage): Promise<boolean>
releaseWorkspaceInviteQuotaV1(reservationId: string): Promise<boolean>
assertMailDeliveryQuotaV1(input: RuntimeMailDeliveryQuotaInput): Promise<RuntimeMailDeliveryQuotaDecision>
commitMailDeliveryQuotaV1(reservationId: string): Promise<boolean>
releaseMailDeliveryQuotaV1(reservationId: string): Promise<boolean>
cleanupExpiredRollingQuota(limit: number): Promise<number>
createAuthChallenge(purpose: string, token: string, payload: any, ttlMs: number): Promise<boolean>
getAuthChallenge(purpose: string, token: string): Promise<any | null>
consumeAuthChallenge(purpose: string, token: string): Promise<any | null>
@@ -254,6 +266,33 @@ export interface CommandResponse {
error?: LicenseError
}
export interface ContentPolicyMatch {
type: string
reason: string
value?: string
span?: ContentPolicyMatchSpan
}
export interface ContentPolicyMatchSpan {
start: number
end: number
}
export interface ContentPolicyScanInput {
value: string
checks?: Array<string>
}
export interface ContentPolicyScanResult {
version: number
original: string
normalized: string
skeleton: string
matched: boolean
matches: Array<ContentPolicyMatch>
flags: Array<string>
}
export interface CoordinationLeaseGrant {
key: string
owner: string
@@ -914,12 +953,62 @@ export interface RuntimeDocHistoryInput {
historyMaxAgeMs: number
}
export interface RuntimeInviteAbuseActionRequired {
action: string
subjectKey: string
evidenceId: string
actionId: string
}
export interface RuntimeInviteAbuseClaimedAction {
action: string
subjectKey: string
evidenceId: string
actionId: string
actorUserId: string
workspaceId: string
}
export interface RuntimeMagicLinkOtpConsumeResult {
ok: boolean
token?: string
reason?: string
}
export interface RuntimeMailDeliveryQuotaDecision {
allowed: boolean
reservationId?: string
mailClass: string
retryAfterSeconds?: number
reason?: string
scopeKey?: string
windowSeconds?: number
limit?: number
current?: number
requested?: number
}
export interface RuntimeMailDeliveryQuotaInput {
requestId?: string
mailName: string
recipient: RuntimeMailDeliveryQuotaRecipientInput
metadata: RuntimeMailDeliveryQuotaMetadataInput
source?: RuntimeQuotaSourceInput
}
export interface RuntimeMailDeliveryQuotaMetadataInput {
actorUserId?: string
workspaceId?: string
notificationId?: string
abuseSubjectKey?: string
}
export interface RuntimeMailDeliveryQuotaRecipientInput {
email: string
domain: string
userId?: string
}
export interface RuntimeMultipartUploadInit {
uploadId: string
expiresAtMs: number
@@ -960,6 +1049,19 @@ export interface RuntimePresignedObjectRequest {
expiresAtMs: number
}
export interface RuntimeQuotaSourceInput {
trusted: boolean
ip?: string
country?: string
asn?: number
rayId?: string
}
export interface RuntimeQuotaTargetDomainInput {
domain: string
count: number
}
export interface RuntimeVerificationTokenRecord {
tokenType: number
token: string
@@ -974,6 +1076,33 @@ export interface RuntimeWorkspaceInviteLinkRecord {
expiresAtMs: number
}
export interface RuntimeWorkspaceInviteQuotaDecision {
allowed: boolean
reservationId?: string
retryAfterSeconds?: number
reason?: string
scopeKey?: string
windowSeconds?: number
limit?: number
current?: number
requested?: number
actionRequired?: RuntimeInviteAbuseActionRequired
}
export interface RuntimeWorkspaceInviteQuotaInput {
actorUserId: string
workspaceId: string
requestId?: string
targetCount: number
targetDomains: Array<RuntimeQuotaTargetDomainInput>
source?: RuntimeQuotaSourceInput
}
export interface RuntimeWorkspaceInviteQuotaUsage {
targetCount: number
targetDomains: Array<RuntimeQuotaTargetDomainInput>
}
export interface RuntimeWorkspaceStatsDailyRecalibrationResult {
processed: number
lastSid: number
@@ -1030,6 +1159,8 @@ export interface SafeFetchResponse {
body: Buffer
}
export declare function scanContentPolicyV1(input: ContentPolicyScanInput): ContentPolicyScanResult
export interface StorageProviderCapabilities {
put: boolean
get: boolean
@@ -0,0 +1,333 @@
use napi_derive::napi;
use unicode_normalization::UnicodeNormalization;
use unicode_skeleton::UnicodeSkeleton;
const VERSION: u32 = 1;
#[napi(object)]
pub struct ContentPolicyScanInput {
pub value: String,
pub checks: Option<Vec<String>>,
}
#[napi(object)]
pub struct ContentPolicyMatchSpan {
pub start: u32,
pub end: u32,
}
#[napi(object)]
pub struct ContentPolicyMatch {
#[napi(js_name = "type")]
pub match_type: String,
pub reason: String,
pub value: Option<String>,
pub span: Option<ContentPolicyMatchSpan>,
}
#[napi(object)]
pub struct ContentPolicyScanResult {
pub version: u32,
pub original: String,
pub normalized: String,
pub skeleton: String,
pub matched: bool,
pub matches: Vec<ContentPolicyMatch>,
pub flags: Vec<String>,
}
#[napi]
pub fn scan_content_policy_v1(input: ContentPolicyScanInput) -> ContentPolicyScanResult {
let original = input.value;
let normalized = normalize_content(&original);
let skeleton = skeleton_content(&normalized);
let mut flags = Vec::new();
if normalized != original {
flags.push("unicode_normalized".to_string());
}
if skeleton != normalized {
flags.push("confusable_skeleton_changed".to_string());
}
let mut matches = Vec::new();
if should_run_check(input.checks.as_deref(), "url_or_domain") {
matches.extend(scan_url_or_domain(&skeleton));
}
ContentPolicyScanResult {
version: VERSION,
original,
normalized,
skeleton,
matched: !matches.is_empty(),
matches,
flags,
}
}
fn should_run_check(checks: Option<&[String]>, check: &str) -> bool {
checks
.map(|checks| checks.iter().any(|configured| configured == check))
.unwrap_or(true)
}
fn normalize_content(value: &str) -> String {
value.nfkc().collect::<String>()
}
fn skeleton_content(value: &str) -> String {
let mut output = String::with_capacity(value.len());
for ch in value.chars() {
push_skeleton_char(&mut output, ch);
}
output
}
fn push_skeleton_char(output: &mut String, ch: char) {
if is_default_ignorable(ch) {
return;
}
if let Some(replacement) = url_punctuation_ascii(ch).or_else(|| enclosed_ascii(ch)) {
output.push_str(replacement);
return;
}
if ch.is_ascii() {
output.extend(ch.to_lowercase());
return;
}
let source = ch.to_string();
let skeleton = source.skeleton_chars().collect::<String>();
if skeleton == source {
output.extend(ch.to_lowercase());
return;
}
for skeleton_ch in skeleton.chars() {
push_skeleton_char(output, skeleton_ch);
}
}
fn is_default_ignorable(ch: char) -> bool {
matches!(
ch,
'\u{00ad}'
| '\u{034f}'
| '\u{061c}'
| '\u{180e}'
| '\u{200b}'..='\u{200f}'
| '\u{202a}'..='\u{202e}'
| '\u{2060}'..='\u{206f}'
| '\u{fe00}'..='\u{fe0f}'
| '\u{feff}'
| '\u{e0100}'..='\u{e01ef}'
)
}
fn enclosed_ascii(ch: char) -> Option<&'static str> {
let code = ch as u32;
if (0x1f150..=0x1f169).contains(&code) {
return ASCII_LOWERCASE.get((code - 0x1f150) as usize).copied();
}
if ch == '⓿' || ch == '🄌' {
return Some("0");
}
for (start, digits) in [
(0x2776, ASCII_DIGITS_1_TO_9),
(0x2780, ASCII_DIGITS_1_TO_9),
(0x278a, ASCII_DIGITS_1_TO_9),
] {
if (start..=start + 8).contains(&code) {
return digits.get((code - start) as usize).copied();
}
}
None
}
const ASCII_LOWERCASE: [&str; 26] = [
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w",
"x", "y", "z",
];
const ASCII_DIGITS_1_TO_9: [&str; 9] = ["1", "2", "3", "4", "5", "6", "7", "8", "9"];
fn url_punctuation_ascii(ch: char) -> Option<&'static str> {
match ch {
// URL punctuation variants that are not always useful in generic skeletons.
'。' | '' | '。' | '' | '‧' | '∙' | '●' | '•' => Some("."),
'' | '' | '' | '' => Some("/"),
'' | '' | 'ː' | '։' => Some(":"),
'' | '' | '' | '' | '—' | '―' | '' | '' | '' => Some("-"),
'_' | '_' => Some("_"),
_ => None,
}
}
fn scan_url_or_domain(value: &str) -> Vec<ContentPolicyMatch> {
let mut matches = Vec::new();
let chars = value.char_indices().collect::<Vec<_>>();
let mut index = 0;
while index < chars.len() {
let start = chars[index].0;
let remaining = &value[start..];
if remaining.starts_with("http://") || remaining.starts_with("https://") || remaining.starts_with("www.") {
let end = consume_non_space(value, start);
matches.push(match_result(value, start, end));
index = char_index_after(&chars, end);
continue;
}
if is_domain_boundary_before(value, start) {
let end = consume_domain(value, start);
if end > start && is_domain_candidate(&value[start..end]) && is_domain_boundary_after(value, end) {
matches.push(match_result(value, start, end));
index = char_index_after(&chars, end);
continue;
}
}
index += 1;
}
matches
}
fn match_result(value: &str, start: usize, end: usize) -> ContentPolicyMatch {
ContentPolicyMatch {
match_type: "url_or_domain".to_string(),
reason: "contains_url_or_domain".to_string(),
value: Some(value[start..end].to_string()),
span: Some(ContentPolicyMatchSpan {
start: start as u32,
end: end as u32,
}),
}
}
fn consume_non_space(value: &str, start: usize) -> usize {
value[start..]
.char_indices()
.find_map(|(offset, ch)| ch.is_whitespace().then_some(start + offset))
.unwrap_or(value.len())
}
fn consume_domain(value: &str, start: usize) -> usize {
value[start..]
.char_indices()
.find_map(|(offset, ch)| (!is_domain_char(ch)).then_some(start + offset))
.unwrap_or(value.len())
}
fn is_domain_candidate(candidate: &str) -> bool {
let labels = candidate.split('.').collect::<Vec<_>>();
if labels.len() < 2 {
return false;
}
let Some(tld) = labels.last() else {
return false;
};
if !(2..=63).contains(&tld.len()) || !tld.chars().all(|ch| ch.is_ascii_alphabetic()) {
return false;
}
labels.iter().all(|label| {
!label.is_empty()
&& label.len() <= 63
&& !label.starts_with('-')
&& !label.ends_with('-')
&& label.chars().all(|ch| ch.is_ascii_alphanumeric() || ch == '-')
})
}
fn is_domain_char(ch: char) -> bool {
ch.is_ascii_alphanumeric() || ch == '-' || ch == '.'
}
fn is_domain_boundary_before(value: &str, start: usize) -> bool {
if start == 0 {
return true;
}
let Some(previous) = value[..start].chars().next_back() else {
return true;
};
!(previous.is_ascii_alphanumeric() || previous == '-' || previous == '_' || previous == '@')
}
fn is_domain_boundary_after(value: &str, end: usize) -> bool {
if end >= value.len() {
return true;
}
let Some(next) = value[end..].chars().next() else {
return true;
};
!(next.is_ascii_alphanumeric() || next == '-' || next == '_' || next == '.')
}
fn char_index_after(chars: &[(usize, char)], byte_index: usize) -> usize {
chars
.iter()
.position(|(offset, _)| *offset >= byte_index)
.unwrap_or(chars.len())
}
#[cfg(test)]
mod tests {
use super::*;
fn scan(value: &str) -> ContentPolicyScanResult {
scan_content_policy_v1(ContentPolicyScanInput {
value: value.to_string(),
checks: None,
})
}
#[test]
fn scans_url_or_domain_cases() {
struct Case {
input: &'static str,
matched: bool,
skeleton: &'static str,
value: Option<&'static str>,
}
for case in [
Case {
input: "https://xamplecom",
matched: true,
skeleton: "https://example.com",
value: Some("https://example.com"),
},
Case {
input: "join 🅠⓿❶.example",
matched: true,
skeleton: "join q01.example",
value: Some("q01.example"),
},
Case {
input: "exa\u{200b}mple.com",
matched: true,
skeleton: "example.com",
value: Some("example.com"),
},
Case {
input: "hello workspace",
matched: false,
skeleton: "hello workspace",
value: None,
},
Case {
input: "user@example.com",
matched: false,
skeleton: "user@example.com",
value: None,
},
] {
let result = scan(case.input);
assert_eq!(result.matched, case.matched, "{}", case.input);
assert_eq!(result.skeleton, case.skeleton, "{}", case.input);
assert_eq!(
result.matches.first().and_then(|matched| matched.value.as_deref()),
case.value,
"{}",
case.input
);
}
}
}
+1
View File
@@ -2,6 +2,7 @@
mod utils;
pub mod content_policy;
pub mod doc;
pub mod doc_loader;
pub mod entitlement;
@@ -4,6 +4,7 @@ mod doc_compactor;
mod doc_storage;
mod gate;
mod housekeeping;
mod rolling_quota;
mod runtime_state;
#[cfg(test)]
mod tests;
@@ -17,8 +18,9 @@ use tokio::sync::Mutex;
use self::types::BackendRuntimeHealth;
pub(crate) use super::types;
use super::{
BackendRuntimeConfig, RuntimeError, RuntimeResult, migrations::migrate_runtime_tables, napi_error, to_napi_error,
pub(super) use super::{
BackendRuntimeConfig, InviteQuotaConfig, RuntimeError, RuntimeResult, migrations::migrate_runtime_tables, napi_error,
to_napi_error,
};
pub(super) fn token_hash(token: &str) -> String {
@@ -0,0 +1,261 @@
use napi::Result;
use sqlx::{PgPool, Row};
use super::{
BackendRuntime, RuntimeError, RuntimeInviteAbuseClaimedAction, RuntimeResult, napi_error, workspace_subject_key,
};
async fn invite_abuse_user_quarantined_or_banned(pool: &PgPool, user_id: &str) -> RuntimeResult<bool> {
let row: Option<i32> = sqlx::query_scalar(
r#"
SELECT 1
FROM runtime_invite_abuse_subjects
WHERE user_id = $1
AND status IN ('quarantined', 'banned')
LIMIT 1
"#,
)
.bind(user_id)
.fetch_optional(pool)
.await
.map_err(|err| RuntimeError::database("failed to load invite abuse subject by user", err))?;
Ok(row.is_some())
}
async fn invite_abuse_workspace_quarantined(pool: &PgPool, workspace_id: &str) -> RuntimeResult<bool> {
let row: Option<i32> = sqlx::query_scalar(
r#"
SELECT 1
FROM runtime_invite_abuse_subjects
WHERE subject_key = $1
AND status = 'quarantined'
LIMIT 1
"#,
)
.bind(workspace_subject_key(workspace_id))
.fetch_optional(pool)
.await
.map_err(|err| RuntimeError::database("failed to load invite abuse workspace subject", err))?;
Ok(row.is_some())
}
async fn claim_invite_abuse_action(pool: &PgPool, action_id: &str, worker_id: &str) -> RuntimeResult<bool> {
let result = sqlx::query(
r#"
UPDATE runtime_invite_abuse_actions action
SET status = 'running',
attempts = attempts + 1,
locked_by = $2,
locked_until = now() + interval '5 minutes',
last_error = NULL,
updated_at = now()
WHERE action.id = $1::bigint
AND action.action IN ('ban_actor', 'quarantine_actor', 'quarantine_workspace', 'quarantine_source_cohort')
AND (
(
action.status IN ('pending', 'retry_wait')
AND (action.next_attempt_at IS NULL OR action.next_attempt_at <= now())
)
OR (
action.status = 'running'
AND action.locked_until IS NOT NULL
AND action.locked_until <= now()
)
)
"#,
)
.bind(action_id)
.bind(worker_id)
.execute(pool)
.await
.map_err(|err| RuntimeError::database("failed to claim invite abuse action", err))?;
Ok(result.rows_affected() > 0)
}
async fn claim_retryable_invite_abuse_actions(
pool: &PgPool,
worker_id: &str,
limit: i64,
) -> RuntimeResult<Vec<RuntimeInviteAbuseClaimedAction>> {
let rows = sqlx::query(
r#"
WITH candidates AS (
SELECT action.id
FROM runtime_invite_abuse_actions action
JOIN runtime_invite_abuse_evidence evidence
ON evidence.id = action.evidence_id
WHERE (
(
action.status IN ('pending', 'retry_wait')
AND (action.next_attempt_at IS NULL OR action.next_attempt_at <= now())
)
OR (
action.status = 'running'
AND action.locked_until IS NOT NULL
AND action.locked_until <= now()
)
)
AND evidence.user_id IS NOT NULL
AND evidence.workspace_id IS NOT NULL
AND action.action IN ('ban_actor', 'quarantine_actor', 'quarantine_workspace', 'quarantine_source_cohort')
ORDER BY COALESCE(action.next_attempt_at, action.created_at), action.id
LIMIT $2
FOR UPDATE SKIP LOCKED
),
claimed AS (
UPDATE runtime_invite_abuse_actions action
SET status = 'running',
attempts = attempts + 1,
locked_by = $1,
locked_until = now() + interval '5 minutes',
last_error = NULL,
updated_at = now()
FROM candidates
WHERE action.id = candidates.id
RETURNING
action.id,
action.subject_key,
action.evidence_id,
action.action
)
SELECT
claimed.action,
claimed.subject_key,
claimed.evidence_id::text AS evidence_id,
claimed.id::text AS action_id,
evidence.user_id AS actor_user_id,
evidence.workspace_id
FROM claimed
JOIN runtime_invite_abuse_evidence evidence
ON evidence.id = claimed.evidence_id
"#,
)
.bind(worker_id)
.bind(limit)
.fetch_all(pool)
.await
.map_err(|err| RuntimeError::database("failed to claim retryable invite abuse actions", err))?;
Ok(
rows
.into_iter()
.map(|row| RuntimeInviteAbuseClaimedAction {
action: row.get("action"),
subject_key: row.get("subject_key"),
evidence_id: row.get("evidence_id"),
action_id: row.get("action_id"),
actor_user_id: row.get("actor_user_id"),
workspace_id: row.get("workspace_id"),
})
.collect(),
)
}
async fn mark_invite_abuse_action(
pool: &PgPool,
action_id: &str,
worker_id: &str,
status: &str,
error: Option<String>,
) -> RuntimeResult<bool> {
let result = match status {
"succeeded" => sqlx::query(
r#"
UPDATE runtime_invite_abuse_actions
SET status = 'succeeded',
next_attempt_at = NULL,
locked_by = NULL,
locked_until = NULL,
last_error = $3,
updated_at = now()
WHERE id = $1::bigint
AND status = 'running'
AND locked_by = $2
"#,
)
.bind(action_id)
.bind(worker_id)
.bind(error)
.execute(pool)
.await
.map_err(|err| RuntimeError::database("failed to mark invite abuse action succeeded", err))?,
"failed" => sqlx::query(
r#"
UPDATE runtime_invite_abuse_actions
SET status = CASE WHEN attempts >= 5 THEN 'failed' ELSE 'retry_wait' END,
next_attempt_at = CASE WHEN attempts >= 5 THEN next_attempt_at ELSE now() + interval '5 minutes' END,
locked_by = NULL,
locked_until = NULL,
last_error = $3,
updated_at = now()
WHERE id = $1::bigint
AND status = 'running'
AND locked_by = $2
"#,
)
.bind(action_id)
.bind(worker_id)
.bind(error)
.execute(pool)
.await
.map_err(|err| RuntimeError::database("failed to mark invite abuse action failed", err))?,
_ => return Err(RuntimeError::invalid_input("invalid invite abuse action status")),
};
Ok(result.rows_affected() > 0)
}
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub async fn is_invite_abuse_user_quarantined_or_banned(&self, user_id: String) -> Result<bool> {
let pool = self.pool().await?;
invite_abuse_user_quarantined_or_banned(&pool, &user_id)
.await
.map_err(Into::into)
}
#[napi]
pub async fn is_invite_abuse_workspace_quarantined(&self, workspace_id: String) -> Result<bool> {
let pool = self.pool().await?;
invite_abuse_workspace_quarantined(&pool, &workspace_id)
.await
.map_err(Into::into)
}
#[napi]
pub async fn claim_invite_abuse_action(&self, action_id: String, worker_id: String) -> Result<bool> {
let pool = self.pool().await?;
claim_invite_abuse_action(&pool, &action_id, &worker_id)
.await
.map_err(Into::into)
}
#[napi]
pub async fn claim_retryable_invite_abuse_actions(
&self,
worker_id: String,
limit: i64,
) -> Result<Vec<RuntimeInviteAbuseClaimedAction>> {
if limit <= 0 {
return Err(napi_error("invite abuse action claim limit must be positive"));
}
let pool = self.pool().await?;
claim_retryable_invite_abuse_actions(&pool, &worker_id, limit)
.await
.map_err(Into::into)
}
#[napi]
pub async fn mark_invite_abuse_action(
&self,
action_id: String,
worker_id: String,
status: String,
error: Option<String>,
) -> Result<bool> {
let pool = self.pool().await?;
mark_invite_abuse_action(&pool, &action_id, &worker_id, &status, error)
.await
.map_err(Into::into)
}
}
@@ -0,0 +1,172 @@
use super::{
InviteQuotaConfig, QuotaViolation, RuntimeMailDeliveryQuotaDecision, RuntimeMailDeliveryQuotaInput, ScopeLimit,
high_risk_domain, normalize_domain, scope, short_hash, source_prefix,
};
#[cfg(test)]
use super::{RuntimeMailDeliveryQuotaMetadataInput, RuntimeMailDeliveryQuotaRecipientInput, RuntimeQuotaSourceInput};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum MailClass {
Auth,
WorkspaceInvitation,
CollaborationNotice,
WorkspaceLifecycle,
BillingLicense,
}
impl MailClass {
pub(super) fn as_str(self) -> &'static str {
match self {
Self::Auth => "auth",
Self::WorkspaceInvitation => "workspace_invitation",
Self::CollaborationNotice => "collaboration_notice",
Self::WorkspaceLifecycle => "workspace_lifecycle",
Self::BillingLicense => "billing_license",
}
}
}
pub(super) fn mail_class(mail_name: &str, config: &InviteQuotaConfig) -> Option<MailClass> {
match config.mail_class_mapping.get(mail_name).map(String::as_str) {
Some("auth") => Some(MailClass::Auth),
Some("workspace_invitation") => Some(MailClass::WorkspaceInvitation),
Some("collaboration_notice") => Some(MailClass::CollaborationNotice),
Some("workspace_lifecycle") => Some(MailClass::WorkspaceLifecycle),
Some("billing_license") => Some(MailClass::BillingLicense),
_ => None,
}
}
pub(super) fn build_mail_scopes(
input: &RuntimeMailDeliveryQuotaInput,
class: MailClass,
config: &InviteQuotaConfig,
) -> Vec<ScopeLimit> {
let recipient_hash = short_hash(&input.recipient.email.trim().to_ascii_lowercase());
let domain = normalize_domain(&input.recipient.domain);
let class_name = class.as_str();
let mut scopes = vec![
scope(
format!("mail:recipient:{recipient_hash}:class:{class_name}"),
3600,
20,
1,
),
scope(
format!("mail:recipient_domain:{domain}:class:{class_name}"),
3600,
250,
1,
),
scope("mail:provider_global:default".to_string(), 60, 500, 1),
];
match class {
MailClass::Auth => {
if let Some(prefix) = source_prefix(input.source.as_ref()) {
scopes.push(scope(format!("mail:source_prefix:{prefix}:class:auth"), 3600, 50, 1));
}
}
MailClass::WorkspaceInvitation => {
if let Some(prefix) = source_prefix(input.source.as_ref()) {
scopes.push(scope(
format!("mail:source_prefix_domain:{prefix}:{domain}:class:{class_name}"),
3600,
if high_risk_domain(&domain, config) { 10 } else { 50 },
1,
));
}
if let Some(subject) = input.metadata.abuse_subject_key.as_deref() {
scopes.push(scope(
format!("mail:abuse_subject:{subject}:class:{class_name}"),
86_400,
0,
1,
));
}
}
MailClass::CollaborationNotice | MailClass::WorkspaceLifecycle => {
if let Some(actor) = input.metadata.actor_user_id.as_deref() {
scopes.push(scope(format!("mail:actor:{actor}:class:{class_name}"), 3600, 200, 1));
}
if let Some(workspace) = input.metadata.workspace_id.as_deref() {
scopes.push(scope(
format!("mail:workspace:{workspace}:class:{class_name}"),
3600,
1000,
1,
));
}
}
MailClass::BillingLicense => {}
}
scopes
}
pub(super) fn decision_from_violation(
violation: QuotaViolation,
class: MailClass,
reason: &str,
) -> RuntimeMailDeliveryQuotaDecision {
RuntimeMailDeliveryQuotaDecision {
allowed: false,
reservation_id: None,
mail_class: class.as_str().to_string(),
retry_after_seconds: Some(60),
reason: Some(reason.to_string()),
scope_key: Some(violation.scope_key),
window_seconds: Some(violation.window_seconds),
limit: Some(violation.limit),
current: Some(violation.current),
requested: Some(violation.requested),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn invite_config() -> InviteQuotaConfig {
InviteQuotaConfig::default()
}
#[test]
fn mail_class_mapping_covers_known_renderers_and_fails_closed() {
let config = invite_config();
for (name, class) in [
("SignIn", MailClass::Auth),
("MemberInvitation", MailClass::WorkspaceInvitation),
("CommentMention", MailClass::CollaborationNotice),
("TeamWorkspaceExpired", MailClass::WorkspaceLifecycle),
("TeamLicense", MailClass::BillingLicense),
] {
assert_eq!(mail_class(name, &config), Some(class), "{name}");
}
assert_eq!(mail_class("TestMail", &config), None);
assert_eq!(mail_class("UnexpectedMail", &config), None);
}
#[test]
fn auth_mail_does_not_expand_actor_or_workspace_scopes() {
let input = RuntimeMailDeliveryQuotaInput {
request_id: None,
mail_name: "SignIn".to_string(),
recipient: RuntimeMailDeliveryQuotaRecipientInput {
email: "user@example.com".to_string(),
domain: "example.com".to_string(),
user_id: Some("u1".to_string()),
},
metadata: RuntimeMailDeliveryQuotaMetadataInput {
actor_user_id: Some("actor".to_string()),
workspace_id: Some("workspace".to_string()),
notification_id: None,
abuse_subject_key: None,
},
source: None::<RuntimeQuotaSourceInput>,
};
let scopes = build_mail_scopes(&input, MailClass::Auth, &invite_config());
assert!(scopes.iter().all(|scope| !scope.scope_key.contains("actor")));
assert!(scopes.iter().all(|scope| !scope.scope_key.contains("workspace")));
}
}
@@ -0,0 +1,175 @@
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
mod invite_abuse_actions;
mod mail_delivery;
mod reservation;
mod workspace_invite;
mod workspace_invite_policy;
use mail_delivery::{build_mail_scopes, decision_from_violation as mail_decision_from_violation, mail_class};
use napi::Result;
use reservation::{
QuotaViolation, ScopeLimit, bucket_seconds, cleanup_expired, commit_reservation, release_reservation, reserve_scopes,
scope,
};
use sha2::{Digest, Sha256};
use workspace_invite_policy::{
ActorFacts, InviteAbuseDecision, InviteActivityFacts, QuotaFacts, WorkspaceFacts, build_invite_scopes,
evaluate_projection, high_confidence_invite_abuse, invite_commit_usage_for_scope, source_cohort_subject_key,
subject_hash, sum_domains,
};
#[cfg(test)]
pub(super) use super::types::{RuntimeMailDeliveryQuotaMetadataInput, RuntimeMailDeliveryQuotaRecipientInput};
pub(super) use super::{
BackendRuntime, InviteQuotaConfig, RuntimeError, RuntimeResult, napi_error,
types::{
RuntimeInviteAbuseActionRequired, RuntimeInviteAbuseClaimedAction, RuntimeMailDeliveryQuotaDecision,
RuntimeMailDeliveryQuotaInput, RuntimeQuotaSourceInput, RuntimeQuotaTargetDomainInput,
RuntimeWorkspaceInviteQuotaDecision, RuntimeWorkspaceInviteQuotaInput, RuntimeWorkspaceInviteQuotaUsage,
},
};
pub(super) fn normalize_domain(domain: &str) -> String {
domain.trim().trim_end_matches('.').to_ascii_lowercase()
}
pub(super) fn high_risk_domain(domain: &str, config: &InviteQuotaConfig) -> bool {
let domain = normalize_domain(domain);
config
.high_risk_target_domains
.iter()
.any(|configured| normalize_domain(configured) == domain)
}
pub(super) fn workspace_subject_key(workspace_id: &str) -> String {
format!("workspace:v1:{}", short_hash(workspace_id))
}
pub(super) fn short_hash(value: &str) -> String {
hex::encode(Sha256::digest(value.as_bytes()))[..24].to_string()
}
pub(super) fn source_prefix(source: Option<&RuntimeQuotaSourceInput>) -> Option<String> {
let source = source?;
if !source.trusted {
return None;
}
let ip = source.ip.as_ref()?.parse::<IpAddr>().ok()?;
match ip {
IpAddr::V4(ip) => {
let octets = ip.octets();
Some(Ipv4Addr::new(octets[0], octets[1], octets[2], 0).to_string() + "/24")
}
IpAddr::V6(ip) => {
let segments = ip.segments();
Some(Ipv6Addr::new(segments[0], segments[1], segments[2], 0, 0, 0, 0, 0).to_string() + "/48")
}
}
}
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub async fn assert_mail_delivery_quota_v1(
&self,
input: RuntimeMailDeliveryQuotaInput,
) -> Result<RuntimeMailDeliveryQuotaDecision> {
let config = self.config()?.invite_quota;
let Some(class) = mail_class(&input.mail_name, &config) else {
return Ok(RuntimeMailDeliveryQuotaDecision {
allowed: false,
reservation_id: None,
mail_class: "unmapped".to_string(),
retry_after_seconds: None,
reason: Some("unmapped_mail_name".to_string()),
scope_key: Some(format!("mail:name:{}", input.mail_name)),
window_seconds: None,
limit: None,
current: None,
requested: Some(1),
});
};
let pool = self.pool().await?;
let scopes = build_mail_scopes(&input, class, &config);
match reserve_scopes(&pool, "mail_delivery", input.request_id.as_deref(), scopes).await? {
Ok(reservation) => Ok(RuntimeMailDeliveryQuotaDecision {
allowed: true,
reservation_id: Some(reservation.reservation_id),
mail_class: class.as_str().to_string(),
retry_after_seconds: None,
reason: None,
scope_key: None,
window_seconds: None,
limit: None,
current: None,
requested: Some(1),
}),
Err(violation) => Ok(mail_decision_from_violation(violation, class, "mail_class")),
}
}
#[napi]
pub async fn commit_mail_delivery_quota_v1(&self, reservation_id: String) -> Result<bool> {
let pool = self.pool().await?;
commit_reservation(&pool, &reservation_id, 1, |_, reserved_count| reserved_count.min(1))
.await
.map_err(Into::into)
}
#[napi]
pub async fn release_mail_delivery_quota_v1(&self, reservation_id: String) -> Result<bool> {
let pool = self.pool().await?;
release_reservation(&pool, &reservation_id).await.map_err(Into::into)
}
#[napi]
pub async fn cleanup_expired_rolling_quota(&self, limit: i64) -> Result<i64> {
if limit <= 0 {
return Err(napi_error("rolling quota cleanup limit must be positive"));
}
let pool = self.pool().await?;
cleanup_expired(&pool, limit).await.map_err(Into::into)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn derives_source_prefix_only_from_trusted_raw_ip() {
assert_eq!(
source_prefix(Some(&RuntimeQuotaSourceInput {
trusted: true,
ip: Some("192.168.12.34".to_string()),
country: None,
asn: None,
ray_id: None,
}))
.as_deref(),
Some("192.168.12.0/24")
);
assert_eq!(
source_prefix(Some(&RuntimeQuotaSourceInput {
trusted: true,
ip: Some("2001:db8:abcd:1234::1".to_string()),
country: None,
asn: None,
ray_id: None,
}))
.as_deref(),
Some("2001:db8:abcd::/48")
);
assert!(
source_prefix(Some(&RuntimeQuotaSourceInput {
trusted: false,
ip: Some("192.168.12.34".to_string()),
country: None,
asn: None,
ray_id: None,
}))
.is_none()
);
}
}
@@ -0,0 +1,344 @@
use chrono::{DateTime, Duration, TimeZone, Utc};
use sha2::{Digest, Sha256};
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
use super::{RuntimeError, RuntimeResult};
const RESERVATION_TTL_SECONDS: i64 = 120;
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct ScopeLimit {
pub(super) scope_key: String,
pub(super) window_seconds: i32,
pub(super) bucket_seconds: i64,
pub(super) limit: i32,
pub(super) requested: i32,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct QuotaViolation {
pub(super) scope_key: String,
pub(super) window_seconds: i32,
pub(super) limit: i32,
pub(super) current: i32,
pub(super) requested: i32,
}
#[derive(Clone, Debug)]
pub(super) struct ReservationDecision {
pub(super) reservation_id: String,
}
pub(super) fn bucket_seconds(window_seconds: i32) -> i64 {
match window_seconds {
60 => 10,
3600 => 5 * 60,
86_400 => 60 * 60,
604_800 => 6 * 60 * 60,
_ => 60,
}
}
fn bucket_start(now: DateTime<Utc>, bucket_seconds: i64) -> DateTime<Utc> {
let timestamp = now.timestamp();
Utc
.timestamp_opt(timestamp - timestamp.rem_euclid(bucket_seconds), 0)
.single()
.expect("valid unix timestamp bucket")
}
pub(super) fn scope(scope_key: String, window_seconds: i32, limit: i32, requested: i32) -> ScopeLimit {
ScopeLimit {
scope_key,
window_seconds,
bucket_seconds: bucket_seconds(window_seconds),
limit,
requested,
}
}
pub(super) async fn reserve_scopes(
pool: &PgPool,
purpose: &str,
request_id: Option<&str>,
scopes: Vec<ScopeLimit>,
) -> RuntimeResult<std::result::Result<ReservationDecision, QuotaViolation>> {
let now: DateTime<Utc> = sqlx::query_scalar("SELECT clock_timestamp()")
.fetch_one(pool)
.await
.map_err(|err| RuntimeError::database("failed to read database clock", err))?;
let mut tx = pool
.begin()
.await
.map_err(|err| RuntimeError::database("failed to begin quota reservation transaction", err))?;
let mut sorted_scope_keys = std::collections::BTreeSet::new();
for scope in &scopes {
sorted_scope_keys.insert(scope.scope_key.clone());
}
for scope_key in sorted_scope_keys {
advisory_lock(&mut tx, &scope_key).await?;
}
let reservation_id = Uuid::new_v4();
for scope in &scopes {
let bucket_start = bucket_start(now, scope.bucket_seconds);
let window_start = now - Duration::seconds(scope.window_seconds as i64);
let committed: i64 = sqlx::query_scalar(
r#"
SELECT COALESCE(SUM(count), 0)::bigint
FROM runtime_rolling_quota_counters
WHERE scope_key = $1
AND window_seconds = $2
AND bucket_start >= $3
"#,
)
.bind(&scope.scope_key)
.bind(scope.window_seconds)
.bind(window_start)
.fetch_one(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to read rolling quota counter", err))?;
let reserved: i64 = sqlx::query_scalar(
r#"
SELECT COALESCE(SUM(count), 0)::bigint
FROM runtime_rolling_quota_reservations
WHERE scope_key = $1
AND window_seconds = $2
AND bucket_start >= $3
AND status = 'reserved'
AND expires_at > $4
"#,
)
.bind(&scope.scope_key)
.bind(scope.window_seconds)
.bind(window_start)
.bind(now)
.fetch_one(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to read rolling quota reservations", err))?;
let current = committed.saturating_add(reserved) as i32;
if current.saturating_add(scope.requested) > scope.limit {
tx.rollback()
.await
.map_err(|err| RuntimeError::database("failed to rollback rejected quota reservation", err))?;
return Ok(Err(QuotaViolation {
scope_key: scope.scope_key.clone(),
window_seconds: scope.window_seconds,
limit: scope.limit,
current,
requested: scope.requested,
}));
}
sqlx::query(
r#"
INSERT INTO runtime_rolling_quota_reservations (
id, scope_key, window_seconds, bucket_start, count, status, purpose, request_id, expires_at
)
VALUES ($1::uuid, $2, $3, $4, $5, 'reserved', $6, $7, $8)
"#,
)
.bind(reservation_id.to_string())
.bind(&scope.scope_key)
.bind(scope.window_seconds)
.bind(bucket_start)
.bind(scope.requested)
.bind(purpose)
.bind(request_id)
.bind(now + Duration::seconds(RESERVATION_TTL_SECONDS))
.execute(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to create rolling quota reservation", err))?;
}
tx.commit()
.await
.map_err(|err| RuntimeError::database("failed to commit quota reservation", err))?;
Ok(Ok(ReservationDecision {
reservation_id: reservation_id.to_string(),
}))
}
async fn advisory_lock(tx: &mut Transaction<'_, Postgres>, scope_key: &str) -> RuntimeResult<()> {
let hash = Sha256::digest(scope_key.as_bytes());
let mut bytes = [0_u8; 8];
bytes.copy_from_slice(&hash[..8]);
let lock_key = i64::from_be_bytes(bytes);
sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(lock_key)
.execute(&mut **tx)
.await
.map_err(|err| RuntimeError::database("failed to acquire rolling quota advisory lock", err))?;
Ok(())
}
pub(super) async fn commit_reservation<F>(
pool: &PgPool,
reservation_id: &str,
settle_usage: i32,
actual_usage_for_scope: F,
) -> RuntimeResult<bool>
where
F: Fn(&str, i32) -> i32,
{
let reservation_id =
Uuid::parse_str(reservation_id).map_err(|_| RuntimeError::invalid_input("invalid reservation id"))?;
let reservation_id = reservation_id.to_string();
let now: DateTime<Utc> = sqlx::query_scalar("SELECT clock_timestamp()")
.fetch_one(pool)
.await
.map_err(|err| RuntimeError::database("failed to read database clock", err))?;
let mut tx = pool
.begin()
.await
.map_err(|err| RuntimeError::database("failed to begin quota commit transaction", err))?;
let rows = sqlx::query(
r#"
SELECT scope_key, window_seconds, bucket_start, count
FROM runtime_rolling_quota_reservations
WHERE id = $1::uuid AND status = 'reserved'
FOR UPDATE
"#,
)
.bind(&reservation_id)
.fetch_all(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to load quota reservation", err))?;
if rows.is_empty() {
tx.rollback()
.await
.map_err(|err| RuntimeError::database("failed to rollback empty quota commit", err))?;
return Ok(false);
}
for row in &rows {
advisory_lock(&mut tx, row.get("scope_key")).await?;
}
for row in &rows {
let reserved_count: i32 = row.get("count");
let scope_key: String = row.get("scope_key");
let count = actual_usage_for_scope(&scope_key, reserved_count)
.min(reserved_count)
.max(0);
if count > 0 {
let window_seconds: i32 = row.get("window_seconds");
sqlx::query(
r#"
INSERT INTO runtime_rolling_quota_counters (scope_key, window_seconds, bucket_start, count, expires_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (scope_key, window_seconds, bucket_start)
DO UPDATE SET count = runtime_rolling_quota_counters.count + EXCLUDED.count,
expires_at = GREATEST(runtime_rolling_quota_counters.expires_at, EXCLUDED.expires_at),
updated_at = EXCLUDED.updated_at
"#,
)
.bind(scope_key)
.bind(window_seconds)
.bind(row.get::<DateTime<Utc>, _>("bucket_start"))
.bind(count)
.bind(now + Duration::seconds(window_seconds as i64 * 2))
.bind(now)
.execute(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to commit rolling quota counter", err))?;
}
}
sqlx::query(
r#"
UPDATE runtime_rolling_quota_reservations
SET status = CASE WHEN $2 > 0 THEN 'committed' ELSE 'released' END,
committed_at = CASE WHEN $2 > 0 THEN $3 ELSE committed_at END,
released_at = CASE WHEN $2 <= 0 THEN $3 ELSE released_at END,
updated_at = $3
WHERE id = $1::uuid AND status = 'reserved'
"#,
)
.bind(&reservation_id)
.bind(settle_usage)
.bind(now)
.execute(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to settle quota reservation", err))?;
tx.commit()
.await
.map_err(|err| RuntimeError::database("failed to commit quota commit transaction", err))?;
Ok(true)
}
pub(super) async fn release_reservation(pool: &PgPool, reservation_id: &str) -> RuntimeResult<bool> {
let reservation_id =
Uuid::parse_str(reservation_id).map_err(|_| RuntimeError::invalid_input("invalid reservation id"))?;
let reservation_id = reservation_id.to_string();
let result = sqlx::query(
r#"
UPDATE runtime_rolling_quota_reservations
SET status = 'released',
released_at = clock_timestamp(),
updated_at = clock_timestamp()
WHERE id = $1::uuid AND status = 'reserved'
"#,
)
.bind(reservation_id)
.execute(pool)
.await
.map_err(|err| RuntimeError::database("failed to release quota reservation", err))?;
Ok(result.rows_affected() > 0)
}
pub(super) async fn cleanup_expired(pool: &PgPool, limit: i64) -> RuntimeResult<i64> {
let released = sqlx::query(
r#"
UPDATE runtime_rolling_quota_reservations
SET status = 'expired',
updated_at = clock_timestamp()
WHERE (id, scope_key, window_seconds, bucket_start) IN (
SELECT id, scope_key, window_seconds, bucket_start
FROM runtime_rolling_quota_reservations
WHERE status = 'reserved' AND expires_at <= clock_timestamp()
LIMIT $1
)
"#,
)
.bind(limit)
.execute(pool)
.await
.map_err(|err| RuntimeError::database("failed to expire rolling quota reservations", err))?
.rows_affected() as i64;
let counters = sqlx::query(
r#"
DELETE FROM runtime_rolling_quota_counters
WHERE (scope_key, window_seconds, bucket_start) IN (
SELECT scope_key, window_seconds, bucket_start
FROM runtime_rolling_quota_counters
WHERE expires_at <= clock_timestamp()
LIMIT $1
)
"#,
)
.bind(limit)
.execute(pool)
.await
.map_err(|err| RuntimeError::database("failed to delete expired rolling quota counters", err))?
.rows_affected() as i64;
Ok(released + counters)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn bucket_boundaries_match_policy_windows() {
let now = Utc.with_ymd_and_hms(2026, 7, 6, 1, 2, 3).single().unwrap();
assert_eq!(bucket_seconds(60), 10);
assert_eq!(bucket_seconds(3600), 300);
assert_eq!(bucket_seconds(86_400), 3600);
assert_eq!(bucket_seconds(604_800), 21_600);
assert_eq!(
bucket_start(now, 300),
Utc.with_ymd_and_hms(2026, 7, 6, 1, 0, 0).single().unwrap()
);
}
}
@@ -0,0 +1,452 @@
use chrono::{DateTime, Utc};
use napi::Result;
use serde_json::json;
use sqlx::{PgPool, Row};
use super::{
ActorFacts, BackendRuntime, InviteAbuseDecision, InviteActivityFacts, InviteQuotaConfig, QuotaFacts, QuotaViolation,
RuntimeError, RuntimeInviteAbuseActionRequired, RuntimeResult, RuntimeWorkspaceInviteQuotaDecision,
RuntimeWorkspaceInviteQuotaInput, RuntimeWorkspaceInviteQuotaUsage, WorkspaceFacts, build_invite_scopes,
commit_reservation, evaluate_projection, high_confidence_invite_abuse, invite_commit_usage_for_scope, napi_error,
normalize_domain, release_reservation, reserve_scopes, short_hash, source_cohort_subject_key, source_prefix,
subject_hash, sum_domains, workspace_subject_key,
};
async fn load_actor(pool: &PgPool, user_id: &str) -> RuntimeResult<ActorFacts> {
let row = sqlx::query(
r#"
SELECT email, created_at, registered, email_verified IS NOT NULL AS email_verified, disabled
FROM users
WHERE id = $1
"#,
)
.bind(user_id)
.fetch_optional(pool)
.await
.map_err(|err| RuntimeError::database("failed to load invite actor", err))?
.ok_or_else(|| RuntimeError::invalid_input("invite actor not found"))?;
Ok(ActorFacts {
email: row.get("email"),
created_at: row.get("created_at"),
registered: row.get("registered"),
email_verified: row.get("email_verified"),
disabled: row.get("disabled"),
})
}
async fn load_workspace(pool: &PgPool, workspace_id: &str) -> RuntimeResult<WorkspaceFacts> {
let row = sqlx::query("SELECT created_at FROM workspaces WHERE id = $1")
.bind(workspace_id)
.fetch_optional(pool)
.await
.map_err(|err| RuntimeError::database("failed to load workspace", err))?
.ok_or_else(|| RuntimeError::invalid_input("workspace not found"))?;
Ok(WorkspaceFacts {
created_at: row.get("created_at"),
})
}
async fn load_invite_activity(
pool: &PgPool,
actor_user_id: &str,
workspace_id: &str,
) -> RuntimeResult<InviteActivityFacts> {
let row = sqlx::query(
r#"
SELECT
COUNT(*) FILTER (
WHERE inviter_user_id = $1 AND created_at >= clock_timestamp() - interval '7 days'
)::int AS actor_created_7d,
COUNT(*) FILTER (
WHERE inviter_user_id = $1 AND accepted_at >= clock_timestamp() - interval '7 days'
)::int AS actor_accepted_7d,
COUNT(*) FILTER (
WHERE workspace_id = $2 AND status IN ('pending', 'waiting_review', 'waiting_seat')
)::int AS workspace_pending,
COUNT(*) FILTER (
WHERE workspace_id = $2 AND created_at >= clock_timestamp() - interval '7 days'
)::int AS workspace_created_7d,
COUNT(*) FILTER (
WHERE workspace_id = $2 AND accepted_at >= clock_timestamp() - interval '7 days'
)::int AS workspace_accepted_7d
FROM workspace_invitations
WHERE inviter_user_id = $1 OR workspace_id = $2
"#,
)
.bind(actor_user_id)
.bind(workspace_id)
.fetch_one(pool)
.await
.map_err(|err| RuntimeError::database("failed to load invite activity facts", err))?;
Ok(InviteActivityFacts {
actor_created_7d: row.get("actor_created_7d"),
actor_accepted_7d: row.get("actor_accepted_7d"),
workspace_pending: row.get("workspace_pending"),
workspace_created_7d: row.get("workspace_created_7d"),
workspace_accepted_7d: row.get("workspace_accepted_7d"),
})
}
async fn load_quota(pool: &PgPool, workspace_id: &str) -> RuntimeResult<Option<QuotaFacts>> {
let row = sqlx::query(
r#"
SELECT plan, owner_user_id, uses_owner_quota, seat_limit, member_count, known, stale, stale_after
FROM effective_workspace_quota_states
WHERE workspace_id = $1
"#,
)
.bind(workspace_id)
.fetch_optional(pool)
.await
.map_err(|err| RuntimeError::database("failed to load workspace quota state", err))?;
Ok(row.map(|row| QuotaFacts {
plan: row.get("plan"),
owner_user_id: row.get("owner_user_id"),
uses_owner_quota: row.get("uses_owner_quota"),
seat_limit: row.get("seat_limit"),
member_count: row.get("member_count"),
known: row.get("known"),
stale: row.get("stale"),
stale_after: row.get("stale_after"),
}))
}
async fn active_subject_status(pool: &PgPool, subject_key: &str) -> RuntimeResult<Option<String>> {
let row = sqlx::query("SELECT status FROM runtime_invite_abuse_subjects WHERE subject_key = $1")
.bind(subject_key)
.fetch_optional(pool)
.await
.map_err(|err| RuntimeError::database("failed to load invite abuse subject", err))?;
Ok(row.map(|row| row.get("status")))
}
async fn record_invite_abuse_action(
pool: &PgPool,
input: &RuntimeWorkspaceInviteQuotaInput,
actor: &ActorFacts,
decision: InviteAbuseDecision,
config: &InviteQuotaConfig,
) -> RuntimeResult<RuntimeInviteAbuseActionRequired> {
let action = decision.action;
let status = if action == "ban_actor" { "banned" } else { "quarantined" };
let source_prefix_hash = source_prefix(input.source.as_ref()).map(|prefix| short_hash(&prefix));
let target_domains = json!(
input
.target_domains
.iter()
.map(|target| json!({
"domain": normalize_domain(&target.domain),
"count": target.count,
}))
.collect::<Vec<_>>()
);
let counters = json!({
"requested": input.target_count,
});
let actor_email_hash = subject_hash(&actor.email, config);
let actor_domain = actor.email.split('@').next_back().map(normalize_domain);
let source_asn = input.source.as_ref().and_then(|source| source.asn).map(i64::from);
let subject_user_id = if decision.subject_kind == "actor_email" {
Some(input.actor_user_id.as_str())
} else {
None
};
let mut tx = pool
.begin()
.await
.map_err(|err| RuntimeError::database("failed to start invite abuse transaction", err))?;
sqlx::query(
r#"
INSERT INTO runtime_invite_abuse_subjects (
subject_key,
kind,
user_id,
actor_email_hash,
email_domain,
first_seen_at,
last_seen_at,
status,
action,
action_reason,
action_at
)
VALUES ($1, $2, $3, $4, $5, now(), now(), $6, $7, $8, now())
ON CONFLICT (subject_key)
DO UPDATE SET
user_id = EXCLUDED.user_id,
actor_email_hash = EXCLUDED.actor_email_hash,
email_domain = EXCLUDED.email_domain,
last_seen_at = now(),
status = EXCLUDED.status,
action = EXCLUDED.action,
action_reason = EXCLUDED.action_reason,
action_at = now(),
updated_at = now()
"#,
)
.bind(&decision.subject_key)
.bind(decision.subject_kind)
.bind(subject_user_id)
.bind(actor_email_hash)
.bind(actor_domain)
.bind(status)
.bind(action)
.bind(decision.reason)
.execute(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to upsert invite abuse subject", err))?;
let evidence_id: i64 = sqlx::query_scalar(
r#"
INSERT INTO runtime_invite_abuse_evidence (
subject_key,
request_id,
workspace_id,
user_id,
actor_email_hash,
source_prefix_hash,
source_asn,
target_domains,
counters,
decision,
reason
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
RETURNING id
"#,
)
.bind(&decision.subject_key)
.bind(&input.request_id)
.bind(&input.workspace_id)
.bind(&input.actor_user_id)
.bind(subject_hash(&actor.email, config))
.bind(source_prefix_hash)
.bind(source_asn)
.bind(target_domains)
.bind(counters)
.bind(action)
.bind(decision.reason)
.fetch_one(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to insert invite abuse evidence", err))?;
let action_id: i64 = sqlx::query_scalar(
r#"
INSERT INTO runtime_invite_abuse_actions (
subject_key,
evidence_id,
action,
status,
next_attempt_at
)
VALUES ($1, $2, $3, 'pending', now())
RETURNING id
"#,
)
.bind(&decision.subject_key)
.bind(evidence_id)
.bind(action)
.fetch_one(&mut *tx)
.await
.map_err(|err| RuntimeError::database("failed to insert invite abuse action", err))?;
tx.commit()
.await
.map_err(|err| RuntimeError::database("failed to commit invite abuse transaction", err))?;
Ok(RuntimeInviteAbuseActionRequired {
action: action.to_string(),
subject_key: decision.subject_key,
evidence_id: evidence_id.to_string(),
action_id: action_id.to_string(),
})
}
fn decision_from_violation(violation: QuotaViolation, reason: &str) -> RuntimeWorkspaceInviteQuotaDecision {
RuntimeWorkspaceInviteQuotaDecision {
allowed: false,
reservation_id: None,
retry_after_seconds: Some(60),
reason: Some(reason.to_string()),
scope_key: Some(violation.scope_key),
window_seconds: Some(violation.window_seconds),
limit: Some(violation.limit),
current: Some(violation.current),
requested: Some(violation.requested),
action_required: None,
}
}
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub async fn assert_workspace_invite_quota_v1(
&self,
input: RuntimeWorkspaceInviteQuotaInput,
) -> Result<RuntimeWorkspaceInviteQuotaDecision> {
if input.target_count <= 0 {
return Err(napi_error("target_count must be positive"));
}
let config = self.config()?.invite_quota;
let pool = self.pool().await?;
let now: DateTime<Utc> = sqlx::query_scalar("SELECT clock_timestamp()")
.fetch_one(&pool)
.await
.map_err(|err| RuntimeError::database("failed to read database clock", err))?;
let actor = load_actor(&pool, &input.actor_user_id).await?;
let actor_subject = subject_hash(&actor.email, &config);
if let Some(status) = active_subject_status(&pool, &actor_subject).await?
&& matches!(status.as_str(), "banned" | "quarantined")
{
return Ok(RuntimeWorkspaceInviteQuotaDecision {
allowed: false,
reservation_id: None,
retry_after_seconds: None,
reason: Some("abuse_subject".to_string()),
scope_key: Some(format!("invite:actor_subject:{actor_subject}")),
window_seconds: None,
limit: None,
current: None,
requested: Some(input.target_count),
action_required: None,
});
}
let workspace_subject = workspace_subject_key(&input.workspace_id);
if let Some(status) = active_subject_status(&pool, &workspace_subject).await?
&& status == "quarantined"
{
return Ok(RuntimeWorkspaceInviteQuotaDecision {
allowed: false,
reservation_id: None,
retry_after_seconds: None,
reason: Some("abuse_workspace".to_string()),
scope_key: Some(format!("invite:workspace_subject:{workspace_subject}")),
window_seconds: None,
limit: None,
current: None,
requested: Some(input.target_count),
action_required: None,
});
}
if let Some(prefix) = source_prefix(input.source.as_ref()) {
for target in &input.target_domains {
let source_subject = source_cohort_subject_key(&prefix, &target.domain);
if let Some(status) = active_subject_status(&pool, &source_subject).await?
&& status == "quarantined"
{
return Ok(RuntimeWorkspaceInviteQuotaDecision {
allowed: false,
reservation_id: None,
retry_after_seconds: None,
reason: Some("abuse_source_cohort".to_string()),
scope_key: Some(format!("invite:source_cohort_subject:{source_subject}")),
window_seconds: None,
limit: None,
current: None,
requested: Some(input.target_count),
action_required: None,
});
}
}
}
let quota = match load_quota(&pool, &input.workspace_id).await? {
Some(quota) => quota,
None => {
return Ok(RuntimeWorkspaceInviteQuotaDecision {
allowed: false,
reservation_id: None,
retry_after_seconds: None,
reason: Some("quota_state_unavailable".to_string()),
scope_key: None,
window_seconds: None,
limit: None,
current: None,
requested: Some(input.target_count),
action_required: None,
});
}
};
if let Some(reason) = evaluate_projection(&quota, now) {
return Ok(RuntimeWorkspaceInviteQuotaDecision {
allowed: false,
reservation_id: None,
retry_after_seconds: None,
reason: Some(reason.to_string()),
scope_key: None,
window_seconds: None,
limit: None,
current: None,
requested: Some(input.target_count),
action_required: None,
});
}
if let Some(abuse_decision) = high_confidence_invite_abuse(&input, &actor, &config) {
let reason = abuse_decision.reason;
let scope_key = match abuse_decision.subject_kind {
"workspace" => format!("invite:workspace_subject:{}", abuse_decision.subject_key),
"source_prefix_domain" => format!("invite:source_cohort_subject:{}", abuse_decision.subject_key),
_ => format!("invite:actor_subject:{}", abuse_decision.subject_key),
};
let action_required = record_invite_abuse_action(&pool, &input, &actor, abuse_decision, &config).await?;
return Ok(RuntimeWorkspaceInviteQuotaDecision {
allowed: false,
reservation_id: None,
retry_after_seconds: None,
reason: Some(reason.to_string()),
scope_key: Some(scope_key),
window_seconds: None,
limit: None,
current: None,
requested: Some(input.target_count),
action_required: Some(action_required),
});
}
let workspace = load_workspace(&pool, &input.workspace_id).await?;
let activity = load_invite_activity(&pool, &input.actor_user_id, &input.workspace_id).await?;
let scopes = build_invite_scopes(&input, &actor, &workspace, &quota, &activity, &config, now)?;
match reserve_scopes(&pool, "workspace_invite", input.request_id.as_deref(), scopes).await? {
Ok(reservation) => Ok(RuntimeWorkspaceInviteQuotaDecision {
allowed: true,
reservation_id: Some(reservation.reservation_id),
retry_after_seconds: None,
reason: None,
scope_key: None,
window_seconds: None,
limit: None,
current: None,
requested: Some(input.target_count),
action_required: None,
}),
Err(violation) => Ok(decision_from_violation(violation, "quota_subject")),
}
}
#[napi]
pub async fn commit_workspace_invite_quota_v1(
&self,
reservation_id: String,
usage: RuntimeWorkspaceInviteQuotaUsage,
) -> Result<bool> {
let domain_usage = sum_domains(&usage.target_domains);
let pool = self.pool().await?;
commit_reservation(&pool, &reservation_id, usage.target_count, |scope_key, _| {
invite_commit_usage_for_scope(scope_key, usage.target_count, &domain_usage)
})
.await
.map_err(Into::into)
}
#[napi]
pub async fn release_workspace_invite_quota_v1(&self, reservation_id: String) -> Result<bool> {
let pool = self.pool().await?;
release_reservation(&pool, &reservation_id).await.map_err(Into::into)
}
}
@@ -0,0 +1,654 @@
use std::collections::BTreeMap;
use chrono::{DateTime, Duration, Utc};
use napi::Result;
use sha2::{Digest, Sha256};
#[cfg(test)]
use super::RuntimeQuotaSourceInput;
use super::{
InviteQuotaConfig, RuntimeQuotaTargetDomainInput, RuntimeWorkspaceInviteQuotaInput, ScopeLimit, bucket_seconds,
high_risk_domain, napi_error, normalize_domain, scope, short_hash, source_prefix, workspace_subject_key,
};
#[derive(Clone, Debug)]
pub(super) struct ActorFacts {
pub(super) email: String,
pub(super) created_at: DateTime<Utc>,
pub(super) registered: bool,
pub(super) email_verified: bool,
pub(super) disabled: bool,
}
#[derive(Clone, Debug)]
pub(super) struct QuotaFacts {
pub(super) plan: String,
pub(super) owner_user_id: Option<String>,
pub(super) uses_owner_quota: bool,
pub(super) seat_limit: i32,
pub(super) member_count: i32,
pub(super) known: bool,
pub(super) stale: bool,
pub(super) stale_after: Option<DateTime<Utc>>,
}
#[derive(Clone, Debug)]
pub(super) struct WorkspaceFacts {
pub(super) created_at: DateTime<Utc>,
}
#[derive(Clone, Debug, Default)]
pub(super) struct InviteActivityFacts {
pub(super) actor_created_7d: i32,
pub(super) actor_accepted_7d: i32,
pub(super) workspace_pending: i32,
pub(super) workspace_created_7d: i32,
pub(super) workspace_accepted_7d: i32,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct InviteAbuseDecision {
pub(super) reason: &'static str,
pub(super) action: &'static str,
pub(super) subject_kind: &'static str,
pub(super) subject_key: String,
}
pub(super) fn subject_hash(email: &str, config: &InviteQuotaConfig) -> String {
let normalized = email.trim().to_ascii_lowercase();
let hash = Sha256::digest(format!("{}:{normalized}", config.subject_hash_salt).as_bytes());
format!("actor_email_sha256:v1:{}", hex::encode(hash))
}
pub(super) fn source_cohort_subject_key(prefix: &str, domain: &str) -> String {
format!(
"source_prefix_domain_sha256:v1:{}",
short_hash(&format!("{}:{}", prefix, normalize_domain(domain)))
)
}
fn quota_subject(workspace_id: &str, quota: &QuotaFacts) -> (String, i32) {
if quota.uses_owner_quota || !quota.plan.to_ascii_lowercase().contains("team") {
(
format!("owner:{}", quota.owner_user_id.as_deref().unwrap_or(workspace_id)),
quota.seat_limit,
)
} else {
(format!("workspace:{workspace_id}"), quota.seat_limit)
}
}
fn plan_ceiling_7d(plan: &str, seat_limit: i32) -> i32 {
let normalized = plan.to_ascii_lowercase();
if normalized.contains("enterprise") || normalized.contains("team") && !normalized.contains("trial") {
seat_limit.saturating_mul(2)
} else if normalized.contains("trial") {
50
} else if normalized.contains("pro") {
20
} else {
10
}
}
fn base_invite_limits(
now: DateTime<Utc>,
actor: &ActorFacts,
workspace: &WorkspaceFacts,
quota: &QuotaFacts,
activity: &InviteActivityFacts,
) -> (i32, i32, i32, i32) {
if actor.disabled || !actor.registered {
return (0, 0, 0, 0);
}
let account_age = now - actor.created_at;
let workspace_age = now - workspace.created_at;
let mut single = 5;
let mut per_hour = 5;
let mut per_day = 15;
let mut per_week = 30;
if account_age < Duration::hours(24) {
return (0, 0, 0, 0);
}
if !actor.email_verified {
single = 1;
per_hour = 1;
per_day = 3;
per_week = 5;
} else if account_age < Duration::days(7) {
single = 3;
per_hour = 3;
per_day = 5;
per_week = 10;
} else if account_age < Duration::days(30) {
single = 5;
per_hour = 5;
per_day = 15;
per_week = 30;
} else {
let plan = quota.plan.to_ascii_lowercase();
if plan.contains("team") && !plan.contains("trial") {
single = quota.seat_limit.min(20);
per_hour = 20;
per_day = 50;
} else if plan.contains("trial") {
single = 10;
per_hour = 10;
per_day = 30;
per_week = 80;
} else if plan.contains("pro") {
single = 8;
per_hour = 8;
per_day = 20;
per_week = 50;
}
}
if workspace_age < Duration::hours(24) || quota.member_count <= 1 {
single = single.min(3);
per_day = per_day.min(10);
} else if workspace_age < Duration::days(7) || quota.member_count <= 5 {
single = single.min((single as f32 * 0.5).ceil() as i32).max(1);
per_hour = per_hour.min((per_hour as f32 * 0.5).ceil() as i32).max(1);
per_day = per_day.min((per_day as f32 * 0.5).ceil() as i32).max(1);
}
if activity.workspace_pending > quota.seat_limit.saturating_mul(2).max(6) {
single = single.min(2);
per_day = per_day.min(5);
}
if activity.actor_created_7d >= 5 && activity.actor_accepted_7d.saturating_mul(10) < activity.actor_created_7d {
single = single.min(2);
per_hour = per_hour.min(2);
per_day = per_day.min(5);
per_week = per_week.min(10);
}
if activity.workspace_created_7d >= 10
&& activity.workspace_accepted_7d.saturating_mul(10) < activity.workspace_created_7d
{
per_day = per_day.min(10);
per_week = per_week.min(20);
}
let seat_7d = plan_ceiling_7d(&quota.plan, quota.seat_limit).min(quota.seat_limit.saturating_mul(2));
(single, per_hour, per_day, per_week.min(seat_7d))
}
pub(super) fn evaluate_projection(quota: &QuotaFacts, now: DateTime<Utc>) -> Option<&'static str> {
if !quota.known {
return Some("quota_state_unavailable");
}
if quota.stale || quota.stale_after.is_some_and(|stale_after| stale_after <= now) {
return Some("quota_projection_stale");
}
None
}
pub(super) fn build_invite_scopes(
input: &RuntimeWorkspaceInviteQuotaInput,
actor: &ActorFacts,
workspace: &WorkspaceFacts,
quota: &QuotaFacts,
activity: &InviteActivityFacts,
config: &InviteQuotaConfig,
now: DateTime<Utc>,
) -> Result<Vec<ScopeLimit>> {
if input.target_count <= 0 {
return Err(napi_error("target_count must be positive"));
}
let (single, per_hour, per_day, per_week) = base_invite_limits(now, actor, workspace, quota, activity);
if input.target_count > single {
return Ok(vec![ScopeLimit {
scope_key: format!("invite:single_request:{}", input.actor_user_id),
window_seconds: 60,
bucket_seconds: bucket_seconds(60),
limit: single,
requested: input.target_count,
}]);
}
let actor_subject = subject_hash(&actor.email, config);
let (quota_subject, seat_limit) = quota_subject(&input.workspace_id, quota);
let quota_subject_7d = plan_ceiling_7d(&quota.plan, seat_limit).min(seat_limit.saturating_mul(2));
let mut scopes = vec![
scope(
format!("invite:user:{}", input.actor_user_id),
3600,
per_hour,
input.target_count,
),
scope(
format!("invite:user:{}", input.actor_user_id),
86_400,
per_day,
input.target_count,
),
scope(
format!("invite:user:{}", input.actor_user_id),
604_800,
per_week,
input.target_count,
),
scope(
format!("invite:actor_subject:{actor_subject}"),
604_800,
per_week,
input.target_count,
),
scope(
format!("invite:workspace:{}", input.workspace_id),
3600,
per_hour.max(5),
input.target_count,
),
scope(
format!("invite:workspace:{}", input.workspace_id),
86_400,
per_day.max(10),
input.target_count,
),
scope(
format!("invite:quota_subject:{quota_subject}"),
604_800,
quota_subject_7d,
input.target_count,
),
];
for target in &input.target_domains {
let domain = normalize_domain(&target.domain);
let limit_7d = if high_risk_domain(&domain, config) {
quota_subject_7d.min((seat_limit / 2).max(2))
} else {
quota_subject_7d
};
scopes.push(scope(
format!("invite:user_domain:{}:{domain}", input.actor_user_id),
86_400,
per_day.min(limit_7d).max(1),
target.count,
));
scopes.push(scope(
format!("invite:workspace_domain:{}:{domain}", input.workspace_id),
86_400,
per_day.max(10).min(limit_7d).max(1),
target.count,
));
scopes.push(scope(
format!("invite:quota_subject_domain:{quota_subject}:{domain}"),
604_800,
limit_7d,
target.count,
));
scopes.push(scope(
format!("invite:target_domain_global:{domain}"),
60,
if high_risk_domain(&domain, config) { 30 } else { 100 },
target.count,
));
scopes.push(scope(
format!("invite:target_domain_global:{domain}"),
86_400,
if high_risk_domain(&domain, config) { 500 } else { 2000 },
target.count,
));
}
if let Some(prefix) = source_prefix(input.source.as_ref()) {
scopes.push(scope(
format!("invite:source_prefix:{prefix}"),
3600,
30,
input.target_count,
));
scopes.push(scope(
format!("invite:source_prefix:{prefix}"),
86_400,
100,
input.target_count,
));
for target in &input.target_domains {
let domain = normalize_domain(&target.domain);
scopes.push(scope(
format!("invite:source_prefix_domain:{prefix}:{domain}"),
3600,
if high_risk_domain(&domain, config) { 5 } else { 15 },
target.count,
));
scopes.push(scope(
format!("invite:source_prefix_domain:{prefix}:{domain}"),
86_400,
if high_risk_domain(&domain, config) { 15 } else { 50 },
target.count,
));
}
}
if input.source.as_ref().is_some_and(|source| source.trusted)
&& let Some(asn) = input.source.as_ref().and_then(|source| source.asn)
{
for target in &input.target_domains {
let domain = normalize_domain(&target.domain);
scopes.push(scope(
format!("invite:source_asn_domain:{asn}:{domain}"),
3600,
if high_risk_domain(&domain, config) { 50 } else { 150 },
target.count,
));
scopes.push(scope(
format!("invite:source_asn_domain:{asn}:{domain}"),
86_400,
if high_risk_domain(&domain, config) { 150 } else { 500 },
target.count,
));
}
}
Ok(scopes)
}
pub(super) fn high_confidence_invite_abuse(
input: &RuntimeWorkspaceInviteQuotaInput,
actor: &ActorFacts,
config: &InviteQuotaConfig,
) -> Option<InviteAbuseDecision> {
let high_risk_domain_counts: Vec<(String, i32)> = input
.target_domains
.iter()
.filter(|target| high_risk_domain(&target.domain, config))
.map(|target| (normalize_domain(&target.domain), target.count.max(0)))
.collect();
let high_risk_targets: i32 = high_risk_domain_counts.iter().map(|(_, count)| *count).sum();
if !actor.email_verified && high_risk_targets >= 3 {
return Some(InviteAbuseDecision {
reason: "unverified_high_risk_domain_burst",
action: "quarantine_actor",
subject_kind: "actor_email",
subject_key: subject_hash(&actor.email, config),
});
}
if input.target_count >= 30 && high_risk_targets * 2 >= input.target_count {
return Some(InviteAbuseDecision {
reason: "workspace_high_risk_domain_burst",
action: "quarantine_workspace",
subject_kind: "workspace",
subject_key: workspace_subject_key(&input.workspace_id),
});
}
if let Some(prefix) = source_prefix(input.source.as_ref())
&& high_risk_targets >= 10
&& high_risk_targets * 2 >= input.target_count
&& let Some((domain, _)) = high_risk_domain_counts.iter().max_by_key(|(_, count)| *count)
{
return Some(InviteAbuseDecision {
reason: "source_high_risk_domain_burst",
action: "quarantine_source_cohort",
subject_kind: "source_prefix_domain",
subject_key: source_cohort_subject_key(&prefix, domain),
});
}
if input.target_count >= 10 && high_risk_targets * 2 >= input.target_count {
return Some(InviteAbuseDecision {
reason: "high_risk_domain_burst",
action: if input.target_count >= 20 {
"ban_actor"
} else {
"quarantine_actor"
},
subject_kind: "actor_email",
subject_key: subject_hash(&actor.email, config),
});
}
None
}
pub(super) fn sum_domains(target_domains: &[RuntimeQuotaTargetDomainInput]) -> BTreeMap<String, i32> {
let mut domains = BTreeMap::new();
for domain in target_domains {
*domains.entry(normalize_domain(&domain.domain)).or_insert(0) += domain.count.max(0);
}
domains
}
pub(super) fn invite_commit_usage_for_scope(
scope_key: &str,
target_count: i32,
domain_usage: &BTreeMap<String, i32>,
) -> i32 {
for (domain, count) in domain_usage {
if scope_key.ends_with(&format!(":{domain}")) {
return *count;
}
}
target_count
}
#[cfg(test)]
mod tests {
use chrono::TimeZone;
use super::*;
fn user(created_at: DateTime<Utc>) -> ActorFacts {
ActorFacts {
email: "actor@example.com".to_string(),
created_at,
registered: true,
email_verified: true,
disabled: false,
}
}
fn workspace(created_at: DateTime<Utc>) -> WorkspaceFacts {
WorkspaceFacts { created_at }
}
fn quota(plan: &str, seat_limit: i32) -> QuotaFacts {
QuotaFacts {
plan: plan.to_string(),
owner_user_id: Some("owner-1".to_string()),
uses_owner_quota: false,
seat_limit,
member_count: 10,
known: true,
stale: false,
stale_after: None,
}
}
fn invite_config() -> InviteQuotaConfig {
InviteQuotaConfig::default()
}
#[test]
fn seat_based_weekly_limit_binds_paid_team_and_high_risk_domain() {
let now = Utc.with_ymd_and_hms(2026, 7, 6, 0, 0, 0).single().unwrap();
let input = RuntimeWorkspaceInviteQuotaInput {
actor_user_id: "u1".to_string(),
workspace_id: "w1".to_string(),
request_id: None,
target_count: 5,
target_domains: vec![RuntimeQuotaTargetDomainInput {
domain: "qq.com".to_string(),
count: 5,
}],
source: None,
};
let scopes = build_invite_scopes(
&input,
&user(now - Duration::days(60)),
&workspace(now - Duration::days(60)),
&quota("paid_team", 10),
&InviteActivityFacts::default(),
&invite_config(),
now,
)
.unwrap();
let quota_subject = scopes
.iter()
.find(|scope| scope.scope_key == "invite:quota_subject:workspace:w1" && scope.window_seconds == 604_800)
.unwrap();
assert_eq!(quota_subject.limit, 20);
let high_risk = scopes
.iter()
.find(|scope| scope.scope_key == "invite:quota_subject_domain:workspace:w1:qq.com")
.unwrap();
assert_eq!(high_risk.limit, 5);
}
#[test]
fn owner_quota_subject_does_not_scale_by_workspace_count() {
let facts = QuotaFacts {
uses_owner_quota: true,
owner_user_id: Some("owner-a".to_string()),
..quota("free", 3)
};
assert_eq!(quota_subject("w1", &facts), ("owner:owner-a".to_string(), 3));
assert_eq!(plan_ceiling_7d("free", 3).min(3 * 2), 6);
}
#[test]
fn low_acceptance_activity_reduces_invite_limits() {
let now = Utc.with_ymd_and_hms(2026, 7, 6, 0, 0, 0).single().unwrap();
let input = RuntimeWorkspaceInviteQuotaInput {
actor_user_id: "u1".to_string(),
workspace_id: "w1".to_string(),
request_id: None,
target_count: 2,
target_domains: vec![RuntimeQuotaTargetDomainInput {
domain: "example.com".to_string(),
count: 2,
}],
source: None,
};
let activity = InviteActivityFacts {
actor_created_7d: 10,
actor_accepted_7d: 0,
..Default::default()
};
let scopes = build_invite_scopes(
&input,
&user(now - Duration::days(60)),
&workspace(now - Duration::days(60)),
&quota("paid_team", 10),
&activity,
&invite_config(),
now,
)
.unwrap();
let day = scopes
.iter()
.find(|scope| scope.scope_key == "invite:user:u1" && scope.window_seconds == 86_400)
.unwrap();
assert_eq!(day.limit, 5);
}
#[test]
fn invite_commit_usage_uses_domain_counts_for_domain_scopes() {
let domain_usage = sum_domains(&[
RuntimeQuotaTargetDomainInput {
domain: "Example.com".to_string(),
count: 2,
},
RuntimeQuotaTargetDomainInput {
domain: "qq.com".to_string(),
count: 1,
},
]);
assert_eq!(
invite_commit_usage_for_scope("invite:user_domain:u1:example.com", 3, &domain_usage),
2
);
assert_eq!(
invite_commit_usage_for_scope("invite:quota_subject:workspace:w1", 3, &domain_usage),
3
);
}
#[test]
fn high_confidence_abuse_selects_workspace_and_source_subjects() {
let config = invite_config();
let mut input = RuntimeWorkspaceInviteQuotaInput {
actor_user_id: "u1".to_string(),
workspace_id: "w1".to_string(),
request_id: None,
target_count: 30,
target_domains: vec![RuntimeQuotaTargetDomainInput {
domain: "qq.com".to_string(),
count: 30,
}],
source: Some(RuntimeQuotaSourceInput {
trusted: true,
ip: Some("192.168.12.34".to_string()),
country: Some("US".to_string()),
asn: Some(13335),
ray_id: Some("ray".to_string()),
}),
};
let workspace_decision = high_confidence_invite_abuse(
&input,
&user(Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).single().unwrap()),
&config,
)
.unwrap();
assert_eq!(workspace_decision.action, "quarantine_workspace");
assert_eq!(workspace_decision.subject_kind, "workspace");
input.target_count = 12;
input.target_domains[0].count = 12;
let source_decision = high_confidence_invite_abuse(
&input,
&user(Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).single().unwrap()),
&config,
)
.unwrap();
assert_eq!(source_decision.action, "quarantine_source_cohort");
assert_eq!(source_decision.subject_kind, "source_prefix_domain");
assert!(
source_decision
.subject_key
.starts_with("source_prefix_domain_sha256:v1:")
);
}
#[test]
fn source_cohort_action_requires_trusted_source() {
let config = invite_config();
let input = RuntimeWorkspaceInviteQuotaInput {
actor_user_id: "u1".to_string(),
workspace_id: "w1".to_string(),
request_id: None,
target_count: 12,
target_domains: vec![RuntimeQuotaTargetDomainInput {
domain: "qq.com".to_string(),
count: 12,
}],
source: Some(RuntimeQuotaSourceInput {
trusted: false,
ip: Some("192.168.12.34".to_string()),
country: Some("US".to_string()),
asn: Some(13335),
ray_id: Some("ray".to_string()),
}),
};
let decision = high_confidence_invite_abuse(
&input,
&user(Utc.with_ymd_and_hms(2026, 7, 1, 0, 0, 0).single().unwrap()),
&config,
)
.unwrap();
assert_eq!(decision.action, "quarantine_actor");
assert_eq!(decision.subject_kind, "actor_email");
}
}
@@ -72,13 +72,129 @@ async fn runtime_from_database_url() -> AnyResult<Option<BackendRuntime>> {
.execute(&pool)
.await
.context("cleanup runtime_leases for backend runtime tests")?;
sqlx::query("DELETE FROM runtime_rolling_quota_reservations WHERE request_id LIKE 'rust-test:%'")
.execute(&pool)
.await
.context("cleanup rolling quota reservations for backend runtime tests")?;
sqlx::query("DELETE FROM runtime_rolling_quota_counters WHERE scope_key LIKE 'invite:%rust-test%'")
.execute(&pool)
.await
.context("cleanup rolling quota counters for backend runtime tests")?;
sqlx::query("DELETE FROM runtime_invite_abuse_actions WHERE subject_key LIKE 'rust-test:%'")
.execute(&pool)
.await
.context("cleanup invite abuse actions for backend runtime tests")?;
sqlx::query("DELETE FROM runtime_invite_abuse_evidence WHERE subject_key LIKE 'rust-test:%'")
.execute(&pool)
.await
.context("cleanup invite abuse evidence for backend runtime tests")?;
sqlx::query(
"DELETE FROM runtime_invite_abuse_subjects WHERE subject_key LIKE 'rust-test:%' OR user_id LIKE 'rust-test:%'",
)
.execute(&pool)
.await
.context("cleanup invite abuse subjects for backend runtime tests")?;
Ok(Some(BackendRuntime {
config: std::sync::RwLock::new(BackendRuntimeConfig { database_url }),
config: std::sync::RwLock::new(BackendRuntimeConfig {
database_url,
invite_quota: Default::default(),
}),
pool: Mutex::new(Some(pool)),
}))
}
async fn insert_invite_quota_fixture(
runtime: &BackendRuntime,
suffix: &str,
stale: bool,
) -> AnyResult<(String, String)> {
let pool = runtime.pool().await.map_err(|err| anyhow!(err.to_string()))?;
let user_id = format!("rust-test:quota:user:{suffix}");
let workspace_id = format!("rust-test:quota:workspace:{suffix}");
let email = format!("rust-test-quota-{suffix}@example.com");
sqlx::query("DELETE FROM effective_workspace_quota_states WHERE workspace_id = $1")
.bind(&workspace_id)
.execute(&pool)
.await?;
sqlx::query("DELETE FROM workspaces WHERE id = $1")
.bind(&workspace_id)
.execute(&pool)
.await?;
sqlx::query("DELETE FROM users WHERE id = $1")
.bind(&user_id)
.execute(&pool)
.await?;
sqlx::query(
r#"
INSERT INTO users (id, name, email, registered, email_verified, disabled, created_at)
VALUES ($1, 'Rust Quota Actor', $2, true, clock_timestamp(), false, clock_timestamp() - interval '60 days')
"#,
)
.bind(&user_id)
.bind(email)
.execute(&pool)
.await?;
sqlx::query(
"INSERT INTO workspaces (id, public, created_at) VALUES ($1, false, clock_timestamp() - interval '60 days')",
)
.bind(&workspace_id)
.execute(&pool)
.await?;
sqlx::query(
r#"
INSERT INTO effective_workspace_quota_states (
workspace_id,
plan,
owner_user_id,
uses_owner_quota,
seat_limit,
member_count,
overcapacity_member_count,
blob_limit,
storage_quota,
used_storage_quota,
history_period_seconds,
readonly,
readonly_reasons,
flags,
known,
stale,
last_reconciled_at,
stale_after
)
VALUES ($1, 'paid_team', $2, false, 10, 3, 0, 1, 1, 0, 1, false, ARRAY[]::TEXT[], '{}'::jsonb, true, $3, clock_timestamp(), clock_timestamp() + interval '1 day')
"#,
)
.bind(&workspace_id)
.bind(&user_id)
.bind(stale)
.execute(&pool)
.await?;
Ok((user_id, workspace_id))
}
fn invite_quota_input(
user_id: &str,
workspace_id: &str,
request_id: &str,
count: i32,
) -> types::RuntimeWorkspaceInviteQuotaInput {
types::RuntimeWorkspaceInviteQuotaInput {
actor_user_id: user_id.to_string(),
workspace_id: workspace_id.to_string(),
request_id: Some(request_id.to_string()),
target_count: count,
target_domains: vec![types::RuntimeQuotaTargetDomainInput {
domain: "example.com".to_string(),
count,
}],
source: None,
}
}
#[tokio::test]
async fn runtime_gate_sql_semantics_are_atomic_and_ttl_bound() {
let _guard = pg_test_lock().lock().await;
@@ -160,6 +276,278 @@ async fn runtime_gate_sql_semantics_are_atomic_and_ttl_bound() {
assert_eq!(runtime.cleanup_expired_runtime_gates(100).await.unwrap(), 0);
}
#[tokio::test]
async fn rolling_quota_sql_state_machine_commits_releases_and_expires() {
let _guard = pg_test_lock().lock().await;
let Some(runtime) = runtime_from_database_url().await.unwrap() else {
eprintln!("skipping postgres integration test: DATABASE_URL is not set");
return;
};
let (user_id, workspace_id) = insert_invite_quota_fixture(&runtime, "state-machine", false)
.await
.unwrap();
let pool = runtime.pool().await.unwrap();
let decision = runtime
.assert_workspace_invite_quota_v1(invite_quota_input(&user_id, &workspace_id, "rust-test:quota:commit", 2))
.await
.unwrap();
assert!(decision.allowed);
let reservation_id = decision.reservation_id.unwrap();
assert!(
runtime
.commit_workspace_invite_quota_v1(
reservation_id,
types::RuntimeWorkspaceInviteQuotaUsage {
target_count: 1,
target_domains: vec![types::RuntimeQuotaTargetDomainInput {
domain: "example.com".to_string(),
count: 1,
}],
},
)
.await
.unwrap()
);
let committed: i64 = sqlx::query_scalar(
r#"
SELECT COALESCE(SUM(count), 0)::bigint
FROM runtime_rolling_quota_counters
WHERE scope_key = $1
"#,
)
.bind(format!("invite:user_domain:{user_id}:example.com"))
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(committed, 1);
let decision = runtime
.assert_workspace_invite_quota_v1(invite_quota_input(
&user_id,
&workspace_id,
"rust-test:quota:release",
1,
))
.await
.unwrap();
let reservation_id = decision.reservation_id.unwrap();
assert!(
runtime
.release_workspace_invite_quota_v1(reservation_id.clone())
.await
.unwrap()
);
let released: String =
sqlx::query_scalar("SELECT status FROM runtime_rolling_quota_reservations WHERE id = $1::uuid LIMIT 1")
.bind(&reservation_id)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(released, "released");
let decision = runtime
.assert_workspace_invite_quota_v1(invite_quota_input(&user_id, &workspace_id, "rust-test:quota:expire", 1))
.await
.unwrap();
let reservation_id = decision.reservation_id.unwrap();
sqlx::query(
"UPDATE runtime_rolling_quota_reservations SET expires_at = clock_timestamp() - interval '1 second' WHERE id = \
$1::uuid",
)
.bind(&reservation_id)
.execute(&pool)
.await
.unwrap();
assert!(runtime.cleanup_expired_rolling_quota(100).await.unwrap() > 0);
let expired: String =
sqlx::query_scalar("SELECT status FROM runtime_rolling_quota_reservations WHERE id = $1::uuid LIMIT 1")
.bind(&reservation_id)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(expired, "expired");
}
#[tokio::test]
async fn rolling_quota_projection_stale_fails_closed() {
let _guard = pg_test_lock().lock().await;
let Some(runtime) = runtime_from_database_url().await.unwrap() else {
eprintln!("skipping postgres integration test: DATABASE_URL is not set");
return;
};
let (user_id, workspace_id) = insert_invite_quota_fixture(&runtime, "stale-projection", true)
.await
.unwrap();
let decision = runtime
.assert_workspace_invite_quota_v1(invite_quota_input(&user_id, &workspace_id, "rust-test:quota:stale", 1))
.await
.unwrap();
assert!(!decision.allowed);
assert_eq!(decision.reason.as_deref(), Some("quota_projection_stale"));
assert!(decision.reservation_id.is_none());
}
#[tokio::test]
async fn invite_abuse_action_sql_state_machine_retries_and_fences_workers() {
let _guard = pg_test_lock().lock().await;
let Some(runtime) = runtime_from_database_url().await.unwrap() else {
eprintln!("skipping postgres integration test: DATABASE_URL is not set");
return;
};
let pool = runtime.pool().await.unwrap();
let actor_id = "rust-test:invite-abuse-action:user";
let workspace_id = "rust-test:invite-abuse-action:workspace";
let subject_key = "rust-test:invite-abuse-action:subject";
let action_id: i64 = sqlx::query_scalar(
r#"
WITH subject AS (
INSERT INTO runtime_invite_abuse_subjects (
subject_key,
kind,
user_id,
actor_email_hash,
status,
first_seen_at,
last_seen_at
)
VALUES ($1, 'actor_email', $2, 'hash', 'quarantined', now(), now())
RETURNING subject_key
),
evidence AS (
INSERT INTO runtime_invite_abuse_evidence (
subject_key,
workspace_id,
user_id,
actor_email_hash,
decision,
reason
)
VALUES ($1, $3, $2, 'hash', 'quarantine_actor', 'test')
RETURNING id
)
INSERT INTO runtime_invite_abuse_actions (
subject_key,
evidence_id,
action,
status
)
SELECT $1, evidence.id, 'quarantine_actor', 'pending'
FROM evidence
RETURNING id
"#,
)
.bind(subject_key)
.bind(actor_id)
.bind(workspace_id)
.fetch_one(&pool)
.await
.unwrap();
assert!(
runtime
.claim_invite_abuse_action(action_id.to_string(), "rust-test:inline-worker".to_string())
.await
.unwrap()
);
assert!(
runtime
.mark_invite_abuse_action(
action_id.to_string(),
"rust-test:inline-worker".to_string(),
"failed".to_string(),
Some("transient cleanup failure".to_string())
)
.await
.unwrap()
);
let waiting = sqlx::query(
r#"
SELECT status, attempts, last_error
FROM runtime_invite_abuse_actions
WHERE id = $1
"#,
)
.bind(action_id)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(waiting.get::<String, _>("status"), "retry_wait");
assert_eq!(waiting.get::<i32, _>("attempts"), 1);
assert_eq!(
waiting.get::<Option<String>, _>("last_error").as_deref(),
Some("transient cleanup failure")
);
sqlx::query(
r#"
UPDATE runtime_invite_abuse_actions
SET next_attempt_at = now() - interval '1 second'
WHERE id = $1
"#,
)
.bind(action_id)
.execute(&pool)
.await
.unwrap();
let claimed = runtime
.claim_retryable_invite_abuse_actions("rust-test:worker".to_string(), 10)
.await
.unwrap();
let current = claimed
.iter()
.find(|action| action.action_id == action_id.to_string())
.expect("retryable action should be claimed");
assert_eq!(current.action, "quarantine_actor");
assert_eq!(current.subject_key, subject_key);
assert_eq!(current.actor_user_id, actor_id);
assert_eq!(current.workspace_id, workspace_id);
assert!(
!runtime
.mark_invite_abuse_action(
action_id.to_string(),
"rust-test:inline-worker".to_string(),
"succeeded".to_string(),
None
)
.await
.unwrap()
);
let still_claimed = sqlx::query(
r#"
SELECT status, locked_by
FROM runtime_invite_abuse_actions
WHERE id = $1
"#,
)
.bind(action_id)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(still_claimed.get::<String, _>("status"), "running");
assert_eq!(
still_claimed.get::<Option<String>, _>("locked_by").as_deref(),
Some("rust-test:worker")
);
assert!(
runtime
.mark_invite_abuse_action(
action_id.to_string(),
"rust-test:worker".to_string(),
"succeeded".to_string(),
None
)
.await
.unwrap()
);
}
#[tokio::test]
async fn coordination_lease_sql_semantics_are_fenced_and_ttl_bound() {
let _guard = pg_test_lock().lock().await;
+93 -1
View File
@@ -1,4 +1,5 @@
use std::{
collections::BTreeMap,
env, fs,
path::{Path, PathBuf},
};
@@ -12,6 +13,35 @@ use super::{RuntimeError, RuntimeResult};
#[derive(Clone, Debug)]
pub(crate) struct BackendRuntimeConfig {
pub(crate) database_url: String,
pub(crate) invite_quota: InviteQuotaConfig,
}
#[derive(Clone, Debug)]
pub(crate) struct InviteQuotaConfig {
pub(crate) high_risk_target_domains: Vec<String>,
pub(crate) subject_hash_salt: String,
pub(crate) mail_class_mapping: BTreeMap<String, String>,
}
impl Default for InviteQuotaConfig {
fn default() -> Self {
Self {
high_risk_target_domains: [
"qq.com",
"proton.me",
"protonmail.com",
"163.com",
"126.com",
"outlook.com",
"hotmail.com",
]
.into_iter()
.map(str::to_string)
.collect(),
subject_hash_salt: "affine-runtime-invite-quota-v1-local".to_string(),
mail_class_mapping: default_mail_class_mapping(),
}
}
}
impl BackendRuntimeConfig {
@@ -20,7 +50,10 @@ impl BackendRuntimeConfig {
let database_url = database_url_from_env()
.or(app_config.database_url())
.unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string());
Ok(Self { database_url })
Ok(Self {
database_url,
invite_quota: app_config.invite_quota_config(),
})
}
pub(crate) async fn with_db_overrides(&self, pool: &PgPool) -> RuntimeResult<Self> {
@@ -30,6 +63,7 @@ impl BackendRuntimeConfig {
// The DB override is loaded after this connection already exists, so it
// must not rewrite the active datasource URL.
database_url: self.database_url.clone(),
invite_quota: app_config.invite_quota_config(),
})
}
}
@@ -53,6 +87,10 @@ impl AppConfigFile {
.and_then(|db| db.datasource_url.clone())
.and_then(non_empty_string)
}
fn invite_quota_config(&self) -> InviteQuotaConfig {
InviteQuotaConfig::default()
}
}
fn database_url_from_env() -> Option<String> {
@@ -86,6 +124,43 @@ impl AppConfigFile {
}
}
fn default_mail_class_mapping() -> BTreeMap<String, String> {
[
("SignIn", "auth"),
("SignUp", "auth"),
("SetPassword", "auth"),
("ChangePassword", "auth"),
("VerifyEmail", "auth"),
("ChangeEmail", "auth"),
("VerifyChangeEmail", "auth"),
("EmailChanged", "auth"),
("MemberInvitation", "workspace_invitation"),
("Mention", "collaboration_notice"),
("Comment", "collaboration_notice"),
("CommentMention", "collaboration_notice"),
("MemberAccepted", "collaboration_notice"),
("LinkInvitationReviewRequest", "collaboration_notice"),
("LinkInvitationApprove", "collaboration_notice"),
("LinkInvitationDecline", "collaboration_notice"),
("MemberLeave", "workspace_lifecycle"),
("MemberRemoved", "workspace_lifecycle"),
("OwnershipTransferred", "workspace_lifecycle"),
("OwnershipReceived", "workspace_lifecycle"),
("TeamWorkspaceUpgraded", "workspace_lifecycle"),
("TeamBecomeAdmin", "workspace_lifecycle"),
("TeamBecomeCollaborator", "workspace_lifecycle"),
("TeamDeleteIn24Hours", "workspace_lifecycle"),
("TeamDeleteInOneMonth", "workspace_lifecycle"),
("TeamWorkspaceDeleted", "workspace_lifecycle"),
("TeamWorkspaceExpireSoon", "workspace_lifecycle"),
("TeamWorkspaceExpired", "workspace_lifecycle"),
("TeamLicense", "billing_license"),
]
.into_iter()
.map(|(mail_name, class)| (mail_name.to_string(), class.to_string()))
.collect()
}
async fn load_app_config_overrides_from_db(pool: &PgPool) -> RuntimeResult<AppConfigFile> {
let rows = match sqlx::query("SELECT id, value FROM app_configs").fetch_all(pool).await {
Ok(rows) => rows,
@@ -197,4 +272,21 @@ mod tests {
Some("postgresql://example/runtime")
);
}
#[test]
fn invite_quota_policy_is_internal_not_app_configurable() {
let app_config = app_config_from_flat_overrides([
("auth.untrustedPolicyOverride", serde_json::json!("runtime-salt-v2")),
("auth.untrustedDomainList", serde_json::json!(["Example.COM."])),
])
.unwrap();
let config = app_config.invite_quota_config();
assert!(!config.high_risk_target_domains.contains(&"example.com".to_string()));
assert_ne!(config.subject_hash_salt, "runtime-salt-v2");
assert_eq!(
config.mail_class_mapping.get("MemberInvitation").map(String::as_str),
Some("workspace_invitation")
);
}
}
+1 -1
View File
@@ -6,5 +6,5 @@ pub(crate) mod error;
pub(crate) mod migrations;
pub(crate) mod types;
pub(crate) use config::BackendRuntimeConfig;
pub(crate) use config::{BackendRuntimeConfig, InviteQuotaConfig};
pub(crate) use error::{RuntimeError, RuntimeResult, napi_error, to_napi_error};
@@ -110,3 +110,90 @@ CREATE TABLE IF NOT EXISTS blob_cleanup_candidates (
CREATE INDEX IF NOT EXISTS blob_cleanup_candidates_run_idx
ON blob_cleanup_candidates (run_id, status);
CREATE TABLE IF NOT EXISTS runtime_rolling_quota_counters (
scope_key TEXT NOT NULL,
window_seconds INTEGER NOT NULL,
bucket_start TIMESTAMPTZ(3) NOT NULL,
count INTEGER NOT NULL CHECK (count >= 0),
expires_at TIMESTAMPTZ(3) NOT NULL,
updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope_key, window_seconds, bucket_start)
);
CREATE INDEX IF NOT EXISTS runtime_rolling_quota_counters_expires_at_idx
ON runtime_rolling_quota_counters (expires_at);
CREATE TABLE IF NOT EXISTS runtime_rolling_quota_reservations (
id UUID NOT NULL DEFAULT gen_random_uuid(),
scope_key TEXT NOT NULL,
window_seconds INTEGER NOT NULL,
bucket_start TIMESTAMPTZ(3) NOT NULL,
count INTEGER NOT NULL CHECK (count >= 0),
status TEXT NOT NULL CHECK (status IN ('reserved', 'committed', 'released', 'expired')),
purpose TEXT NOT NULL,
request_id TEXT,
expires_at TIMESTAMPTZ(3) NOT NULL,
committed_at TIMESTAMPTZ(3),
released_at TIMESTAMPTZ(3),
created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, scope_key, window_seconds, bucket_start)
);
CREATE INDEX IF NOT EXISTS runtime_rolling_quota_reservations_status_expires_at_idx
ON runtime_rolling_quota_reservations (status, expires_at);
CREATE INDEX IF NOT EXISTS runtime_rolling_quota_reservations_scope_idx
ON runtime_rolling_quota_reservations (scope_key, window_seconds, bucket_start)
WHERE status = 'reserved';
CREATE TABLE IF NOT EXISTS runtime_invite_abuse_subjects (
subject_key TEXT PRIMARY KEY,
kind TEXT NOT NULL,
user_id TEXT,
actor_email_hash TEXT,
email_domain TEXT,
first_seen_at TIMESTAMPTZ(3) NOT NULL,
last_seen_at TIMESTAMPTZ(3) NOT NULL,
status TEXT NOT NULL DEFAULT 'active',
action TEXT,
action_reason TEXT,
action_at TIMESTAMPTZ(3),
created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS runtime_invite_abuse_evidence (
id BIGSERIAL PRIMARY KEY,
subject_key TEXT NOT NULL REFERENCES runtime_invite_abuse_subjects(subject_key),
request_id TEXT,
workspace_id TEXT,
user_id TEXT,
actor_email_hash TEXT,
source_prefix_hash TEXT,
source_asn BIGINT,
target_domains JSONB NOT NULL DEFAULT '[]'::jsonb,
counters JSONB NOT NULL DEFAULT '{}'::jsonb,
decision TEXT NOT NULL,
reason TEXT NOT NULL,
created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS runtime_invite_abuse_actions (
id BIGSERIAL PRIMARY KEY,
subject_key TEXT NOT NULL REFERENCES runtime_invite_abuse_subjects(subject_key),
evidence_id BIGINT NOT NULL REFERENCES runtime_invite_abuse_evidence(id),
action TEXT NOT NULL CHECK (action IN ('ban_actor', 'quarantine_actor', 'quarantine_workspace', 'quarantine_source_cohort')),
status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'succeeded', 'retry_wait', 'failed')),
attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0),
next_attempt_at TIMESTAMPTZ(3),
locked_by TEXT,
locked_until TIMESTAMPTZ(3),
last_error TEXT,
created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS runtime_invite_abuse_actions_status_next_attempt_idx
ON runtime_invite_abuse_actions (status, next_attempt_at);
@@ -14,6 +14,107 @@ pub struct BackendRuntimeHealth {
pub database_connected: bool,
}
#[napi_derive::napi(object)]
pub struct RuntimeQuotaTargetDomainInput {
pub domain: String,
pub count: i32,
}
#[napi_derive::napi(object)]
pub struct RuntimeQuotaSourceInput {
pub trusted: bool,
pub ip: Option<String>,
pub country: Option<String>,
pub asn: Option<u32>,
pub ray_id: Option<String>,
}
#[napi_derive::napi(object)]
pub struct RuntimeWorkspaceInviteQuotaInput {
pub actor_user_id: String,
pub workspace_id: String,
pub request_id: Option<String>,
pub target_count: i32,
pub target_domains: Vec<RuntimeQuotaTargetDomainInput>,
pub source: Option<RuntimeQuotaSourceInput>,
}
#[napi_derive::napi(object)]
pub struct RuntimeWorkspaceInviteQuotaUsage {
pub target_count: i32,
pub target_domains: Vec<RuntimeQuotaTargetDomainInput>,
}
#[napi_derive::napi(object)]
pub struct RuntimeInviteAbuseActionRequired {
pub action: String,
pub subject_key: String,
pub evidence_id: String,
pub action_id: String,
}
#[napi_derive::napi(object)]
pub struct RuntimeInviteAbuseClaimedAction {
pub action: String,
pub subject_key: String,
pub evidence_id: String,
pub action_id: String,
pub actor_user_id: String,
pub workspace_id: String,
}
#[napi_derive::napi(object)]
pub struct RuntimeWorkspaceInviteQuotaDecision {
pub allowed: bool,
pub reservation_id: Option<String>,
pub retry_after_seconds: Option<i32>,
pub reason: Option<String>,
pub scope_key: Option<String>,
pub window_seconds: Option<i32>,
pub limit: Option<i32>,
pub current: Option<i32>,
pub requested: Option<i32>,
pub action_required: Option<RuntimeInviteAbuseActionRequired>,
}
#[napi_derive::napi(object)]
pub struct RuntimeMailDeliveryQuotaMetadataInput {
pub actor_user_id: Option<String>,
pub workspace_id: Option<String>,
pub notification_id: Option<String>,
pub abuse_subject_key: Option<String>,
}
#[napi_derive::napi(object)]
pub struct RuntimeMailDeliveryQuotaRecipientInput {
pub email: String,
pub domain: String,
pub user_id: Option<String>,
}
#[napi_derive::napi(object)]
pub struct RuntimeMailDeliveryQuotaInput {
pub request_id: Option<String>,
pub mail_name: String,
pub recipient: RuntimeMailDeliveryQuotaRecipientInput,
pub metadata: RuntimeMailDeliveryQuotaMetadataInput,
pub source: Option<RuntimeQuotaSourceInput>,
}
#[napi_derive::napi(object)]
pub struct RuntimeMailDeliveryQuotaDecision {
pub allowed: bool,
pub reservation_id: Option<String>,
pub mail_class: String,
pub retry_after_seconds: Option<i32>,
pub reason: Option<String>,
pub scope_key: Option<String>,
pub window_seconds: Option<i32>,
pub limit: Option<i32>,
pub current: Option<i32>,
pub requested: Option<i32>,
}
#[napi_derive::napi(object)]
pub struct CoordinationLeaseGrant {
pub key: String,
@@ -0,0 +1,132 @@
-- CreateTable
CREATE TABLE "mail_deliveries" (
"id" UUID NOT NULL DEFAULT gen_random_uuid(),
"mail_name" TEXT NOT NULL,
"mail_class" TEXT NOT NULL,
"priority" TEXT NOT NULL DEFAULT 'normal',
"status" TEXT NOT NULL,
"dedupe_key" TEXT,
"recipient_email" TEXT,
"recipient_hash" TEXT NOT NULL,
"recipient_domain" TEXT NOT NULL,
"recipient_user_id" VARCHAR,
"actor_user_id" VARCHAR,
"workspace_id" VARCHAR,
"notification_id" VARCHAR,
"abuse_subject_key" TEXT,
"quota_reservation_id" UUID,
"quota_decision" JSONB,
"payload" JSONB,
"send_after" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"expires_at" TIMESTAMPTZ(3),
"attempt_count" INTEGER NOT NULL DEFAULT 0,
"max_attempts" INTEGER NOT NULL DEFAULT 3,
"locked_by" TEXT,
"locked_until" TIMESTAMPTZ(3),
"first_attempt_at" TIMESTAMPTZ(3),
"last_attempt_at" TIMESTAMPTZ(3),
"sent_at" TIMESTAMPTZ(3),
"settled_at" TIMESTAMPTZ(3),
"canceled_at" TIMESTAMPTZ(3),
"failed_at" TIMESTAMPTZ(3),
"provider_message_id" TEXT,
"provider_response" TEXT,
"last_error_code" TEXT,
"last_error" TEXT,
"retention_state" TEXT NOT NULL DEFAULT 'full',
"anonymized_at" TIMESTAMPTZ(3),
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "mail_deliveries_pkey" PRIMARY KEY ("id"),
CONSTRAINT "mail_deliveries_priority_check" CHECK ("priority" IN ('critical', 'high', 'normal', 'low')),
CONSTRAINT "mail_deliveries_status_check" CHECK ("status" IN ('queued', 'sending', 'retry_wait', 'sent', 'skipped', 'failed', 'canceled')),
CONSTRAINT "mail_deliveries_attempt_count_check" CHECK ("attempt_count" >= 0),
CONSTRAINT "mail_deliveries_max_attempts_check" CHECK ("max_attempts" >= 0),
CONSTRAINT "mail_deliveries_retention_state_check" CHECK ("retention_state" IN ('full', 'anonymized')),
CONSTRAINT "mail_deliveries_payload_retention_check" CHECK (
(
"status" IN ('queued', 'sending', 'retry_wait')
AND "retention_state" = 'full'
AND "recipient_email" IS NOT NULL
AND "payload" IS NOT NULL
)
OR (
"status" IN ('sent', 'skipped', 'failed', 'canceled')
AND "retention_state" = 'anonymized'
AND "recipient_email" IS NULL
AND "payload" IS NULL
AND "anonymized_at" IS NOT NULL
)
)
);
-- CreateIndex
CREATE UNIQUE INDEX "mail_deliveries_dedupe_key_idx"
ON "mail_deliveries"("dedupe_key")
WHERE "dedupe_key" IS NOT NULL;
-- CreateIndex
CREATE INDEX "mail_deliveries_ready_idx"
ON "mail_deliveries"("status", "send_after", "locked_until");
-- CreateIndex
CREATE INDEX "mail_deliveries_pending_expires_at_idx"
ON "mail_deliveries"("expires_at")
WHERE "status" IN ('queued', 'sending', 'retry_wait') AND "expires_at" IS NOT NULL;
-- CreateIndex
CREATE INDEX "mail_deliveries_created_at_idx" ON "mail_deliveries"("created_at");
-- CreateIndex
CREATE INDEX "mail_deliveries_mail_name_status_created_at_idx" ON "mail_deliveries"("mail_name", "status", "created_at");
-- CreateIndex
CREATE INDEX "mail_deliveries_mail_class_status_created_at_idx" ON "mail_deliveries"("mail_class", "status", "created_at");
-- CreateIndex
CREATE INDEX "mail_deliveries_status_created_at_idx" ON "mail_deliveries"("status", "created_at");
-- CreateIndex
CREATE INDEX "mail_deliveries_settled_at_idx" ON "mail_deliveries"("settled_at");
-- CreateIndex
CREATE INDEX "mail_deliveries_settled_name_status_idx"
ON "mail_deliveries"("settled_at", "mail_name", "status")
WHERE "settled_at" IS NOT NULL;
-- CreateIndex
CREATE INDEX "mail_deliveries_recipient_domain_created_at_idx" ON "mail_deliveries"("recipient_domain", "created_at");
-- CreateIndex
CREATE INDEX "mail_deliveries_workspace_id_created_at_idx" ON "mail_deliveries"("workspace_id", "created_at");
-- CreateIndex
CREATE INDEX "mail_deliveries_actor_user_id_created_at_idx" ON "mail_deliveries"("actor_user_id", "created_at");
-- CreateIndex
CREATE INDEX "mail_deliveries_abuse_subject_key_created_at_idx" ON "mail_deliveries"("abuse_subject_key", "created_at");
-- CreateIndex
CREATE INDEX "workspace_invitations_inviter_created_at_idx"
ON "workspace_invitations"("inviter_user_id", "created_at");
-- CreateIndex
CREATE INDEX "workspace_invitations_inviter_status_created_at_idx"
ON "workspace_invitations"("inviter_user_id", "status", "created_at");
-- CreateIndex
CREATE INDEX "workspace_invitations_workspace_inviter_created_at_idx"
ON "workspace_invitations"("workspace_id", "inviter_user_id", "created_at");
-- CreateIndex
CREATE INDEX "workspace_invitations_workspace_status_created_at_idx"
ON "workspace_invitations"("workspace_id", "status", "created_at");
-- CreateIndex
CREATE INDEX "workspace_invitations_workspace_accepted_at_idx"
ON "workspace_invitations"("workspace_id", "accepted_at");
-- CreateIndex
CREATE INDEX "workspace_members_workspace_state_created_at_idx"
ON "workspace_members"("workspace_id", "state", "created_at");
+52
View File
@@ -360,6 +360,58 @@ model DocAccessPolicy {
@@map("doc_access_policies")
}
model MailDelivery {
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
mailName String @map("mail_name") @db.Text
mailClass String @map("mail_class") @db.Text
priority String @default("normal") @db.Text
status String @db.Text
dedupeKey String? @map("dedupe_key") @db.Text
recipientEmail String? @map("recipient_email") @db.Text
recipientHash String @map("recipient_hash") @db.Text
recipientDomain String @map("recipient_domain") @db.Text
recipientUserId String? @map("recipient_user_id") @db.VarChar
actorUserId String? @map("actor_user_id") @db.VarChar
workspaceId String? @map("workspace_id") @db.VarChar
notificationId String? @map("notification_id") @db.VarChar
abuseSubjectKey String? @map("abuse_subject_key") @db.Text
quotaReservationId String? @map("quota_reservation_id") @db.Uuid
quotaDecision Json? @map("quota_decision") @db.JsonB
payload Json? @db.JsonB
sendAfter DateTime @default(now()) @map("send_after") @db.Timestamptz(3)
expiresAt DateTime? @map("expires_at") @db.Timestamptz(3)
attemptCount Int @default(0) @map("attempt_count") @db.Integer
maxAttempts Int @default(3) @map("max_attempts") @db.Integer
lockedBy String? @map("locked_by") @db.Text
lockedUntil DateTime? @map("locked_until") @db.Timestamptz(3)
firstAttemptAt DateTime? @map("first_attempt_at") @db.Timestamptz(3)
lastAttemptAt DateTime? @map("last_attempt_at") @db.Timestamptz(3)
sentAt DateTime? @map("sent_at") @db.Timestamptz(3)
settledAt DateTime? @map("settled_at") @db.Timestamptz(3)
canceledAt DateTime? @map("canceled_at") @db.Timestamptz(3)
failedAt DateTime? @map("failed_at") @db.Timestamptz(3)
providerMessageId String? @map("provider_message_id") @db.Text
providerResponse String? @map("provider_response") @db.Text
lastErrorCode String? @map("last_error_code") @db.Text
lastError String? @map("last_error") @db.Text
retentionState String @default("full") @map("retention_state") @db.Text
anonymizedAt DateTime? @map("anonymized_at") @db.Timestamptz(3)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
@@index([createdAt])
@@index([mailName, status, createdAt])
@@index([mailClass, status, createdAt])
@@index([status, createdAt])
@@index([status, sendAfter, lockedUntil], map: "mail_deliveries_ready_idx")
@@index([settledAt])
@@index([recipientDomain, createdAt])
@@index([workspaceId, createdAt])
@@index([actorUserId, createdAt])
@@index([abuseSubjectKey, createdAt])
@@map("mail_deliveries")
}
model DocGrant {
workspaceId String @map("workspace_id") @db.VarChar
docId String @map("doc_id") @db.VarChar
@@ -110,7 +110,7 @@ test('set and change password', async t => {
const u1 = await app.signupV1(u1Email);
await sendSetPasswordEmail(app, u1Email, '/password-change');
const setPasswordMail = app.mails.last('ChangePassword');
const setPasswordMail = app.mails.last('SetPassword');
const link = new URL(setPasswordMail.props.url);
const setPasswordToken = link.searchParams.get('token');
@@ -4,7 +4,9 @@ import {
getCurrentUserQuery,
getWorkspaceQuery,
} from '@affine/graphql';
import { WorkspaceMemberStatus } from '@prisma/client';
import { WorkspaceRole } from '../../../models';
import { app, e2e, Mockers } from '../test';
const admin = await app.create(Mockers.User, {
@@ -107,6 +109,31 @@ e2e('should ban account', async t => {
t.is(banUser.disabled, true);
});
e2e('should ban account with pending workspace invitation', async t => {
const owner = await app.create(Mockers.User);
const user = await app.create(Mockers.User);
const workspace = await app.create(Mockers.Workspace, { owner });
await app.models.workspaceInvitation.set(
workspace.id,
user.id,
WorkspaceRole.Collaborator,
WorkspaceMemberStatus.Pending,
{ inviterId: owner.id }
);
await app.login(admin);
const { banUser } = await app.gql({
query: disableUserMutation,
variables: {
id: user.id,
},
});
t.is(banUser.disabled, true);
});
e2e('should not login banned account', async t => {
const user = await app.create(Mockers.User);
@@ -2,17 +2,17 @@ import { interval, map, take, takeUntil } from 'rxjs';
import Sinon from 'sinon';
import { Mailer } from '../../core/mail';
import type { SendMailCommand } from '../../core/mail/types';
import { MailName } from '../../mails';
export class MockMailer {
send = Sinon.createStubInstance(Mailer).send.resolves(true);
trySend(command: Jobs['notification.sendMail']) {
skip = Sinon.createStubInstance(Mailer).skip.resolves(false);
trySend(command: SendMailCommand) {
return this.send(command, true);
}
last<Mail extends MailName>(
name: Mail
): Extract<Jobs['notification.sendMail'], { name: Mail }> {
last<Mail extends MailName>(name: Mail): SendMailCommand & { name: Mail } {
const last = this.send.lastCall.args[0];
if (!last) {
@@ -23,22 +23,26 @@ export class MockMailer {
throw new Error(`Mail name mismatch: ${last.name} !== ${name}`);
}
return last as any;
return last as SendMailCommand & { name: Mail };
}
waitFor<Mail extends MailName>(
name: Mail,
timeout: number = 1000
): Promise<Extract<Jobs['notification.sendMail'], { name: Mail }>> {
const { promise, reject, resolve } = Promise.withResolvers<any>();
): Promise<SendMailCommand & { name: Mail }> {
const { promise, reject, resolve } = Promise.withResolvers<
SendMailCommand & { name: Mail }
>();
interval(10)
.pipe(
take(Math.floor(timeout / 10)),
takeUntil(promise),
map(() => {
const last = this.send.lastCall.args[0];
return last.name === name ? last : undefined;
const last = this.send.lastCall?.args[0];
return last?.name === name
? (last as SendMailCommand & { name: Mail })
: undefined;
})
)
.subscribe({
@@ -102,4 +102,12 @@ defineModuleConfig('job', {
},
schema,
},
'queues.inviteAbuse': {
desc: 'The config for invite abuse disposition job queue',
default: {
concurrency: 1,
},
schema,
},
});
@@ -30,6 +30,7 @@ export enum Queue {
INDEXER = 'indexer',
CALENDAR = 'calendar',
BACKENDRUNTIME = 'backendRuntime',
INVITE_ABUSE = 'inviteAbuse',
}
export const QUEUES = Object.values(Queue);
@@ -12,6 +12,9 @@ export interface AuthConfig {
requireEmailDomainVerification: boolean;
requireEmailVerification: boolean;
newAccountShareActionDelay: number;
trustedCloudflareHeaders: boolean;
inviteQuotaShadowMode: boolean;
inviteQuotaFailOpenOnRuntimeError: boolean;
passwordRequirements: ConfigItem<{
min: number;
max: number;
@@ -46,6 +49,21 @@ defineModuleConfig('auth', {
default: 24 * 60 * 60,
shape: z.number().int().min(0),
},
trustedCloudflareHeaders: {
desc: 'Whether request abuse source facts should trust Cloudflare headers from the origin edge.',
default: false,
shape: z.boolean(),
},
inviteQuotaShadowMode: {
desc: 'Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions.',
default: false,
shape: z.boolean(),
},
inviteQuotaFailOpenOnRuntimeError: {
desc: 'Whether workspace invite quota should fail open when native runtime admission is unavailable. Keep disabled for production.',
default: false,
shape: z.boolean(),
},
passwordRequirements: {
desc: 'The password strength requirements when set new password.',
default: {
@@ -15,6 +15,7 @@ import type { Request, Response } from 'express';
import {
ActionForbidden,
Config,
EmailTokenNotFound,
getRequestCookie,
InvalidAuthState,
@@ -25,6 +26,7 @@ import {
} from '../../base';
import { Models } from '../../models';
import { validators } from '../utils/validators';
import { getAbuseRequestSource } from '../workspaces/abuse';
import { Public } from './guard';
import { MagicLinkAuthService } from './magic-link';
import { AuthMethodsService } from './methods';
@@ -79,7 +81,8 @@ export class AuthController {
private readonly openApp: OpenAppAuthService,
private readonly authMethods: AuthMethodsService,
private readonly sessionExchange: SessionExchangeService,
private readonly models: Models
private readonly models: Models,
private readonly config: Config
) {
if (env.dev) {
// set DNS servers in dev mode
@@ -135,6 +138,7 @@ export class AuthController {
);
} else {
await this.sendMagicLink(
req,
res,
credential.email,
credential.callbackUrl,
@@ -163,12 +167,15 @@ export class AuthController {
}
async sendMagicLink(
req: Request,
res: Response,
email: string,
callbackUrl = '/magic-link',
clientNonce?: string
) {
const payload = await this.magicLink.send(email, callbackUrl, clientNonce);
const payload = await this.magicLink.send(email, callbackUrl, clientNonce, {
source: getAbuseRequestSource(req, this.config),
});
res.status(HttpStatus.OK).send(payload);
}
@@ -12,6 +12,7 @@ import {
WrongSignInCredentials,
} from '../../base';
import { Models, TokenType } from '../../models';
import type { MailDeliveryMetadata } from '../mail/types';
import { validators } from '../utils/validators';
import { verifyEmailDomainRecords } from './email-domain';
import type { VerifiedIdentity } from './identity';
@@ -29,7 +30,12 @@ export class MagicLinkAuthService {
private readonly crypto: CryptoHelper
) {}
async send(email: string, callbackUrl = '/magic-link', clientNonce?: string) {
async send(
email: string,
callbackUrl = '/magic-link',
clientNonce?: string,
metadata?: Pick<MailDeliveryMetadata, 'source'>
) {
validators.assertValidEmail(email);
if (!this.url.isAllowedCallbackUrl(callbackUrl)) {
@@ -57,21 +63,33 @@ export class MagicLinkAuthService {
}
const ttlInSec = 30 * 60;
const token = await this.models.verificationToken.create(
TokenType.SignIn,
email,
ttlInSec
);
const { token, expiresAt: tokenExpiresAt } =
await this.models.verificationToken.createWithExpiresAt(
TokenType.SignIn,
email,
ttlInSec
);
const otp = this.crypto.otp();
await this.models.magicLinkOtp.upsert(email, otp, token, clientNonce);
const { expiresAt: otpExpiresAt } = await this.models.magicLinkOtp.upsert(
email,
otp,
token,
clientNonce
);
const magicLink = this.url.link(callbackUrl, { token: otp, email });
if (env.dev) {
this.logger.debug(`Magic link: ${magicLink}`);
}
await this.auth.sendSignInEmail(email, magicLink, otp, !user);
await this.auth.sendSignInEmail(email, magicLink, otp, !user, {
...metadata,
expiresAt:
tokenExpiresAt.getTime() < otpExpiresAt.getTime()
? tokenExpiresAt
: otpExpiresAt,
});
return { email };
}
@@ -1,5 +1,6 @@
import {
Args,
Context,
Field,
Mutation,
ObjectType,
@@ -11,6 +12,7 @@ import {
import {
ActionForbidden,
Config,
EmailAlreadyUsed,
EmailTokenNotFound,
EmailVerificationRequired,
@@ -21,10 +23,13 @@ import {
Throttle,
URLHelper,
} from '../../base';
import type { GraphqlContext } from '../../base/graphql';
import { Models, TokenType } from '../../models';
import { Admin } from '../common';
import type { MailDeliveryMetadata } from '../mail/types';
import { UserType } from '../user/types';
import { validators } from '../utils/validators';
import { getAbuseRequestSource } from '../workspaces/abuse';
import { Public } from './guard';
import { AuthService } from './service';
import { CurrentUser } from './session';
@@ -47,9 +52,20 @@ export class AuthResolver {
constructor(
private readonly url: URLHelper,
private readonly auth: AuthService,
private readonly models: Models
private readonly models: Models,
private readonly config: Config
) {}
private mailMetadata(
context: GraphqlContext,
expiresAt?: Date
): MailDeliveryMetadata {
return {
source: getAbuseRequestSource(context.req, this.config),
expiresAt,
};
}
@SkipThrottle()
@Public()
@Query(() => UserType, {
@@ -145,23 +161,30 @@ export class AuthResolver {
@CurrentUser() user: CurrentUser,
@Args('callbackUrl') callbackUrl: string,
@Args('email', {
type: () => String,
nullable: true,
deprecationReason: 'fetched from signed in user',
})
_email?: string
_email: string | undefined,
@Context() context: GraphqlContext
) {
if (!user.emailVerified) {
throw new EmailVerificationRequired();
}
const token = await this.models.verificationToken.create(
TokenType.ChangePassword,
user.id
);
const { token, expiresAt } =
await this.models.verificationToken.createWithExpiresAt(
TokenType.ChangePassword,
user.id
);
const url = this.url.safeLink(callbackUrl, { userId: user.id, token });
return await this.auth.sendChangePasswordEmail(user.email, url);
return await this.auth.sendChangePasswordEmail(
user.email,
url,
this.mailMetadata(context, expiresAt)
);
}
@Mutation(() => Boolean)
@@ -169,12 +192,26 @@ export class AuthResolver {
@CurrentUser() user: CurrentUser,
@Args('callbackUrl') callbackUrl: string,
@Args('email', {
type: () => String,
nullable: true,
deprecationReason: 'fetched from signed in user',
})
_email?: string
_email: string | undefined,
@Context() context: GraphqlContext
) {
return this.sendChangePasswordEmail(user, callbackUrl);
const { token, expiresAt } =
await this.models.verificationToken.createWithExpiresAt(
TokenType.ChangePassword,
user.id
);
const url = this.url.safeLink(callbackUrl, { userId: user.id, token });
return await this.auth.sendSetPasswordEmail(
user.email,
url,
this.mailMetadata(context, expiresAt)
);
}
// The change email step is:
@@ -187,20 +224,26 @@ export class AuthResolver {
@Mutation(() => Boolean)
async sendChangeEmail(
@CurrentUser() user: CurrentUser,
@Args('callbackUrl') callbackUrl: string
@Args('callbackUrl') callbackUrl: string,
@Context() context: GraphqlContext
) {
if (!user.emailVerified) {
throw new EmailVerificationRequired();
}
const token = await this.models.verificationToken.create(
TokenType.ChangeEmail,
user.id
);
const { token, expiresAt } =
await this.models.verificationToken.createWithExpiresAt(
TokenType.ChangeEmail,
user.id
);
const url = this.url.safeLink(callbackUrl, { token });
return await this.auth.sendChangeEmail(user.email, url);
return await this.auth.sendChangeEmail(
user.email,
url,
this.mailMetadata(context, expiresAt)
);
}
@Mutation(() => Boolean)
@@ -208,7 +251,8 @@ export class AuthResolver {
@CurrentUser() user: CurrentUser,
@Args('token') token: string,
@Args('email') email: string,
@Args('callbackUrl') callbackUrl: string
@Args('callbackUrl') callbackUrl: string,
@Context() context: GraphqlContext
) {
if (!token) {
throw new EmailTokenNotFound();
@@ -237,31 +281,42 @@ export class AuthResolver {
}
}
const verifyEmailToken = await this.models.verificationToken.create(
TokenType.VerifyEmail,
user.id
);
const { token: verifyEmailToken, expiresAt } =
await this.models.verificationToken.createWithExpiresAt(
TokenType.VerifyEmail,
user.id
);
const url = this.url.safeLink(callbackUrl, {
token: verifyEmailToken,
email,
});
return await this.auth.sendVerifyChangeEmail(email, url);
return await this.auth.sendVerifyChangeEmail(
email,
url,
this.mailMetadata(context, expiresAt)
);
}
@Mutation(() => Boolean)
async sendVerifyEmail(
@CurrentUser() user: CurrentUser,
@Args('callbackUrl') callbackUrl: string
@Args('callbackUrl') callbackUrl: string,
@Context() context: GraphqlContext
) {
const token = await this.models.verificationToken.create(
TokenType.VerifyEmail,
user.id
);
const { token, expiresAt } =
await this.models.verificationToken.createWithExpiresAt(
TokenType.VerifyEmail,
user.id
);
const url = this.url.safeLink(callbackUrl, { token });
return await this.auth.sendVerifyEmail(user.email, url);
return await this.auth.sendVerifyEmail(
user.email,
url,
this.mailMetadata(context, expiresAt)
);
}
@Mutation(() => Boolean)
@@ -7,6 +7,7 @@ import { assign, pick } from 'lodash-es';
import { Config, SignUpForbidden } from '../../base';
import { Models, type User, type UserSession } from '../../models';
import { Mailer } from '../mail/mailer';
import type { MailDeliveryMetadata } from '../mail/types';
import { createDevUsers } from './dev';
import type { VerifiedIdentity } from './identity';
import {
@@ -312,49 +313,74 @@ export class AuthService implements OnApplicationBootstrap {
});
}
async sendChangePasswordEmail(email: string, callbackUrl: string) {
async sendChangePasswordEmail(
email: string,
callbackUrl: string,
metadata?: MailDeliveryMetadata
) {
return await this.mailer.send({
name: 'ChangePassword',
to: email,
props: {
url: callbackUrl,
},
metadata,
});
}
async sendSetPasswordEmail(email: string, callbackUrl: string) {
async sendSetPasswordEmail(
email: string,
callbackUrl: string,
metadata?: MailDeliveryMetadata
) {
return await this.mailer.send({
name: 'SetPassword',
to: email,
props: {
url: callbackUrl,
},
metadata,
});
}
async sendChangeEmail(email: string, callbackUrl: string) {
async sendChangeEmail(
email: string,
callbackUrl: string,
metadata?: MailDeliveryMetadata
) {
return await this.mailer.send({
name: 'ChangeEmail',
to: email,
props: {
url: callbackUrl,
},
metadata,
});
}
async sendVerifyChangeEmail(email: string, callbackUrl: string) {
async sendVerifyChangeEmail(
email: string,
callbackUrl: string,
metadata?: MailDeliveryMetadata
) {
return await this.mailer.send({
name: 'VerifyChangeEmail',
to: email,
props: {
url: callbackUrl,
},
metadata,
});
}
async sendVerifyEmail(email: string, callbackUrl: string) {
async sendVerifyEmail(
email: string,
callbackUrl: string,
metadata?: MailDeliveryMetadata
) {
return await this.mailer.send({
name: 'VerifyEmail',
to: email,
props: {
url: callbackUrl,
},
metadata,
});
}
async sendNotificationChangeEmail(email: string) {
@@ -371,7 +397,8 @@ export class AuthService implements OnApplicationBootstrap {
email: string,
link: string,
otp: string,
signUp: boolean
signUp: boolean,
metadata?: MailDeliveryMetadata
) {
return await this.mailer.send({
name: signUp ? 'SignUp' : 'SignIn',
@@ -381,6 +408,7 @@ export class AuthService implements OnApplicationBootstrap {
otp,
serverName: this.getServerName(),
},
metadata,
});
}
}
@@ -15,6 +15,7 @@ interface Context {
runtime: {
cleanupExpiredRuntimeStates: Sinon.SinonStub;
cleanupExpiredRuntimeGates: Sinon.SinonStub;
cleanupExpiredRollingQuota: Sinon.SinonStub;
};
}
@@ -24,6 +25,7 @@ test.before(async t => {
t.context.runtime = {
cleanupExpiredRuntimeStates: Sinon.stub(),
cleanupExpiredRuntimeGates: Sinon.stub(),
cleanupExpiredRollingQuota: Sinon.stub(),
};
t.context.module = await createTestingModule({
imports: [ScheduleModule.forRoot(), BackendRuntimeModule],
@@ -39,6 +41,7 @@ test.before(async t => {
test.beforeEach(t => {
t.context.runtime.cleanupExpiredRuntimeStates.reset();
t.context.runtime.cleanupExpiredRuntimeGates.reset();
t.context.runtime.cleanupExpiredRollingQuota.reset();
});
test.after.always(async t => {
@@ -49,9 +52,11 @@ test('backend-runtime housekeeping cleans runtime state and gate batches', async
t.context.runtime.cleanupExpiredRuntimeStates.onCall(0).resolves(1000);
t.context.runtime.cleanupExpiredRuntimeStates.onCall(1).resolves(2);
t.context.runtime.cleanupExpiredRuntimeGates.resolves(1);
t.context.runtime.cleanupExpiredRollingQuota.resolves(1);
await t.context.job.cleanExpiredRuntimeHousekeeping();
t.is(t.context.runtime.cleanupExpiredRuntimeStates.callCount, 2);
t.is(t.context.runtime.cleanupExpiredRuntimeGates.callCount, 1);
t.is(t.context.runtime.cleanupExpiredRollingQuota.callCount, 1);
});
@@ -10,4 +10,15 @@ import { BackendRuntimeProvider } from './provider';
})
export class BackendRuntimeModule {}
export { BackendRuntimeProvider } from './provider';
export {
BackendRuntimeProvider,
type RuntimeInviteAbuseAction,
type RuntimeInviteAbuseClaimedAction,
type RuntimeMailDeliveryQuotaDecision,
type RuntimeMailDeliveryQuotaInput,
type RuntimeQuotaSourceInput,
type RuntimeQuotaTargetDomainInput,
type RuntimeWorkspaceInviteQuotaDecision,
type RuntimeWorkspaceInviteQuotaInput,
type RuntimeWorkspaceInviteQuotaUsage,
} from './provider';
@@ -38,9 +38,12 @@ export class BackendRuntimeHousekeepingJob {
const gates = await this.cleanBatches(() =>
this.rt.cleanupExpiredRuntimeGates(1000)
);
const rollingQuota = await this.cleanBatches(() =>
this.rt.cleanupExpiredRollingQuota(1000)
);
this.logger.log(
`cleaned runtime housekeeping states=${states} gates=${gates}`
`cleaned runtime housekeeping states=${states} gates=${gates} rollingQuota=${rollingQuota}`
);
}
@@ -10,6 +10,187 @@ import { BackendRuntime, type BackendRuntimeHealth } from '../../native';
type RuntimeInstance = InstanceType<typeof BackendRuntime>;
export type RuntimeQuotaTargetDomainInput = {
domain: string;
count: number;
};
export type RuntimeQuotaSourceInput = {
trusted: boolean;
ip?: string;
country?: string;
asn?: number;
rayId?: string;
};
export type RuntimeWorkspaceInviteQuotaInput = {
actorUserId: string;
workspaceId: string;
requestId?: string;
targetCount: number;
targetDomains: RuntimeQuotaTargetDomainInput[];
source?: RuntimeQuotaSourceInput;
};
export type RuntimeWorkspaceInviteQuotaUsage = {
targetCount: number;
targetDomains: RuntimeQuotaTargetDomainInput[];
};
export type RuntimeInviteAbuseAction =
| 'ban_actor'
| 'quarantine_actor'
| 'quarantine_workspace'
| 'quarantine_source_cohort';
const RUNTIME_INVITE_ABUSE_ACTIONS = new Set<RuntimeInviteAbuseAction>([
'ban_actor',
'quarantine_actor',
'quarantine_workspace',
'quarantine_source_cohort',
]);
export type RuntimeInviteAbuseClaimedAction = {
action: RuntimeInviteAbuseAction;
subjectKey: string;
evidenceId: string;
actionId: string;
actorUserId: string;
workspaceId: string;
};
type NativeRuntimeInviteAbuseClaimedAction = Omit<
RuntimeInviteAbuseClaimedAction,
'action'
> & {
action: string;
};
export type RuntimeWorkspaceInviteQuotaDecision = {
allowed: boolean;
reservationId?: string;
retryAfterSeconds?: number;
reason?: string;
scopeKey?: string;
windowSeconds?: number;
limit?: number;
current?: number;
requested?: number;
actionRequired?: {
action: RuntimeInviteAbuseAction;
subjectKey: string;
evidenceId: string;
actionId: string;
};
};
type NativeRuntimeInviteAbuseActionRequired = Omit<
NonNullable<RuntimeWorkspaceInviteQuotaDecision['actionRequired']>,
'action'
> & {
action: string;
};
type NativeRuntimeWorkspaceInviteQuotaDecision = Omit<
RuntimeWorkspaceInviteQuotaDecision,
'actionRequired'
> & {
actionRequired?: NativeRuntimeInviteAbuseActionRequired;
};
export type RuntimeMailDeliveryQuotaInput = {
requestId?: string;
mailName: string;
recipient: {
email: string;
domain: string;
userId?: string;
};
metadata: {
actorUserId?: string;
workspaceId?: string;
notificationId?: string;
abuseSubjectKey?: string;
};
source?: RuntimeQuotaSourceInput;
};
export type RuntimeMailDeliveryQuotaDecision = {
allowed: boolean;
reservationId?: string;
mailClass: string;
retryAfterSeconds?: number;
reason?: string;
scopeKey?: string;
windowSeconds?: number;
limit?: number;
current?: number;
requested?: number;
};
type RuntimeQuotaMethods = RuntimeInstance & {
assertWorkspaceInviteQuotaV1(
input: RuntimeWorkspaceInviteQuotaInput
): Promise<NativeRuntimeWorkspaceInviteQuotaDecision>;
commitWorkspaceInviteQuotaV1(
reservationId: string,
usage: RuntimeWorkspaceInviteQuotaUsage
): Promise<boolean>;
releaseWorkspaceInviteQuotaV1(reservationId: string): Promise<boolean>;
assertMailDeliveryQuotaV1(
input: RuntimeMailDeliveryQuotaInput
): Promise<RuntimeMailDeliveryQuotaDecision>;
commitMailDeliveryQuotaV1(reservationId: string): Promise<boolean>;
releaseMailDeliveryQuotaV1(reservationId: string): Promise<boolean>;
cleanupExpiredRollingQuota(limit: number): Promise<number>;
isInviteAbuseUserQuarantinedOrBanned(userId: string): Promise<boolean>;
isInviteAbuseWorkspaceQuarantined(workspaceId: string): Promise<boolean>;
claimInviteAbuseAction(actionId: string, workerId: string): Promise<boolean>;
claimRetryableInviteAbuseActions(
workerId: string,
limit: number
): Promise<NativeRuntimeInviteAbuseClaimedAction[]>;
markInviteAbuseAction(
actionId: string,
workerId: string,
status: 'succeeded' | 'failed',
error?: string | null
): Promise<boolean>;
};
function normalizeInviteAbuseAction(action: string): RuntimeInviteAbuseAction {
if (RUNTIME_INVITE_ABUSE_ACTIONS.has(action as RuntimeInviteAbuseAction)) {
return action as RuntimeInviteAbuseAction;
}
throw new Error(`Unknown invite abuse action: ${action}`);
}
function normalizeWorkspaceInviteQuotaDecision(
decision: NativeRuntimeWorkspaceInviteQuotaDecision
): RuntimeWorkspaceInviteQuotaDecision {
const { actionRequired, ...rest } = decision;
if (!actionRequired) {
return rest;
}
return {
...rest,
actionRequired: {
...actionRequired,
action: normalizeInviteAbuseAction(actionRequired.action),
},
};
}
function normalizeClaimedInviteAbuseAction(
action: NativeRuntimeInviteAbuseClaimedAction
): RuntimeInviteAbuseClaimedAction {
return {
...action,
action: normalizeInviteAbuseAction(action.action),
};
}
@Injectable()
export class BackendRuntimeProvider
implements OnApplicationBootstrap, OnApplicationShutdown
@@ -66,6 +247,102 @@ export class BackendRuntimeProvider
);
}
async assertWorkspaceInviteQuotaV1(
input: RuntimeWorkspaceInviteQuotaInput
): Promise<RuntimeWorkspaceInviteQuotaDecision> {
return normalizeWorkspaceInviteQuotaDecision(
await this.measured('assertWorkspaceInviteQuotaV1', rt =>
this.quotaRuntime(rt).assertWorkspaceInviteQuotaV1(input)
)
);
}
async commitWorkspaceInviteQuotaV1(
reservationId: string,
usage: RuntimeWorkspaceInviteQuotaUsage
): Promise<boolean> {
return await this.measured('commitWorkspaceInviteQuotaV1', rt =>
this.quotaRuntime(rt).commitWorkspaceInviteQuotaV1(reservationId, usage)
);
}
async releaseWorkspaceInviteQuotaV1(reservationId: string): Promise<boolean> {
return await this.measured('releaseWorkspaceInviteQuotaV1', rt =>
this.quotaRuntime(rt).releaseWorkspaceInviteQuotaV1(reservationId)
);
}
async assertMailDeliveryQuotaV1(
input: RuntimeMailDeliveryQuotaInput
): Promise<RuntimeMailDeliveryQuotaDecision> {
return await this.measured('assertMailDeliveryQuotaV1', rt =>
this.quotaRuntime(rt).assertMailDeliveryQuotaV1(input)
);
}
async commitMailDeliveryQuotaV1(reservationId: string): Promise<boolean> {
return await this.measured('commitMailDeliveryQuotaV1', rt =>
this.quotaRuntime(rt).commitMailDeliveryQuotaV1(reservationId)
);
}
async releaseMailDeliveryQuotaV1(reservationId: string): Promise<boolean> {
return await this.measured('releaseMailDeliveryQuotaV1', rt =>
this.quotaRuntime(rt).releaseMailDeliveryQuotaV1(reservationId)
);
}
async cleanupExpiredRollingQuota(limit: number) {
return await this.measured('cleanupExpiredRollingQuota', rt =>
this.quotaRuntime(rt).cleanupExpiredRollingQuota(limit)
);
}
async isInviteAbuseUserQuarantinedOrBanned(userId: string) {
return await this.measured('isInviteAbuseUserQuarantinedOrBanned', rt =>
this.quotaRuntime(rt).isInviteAbuseUserQuarantinedOrBanned(userId)
);
}
async isInviteAbuseWorkspaceQuarantined(workspaceId: string) {
return await this.measured('isInviteAbuseWorkspaceQuarantined', rt =>
this.quotaRuntime(rt).isInviteAbuseWorkspaceQuarantined(workspaceId)
);
}
async claimInviteAbuseAction(actionId: string, workerId: string) {
return await this.measured('claimInviteAbuseAction', rt =>
this.quotaRuntime(rt).claimInviteAbuseAction(actionId, workerId)
);
}
async claimRetryableInviteAbuseActions(
workerId: string,
limit: number
): Promise<RuntimeInviteAbuseClaimedAction[]> {
return (
await this.measured('claimRetryableInviteAbuseActions', rt =>
this.quotaRuntime(rt).claimRetryableInviteAbuseActions(workerId, limit)
)
).map(normalizeClaimedInviteAbuseAction);
}
async markInviteAbuseAction(
actionId: string,
workerId: string,
status: 'succeeded' | 'failed',
error?: string | null
) {
return await this.measured('markInviteAbuseAction', rt =>
this.quotaRuntime(rt).markInviteAbuseAction(
actionId,
workerId,
status,
error
)
);
}
private async measured<T>(
method: string,
fn: (runtime: RuntimeInstance) => Promise<T>
@@ -78,6 +355,10 @@ export class BackendRuntimeProvider
)();
}
private quotaRuntime(runtime: RuntimeInstance): RuntimeQuotaMethods {
return runtime as unknown as RuntimeQuotaMethods;
}
private async runMigrationsOnce() {
if (this.migrationsStarted) {
return;
@@ -0,0 +1,14 @@
import { scanContentPolicyV1 } from '../native';
export function scanContentPolicy(value: string | null | undefined) {
return scanContentPolicyV1({
value: value ?? '',
checks: ['url_or_domain'],
});
}
export function containsUrlOrDomain(value: string | null | undefined) {
return scanContentPolicy(value).matches.some(
match => match.type === 'url_or_domain'
);
}
@@ -1,62 +1,70 @@
import { Prisma, PrismaClient } from '@prisma/client';
import test from 'ava';
import Sinon from 'sinon';
import { Mockers } from '../../../__tests__/mocks';
import { createTestingModule } from '../../../__tests__/utils';
import { Cache } from '../../../base';
import { Models } from '../../../models';
import { MailJob } from '../job';
import { MailSender } from '../sender';
let module: Awaited<ReturnType<typeof createTestingModule>>;
let cache: Cache;
let mailJob: MailJob;
let sender: MailSender;
let models: Models;
let db: PrismaClient;
test.before(async () => {
module = await createTestingModule();
cache = module.get(Cache);
mailJob = module.get(MailJob);
sender = module.get(MailSender);
models = module.get(Models);
db = module.get(PrismaClient);
});
test.after.always(async () => {
await module.close();
});
test.afterEach(() => {
test.afterEach.always(async () => {
Sinon.restore();
await db.mailDelivery.deleteMany();
});
test('should clear pending mail records when user is deleted', async t => {
async function createDelivery(
input: {
name: 'SignIn' | 'VerifyEmail' | 'MemberInvitation';
to: string;
props: Record<string, unknown>;
},
overrides: Partial<Parameters<Models['mailDelivery']['create']>[0]> = {}
) {
const mailClass =
input.name === 'MemberInvitation' ? 'workspace_invitation' : 'auth';
return await models.mailDelivery.create({
mailName: input.name,
mailClass,
priority: mailClass === 'auth' ? 'critical' : 'normal',
recipientEmail: input.to,
payload: input as Prisma.JsonObject,
...overrides,
});
}
async function delivery(id: string) {
return await db.mailDelivery.findUniqueOrThrow({ where: { id } });
}
test('should cancel pending mail deliveries when user is deleted', async t => {
const user = await module.create(Mockers.User);
const another = await module.create(Mockers.User);
const sendMailKey = 'mailjob:sendMail';
const retryMailKey = 'mailjob:sendMail:retry';
const userKey = `${sendMailKey}:SignIn:${user.email}`;
const userRetryKey = `${sendMailKey}:VerifyEmail:${user.email}`;
const anotherKey = `${sendMailKey}:SignIn:${another.email}`;
const senderRetryKey = `${retryMailKey}:MemberInvitation:invited@affine.pro`;
await cache.mapSet(sendMailKey, userKey, 1);
await cache.mapSet(sendMailKey, anotherKey, 1);
await cache.mapSet(
retryMailKey,
userRetryKey,
JSON.stringify({
startTime: Date.now(),
name: 'VerifyEmail',
to: user.email,
props: { url: 'https://affine.pro/verify' },
})
);
await cache.mapSet(
retryMailKey,
senderRetryKey,
JSON.stringify({
startTime: Date.now(),
const recipientDelivery = await createDelivery({
name: 'SignIn',
to: user.email,
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
});
const senderDelivery = await createDelivery(
{
name: 'MemberInvitation',
to: 'invited@affine.pro',
props: {
@@ -64,153 +72,126 @@ test('should clear pending mail records when user is deleted', async t => {
workspace: { $$workspaceId: 'workspace-id' },
url: 'https://affine.pro/invite',
},
})
},
{ actorUserId: user.id }
);
const anotherDelivery = await createDelivery({
name: 'SignIn',
to: another.email,
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
});
await mailJob.onUserDeleted({ ...user, ownedWorkspaces: [] });
t.true(module.queue.removeWhere.calledOnce);
t.is(module.queue.removeWhere.firstCall.args[0], 'notification.sendMail');
const shouldRemove = module.queue.removeWhere.firstCall.args[1];
t.true(
await shouldRemove({
to: user.email,
} as Jobs['notification.sendMail'])
);
t.true(
await shouldRemove({
name: 'MemberInvitation',
to: 'invited@affine.pro',
props: {
user: { $$userId: user.id },
workspace: { $$workspaceId: 'workspace-id' },
url: 'https://affine.pro/invite',
},
} as Jobs['notification.sendMail'])
);
t.false(
await shouldRemove({
to: another.email,
} as Jobs['notification.sendMail'])
);
t.is(await cache.mapGet(sendMailKey, userKey), undefined);
t.is(await cache.mapGet(retryMailKey, userRetryKey), undefined);
t.is(await cache.mapGet(retryMailKey, senderRetryKey), undefined);
t.is(await cache.mapGet(sendMailKey, anotherKey), 1);
t.is((await delivery(recipientDelivery.id)).status, 'canceled');
t.is((await delivery(senderDelivery.id)).status, 'canceled');
t.is((await delivery(anotherDelivery.id)).status, 'queued');
t.is((await delivery(recipientDelivery.id)).recipientEmail, null);
t.is((await delivery(senderDelivery.id)).payload, null);
});
test('should skip queued mail for disabled recipient', async t => {
const user = await module.create(Mockers.User, { disabled: true });
const send = Sinon.stub(sender, 'send').resolves(true);
await mailJob.sendMail({
startTime: Date.now(),
const send = Sinon.stub(sender, 'send').resolves({
status: 'accepted',
retryable: false,
});
const row = await createDelivery({
name: 'SignIn',
to: user.email,
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
});
await mailJob.processReadyDeliveries();
const updated = await delivery(row.id);
t.false(send.called);
t.truthy(await models.user.get(user.id, { withDisabled: true }));
t.is(updated.status, 'skipped');
t.is(updated.lastErrorCode, 'disabled_recipient');
t.is(updated.recipientEmail, null);
t.is(updated.payload, null);
});
test('should drop expired mail retry', async t => {
const send = Sinon.stub(sender, 'send').resolves(true);
await mailJob.sendMail({
startTime: Date.now() - 25 * 60 * 60 * 1000,
name: 'SignIn',
to: 'expired-retry@example.com',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
test('should not create sendable row for expired mail', async t => {
const send = Sinon.stub(sender, 'send').resolves({
status: 'accepted',
retryable: false,
});
t.false(send.called);
});
test('should drop time-sensitive mail after its business expiration', async t => {
const send = Sinon.stub(sender, 'send').resolves(true);
await mailJob.sendMail({
startTime: Date.now() - 31 * 60 * 1000,
name: 'SignIn',
to: 'expired-sign-in@example.com',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
const row = await createDelivery(
{
name: 'SignIn',
to: 'expired-retry@example.com',
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
},
});
{ expiresAt: new Date(Date.now() - 1) }
);
await mailJob.processReadyDeliveries();
const updated = await delivery(row.id);
t.false(send.called);
t.is(updated.status, 'failed');
t.is(updated.lastErrorCode, 'expired');
t.is(updated.retentionState, 'anonymized');
});
test('should use explicit mail expiration when provided', async t => {
const send = Sinon.stub(sender, 'send').resolves(true);
test('should not claim delivery when max attempts is zero', async t => {
const send = Sinon.stub(sender, 'send').resolves({
status: 'accepted',
retryable: false,
});
const row = await createDelivery(
{
name: 'SignIn',
to: 'max-attempts@example.com',
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
},
{ maxAttempts: 0 }
);
await mailJob.sendMail({
startTime: Date.now(),
expiresAt: Date.now() - 1,
name: 'MemberInvitation',
to: 'expired-invitation@example.com',
props: {
user: {
$$userId: 'owner-id',
await mailJob.processReadyDeliveries();
t.false(send.called);
t.is((await delivery(row.id)).status, 'queued');
});
test('should retry retryable send failures without mutating stored dynamic props', async t => {
const owner = await module.create(Mockers.User);
const member = await module.create(Mockers.User);
const workspace = await module.create(Mockers.Workspace, {
owner: { id: owner.id },
name: 'Safe Workspace',
});
Sinon.stub(sender, 'send').resolves({
status: 'failed',
retryable: true,
errorCode: 'transport_failed',
error: 'temporary failure',
});
const row = await createDelivery(
{
name: 'MemberInvitation',
to: member.email,
props: {
user: { $$userId: owner.id },
workspace: { $$workspaceId: workspace.id },
url: 'https://affine.pro/invite/test',
},
workspace: {
$$workspaceId: 'workspace-id',
},
url: 'https://affine.pro/invite/test',
},
{ actorUserId: owner.id, workspaceId: workspace.id }
);
await mailJob.processReadyDeliveries();
const updated = await delivery(row.id);
t.is(updated.status, 'retry_wait');
t.is(updated.attemptCount, 1);
t.like(updated.payload as object, {
props: {
user: { $$userId: owner.id },
workspace: { $$workspaceId: workspace.id },
},
});
t.false(send.called);
});
test('should drop mail retry after max attempts', async t => {
const send = Sinon.stub(sender, 'send').resolves(true);
await mailJob.sendMail({
startTime: Date.now(),
retryCount: 12,
name: 'SignIn',
to: 'max-retry@example.com',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
});
t.false(send.called);
});
test('should requeue legacy stringified retry mail', async t => {
const retryMailKey = 'mailjob:sendMail:retry';
const job: Jobs['notification.sendMail'] = {
startTime: Date.now(),
name: 'SignIn',
to: 'legacy-retry@example.com',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
};
const cacheKey = `${retryMailKey}:SignIn:${job.to}`;
Sinon.stub(cache, 'mapRandomKey')
.onFirstCall()
.resolves(cacheKey)
.onSecondCall()
.resolves(undefined);
await cache.mapSet(retryMailKey, cacheKey, JSON.stringify(job));
await mailJob.sendRetryMails();
t.true(module.queue.add.calledWith('notification.sendMail', job));
t.is(await cache.mapGet(retryMailKey, cacheKey), undefined);
});
test('should skip member invitation mail when rendered workspace name contains domain', async t => {
@@ -218,53 +199,123 @@ test('should skip member invitation mail when rendered workspace name contains d
const member = await module.create(Mockers.User);
const workspace = await module.create(Mockers.Workspace, {
owner: { id: owner.id },
name: 'BTC https://spam.example',
name: 'BTC example.com',
});
const send = Sinon.stub(sender, 'send').resolves(true);
await mailJob.sendMail({
startTime: Date.now(),
name: 'MemberInvitation',
to: member.email,
props: {
user: {
$$userId: owner.id,
const send = Sinon.stub(sender, 'send').resolves({
status: 'accepted',
retryable: false,
});
const row = await createDelivery(
{
name: 'MemberInvitation',
to: member.email,
props: {
user: { $$userId: owner.id },
workspace: { $$workspaceId: workspace.id },
url: 'https://affine.pro/invite/test',
},
workspace: {
$$workspaceId: workspace.id,
},
url: 'https://affine.pro/invite/test',
},
});
{ actorUserId: owner.id, workspaceId: workspace.id }
);
await mailJob.processReadyDeliveries();
const updated = await delivery(row.id);
t.false(send.called);
t.is(updated.status, 'skipped');
t.is(updated.lastErrorCode, 'dynamic_props_missing');
});
test('should keep dynamic mail props untouched for retry', async t => {
const owner = await module.create(Mockers.User);
const member = await module.create(Mockers.User);
const workspace = await module.create(Mockers.Workspace, {
owner: { id: owner.id },
name: 'Safe Workspace',
test('should mark accepted mail as sent and anonymize sendable payload', async t => {
const user = await module.create(Mockers.User);
Sinon.stub(sender, 'send').resolves({
status: 'accepted',
retryable: false,
providerMessageId: 'message-id',
providerResponse: '250 ok',
});
const row = await createDelivery({
name: 'SignIn',
to: user.email,
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
});
Sinon.stub(sender, 'send').resolves(false);
const job: Jobs['notification.sendMail'] = {
startTime: Date.now(),
name: 'MemberInvitation',
to: member.email,
props: {
user: {
$$userId: owner.id,
},
workspace: {
$$workspaceId: workspace.id,
},
url: 'https://affine.pro/invite/test',
},
};
await mailJob.sendMail(job);
await mailJob.processReadyDeliveries();
t.deepEqual(job.props.user, { $$userId: owner.id });
t.deepEqual(job.props.workspace, { $$workspaceId: workspace.id });
const updated = await delivery(row.id);
t.is(updated.status, 'sent');
t.is(updated.providerMessageId, 'message-id');
t.is(updated.recipientEmail, null);
t.is(updated.payload, null);
t.is(updated.retentionState, 'anonymized');
});
test('should claim critical priority before lower priority rows', async t => {
Sinon.stub(sender, 'send').resolves({
status: 'accepted',
retryable: false,
});
const low = await createDelivery(
{
name: 'SignIn',
to: 'low-priority@example.com',
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
},
{ priority: 'low' }
);
const critical = await createDelivery({
name: 'SignIn',
to: 'critical-priority@example.com',
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
});
await mailJob.processReadyDeliveries(1);
t.is((await delivery(critical.id)).status, 'sent');
t.is((await delivery(low.id)).status, 'queued');
});
test('should reclaim expired sending lease', async t => {
Sinon.stub(sender, 'send').resolves({
status: 'accepted',
retryable: false,
});
const row = await createDelivery({
name: 'SignIn',
to: 'reclaim@example.com',
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
});
await db.mailDelivery.update({
where: { id: row.id },
data: {
status: 'sending',
lockedBy: 'dead-worker',
lockedUntil: new Date(Date.now() - 1000),
},
});
await mailJob.processReadyDeliveries();
t.is((await delivery(row.id)).status, 'sent');
});
test('should delete retained anonymized terminal rows on worker tick', async t => {
const row = await createDelivery(
{
name: 'SignIn',
to: 'retention@example.com',
props: { url: 'https://affine.pro/sign-in', otp: '123456' },
},
{ status: 'skipped' }
);
await db.mailDelivery.update({
where: { id: row.id },
data: {
settledAt: new Date(Date.now() - 31 * 24 * 60 * 60 * 1000),
},
});
await mailJob.sendPendingMails();
t.is(await db.mailDelivery.count({ where: { id: row.id } }), 0);
});
@@ -0,0 +1,316 @@
import { createHash, createHmac } from 'node:crypto';
import { PrismaClient } from '@prisma/client';
import ava, { TestFn } from 'ava';
import Sinon from 'sinon';
import {
createTestingModule,
type TestingModule,
} from '../../../__tests__/utils';
import { CryptoHelper } from '../../../base';
import { Models } from '../../../models';
import { BackendRuntimeProvider } from '../../backend-runtime';
import { Mailer } from '../mailer';
import { MailSender } from '../sender';
interface Context {
module: TestingModule;
mailer: Mailer;
models: Models;
db: PrismaClient;
crypto: CryptoHelper;
runtime: {
assertMailDeliveryQuotaV1: Sinon.SinonStub;
commitMailDeliveryQuotaV1: Sinon.SinonStub;
releaseMailDeliveryQuotaV1: Sinon.SinonStub;
};
}
const test = ava as TestFn<Context>;
test.before(async t => {
t.context.runtime = {
assertMailDeliveryQuotaV1: Sinon.stub(),
commitMailDeliveryQuotaV1: Sinon.stub(),
releaseMailDeliveryQuotaV1: Sinon.stub(),
};
t.context.module = await createTestingModule({
tapModule: builder => {
builder
.overrideProvider(Mailer)
.useClass(Mailer)
.overrideProvider(MailSender)
.useValue({ configured: true })
.overrideProvider(BackendRuntimeProvider)
.useValue(t.context.runtime);
},
});
t.context.mailer = t.context.module.get(Mailer);
t.context.models = t.context.module.get(Models);
t.context.db = t.context.module.get(PrismaClient);
t.context.crypto = t.context.module.get(CryptoHelper);
});
test.beforeEach(t => {
t.context.runtime.assertMailDeliveryQuotaV1.reset();
t.context.runtime.commitMailDeliveryQuotaV1.reset();
t.context.runtime.releaseMailDeliveryQuotaV1.reset();
t.context.runtime.assertMailDeliveryQuotaV1.resolves({
allowed: true,
reservationId: '00000000-0000-0000-0000-000000000001',
mailClass: 'auth',
});
t.context.runtime.commitMailDeliveryQuotaV1.resolves(true);
t.context.runtime.releaseMailDeliveryQuotaV1.resolves(true);
});
test.afterEach.always(async t => {
Sinon.restore();
await t.context.db.mailDelivery.deleteMany();
});
test.after.always(async t => {
await t.context.module.close();
});
test('trySend creates a delivery row and commits quota reservation', async t => {
const sent = await t.context.mailer.trySend({
name: 'SignIn',
to: 'auth-user@example.com',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
metadata: {
dedupeKey: 'signin:auth-user@example.com:1',
recipientUserId: 'user-1',
source: { trusted: false },
},
});
const row = await t.context.db.mailDelivery.findFirstOrThrow({
where: { dedupeKey: 'signin:auth-user@example.com:1' },
});
t.true(sent);
t.is(row.status, 'queued');
t.is(row.recipientUserId, 'user-1');
t.is(row.mailClass, 'auth');
t.is(
row.recipientHash,
createHmac('sha256', t.context.crypto.keyPair.sha256.privateKey)
.update('auth-user@example.com')
.digest('hex')
);
t.not(
row.recipientHash,
createHash('sha256').update('auth-user@example.com').digest('hex')
);
t.true(
t.context.runtime.commitMailDeliveryQuotaV1.calledOnceWithExactly(
'00000000-0000-0000-0000-000000000001'
)
);
});
test('cancelByRecipient matches the keyed recipient hash', async t => {
await t.context.mailer.trySend({
name: 'SignIn',
to: ' Delete-Me@Example.COM ',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
metadata: {
dedupeKey: 'signin:delete-me@example.com:1',
source: { trusted: false },
},
});
await t.context.models.mailDelivery.cancelByRecipient(
'delete-me@example.com'
);
const row = await t.context.db.mailDelivery.findFirstOrThrow({
where: { dedupeKey: 'signin:delete-me@example.com:1' },
});
t.is(
row.recipientHash,
createHmac('sha256', t.context.crypto.keyPair.sha256.privateKey)
.update('delete-me@example.com')
.digest('hex')
);
t.is(row.status, 'canceled');
t.is(row.lastErrorCode, 'recipient_deleted');
});
test('dedupe replay does not consume another mail quota reservation', async t => {
const command = {
name: 'SignIn' as const,
to: 'dedupe@example.com',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
metadata: {
dedupeKey: 'signin:dedupe@example.com:1',
source: { trusted: false },
},
};
t.true(await t.context.mailer.trySend(command));
t.true(await t.context.mailer.trySend(command));
t.is(t.context.runtime.assertMailDeliveryQuotaV1.callCount, 1);
t.is(t.context.runtime.commitMailDeliveryQuotaV1.callCount, 1);
t.is(
await t.context.db.mailDelivery.count({
where: { dedupeKey: command.metadata.dedupeKey },
}),
1
);
});
test('quota denial records skipped deliveries without committing quota', async t => {
for (const quota of [
{
email: 'limited@example.com',
mailClass: 'auth',
reason: 'recipient_rate_limited',
},
{
email: 'unmapped@example.com',
mailClass: 'unknown',
reason: 'unmapped_mail_name',
},
] as const) {
t.context.runtime.assertMailDeliveryQuotaV1.resolves({
allowed: false,
mailClass: quota.mailClass,
reason: quota.reason,
});
const dedupeKey = `signin:${quota.email}:1`;
const sent = await t.context.mailer.trySend({
name: 'SignIn',
to: quota.email,
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
metadata: {
dedupeKey,
source: { trusted: false },
},
});
const row = await t.context.db.mailDelivery.findFirstOrThrow({
where: { dedupeKey },
});
t.false(sent, quota.reason);
t.is(row.status, 'skipped', quota.reason);
t.is(row.mailClass, quota.mailClass, quota.reason);
t.is(row.lastErrorCode, quota.reason, quota.reason);
t.is(row.recipientEmail, null, quota.reason);
t.false(t.context.runtime.commitMailDeliveryQuotaV1.called, quota.reason);
t.context.runtime.assertMailDeliveryQuotaV1.resetHistory();
}
});
test('skip records terminal delivery without quota admission', async t => {
const sent = await t.context.mailer.skip(
{
name: 'MemberInvitation',
to: 'skip@example.com',
props: {
url: 'https://affine.pro/invite',
user: { $$userId: 'actor-1' },
workspace: { $$workspaceId: 'workspace-1' },
},
metadata: {
dedupeKey: 'invite:skip@example.com:1',
recipientUserId: 'user-1',
actorUserId: 'actor-1',
workspaceId: 'workspace-1',
source: { trusted: false },
},
},
{
mailClass: 'workspace_invitation',
reason: 'workspace_name_contains_domain',
}
);
const row = await t.context.db.mailDelivery.findFirstOrThrow({
where: { dedupeKey: 'invite:skip@example.com:1' },
});
t.false(sent);
t.is(row.status, 'skipped');
t.is(row.mailClass, 'workspace_invitation');
t.is(row.lastErrorCode, 'workspace_name_contains_domain');
t.is(row.maxAttempts, 0);
t.false(t.context.runtime.assertMailDeliveryQuotaV1.called);
});
test('send releases quota reservation and rethrows when ledger write fails', async t => {
Sinon.stub(t.context.models.mailDelivery, 'create').rejects(
new Error('ledger failed')
);
await t.throwsAsync(
t.context.mailer.send({
name: 'SignIn',
to: 'throw@example.com',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
metadata: {
source: { trusted: false },
},
}),
{ message: 'ledger failed' }
);
t.true(
t.context.runtime.releaseMailDeliveryQuotaV1.calledOnceWithExactly(
'00000000-0000-0000-0000-000000000001'
)
);
});
test('send cancels queued delivery when quota commit fails', async t => {
t.context.runtime.commitMailDeliveryQuotaV1.rejects(
new Error('commit failed')
);
await t.throwsAsync(
t.context.mailer.send({
name: 'SignIn',
to: 'commit-failed@example.com',
props: {
url: 'https://affine.pro/sign-in',
otp: '123456',
},
metadata: {
dedupeKey: 'signin:commit-failed@example.com:1',
source: { trusted: false },
},
}),
{ message: 'commit failed' }
);
const row = await t.context.db.mailDelivery.findFirstOrThrow({
where: { dedupeKey: 'signin:commit-failed@example.com:1' },
});
t.is(row.status, 'canceled');
t.is(row.lastErrorCode, 'quota_commit_failed');
t.true(
t.context.runtime.releaseMailDeliveryQuotaV1.calledOnceWithExactly(
'00000000-0000-0000-0000-000000000001'
)
);
});
@@ -0,0 +1,257 @@
import { PrismaClient } from '@prisma/client';
import test from 'ava';
import { createApp, type TestingApp } from '../../../__tests__/e2e/test';
import { Mockers } from '../../../__tests__/mocks';
let app: TestingApp;
function startOfUtcHour(value: Date) {
return new Date(
Date.UTC(
value.getUTCFullYear(),
value.getUTCMonth(),
value.getUTCDate(),
value.getUTCHours()
)
);
}
test.before(async () => {
app = await createApp();
});
test.beforeEach(async () => {
await app.get(PrismaClient).mailDelivery.deleteMany();
});
test.after.always(async () => {
await app.close();
});
test.afterEach.always(async () => {
await app.get(PrismaClient).mailDelivery.deleteMany();
});
test('sendTestEmail rejects non-admin users before SMTP or ledger', async t => {
const user = await app.create(Mockers.User);
await app.login(user);
const response = await app.request('post', '/graphql').send({
query: /* GraphQL */ `
mutation SendTestEmail($config: JSONObject!) {
sendTestEmail(config: $config)
}
`,
variables: {
config: {
name: '',
host: 'smtp.example.com',
port: 587,
username: 'user',
password: 'password',
ignoreTLS: false,
sender: 'AFFiNE <noreply@example.com>',
},
},
});
t.is(response.status, 200);
t.truthy(response.body.errors);
t.is(await app.get(PrismaClient).mailDelivery.count(), 0);
});
test('adminMailDeliveries returns timeline series for status type and outcome', async t => {
const db = app.get(PrismaClient);
const admin = await app.create(Mockers.User, {
feature: 'administrator',
});
const now = new Date();
const createdAt = new Date(startOfUtcHour(now).getTime() - 60 * 60 * 1000);
const base = {
priority: 'normal',
recipientHash: 'hash',
recipientDomain: 'example.com',
sendAfter: now,
retentionState: 'anonymized',
anonymizedAt: now,
createdAt,
updatedAt: now,
};
await db.mailDelivery.createMany({
data: [
{
...base,
mailName: 'SignIn',
mailClass: 'auth',
status: 'sent',
settledAt: now,
sentAt: now,
},
{
...base,
mailName: 'MemberInvitation',
mailClass: 'workspace_invitation',
status: 'failed',
settledAt: now,
failedAt: now,
lastErrorCode: 'transport_failed',
},
{
...base,
mailName: 'Mention',
mailClass: 'notification',
status: 'queued',
recipientEmail: 'queued@example.com',
payload: {},
retentionState: 'full',
anonymizedAt: null,
},
],
});
await app.login(admin);
const response = await app.request('post', '/graphql').send({
query: /* GraphQL */ `
query AdminMailDeliveries($input: AdminMailDeliveriesInput) {
adminMailDeliveries(input: $input) {
window {
bucket
effectiveSize
}
summary {
total
sent
failed
queued
successRate
}
byStatus {
key
total
points {
bucket
count
}
}
byType {
key
total
}
byOutcome {
key
total
}
}
}
`,
variables: {
input: {
hours: 24,
},
},
});
t.is(response.status, 200);
t.falsy(response.body.errors);
const analytics = response.body.data.adminMailDeliveries;
t.is(analytics.window.bucket, 'Hour');
t.is(analytics.window.effectiveSize, 24);
t.like(analytics.summary, {
total: 3,
sent: 1,
failed: 1,
queued: 1,
successRate: 0.5,
});
t.true(
analytics.byStatus.some((series: { key: string; total: number }) => {
return series.key === 'sent' && series.total === 1;
})
);
const sent = analytics.byStatus.find(
(series: { key: string; points: { bucket: string; count: number }[] }) =>
series.key === 'sent'
);
t.deepEqual(
sent?.points.filter((point: { count: number }) => point.count > 0),
[{ bucket: createdAt.toISOString(), count: 1 }]
);
t.true(
analytics.byType.some(
(series: { key: string; total: number }) =>
series.key === 'workspace_invitation' && series.total === 1
)
);
t.true(
analytics.byOutcome.some(
(series: { key: string; total: number }) =>
series.key === 'pending' && series.total === 1
)
);
});
test('adminMailDeliveries uses day buckets for seven day window', async t => {
const admin = await app.create(Mockers.User, {
feature: 'administrator',
});
await app.login(admin);
const response = await app.request('post', '/graphql').send({
query: /* GraphQL */ `
query AdminMailDeliveries($input: AdminMailDeliveriesInput) {
adminMailDeliveries(input: $input) {
window {
bucket
effectiveSize
}
}
}
`,
variables: {
input: {
hours: 168,
},
},
});
t.is(response.status, 200);
t.falsy(response.body.errors);
t.like(response.body.data.adminMailDeliveries.window, {
bucket: 'Day',
effectiveSize: 7,
});
});
test('adminMailDeliveries rejects unsupported window sizes', async t => {
const admin = await app.create(Mockers.User, {
feature: 'administrator',
});
await app.login(admin);
const response = await app.request('post', '/graphql').send({
query: /* GraphQL */ `
query AdminMailDeliveries($input: AdminMailDeliveriesInput) {
adminMailDeliveries(input: $input) {
window {
requestedSize
}
}
}
`,
variables: {
input: {
hours: 48,
},
},
});
t.is(response.status, 200);
t.truthy(response.body.errors);
t.regex(
response.body.errors[0].message,
/Mail delivery analytics window must be 24 or 168 hours/
);
});
@@ -16,6 +16,11 @@ declare global {
};
fallbackDomains: ConfigItem<string[]>;
deliveryWorker: {
batchSize: number;
leaseMs: number;
retentionDays: number;
};
fallbackSMTP: {
name: string;
host: string;
@@ -71,6 +76,21 @@ defineModuleConfig('mailer', {
default: [],
shape: z.array(z.string()),
},
'deliveryWorker.batchSize': {
desc: 'Number of mail delivery rows claimed by each worker tick.',
default: 50,
shape: z.number().int().min(1).max(1000),
},
'deliveryWorker.leaseMs': {
desc: 'Mail delivery worker lease duration in milliseconds.',
default: 2 * 60 * 1000,
shape: z.number().int().min(1000),
},
'deliveryWorker.retentionDays': {
desc: 'Days to retain anonymized terminal mail delivery ledger rows.',
default: 30,
shape: z.number().int().min(1),
},
'fallbackSMTP.name': {
desc: 'Hostname used for fallback SMTP HELO/EHLO (e.g. mail.example.com). Leave empty to use the system hostname.',
default: '',
@@ -2,6 +2,7 @@ import './config';
import { Module } from '@nestjs/common';
import { BackendRuntimeModule } from '../backend-runtime';
import { DocStorageModule } from '../doc';
import { StorageModule } from '../storage';
import { MailJob } from './job';
@@ -10,7 +11,7 @@ import { MailResolver } from './resolver';
import { MailSender } from './sender';
@Module({
imports: [DocStorageModule, StorageModule],
imports: [BackendRuntimeModule, DocStorageModule, StorageModule],
providers: [MailSender, Mailer, MailJob, MailResolver],
exports: [Mailer],
})
+178 -243
View File
@@ -1,106 +1,184 @@
import { randomUUID } from 'node:crypto';
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { getStreamAsBuffer } from 'get-stream';
import { Cache, JOB_SIGNAL, JobQueue, OnEvent, OnJob, sleep } from '../../base';
import { type MailName, MailProps, Renderers } from '../../mails';
import { Config, metrics, OnEvent } from '../../base';
import { type MailName, Renderers } from '../../mails';
import { UserProps, WorkspaceProps } from '../../mails/components';
import { Models } from '../../models';
import { MailDeliveryRow, Models } from '../../models';
import { containsUrlOrDomain } from '../content-policy';
import { DocReader } from '../doc/reader';
import { WorkspaceBlobStorage } from '../storage';
import { containsUrlOrDomain } from '../workspaces/abuse';
import { MailSender, SendOptions } from './sender';
import { SendMailPayload } from './types';
type DynamicallyFetchedProps<Props> = {
[Key in keyof Props]: Props[Key] extends infer Prop
? Prop extends UserProps
? {
$$userId: string;
} & Omit<Prop, 'email' | 'avatar'>
: Prop extends WorkspaceProps
? {
$$workspaceId: string;
} & Omit<Prop, 'name' | 'avatar'>
: Prop
: never;
};
type SendMailJob<Mail extends MailName = MailName, Props = MailProps<Mail>> = {
name: Mail;
to: string;
// NOTE(@forehalo):
// workspace avatar currently send as base64 img instead of a avatar url,
// so the content might be too large to be put in job payload.
props: DynamicallyFetchedProps<Props>;
};
declare global {
interface Jobs {
'notification.sendMail': {
startTime: number;
retryCount?: number;
expiresAt?: number;
} & {
[K in MailName]: SendMailJob<K>;
}[MailName];
}
}
const sendMailKey = 'mailjob:sendMail';
const retryMailKey = 'mailjob:sendMail:retry';
const sendMailCacheKey = (name: string, to: string) =>
`${sendMailKey}:${name}:${to}`;
const retryMaxPerTick = 20;
const retryFirstTime = 3;
const retryMaxAttempts = 12;
const retryMaxAge = 24 * 60 * 60 * 1000;
const magicLinkExpiresIn = 30 * 60 * 1000;
const mailExpiresIn: Partial<Record<MailName, number>> = {
SignIn: magicLinkExpiresIn,
SignUp: magicLinkExpiresIn,
SetPassword: magicLinkExpiresIn,
ChangePassword: magicLinkExpiresIn,
VerifyEmail: magicLinkExpiresIn,
ChangeEmail: magicLinkExpiresIn,
VerifyChangeEmail: magicLinkExpiresIn,
type DynamicProp = Record<string, unknown> & {
$$workspaceId?: string;
$$userId?: string;
};
@Injectable()
export class MailJob {
private readonly logger = new Logger('MailJob');
private readonly logger = new Logger('MailDeliveryWorker');
private readonly workerId = `mail-delivery-${process.pid}-${randomUUID()}`;
constructor(
private readonly cache: Cache,
private readonly queue: JobQueue,
private readonly sender: MailSender,
private readonly doc: DocReader,
private readonly workspaceBlob: WorkspaceBlobStorage,
private readonly models: Models
private readonly models: Models,
private readonly config: Config
) {}
private calculateRetryDelay(startTime: number) {
const elapsed = Date.now() - startTime;
return Math.min(30 * 1000, Math.round(elapsed / 2000) * 1000);
@OnEvent('user.deleted')
async onUserDeleted(user: Events['user.deleted']) {
await Promise.all([
this.models.mailDelivery.cancelByRecipient(user.email),
this.models.mailDelivery.cancelMemberInvitationByActor(user.id),
]);
}
private getRetryExhaustedReason({
startTime,
retryCount,
expiresAt,
name,
}: Jobs['notification.sendMail']) {
const expiredAt =
expiresAt ?? startTime + (mailExpiresIn[name] ?? retryMaxAge);
if (Date.now() > expiredAt) {
return 'expired';
@Cron(CronExpression.EVERY_MINUTE)
async sendPendingMails() {
await this.processReadyDeliveries();
await this.cleanupRetainedDeliveries();
await this.recordDeliveryMetrics();
}
async processReadyDeliveries(
batchSize = this.config.mailer.deliveryWorker.batchSize
) {
const rows = await this.models.mailDelivery.claimReady(this.workerId, {
batchSize,
leaseMs: this.config.mailer.deliveryWorker.leaseMs,
});
for (const row of rows) {
await this.processDelivery(row);
}
return rows.length;
}
private async cleanupRetainedDeliveries() {
const retentionMs =
this.config.mailer.deliveryWorker.retentionDays * 24 * 60 * 60 * 1000;
const before = new Date(Date.now() - retentionMs);
await this.models.mailDelivery.deleteAnonymizedBefore(
before,
this.config.mailer.deliveryWorker.batchSize
);
}
private async recordDeliveryMetrics() {
const snapshot = await this.models.mailDelivery.metricsSnapshot();
metrics.mail.gauge('retry_wait_backlog').record(snapshot.retryWait);
metrics.mail.gauge('failed_recent').record(snapshot.failedRecent);
metrics.mail.gauge('expired_lease_backlog').record(snapshot.expiredLeases);
metrics.mail.histogram('ready_delay_ms').record(snapshot.readyDelayMs);
}
private async processDelivery(row: MailDeliveryRow) {
if (!row.recipientEmail || !row.payload) {
await this.models.mailDelivery.markSkipped(
row.id,
this.workerId,
'missing_payload'
);
return;
}
if (row.expiresAt && row.expiresAt.getTime() <= Date.now()) {
await this.models.mailDelivery.markFailed(
row.id,
this.workerId,
'expired'
);
return;
}
if (await this.shouldSkipRecipient(row.recipientEmail)) {
await this.models.mailDelivery.markSkipped(
row.id,
this.workerId,
'disabled_recipient'
);
return;
}
if ((retryCount ?? 0) > retryMaxAttempts) {
return 'max attempts reached';
const payload = row.payload as SendMailPayload;
const rendered = await this.renderPayload(payload);
if (!rendered) {
await this.models.mailDelivery.markSkipped(
row.id,
this.workerId,
'dynamic_props_missing'
);
return;
}
if (row.expiresAt && row.expiresAt.getTime() <= Date.now()) {
await this.models.mailDelivery.markFailed(
row.id,
this.workerId,
'expired'
);
return;
}
return;
const attempt = await this.models.mailDelivery.markAttemptStarted(
row.id,
this.workerId
);
if (!attempt) {
return;
}
const result = await this.sender.send(payload.name, {
to: row.recipientEmail,
...rendered.options,
...rendered.content,
});
if (result.status === 'accepted') {
await this.models.mailDelivery.markSent(row.id, this.workerId, {
providerMessageId: result.providerMessageId,
providerResponse: result.providerResponse,
});
return;
}
const nextSendAfter = this.nextSendAfter(attempt.attemptCount);
const canRetry =
result.retryable &&
attempt.attemptCount < attempt.maxAttempts &&
(!attempt.expiresAt || nextSendAfter < attempt.expiresAt);
if (canRetry) {
const retry = await this.models.mailDelivery.markRetry(
row.id,
this.workerId,
{
sendAfter: nextSendAfter,
errorCode: result.errorCode,
error: result.error,
}
);
if (retry) {
return;
}
}
await this.models.mailDelivery.markFailed(
row.id,
this.workerId,
attempt.expiresAt && nextSendAfter >= attempt.expiresAt
? 'expired'
: (result.errorCode ?? result.status),
result.error ?? result.providerResponse ?? undefined
);
}
private nextSendAfter(attemptCount: number) {
const delayMs = Math.min(30 * 60 * 1000, 2 ** attemptCount * 30 * 1000);
return new Date(Date.now() + delayMs);
}
private async shouldSkipRecipient(to: string) {
@@ -111,72 +189,16 @@ export class MailJob {
return user?.disabled === true;
}
private async deleteRecipientMailCache(to: string) {
const suffix = `:${to}`;
await Promise.all(
[sendMailKey, retryMailKey].map(async map => {
const keys = await this.cache.mapKeys(map);
await Promise.all(
keys
.filter(key => key.endsWith(suffix))
.map(key => this.cache.mapDelete(map, key))
);
})
);
}
private isInvitationMailSentByUser(
job: Jobs['notification.sendMail'],
userId: string
) {
return (
job.name === 'MemberInvitation' &&
'user' in job.props &&
typeof job.props.user === 'object' &&
job.props.user !== null &&
'$$userId' in job.props.user &&
job.props.user.$$userId === userId
);
}
private async deleteInvitationMailCacheBySender(userId: string) {
const keys = await this.cache.mapKeys(retryMailKey);
await Promise.all(
keys.map(async key => {
const job = await this.cache.mapGet<
Jobs['notification.sendMail'] | string
>(retryMailKey, key);
const jobData =
typeof job === 'string'
? (JSON.parse(job) as Jobs['notification.sendMail'])
: job;
if (jobData && this.isInvitationMailSentByUser(jobData, userId)) {
await this.cache.mapDelete(retryMailKey, key);
}
})
);
}
private async sendMailInternal({
startTime,
name,
to,
props,
}: Jobs['notification.sendMail']) {
if (await this.shouldSkipRecipient(to)) {
this.logger.debug(`Skip mail [${name}] to disabled user [${to}]`);
return;
}
private async renderPayload(payload: SendMailPayload) {
let options: Partial<SendOptions> = {};
const renderedProps = { ...props };
const renderedProps = { ...payload.props };
for (const key in renderedProps) {
// @ts-expect-error allow
const val = renderedProps[key];
const val = renderedProps[key as keyof typeof renderedProps] as
| DynamicProp
| undefined;
if (val && typeof val === 'object') {
if ('$$workspaceId' in val) {
if (typeof val.$$workspaceId === 'string') {
const workspaceProps = await this.fetchWorkspaceProps(
val.$$workspaceId
);
@@ -186,11 +208,11 @@ export class MailJob {
}
if (
name === 'MemberInvitation' &&
payload.name === 'MemberInvitation' &&
containsUrlOrDomain(workspaceProps.name)
) {
this.logger.warn(
`Skip mail [${name}] to [${to}], reason=workspace name contains url or domain`
`Skip mail [${payload.name}] to [${payload.to}], reason=workspace name contains url or domain`
);
return;
}
@@ -206,57 +228,42 @@ export class MailJob {
];
workspaceProps.avatar = 'cid:workspaceAvatar';
}
// @ts-expect-error replacement
renderedProps[key] = workspaceProps;
} else if ('$$userId' in val) {
Object.assign(val, workspaceProps);
delete val.$$workspaceId;
} else if (typeof val.$$userId === 'string') {
const userProps = await this.fetchUserProps(val.$$userId);
if (!userProps) {
return;
}
// @ts-expect-error replacement
renderedProps[key] = userProps;
Object.assign(val, userProps);
delete val.$$userId;
}
}
}
if (
name === 'MemberInvitation' &&
payload.name === 'MemberInvitation' &&
'workspace' in renderedProps &&
containsUrlOrDomain(
(renderedProps.workspace as WorkspaceProps | undefined)?.name
)
) {
this.logger.warn(
`Skip mail [${name}] to [${to}], reason=workspace name contains url or domain`
`Skip mail [${payload.name}] to [${payload.to}], reason=workspace name contains url or domain`
);
return;
}
try {
const result = await this.sender.send(name, {
to,
...(await Renderers[name](
// @ts-expect-error the job trigger part has been typechecked
renderedProps
)),
...options,
});
if (!result) {
// wait for a while before retrying
const retryDelay = this.calculateRetryDelay(startTime);
await sleep(retryDelay);
return JOB_SIGNAL.Retry;
}
return undefined;
} catch (e) {
this.logger.error(`Failed to send mail [${name}] to [${to}]`, e);
// wait for a while before retrying
const retryDelay = this.calculateRetryDelay(startTime);
await sleep(retryDelay);
return JOB_SIGNAL.Retry;
}
return {
options,
content: await (
Renderers[payload.name] as (
props: unknown
) => ReturnType<(typeof Renderers)[MailName]>
)(renderedProps),
};
}
private async fetchWorkspaceProps(workspaceId: string) {
@@ -294,76 +301,4 @@ export class MailJob {
return { email: user.email } satisfies UserProps;
}
@OnJob('notification.sendMail')
async sendMail(job: Jobs['notification.sendMail']) {
const cacheKey = sendMailCacheKey(job.name, job.to);
job.retryCount = (job.retryCount ?? 0) + 1;
const exhaustedReason = this.getRetryExhaustedReason(job);
if (exhaustedReason) {
this.logger.warn(
`Drop mail [${job.name}] to [${job.to}], reason=${exhaustedReason}`
);
await Promise.all([
this.cache.mapDelete(sendMailKey, cacheKey),
this.cache.mapDelete(retryMailKey, cacheKey),
]);
return;
}
const retried = await this.cache.mapIncrease(sendMailKey, cacheKey, 1);
if (retried <= retryFirstTime) {
const ret = await this.sendMailInternal(job);
if (!ret) await this.cache.mapDelete(sendMailKey, cacheKey);
return ret;
}
await this.cache.mapSet(retryMailKey, cacheKey, job);
await this.cache.mapDelete(sendMailKey, cacheKey);
return undefined;
}
@OnEvent('user.deleted')
async onUserDeleted(user: Events['user.deleted']) {
await Promise.all([
this.deleteRecipientMailCache(user.email),
this.deleteInvitationMailCacheBySender(user.id),
this.queue.removeWhere(
'notification.sendMail',
job =>
job.to === user.email || this.isInvitationMailSentByUser(job, user.id)
),
]);
}
@Cron(CronExpression.EVERY_MINUTE)
async sendRetryMails() {
// pick random one from the retry map
let processed = 0;
let key = await this.cache.mapRandomKey(retryMailKey);
while (key && processed < retryMaxPerTick) {
try {
const job = await this.cache.mapGet<
Jobs['notification.sendMail'] | string
>(retryMailKey, key);
if (job) {
const jobData =
typeof job === 'string'
? (JSON.parse(job) as Jobs['notification.sendMail'])
: job;
await this.queue.add('notification.sendMail', jobData);
// wait for a while before retrying
const retryDelay = this.calculateRetryDelay(jobData.startTime);
await sleep(retryDelay);
}
await this.cache.mapDelete(retryMailKey, key);
} catch (e) {
this.logger.error(
`Failed to re-queue retry mail job for key [${key}]`,
e
);
}
key = await this.cache.mapRandomKey(retryMailKey);
processed++;
}
}
}
+161 -15
View File
@@ -1,13 +1,50 @@
import { Injectable } from '@nestjs/common';
import { EmailServiceNotConfigured, JobQueue } from '../../base';
import { EmailServiceNotConfigured } from '../../base';
import type { MailName } from '../../mails';
import { Models } from '../../models';
import { BackendRuntimeProvider } from '../backend-runtime';
import { MailSender } from './sender';
import type { SendMailCommand } from './types';
type MailCommand = SendMailCommand;
type SkipMailOptions = {
mailClass: string;
reason: string;
priority?: 'critical' | 'high' | 'normal' | 'low';
};
function recipientDomain(email: string) {
const parts = email.trim().toLowerCase().split('@');
return parts.length === 2 ? parts[1] : '';
}
function defaultPriority(mailClass: string): 'critical' | 'high' | 'normal' {
if (mailClass === 'auth') {
return 'critical';
}
if (mailClass === 'billing_license') {
return 'high';
}
return 'normal';
}
function serializePayload(command: MailCommand) {
return JSON.parse(
JSON.stringify({
name: command.name,
to: command.to,
props: command.props,
})
);
}
@Injectable()
export class Mailer {
constructor(
private readonly queue: JobQueue,
private readonly sender: MailSender
private readonly sender: MailSender,
private readonly models: Models,
private readonly runtime: BackendRuntimeProvider
) {}
/**
@@ -15,14 +52,43 @@ export class Mailer {
*
* @note never throw
*/
async trySend(command: Omit<Jobs['notification.sendMail'], 'startTime'>) {
async trySend(command: MailCommand) {
return this.send(command, true);
}
async send(
command: Omit<Jobs['notification.sendMail'], 'startTime'>,
suppressError = false
) {
async skip(command: MailCommand, options: SkipMailOptions) {
const metadata = command.metadata ?? {};
const deduped = metadata.dedupeKey
? await this.models.mailDelivery.findByDedupeKey(metadata.dedupeKey)
: null;
if (deduped) {
return false;
}
await this.models.mailDelivery.create({
mailName: command.name,
mailClass: options.mailClass,
priority:
options.priority ??
metadata.priority ??
defaultPriority(options.mailClass),
status: 'skipped',
dedupeKey: metadata.dedupeKey,
recipientEmail: command.to,
recipientUserId: metadata.recipientUserId,
actorUserId: metadata.actorUserId,
workspaceId: metadata.workspaceId,
notificationId: metadata.notificationId,
abuseSubjectKey: metadata.abuseSubjectKey,
payload: serializePayload(command),
expiresAt: metadata.expiresAt,
maxAttempts: 0,
lastErrorCode: options.reason,
});
return false;
}
async send(command: MailCommand, suppressError = false) {
if (!this.sender.configured) {
if (suppressError) {
return false;
@@ -30,15 +96,95 @@ export class Mailer {
throw new EmailServiceNotConfigured();
}
let reservationId: string | undefined;
let deliveryId: string | undefined;
try {
await this.queue.add(
'notification.sendMail',
Object.assign({}, command, {
startTime: Date.now(),
}) as Jobs['notification.sendMail']
);
const metadata = command.metadata ?? {};
const deduped = metadata.dedupeKey
? await this.models.mailDelivery.findByDedupeKey(metadata.dedupeKey)
: null;
if (deduped) {
return !['failed', 'canceled', 'skipped'].includes(deduped.status);
}
const decision = await this.runtime.assertMailDeliveryQuotaV1({
mailName: command.name as MailName,
recipient: {
email: command.to,
domain: recipientDomain(command.to),
userId: metadata.recipientUserId,
},
metadata: {
actorUserId: metadata.actorUserId,
workspaceId: metadata.workspaceId,
notificationId: metadata.notificationId,
abuseSubjectKey: metadata.abuseSubjectKey,
},
source: metadata.source,
});
reservationId = decision.reservationId;
if (!decision.allowed) {
await this.models.mailDelivery.create({
mailName: command.name,
mailClass: decision.mailClass,
priority: metadata.priority ?? defaultPriority(decision.mailClass),
status: 'skipped',
dedupeKey: metadata.dedupeKey,
recipientEmail: command.to,
recipientUserId: metadata.recipientUserId,
actorUserId: metadata.actorUserId,
workspaceId: metadata.workspaceId,
notificationId: metadata.notificationId,
abuseSubjectKey: metadata.abuseSubjectKey,
quotaDecision: decision,
payload: serializePayload(command),
expiresAt: metadata.expiresAt,
maxAttempts: metadata.maxAttempts,
lastErrorCode: decision.reason ?? 'quota_denied',
});
return false;
}
const delivery = await this.models.mailDelivery.create({
mailName: command.name,
mailClass: decision.mailClass,
priority: metadata.priority ?? defaultPriority(decision.mailClass),
dedupeKey: metadata.dedupeKey,
recipientEmail: command.to,
recipientUserId: metadata.recipientUserId,
actorUserId: metadata.actorUserId,
workspaceId: metadata.workspaceId,
notificationId: metadata.notificationId,
abuseSubjectKey: metadata.abuseSubjectKey,
quotaReservationId: decision.reservationId,
quotaDecision: decision,
payload: serializePayload(command),
expiresAt: metadata.expiresAt,
maxAttempts: metadata.maxAttempts,
});
deliveryId = delivery.id;
if (decision.reservationId) {
if (delivery.quotaReservationId === decision.reservationId) {
await this.runtime.commitMailDeliveryQuotaV1(decision.reservationId);
} else {
await this.runtime.releaseMailDeliveryQuotaV1(decision.reservationId);
}
}
return true;
} catch {
} catch (error) {
if (deliveryId) {
await this.models.mailDelivery.cancelById(
deliveryId,
'quota_commit_failed'
);
}
if (reservationId) {
await this.runtime.releaseMailDeliveryQuotaV1(reservationId);
}
if (!suppressError) {
throw error;
}
return false;
}
}
@@ -1,15 +1,212 @@
import { Args, Mutation, Resolver } from '@nestjs/graphql';
import {
Args,
Field,
Float,
InputType,
Int,
Mutation,
ObjectType,
Query,
Resolver,
} from '@nestjs/graphql';
import { GraphQLJSONObject } from 'graphql-scalars';
import { BadRequest } from '../../base';
import { Renderers } from '../../mails';
import { Models } from '../../models';
import { CurrentUser } from '../auth/session';
import { Admin } from '../common';
import {
TimeBucket,
TimeWindow,
} from '../workspaces/resolvers/analytics-types';
import { MailSender } from './sender';
@InputType()
class AdminMailDeliveriesInput {
@Field(() => Int, { defaultValue: 24 })
hours!: number;
}
@ObjectType()
class AdminMailDeliveryPoint {
@Field(() => Date)
bucket!: Date;
@Field(() => Int)
count!: number;
}
@ObjectType()
class AdminMailDeliverySeries {
@Field()
key!: string;
@Field()
label!: string;
@Field(() => Int)
total!: number;
@Field(() => [AdminMailDeliveryPoint])
points!: AdminMailDeliveryPoint[];
}
@ObjectType()
class AdminMailDeliverySummary {
@Field(() => Int)
total!: number;
@Field(() => Int)
sent!: number;
@Field(() => Int)
failed!: number;
@Field(() => Int)
skipped!: number;
@Field(() => Int)
canceled!: number;
@Field(() => Int)
queued!: number;
@Field(() => Int)
sending!: number;
@Field(() => Int)
retryWait!: number;
@Field(() => Float)
successRate!: number;
}
@ObjectType()
class AdminMailDeliveryAnalytics {
@Field(() => TimeWindow)
window!: TimeWindow;
@Field(() => AdminMailDeliverySummary)
summary!: AdminMailDeliverySummary;
@Field(() => [AdminMailDeliverySeries])
byStatus!: AdminMailDeliverySeries[];
@Field(() => [AdminMailDeliverySeries])
byType!: AdminMailDeliverySeries[];
@Field(() => [AdminMailDeliverySeries])
byOutcome!: AdminMailDeliverySeries[];
}
const MAIL_STATUSES = [
'sent',
'failed',
'skipped',
'canceled',
'queued',
'sending',
'retry_wait',
] as const;
const STATUS_LABELS = {
sent: 'Sent',
failed: 'Failed',
skipped: 'Skipped',
canceled: 'Canceled',
queued: 'Queued',
sending: 'Sending',
retry_wait: 'Retry wait',
} satisfies Record<(typeof MAIL_STATUSES)[number], string>;
function startOfUtcHour(value: Date) {
return new Date(
Date.UTC(
value.getUTCFullYear(),
value.getUTCMonth(),
value.getUTCDate(),
value.getUTCHours()
)
);
}
function startOfUtcDay(value: Date) {
return new Date(
Date.UTC(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate())
);
}
function addUtcHours(value: Date, hours: number) {
return new Date(value.getTime() + hours * 60 * 60 * 1000);
}
function addUtcDays(value: Date, days: number) {
return new Date(value.getTime() + days * 24 * 60 * 60 * 1000);
}
function normalizeMailWindow(hours: number | undefined) {
if (hours !== undefined && hours !== 24 && hours !== 24 * 7) {
throw new BadRequest(
'Mail delivery analytics window must be 24 or 168 hours.'
);
}
const requestedHours = hours ?? 24;
const bucket = requestedHours > 24 ? ('day' as const) : ('hour' as const);
const now = new Date();
const to =
bucket === 'hour'
? addUtcHours(startOfUtcHour(now), 1)
: addUtcDays(startOfUtcDay(now), 1);
const effectiveSize =
bucket === 'hour' ? requestedHours : requestedHours / 24;
const from =
bucket === 'hour'
? addUtcHours(to, -effectiveSize)
: addUtcDays(to, -effectiveSize);
return {
from,
to,
bucket,
requestedHours,
effectiveSize,
};
}
function buildBuckets(input: ReturnType<typeof normalizeMailWindow>) {
return Array.from({ length: input.effectiveSize }, (_, index) =>
input.bucket === 'hour'
? addUtcHours(input.from, index)
: addUtcDays(input.from, index)
);
}
function bucketKey(value: Date) {
return value.toISOString();
}
function buildSeries(
keys: { key: string; label: string }[],
buckets: Date[],
counts: Map<string, number>
): AdminMailDeliverySeries[] {
return keys.map(({ key, label }) => {
let total = 0;
const points = buckets.map(bucket => {
const count = counts.get(`${key}:${bucketKey(bucket)}`) ?? 0;
total += count;
return { bucket, count };
});
return { key, label, total, points };
});
}
@Admin()
@Resolver(() => Boolean)
export class MailResolver {
constructor(private readonly models: Models) {}
@Mutation(() => Boolean)
async sendTestEmail(
@CurrentUser() user: CurrentUser,
@@ -46,4 +243,111 @@ export class MailResolver {
return true;
}
@Query(() => AdminMailDeliveryAnalytics, {
description: 'Aggregate mail delivery timeline facts for admin panel',
})
async adminMailDeliveries(
@Args('input', { nullable: true, type: () => AdminMailDeliveriesInput })
input?: AdminMailDeliveriesInput
) {
const window = normalizeMailWindow(input?.hours);
const buckets = buildBuckets(window);
const rows = await this.models.mailDelivery.adminAggregate({
from: window.from,
to: window.to,
bucket: window.bucket,
});
const statusCounts = new Map<string, number>();
const classCounts = new Map<string, number>();
const outcomeCounts = new Map<string, number>();
const classTotals = new Map<string, number>();
const summary = {
total: 0,
sent: 0,
failed: 0,
skipped: 0,
canceled: 0,
queued: 0,
sending: 0,
retryWait: 0,
successRate: 0,
};
for (const row of rows) {
const bucket = bucketKey(row.bucket);
summary.total += row.count;
if (row.status === 'retry_wait') {
summary.retryWait += row.count;
} else {
summary[row.status] += row.count;
}
statusCounts.set(
`${row.status}:${bucket}`,
(statusCounts.get(`${row.status}:${bucket}`) ?? 0) + row.count
);
classCounts.set(
`${row.mailClass}:${bucket}`,
(classCounts.get(`${row.mailClass}:${bucket}`) ?? 0) + row.count
);
classTotals.set(
row.mailClass,
(classTotals.get(row.mailClass) ?? 0) + row.count
);
const outcome =
row.status === 'sent'
? 'successful'
: row.status === 'queued' ||
row.status === 'sending' ||
row.status === 'retry_wait'
? 'pending'
: 'unsuccessful';
outcomeCounts.set(
`${outcome}:${bucket}`,
(outcomeCounts.get(`${outcome}:${bucket}`) ?? 0) + row.count
);
}
const terminal =
summary.sent + summary.failed + summary.skipped + summary.canceled;
summary.successRate = terminal ? summary.sent / terminal : 0;
const topClasses = [...classTotals.entries()]
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
.slice(0, 6)
.map(([key]) => ({ key, label: key }));
return {
window: {
from: window.from,
to: window.to,
timezone: 'UTC',
bucket: window.bucket === 'hour' ? TimeBucket.Hour : TimeBucket.Day,
requestedSize: window.requestedHours,
effectiveSize: window.effectiveSize,
},
summary,
byStatus: buildSeries(
MAIL_STATUSES.map(status => ({
key: status,
label: STATUS_LABELS[status],
})),
buckets,
statusCounts
),
byType: buildSeries(topClasses, buckets, classCounts),
byOutcome: buildSeries(
[
{ key: 'successful', label: 'Successful' },
{ key: 'unsuccessful', label: 'Unsuccessful' },
{ key: 'pending', label: 'Pending' },
],
buckets,
outcomeCounts
),
};
}
}
@@ -17,8 +17,24 @@ export type SendOptions = Omit<SendMailOptions, 'to' | 'subject' | 'html'> & {
html: string;
};
export type MailSendResult =
| {
status: 'accepted';
providerMessageId?: string;
providerResponse?: string;
retryable: false;
}
| {
status: 'rejected' | 'failed';
providerResponse?: string;
retryable: boolean;
errorCode?: string;
error?: string;
};
function configToSMTPOptions(
config: AppConfig['mailer']['SMTP']
config: AppConfig['mailer']['SMTP'],
timeoutMs = 30 * 1000
): SMTPTransport.Options {
const name = resolveSMTPHeloHostname(config.name);
@@ -33,6 +49,9 @@ function configToSMTPOptions(
user: config.username,
pass: config.password,
},
connectionTimeout: timeoutMs,
greetingTimeout: timeoutMs,
socketTimeout: timeoutMs,
};
}
@@ -68,7 +87,11 @@ export class MailSender {
private setup() {
const { SMTP, fallbackDomains, fallbackSMTP } = this.config.mailer;
const opts = configToSMTPOptions(SMTP);
const timeoutMs = Math.max(
1000,
Math.min(30 * 1000, this.config.mailer.deliveryWorker.leaseMs - 1000)
);
const opts = configToSMTPOptions(SMTP, timeoutMs);
if (SMTP.host) {
this.smtp = createTransport(opts);
@@ -76,7 +99,9 @@ export class MailSender {
this.logger.warn(
`Fallback SMTP is configured for domains: ${fallbackDomains.join(', ')}`
);
this.fallbackSMTP = createTransport(configToSMTPOptions(fallbackSMTP));
this.fallbackSMTP = createTransport(
configToSMTPOptions(fallbackSMTP, timeoutMs)
);
}
} else if (env.dev) {
createTestAccount((err, account) => {
@@ -107,17 +132,25 @@ export class MailSender {
return [this.smtp, SMTP.sender] as const;
}
async send(name: string, options: SendOptions) {
async send(name: string, options: SendOptions): Promise<MailSendResult> {
const [, domain, ...rest] = options.to.split('@');
if (rest.length || !domain) {
this.logger.error(`Invalid email address: ${options.to}`);
return null;
return {
status: 'rejected',
retryable: false,
errorCode: 'invalid_recipient',
};
}
const [smtpClient, from] = this.getSender(domain);
if (!smtpClient) {
this.logger.warn(`Mailer SMTP transport is not configured to send mail.`);
return null;
return {
status: 'failed',
retryable: true,
errorCode: 'smtp_not_configured',
};
}
metrics.mail.counter('send_total').add(1, { name });
@@ -129,7 +162,12 @@ export class MailSender {
this.logger.error(
`Mail [${name}] rejected with response: ${result.response}`
);
return false;
return {
status: 'rejected',
providerResponse: result.response,
retryable: false,
errorCode: 'provider_rejected',
};
}
metrics.mail.counter('accepted_total').add(1, { name });
@@ -140,11 +178,22 @@ export class MailSender {
);
}
return true;
return {
status: 'accepted',
providerMessageId:
typeof result.messageId === 'string' ? result.messageId : undefined,
providerResponse: result.response,
retryable: false,
};
} catch (e) {
metrics.mail.counter('failed_total').add(1, { name });
this.logger.error(`Failed to send mail [${name}].`, e);
return false;
return {
status: 'failed',
retryable: true,
errorCode: 'transport_failed',
error: e instanceof Error ? e.message : String(e),
};
}
}
}
@@ -0,0 +1,45 @@
import type { MailName, MailProps } from '../../mails';
import type { UserProps, WorkspaceProps } from '../../mails/components';
import type { RuntimeQuotaSourceInput } from '../backend-runtime';
export type MailDeliveryMetadata = {
dedupeKey?: string;
recipientUserId?: string;
actorUserId?: string;
workspaceId?: string;
notificationId?: string;
abuseSubjectKey?: string;
source?: RuntimeQuotaSourceInput;
expiresAt?: Date;
maxAttempts?: number;
priority?: 'critical' | 'high' | 'normal' | 'low';
};
export type DynamicallyFetchedProps<Props> = {
[Key in keyof Props]: Props[Key] extends infer Prop
? Prop extends UserProps
? {
$$userId: string;
} & Omit<Prop, 'email' | 'avatar'>
: Prop extends WorkspaceProps
? {
$$workspaceId: string;
} & Omit<Prop, 'name' | 'avatar'>
: Prop
: never;
};
export type SendMailPayload<
Mail extends MailName = MailName,
Props = MailProps<Mail>,
> = {
name: Mail;
to: string;
props: DynamicallyFetchedProps<Props>;
};
export type SendMailCommand = {
[K in MailName]: SendMailPayload<K>;
}[MailName] & {
metadata?: MailDeliveryMetadata;
};
@@ -4,7 +4,7 @@ import { mock } from 'node:test';
import test from 'ava';
import { createModule } from '../../../__tests__/create-module';
import { Mockers } from '../../../__tests__/mocks';
import { Mockers, MockMailer } from '../../../__tests__/mocks';
import { Due, NotificationNotFound } from '../../../base';
import {
DocMode,
@@ -18,6 +18,7 @@ import {
import { DocStorageModule } from '../../doc';
import { FeatureModule } from '../../features';
import { MailModule } from '../../mail';
import { Mailer } from '../../mail/mailer';
import { PermissionModule } from '../../permission';
import { StorageModule } from '../../storage';
import { NotificationModule } from '../index';
@@ -33,9 +34,13 @@ const module = await createModule({
NotificationModule,
],
providers: [NotificationService],
tapModule: builder => {
builder.overrideProvider(Mailer).useValue(new MockMailer());
},
});
const notificationService = module.get(NotificationService);
const models = module.get(Models);
const mailer = module.get(Mailer) as unknown as MockMailer;
let owner: User;
let member: User;
@@ -82,9 +87,9 @@ test('should create invitation notification and email', async t => {
t.is(notification!.body.inviteId, inviteId);
// should send invitation email
const invitationMail = module.queue.last('notification.sendMail');
t.is(invitationMail.payload.to, member.email);
t.is(invitationMail.payload.name, 'MemberInvitation');
const invitationMail = mailer.last('MemberInvitation');
t.is(invitationMail.to, member.email);
t.is(invitationMail.name, 'MemberInvitation');
});
test('should not send invitation email when workspace name contains domain', async t => {
@@ -92,10 +97,11 @@ test('should not send invitation email when workspace name contains domain', asy
owner: {
id: owner.id,
},
name: 'BTC https://spam.example',
name: 'BTC example.com',
});
const inviteId = randomUUID();
const invitationMailCount = module.queue.count('notification.sendMail');
const invitationMailCount = mailer.send.callCount;
const skippedMailCount = mailer.skip.callCount;
const notification = await notificationService.createInvitation({
userId: member.id,
@@ -107,7 +113,9 @@ test('should not send invitation email when workspace name contains domain', asy
});
t.truthy(notification);
t.is(module.queue.count('notification.sendMail'), invitationMailCount);
t.is(mailer.send.callCount, invitationMailCount);
t.is(mailer.skip.callCount, skippedMailCount + 1);
t.is(mailer.skip.lastCall.args[1].reason, 'workspace_name_contains_domain');
});
test('should not send invitation email if user setting is not to receive invitation email', async t => {
@@ -116,7 +124,8 @@ test('should not send invitation email if user setting is not to receive invitat
userId: member.id,
receiveInvitationEmail: false,
});
const invitationMailCount = module.queue.count('notification.sendMail');
const invitationMailCount = mailer.send.callCount;
const skippedMailCount = mailer.skip.callCount;
const notification = await notificationService.createInvitation({
userId: member.id,
@@ -130,7 +139,9 @@ test('should not send invitation email if user setting is not to receive invitat
t.truthy(notification);
// no new invitation email should be sent
t.is(module.queue.count('notification.sendMail'), invitationMailCount);
t.is(mailer.send.callCount, invitationMailCount);
t.is(mailer.skip.callCount, skippedMailCount + 1);
t.is(mailer.skip.lastCall.args[1].reason, 'recipient_notification_disabled');
});
test('should not create invitation notification if user is already a member', async t => {
@@ -174,9 +185,9 @@ test('should create invitation accepted notification and email', async t => {
t.is(notification!.body.inviteId, inviteId);
// should send email
const invitationAcceptedMail = module.queue.last('notification.sendMail');
t.is(invitationAcceptedMail.payload.to, owner.email);
t.is(invitationAcceptedMail.payload.name, 'MemberAccepted');
const invitationAcceptedMail = mailer.last('MemberAccepted');
t.is(invitationAcceptedMail.to, owner.email);
t.is(invitationAcceptedMail.name, 'MemberAccepted');
});
test('should not send invitation accepted email if user settings is not receive invitation email', async t => {
@@ -189,9 +200,7 @@ test('should not send invitation accepted email if user settings is not receive
userId: owner.id,
receiveInvitationEmail: false,
});
const invitationAcceptedMailCount = module.queue.count(
'notification.sendMail'
);
const invitationAcceptedMailCount = mailer.send.callCount;
const notification = await notificationService.createInvitationAccepted({
userId: owner.id,
@@ -205,10 +214,7 @@ test('should not send invitation accepted email if user settings is not receive
t.truthy(notification);
// no new invitation accepted email should be sent
t.is(
module.queue.count('notification.sendMail'),
invitationAcceptedMailCount
);
t.is(mailer.send.callCount, invitationAcceptedMailCount);
});
test('should not create invitation accepted notification if user is not an active member', async t => {
@@ -286,11 +292,11 @@ test('should create invitation review request notification if user is not an act
t.is(notification!.body.inviteId, inviteId);
// should send email
const invitationReviewRequestMail = module.queue.last(
'notification.sendMail'
const invitationReviewRequestMail = mailer.last(
'LinkInvitationReviewRequest'
);
t.is(invitationReviewRequestMail.payload.to, owner.email);
t.is(invitationReviewRequestMail.payload.name, 'LinkInvitationReviewRequest');
t.is(invitationReviewRequestMail.to, owner.email);
t.is(invitationReviewRequestMail.name, 'LinkInvitationReviewRequest');
});
test('should not create invitation review request notification if user is an active member', async t => {
@@ -336,11 +342,9 @@ test('should create invitation review approved notification if user is an active
t.is(notification!.body.inviteId, inviteId);
// should send email
const invitationReviewApprovedMail = module.queue.last(
'notification.sendMail'
);
t.is(invitationReviewApprovedMail.payload.to, member.email);
t.is(invitationReviewApprovedMail.payload.name, 'LinkInvitationApprove');
const invitationReviewApprovedMail = mailer.last('LinkInvitationApprove');
t.is(invitationReviewApprovedMail.to, member.email);
t.is(invitationReviewApprovedMail.name, 'LinkInvitationApprove');
});
test('should not create invitation review approved notification if user is not an active member', async t => {
@@ -382,11 +386,9 @@ test('should create invitation review declined notification if user is not an ac
t.is(notification!.body.createdByUserId, owner.id);
// should send email
const invitationReviewDeclinedMail = module.queue.last(
'notification.sendMail'
);
t.is(invitationReviewDeclinedMail.payload.to, member.email);
t.is(invitationReviewDeclinedMail.payload.name, 'LinkInvitationDecline');
const invitationReviewDeclinedMail = mailer.last('LinkInvitationDecline');
t.is(invitationReviewDeclinedMail.to, member.email);
t.is(invitationReviewDeclinedMail.name, 'LinkInvitationDecline');
});
test('should not create invitation review declined notification if user is an active member', async t => {
@@ -618,12 +620,12 @@ test('should send mention email by user setting', async t => {
t.truthy(notification);
// should send mention email
const mentionMail = module.queue.last('notification.sendMail');
t.is(mentionMail.payload.to, member.email);
t.is(mentionMail.payload.name, 'Mention');
const mentionMail = mailer.last('Mention');
t.is(mentionMail.to, member.email);
t.is(mentionMail.name, 'Mention');
// update user setting to not receive mention email
const mentionMailCount = module.queue.count('notification.sendMail');
const mentionMailCount = mailer.send.callCount;
await module.create(Mockers.UserSettings, {
userId: member.id,
receiveMentionEmail: false,
@@ -644,7 +646,7 @@ test('should send mention email by user setting', async t => {
});
// should not send mention email
t.is(module.queue.count('notification.sendMail'), mentionMailCount);
t.is(mailer.send.callCount, mentionMailCount);
});
test('should send mention email with use client doc title if server doc title is empty', async t => {
@@ -672,11 +674,10 @@ test('should send mention email with use client doc title if server doc title is
t.truthy(notification);
const mentionMail = module.queue.last('notification.sendMail');
t.is(mentionMail.payload.to, member.email);
t.is(mentionMail.payload.name, 'Mention');
// @ts-expect-error - payload is not typed
t.is(mentionMail.payload.props.doc.title, 'doc-title-1');
const mentionMail = mailer.last('Mention');
t.is(mentionMail.to, member.email);
t.is(mentionMail.name, 'Mention');
t.is(mentionMail.props.doc.title, 'doc-title-1');
});
test('should send comment notification and email', async t => {
@@ -699,9 +700,9 @@ test('should send comment notification and email', async t => {
t.truthy(notification);
const commentMail = module.queue.last('notification.sendMail');
t.is(commentMail.payload.to, member.email);
t.is(commentMail.payload.name, 'Comment');
const commentMail = mailer.last('Comment');
t.is(commentMail.to, member.email);
t.is(commentMail.name, 'Comment');
});
test('should send comment mention notification and email', async t => {
@@ -729,9 +730,9 @@ test('should send comment mention notification and email', async t => {
t.truthy(notification);
const commentMentionMail = module.queue.last('notification.sendMail');
t.is(commentMentionMail.payload.to, member.email);
t.is(commentMentionMail.payload.name, 'CommentMention');
const commentMentionMail = mailer.last('CommentMention');
t.is(commentMentionMail.to, member.email);
t.is(commentMentionMail.name, 'CommentMention');
});
test('should send comment email by user setting', async t => {
@@ -753,12 +754,12 @@ test('should send comment email by user setting', async t => {
t.truthy(notification);
const commentMail = module.queue.last('notification.sendMail');
t.is(commentMail.payload.to, member.email);
t.is(commentMail.payload.name, 'Comment');
const commentMail = mailer.last('Comment');
t.is(commentMail.to, member.email);
t.is(commentMail.name, 'Comment');
// update user setting to not receive comment email
const commentMailCount = module.queue.count('notification.sendMail');
const commentMailCount = mailer.send.callCount;
await module.create(Mockers.UserSettings, {
userId: member.id,
receiveCommentEmail: false,
@@ -779,5 +780,5 @@ test('should send comment email by user setting', async t => {
});
// should not send comment email
t.is(module.queue.count('notification.sendMail'), commentMailCount);
t.is(mailer.send.callCount, commentMailCount);
});
@@ -15,15 +15,17 @@ import {
UnionNotificationBody,
Workspace,
} from '../../models';
import { BackendRuntimeProvider } from '../backend-runtime';
import { containsUrlOrDomain } from '../content-policy';
import { DocReader } from '../doc';
import { Mailer } from '../mail';
import type { SendMailCommand } from '../mail/types';
import { realtimeNotificationRoom, RealtimePublisher } from '../realtime';
import { generateDocPath } from '../utils/doc';
import {
generateWorkspaceSettingsPath,
WorkspaceSettingsTab,
} from '../utils/workspace';
import { containsUrlOrDomain } from '../workspaces/abuse';
@Injectable()
export class NotificationService {
@@ -34,7 +36,8 @@ export class NotificationService {
private readonly docReader: DocReader,
private readonly mailer: Mailer,
private readonly url: URLHelper,
private readonly realtime: RealtimePublisher
private readonly realtime: RealtimePublisher,
private readonly runtime: BackendRuntimeProvider
) {}
async cleanExpiredNotifications() {
@@ -55,23 +58,21 @@ export class NotificationService {
const notification = isMention
? await this.models.notification.createCommentMention(input)
: await this.models.notification.createComment(input);
await this.sendCommentEmail(input, isMention);
await this.sendCommentEmail(input, isMention, notification.id);
await this.publishCountChanged(input.userId, 'created');
return notification;
}
private async sendCommentEmail(
input: CommentNotificationCreate,
isMention?: boolean
isMention?: boolean,
notificationId?: string
) {
const userSetting = await this.models.userSettings.get(input.userId);
if (!userSetting.receiveCommentEmail) {
return;
}
const receiver = await this.models.user.getWorkspaceUser(input.userId);
if (!receiver) {
return;
}
const userSetting = await this.models.userSettings.get(input.userId);
const doc = await this.models.doc.getMeta(
input.body.workspaceId,
input.body.doc.id
@@ -88,7 +89,7 @@ export class NotificationService {
replyId: input.body.replyId,
})
);
await this.mailer.trySend({
const command: SendMailCommand = {
name: isMention ? 'CommentMention' : 'Comment',
to: receiver.email,
props: {
@@ -100,26 +101,44 @@ export class NotificationService {
url,
},
},
});
metadata: {
dedupeKey: notificationId
? `notification:${notificationId}:mail:${isMention ? 'CommentMention' : 'Comment'}`
: undefined,
recipientUserId: receiver.id,
actorUserId: input.body.createdByUserId,
workspaceId: input.body.workspaceId,
notificationId,
source: { trusted: false },
},
};
if (!userSetting.receiveCommentEmail) {
await this.mailer.skip(command, {
mailClass: 'collaboration_notice',
reason: 'recipient_notification_disabled',
});
return;
}
await this.trySendMail(command, 'collaboration_notice');
this.logger.debug(`Comment email sent to user ${receiver.id}`);
}
async createMention(input: MentionNotificationCreate) {
const notification = await this.models.notification.createMention(input);
await this.sendMentionEmail(input);
await this.sendMentionEmail(input, notification.id);
await this.publishCountChanged(input.userId, 'created');
return notification;
}
private async sendMentionEmail(input: MentionNotificationCreate) {
const userSetting = await this.models.userSettings.get(input.userId);
if (!userSetting.receiveMentionEmail) {
return;
}
private async sendMentionEmail(
input: MentionNotificationCreate,
notificationId?: string
) {
const receiver = await this.models.user.getWorkspaceUser(input.userId);
if (!receiver) {
return;
}
const userSetting = await this.models.userSettings.get(input.userId);
const doc = await this.models.doc.getMeta(
input.body.workspaceId,
input.body.doc.id
@@ -134,7 +153,7 @@ export class NotificationService {
elementId: input.body.doc.elementId,
})
);
await this.mailer.trySend({
const command: SendMailCommand = {
name: 'Mention',
to: receiver.email,
props: {
@@ -146,7 +165,25 @@ export class NotificationService {
url,
},
},
});
metadata: {
dedupeKey: notificationId
? `notification:${notificationId}:mail:Mention`
: undefined,
recipientUserId: receiver.id,
actorUserId: input.body.createdByUserId,
workspaceId: input.body.workspaceId,
notificationId,
source: { trusted: false },
},
};
if (!userSetting.receiveMentionEmail) {
await this.mailer.skip(command, {
mailClass: 'collaboration_notice',
reason: 'recipient_notification_disabled',
});
return;
}
await this.trySendMail(command, 'collaboration_notice');
this.logger.debug(`Mention email sent to user ${receiver.id}`);
}
@@ -161,36 +198,25 @@ export class NotificationService {
input,
NotificationType.Invitation
);
await this.sendInvitationEmail(input);
await this.sendInvitationEmail(input, notification.id);
await this.publishCountChanged(input.userId, 'created');
return notification;
}
private async sendInvitationEmail(input: InvitationNotificationCreate) {
const workspace = await this.docReader.getWorkspaceContent(
input.body.workspaceId
);
if (containsUrlOrDomain(workspace?.name)) {
this.logger.warn(
`Skip invitation email for workspace ${input.body.workspaceId}, reason=workspace name contains url or domain`
);
return;
}
private async sendInvitationEmail(
input: InvitationNotificationCreate,
notificationId?: string
) {
const inviteUrl = this.url.link(`/invite/${input.body.inviteId}`);
if (env.dev) {
// make it easier to test in dev mode
this.logger.debug(`Invite link: ${inviteUrl}`);
}
const userSetting = await this.models.userSettings.get(input.userId);
if (!userSetting.receiveInvitationEmail) {
return;
}
const receiver = await this.models.user.getWorkspaceUser(input.userId);
if (!receiver) {
return;
}
await this.mailer.trySend({
const command: SendMailCommand = {
name: 'MemberInvitation',
to: receiver.email,
props: {
@@ -202,7 +228,40 @@ export class NotificationService {
},
url: inviteUrl,
},
});
metadata: {
dedupeKey: notificationId
? `notification:${notificationId}:mail:MemberInvitation`
: `invite:${input.body.inviteId}:mail:MemberInvitation`,
recipientUserId: receiver.id,
actorUserId: input.body.createdByUserId,
workspaceId: input.body.workspaceId,
notificationId,
source: { trusted: false },
},
};
const workspace = await this.docReader.getWorkspaceContent(
input.body.workspaceId
);
if (containsUrlOrDomain(workspace?.name)) {
this.logger.warn(
`Skip invitation email for workspace ${input.body.workspaceId}, reason=workspace name contains url or domain`
);
await this.mailer.skip(command, {
mailClass: 'workspace_invitation',
reason: 'workspace_name_contains_domain',
});
return;
}
const userSetting = await this.models.userSettings.get(input.userId);
if (!userSetting.receiveInvitationEmail) {
await this.mailer.skip(command, {
mailClass: 'workspace_invitation',
reason: 'recipient_notification_disabled',
});
return;
}
await this.trySendMail(command, 'workspace_invitation');
this.logger.debug(
`Invitation email sent to user ${receiver.id} for workspace ${input.body.workspaceId}`
);
@@ -223,26 +282,23 @@ export class NotificationService {
input,
NotificationType.InvitationAccepted
);
await this.sendInvitationAcceptedEmail(input);
await this.sendInvitationAcceptedEmail(input, notification.id);
await this.publishCountChanged(input.userId, 'created');
return notification;
}
private async sendInvitationAcceptedEmail(
input: InvitationNotificationCreate
input: InvitationNotificationCreate,
notificationId?: string
) {
const inviterUserId = input.userId;
const inviteeUserId = input.body.createdByUserId;
const workspaceId = input.body.workspaceId;
const userSetting = await this.models.userSettings.get(inviterUserId);
if (!userSetting.receiveInvitationEmail) {
return;
}
const inviter = await this.models.user.getWorkspaceUser(inviterUserId);
if (!inviter) {
return;
}
await this.mailer.trySend({
const command: SendMailCommand = {
name: 'MemberAccepted',
to: inviter.email,
props: {
@@ -259,7 +315,26 @@ export class NotificationService {
})
),
},
});
metadata: {
dedupeKey: notificationId
? `notification:${notificationId}:mail:MemberAccepted`
: undefined,
recipientUserId: inviter.id,
actorUserId: inviteeUserId,
workspaceId,
notificationId,
source: { trusted: false },
},
};
const userSetting = await this.models.userSettings.get(inviterUserId);
if (!userSetting.receiveInvitationEmail) {
await this.mailer.skip(command, {
mailClass: 'workspace_lifecycle',
reason: 'recipient_notification_disabled',
});
return;
}
await this.trySendMail(command, 'workspace_lifecycle');
this.logger.debug(
`Invitation accepted email sent to user ${inviter.id} for workspace ${workspaceId}`
);
@@ -297,13 +372,14 @@ export class NotificationService {
input,
NotificationType.InvitationReviewRequest
);
await this.sendInvitationReviewRequestEmail(input);
await this.sendInvitationReviewRequestEmail(input, notification.id);
await this.publishCountChanged(input.userId, 'created');
return notification;
}
private async sendInvitationReviewRequestEmail(
input: InvitationNotificationCreate
input: InvitationNotificationCreate,
notificationId?: string
) {
const inviteeUserId = input.body.createdByUserId;
const reviewerUserId = input.userId;
@@ -312,24 +388,37 @@ export class NotificationService {
if (!reviewer) {
return;
}
await this.mailer.trySend({
name: 'LinkInvitationReviewRequest',
to: reviewer.email,
props: {
user: {
$$userId: inviteeUserId,
await this.trySendMail(
{
name: 'LinkInvitationReviewRequest',
to: reviewer.email,
props: {
user: {
$$userId: inviteeUserId,
},
workspace: {
$$workspaceId: workspaceId,
},
url: this.url.link(
generateWorkspaceSettingsPath({
workspaceId,
tab: WorkspaceSettingsTab.members,
})
),
},
workspace: {
$$workspaceId: workspaceId,
metadata: {
dedupeKey: notificationId
? `notification:${notificationId}:mail:LinkInvitationReviewRequest`
: undefined,
recipientUserId: reviewer.id,
actorUserId: inviteeUserId,
workspaceId,
notificationId,
source: { trusted: false },
},
url: this.url.link(
generateWorkspaceSettingsPath({
workspaceId,
tab: WorkspaceSettingsTab.members,
})
),
},
});
'workspace_invitation'
);
this.logger.debug(
`Invitation review request email sent to user ${reviewer.id} for workspace ${workspaceId}`
);
@@ -346,13 +435,14 @@ export class NotificationService {
input,
NotificationType.InvitationReviewApproved
);
await this.sendInvitationReviewApprovedEmail(input);
await this.sendInvitationReviewApprovedEmail(input, notification.id);
await this.publishCountChanged(input.userId, 'created');
return notification;
}
private async sendInvitationReviewApprovedEmail(
input: InvitationNotificationCreate
input: InvitationNotificationCreate,
notificationId?: string
) {
const workspaceId = input.body.workspaceId;
const receiverUserId = input.userId;
@@ -360,16 +450,29 @@ export class NotificationService {
if (!receiver) {
return;
}
await this.mailer.trySend({
name: 'LinkInvitationApprove',
to: receiver.email,
props: {
workspace: {
$$workspaceId: workspaceId,
await this.trySendMail(
{
name: 'LinkInvitationApprove',
to: receiver.email,
props: {
workspace: {
$$workspaceId: workspaceId,
},
url: this.url.link(`/workspace/${workspaceId}`),
},
metadata: {
dedupeKey: notificationId
? `notification:${notificationId}:mail:LinkInvitationApprove`
: undefined,
recipientUserId: receiver.id,
actorUserId: input.body.createdByUserId,
workspaceId,
notificationId,
source: { trusted: false },
},
url: this.url.link(`/workspace/${workspaceId}`),
},
});
'workspace_invitation'
);
this.logger.debug(
`Invitation review approved email sent to user ${receiver.id} for workspace ${workspaceId}`
);
@@ -386,13 +489,14 @@ export class NotificationService {
await this.ensureWorkspaceContentExists(workspaceId);
const notification =
await this.models.notification.createInvitationReviewDeclined(input);
await this.sendInvitationReviewDeclinedEmail(input);
await this.sendInvitationReviewDeclinedEmail(input, notification.id);
await this.publishCountChanged(input.userId, 'created');
return notification;
}
private async sendInvitationReviewDeclinedEmail(
input: InvitationReviewDeclinedNotificationCreate
input: InvitationReviewDeclinedNotificationCreate,
notificationId?: string
) {
const workspaceId = input.body.workspaceId;
const receiverUserId = input.userId;
@@ -400,15 +504,28 @@ export class NotificationService {
if (!receiver) {
return;
}
await this.mailer.trySend({
name: 'LinkInvitationDecline',
to: receiver.email,
props: {
workspace: {
$$workspaceId: workspaceId,
await this.trySendMail(
{
name: 'LinkInvitationDecline',
to: receiver.email,
props: {
workspace: {
$$workspaceId: workspaceId,
},
},
metadata: {
dedupeKey: notificationId
? `notification:${notificationId}:mail:LinkInvitationDecline`
: undefined,
recipientUserId: receiver.id,
actorUserId: input.body.createdByUserId,
workspaceId,
notificationId,
source: { trusted: false },
},
},
});
'workspace_invitation'
);
this.logger.debug(
`Invitation review declined email sent to user ${receiver.id} for workspace ${workspaceId}`
);
@@ -539,4 +656,32 @@ export class NotificationService {
);
return !!isActive;
}
private async trySendMail(command: SendMailCommand, mailClass: string) {
const actorUserId = command.metadata?.actorUserId;
if (
actorUserId &&
(await this.runtime.isInviteAbuseUserQuarantinedOrBanned(actorUserId))
) {
await this.mailer.skip(command, {
mailClass,
reason: 'actor_quarantined',
});
return false;
}
const workspaceId = command.metadata?.workspaceId;
if (
workspaceId &&
(await this.runtime.isInviteAbuseWorkspaceQuarantined(workspaceId))
) {
await this.mailer.skip(command, {
mailClass,
reason: 'workspace_quarantined',
});
return false;
}
return await this.mailer.trySend(command);
}
}
@@ -1,36 +1,390 @@
import { createHash } from 'node:crypto';
import {
createInviteLinkMutation,
inviteByEmailsMutation,
WorkspaceInviteLinkExpireTime,
} from '@affine/graphql';
import { PrismaClient, WorkspaceMemberStatus } from '@prisma/client';
import test from 'ava';
import type { Request } from 'express';
import Sinon from 'sinon';
import { canUserExecuteLimitedActions, containsUrlOrDomain } from '../abuse';
import { createApp, type TestingApp } from '../../../__tests__/e2e/test';
import { Mockers } from '../../../__tests__/mocks';
import { Config } from '../../../base';
import { ActionForbidden, TooManyRequest } from '../../../base/error';
import { Models, WorkspaceRole } from '../../../models';
import {
getAbuseRequestSource,
InviteAbuseDispositionService,
InviteQuotaAssertService,
} from '../abuse';
test('should detect links and bare domains in workspace names', t => {
t.true(containsUrlOrDomain('BTC https://spam.example'));
t.true(containsUrlOrDomain('Join spam.example now'));
t.true(containsUrlOrDomain('Join spam.example, ltd'));
t.true(containsUrlOrDomain('Join spam.example。'));
t.true(containsUrlOrDomain('www.spam.example'));
let app: TestingApp;
const quota = {
assertWorkspaceInviteQuota: Sinon.stub(),
commitWorkspaceInviteQuota: Sinon.stub(),
releaseWorkspaceInviteQuota: Sinon.stub(),
};
function workspaceSubjectKey(workspaceId: string) {
return `workspace:v1:${createHash('sha256').update(workspaceId).digest('hex').slice(0, 24)}`;
}
test.before(async () => {
app = await createApp({
tapModule: builder => {
builder.overrideProvider(InviteQuotaAssertService).useValue(quota);
},
});
});
test('should not detect email addresses or partial domain words', t => {
t.false(containsUrlOrDomain('Contact user@spam.example'));
t.false(containsUrlOrDomain('spam.example_btc'));
test.beforeEach(() => {
quota.assertWorkspaceInviteQuota.reset();
quota.commitWorkspaceInviteQuota.reset();
quota.releaseWorkspaceInviteQuota.reset();
});
test('should check account age for share actions', t => {
const minimumAccountAgeMs = 24 * 60 * 60 * 1000;
test.after.always(async () => {
await app.close();
});
t.false(
canUserExecuteLimitedActions({ createdAt: new Date() }, minimumAccountAgeMs)
);
t.true(
canUserExecuteLimitedActions(
{
createdAt: new Date(Date.now() - minimumAccountAgeMs - 1),
test('invite quota rejection has no invite side effects', async t => {
quota.assertWorkspaceInviteQuota.rejects(new TooManyRequest());
const models = app.get(Models);
const owner = await app.create(Mockers.User);
const workspace = await app.create(Mockers.Workspace, {
owner: { id: owner.id },
});
const targetEmail = `quota-${Date.now()}@example.com`;
await app.login(owner);
await t.throwsAsync(
app.gql({
query: inviteByEmailsMutation,
variables: {
workspaceId: workspace.id,
emails: [targetEmail],
},
minimumAccountAgeMs
)
})
);
t.is(await models.user.getUserByEmail(targetEmail), null);
t.is(await models.workspaceUser.count(workspace.id), 1);
t.is(app.mails.send.callCount, 0);
t.is(app.queue.count('notification.sendInvitation'), 0);
t.is(quota.commitWorkspaceInviteQuota.callCount, 0);
t.is(quota.releaseWorkspaceInviteQuota.callCount, 0);
});
test('abuse request source trusts Cloudflare facts only when configured', t => {
const config = app.get(Config);
const previousTrusted = config.auth.trustedCloudflareHeaders;
const req = {
ip: '10.0.0.1',
get(name: string) {
return (
{
'CF-Connecting-IP': '114.51.41.91',
'CF-IPCountry': 'JP',
'CF-Ray': 'ray-id',
'x-affine-cf-asn': '4294967295',
'X-Forwarded-For': '198.51.100.9',
} satisfies Record<string, string>
)[name];
},
} as Request;
try {
config.auth.trustedCloudflareHeaders = false;
t.deepEqual(getAbuseRequestSource(req, config), { trusted: false });
config.auth.trustedCloudflareHeaders = true;
t.deepEqual(getAbuseRequestSource(req, config), {
trusted: true,
ip: '114.51.41.91',
country: 'JP',
asn: 4294967295,
rayId: 'ray-id',
});
} finally {
config.auth.trustedCloudflareHeaders = previousTrusted;
}
});
test('invite quota rejection keeps mapped response when disposition fails', async t => {
const service = new InviteQuotaAssertService(
app.get(Config),
{
getWorkspaceSeatQuota: Sinon.stub().resolves({
memberLimit: 10,
memberCount: 1,
}),
} as any,
{
assertWorkspaceInviteQuotaV1: Sinon.stub().resolves({
allowed: false,
reason: 'abuse_subject',
actionRequired: {
action: 'quarantine_actor',
actionId: '1',
subjectKey: 'subject',
evidenceId: '1',
},
}),
} as any,
{
execute: Sinon.stub().rejects(new Error('disposition failed')),
} as any
);
await t.throwsAsync(
service.assertWorkspaceInviteQuota({
actorUserId: 'actor',
workspaceId: 'workspace',
targetCount: 1,
targetDomains: [{ domain: 'example.com', count: 1 }],
}),
{ instanceOf: ActionForbidden }
);
});
test('should skip account age check when share action delay is disabled', t => {
t.true(canUserExecuteLimitedActions({ createdAt: new Date() }, 0));
test('abuse disposition applies action scope to invitation artifacts', async t => {
const models = app.get(Models);
const db = app.get(PrismaClient);
const disposition = app.get(InviteAbuseDispositionService);
for (const scenario of [
{
name: 'actor',
subjectKey: 'actor-subject',
subjectKind: 'actor_email',
action: 'quarantine_actor',
},
{
name: 'workspace',
subjectKey: 'workspace-subject',
subjectKind: 'workspace',
action: 'quarantine_workspace',
},
] as const) {
const actor = await app.create(Mockers.User);
const invitee = await app.create(Mockers.User);
const workspace = await app.create(Mockers.Workspace, { owner: actor });
const anotherWorkspace = await app.create(Mockers.Workspace, {
owner: actor,
});
await models.workspaceInvitation.set(
workspace.id,
invitee.id,
WorkspaceRole.Collaborator,
WorkspaceMemberStatus.Pending,
{ inviterId: actor.id }
);
await models.workspaceInvitation.set(
anotherWorkspace.id,
invitee.id,
WorkspaceRole.Collaborator,
WorkspaceMemberStatus.Pending,
{ inviterId: actor.id }
);
const canceledDelivery = await models.mailDelivery.create({
mailName: 'MemberInvitation',
mailClass: 'workspace_invitation',
priority: 'normal',
recipientEmail: invitee.email,
actorUserId: actor.id,
workspaceId: workspace.id,
abuseSubjectKey: scenario.subjectKey,
payload: {
name: 'MemberInvitation',
to: invitee.email,
props: {
url: 'https://affine.pro/invite',
user: { $$userId: actor.id },
workspace: { $$workspaceId: workspace.id },
},
},
});
const otherDelivery =
scenario.name === 'workspace'
? await models.mailDelivery.create({
mailName: 'MemberInvitation',
mailClass: 'workspace_invitation',
priority: 'normal',
recipientEmail: invitee.email,
actorUserId: actor.id,
workspaceId: anotherWorkspace.id,
abuseSubjectKey: 'other-workspace-subject',
payload: {
name: 'MemberInvitation',
to: invitee.email,
props: {
url: 'https://affine.pro/invite',
user: { $$userId: actor.id },
workspace: { $$workspaceId: anotherWorkspace.id },
},
},
})
: null;
const [{ id: actionId }] = await db.$queryRaw<Array<{ id: bigint }>>`
WITH subject AS (
INSERT INTO runtime_invite_abuse_subjects (
subject_key,
kind,
user_id,
actor_email_hash,
status,
first_seen_at,
last_seen_at
)
VALUES (${scenario.subjectKey}, ${scenario.subjectKind}, ${actor.id}, 'hash', 'quarantined', now(), now())
ON CONFLICT (subject_key) DO NOTHING
RETURNING subject_key
),
evidence AS (
INSERT INTO runtime_invite_abuse_evidence (
subject_key,
workspace_id,
user_id,
actor_email_hash,
decision,
reason
)
VALUES (${scenario.subjectKey}, ${workspace.id}, ${actor.id}, 'hash', ${scenario.action}, 'test')
RETURNING id
)
INSERT INTO runtime_invite_abuse_actions (
subject_key,
evidence_id,
action,
status
)
SELECT ${scenario.subjectKey}, evidence.id, ${scenario.action}, 'pending'
FROM evidence
RETURNING id
`;
await disposition.execute({
actorUserId: actor.id,
workspaceId: workspace.id,
actionRequired: {
action: scenario.action,
subjectKey: scenario.subjectKey,
evidenceId: '1',
actionId: actionId.toString(),
},
});
if (scenario.name === 'actor') {
t.is(
await db.workspaceInvitation.count({
where: { inviterUserId: actor.id },
}),
0,
scenario.action
);
} else {
t.is(
await db.workspaceInvitation.count({
where: { workspaceId: workspace.id },
}),
0,
scenario.action
);
t.is(
await db.workspaceInvitation.count({
where: { workspaceId: anotherWorkspace.id },
}),
1,
scenario.action
);
}
t.is(
(
await db.mailDelivery.findUniqueOrThrow({
where: { id: canceledDelivery.id },
})
).status,
'canceled',
scenario.action
);
if (otherDelivery) {
t.is(
(
await db.mailDelivery.findUniqueOrThrow({
where: { id: otherDelivery.id },
})
).status,
'queued',
scenario.action
);
}
}
});
test('workspace quarantine blocks invite link creation', async t => {
const db = app.get(PrismaClient);
const config = app.get(Config);
const owner = await app.create(Mockers.User);
const workspace = await app.create(Mockers.Workspace, { owner });
const subjectKey = workspaceSubjectKey(workspace.id);
await db.$executeRaw`
INSERT INTO runtime_invite_abuse_subjects (
subject_key,
kind,
status,
first_seen_at,
last_seen_at
)
VALUES (${subjectKey}, 'workspace', 'quarantined', now(), now())
ON CONFLICT (subject_key)
DO UPDATE SET
status = 'quarantined',
updated_at = now()
`;
const previousDelay = config.auth.newAccountShareActionDelay;
config.auth.newAccountShareActionDelay = 0;
try {
await app.login(owner);
await t.throwsAsync(
app.gql({
query: createInviteLinkMutation,
variables: {
workspaceId: workspace.id,
expireTime: WorkspaceInviteLinkExpireTime.OneDay,
},
})
);
} finally {
config.auth.newAccountShareActionDelay = previousDelay;
}
});
test('domain workspace name blocks invite link creation', async t => {
const config = app.get(Config);
const owner = await app.create(Mockers.User);
const workspace = await app.create(Mockers.Workspace, {
owner,
name: 'Join example.com',
});
const previousDelay = config.auth.newAccountShareActionDelay;
config.auth.newAccountShareActionDelay = 0;
try {
await app.login(owner);
await t.throwsAsync(
app.gql({
query: createInviteLinkMutation,
variables: {
workspaceId: workspace.id,
expireTime: WorkspaceInviteLinkExpireTime.OneDay,
},
})
);
} finally {
config.auth.newAccountShareActionDelay = previousDelay;
}
});
@@ -1,8 +1,41 @@
const URL_OR_DOMAIN_PATTERN =
/(?:https?:\/\/|www\.|(?<![@\w-])(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}(?=$|[^\p{L}\p{N}._-]))/iu;
import { createHash, randomUUID } from 'node:crypto';
export function containsUrlOrDomain(value: string | null | undefined) {
return URL_OR_DOMAIN_PATTERN.test(value ?? '');
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import type { Request } from 'express';
import {
ActionForbidden,
Config,
getRequestClientIp,
JobQueue,
metrics,
OnJob,
TooManyRequest,
type UserFriendlyError,
} from '../../base';
import { Models } from '../../models';
import {
BackendRuntimeProvider,
type RuntimeQuotaSourceInput,
type RuntimeQuotaTargetDomainInput,
type RuntimeWorkspaceInviteQuotaDecision,
} from '../backend-runtime';
import { QuotaService } from '../quota/service';
type ActionRequired = NonNullable<
RuntimeWorkspaceInviteQuotaDecision['actionRequired']
>;
export type InviteQuotaAdmission = {
reservationId?: string;
decision: RuntimeWorkspaceInviteQuotaDecision;
};
declare global {
interface Jobs {
'inviteAbuse.executePendingActions': {};
}
}
export function canUserExecuteLimitedActions(
@@ -12,3 +45,344 @@ export function canUserExecuteLimitedActions(
if (minimumAccountAgeMs <= 0) return true;
return Date.now() - user.createdAt.getTime() >= minimumAccountAgeMs;
}
function parseAsn(value: string | undefined) {
if (!value) {
return;
}
const asn = Number(value);
return Number.isSafeInteger(asn) && asn > 0 && asn <= 0xffffffff
? asn
: undefined;
}
export function getAbuseRequestSource(
req: Request | undefined,
config: Config
): RuntimeQuotaSourceInput {
if (!req || !config.auth.trustedCloudflareHeaders) {
return { trusted: false };
}
return {
trusted: true,
ip: getRequestClientIp(req),
country: req.get('CF-IPCountry')?.trim() || undefined,
asn: parseAsn(req.get('x-affine-cf-asn')),
rayId: req.get('CF-Ray')?.trim() || undefined,
};
}
function hashLogValue(value: string | undefined) {
if (!value) {
return undefined;
}
return createHash('sha256').update(value).digest('hex').slice(0, 24);
}
@Injectable()
export class InviteAbuseDispositionService {
private readonly logger = new Logger(InviteAbuseDispositionService.name);
private readonly workerOwnerId = randomUUID();
private readonly inlineWorkerId = `node-inline:${this.workerOwnerId}`;
constructor(
private readonly models: Models,
private readonly runtime: BackendRuntimeProvider,
private readonly queue: JobQueue
) {}
async execute(input: {
actionRequired?: ActionRequired;
actorUserId: string;
workspaceId: string;
alreadyClaimed?: boolean;
workerId?: string;
}) {
const { actionRequired } = input;
if (!actionRequired) {
return;
}
const workerId = input.workerId ?? this.inlineWorkerId;
try {
if (!input.alreadyClaimed) {
const claimed = await this.runtime.claimInviteAbuseAction(
actionRequired.actionId,
workerId
);
if (!claimed) {
return;
}
}
switch (actionRequired.action) {
case 'ban_actor':
await this.cancelPendingActorArtifacts(input, actionRequired);
await this.models.session.deleteUserSessions(input.actorUserId);
await this.models.user.ban(input.actorUserId);
break;
case 'quarantine_actor':
await this.cancelPendingActorArtifacts(input, actionRequired);
await this.models.session.deleteUserSessions(input.actorUserId);
break;
case 'quarantine_workspace':
await this.cancelPendingWorkspaceArtifacts(input.workspaceId);
break;
case 'quarantine_source_cohort':
await this.models.mailDelivery.cancelByAbuseSubject(
actionRequired.subjectKey,
'source_cohort_quarantined'
);
break;
default:
throw new Error(
`Unknown invite abuse action: ${actionRequired.action}`
);
}
await this.runtime.markInviteAbuseAction(
actionRequired.actionId,
workerId,
'succeeded'
);
} catch (error) {
const message =
error instanceof Error ? error.message : 'Failed to execute action.';
await this.runtime.markInviteAbuseAction(
actionRequired.actionId,
workerId,
'failed',
message
);
this.logger.error('Failed to execute invite abuse disposition', {
userId: input.actorUserId,
workspaceId: input.workspaceId,
action: actionRequired.action,
actionId: actionRequired.actionId,
evidenceId: actionRequired.evidenceId,
subjectKey: actionRequired.subjectKey,
error: message,
});
throw error;
}
}
@Cron(CronExpression.EVERY_MINUTE)
async enqueuePendingActions() {
await this.queue.add(
'inviteAbuse.executePendingActions',
{},
{ jobId: 'invite-abuse-execute-pending-actions' }
);
}
@OnJob('inviteAbuse.executePendingActions')
async executePendingActions() {
const workerId = `node:${this.workerOwnerId}`;
const actions = await this.runtime.claimRetryableInviteAbuseActions(
workerId,
20
);
for (const action of actions) {
await this.execute({
actorUserId: action.actorUserId,
workspaceId: action.workspaceId,
alreadyClaimed: true,
workerId,
actionRequired: {
action: action.action,
subjectKey: action.subjectKey,
evidenceId: action.evidenceId,
actionId: action.actionId,
},
});
}
}
private async cancelPendingActorArtifacts(
input: {
actorUserId: string;
workspaceId: string;
},
action: ActionRequired
) {
await this.models.mailDelivery.cancelByActor(input.actorUserId);
await this.models.mailDelivery.cancelByWorkspace(input.workspaceId);
await this.models.mailDelivery.cancelByAbuseSubject(action.subjectKey);
await this.models.workspaceInvitation.cancelPendingByActor(
input.actorUserId
);
}
private async cancelPendingWorkspaceArtifacts(workspaceId: string) {
await this.models.mailDelivery.cancelByWorkspace(workspaceId);
await this.models.workspaceInvitation.cancelPendingByWorkspace(workspaceId);
}
}
@Injectable()
export class InviteQuotaAssertService {
private readonly logger = new Logger(InviteQuotaAssertService.name);
constructor(
private readonly config: Config,
private readonly quota: QuotaService,
private readonly runtime: BackendRuntimeProvider,
private readonly disposition: InviteAbuseDispositionService
) {}
async assertWorkspaceInviteQuota(input: {
actorUserId: string;
workspaceId: string;
requestId?: string;
targetCount: number;
targetDomains: RuntimeQuotaTargetDomainInput[];
source?: RuntimeQuotaSourceInput;
}): Promise<InviteQuotaAdmission> {
const start = Date.now();
const seatQuota = await this.quota.getWorkspaceSeatQuota(input.workspaceId);
let decision: RuntimeWorkspaceInviteQuotaDecision;
try {
decision = await this.runtime.assertWorkspaceInviteQuotaV1(input);
} catch (error) {
const message =
error instanceof Error ? error.message : 'native assert failed';
metrics.workspace.counter('invite_quota_runtime_fallback').add(1, {
mode: this.config.auth.inviteQuotaFailOpenOnRuntimeError
? 'fail_open'
: 'fail_closed',
});
this.logger.error('Workspace invite quota native assert failed', {
userId: input.actorUserId,
workspaceId: input.workspaceId,
targetCount: input.targetCount,
targetDomainsSummary: input.targetDomains,
sourceTrusted: input.source?.trusted ?? false,
country: input.source?.country,
asn: input.source?.asn,
requestId: input.requestId,
cfRay: input.source?.rayId,
error: message,
});
if (this.config.auth.inviteQuotaFailOpenOnRuntimeError) {
return { decision: { allowed: true, requested: input.targetCount } };
}
throw new TooManyRequest();
}
metrics.workspace
.counter('invite_quota_requested_targets')
.add(input.targetCount);
metrics.workspace
.histogram('invite_quota_counter_latency_ms')
.record(Date.now() - start);
if (!decision.allowed) {
metrics.workspace.counter('invite_quota_rejected').add(1, {
reason: decision.reason ?? 'unknown',
});
metrics.workspace.counter('invite_quota_reject_by_reason').add(1, {
reason: decision.reason ?? 'unknown',
});
if (this.config.auth.inviteQuotaShadowMode) {
this.logger.warn('Workspace invite quota shadow rejected', {
userId: input.actorUserId,
workspaceId: input.workspaceId,
targetCount: input.targetCount,
targetDomainsSummary: input.targetDomains,
sourceTrusted: input.source?.trusted ?? false,
country: input.source?.country,
asn: input.source?.asn,
reason: decision.reason,
scopeKeyHash: hashLogValue(decision.scopeKey),
limit: decision.limit,
current: decision.current,
requested: decision.requested,
memberLimit: seatQuota.memberLimit,
memberCount: seatQuota.memberCount,
retryAfter: decision.retryAfterSeconds,
requestId: input.requestId,
cfRay: input.source?.rayId,
wouldDispose: decision.actionRequired?.action,
});
return { decision };
}
try {
await this.disposition.execute({
actionRequired: decision.actionRequired,
actorUserId: input.actorUserId,
workspaceId: input.workspaceId,
});
} catch (error) {
this.logger.error('Workspace invite quota disposition failed', {
userId: input.actorUserId,
workspaceId: input.workspaceId,
action: decision.actionRequired?.action,
actionId: decision.actionRequired?.actionId,
reason: decision.reason,
error: error instanceof Error ? error.message : 'Unknown error',
});
}
this.logger.warn('Workspace invite quota rejected', {
userId: input.actorUserId,
workspaceId: input.workspaceId,
targetCount: input.targetCount,
targetDomainsSummary: input.targetDomains,
sourceTrusted: input.source?.trusted ?? false,
country: input.source?.country,
asn: input.source?.asn,
reason: decision.reason,
scopeKeyHash: hashLogValue(decision.scopeKey),
limit: decision.limit,
current: decision.current,
requested: decision.requested,
memberLimit: seatQuota.memberLimit,
memberCount: seatQuota.memberCount,
retryAfter: decision.retryAfterSeconds,
requestId: input.requestId,
cfRay: input.source?.rayId,
});
throw this.mapDecision(decision);
}
metrics.workspace.counter('invite_quota_allowed').add(1);
return {
reservationId: decision.reservationId,
decision,
};
}
async commitWorkspaceInviteQuota(
reservationId: string | undefined,
usage: {
targetCount: number;
targetDomains: RuntimeQuotaTargetDomainInput[];
}
) {
if (!reservationId) {
return false;
}
return await this.runtime.commitWorkspaceInviteQuotaV1(
reservationId,
usage
);
}
async releaseWorkspaceInviteQuota(reservationId: string | undefined) {
if (!reservationId) {
return false;
}
return await this.runtime.releaseWorkspaceInviteQuotaV1(reservationId);
}
private mapDecision(
decision: RuntimeWorkspaceInviteQuotaDecision
): UserFriendlyError {
if (decision.reason === 'abuse_subject' || decision.actionRequired) {
return new ActionForbidden('This feature is temporarily unavailable.');
}
return new TooManyRequest();
}
}
@@ -79,6 +79,11 @@ export class WorkspaceEvents {
$$workspaceId: workspaceId,
},
},
metadata: {
workspaceId,
recipientUserId: userId,
source: { trusted: false },
},
});
}
@@ -9,6 +9,10 @@ import { PermissionModule } from '../permission';
import { QuotaModule } from '../quota';
import { StorageModule } from '../storage';
import { UserModule } from '../user';
import {
InviteAbuseDispositionService,
InviteQuotaAssertService,
} from './abuse';
import { WorkspacesController } from './controller';
import { WorkspaceEvents } from './event';
import { WorkspaceRealtimeModule } from './realtime.module';
@@ -46,6 +50,8 @@ import { WorkspaceStatsJob } from './stats.job';
DocHistoryResolver,
WorkspaceBlobResolver,
WorkspaceService,
InviteAbuseDispositionService,
InviteQuotaAssertService,
WorkspaceEvents,
AdminWorkspaceResolver,
WorkspaceStatsJob,
@@ -54,6 +60,11 @@ import { WorkspaceStatsJob } from './stats.job';
})
export class WorkspaceModule {}
export {
getAbuseRequestSource,
InviteAbuseDispositionService,
InviteQuotaAssertService,
} from './abuse';
export { WorkspaceRealtimeModule } from './realtime.module';
export { WorkspaceService } from './service';
export { InvitationType, WorkspaceType } from './types';
@@ -2,6 +2,7 @@ import { Field, Int, ObjectType, registerEnumType } from '@nestjs/graphql';
export enum TimeBucket {
Minute = 'Minute',
Hour = 'Hour',
Day = 'Day',
}
@@ -35,6 +35,7 @@ import {
import { PageInfo } from '../../../base/graphql/pagination';
import { Models, PublicDocMode } from '../../../models';
import { CurrentUser } from '../../auth';
import { BackendRuntimeProvider } from '../../backend-runtime';
import { Editor } from '../../doc';
import {
DOC_ACTIONS,
@@ -303,13 +304,34 @@ export class WorkspaceDocResolver {
private readonly models: Models,
private readonly cache: Cache,
private readonly event: EventBus,
private readonly config: Config
private readonly config: Config,
private readonly runtime: BackendRuntimeProvider
) {}
private async assertCanShare(
userId: string,
context: { workspaceId: string; docId: string; action: 'publishDoc' }
) {
if (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(userId)) {
this.logger.warn('Share action blocked for quarantined actor', {
userId,
...context,
});
throw new ActionForbidden(
'This feature is temporarily unavailable for you.'
);
}
if (
await this.runtime.isInviteAbuseWorkspaceQuarantined(context.workspaceId)
) {
this.logger.warn('Share action blocked for quarantined workspace', {
userId,
...context,
});
throw new ActionForbidden(
'This feature is temporarily unavailable for you.'
);
}
const user = await this.models.user.get(userId);
const newAccountAgeMs = this.config.auth.newAccountShareActionDelay * 1000;
if (!user || !canUserExecuteLimitedActions(user, newAccountAgeMs)) {
@@ -1,6 +1,7 @@
import { Logger } from '@nestjs/common';
import {
Args,
Context,
Int,
Mutation,
Parent,
@@ -24,6 +25,7 @@ import {
CanNotRevokeYourself,
Config,
EventBus,
getRequestTrackerId,
InvalidInvitation,
isValidCacheTtl,
mapAnyError,
@@ -38,8 +40,11 @@ import {
URLHelper,
UserNotFound,
} from '../../../base';
import type { GraphqlContext } from '../../../base/graphql';
import { Models } from '../../../models';
import { CurrentUser, Public } from '../../auth';
import { BackendRuntimeProvider } from '../../backend-runtime';
import { containsUrlOrDomain } from '../../content-policy';
import {
PermissionAccess,
WorkspacePolicyService,
@@ -48,7 +53,11 @@ import {
import { QuotaService } from '../../quota';
import { UserType } from '../../user';
import { validators } from '../../utils/validators';
import { canUserExecuteLimitedActions, containsUrlOrDomain } from '../abuse';
import {
canUserExecuteLimitedActions,
getAbuseRequestSource,
InviteQuotaAssertService,
} from '../abuse';
import { WorkspaceService } from '../service';
import {
InvitationType,
@@ -59,6 +68,27 @@ import {
WorkspaceType,
} from '../types';
type InviteCandidate = {
index: number;
email: string;
normalizedEmail: string;
domain: string;
target?: { id: string };
};
function emailDomain(email: string) {
const parts = email.split('@');
return parts.length === 2 ? parts[1] : '';
}
function aggregateTargetDomains(candidates: InviteCandidate[]) {
const domains = new Map<string, number>();
for (const candidate of candidates) {
domains.set(candidate.domain, (domains.get(candidate.domain) ?? 0) + 1);
}
return Array.from(domains, ([domain, count]) => ({ domain, count }));
}
/**
* Workspace team resolver
* Public apis rate limit: 10 req/m
@@ -78,16 +108,39 @@ export class WorkspaceMemberResolver {
private readonly policy: WorkspacePolicyService,
private readonly workspaceService: WorkspaceService,
private readonly quota: QuotaService,
private readonly config: Config
private readonly config: Config,
private readonly inviteQuota: InviteQuotaAssertService,
private readonly runtime: BackendRuntimeProvider
) {}
private async assertCanInviteOrShare(
userId: string,
context: {
workspaceId: string;
action: 'inviteMembers' | 'createInviteLink';
action: 'createInviteLink';
}
) {
if (await this.runtime.isInviteAbuseUserQuarantinedOrBanned(userId)) {
this.logger.warn('Share action blocked for quarantined actor', {
userId,
...context,
});
throw new ActionForbidden(
'This feature is temporarily unavailable for you.'
);
}
if (
await this.runtime.isInviteAbuseWorkspaceQuarantined(context.workspaceId)
) {
this.logger.warn('Share action blocked for quarantined workspace', {
userId,
...context,
});
throw new ActionForbidden(
'This feature is temporarily unavailable for you.'
);
}
// Member invites are owned by native quota; this guard stays for invite links until share/link actions migrate.
const user = await this.models.user.get(userId);
const newAccountAgeMs = this.config.auth.newAccountShareActionDelay * 1000;
if (!user || !canUserExecuteLimitedActions(user, newAccountAgeMs)) {
@@ -182,6 +235,7 @@ export class WorkspaceMemberResolver {
@Mutation(() => [InviteResult])
async inviteMembers(
@CurrentUser() me: CurrentUser,
@Context() context: GraphqlContext,
@Args('workspaceId') workspaceId: string,
@Args({ name: 'emails', type: () => [String] }) emails: string[]
): Promise<InviteResult[]> {
@@ -189,16 +243,54 @@ export class WorkspaceMemberResolver {
.user(me.id)
.workspace(workspaceId)
.assert('Workspace.Users.Manage');
await this.assertCanInviteOrShare(me.id, {
workspaceId,
action: 'inviteMembers',
});
await this.assertWorkspaceNameCanInvite(workspaceId);
if (emails.length > 512) {
throw new TooManyRequest();
}
const results: InviteResult[] = emails.map(email => ({ email }));
const candidates: InviteCandidate[] = [];
const seen = new Set<string>();
for (const [index, email] of emails.entries()) {
try {
const normalizedEmail = email.trim().toLowerCase();
validators.assertValidEmail(normalizedEmail);
if (seen.has(normalizedEmail)) {
throw new ActionForbidden('Duplicate invite email.');
}
seen.add(normalizedEmail);
const target = await this.models.user.getUserByEmail(normalizedEmail);
if (target) {
const originRecord = await this.models.workspaceUser.get(
workspaceId,
target.id
);
if (originRecord) {
throw new AlreadyInSpace({ spaceId: workspaceId });
}
}
candidates.push({
index,
email,
normalizedEmail,
domain: emailDomain(normalizedEmail),
target: target ? { id: target.id } : undefined,
});
} catch (error) {
results[index] = {
email,
error: mapAnyError(error),
};
}
}
if (candidates.length === 0) {
return results;
}
// lock to prevent concurrent invite
const lockFlag = `invite:${workspaceId}`;
await using lock = await this.mutex.acquire(lockFlag);
@@ -206,49 +298,68 @@ export class WorkspaceMemberResolver {
throw new TooManyRequest();
}
const admission = await this.inviteQuota.assertWorkspaceInviteQuota({
actorUserId: me.id,
workspaceId,
requestId: getRequestTrackerId(context.req),
targetCount: candidates.length,
targetDomains: aggregateTargetDomains(candidates),
source: getAbuseRequestSource(context.req, this.config),
});
const quota = await this.quota.getWorkspaceSeatQuota(workspaceId);
const isTeam = await this.workspaceService.isTeamWorkspace(workspaceId);
const successfulCandidates: InviteCandidate[] = [];
let reservationSettled = false;
const results: InviteResult[] = [];
try {
for (const candidate of candidates) {
try {
let target = candidate.target;
if (!target) {
target = await this.models.user.create({
email: candidate.normalizedEmail,
registered: false,
});
}
for (const [idx, email] of emails.entries()) {
try {
validators.assertValidEmail(email);
let target = await this.models.user.getUserByEmail(email);
if (target) {
const originRecord = await this.models.workspaceUser.get(
const existingMember = await this.models.workspaceUser.get(
workspaceId,
target.id
);
// only invite if the user is not already in the workspace
if (originRecord) {
if (existingMember) {
throw new AlreadyInSpace({ spaceId: workspaceId });
}
} else {
target = await this.models.user.create({
email,
registered: false,
});
}
// no need to check quota, directly go allocating seat path
if (isTeam) {
const role = await this.models.workspaceUser.set(
workspaceId,
target.id,
WorkspaceRole.Collaborator,
{
status: WorkspaceMemberStatus.AllocatingSeat,
source: WorkspaceMemberSource.Email,
inviterId: me.id,
if (!isTeam) {
const needMoreSeat =
quota.memberCount + successfulCandidates.length + 1 >
quota.memberLimit;
if (needMoreSeat) {
throw new NoMoreSeat({ spaceId: workspaceId });
}
);
await this.allocateAvailableTeamSeats(workspaceId, quota.memberLimit);
results.push({ email, inviteId: role.id });
} else {
const needMoreSeat = quota.memberCount + idx + 1 > quota.memberLimit;
if (needMoreSeat) {
throw new NoMoreSeat({ spaceId: workspaceId });
}
// no need to check quota, directly go allocating seat path
if (isTeam) {
const role = await this.models.workspaceUser.set(
workspaceId,
target.id,
WorkspaceRole.Collaborator,
{
status: WorkspaceMemberStatus.AllocatingSeat,
source: WorkspaceMemberSource.Email,
inviterId: me.id,
}
);
await this.allocateAvailableTeamSeats(
workspaceId,
quota.memberLimit
);
results[candidate.index] = {
email: candidate.email,
inviteId: role.id,
};
successfulCandidates.push(candidate);
} else {
const role = await this.models.workspaceUser.set(
workspaceId,
@@ -264,17 +375,39 @@ export class WorkspaceMemberResolver {
inviteId: role.id,
inviterId: me.id,
});
results.push({
email,
results[candidate.index] = {
email: candidate.email,
inviteId: role.id,
});
};
successfulCandidates.push(candidate);
}
} catch (error) {
results[candidate.index] = {
email: candidate.email,
error: mapAnyError(error),
};
}
} catch (error) {
results.push({
email,
error: mapAnyError(error),
});
}
if (successfulCandidates.length > 0) {
await this.inviteQuota.commitWorkspaceInviteQuota(
admission.reservationId,
{
targetCount: successfulCandidates.length,
targetDomains: aggregateTargetDomains(successfulCandidates),
}
);
} else {
await this.inviteQuota.releaseWorkspaceInviteQuota(
admission.reservationId
);
}
reservationSettled = true;
} finally {
if (!reservationSettled) {
await this.inviteQuota.releaseWorkspaceInviteQuota(
admission.reservationId
);
}
}
@@ -7,8 +7,10 @@ import {
DEFAULT_WORKSPACE_NAME,
Models,
} from '../../models';
import { BackendRuntimeProvider } from '../backend-runtime';
import { DocReader } from '../doc';
import { Mailer } from '../mail';
import type { SendMailCommand } from '../mail/types';
import { WorkspaceRole } from '../permission';
import { QuotaStateService } from '../quota/state';
import { WorkspaceBlobStorage } from '../storage';
@@ -32,7 +34,8 @@ export class WorkspaceService {
private readonly blobStorage: WorkspaceBlobStorage,
private readonly mailer: Mailer,
private readonly queue: JobQueue,
private readonly quotaState: QuotaStateService
private readonly quotaState: QuotaStateService,
private readonly runtime: BackendRuntimeProvider
) {}
async getInviteInfo(inviteId: string): Promise<InviteInfo> {
@@ -111,7 +114,7 @@ export class WorkspaceService {
const admins = await this.models.workspaceUser.getAdmins(workspaceId);
const link = this.url.link(`/workspace/${workspaceId}`);
await this.mailer.trySend({
await this.trySendWorkspaceMail({
name: 'TeamWorkspaceUpgraded',
to: owner.email,
props: {
@@ -121,11 +124,16 @@ export class WorkspaceService {
isOwner: true,
url: link,
},
metadata: {
workspaceId,
recipientUserId: owner.id,
source: { trusted: false },
},
});
await Promise.allSettled(
admins.map(async user => {
await this.mailer.trySend({
await this.trySendWorkspaceMail({
name: 'TeamWorkspaceUpgraded',
to: user.email,
props: {
@@ -135,6 +143,11 @@ export class WorkspaceService {
isOwner: false,
url: link,
},
metadata: {
workspaceId,
recipientUserId: user.id,
source: { trusted: false },
},
});
})
);
@@ -192,7 +205,7 @@ export class WorkspaceService {
}
if (ws.role === WorkspaceRole.Admin) {
await this.mailer.trySend({
await this.trySendWorkspaceMail({
name: 'TeamBecomeAdmin',
to: user.email,
props: {
@@ -201,9 +214,14 @@ export class WorkspaceService {
},
url: this.url.link(`/workspace/${ws.id}`),
},
metadata: {
workspaceId: ws.id,
recipientUserId: user.id,
source: { trusted: false },
},
});
} else {
await this.mailer.trySend({
await this.trySendWorkspaceMail({
name: 'TeamBecomeCollaborator',
to: user.email,
props: {
@@ -212,12 +230,17 @@ export class WorkspaceService {
},
url: this.url.link(`/workspace/${ws.id}`),
},
metadata: {
workspaceId: ws.id,
recipientUserId: user.id,
source: { trusted: false },
},
});
}
}
async sendOwnershipTransferredEmail(email: string, ws: { id: string }) {
await this.mailer.trySend({
await this.trySendWorkspaceMail({
name: 'OwnershipTransferred',
to: email,
props: {
@@ -225,11 +248,15 @@ export class WorkspaceService {
$$workspaceId: ws.id,
},
},
metadata: {
workspaceId: ws.id,
source: { trusted: false },
},
});
}
async sendOwnershipReceivedEmail(email: string, ws: { id: string }) {
await this.mailer.trySend({
await this.trySendWorkspaceMail({
name: 'OwnershipReceived',
to: email,
props: {
@@ -237,12 +264,16 @@ export class WorkspaceService {
$$workspaceId: ws.id,
},
},
metadata: {
workspaceId: ws.id,
source: { trusted: false },
},
});
}
async sendLeaveEmail(workspaceId: string, userId: string) {
const owner = await this.models.workspaceUser.getOwner(workspaceId);
await this.mailer.trySend({
await this.trySendWorkspaceMail({
name: 'MemberLeave',
to: owner.email,
props: {
@@ -253,9 +284,41 @@ export class WorkspaceService {
$$userId: userId,
},
},
metadata: {
workspaceId,
recipientUserId: owner.id,
actorUserId: userId,
source: { trusted: false },
},
});
}
private async trySendWorkspaceMail(command: SendMailCommand) {
const actorUserId = command.metadata?.actorUserId;
if (
actorUserId &&
(await this.runtime.isInviteAbuseUserQuarantinedOrBanned(actorUserId))
) {
await this.mailer.skip(command, {
mailClass: 'workspace_lifecycle',
reason: 'actor_quarantined',
});
return false;
}
const workspaceId = command.metadata?.workspaceId;
if (
workspaceId &&
(await this.runtime.isInviteAbuseWorkspaceQuarantined(workspaceId))
) {
await this.mailer.skip(command, {
mailClass: 'workspace_lifecycle',
reason: 'workspace_quarantined',
});
return false;
}
return await this.mailer.trySend(command);
}
async allocateSeats(workspaceId: string, quantity: number) {
const pendings = await this.models.workspaceUser.allocateSeats(
workspaceId,
@@ -29,6 +29,7 @@ import { DocUserModel } from './doc-user';
import { FeatureModel } from './feature';
import { HistoryModel } from './history';
import { MagicLinkOtpModel } from './magic-link-otp';
import { MailDeliveryModel } from './mail-delivery';
import { NotificationModel } from './notification';
import { PermissionProjectionModel } from './permission-projection';
import {
@@ -57,6 +58,7 @@ const MODELS = {
session: SessionModel,
verificationToken: VerificationTokenModel,
magicLinkOtp: MagicLinkOtpModel,
mailDelivery: MailDeliveryModel,
feature: FeatureModel,
workspace: WorkspaceModel,
userFeature: UserFeatureModel,
@@ -165,6 +167,7 @@ export * from './doc-user';
export * from './feature';
export * from './history';
export * from './magic-link-otp';
export * from './mail-delivery';
export * from './notification';
export * from './permission-projection';
export * from './permission-write';
@@ -36,6 +36,7 @@ export class MagicLinkOtpModel extends BaseModel {
create: { email, otpHash, token, clientNonce, expiresAt, attempts: 0 },
update: { otpHash, token, clientNonce, expiresAt, attempts: 0 },
});
return { expiresAt };
}
@Transactional()
@@ -0,0 +1,665 @@
import { createHmac } from 'node:crypto';
import { Injectable } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { CryptoHelper } from '../base';
import { BaseModel } from './base';
export type MailDeliveryStatus =
| 'queued'
| 'sending'
| 'retry_wait'
| 'sent'
| 'skipped'
| 'failed'
| 'canceled';
export type MailDeliveryPriority = 'critical' | 'high' | 'normal' | 'low';
export type MailDeliveryRow = {
id: string;
mailName: string;
mailClass: string;
priority: MailDeliveryPriority;
status: MailDeliveryStatus;
dedupeKey: string | null;
recipientEmail: string | null;
recipientHash: string;
recipientDomain: string;
recipientUserId: string | null;
actorUserId: string | null;
workspaceId: string | null;
notificationId: string | null;
abuseSubjectKey: string | null;
quotaReservationId: string | null;
quotaDecision: Prisma.JsonValue | null;
payload: Prisma.JsonValue | null;
sendAfter: Date;
expiresAt: Date | null;
attemptCount: number;
maxAttempts: number;
lockedBy: string | null;
lockedUntil: Date | null;
firstAttemptAt: Date | null;
lastAttemptAt: Date | null;
sentAt: Date | null;
settledAt: Date | null;
canceledAt: Date | null;
failedAt: Date | null;
providerMessageId: string | null;
providerResponse: string | null;
lastErrorCode: string | null;
lastError: string | null;
retentionState: 'full' | 'anonymized';
anonymizedAt: Date | null;
createdAt: Date;
updatedAt: Date;
};
export type MailDeliveryCreateInput = {
mailName: string;
mailClass: string;
priority: MailDeliveryPriority;
status?: Extract<MailDeliveryStatus, 'queued' | 'skipped' | 'failed'>;
dedupeKey?: string;
recipientEmail: string;
recipientUserId?: string;
actorUserId?: string;
workspaceId?: string;
notificationId?: string;
abuseSubjectKey?: string;
quotaReservationId?: string;
quotaDecision?: Prisma.JsonValue;
payload: Prisma.JsonValue;
sendAfter?: Date;
expiresAt?: Date;
maxAttempts?: number;
lastErrorCode?: string;
lastError?: string;
};
export type MailDeliveryAdminAggregate = {
bucket: Date;
mailName: string;
mailClass: string;
status: MailDeliveryStatus;
count: number;
};
export type MailDeliveryMetricsSnapshot = {
retryWait: number;
failedRecent: number;
expiredLeases: number;
readyDelayMs: number;
};
const TERMINAL_STATUSES = new Set<MailDeliveryStatus>([
'sent',
'skipped',
'failed',
'canceled',
]);
function recipientDomain(email: string) {
const parts = email.trim().toLowerCase().split('@');
return parts.length === 2 ? parts[1] : '';
}
function redact(value: string | undefined | null) {
if (!value) {
return null;
}
return value
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '[email]')
.replace(/\b\d{6,}\b/g, '[code]')
.replace(/https?:\/\/\S+/gi, '[url]')
.slice(0, 1000);
}
function mapRow(row: Record<string, unknown>): MailDeliveryRow {
return {
id: row.id as string,
mailName: row.mailName as string,
mailClass: row.mailClass as string,
priority: row.priority as MailDeliveryPriority,
status: row.status as MailDeliveryStatus,
dedupeKey: row.dedupeKey as string | null,
recipientEmail: row.recipientEmail as string | null,
recipientHash: row.recipientHash as string,
recipientDomain: row.recipientDomain as string,
recipientUserId: row.recipientUserId as string | null,
actorUserId: row.actorUserId as string | null,
workspaceId: row.workspaceId as string | null,
notificationId: row.notificationId as string | null,
abuseSubjectKey: row.abuseSubjectKey as string | null,
quotaReservationId: row.quotaReservationId as string | null,
quotaDecision: row.quotaDecision as Prisma.JsonValue | null,
payload: row.payload as Prisma.JsonValue | null,
sendAfter: row.sendAfter as Date,
expiresAt: row.expiresAt as Date | null,
attemptCount: row.attemptCount as number,
maxAttempts: row.maxAttempts as number,
lockedBy: row.lockedBy as string | null,
lockedUntil: row.lockedUntil as Date | null,
firstAttemptAt: row.firstAttemptAt as Date | null,
lastAttemptAt: row.lastAttemptAt as Date | null,
sentAt: row.sentAt as Date | null,
settledAt: row.settledAt as Date | null,
canceledAt: row.canceledAt as Date | null,
failedAt: row.failedAt as Date | null,
providerMessageId: row.providerMessageId as string | null,
providerResponse: row.providerResponse as string | null,
lastErrorCode: row.lastErrorCode as string | null,
lastError: row.lastError as string | null,
retentionState: row.retentionState as 'full' | 'anonymized',
anonymizedAt: row.anonymizedAt as Date | null,
createdAt: row.createdAt as Date,
updatedAt: row.updatedAt as Date,
};
}
const SELECT_FIELDS = Prisma.sql`
id::text AS "id",
mail_name AS "mailName",
mail_class AS "mailClass",
priority,
status,
dedupe_key AS "dedupeKey",
recipient_email AS "recipientEmail",
recipient_hash AS "recipientHash",
recipient_domain AS "recipientDomain",
recipient_user_id AS "recipientUserId",
actor_user_id AS "actorUserId",
workspace_id AS "workspaceId",
notification_id AS "notificationId",
abuse_subject_key AS "abuseSubjectKey",
quota_reservation_id::text AS "quotaReservationId",
quota_decision AS "quotaDecision",
payload,
send_after AS "sendAfter",
expires_at AS "expiresAt",
attempt_count AS "attemptCount",
max_attempts AS "maxAttempts",
locked_by AS "lockedBy",
locked_until AS "lockedUntil",
first_attempt_at AS "firstAttemptAt",
last_attempt_at AS "lastAttemptAt",
sent_at AS "sentAt",
settled_at AS "settledAt",
canceled_at AS "canceledAt",
failed_at AS "failedAt",
provider_message_id AS "providerMessageId",
provider_response AS "providerResponse",
last_error_code AS "lastErrorCode",
last_error AS "lastError",
retention_state AS "retentionState",
anonymized_at AS "anonymizedAt",
created_at AS "createdAt",
updated_at AS "updatedAt"
`;
@Injectable()
export class MailDeliveryModel extends BaseModel {
constructor(private readonly crypto: CryptoHelper) {
super();
}
private recipientHash(email: string) {
return createHmac('sha256', this.crypto.keyPair.sha256.privateKey)
.update(email.trim().toLowerCase())
.digest('hex');
}
async create(input: MailDeliveryCreateInput): Promise<MailDeliveryRow> {
const now = new Date();
const sendAfter = input.sendAfter ?? now;
const status = input.status ?? 'queued';
const expired =
input.expiresAt &&
(input.expiresAt.getTime() <= now.getTime() ||
sendAfter.getTime() >= input.expiresAt.getTime());
const finalStatus = expired ? 'failed' : status;
const terminal = TERMINAL_STATUSES.has(finalStatus);
const lastErrorCode = expired
? 'expired'
: (input.lastErrorCode ?? (status === 'skipped' ? 'skipped' : null));
const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
INSERT INTO mail_deliveries (
mail_name,
mail_class,
priority,
status,
dedupe_key,
recipient_email,
recipient_hash,
recipient_domain,
recipient_user_id,
actor_user_id,
workspace_id,
notification_id,
abuse_subject_key,
quota_reservation_id,
quota_decision,
payload,
send_after,
expires_at,
max_attempts,
settled_at,
failed_at,
last_error_code,
last_error,
retention_state,
anonymized_at
)
VALUES (
${input.mailName},
${input.mailClass},
${input.priority},
${finalStatus},
${input.dedupeKey ?? null},
${terminal ? null : input.recipientEmail},
${this.recipientHash(input.recipientEmail)},
${recipientDomain(input.recipientEmail)},
${input.recipientUserId ?? null},
${input.actorUserId ?? null},
${input.workspaceId ?? null},
${input.notificationId ?? null},
${input.abuseSubjectKey ?? null},
${input.quotaReservationId ?? null}::uuid,
${input.quotaDecision ?? null},
${terminal ? null : input.payload},
${sendAfter},
${input.expiresAt ?? null},
${input.maxAttempts ?? 3},
${terminal ? now : null},
${finalStatus === 'failed' ? now : null},
${lastErrorCode},
${redact(input.lastError)},
${terminal ? 'anonymized' : 'full'},
${terminal ? now : null}
)
ON CONFLICT (dedupe_key) WHERE dedupe_key IS NOT NULL
DO NOTHING
RETURNING ${SELECT_FIELDS}
`;
if (rows[0]) {
return mapRow(rows[0]);
}
if (!input.dedupeKey) {
throw new Error('Failed to create mail delivery.');
}
const existing = await this.findByDedupeKey(input.dedupeKey);
if (!existing) {
throw new Error('Failed to find deduped mail delivery.');
}
return existing;
}
async findByDedupeKey(dedupeKey: string) {
const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
SELECT ${SELECT_FIELDS}
FROM mail_deliveries
WHERE dedupe_key = ${dedupeKey}
LIMIT 1
`;
return rows[0] ? mapRow(rows[0]) : null;
}
async claimReady(
workerId: string,
options: { batchSize: number; leaseMs: number }
) {
await this.markExpiredPending(options.batchSize);
const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
UPDATE mail_deliveries
SET status = 'sending',
locked_by = ${workerId},
locked_until = now() + (${options.leaseMs}::text || ' milliseconds')::interval,
updated_at = now()
WHERE id IN (
SELECT id
FROM mail_deliveries
WHERE (
status IN ('queued', 'retry_wait')
OR (status = 'sending' AND locked_until < now())
)
AND send_after <= now()
AND (expires_at IS NULL OR expires_at > now())
AND (locked_until IS NULL OR locked_until < now())
AND max_attempts > attempt_count
ORDER BY
CASE priority
WHEN 'critical' THEN 0
WHEN 'high' THEN 1
WHEN 'normal' THEN 2
ELSE 3
END,
send_after,
created_at
FOR UPDATE SKIP LOCKED
LIMIT ${options.batchSize}
)
RETURNING ${SELECT_FIELDS}
`;
return rows.map(mapRow);
}
async markAttemptStarted(id: string, workerId: string) {
const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
UPDATE mail_deliveries
SET attempt_count = attempt_count + 1,
first_attempt_at = COALESCE(first_attempt_at, now()),
last_attempt_at = now(),
updated_at = now()
WHERE id = ${id}::uuid
AND status = 'sending'
AND locked_by = ${workerId}
AND (expires_at IS NULL OR expires_at > now())
AND max_attempts > attempt_count
RETURNING ${SELECT_FIELDS}
`;
return rows[0] ? mapRow(rows[0]) : null;
}
async markSent(
id: string,
workerId: string,
result: {
providerMessageId?: string | null;
providerResponse?: string | null;
}
) {
return await this.markTerminal(id, workerId, 'sent', {
providerMessageId: result.providerMessageId,
providerResponse: redact(result.providerResponse),
});
}
async markRetry(
id: string,
workerId: string,
input: {
sendAfter: Date;
errorCode?: string | null;
error?: string | null;
}
) {
const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
UPDATE mail_deliveries
SET status = 'retry_wait',
send_after = ${input.sendAfter},
locked_by = NULL,
locked_until = NULL,
last_error_code = ${input.errorCode ?? null},
last_error = ${redact(input.error)},
updated_at = now()
WHERE id = ${id}::uuid
AND status = 'sending'
AND locked_by = ${workerId}
AND (expires_at IS NULL OR ${input.sendAfter} < expires_at)
AND attempt_count < max_attempts
RETURNING ${SELECT_FIELDS}
`;
return rows[0] ? mapRow(rows[0]) : null;
}
async markSkipped(
id: string,
workerId: string,
reason: string,
detail?: string
) {
return await this.markTerminal(id, workerId, 'skipped', {
lastErrorCode: reason,
lastError: redact(detail),
});
}
async markFailed(
id: string,
workerId: string,
errorCode: string,
error?: string
) {
return await this.markTerminal(id, workerId, 'failed', {
lastErrorCode: errorCode,
lastError: redact(error),
});
}
async markCanceled(id: string, workerId: string, reason = 'canceled') {
return await this.markTerminal(id, workerId, 'canceled', {
lastErrorCode: reason,
});
}
async cancelByRecipient(
recipientEmail: string,
reason = 'recipient_deleted'
) {
return await this.cancelWhere(
Prisma.sql`recipient_hash = ${this.recipientHash(recipientEmail)}`,
reason
);
}
async cancelById(id: string, reason = 'canceled') {
return await this.cancelWhere(Prisma.sql`id = ${id}::uuid`, reason);
}
async cancelMemberInvitationByActor(
actorUserId: string,
reason = 'actor_deleted'
) {
return await this.cancelWhere(
Prisma.sql`mail_name = 'MemberInvitation' AND actor_user_id = ${actorUserId}`,
reason
);
}
async cancelByActor(actorUserId: string, reason = 'actor_quarantined') {
return await this.cancelWhere(
Prisma.sql`actor_user_id = ${actorUserId}`,
reason
);
}
async cancelByWorkspace(
workspaceId: string,
reason = 'workspace_quarantined'
) {
return await this.cancelWhere(
Prisma.sql`workspace_id = ${workspaceId}`,
reason
);
}
async cancelByAbuseSubject(
subjectKey: string,
reason = 'abuse_subject_quarantined'
) {
return await this.cancelWhere(
Prisma.sql`abuse_subject_key = ${subjectKey}`,
reason
);
}
async markExpiredPending(limit: number) {
const result = await this.db.$executeRaw`
UPDATE mail_deliveries
SET status = 'failed',
failed_at = now(),
settled_at = now(),
locked_by = NULL,
locked_until = NULL,
recipient_email = NULL,
payload = NULL,
retention_state = 'anonymized',
anonymized_at = now(),
last_error_code = 'expired',
updated_at = now()
WHERE id IN (
SELECT id
FROM mail_deliveries
WHERE (
status IN ('queued', 'retry_wait')
OR (status = 'sending' AND locked_until < now())
)
AND expires_at IS NOT NULL
AND expires_at <= now()
LIMIT ${limit}
)
`;
return Number(result);
}
async adminAggregate(input: {
from: Date;
to: Date;
bucket: 'hour' | 'day';
}) {
const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
WITH events AS (
SELECT
date_trunc(${input.bucket}, created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AS bucket,
mail_name,
mail_class,
status,
COUNT(*)::int AS count
FROM mail_deliveries
WHERE created_at >= ${input.from}
AND created_at < ${input.to}
GROUP BY
bucket,
mail_name,
mail_class,
status
)
SELECT
bucket AS "bucket",
mail_name AS "mailName",
mail_class AS "mailClass",
status,
count
FROM events
ORDER BY bucket ASC, count DESC, "mailName" ASC
`;
return rows.map(row => ({
bucket: row.bucket as Date,
mailName: row.mailName as string,
mailClass: row.mailClass as string,
status: row.status as MailDeliveryStatus,
count: row.count as number,
})) satisfies MailDeliveryAdminAggregate[];
}
async deleteAnonymizedBefore(before: Date, limit: number) {
const result = await this.db.$executeRaw`
DELETE FROM mail_deliveries
WHERE id IN (
SELECT id
FROM mail_deliveries
WHERE retention_state = 'anonymized'
AND settled_at IS NOT NULL
AND settled_at < ${before}
ORDER BY settled_at
LIMIT ${limit}
)
`;
return Number(result);
}
async metricsSnapshot(): Promise<MailDeliveryMetricsSnapshot> {
const rows = await this.db.$queryRaw<
Array<{
retryWait: number;
failedRecent: number;
expiredLeases: number;
readyDelayMs: number;
}>
>`
SELECT
COUNT(*) FILTER (WHERE status = 'retry_wait')::int AS "retryWait",
COUNT(*) FILTER (
WHERE status = 'failed' AND failed_at >= now() - interval '1 hour'
)::int AS "failedRecent",
COUNT(*) FILTER (
WHERE status = 'sending' AND locked_until < now()
)::int AS "expiredLeases",
COALESCE(MAX(EXTRACT(EPOCH FROM now() - send_after) * 1000) FILTER (
WHERE status IN ('queued', 'retry_wait') AND send_after <= now()
), 0)::int AS "readyDelayMs"
FROM mail_deliveries
`;
return (
rows[0] ?? {
retryWait: 0,
failedRecent: 0,
expiredLeases: 0,
readyDelayMs: 0,
}
);
}
private async markTerminal(
id: string,
workerId: string,
status: Extract<
MailDeliveryStatus,
'sent' | 'skipped' | 'failed' | 'canceled'
>,
input: {
providerMessageId?: string | null;
providerResponse?: string | null;
lastErrorCode?: string | null;
lastError?: string | null;
}
) {
const rows = await this.db.$queryRaw<Array<Record<string, unknown>>>`
UPDATE mail_deliveries
SET status = ${status},
sent_at = CASE WHEN ${status} = 'sent' THEN now() ELSE sent_at END,
failed_at = CASE WHEN ${status} = 'failed' THEN now() ELSE failed_at END,
canceled_at = CASE WHEN ${status} = 'canceled' THEN now() ELSE canceled_at END,
settled_at = now(),
locked_by = NULL,
locked_until = NULL,
recipient_email = NULL,
payload = NULL,
retention_state = 'anonymized',
anonymized_at = now(),
provider_message_id = ${input.providerMessageId ?? null},
provider_response = ${input.providerResponse ?? null},
last_error_code = ${input.lastErrorCode ?? null},
last_error = ${input.lastError ?? null},
updated_at = now()
WHERE id = ${id}::uuid
AND status = 'sending'
AND locked_by = ${workerId}
RETURNING ${SELECT_FIELDS}
`;
return rows[0] ? mapRow(rows[0]) : null;
}
private async cancelWhere(where: Prisma.Sql, reason: string) {
const result = await this.db.$executeRaw`
UPDATE mail_deliveries
SET status = 'canceled',
canceled_at = now(),
settled_at = now(),
locked_by = NULL,
locked_until = NULL,
recipient_email = NULL,
payload = NULL,
retention_state = 'anonymized',
anonymized_at = now(),
last_error_code = ${reason},
updated_at = now()
WHERE status IN ('queued', 'retry_wait', 'sending')
AND ${where}
`;
return Number(result);
}
}
@@ -358,6 +358,32 @@ export class WorkspaceInvitationModel extends BaseModel {
});
}
@Transactional()
async cancelPendingByActor(actorUserId: string) {
await this.models.permissionProjection.markNewWriteOrigin();
return await this.db.workspaceInvitation.deleteMany({
where: {
inviterUserId: actorUserId,
status: {
in: ['pending', 'waiting_review', 'waiting_seat'],
},
},
});
}
@Transactional()
async cancelPendingByWorkspace(workspaceId: string) {
await this.models.permissionProjection.markNewWriteOrigin();
return await this.db.workspaceInvitation.deleteMany({
where: {
workspaceId,
status: {
in: ['pending', 'waiting_review', 'waiting_seat'],
},
},
});
}
private async supportsCurrentInvitationColumns() {
this.hasCurrentColumns ??= this.db.$queryRaw<Array<{ exists: boolean }>>`
SELECT EXISTS (
@@ -28,17 +28,31 @@ export class VerificationTokenModel extends BaseModel {
type: TokenType,
credential?: string,
ttlInSec: number = 30 * 60
) {
const { token } = await this.createWithExpiresAt(
type,
credential,
ttlInSec
);
return token;
}
async createWithExpiresAt(
type: TokenType,
credential?: string,
ttlInSec: number = 30 * 60
) {
const plaintextToken = randomUUID();
const expiresAt = new Date(Date.now() + ttlInSec * 1000);
const { token } = await this.db.verificationToken.create({
data: {
type,
token: plaintextToken,
credential,
expiresAt: new Date(Date.now() + ttlInSec * 1000),
expiresAt,
},
});
return this.crypto.encrypt(token);
return { token: this.crypto.encrypt(token), expiresAt };
}
/**
+5
View File
@@ -11,6 +11,8 @@ import serverNativeModule, {
type CapabilityAttachmentContract,
type CapabilityModelCapability,
type CommandResponse,
type ContentPolicyScanInput,
type ContentPolicyScanResult,
type ImageInspection,
type ImageInspectionOptions,
type LicenseError,
@@ -78,6 +80,8 @@ export type {
CapabilityAttachmentContract,
CapabilityModelCapability,
CommandResponse,
ContentPolicyScanInput,
ContentPolicyScanResult,
ImageInspection,
ImageInspectionOptions,
LicenseError,
@@ -255,6 +259,7 @@ export const inspectImageForProxy = serverNativeModule.inspectImageForProxy;
export const fetchRemoteAttachment = serverNativeModule.fetchRemoteAttachment;
export const inferRemoteMimeType = serverNativeModule.inferRemoteMimeType;
export const assertSafeUrl = serverNativeModule.assertSafeUrl;
export const scanContentPolicyV1 = serverNativeModule.scanContentPolicyV1;
export const safeFetch = serverNativeModule.safeFetch;
export const activateLicense = serverNativeModule.activateLicense;
export const checkLicenseHealth = serverNativeModule.checkLicenseHealth;
@@ -148,6 +148,10 @@ export class SelfhostTeamSubscriptionManager extends SubscriptionManager {
name: 'TeamLicense',
to: userEmail,
props: { license: key },
metadata: {
dedupeKey: `selfhost-license:${key}`,
source: { trusted: false },
},
});
await this.upsertStripeProviderSubscription(
+40
View File
@@ -105,6 +105,42 @@ type AdminLicensePreview {
workspaceId: String!
}
input AdminMailDeliveriesInput {
hours: Int! = 24
}
type AdminMailDeliveryAnalytics {
byOutcome: [AdminMailDeliverySeries!]!
byStatus: [AdminMailDeliverySeries!]!
byType: [AdminMailDeliverySeries!]!
summary: AdminMailDeliverySummary!
window: TimeWindow!
}
type AdminMailDeliveryPoint {
bucket: DateTime!
count: Int!
}
type AdminMailDeliverySeries {
key: String!
label: String!
points: [AdminMailDeliveryPoint!]!
total: Int!
}
type AdminMailDeliverySummary {
canceled: Int!
failed: Int!
queued: Int!
retryWait: Int!
sending: Int!
sent: Int!
skipped: Int!
successRate: Float!
total: Int!
}
type AdminSharedLinkTopItem {
docId: String!
guestViews: SafeInt!
@@ -1952,6 +1988,9 @@ type Query {
"""Get aggregated dashboard metrics for admin panel"""
adminDashboard(input: AdminDashboardInput): AdminDashboard!
"""Aggregate mail delivery timeline facts for admin panel"""
adminMailDeliveries(input: AdminMailDeliveriesInput): AdminMailDeliveryAnalytics!
"""Get workspace detail for admin"""
adminWorkspace(id: String!): AdminWorkspace
@@ -2420,6 +2459,7 @@ type TestWorkspaceByokConfigResultType {
enum TimeBucket {
Day
Hour
Minute
}
@@ -0,0 +1,50 @@
query adminMailDeliveries($input: AdminMailDeliveriesInput) {
adminMailDeliveries(input: $input) {
window {
from
to
timezone
bucket
requestedSize
effectiveSize
}
summary {
total
sent
failed
skipped
canceled
queued
sending
retryWait
successRate
}
byStatus {
key
label
total
points {
bucket
count
}
}
byType {
key
label
total
points {
bucket
count
}
}
byOutcome {
key
label
total
points {
bucket
count
}
}
}
}
@@ -191,6 +191,61 @@ export const adminDashboardQuery = {
}`,
};
export const adminMailDeliveriesQuery = {
id: 'adminMailDeliveriesQuery' as const,
op: 'adminMailDeliveries',
query: `query adminMailDeliveries($input: AdminMailDeliveriesInput) {
adminMailDeliveries(input: $input) {
window {
from
to
timezone
bucket
requestedSize
effectiveSize
}
summary {
total
sent
failed
skipped
canceled
queued
sending
retryWait
successRate
}
byStatus {
key
label
total
points {
bucket
count
}
}
byType {
key
label
total
points {
bucket
count
}
}
byOutcome {
key
label
total
points {
bucket
count
}
}
}
}`,
};
export const adminServerConfigQuery = {
id: 'adminServerConfigQuery' as const,
op: 'adminServerConfig',
+117
View File
@@ -147,6 +147,46 @@ export interface AdminLicensePreview {
workspaceId: Scalars['String']['output'];
}
export interface AdminMailDeliveriesInput {
hours?: Scalars['Int']['input'];
}
export interface AdminMailDeliveryAnalytics {
__typename?: 'AdminMailDeliveryAnalytics';
byOutcome: Array<AdminMailDeliverySeries>;
byStatus: Array<AdminMailDeliverySeries>;
byType: Array<AdminMailDeliverySeries>;
summary: AdminMailDeliverySummary;
window: TimeWindow;
}
export interface AdminMailDeliveryPoint {
__typename?: 'AdminMailDeliveryPoint';
bucket: Scalars['DateTime']['output'];
count: Scalars['Int']['output'];
}
export interface AdminMailDeliverySeries {
__typename?: 'AdminMailDeliverySeries';
key: Scalars['String']['output'];
label: Scalars['String']['output'];
points: Array<AdminMailDeliveryPoint>;
total: Scalars['Int']['output'];
}
export interface AdminMailDeliverySummary {
__typename?: 'AdminMailDeliverySummary';
canceled: Scalars['Int']['output'];
failed: Scalars['Int']['output'];
queued: Scalars['Int']['output'];
retryWait: Scalars['Int']['output'];
sending: Scalars['Int']['output'];
sent: Scalars['Int']['output'];
skipped: Scalars['Int']['output'];
successRate: Scalars['Float']['output'];
total: Scalars['Int']['output'];
}
export interface AdminSharedLinkTopItem {
__typename?: 'AdminSharedLinkTopItem';
docId: Scalars['String']['output'];
@@ -2620,6 +2660,8 @@ export interface Query {
adminAllSharedLinks: PaginatedAdminAllSharedLink;
/** Get aggregated dashboard metrics for admin panel */
adminDashboard: AdminDashboard;
/** Aggregate mail delivery timeline facts for admin panel */
adminMailDeliveries: AdminMailDeliveryAnalytics;
/** Get workspace detail for admin */
adminWorkspace: Maybe<AdminWorkspace>;
/** List workspaces for admin */
@@ -2677,6 +2719,10 @@ export interface QueryAdminDashboardArgs {
input?: InputMaybe<AdminDashboardInput>;
}
export interface QueryAdminMailDeliveriesArgs {
input?: InputMaybe<AdminMailDeliveriesInput>;
}
export interface QueryAdminWorkspaceArgs {
id: Scalars['String']['input'];
}
@@ -3143,6 +3189,7 @@ export interface TestWorkspaceByokConfigResultType {
export enum TimeBucket {
Day = 'Day',
Hour = 'Hour',
Minute = 'Minute',
}
@@ -3907,6 +3954,71 @@ export type AdminDashboardQuery = {
};
};
export type AdminMailDeliveriesQueryVariables = Exact<{
input?: InputMaybe<AdminMailDeliveriesInput>;
}>;
export type AdminMailDeliveriesQuery = {
__typename?: 'Query';
adminMailDeliveries: {
__typename?: 'AdminMailDeliveryAnalytics';
window: {
__typename?: 'TimeWindow';
from: string;
to: string;
timezone: string;
bucket: TimeBucket;
requestedSize: number;
effectiveSize: number;
};
summary: {
__typename?: 'AdminMailDeliverySummary';
total: number;
sent: number;
failed: number;
skipped: number;
canceled: number;
queued: number;
sending: number;
retryWait: number;
successRate: number;
};
byStatus: Array<{
__typename?: 'AdminMailDeliverySeries';
key: string;
label: string;
total: number;
points: Array<{
__typename?: 'AdminMailDeliveryPoint';
bucket: string;
count: number;
}>;
}>;
byType: Array<{
__typename?: 'AdminMailDeliverySeries';
key: string;
label: string;
total: number;
points: Array<{
__typename?: 'AdminMailDeliveryPoint';
bucket: string;
count: number;
}>;
}>;
byOutcome: Array<{
__typename?: 'AdminMailDeliverySeries';
key: string;
label: string;
total: number;
points: Array<{
__typename?: 'AdminMailDeliveryPoint';
bucket: string;
count: number;
}>;
}>;
};
};
export type AdminServerConfigQueryVariables = Exact<{ [key: string]: never }>;
export type AdminServerConfigQuery = {
@@ -7718,6 +7830,11 @@ export type Queries =
variables: AdminDashboardQueryVariables;
response: AdminDashboardQuery;
}
| {
name: 'adminMailDeliveriesQuery';
variables: AdminMailDeliveriesQueryVariables;
response: AdminMailDeliveriesQuery;
}
| {
name: 'adminServerConfigQuery';
variables: AdminServerConfigQueryVariables;
+28
View File
@@ -50,6 +50,10 @@
"queues.backendRuntime": {
"type": "Object",
"desc": "The config for backend runtime job queue"
},
"queues.inviteAbuse": {
"type": "Object",
"desc": "The config for invite abuse disposition job queue"
}
},
"throttle": {
@@ -87,6 +91,18 @@
"type": "Number",
"desc": "Minimum account age in seconds before new accounts can invite members or create share links."
},
"trustedCloudflareHeaders": {
"type": "Boolean",
"desc": "Whether request abuse source facts should trust Cloudflare headers from the origin edge."
},
"inviteQuotaShadowMode": {
"type": "Boolean",
"desc": "Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions."
},
"inviteQuotaFailOpenOnRuntimeError": {
"type": "Boolean",
"desc": "Whether workspace invite quota should fail open when native runtime admission is unavailable. Keep disabled for production."
},
"passwordRequirements": {
"type": "Object",
"desc": "The password strength requirements when set new password."
@@ -140,6 +156,18 @@
"type": "Array",
"desc": "The emails from these domains are always sent using the fallback SMTP server."
},
"deliveryWorker.batchSize": {
"type": "Number",
"desc": "Number of mail delivery rows claimed by each worker tick."
},
"deliveryWorker.leaseMs": {
"type": "Number",
"desc": "Mail delivery worker lease duration in milliseconds."
},
"deliveryWorker.retentionDays": {
"type": "Number",
"desc": "Days to retain anonymized terminal mail delivery ledger rows."
},
"fallbackSMTP.name": {
"type": "String",
"desc": "Hostname used for fallback SMTP HELO/EHLO (e.g. mail.example.com). Leave empty to use the system hostname."
@@ -87,16 +87,91 @@ const dashboardData = {
},
};
const mailDeliveryData = {
adminMailDeliveries: {
window: {
from: '2026-02-15T20:00:00.000Z',
to: '2026-02-16T20:00:00.000Z',
timezone: 'UTC',
bucket: 'Hour',
requestedSize: 24,
effectiveSize: 24,
},
summary: {
total: 4,
sent: 2,
failed: 1,
skipped: 0,
canceled: 0,
queued: 1,
sending: 0,
retryWait: 0,
successRate: 2 / 3,
},
byStatus: [
{
key: 'sent',
label: 'Sent',
total: 2,
points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 2 }],
},
{
key: 'failed',
label: 'Failed',
total: 1,
points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 1 }],
},
{
key: 'queued',
label: 'Queued',
total: 1,
points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 1 }],
},
],
byType: [
{
key: 'auth',
label: 'auth',
total: 2,
points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 2 }],
},
],
byOutcome: [
{
key: 'successful',
label: 'Successful',
total: 2,
points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 2 }],
},
{
key: 'unsuccessful',
label: 'Unsuccessful',
total: 1,
points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 1 }],
},
{
key: 'pending',
label: 'Pending',
total: 1,
points: [{ bucket: '2026-02-16T19:00:00.000Z', count: 1 }],
},
],
},
};
describe('DashboardPage', () => {
beforeEach(() => {
(globalThis as any).environment = {
isSelfHosted: true,
};
useQueryMock.mockReset();
useQueryMock.mockReturnValue({
data: dashboardData,
useQueryMock.mockImplementation(({ query }: { query: { id: string } }) => ({
data:
query.id === 'adminMailDeliveriesQuery'
? mailDeliveryData
: dashboardData,
isValidating: false,
});
}));
mutateQueryResourceMock.mockReset();
});
@@ -125,4 +200,14 @@ describe('DashboardPage', () => {
expect(styles).toContain('--color-secondary: var(--muted-foreground);');
expect(styles).not.toContain('hsl(var(--primary))');
});
test('renders mail delivery analytics controls and summary', () => {
const { getByText } = render(<DashboardPage />);
expect(getByText('Email Delivery Trend')).toBeTruthy();
expect(getByText('Mail Delivery')).toBeTruthy();
expect(getByText('24 hours')).toBeTruthy();
expect(getByText('Success rate')).toBeTruthy();
expect(getByText('66.7%')).toBeTruthy();
});
});
@@ -46,12 +46,17 @@ import {
} from '@affine/admin/components/ui/table';
import { useMutation } from '@affine/admin/use-mutation';
import { useQuery } from '@affine/admin/use-query';
import { adminDashboardQuery, previewLicenseMutation } from '@affine/graphql';
import {
adminDashboardQuery,
adminMailDeliveriesQuery,
previewLicenseMutation,
} from '@affine/graphql';
import { ROUTES } from '@affine/routes';
import {
ChevronDownIcon,
DatabaseIcon,
FileSearchIcon,
MailIcon,
MessageSquareTextIcon,
RefreshCwIcon,
UsersIcon,
@@ -166,6 +171,7 @@ const utcDateFormatter = new Intl.DateTimeFormat('en-US', {
const STORAGE_DAY_OPTIONS = [7, 14, 30, 60, 90] as const;
const SYNC_HOUR_OPTIONS = [1, 6, 12, 24, 48, 72] as const;
const SHARED_DAY_OPTIONS = [7, 14, 28, 60, 90] as const;
type MailChartMode = 'status' | 'type' | 'outcome';
type DualNumberPoint = {
label: string;
@@ -180,6 +186,18 @@ type TrendPoint = {
secondary?: number;
};
type MultiTrendPoint = {
x: number;
label: string;
} & Record<string, number | string>;
type MultiTrendSeries = {
key: string;
label: string;
color: string;
total: number;
};
type LicensePreview = {
id: string;
workspaceId: string;
@@ -362,6 +380,110 @@ function TrendChart({
);
}
function MultiTrendChart({
ariaLabel,
points,
series,
valueFormatter,
}: {
ariaLabel: string;
points: MultiTrendPoint[];
series: MultiTrendSeries[];
valueFormatter: (value: number) => string;
}) {
const visibleSeries = series.filter(item => item.total > 0).slice(0, 4);
if (points.length === 0 || visibleSeries.length === 0) {
return (
<div className="flex h-44 items-center justify-center rounded-lg border border-dashed border-border/60 bg-muted/15 text-sm text-muted-foreground">
No mail deliveries in this window
</div>
);
}
const chartPoints =
points.length === 1
? [points[0], { ...points[0], x: Number(points[0].x) + 1 }]
: points;
const config: ChartConfig = Object.fromEntries(
visibleSeries.map(item => [
item.key,
{
label: item.label,
color: item.color,
},
])
);
return (
<ChartContainer
config={config}
className="h-44 w-full"
aria-label={ariaLabel}
role="img"
>
<LineChart
data={chartPoints}
margin={{ top: 8, right: 0, bottom: 0, left: 0 }}
>
<CartesianGrid
vertical={false}
stroke="var(--border)"
strokeDasharray="3 4"
/>
<XAxis
dataKey="x"
type="number"
hide
allowDecimals={false}
domain={['dataMin', 'dataMax']}
/>
<YAxis
hide
domain={[
0,
(max: number) => {
if (max <= 0) {
return 1;
}
return Math.ceil(max * 1.1);
},
]}
/>
<ChartTooltip
cursor={{
stroke: 'var(--border)',
strokeDasharray: '4 4',
strokeWidth: 1,
}}
content={
<ChartTooltipContent
labelFormatter={(_, payload) => {
const item = payload?.[0];
return item?.payload?.label ?? '';
}}
valueFormatter={value => valueFormatter(value)}
/>
}
/>
{visibleSeries.map(item => (
<Line
key={item.key}
dataKey={item.key}
type="monotone"
stroke={`var(--color-${item.key})`}
strokeWidth={item.key === visibleSeries[0]?.key ? 3 : 2}
dot={false}
activeDot={{ r: 3 }}
connectNulls
isAnimationActive={false}
/>
))}
</LineChart>
</ChartContainer>
);
}
function PrimaryMetricCard({
value,
description,
@@ -458,6 +580,37 @@ function WindowSelect({
);
}
function MailWindowSelect({
value,
onChange,
}: {
value: number;
onChange: (value: number) => void;
}) {
return (
<div className="flex min-w-0 flex-col gap-2">
<Label
htmlFor="mail-delivery-window"
className="text-xs uppercase tracking-wide text-muted-foreground"
>
Mail Delivery
</Label>
<Select
value={String(value)}
onValueChange={next => onChange(Number(next))}
>
<SelectTrigger id="mail-delivery-window">
<SelectValue placeholder="Select mail window…" />
</SelectTrigger>
<SelectContent>
<SelectItem value="24">24 hours</SelectItem>
<SelectItem value="168">7 days</SelectItem>
</SelectContent>
</Select>
</div>
);
}
function LicensePreviewDialog({
license,
onOpenChange,
@@ -830,10 +983,244 @@ function TopSharedLinksSection({
);
}
function mailWindowLabel(hours: number) {
return hours === 24 ? '24h' : '7d';
}
function mailBucketLabel(value: string, hours: number) {
return hours === 24 ? formatDateTime(value) : formatDate(value);
}
function toMailChartPoints(
series: {
key: string;
points: { bucket: string; count: number }[];
}[],
hours: number
) {
const first = series[0]?.points ?? [];
return first.map((point, index) => {
const row: MultiTrendPoint = {
x: index,
label: mailBucketLabel(point.bucket, hours),
};
for (const item of series) {
row[item.key] = item.points[index]?.count ?? 0;
}
return row;
});
}
function seriesColor(key: string, index: number) {
const colors: Record<string, string> = {
sent: 'var(--primary)',
failed: 'var(--destructive)',
retry_wait: 'var(--muted-foreground)',
queued: 'var(--foreground)',
pending_status: 'var(--muted-foreground)',
successful: 'var(--primary)',
unsuccessful: 'var(--destructive)',
pending: 'var(--muted-foreground)',
auth: 'var(--primary)',
notification: 'var(--muted-foreground)',
workspace_invitation: 'var(--foreground)',
};
return (
colors[key] ??
['var(--primary)', 'var(--muted-foreground)', 'var(--foreground)'][
index % 3
]
);
}
function combineMailSeries(
key: string,
label: string,
series: { points: { bucket: string; count: number }[] }[]
) {
const first = series[0];
return {
key,
label,
total: series.reduce(
(total, item) =>
total + item.points.reduce((sum, point) => sum + point.count, 0),
0
),
points:
first?.points.map((point, index) => ({
bucket: point.bucket,
count: series.reduce(
(total, item) => total + (item.points[index]?.count ?? 0),
0
),
})) ?? [],
};
}
function MailDeliverySection({ hours }: { hours: number }) {
const [mode, setMode] = useState<MailChartMode>('status');
const { data } = useQuery(
{
query: adminMailDeliveriesQuery,
variables: {
input: { hours },
},
},
{
keepPreviousData: true,
revalidateOnFocus: false,
revalidateIfStale: true,
revalidateOnReconnect: true,
}
);
const analytics = data.adminMailDeliveries;
const sourceSeries =
mode === 'type'
? analytics.byType
: mode === 'outcome'
? analytics.byOutcome
: [
analytics.byStatus.find(series => series.key === 'sent'),
analytics.byStatus.find(series => series.key === 'failed'),
combineMailSeries(
'pending_status',
'Pending',
analytics.byStatus.filter(series =>
['queued', 'sending'].includes(series.key)
)
),
analytics.byStatus.find(series => series.key === 'retry_wait'),
].filter(series => series !== undefined);
const chartSeries = sourceSeries.map((series, index) => ({
key: series.key,
label: series.label,
total: series.total,
color: seriesColor(series.key, index),
}));
const chartPoints = toMailChartPoints(sourceSeries, hours);
const failedLike =
analytics.summary.failed +
analytics.summary.skipped +
analytics.summary.canceled;
const pending =
analytics.summary.queued +
analytics.summary.sending +
analytics.summary.retryWait;
return (
<Card className="border-border/60 bg-card shadow-1">
<CardHeader className="flex flex-col gap-3 pb-4 md:flex-row md:items-start md:justify-between">
<div className="space-y-1.5">
<CardTitle className="flex items-center gap-2 text-base">
<MailIcon className="h-4 w-4" aria-hidden="true" />
Email Delivery Trend
</CardTitle>
<CardDescription>
{mailWindowLabel(hours)} at{' '}
{analytics.window.bucket === 'Hour' ? 'hour' : 'day'} bucket in UTC
</CardDescription>
</div>
<Select
value={mode}
onValueChange={value => setMode(value as MailChartMode)}
>
<SelectTrigger className="w-full md:w-44">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="status">Status</SelectItem>
<SelectItem value="type">Mail type</SelectItem>
<SelectItem value="outcome">Success / failure</SelectItem>
</SelectContent>
</Select>
</CardHeader>
<CardContent className="space-y-4">
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
<div className="rounded-md border border-border/60 bg-muted/15 px-3 py-2">
<div className="text-xs text-muted-foreground">Sent</div>
<div className="mt-1 text-xl font-semibold tabular-nums">
{compactFormatter.format(analytics.summary.sent)}
</div>
</div>
<div className="rounded-md border border-border/60 bg-muted/15 px-3 py-2">
<div className="text-xs text-muted-foreground">Not delivered</div>
<div className="mt-1 text-xl font-semibold tabular-nums">
{compactFormatter.format(failedLike)}
</div>
</div>
<div className="rounded-md border border-border/60 bg-muted/15 px-3 py-2">
<div className="text-xs text-muted-foreground">Pending</div>
<div className="mt-1 text-xl font-semibold tabular-nums">
{compactFormatter.format(pending)}
</div>
</div>
<div className="rounded-md border border-border/60 bg-muted/15 px-3 py-2">
<div className="text-xs text-muted-foreground">Success rate</div>
<div className="mt-1 text-xl font-semibold tabular-nums">
{(analytics.summary.successRate * 100).toFixed(1)}%
</div>
</div>
</div>
<MultiTrendChart
ariaLabel="Email delivery trend"
points={chartPoints}
series={chartSeries}
valueFormatter={value => intFormatter.format(value)}
/>
<div className="flex flex-wrap items-center gap-4 text-xs text-muted-foreground">
{chartSeries
.filter(series => series.total > 0)
.slice(0, 4)
.map(series => (
<div key={series.key} className="flex items-center gap-2">
<span
className="h-2 w-2 rounded-full"
style={{ backgroundColor: series.color }}
aria-hidden="true"
/>
{series.label}: {compactFormatter.format(series.total)}
</div>
))}
</div>
<div className="flex justify-between text-xs text-muted-foreground tabular-nums">
<span>{mailBucketLabel(analytics.window.from, hours)}</span>
<span>{mailBucketLabel(analytics.window.to, hours)}</span>
</div>
</CardContent>
</Card>
);
}
function MailDeliveryCardSkeleton() {
return (
<Card className="border-border/60 bg-card shadow-1">
<CardHeader>
<Skeleton className="h-5 w-44" />
<Skeleton className="h-4 w-64" />
</CardHeader>
<CardContent className="space-y-4">
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
<Skeleton className="h-16 w-full" />
<Skeleton className="h-16 w-full" />
<Skeleton className="h-16 w-full" />
<Skeleton className="h-16 w-full" />
</div>
<Skeleton className="h-44 w-full" />
</CardContent>
</Card>
);
}
function DashboardPageContent() {
const [storageHistoryDays, setStorageHistoryDays] = useState<number>(30);
const [syncHistoryHours, setSyncHistoryHours] = useState<number>(48);
const [sharedLinkWindowDays, setSharedLinkWindowDays] = useState<number>(28);
const [mailDeliveryHours, setMailDeliveryHours] = useState<number>(24);
const shouldShowTopSharedLinks = !environment.isSelfHosted;
const revalidateQueryResource = useMutateQueryResource();
@@ -902,6 +1289,7 @@ function DashboardPageContent() {
isValidating={isValidating}
onRefresh={() => {
revalidateQueryResource(adminDashboardQuery).catch(() => {});
revalidateQueryResource(adminMailDeliveriesQuery).catch(() => {});
}}
/>
}
@@ -916,7 +1304,7 @@ function DashboardPageContent() {
automatically.
</CardDescription>
</CardHeader>
<CardContent className="grid grid-cols-1 items-end gap-3 md:grid-cols-2 lg:grid-cols-3">
<CardContent className="grid grid-cols-1 items-end gap-3 md:grid-cols-2 lg:grid-cols-4">
<WindowSelect
id="storage-history-window"
label="Storage History"
@@ -941,6 +1329,10 @@ function DashboardPageContent() {
unit="days"
onChange={setSharedLinkWindowDays}
/>
<MailWindowSelect
value={mailDeliveryHours}
onChange={setMailDeliveryHours}
/>
</CardContent>
</Card>
@@ -1035,6 +1427,10 @@ function DashboardPageContent() {
</Card>
</div>
<Suspense fallback={<MailDeliveryCardSkeleton />}>
<MailDeliverySection hours={mailDeliveryHours} />
</Suspense>
{shouldShowTopSharedLinks ? (
<Suspense fallback={<TopSharedLinksCardSkeleton />}>
<TopSharedLinksSection
@@ -1,5 +1,5 @@
{
"ar": 94,
"ar": 95,
"ca": 92,
"da": 4,
"de": 100,
@@ -7,7 +7,7 @@
"en": 100,
"es-AR": 91,
"es-CL": 92,
"es": 90,
"es": 91,
"fa": 90,
"fr": 94,
"hi": 1,
-8
View File
@@ -2506,14 +2506,6 @@ export function useAFFiNEI18N(): {
* `Import your Notion data. Supported import formats: HTML with subpages.`
*/
["com.affine.import.notion.tooltip"](): string;
/**
* `Notion (Markdown, .zip)`
*/
["com.affine.import.notion-markdown"](): string;
/**
* `Import a Notion Markdown export zip with subpages and attachments.`
*/
["com.affine.import.notion-markdown.tooltip"](): string;
/**
* `OneNote (Experimental)`
*/
@@ -625,8 +625,6 @@
"com.affine.import.modal.tip": "If you'd like to request support for additional file types, feel free to let us know on",
"com.affine.import.notion": "Notion (Experimental)",
"com.affine.import.notion.tooltip": "Import your Notion data. Supported import formats: HTML with subpages.",
"com.affine.import.notion-markdown": "Notion (Markdown, .zip)",
"com.affine.import.notion-markdown.tooltip": "Import a Notion Markdown export zip with subpages and attachments.",
"com.affine.import.onenote": "OneNote (Experimental)",
"com.affine.import.onenote.tooltip": "Import a OneNote .one, .onetoc2, or .onepkg file. Available in the desktop app.",
"com.affine.import.onenote.desktop-only": "This format importer is available in the AFFiNE desktop app.",