Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-05-08 22:07:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): allow custom R2 jurisdictional endpoint (#14848)

Browse Source 
## Summary
This PR fixes `cloudflare-r2` storage configuration so jurisdictional R2
endpoints (for example EU buckets) work correctly.

Closes #14847

## Problem
`cloudflare-r2` currently ignores `config.endpoint` and always uses:

`https://<accountId>.r2.cloudflarestorage.com`

That breaks uploads for jurisdictional buckets that require endpoints
like:

`https://<accountId>.eu.r2.cloudflarestorage.com`

## Changes
- Updated `R2StorageProvider` endpoint resolution:
  - use `config.endpoint` when provided
- otherwise fall back to `https://${accountId}.r2.cloudflarestorage.com`
- Kept `forcePathStyle: true` behavior unchanged
- Updated validation to require `accountId` **or** `endpoint`
- Improved storage schema descriptions to mention jurisdiction endpoints
- Added focused unit tests for:
  - default account endpoint behavior
  - custom jurisdiction endpoint behavior

## Backward Compatibility
- Existing R2 configs that only provide `accountId` continue to work
exactly as before.
- New behavior only applies when a custom `config.endpoint` is
explicitly set.

## Tests
- Added: `packages/backend/server/src/base/storage/__tests__/r2.spec.ts`
- Verifies both default and custom endpoint selection paths.

_Disclaimer: parts of this PR were implemented with AI assistance._

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Cloudflare R2 config adds an optional "jurisdiction" (EU) option and
consistent endpoint derivation for S3-compatible providers.

* **Documentation**
* Storage configuration schemas clarified: S3 endpoint is
optional/derived from region; R2 endpoint removed from schema and
jurisdiction documented.

* **Tests**
* Added tests validating R2 endpoint selection for default,
EU-jurisdiction, undefined-jurisdiction, and missing-account scenarios.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: DarkSky <25152247+darkskygit@users.noreply.github.com>
This commit is contained in:
Remi Huigen
2026-05-03 18:22:51 +02:00
committed by GitHub
parent fb6291cb15
commit fa8f1a096c
5 changed files with 184 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
.docker/selfhost/schema.json
View File

@@ -353,7 +353,7 @@
"properties": {
"endpoint": {
"type": "string",
"description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://<account>.r2.cloudflarestorage.com\"."
"description": "The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region."
},
"region": {
"type": "string",
@@ -420,10 +420,6 @@
"type": "object",
"description": "The config for the S3 compatible storage provider.",
"properties": {
"endpoint": {
"type": "string",
"description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://<account>.r2.cloudflarestorage.com\"."
},
"region": {
"type": "string",
"description": "The region for the storage provider. Example: \"us-east-1\" or \"auto\" for R2."
@@ -490,6 +486,13 @@
"description": "The presigned key for the cloudflare r2 storage provider."
}
}
},
"jurisdiction": {
"type": "string",
"enum": [
"eu"
],
"description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets."
}
}
}
@@ -548,7 +551,7 @@
"properties": {
"endpoint": {
"type": "string",
"description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://<account>.r2.cloudflarestorage.com\"."
"description": "The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region."
},
"region": {
"type": "string",
@@ -615,10 +618,6 @@
"type": "object",
"description": "The config for the S3 compatible storage provider.",
"properties": {
"endpoint": {
"type": "string",
"description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://<account>.r2.cloudflarestorage.com\"."
},
"region": {
"type": "string",
"description": "The region for the storage provider. Example: \"us-east-1\" or \"auto\" for R2."
@@ -685,6 +684,13 @@
"description": "The presigned key for the cloudflare r2 storage provider."
}
}
},
"jurisdiction": {
"type": "string",
"enum": [
"eu"
],
"description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets."
}
}
}
@@ -1192,7 +1198,7 @@
"properties": {
"endpoint": {
"type": "string",
"description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://<account>.r2.cloudflarestorage.com\"."
"description": "The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region."
},
"region": {
"type": "string",
@@ -1259,10 +1265,6 @@
"type": "object",
"description": "The config for the S3 compatible storage provider.",
"properties": {
"endpoint": {
"type": "string",
"description": "The S3 compatible endpoint. Example: \"https://s3.us-east-1.amazonaws.com\" or \"https://<account>.r2.cloudflarestorage.com\"."
},
"region": {
"type": "string",
"description": "The region for the storage provider. Example: \"us-east-1\" or \"auto\" for R2."
@@ -1329,6 +1331,13 @@
"description": "The presigned key for the cloudflare r2 storage provider."
}
}
},
"jurisdiction": {
"type": "string",
"enum": [
"eu"
],
"description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets."
}
}
}

85
packages/backend/server/src/base/storage/__tests__/r2.spec.ts Normal file
View File

@@ -0,0 +1,85 @@
import test from 'ava';
import { R2StorageProvider } from '../providers/r2';
function endpointOf(provider: R2StorageProvider) {
return provider.endpointUrl;
}
test('R2 provider should use account endpoint by default', t => {
const provider = new R2StorageProvider(
{
accountId: 'test-account',
region: 'auto',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
},
'test-bucket'
);
t.is(
endpointOf(provider),
'https://test-account.r2.cloudflarestorage.com/test-bucket'
);
});
test('R2 provider should append jurisdiction suffix for EU buckets', t => {
const provider = new R2StorageProvider(
{
accountId: 'test-account',
jurisdiction: 'eu',
region: 'auto',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
},
'test-bucket'
);
t.is(
endpointOf(provider),
'https://test-account.eu.r2.cloudflarestorage.com/test-bucket'
);
});
test('R2 provider should throw when accountId is missing', t => {
t.throws(
() =>
new R2StorageProvider(
{
region: 'auto',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
} as any,
'test-bucket'
)
);
});
test(
'R2 provider should use default endpoint when jurisdiction is explicitly undefined',
t => {
const provider = new R2StorageProvider(
{
accountId: 'test-account',
jurisdiction: undefined,
region: 'auto',
credentials: {
accessKeyId: 'test',
secretAccessKey: 'test',
},
},
'test-bucket'
);
t.is(
endpointOf(provider),
'https://test-account.r2.cloudflarestorage.com/test-bucket'
);
}
);

27
packages/backend/server/src/base/storage/providers/index.ts
View File

@@ -3,7 +3,11 @@ import { Type } from '@nestjs/common';
import { JSONSchema } from '../../config';
import { FsStorageConfig, FsStorageProvider } from './fs';
import { StorageProvider } from './provider';
import { R2StorageConfig, R2StorageProvider } from './r2';
import {
R2_JURISDICTIONS,
R2StorageConfig,
R2StorageProvider,
} from './r2';
import { S3StorageConfig, S3StorageProvider } from './s3';
export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2';
@@ -38,7 +42,7 @@ const S3ConfigSchema: JSONSchema = {
endpoint: {
type: 'string',
description:
'The S3 compatible endpoint. Example: "https://s3.us-east-1.amazonaws.com" or "https://<account>.r2.cloudflarestorage.com".',
'The S3 compatible endpoint (used by aws-s3 provider). Optional; if omitted, endpoint is derived from region.',
},
region: {
type: 'string',
@@ -89,6 +93,17 @@ const S3ConfigSchema: JSONSchema = {
},
};
const S3ConfigPropertiesWithoutEndpoint = Object.fromEntries(
Object.entries(
(
S3ConfigSchema as {
type: 'object';
properties?: Record<string, JSONSchema>;
}
).properties ?? {}
).filter(([key]) => key !== 'endpoint')
) as Record<string, JSONSchema>;
export const StorageJSONSchema: JSONSchema = {
oneOf: [
{
@@ -137,12 +152,18 @@ export const StorageJSONSchema: JSONSchema = {
config: {
...S3ConfigSchema,
properties: {
...S3ConfigSchema.properties,
...S3ConfigPropertiesWithoutEndpoint,
accountId: {
type: 'string' as const,
description:
'The account id for the cloudflare r2 storage provider.',
},
jurisdiction: {
type: 'string' as const,
enum: [...R2_JURISDICTIONS],
description:
'Optional jurisdiction for the cloudflare r2 endpoint. Set to "eu" for EU buckets.',
},
usePresignedURL: {
type: 'object' as const,
description:

16
packages/backend/server/src/base/storage/providers/r2.ts
View File

@@ -15,8 +15,15 @@ import {
SIGNED_URL_EXPIRED,
} from './utils';
export interface R2StorageConfig extends S3StorageConfig {
export const R2_JURISDICTIONS = ['eu'] as const;
type R2Jurisdiction = (typeof R2_JURISDICTIONS)[number];
export interface R2StorageConfig extends Omit<
S3StorageConfig,
'endpoint' | 'forcePathStyle'
> {
accountId: string;
jurisdiction?: R2Jurisdiction;
usePresignedURL?: {
enabled: boolean;
urlPrefix?: string;
@@ -33,11 +40,16 @@ export class R2StorageProvider extends S3StorageProvider {
bucket: string
) {
assert(config.accountId, 'accountId is required for R2 storage provider');
const account = config.jurisdiction
? `${config.accountId}.${config.jurisdiction}`
: config.accountId;
const endpoint = `https://${account}.r2.cloudflarestorage.com`;
super(
{
...config,
forcePathStyle: true,
endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
endpoint,
},
bucket
);

37
packages/backend/server/src/base/storage/providers/s3.ts
View File

@@ -47,10 +47,46 @@ function resolveEndpoint(config: S3StorageConfig) {
return `https://s3.${config.region}.amazonaws.com`;
}
function joinPath(basePath: string, suffix: string) {
const trimmedBase = basePath.endsWith('/') ? basePath.slice(0, -1) : basePath;
const trimmedSuffix = suffix.startsWith('/') ? suffix.slice(1) : suffix;
if (!trimmedBase) {
return `/${trimmedSuffix}`;
}
if (!trimmedSuffix) {
return trimmedBase;
}
return `${trimmedBase}/${trimmedSuffix}`;
}
function composeEndpointUrl(config: S3CompatConfig) {
const url = new URL(config.endpoint);
if (config.forcePathStyle) {
const firstSegment = url.pathname.split('/').find(Boolean);
if (firstSegment !== config.bucket) {
url.pathname = joinPath(url.pathname, config.bucket);
}
return url.toString();
}
const firstSegment = url.pathname.split('/').find(Boolean);
const hostHasBucket = url.hostname.startsWith(`${config.bucket}.`);
const pathHasBucket = firstSegment === config.bucket;
if (!hostHasBucket && !pathHasBucket) {
url.hostname = `${config.bucket}.${url.hostname}`;
}
return url.toString();
}
export class S3StorageProvider implements StorageProvider {
protected logger: Logger;
protected client: S3CompatClient;
private readonly usePresignedURL: boolean;
private readonly endpoint: string;
get endpointUrl() {
return this.endpoint;
}
constructor(
config: S3StorageConfig,
@@ -69,6 +105,7 @@ export class S3StorageProvider implements StorageProvider {
},
};
this.endpoint = composeEndpointUrl(compatConfig);
this.client = createS3CompatClient(compatConfig, credentials);
this.usePresignedURL = usePresignedURL?.enabled ?? false;
this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);