diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index eb1d8ff3b1..6664d01481 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -494,11 +494,11 @@ }, "urlPrefix": { "type": "string", - "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured." }, "signKey": { "type": "string", - "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix." } } } @@ -577,11 +577,11 @@ }, "urlPrefix": { "type": "string", - "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured." }, "signKey": { "type": "string", - "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix." } } }, @@ -741,11 +741,11 @@ }, "urlPrefix": { "type": "string", - "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured." }, "signKey": { "type": "string", - "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix." } } } @@ -824,11 +824,11 @@ }, "urlPrefix": { "type": "string", - "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured." }, "signKey": { "type": "string", - "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix." } } }, @@ -1294,11 +1294,11 @@ }, "urlPrefix": { "type": "string", - "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured." }, "signKey": { "type": "string", - "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix." } } } @@ -1377,11 +1377,11 @@ }, "urlPrefix": { "type": "string", - "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured." }, "signKey": { "type": "string", - "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix." } } }, diff --git a/packages/backend/native/src/runtime/object_storage/config.rs b/packages/backend/native/src/runtime/object_storage/config.rs index 49d74aff3a..77f2673578 100644 --- a/packages/backend/native/src/runtime/object_storage/config.rs +++ b/packages/backend/native/src/runtime/object_storage/config.rs @@ -1,13 +1,20 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use hmac::{Hmac, KeyInit, Mac}; use rusty_s3::{Bucket, Credentials, UrlStyle}; use serde::Deserialize; +use sha2::Sha256; use url::Url; use super::{ client::ObjectStorageClient, error::{ObjectStorageError, ObjectStorageResult}, - types::StorageProviderConfig, + types::{ObjectKey, PresignedObjectRequest, StorageProviderConfig}, }; +type HmacSha256 = Hmac; + #[derive(Clone, Debug)] pub(crate) struct ObjectStorageConfig { pub(crate) provider: String, @@ -24,6 +31,8 @@ pub(crate) struct ObjectStorageConfig { pub(crate) presign_sign_content_type_for_put: Option, pub(crate) use_presigned_url: bool, pub(crate) proxy_upload: bool, + pub(crate) custom_get_url_prefix: Option, + pub(crate) custom_get_sign_key: Option, } #[derive(Debug, Deserialize)] @@ -124,6 +133,8 @@ impl ObjectStorageConfig { presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put), use_presigned_url: config.use_presigned_url.map(|v| v.enabled).unwrap_or(false), proxy_upload: false, + custom_get_url_prefix: None, + custom_get_sign_key: None, })) } @@ -135,17 +146,25 @@ impl ObjectStorageConfig { Some(R2Jurisdiction::Default) | None => config.account_id, }; let credentials = config.credentials.unwrap_or_default(); - let (use_presigned_url, proxy_upload) = config + let (use_presigned_url, proxy_upload, custom_get_url_prefix, custom_get_sign_key) = config .use_presigned_url .map(|value| { + let url_prefix = value.url_prefix.filter(|prefix| !prefix.is_empty()); + let sign_key = value.sign_key.filter(|key| !key.is_empty()); + let custom_get_enabled = value.enabled && url_prefix.is_some() && sign_key.is_some(); + let (custom_get_url_prefix, custom_get_sign_key) = if custom_get_enabled { + (url_prefix, sign_key) + } else { + (None, None) + }; ( value.enabled, - value.enabled - && value.url_prefix.as_ref().is_some_and(|prefix| !prefix.is_empty()) - && value.sign_key.as_ref().is_some_and(|key| !key.is_empty()), + custom_get_enabled, + custom_get_url_prefix, + custom_get_sign_key, ) }) - .unwrap_or((false, false)); + .unwrap_or((false, false, None, None)); Ok(Some(Self { provider: storage.provider, @@ -162,6 +181,57 @@ impl ObjectStorageConfig { presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put), use_presigned_url, proxy_upload, + custom_get_url_prefix, + custom_get_sign_key, + })) + } + + pub(crate) fn custom_presign_get(&self, key: &ObjectKey) -> ObjectStorageResult> { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| ObjectStorageError::Config(format!("system time before unix epoch: {err}")))? + .as_secs(); + self.custom_presign_get_at(key, timestamp) + } + + pub(super) fn custom_presign_get_at( + &self, + key: &ObjectKey, + timestamp: u64, + ) -> ObjectStorageResult> { + let (Some(prefix), Some(sign_key)) = ( + self.custom_get_url_prefix.as_deref(), + self.custom_get_sign_key.as_deref(), + ) else { + return Ok(None); + }; + let mut url = Url::parse(prefix) + .map_err(|err| ObjectStorageError::Config(format!("invalid object storage URL prefix: {err}")))?; + if !matches!(url.scheme(), "http" | "https") || url.query().is_some() || url.fragment().is_some() { + return Err(ObjectStorageError::Config( + "object storage URL prefix must be an HTTP(S) URL without query or fragment".to_string(), + )); + } + url + .path_segments_mut() + .map_err(|_| ObjectStorageError::Config("object storage URL prefix cannot be a base URL".to_string()))? + .pop_if_empty() + .extend(key.as_str().split('/')); + let payload = format!("{}{timestamp}", url.path()); + let mut mac = HmacSha256::new_from_slice(sign_key.as_bytes()) + .map_err(|err| ObjectStorageError::Config(format!("invalid object storage signing key: {err}")))?; + mac.update(payload.as_bytes()); + let signature = STANDARD.encode(mac.finalize().into_bytes()); + url + .query_pairs_mut() + .append_pair("sign", &format!("{timestamp}-{signature}")); + + Ok(Some(PresignedObjectRequest { + url: url.to_string(), + headers: Default::default(), + expires_at_ms: i64::try_from(timestamp.saturating_add(self.presign_expires_in_seconds.unwrap_or(60))) + .unwrap_or(i64::MAX) + .saturating_mul(1000), })) } diff --git a/packages/backend/native/src/runtime/object_storage/service.rs b/packages/backend/native/src/runtime/object_storage/service.rs index f9d6fce6cc..2535cad9db 100644 --- a/packages/backend/native/src/runtime/object_storage/service.rs +++ b/packages/backend/native/src/runtime/object_storage/service.rs @@ -182,12 +182,17 @@ impl ObjectStorageService { ) -> RuntimeResult> { match self.backend_for_scope(locator.scope)? { StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(None), - StorageBackendConfig::S3(config) => config - .build_client()? - .presign_get(&locator.key) - .await - .map(Some) - .map_err(Into::into), + StorageBackendConfig::S3(config) => { + if let Some(request) = config.custom_presign_get(&locator.key)? { + return Ok(Some(request)); + } + config + .build_client()? + .presign_get(&locator.key) + .await + .map(Some) + .map_err(Into::into) + } } } diff --git a/packages/backend/native/src/runtime/object_storage/tests.rs b/packages/backend/native/src/runtime/object_storage/tests.rs index a747c0a1b7..c2ca7b5b93 100644 --- a/packages/backend/native/src/runtime/object_storage/tests.rs +++ b/packages/backend/native/src/runtime/object_storage/tests.rs @@ -261,7 +261,7 @@ fn resolves_r2_proxy_upload_capability_from_config_json_shape() { }, "usePresignedURL": { "enabled": true, - "urlPrefix": "https://cdn.example.com", + "urlPrefix": "https://cdn.example.com/storage/", "signKey": "secret" } }), @@ -271,6 +271,50 @@ fn resolves_r2_proxy_upload_capability_from_config_json_shape() { assert!(config.use_presigned_url); assert!(config.proxy_upload); + let request = config + .custom_presign_get_at(&ObjectKey::new("workspace/blob.m4a").unwrap(), 1_700_000_000) + .unwrap() + .unwrap(); + let url = url::Url::parse(&request.url).unwrap(); + assert_eq!(url.origin().ascii_serialization(), "https://cdn.example.com"); + assert_eq!(url.path(), "/storage/workspace/blob.m4a"); + assert_eq!( + url.query_pairs().find(|(key, _)| key == "sign").unwrap().1, + "1700000000-01IngHvoE2trslxVyYUzfWkhgdlYdpcRXcpSYqZ9gkc=" + ); + + for use_presigned_url in [ + serde_json::json!({ + "enabled": true, + "urlPrefix": "https://cdn.example.com" + }), + serde_json::json!({ + "enabled": true, + "urlPrefix": "https://cdn.example.com", + "signKey": "" + }), + ] { + let storage = StorageProviderConfig { + provider: "cloudflare-r2".to_string(), + bucket: "workspace-blobs".to_string(), + config: serde_json::json!({ + "accountId": "account", + "credentials": { + "accessKeyId": "key", + "secretAccessKey": "secret" + }, + "usePresignedURL": use_presigned_url + }), + }; + let config = ObjectStorageConfig::from_r2_config(storage).unwrap().unwrap(); + assert!(!config.proxy_upload); + assert!( + config + .custom_presign_get_at(&ObjectKey::new("workspace/blob.m4a").unwrap(), 1_700_000_000) + .unwrap() + .is_none() + ); + } } #[test] diff --git a/packages/backend/native/src/runtime/storage_runtime/capabilities.rs b/packages/backend/native/src/runtime/storage_runtime/capabilities.rs index cca2afd881..49ff6035b2 100644 --- a/packages/backend/native/src/runtime/storage_runtime/capabilities.rs +++ b/packages/backend/native/src/runtime/storage_runtime/capabilities.rs @@ -96,6 +96,8 @@ mod tests { presign_sign_content_type_for_put: Some(true), use_presigned_url: true, proxy_upload: false, + custom_get_url_prefix: None, + custom_get_sign_key: None, })); assert!(capabilities.presign_put); @@ -121,6 +123,8 @@ mod tests { presign_sign_content_type_for_put: Some(true), use_presigned_url: true, proxy_upload: true, + custom_get_url_prefix: None, + custom_get_sign_key: None, })); assert!(capabilities.proxy_upload); diff --git a/packages/backend/server/src/base/storage/providers/index.ts b/packages/backend/server/src/base/storage/providers/index.ts index 7d07dc1943..410176d259 100644 --- a/packages/backend/server/src/base/storage/providers/index.ts +++ b/packages/backend/server/src/base/storage/providers/index.ts @@ -127,12 +127,12 @@ const S3ConfigSchema: JSONSchema = { urlPrefix: { type: 'string', description: - 'Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured.', + 'Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured.', }, signKey: { type: 'string', description: - 'Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin.', + 'Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix.', }, }, },