Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-18 18:46:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): impl storage runtime (#15181)

Browse Source 
#### PR Dependency Tree


* **PR #15181** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added an additional storage backend option: asset-pack based storage
(provider for avatar, blob, and copilot).
* Introduced a dedicated storage runtime with provider capability
reporting and expanded object operations (put/head/get/list/delete),
including presigned and multipart flows where supported.
* Cloudflare R2 `jurisdiction` now uses an explicit default when
omitted.
* **Bug Fixes**
  * Broadened avatar access to allow both fs and asset-pack providers.
* Improved workspace blob upload completion validation and handling when
stored objects are missing or mismatched.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-07-01 22:24:10 +08:00
committed by GitHub
parent da7d438377
commit 8ebdb7452f
102 changed files with 6487 additions and 4508 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/native/src/backend_runtime/blob_complete.rs
-296
View File
@@ -1,296 +0,0 @@
use std::{
fs,
path::{Path, PathBuf},
};
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use napi::Result;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use super::{BackendRuntime, error::napi_error, types::RuntimeBlobCompleteResult};
const MAX_BLOB_SIZE: i64 = i32::MAX as i64;
fn object_missing_error(err: &napi::Error) -> bool {
let message = err.to_string();
message.contains("NoSuchKey") || message.contains("NotFound") || message.contains("not found")
}
fn blob_complete_failure(reason: &str) -> RuntimeBlobCompleteResult {
RuntimeBlobCompleteResult {
ok: false,
reason: Some(reason.to_string()),
content_type: None,
content_length: None,
last_modified_ms: None,
}
}
fn blob_complete_success(
content_type: String,
content_length: i64,
last_modified_ms: i64,
) -> RuntimeBlobCompleteResult {
RuntimeBlobCompleteResult {
ok: true,
reason: None,
content_type: Some(content_type),
content_length: Some(content_length),
last_modified_ms: Some(last_modified_ms),
}
}
fn normalize_base64_url_key(key: &str) -> &str {
key.trim_end_matches('=')
}
fn sha256_base64_url(body: &[u8]) -> String {
URL_SAFE_NO_PAD.encode(Sha256::digest(body))
}
fn sha256_base64_url_matches(body: &[u8], key: &str) -> bool {
sha256_base64_url(body) == normalize_base64_url_key(key)
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct FsBlobMetadata {
content_type: String,
content_length: i64,
last_modified: i64,
}
fn normalize_storage_key(key: &str) -> Result<Vec<String>> {
let normalized = key.replace('\\', "/");
let segments = normalized.split('/').map(ToString::to_string).collect::<Vec<_>>();
if normalized.is_empty()
|| normalized.starts_with('/')
|| segments
.iter()
.any(|segment| segment.is_empty() || segment == "." || segment == "..")
{
return Err(napi_error(format!("Invalid storage key: {key}")));
}
Ok(segments)
}
fn fs_bucket_path(root: &str, bucket: &str) -> PathBuf {
if let Some(stripped) = root.strip_prefix("~/")
&& let Ok(Some(home)) = homedir::my_home()
{
return home.join(stripped).join(bucket);
}
Path::new(root).join(bucket)
}
fn fs_object_path(root: &str, bucket: &str, key: &str) -> Result<PathBuf> {
let mut path = fs_bucket_path(root, bucket);
for segment in normalize_storage_key(key)? {
path.push(segment);
}
Ok(path)
}
fn read_fs_metadata(path: &Path) -> Result<Option<FsBlobMetadata>> {
let metadata_path = PathBuf::from(format!("{}.metadata.json", path.display()));
let raw = match fs::read_to_string(metadata_path) {
Ok(raw) => raw,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(err) => {
return Err(napi_error(format!("BlobComplete read fs metadata failed: {err}")));
}
};
serde_json::from_str(&raw).map(Some).map_err(|err| {
napi_error(format!(
"BlobComplete parse fs metadata failed for {}: {err}",
path.display()
))
})
}
async fn upsert_completed_blob(
runtime: &BackendRuntime,
workspace_id: &str,
key: &str,
mime: &str,
size: i64,
) -> Result<()> {
if !(0..=MAX_BLOB_SIZE).contains(&size) {
return Err(napi_error("BlobComplete size exceeds limit"));
}
let size = i32::try_from(size).map_err(|_| napi_error("BlobComplete size exceeds limit"))?;
sqlx::query(
r#"
INSERT INTO blobs (workspace_id, key, mime, size, status, upload_id)
VALUES ($1, $2, $3, $4, 'completed', NULL)
ON CONFLICT (workspace_id, key)
DO UPDATE SET
mime = EXCLUDED.mime,
size = EXCLUDED.size,
status = EXCLUDED.status,
upload_id = NULL
"#,
)
.bind(workspace_id)
.bind(key)
.bind(mime)
.bind(size)
.execute(&runtime.pool().await?)
.await
.map_err(|err| napi_error(format!("BlobComplete upsert metadata failed: {err}")))?;
Ok(())
}
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub async fn complete_blob_upload(
&self,
workspace_id: String,
key: String,
expected_size: i64,
expected_mime: String,
) -> Result<RuntimeBlobCompleteResult> {
if !(0..=MAX_BLOB_SIZE).contains(&expected_size) {
return Ok(blob_complete_failure("size_too_large"));
}
let object_key = format!("{workspace_id}/{key}");
let object = match self.object_storage_get(object_key.clone()).await {
Ok(Some(object)) => object,
Ok(None) => return Ok(blob_complete_failure("not_found")),
Err(err) if object_missing_error(&err) => return Ok(blob_complete_failure("not_found")),
Err(err) => return Err(err),
};
if !(0..=MAX_BLOB_SIZE).contains(&object.metadata.content_length) {
match self.object_storage_delete(object_key).await {
Ok(()) => {}
Err(err) if object_missing_error(&err) => {}
Err(err) => return Err(err),
}
return Ok(blob_complete_failure("size_too_large"));
}
if object.metadata.content_length != expected_size {
return Ok(blob_complete_failure("size_mismatch"));
}
if !expected_mime.is_empty() && object.metadata.content_type != expected_mime {
return Ok(blob_complete_failure("mime_mismatch"));
}
if !sha256_base64_url_matches(&object.body, &key) {
match self.object_storage_delete(object_key).await {
Ok(()) => {}
Err(err) if object_missing_error(&err) => {}
Err(err) => return Err(err),
}
return Ok(blob_complete_failure("checksum_mismatch"));
}
upsert_completed_blob(
self,
&workspace_id,
&key,
&object.metadata.content_type,
object.metadata.content_length,
)
.await?;
Ok(blob_complete_success(
object.metadata.content_type,
object.metadata.content_length,
object.metadata.last_modified_ms,
))
}
#[napi]
pub async fn complete_fs_blob_upload(
&self,
root: String,
bucket: String,
workspace_id: String,
key: String,
expected_size: i64,
expected_mime: String,
) -> Result<RuntimeBlobCompleteResult> {
if !(0..=MAX_BLOB_SIZE).contains(&expected_size) {
return Ok(blob_complete_failure("size_too_large"));
}
let storage_key = format!("{workspace_id}/{key}");
let path = fs_object_path(&root, &bucket, &storage_key)?;
let metadata = match read_fs_metadata(&path)? {
Some(metadata) => metadata,
None => return Ok(blob_complete_failure("not_found")),
};
if !(0..=MAX_BLOB_SIZE).contains(&metadata.content_length) {
let _ = fs::remove_file(&path);
let _ = fs::remove_file(PathBuf::from(format!("{}.metadata.json", path.display())));
return Ok(blob_complete_failure("size_too_large"));
}
if metadata.content_length != expected_size {
return Ok(blob_complete_failure("size_mismatch"));
}
if !expected_mime.is_empty() && metadata.content_type != expected_mime {
return Ok(blob_complete_failure("mime_mismatch"));
}
let body = match fs::read(&path) {
Ok(body) => body,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(blob_complete_failure("not_found")),
Err(err) => return Err(napi_error(format!("BlobComplete read fs object failed: {err}"))),
};
if !sha256_base64_url_matches(&body, &key) {
let _ = fs::remove_file(&path);
let _ = fs::remove_file(PathBuf::from(format!("{}.metadata.json", path.display())));
return Ok(blob_complete_failure("checksum_mismatch"));
}
upsert_completed_blob(
self,
&workspace_id,
&key,
&metadata.content_type,
metadata.content_length,
)
.await?;
Ok(blob_complete_success(
metadata.content_type,
metadata.content_length,
metadata.last_modified,
))
}
}
#[cfg(test)]
mod tests {
use super::{sha256_base64_url, sha256_base64_url_matches};
#[test]
fn sha256_base64_url_omits_padding() {
assert_eq!(
sha256_base64_url(b"hello"),
"LPJNul-wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ"
);
}
#[test]
fn sha256_base64_url_matches_legacy_padding() {
assert!(sha256_base64_url_matches(
b"hello",
"LPJNul-wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="
));
}
}
packages/backend/native/src/backend_runtime/error.rs
-5
View File
@@ -1,5 +0,0 @@
use napi::{Error, Status};
pub(super) fn napi_error(message: impl Into<String>) -> Error {
Error::new(Status::GenericFailure, message.into())
}
packages/backend/native/src/backend_runtime/object_storage/client.rs
-389
View File
@@ -1,389 +0,0 @@
use std::{
collections::HashMap,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use aws_sdk_s3::{
Client as S3Client, presigning::PresigningConfig, primitives::ByteStream, types::CompletedMultipartUpload,
};
use napi::Result;
use super::types::{
MultipartUploadInitResult, MultipartUploadPart, ObjectGetResult, ObjectListEntry, ObjectListPage, ObjectMetadata,
ObjectPutMetadata, PresignedObjectRequest, completed_multipart_parts, trim_etag,
};
use crate::backend_runtime::error::napi_error;
#[derive(Clone)]
pub(super) struct ObjectStorageClient {
client: S3Client,
bucket: String,
presign_expires_in_seconds: u64,
presign_sign_content_type_for_put: bool,
}
impl ObjectStorageClient {
pub(super) fn new(
config: aws_sdk_s3::Config,
bucket: String,
presign_expires_in_seconds: u64,
presign_sign_content_type_for_put: bool,
) -> Self {
Self {
client: S3Client::from_conf(config),
bucket,
presign_expires_in_seconds,
presign_sign_content_type_for_put,
}
}
pub(super) fn non_destructive_health(&self) -> bool {
let _ = &self.client;
!self.bucket.is_empty()
}
pub(super) async fn put(&self, key: &str, body: Vec<u8>, metadata: ObjectPutMetadata) -> Result<()> {
let content_length = metadata.content_length.unwrap_or(body.len() as i64);
let content_type = metadata
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string());
let mut request = self
.client
.put_object()
.bucket(&self.bucket)
.key(key)
.body(ByteStream::from(body))
.content_type(content_type)
.content_length(content_length);
if let Some(checksum) = metadata.checksum_crc32 {
request = request.checksum_crc32(checksum);
}
request
.send()
.await
.map_err(|err| napi_error(format!("ObjectStorage put failed for {key}: {err:?}")))?;
Ok(())
}
pub(super) async fn presign_put(&self, key: &str, metadata: ObjectPutMetadata) -> Result<PresignedObjectRequest> {
let content_type = metadata
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string());
let expires_at_ms = expires_at_ms(self.presign_expires_in_seconds)?;
let config = PresigningConfig::expires_in(Duration::from_secs(self.presign_expires_in_seconds))
.map_err(|err| napi_error(format!("ObjectStorage presign config failed: {err}")))?;
let mut request = self.client.put_object().bucket(&self.bucket).key(key);
if self.presign_sign_content_type_for_put {
request = request.content_type(content_type.clone());
}
if let Some(content_length) = metadata.content_length {
request = request.content_length(content_length);
}
let presigned = request
.presigned(config)
.await
.map_err(|err| napi_error(format!("ObjectStorage presign put failed for {key}: {err}")))?;
let mut headers = presigned_headers(&presigned);
headers.insert("Content-Type".to_string(), content_type);
Ok(PresignedObjectRequest {
url: presigned.uri().to_string(),
headers,
expires_at_ms,
})
}
pub(super) async fn create_multipart_upload(
&self,
key: &str,
metadata: ObjectPutMetadata,
) -> Result<Option<MultipartUploadInitResult>> {
let content_type = metadata
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string());
let result = self
.client
.create_multipart_upload()
.bucket(&self.bucket)
.key(key)
.content_type(content_type)
.send()
.await
.map_err(|err| {
napi_error(format!(
"ObjectStorage create multipart upload failed for {key}: {err:?}"
))
})?;
let expires_at_ms = expires_at_ms(self.presign_expires_in_seconds)?;
Ok(result.upload_id.map(|upload_id| MultipartUploadInitResult {
upload_id,
expires_at_ms,
}))
}
pub(super) async fn presign_upload_part(
&self,
key: &str,
upload_id: &str,
part_number: i32,
) -> Result<PresignedObjectRequest> {
let expires_at_ms = expires_at_ms(self.presign_expires_in_seconds)?;
let config = PresigningConfig::expires_in(Duration::from_secs(self.presign_expires_in_seconds))
.map_err(|err| napi_error(format!("ObjectStorage presign config failed: {err}")))?;
let presigned = self
.client
.upload_part()
.bucket(&self.bucket)
.key(key)
.upload_id(upload_id)
.part_number(part_number)
.presigned(config)
.await
.map_err(|err| napi_error(format!("ObjectStorage presign upload part failed for {key}: {err}")))?;
Ok(PresignedObjectRequest {
url: presigned.uri().to_string(),
headers: presigned_headers(&presigned),
expires_at_ms,
})
}
pub(super) async fn list_multipart_upload_parts(
&self,
key: &str,
upload_id: &str,
) -> Result<Vec<MultipartUploadPart>> {
let result = self
.client
.list_parts()
.bucket(&self.bucket)
.key(key)
.upload_id(upload_id)
.send()
.await
.map_err(|err| {
napi_error(format!(
"ObjectStorage list multipart upload parts failed for {key}: {err}"
))
})?;
Ok(
result
.parts()
.iter()
.filter_map(|part| {
Some(MultipartUploadPart {
part_number: part.part_number?,
etag: trim_etag(part.e_tag.as_deref().unwrap_or_default()),
})
})
.collect(),
)
}
pub(super) async fn complete_multipart_upload(
&self,
key: &str,
upload_id: &str,
parts: Vec<MultipartUploadPart>,
) -> Result<()> {
let ordered_parts = completed_multipart_parts(parts);
self
.client
.complete_multipart_upload()
.bucket(&self.bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.set_parts(Some(ordered_parts))
.build(),
)
.send()
.await
.map_err(|err| {
napi_error(format!(
"ObjectStorage complete multipart upload failed for {key}: {err}"
))
})?;
Ok(())
}
pub(super) async fn abort_multipart_upload(&self, key: &str, upload_id: &str) -> Result<()> {
self
.client
.abort_multipart_upload()
.bucket(&self.bucket)
.key(key)
.upload_id(upload_id)
.send()
.await
.map_err(|err| {
napi_error(format!(
"ObjectStorage abort multipart upload failed for {key}: {err:?}"
))
})?;
Ok(())
}
pub(super) async fn head(&self, key: &str) -> Result<Option<ObjectMetadata>> {
let result = match self.client.head_object().bucket(&self.bucket).key(key).send().await {
Ok(result) => result,
Err(err) if is_not_found_error(&err) => return Ok(None),
Err(err) => return Err(napi_error(format!("ObjectStorage head failed for {key}: {err:?}"))),
};
Ok(Some(ObjectMetadata {
content_type: result
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string()),
content_length: result.content_length.unwrap_or(0),
last_modified_ms: optional_datetime_ms(result.last_modified),
checksum_crc32: result.checksum_crc32,
}))
}
pub(super) async fn get(&self, key: &str) -> Result<Option<ObjectGetResult>> {
let result = match self.client.get_object().bucket(&self.bucket).key(key).send().await {
Ok(result) => result,
Err(err) if is_not_found_error(&err) => return Ok(None),
Err(err) => return Err(napi_error(format!("ObjectStorage get failed for {key}: {err:?}"))),
};
let metadata = ObjectMetadata {
content_type: result
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string()),
content_length: result.content_length.unwrap_or(0),
last_modified_ms: optional_datetime_ms(result.last_modified),
checksum_crc32: result.checksum_crc32,
};
let body = result
.body
.collect()
.await
.map_err(|err| napi_error(format!("ObjectStorage read body failed for {key}: {err}")))?
.into_bytes()
.to_vec();
Ok(Some(ObjectGetResult { body, metadata }))
}
pub(super) async fn list(&self, prefix: Option<String>) -> Result<Vec<ObjectListEntry>> {
let mut entries = Vec::new();
let mut token = None;
loop {
let mut request = self.client.list_objects_v2().bucket(&self.bucket);
if let Some(prefix) = &prefix {
request = request.prefix(prefix);
}
if let Some(next_token) = token {
request = request.continuation_token(next_token);
}
let result = request
.send()
.await
.map_err(|err| napi_error(format!("ObjectStorage list failed: {err:?}")))?;
entries.extend(result.contents().iter().filter_map(|object| {
Some(ObjectListEntry {
key: object.key.as_ref()?.clone(),
content_length: object.size.unwrap_or(0),
last_modified_ms: optional_datetime_ms(object.last_modified),
})
}));
if result.is_truncated.unwrap_or(false) {
token = result.next_continuation_token;
} else {
break;
}
}
Ok(entries)
}
pub(super) async fn list_page(
&self,
prefix: Option<String>,
continuation_token: Option<String>,
start_after: Option<String>,
max_keys: i32,
) -> Result<ObjectListPage> {
let mut request = self.client.list_objects_v2().bucket(&self.bucket).max_keys(max_keys);
if let Some(prefix) = prefix {
request = request.prefix(prefix);
}
if let Some(continuation_token) = continuation_token {
request = request.continuation_token(continuation_token);
} else if let Some(start_after) = start_after {
request = request.start_after(start_after);
}
let result = request
.send()
.await
.map_err(|err| napi_error(format!("ObjectStorage list page failed: {err:?}")))?;
Ok(ObjectListPage {
entries: result
.contents()
.iter()
.filter_map(|object| {
Some(ObjectListEntry {
key: object.key.as_ref()?.clone(),
content_length: object.size.unwrap_or(0),
last_modified_ms: optional_datetime_ms(object.last_modified),
})
})
.collect(),
next_continuation_token: result.next_continuation_token,
})
}
pub(super) async fn delete(&self, key: &str) -> Result<()> {
self
.client
.delete_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(|err| napi_error(format!("ObjectStorage delete failed for {key}: {err:?}")))?;
Ok(())
}
}
fn is_not_found_error(error: &impl std::fmt::Debug) -> bool {
let message = format!("{error:?}");
message.contains("NoSuchKey") || (message.contains("NotFound") && !message.contains("NoSuchBucket"))
}
fn expires_at_ms(expires_in_seconds: u64) -> Result<i64> {
let expires_at = SystemTime::now()
.checked_add(Duration::from_secs(expires_in_seconds))
.ok_or_else(|| napi_error("ObjectStorage presign expiration overflow"))?;
system_time_ms(expires_at)
}
fn system_time_ms(time: SystemTime) -> Result<i64> {
let duration = time
.duration_since(UNIX_EPOCH)
.map_err(|err| napi_error(format!("system time before unix epoch: {err}")))?;
Ok(duration.as_millis() as i64)
}
fn optional_datetime_ms(time: Option<aws_sdk_s3::primitives::DateTime>) -> i64 {
time.and_then(|value| value.to_millis().ok()).unwrap_or(0)
}
fn presigned_headers(request: &aws_sdk_s3::presigning::PresignedRequest) -> HashMap<String, String> {
request
.headers()
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect()
}
packages/backend/native/src/backend_runtime/object_storage/mod.rs
-197
View File
@@ -1,197 +0,0 @@
mod client;
mod config;
#[cfg(test)]
mod tests;
mod types;
use client::ObjectStorageClient;
pub(super) use config::ObjectStorageConfig;
use napi::{Result, bindgen_prelude::Buffer};
use types::ObjectListPage;
pub(super) use types::StorageProviderConfig;
use super::{
BackendRuntime,
types::{
RuntimeMultipartUploadInit, RuntimeMultipartUploadPart, RuntimeObjectGetResult, RuntimeObjectListEntry,
RuntimeObjectMetadata, RuntimeObjectStorageHealth, RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest,
},
};
#[napi_derive::napi]
impl BackendRuntime {
fn object_storage_client(&self) -> Result<ObjectStorageClient> {
let storage = self
.config()?
.storage
.ok_or_else(|| super::error::napi_error("ObjectStorageClient is not configured"))?;
storage.build_client()
}
pub(super) async fn object_storage_delete_object(&self, key: &str) -> Result<()> {
self.object_storage_client()?.delete(key).await
}
pub(super) async fn object_storage_list_page(
&self,
prefix: Option<String>,
continuation_token: Option<String>,
start_after: Option<String>,
max_keys: i32,
) -> Result<ObjectListPage> {
self
.object_storage_client()?
.list_page(prefix, continuation_token, start_after, max_keys)
.await
}
pub(super) async fn object_storage_abort_upload(&self, key: &str, upload_id: &str) -> Result<()> {
self
.object_storage_client()?
.abort_multipart_upload(key, upload_id)
.await
}
#[napi]
pub fn object_storage_health(&self) -> RuntimeObjectStorageHealth {
match self.config().ok().and_then(|config| config.storage) {
Some(storage) => storage.health(),
None => RuntimeObjectStorageHealth {
configured: false,
provider: None,
bucket: None,
endpoint: None,
region: None,
has_credentials: false,
force_path_style: false,
request_timeout_ms: None,
min_part_size: None,
presign_expires_in_seconds: None,
presign_sign_content_type_for_put: None,
use_presigned_url: false,
client_buildable: false,
},
}
}
#[napi]
pub async fn object_storage_put(
&self,
key: String,
body: Buffer,
metadata: Option<RuntimeObjectStoragePutOptions>,
) -> Result<()> {
self
.object_storage_client()?
.put(&key, body.to_vec(), metadata.map(Into::into).unwrap_or_default())
.await
}
#[napi]
pub async fn object_storage_presign_put(
&self,
key: String,
metadata: Option<RuntimeObjectStoragePutOptions>,
) -> Result<RuntimePresignedObjectRequest> {
self
.object_storage_client()?
.presign_put(&key, metadata.map(Into::into).unwrap_or_default())
.await?
.try_into()
}
#[napi]
pub async fn object_storage_create_multipart_upload(
&self,
key: String,
metadata: Option<RuntimeObjectStoragePutOptions>,
) -> Result<Option<RuntimeMultipartUploadInit>> {
Ok(
self
.object_storage_client()?
.create_multipart_upload(&key, metadata.map(Into::into).unwrap_or_default())
.await?
.map(Into::into),
)
}
#[napi]
pub async fn object_storage_presign_upload_part(
&self,
key: String,
upload_id: String,
part_number: i32,
) -> Result<RuntimePresignedObjectRequest> {
self
.object_storage_client()?
.presign_upload_part(&key, &upload_id, part_number)
.await?
.try_into()
}
#[napi]
pub async fn object_storage_list_multipart_upload_parts(
&self,
key: String,
upload_id: String,
) -> Result<Vec<RuntimeMultipartUploadPart>> {
Ok(
self
.object_storage_client()?
.list_multipart_upload_parts(&key, &upload_id)
.await?
.into_iter()
.map(Into::into)
.collect(),
)
}
#[napi]
pub async fn object_storage_complete_multipart_upload(
&self,
key: String,
upload_id: String,
parts: Vec<RuntimeMultipartUploadPart>,
) -> Result<()> {
self
.object_storage_client()?
.complete_multipart_upload(&key, &upload_id, parts.into_iter().map(Into::into).collect())
.await
}
#[napi]
pub async fn object_storage_abort_multipart_upload(&self, key: String, upload_id: String) -> Result<()> {
self
.object_storage_client()?
.abort_multipart_upload(&key, &upload_id)
.await
}
#[napi]
pub async fn object_storage_head(&self, key: String) -> Result<Option<RuntimeObjectMetadata>> {
Ok(self.object_storage_client()?.head(&key).await?.map(Into::into))
}
#[napi]
pub async fn object_storage_get(&self, key: String) -> Result<Option<RuntimeObjectGetResult>> {
Ok(self.object_storage_client()?.get(&key).await?.map(Into::into))
}
#[napi]
pub async fn object_storage_list(&self, prefix: Option<String>) -> Result<Vec<RuntimeObjectListEntry>> {
Ok(
self
.object_storage_client()?
.list(prefix)
.await?
.into_iter()
.map(Into::into)
.collect(),
)
}
#[napi]
pub async fn object_storage_delete(&self, key: String) -> Result<()> {
self.object_storage_client()?.delete(&key).await
}
}
packages/backend/native/src/backend_runtime/object_storage/tests.rs
-129
View File
@@ -1,129 +0,0 @@
use super::{
config::ObjectStorageConfig,
types::{MultipartUploadPart, ObjectPutMetadata, StorageProviderConfig, completed_multipart_parts, trim_etag},
};
#[test]
fn resolves_r2_config_from_config_json_shape() {
let storage = StorageProviderConfig {
provider: "cloudflare-r2".to_string(),
bucket: "workspace-blobs".to_string(),
config: serde_json::json!({
"accountId": "account",
"jurisdiction": "eu",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"usePresignedURL": {
"enabled": true
}
}),
};
let config = ObjectStorageConfig::from_r2_config(storage).unwrap().unwrap();
assert_eq!(config.provider, "cloudflare-r2");
assert_eq!(config.bucket, "workspace-blobs");
assert_eq!(
config.endpoint.as_deref(),
Some("https://account.eu.r2.cloudflarestorage.com")
);
assert_eq!(config.region.as_deref(), Some("auto"));
assert!(config.force_path_style);
assert!(config.use_presigned_url);
assert_eq!(config.access_key_id.as_deref(), Some("key"));
}
#[test]
fn resolves_s3_config_from_config_json_shape() {
let storage = StorageProviderConfig {
provider: "aws-s3".to_string(),
bucket: "workspace-blobs".to_string(),
config: serde_json::json!({
"region": "us-west-2",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret",
"sessionToken": "session"
},
"forcePathStyle": true,
"requestTimeoutMs": 1000,
"minPartSize": 1024,
"presign": {
"expiresInSeconds": 60,
"signContentTypeForPut": false
}
}),
};
let config = ObjectStorageConfig::from_s3_config(storage).unwrap().unwrap();
assert_eq!(config.provider, "aws-s3");
assert_eq!(config.endpoint.as_deref(), Some("https://s3.us-west-2.amazonaws.com"));
assert_eq!(config.session_token.as_deref(), Some("session"));
assert!(config.force_path_style);
assert_eq!(config.request_timeout_ms, Some(1000));
assert_eq!(config.min_part_size, Some(1024));
assert_eq!(config.presign_expires_in_seconds, Some(60));
assert_eq!(config.presign_sign_content_type_for_put, Some(false));
}
#[tokio::test]
async fn object_storage_presign_put_returns_sigv4_url_and_headers() {
let storage = StorageProviderConfig {
provider: "aws-s3".to_string(),
bucket: "test-bucket".to_string(),
config: serde_json::json!({
"region": "us-east-1",
"endpoint": "https://s3.us-east-1.amazonaws.com",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"presign": {
"expiresInSeconds": 60
}
}),
};
let config = ObjectStorageConfig::from_s3_config(storage).unwrap().unwrap();
let Ok(Ok(client)) = std::panic::catch_unwind(|| config.build_client()) else {
eprintln!("skipping object storage presign test: S3 client cannot be built in this environment");
return;
};
let result = client
.presign_put(
"key",
ObjectPutMetadata {
content_type: Some("text/plain".to_string()),
..Default::default()
},
)
.await
.unwrap();
assert!(result.url.contains("X-Amz-Algorithm=AWS4-HMAC-SHA256"));
assert!(result.url.contains("X-Amz-SignedHeaders="));
assert_eq!(
result.headers.get("Content-Type").map(String::as_str),
Some("text/plain")
);
assert!(result.expires_at_ms > 0);
}
#[test]
fn object_storage_orders_completed_multipart_parts_and_trims_etags() {
let parts = completed_multipart_parts(vec![
MultipartUploadPart {
part_number: 2,
etag: trim_etag("\"b\""),
},
MultipartUploadPart {
part_number: 1,
etag: trim_etag("a"),
},
]);
assert_eq!(parts[0].part_number, Some(1));
assert_eq!(parts[0].e_tag.as_deref(), Some("a"));
assert_eq!(parts[1].part_number, Some(2));
assert_eq!(parts[1].e_tag.as_deref(), Some("b"));
}
packages/backend/native/src/backend_runtime/object_storage/types.rs
-171
View File
@@ -1,171 +0,0 @@
use std::collections::HashMap;
use aws_sdk_s3::types::CompletedPart;
use napi::Result;
use serde::Deserialize;
use crate::backend_runtime::{
error::napi_error,
types::{
RuntimeMultipartUploadInit, RuntimeMultipartUploadPart, RuntimeObjectGetResult, RuntimeObjectListEntry,
RuntimeObjectMetadata, RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest,
},
};
#[derive(Clone, Debug, Default)]
pub(super) struct ObjectPutMetadata {
pub(super) content_type: Option<String>,
pub(super) content_length: Option<i64>,
pub(super) checksum_crc32: Option<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub(super) struct ObjectMetadata {
pub(super) content_type: String,
pub(super) content_length: i64,
pub(super) last_modified_ms: i64,
pub(super) checksum_crc32: Option<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub(in crate::backend_runtime) struct ObjectListEntry {
pub(in crate::backend_runtime) key: String,
pub(in crate::backend_runtime) content_length: i64,
pub(in crate::backend_runtime) last_modified_ms: i64,
}
#[derive(Clone, Debug, PartialEq)]
pub(in crate::backend_runtime) struct ObjectListPage {
pub(in crate::backend_runtime) entries: Vec<ObjectListEntry>,
pub(in crate::backend_runtime) next_continuation_token: Option<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub(super) struct ObjectGetResult {
pub(super) body: Vec<u8>,
pub(super) metadata: ObjectMetadata,
}
#[derive(Clone, Debug, PartialEq)]
pub(super) struct PresignedObjectRequest {
pub(super) url: String,
pub(super) headers: HashMap<String, String>,
pub(super) expires_at_ms: i64,
}
#[derive(Clone, Debug, PartialEq)]
pub(super) struct MultipartUploadInitResult {
pub(super) upload_id: String,
pub(super) expires_at_ms: i64,
}
#[derive(Clone, Debug, PartialEq)]
pub(super) struct MultipartUploadPart {
pub(super) part_number: i32,
pub(super) etag: String,
}
#[derive(Clone, Debug, Deserialize)]
pub(in crate::backend_runtime) struct StorageProviderConfig {
pub(super) provider: String,
pub(super) bucket: String,
#[serde(default)]
pub(super) config: serde_json::Value,
}
pub(super) fn trim_etag(etag: &str) -> String {
etag.trim_matches('"').to_string()
}
pub(super) fn completed_multipart_parts(mut parts: Vec<MultipartUploadPart>) -> Vec<CompletedPart> {
parts.sort_by_key(|part| part.part_number);
parts
.into_iter()
.map(|part| {
CompletedPart::builder()
.part_number(part.part_number)
.e_tag(part.etag)
.build()
})
.collect()
}
impl From<RuntimeObjectStoragePutOptions> for ObjectPutMetadata {
fn from(options: RuntimeObjectStoragePutOptions) -> Self {
Self {
content_type: options.content_type,
content_length: options.content_length,
checksum_crc32: options.checksum_crc32,
}
}
}
impl From<ObjectMetadata> for RuntimeObjectMetadata {
fn from(metadata: ObjectMetadata) -> Self {
Self {
content_type: metadata.content_type,
content_length: metadata.content_length,
last_modified_ms: metadata.last_modified_ms,
checksum_crc32: metadata.checksum_crc32,
}
}
}
impl From<ObjectListEntry> for RuntimeObjectListEntry {
fn from(entry: ObjectListEntry) -> Self {
Self {
key: entry.key,
content_length: entry.content_length,
last_modified_ms: entry.last_modified_ms,
}
}
}
impl TryFrom<PresignedObjectRequest> for RuntimePresignedObjectRequest {
type Error = napi::Error;
fn try_from(request: PresignedObjectRequest) -> Result<Self> {
Ok(Self {
url: request.url,
headers_json: serde_json::to_string(&request.headers)
.map_err(|err| napi_error(format!("ObjectStorage headers serialization failed: {err}")))?,
expires_at_ms: request.expires_at_ms,
})
}
}
impl From<ObjectGetResult> for RuntimeObjectGetResult {
fn from(result: ObjectGetResult) -> Self {
Self {
body: result.body.into(),
metadata: result.metadata.into(),
}
}
}
impl From<MultipartUploadInitResult> for RuntimeMultipartUploadInit {
fn from(init: MultipartUploadInitResult) -> Self {
Self {
upload_id: init.upload_id,
expires_at_ms: init.expires_at_ms,
}
}
}
impl From<RuntimeMultipartUploadPart> for MultipartUploadPart {
fn from(part: RuntimeMultipartUploadPart) -> Self {
Self {
part_number: part.part_number,
etag: part.etag,
}
}
}
impl From<MultipartUploadPart> for RuntimeMultipartUploadPart {
fn from(part: MultipartUploadPart) -> Self {
Self {
part_number: part.part_number,
etag: part.etag,
}
}
}
packages/backend/native/src/lib.rs
+1 -1
View File
@@ -2,7 +2,6 @@
mod utils;
pub mod backend_runtime;
pub mod doc;
pub mod doc_loader;
pub mod entitlement;
@@ -13,6 +12,7 @@ pub mod image;
pub mod license;
pub mod llm;
pub mod permission;
pub mod runtime;
pub mod safe_fetch;
pub mod tiktoken;
packages/backend/native/src/license.rs
+2 -5
View File
@@ -1,7 +1,7 @@
use std::{
collections::HashMap,
sync::{Mutex, OnceLock},
time::{Duration, SystemTime, UNIX_EPOCH},
time::{Duration, SystemTime},
};
use anyhow::{Context, Result as AnyResult, bail};
@@ -458,10 +458,7 @@ fn parse_future_end_at(value: &serde_json::Value) -> AnyResult<f64> {
}
fn now_millis() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as f64
crate::utils::system_time_millis(SystemTime::now()).unwrap_or_default() as f64
}
fn affine_pro_ech_config() -> AnyResult<Vec<u8>> {
packages/backend/native/src/backend_runtime/constants.rs → packages/backend/native/src/runtime/backend_runtime/constants.rs
-1
View File
@@ -7,4 +7,3 @@ pub(super) const WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE: &str = "workspace_invi
pub(super) const WORKSPACE_STATS_LEASE_KEY: &str = "workspace:admin-stats:refresh";
pub(super) const WORKSPACE_STATS_LOCK_NAMESPACE: i64 = 97_301;
pub(super) const WORKSPACE_STATS_REFRESH_LOCK_KEY: i64 = 1;
pub(super) const RUNTIME_MIGRATIONS: &str = include_str!("sql/runtime_migrations.sql");
packages/backend/native/src/backend_runtime/coordination_lease.rs → packages/backend/native/src/runtime/backend_runtime/coordination_lease.rs
+43 -18
View File
@@ -1,7 +1,7 @@
use napi::Result;
use sqlx::{FromRow, PgPool};
use super::{BackendRuntime, error::napi_error, types::CoordinationLeaseGrant};
use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error, types::CoordinationLeaseGrant};
#[derive(FromRow)]
struct LeaseGrantRow {
@@ -17,7 +17,7 @@ impl CoordinationLeaseStore {
Self { pool }
}
async fn acquire(&self, key: String, owner: String, ttl_ms: i64) -> Result<Option<CoordinationLeaseGrant>> {
async fn acquire(&self, key: String, owner: String, ttl_ms: i64) -> RuntimeResult<Option<CoordinationLeaseGrant>> {
let row = sqlx::query_as::<_, LeaseGrantRow>(
r#"
INSERT INTO runtime_leases (key, owner, fencing_token, expires_at)
@@ -36,7 +36,7 @@ impl CoordinationLeaseStore {
.bind(ttl_ms as f64)
.fetch_optional(&self.pool)
.await
.map_err(|err| napi_error(format!("CoordinationLease acquire failed: {err}")))?;
.map_err(|err| RuntimeError::database("CoordinationLease acquire failed", err))?;
Ok(row.map(|row| CoordinationLeaseGrant {
key,
@@ -45,7 +45,7 @@ impl CoordinationLeaseStore {
}))
}
async fn release(&self, key: &str, owner: &str, fencing_token: i64) -> Result<bool> {
async fn release(&self, key: &str, owner: &str, fencing_token: i64) -> RuntimeResult<bool> {
let result = sqlx::query(
r#"
DELETE FROM runtime_leases
@@ -57,12 +57,12 @@ impl CoordinationLeaseStore {
.bind(fencing_token)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("CoordinationLease release failed: {err}")))?;
.map_err(|err| RuntimeError::database("CoordinationLease release failed", err))?;
Ok(result.rows_affected() == 1)
}
async fn renew(&self, key: &str, owner: &str, fencing_token: i64, ttl_ms: i64) -> Result<bool> {
async fn renew(&self, key: &str, owner: &str, fencing_token: i64, ttl_ms: i64) -> RuntimeResult<bool> {
let result = sqlx::query(
r#"
UPDATE runtime_leases
@@ -80,7 +80,7 @@ impl CoordinationLeaseStore {
.bind(ttl_ms as f64)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("CoordinationLease renew failed: {err}")))?;
.map_err(|err| RuntimeError::database("CoordinationLease renew failed", err))?;
Ok(result.rows_affected() == 1)
}
@@ -88,6 +88,35 @@ impl CoordinationLeaseStore {
#[napi_derive::napi]
impl BackendRuntime {
pub(crate) async fn acquire_coordination_lease_inner(
&self,
key: String,
owner: String,
ttl_ms: i64,
) -> RuntimeResult<Option<CoordinationLeaseGrant>> {
if ttl_ms <= 0 {
return Err(RuntimeError::invalid_input("coordination lease ttl must be positive"));
}
if owner.is_empty() {
return Err(RuntimeError::invalid_input("coordination lease owner is required"));
}
CoordinationLeaseStore::new(self.pool().await?)
.acquire(key, owner, ttl_ms)
.await
}
pub(crate) async fn release_coordination_lease_inner(
&self,
key: String,
owner: String,
fencing_token: i64,
) -> RuntimeResult<bool> {
CoordinationLeaseStore::new(self.pool().await?)
.release(&key, &owner, fencing_token)
.await
}
#[napi]
pub async fn acquire_coordination_lease(
&self,
@@ -95,16 +124,10 @@ impl BackendRuntime {
owner: String,
ttl_ms: i64,
) -> Result<Option<CoordinationLeaseGrant>> {
if ttl_ms <= 0 {
return Err(napi_error("coordination lease ttl must be positive"));
}
if owner.is_empty() {
return Err(napi_error("coordination lease owner is required"));
}
CoordinationLeaseStore::new(self.pool().await?)
.acquire(key, owner, ttl_ms)
self
.acquire_coordination_lease_inner(key, owner, ttl_ms)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -114,9 +137,10 @@ impl BackendRuntime {
owner: String,
#[napi(ts_arg_type = "bigint | number")] fencing_token: i64,
) -> Result<bool> {
CoordinationLeaseStore::new(self.pool().await?)
.release(&key, &owner, fencing_token)
self
.release_coordination_lease_inner(key, owner, fencing_token)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -134,5 +158,6 @@ impl BackendRuntime {
CoordinationLeaseStore::new(self.pool().await?)
.renew(&key, &owner, fencing_token, ttl_ms)
.await
.map_err(napi::Error::from)
}
}
packages/backend/native/src/backend_runtime/doc_compactor.rs → packages/backend/native/src/runtime/backend_runtime/doc_compactor.rs
+31 -31
View File
@@ -1,9 +1,8 @@
use chrono::{DateTime, Duration, Utc};
use napi::Result;
use sqlx::{FromRow, PgPool, Postgres, Row, Transaction};
use y_octo::Doc;
use super::{BackendRuntime, error::napi_error, types::RuntimeDocCompactionResult};
use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error, types::RuntimeDocCompactionResult};
#[derive(FromRow)]
struct SnapshotRow {
@@ -35,7 +34,7 @@ impl DocCompactorStore {
batch_limit: i64,
history_min_interval_ms: i64,
history_max_age_seconds: i64,
) -> Result<(i64, bool)> {
) -> RuntimeResult<(i64, bool)> {
compact_doc(
self.pool.clone(),
workspace_id,
@@ -52,31 +51,32 @@ fn is_empty_doc(bin: &[u8]) -> bool {
bin.is_empty() || (bin.len() == 1 && bin[0] == 0) || (bin.len() == 2 && bin[0] == 0 && bin[1] == 0)
}
fn apply_updates(updates: impl IntoIterator<Item = Vec<u8>>) -> Result<Vec<u8>> {
fn apply_updates(updates: impl IntoIterator<Item = Vec<u8>>) -> RuntimeResult<Vec<u8>> {
let mut doc = Doc::default();
for update in updates {
doc
.apply_update_from_binary_v1(&update)
.map_err(|err| napi_error(format!("DocCompactor merge failed: {err}")))?;
.map_err(|err| RuntimeError::invalid_state(format!("DocCompactor merge failed: {err}")))?;
}
doc
.encode_update_v1()
.map_err(|err| napi_error(format!("DocCompactor encode failed: {err}")))
.map_err(|err| RuntimeError::invalid_state(format!("DocCompactor encode failed: {err}")))
}
fn checked_milliseconds(value: i64, field: &str) -> Result<Duration> {
Duration::try_milliseconds(value).ok_or_else(|| napi_error(format!("DocCompactor {field} is too large")))
fn checked_milliseconds(value: i64, field: &str) -> RuntimeResult<Duration> {
Duration::try_milliseconds(value)
.ok_or_else(|| RuntimeError::invalid_input(format!("DocCompactor {field} is too large")))
}
fn checked_seconds(value: i64, field: &str) -> Result<Duration> {
Duration::try_seconds(value).ok_or_else(|| napi_error(format!("DocCompactor {field} is too large")))
fn checked_seconds(value: i64, field: &str) -> RuntimeResult<Duration> {
Duration::try_seconds(value).ok_or_else(|| RuntimeError::invalid_input(format!("DocCompactor {field} is too large")))
}
async fn load_snapshot(
tx: &mut Transaction<'_, Postgres>,
workspace_id: &str,
doc_id: &str,
) -> Result<Option<SnapshotRow>> {
) -> RuntimeResult<Option<SnapshotRow>> {
sqlx::query_as::<_, SnapshotRow>(
r#"
SELECT blob, updated_at, updated_by
@@ -89,7 +89,7 @@ async fn load_snapshot(
.bind(doc_id)
.fetch_optional(&mut **tx)
.await
.map_err(|err| napi_error(format!("DocCompactor load snapshot failed: {err}")))
.map_err(|err| RuntimeError::database("DocCompactor load snapshot failed", err))
}
async fn load_updates(
@@ -97,7 +97,7 @@ async fn load_updates(
workspace_id: &str,
doc_id: &str,
batch_limit: i64,
) -> Result<Vec<UpdateRow>> {
) -> RuntimeResult<Vec<UpdateRow>> {
sqlx::query_as::<_, UpdateRow>(
r#"
SELECT blob, created_at, created_by
@@ -113,7 +113,7 @@ async fn load_updates(
.bind(batch_limit)
.fetch_all(&mut **tx)
.await
.map_err(|err| napi_error(format!("DocCompactor load updates failed: {err}")))
.map_err(|err| RuntimeError::database("DocCompactor load updates failed", err))
}
async fn upsert_snapshot(
@@ -123,7 +123,7 @@ async fn upsert_snapshot(
blob: &[u8],
timestamp: DateTime<Utc>,
editor: Option<&str>,
) -> Result<bool> {
) -> RuntimeResult<bool> {
if is_empty_doc(blob) {
return Ok(false);
}
@@ -154,7 +154,7 @@ async fn upsert_snapshot(
.bind(editor)
.fetch_optional(&mut **tx)
.await
.map_err(|err| napi_error(format!("DocCompactor upsert snapshot failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocCompactor upsert snapshot failed", err))?;
Ok(row.is_some())
}
@@ -165,7 +165,7 @@ async fn should_create_history(
workspace_id: &str,
doc_id: &str,
history_min_interval_ms: i64,
) -> Result<bool> {
) -> RuntimeResult<bool> {
if is_empty_doc(&snapshot.blob) {
return Ok(false);
}
@@ -183,7 +183,7 @@ async fn should_create_history(
.bind(doc_id)
.fetch_optional(&mut **tx)
.await
.map_err(|err| napi_error(format!("DocCompactor load latest history failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocCompactor load latest history failed", err))?;
let Some(row) = row else {
return Ok(true);
@@ -198,7 +198,7 @@ async fn should_create_history(
let threshold = snapshot
.updated_at
.checked_sub_signed(min_interval)
.ok_or_else(|| napi_error("DocCompactor history interval is out of range"))?;
.ok_or_else(|| RuntimeError::invalid_input("DocCompactor history interval is out of range"))?;
Ok(last_timestamp < threshold)
}
@@ -209,7 +209,7 @@ async fn create_history(
doc_id: &str,
snapshot: &SnapshotRow,
max_age_seconds: i64,
) -> Result<bool> {
) -> RuntimeResult<bool> {
if max_age_seconds <= 0 {
return Ok(false);
}
@@ -217,7 +217,7 @@ async fn create_history(
let max_age = checked_seconds(max_age_seconds, "history max age")?;
let expired_at = Utc::now()
.checked_add_signed(max_age)
.ok_or_else(|| napi_error("DocCompactor history max age is out of range"))?;
.ok_or_else(|| RuntimeError::invalid_input("DocCompactor history max age is out of range"))?;
sqlx::query(
r#"
INSERT INTO snapshot_histories
@@ -236,7 +236,7 @@ async fn create_history(
.bind(snapshot.updated_by.as_deref())
.execute(&mut **tx)
.await
.map_err(|err| napi_error(format!("DocCompactor create history failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocCompactor create history failed", err))?;
Ok(true)
}
@@ -246,7 +246,7 @@ async fn delete_updates(
workspace_id: &str,
doc_id: &str,
timestamps: &[DateTime<Utc>],
) -> Result<i64> {
) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
DELETE FROM updates
@@ -260,7 +260,7 @@ async fn delete_updates(
.bind(timestamps)
.execute(&mut **tx)
.await
.map_err(|err| napi_error(format!("DocCompactor delete updates failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocCompactor delete updates failed", err))?;
Ok(result.rows_affected() as i64)
}
@@ -272,18 +272,18 @@ async fn compact_doc(
batch_limit: i64,
history_min_interval_ms: i64,
history_max_age_seconds: i64,
) -> Result<(i64, bool)> {
) -> RuntimeResult<(i64, bool)> {
let mut tx = pool
.begin()
.await
.map_err(|err| napi_error(format!("DocCompactor begin transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocCompactor begin transaction failed", err))?;
let snapshot = load_snapshot(&mut tx, workspace_id, doc_id).await?;
let updates = load_updates(&mut tx, workspace_id, doc_id, batch_limit).await?;
if updates.is_empty() {
tx.commit()
.await
.map_err(|err| napi_error(format!("DocCompactor commit transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocCompactor commit transaction failed", err))?;
return Ok((0, false));
}
@@ -323,7 +323,7 @@ async fn compact_doc(
tx.commit()
.await
.map_err(|err| napi_error(format!("DocCompactor commit transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocCompactor commit transaction failed", err))?;
Ok((deleted, history_created))
}
@@ -350,7 +350,7 @@ impl BackendRuntime {
history_max_age_seconds: i64,
owner: String,
lease_ttl_ms: i64,
) -> Result<RuntimeDocCompactionResult> {
) -> napi::Result<RuntimeDocCompactionResult> {
if batch_limit <= 0 {
return Err(napi_error("doc compactor batch limit must be positive"));
}
@@ -365,7 +365,7 @@ impl BackendRuntime {
let max_age = checked_seconds(history_max_age_seconds, "history max age")?;
Utc::now()
.checked_add_signed(max_age)
.ok_or_else(|| napi_error("DocCompactor history max age is out of range"))?;
.ok_or_else(|| RuntimeError::invalid_input("DocCompactor history max age is out of range"))?;
}
let lease_key = format!("doc:update:{workspace_id}:{doc_id}");
@@ -394,7 +394,7 @@ impl BackendRuntime {
.release_coordination_lease(lease.key, lease.owner, lease.fencing_token)
.await?;
if !released {
return Err(napi_error("DocCompactor failed to release coordination lease"));
return Err(RuntimeError::invalid_state("DocCompactor failed to release coordination lease").into());
}
let (updates_merged, history_created) = result?;
packages/backend/native/src/backend_runtime/doc_storage.rs → packages/backend/native/src/runtime/backend_runtime/doc_storage.rs
+20 -16
View File
@@ -1,14 +1,18 @@
use chrono::{DateTime, Duration, Utc};
use napi::{Result, bindgen_prelude::Buffer};
use napi::bindgen_prelude::Buffer;
use sqlx::{PgPool, Row};
use super::{BackendRuntime, error::napi_error, types::RuntimeDocHistoryInput};
use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error, types::RuntimeDocHistoryInput};
fn is_empty_doc(bin: &[u8]) -> bool {
bin.is_empty() || (bin.len() == 1 && bin[0] == 0) || (bin.len() == 2 && bin[0] == 0 && bin[1] == 0)
}
async fn latest_history_timestamp(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result<Option<DateTime<Utc>>> {
async fn latest_history_timestamp(
pool: &PgPool,
workspace_id: &str,
doc_id: &str,
) -> RuntimeResult<Option<DateTime<Utc>>> {
sqlx::query(
r#"
SELECT timestamp
@@ -23,7 +27,7 @@ async fn latest_history_timestamp(pool: &PgPool, workspace_id: &str, doc_id: &st
.fetch_optional(pool)
.await
.map(|row| row.map(|row| row.get("timestamp")))
.map_err(|err| napi_error(format!("DocStorage load latest history failed: {err}")))
.map_err(|err| RuntimeError::database("DocStorage load latest history failed", err))
}
#[napi_derive::napi]
@@ -36,13 +40,13 @@ impl BackendRuntime {
blob: Buffer,
timestamp_ms: i64,
editor_id: Option<String>,
) -> Result<bool> {
) -> napi::Result<bool> {
if is_empty_doc(blob.as_ref()) {
return Ok(false);
}
let timestamp = DateTime::<Utc>::from_timestamp_millis(timestamp_ms)
.ok_or_else(|| napi_error(format!("Invalid doc snapshot timestamp: {timestamp_ms}")))?;
.ok_or_else(|| RuntimeError::invalid_input(format!("Invalid doc snapshot timestamp: {timestamp_ms}")))?;
let pool = self.pool().await?;
let row = sqlx::query(
r#"
@@ -70,13 +74,13 @@ impl BackendRuntime {
.bind(editor_id.as_deref())
.fetch_optional(&pool)
.await
.map_err(|err| napi_error(format!("DocStorage upsert snapshot failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocStorage upsert snapshot failed", err))?;
Ok(row.is_some())
}
#[napi]
pub async fn create_doc_history(&self, input: RuntimeDocHistoryInput) -> Result<bool> {
pub async fn create_doc_history(&self, input: RuntimeDocHistoryInput) -> napi::Result<bool> {
if input.history_min_interval_ms < 0 {
return Err(napi_error("doc history interval must be non-negative"));
}
@@ -85,7 +89,7 @@ impl BackendRuntime {
}
let timestamp = DateTime::<Utc>::from_timestamp_millis(input.timestamp_ms)
.ok_or_else(|| napi_error(format!("Invalid doc history timestamp: {}", input.timestamp_ms)))?;
.ok_or_else(|| RuntimeError::invalid_input(format!("Invalid doc history timestamp: {}", input.timestamp_ms)))?;
let pool = self.pool().await?;
let should_create = match latest_history_timestamp(&pool, &input.workspace_id, &input.doc_id).await? {
None => true,
@@ -118,41 +122,41 @@ impl BackendRuntime {
.bind(input.editor_id.as_deref())
.execute(&pool)
.await
.map_err(|err| napi_error(format!("DocStorage create history failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocStorage create history failed", err))?;
Ok(true)
}
#[napi]
pub async fn delete_doc_storage(&self, workspace_id: String, doc_id: String) -> Result<()> {
pub async fn delete_doc_storage(&self, workspace_id: String, doc_id: String) -> napi::Result<()> {
let pool = self.pool().await?;
let mut tx = pool
.begin()
.await
.map_err(|err| napi_error(format!("DocStorage delete begin transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocStorage delete begin transaction failed", err))?;
sqlx::query("DELETE FROM snapshots WHERE workspace_id = $1 AND guid = $2")
.bind(&workspace_id)
.bind(&doc_id)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("DocStorage delete snapshot failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocStorage delete snapshot failed", err))?;
sqlx::query("DELETE FROM updates WHERE workspace_id = $1 AND guid = $2")
.bind(&workspace_id)
.bind(&doc_id)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("DocStorage delete updates failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocStorage delete updates failed", err))?;
sqlx::query("DELETE FROM snapshot_histories WHERE workspace_id = $1 AND guid = $2")
.bind(&workspace_id)
.bind(&doc_id)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("DocStorage delete histories failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocStorage delete histories failed", err))?;
tx.commit()
.await
.map_err(|err| napi_error(format!("DocStorage delete commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("DocStorage delete commit failed", err))?;
Ok(())
}
}
packages/backend/native/src/backend_runtime/gate.rs → packages/backend/native/src/runtime/backend_runtime/gate.rs
+13 -9
View File
@@ -1,7 +1,7 @@
use napi::Result;
use sqlx::PgPool;
use super::{BackendRuntime, error::napi_error};
use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error};
struct RuntimeGateStore {
pool: PgPool,
@@ -12,18 +12,18 @@ impl RuntimeGateStore {
Self { pool }
}
async fn put_if_absent(&self, key: &str, ttl_ms: i64) -> Result<bool> {
async fn put_if_absent(&self, key: &str, ttl_ms: i64) -> RuntimeResult<bool> {
let mut tx = self
.pool
.begin()
.await
.map_err(|err| napi_error(format!("RuntimeGate transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeGate transaction failed", err))?;
sqlx::query("DELETE FROM runtime_gates WHERE key = $1 AND expires_at <= CURRENT_TIMESTAMP")
.bind(key)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("RuntimeGate expired cleanup failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeGate expired cleanup failed", err))?;
let inserted = sqlx::query(
r#"
@@ -36,18 +36,18 @@ impl RuntimeGateStore {
.bind(ttl_ms as f64)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("RuntimeGate put_if_absent failed: {err}")))?
.map_err(|err| RuntimeError::database("RuntimeGate put_if_absent failed", err))?
.rows_affected()
== 1;
tx.commit()
.await
.map_err(|err| napi_error(format!("RuntimeGate transaction commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeGate transaction commit failed", err))?;
Ok(inserted)
}
async fn cleanup_expired(&self, limit: i64) -> Result<i64> {
async fn cleanup_expired(&self, limit: i64) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
DELETE FROM runtime_gates
@@ -62,7 +62,7 @@ impl RuntimeGateStore {
.bind(limit)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("RuntimeGate cleanup failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeGate cleanup failed", err))?;
Ok(result.rows_affected() as i64)
}
@@ -78,6 +78,7 @@ impl BackendRuntime {
RuntimeGateStore::new(self.pool().await?)
.put_if_absent(&key, ttl_ms)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -85,6 +86,9 @@ impl BackendRuntime {
if limit <= 0 {
return Err(napi_error("runtime gate cleanup limit must be positive"));
}
RuntimeGateStore::new(self.pool().await?).cleanup_expired(limit).await
RuntimeGateStore::new(self.pool().await?)
.cleanup_expired(limit)
.await
.map_err(napi::Error::from)
}
}
packages/backend/native/src/backend_runtime/housekeeping.rs → packages/backend/native/src/runtime/backend_runtime/housekeeping.rs
+7 -5
View File
@@ -1,7 +1,7 @@
use napi::Result;
use sqlx::PgPool;
use super::{BackendRuntime, error::napi_error};
use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error};
struct HousekeepingStore {
pool: PgPool,
@@ -12,7 +12,7 @@ impl HousekeepingStore {
Self { pool }
}
async fn cleanup_expired_user_sessions(&self, limit: i64) -> Result<i64> {
async fn cleanup_expired_user_sessions(&self, limit: i64) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
DELETE FROM user_sessions
@@ -27,12 +27,12 @@ impl HousekeepingStore {
.bind(limit)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("Housekeeping user sessions cleanup failed: {err}")))?;
.map_err(|err| RuntimeError::database("Housekeeping user sessions cleanup failed", err))?;
Ok(result.rows_affected() as i64)
}
async fn cleanup_expired_snapshot_histories(&self, limit: i64) -> Result<i64> {
async fn cleanup_expired_snapshot_histories(&self, limit: i64) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
DELETE FROM snapshot_histories
@@ -48,7 +48,7 @@ impl HousekeepingStore {
.bind(limit)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("Housekeeping snapshot histories cleanup failed: {err}")))?;
.map_err(|err| RuntimeError::database("Housekeeping snapshot histories cleanup failed", err))?;
Ok(result.rows_affected() as i64)
}
@@ -65,6 +65,7 @@ impl BackendRuntime {
HousekeepingStore::new(self.pool().await?)
.cleanup_expired_user_sessions(limit)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -76,5 +77,6 @@ impl BackendRuntime {
HousekeepingStore::new(self.pool().await?)
.cleanup_expired_snapshot_histories(limit)
.await
.map_err(napi::Error::from)
}
}
packages/backend/native/src/backend_runtime/mod.rs → packages/backend/native/src/runtime/backend_runtime/mod.rs
+20 -38
View File
@@ -1,23 +1,13 @@
mod blob_cleanup;
mod blob_complete;
mod blob_reclaimer;
mod blob_reconciliation;
mod config;
mod constants;
mod coordination_lease;
mod doc_blob_refs;
mod doc_compactor;
mod doc_storage;
mod error;
mod gate;
mod housekeeping;
mod object_storage;
mod runtime_state;
#[cfg(test)]
mod tests;
mod types;
mod workspace_stats;
use std::{sync::RwLock, time::Duration};
use napi::Result;
@@ -25,7 +15,11 @@ use sha2::{Digest, Sha256};
use sqlx::{PgPool, Row, postgres::PgPoolOptions};
use tokio::sync::Mutex;
use self::{config::RuntimeConfig, constants::RUNTIME_MIGRATIONS, error::napi_error, types::BackendRuntimeHealth};
use self::types::BackendRuntimeHealth;
pub(crate) use super::types;
use super::{
BackendRuntimeConfig, RuntimeError, RuntimeResult, migrations::migrate_runtime_tables, napi_error, to_napi_error,
};
pub(super) fn token_hash(token: &str) -> String {
hex::encode(Sha256::digest(token.as_bytes()))
@@ -33,7 +27,7 @@ pub(super) fn token_hash(token: &str) -> String {
#[napi_derive::napi]
pub struct BackendRuntime {
config: RwLock<RuntimeConfig>,
config: RwLock<BackendRuntimeConfig>,
pool: Mutex<Option<PgPool>>,
}
@@ -42,13 +36,17 @@ impl BackendRuntime {
#[napi(constructor)]
pub fn new() -> Result<Self> {
Ok(Self {
config: RwLock::new(RuntimeConfig::from_config_files()?),
config: RwLock::new(BackendRuntimeConfig::from_config_files().map_err(to_napi_error)?),
pool: Mutex::new(None),
})
}
#[napi]
pub async fn start(&self) -> Result<()> {
self.start_inner().await.map_err(to_napi_error)
}
async fn start_inner(&self) -> RuntimeResult<()> {
let mut guard = self.pool.lock().await;
if guard.is_some() {
return Ok(());
@@ -60,12 +58,12 @@ impl BackendRuntime {
.acquire_timeout(Duration::from_secs(5))
.connect(&database_url)
.await
.map_err(|err| napi_error(format!("BackendRuntime failed to connect postgres: {err}")))?;
.map_err(|err| RuntimeError::database("BackendRuntime failed to connect postgres", err))?;
sqlx::query("SELECT 1")
.execute(&pool)
.await
.map_err(|err| napi_error(format!("BackendRuntime postgres health check failed: {err}")))?;
.map_err(|err| RuntimeError::database("BackendRuntime postgres health check failed", err))?;
let config = self.config()?.with_db_overrides(&pool).await?;
self.update_config(config)?;
@@ -98,54 +96,38 @@ impl BackendRuntime {
Ok(BackendRuntimeHealth {
started: pool.is_some(),
database_connected,
object_storage_configured: self.config()?.storage.is_some(),
})
}
#[napi]
pub async fn run_migrations(&self) -> Result<()> {
let pool = self.pool().await?;
migrate_runtime_tables(&pool).await
migrate_runtime_tables(&pool).await.map_err(to_napi_error)
}
async fn pool(&self) -> Result<PgPool> {
pub(crate) async fn pool(&self) -> RuntimeResult<PgPool> {
self
.pool
.lock()
.await
.as_ref()
.cloned()
.ok_or_else(|| napi_error("BackendRuntime must be started before using postgres operations"))
.ok_or_else(|| RuntimeError::invalid_state("BackendRuntime must be started before using postgres operations"))
}
pub(in crate::backend_runtime) fn config(&self) -> Result<RuntimeConfig> {
pub(crate) fn config(&self) -> RuntimeResult<BackendRuntimeConfig> {
self
.config
.read()
.map(|config| config.clone())
.map_err(|_| napi_error("BackendRuntime config lock poisoned"))
.map_err(|_| RuntimeError::invalid_state("BackendRuntime config lock poisoned"))
}
fn update_config(&self, config: RuntimeConfig) -> Result<()> {
fn update_config(&self, config: BackendRuntimeConfig) -> RuntimeResult<()> {
*self
.config
.write()
.map_err(|_| napi_error("BackendRuntime config lock poisoned"))? = config;
.map_err(|_| RuntimeError::invalid_state("BackendRuntime config lock poisoned"))? = config;
Ok(())
}
}
async fn migrate_runtime_tables(pool: &PgPool) -> Result<()> {
for statement in RUNTIME_MIGRATIONS
.split(';')
.map(str::trim)
.filter(|statement| !statement.is_empty())
{
sqlx::query(statement)
.execute(pool)
.await
.map_err(|err| napi_error(format!("BackendRuntime migration failed: {err}")))?;
}
Ok(())
}
packages/backend/native/src/backend_runtime/runtime_state/auth_challenge.rs → packages/backend/native/src/runtime/backend_runtime/runtime_state/auth_challenge.rs
+1 -3
View File
@@ -1,6 +1,4 @@
use napi::Result;
use super::{auth_challenge_purpose, dto::RuntimeStateRows};
use super::{Result, auth_challenge_purpose, dto::RuntimeStateRows};
pub(super) async fn create(
rows: &RuntimeStateRows,
packages/backend/native/src/backend_runtime/runtime_state/byok_local_lease.rs → packages/backend/native/src/runtime/backend_runtime/runtime_state/byok_local_lease.rs
+11 -19
View File
@@ -1,10 +1,6 @@
use napi::Result;
use super::dto::{RuntimeStateInsertPayload, RuntimeStatePayloadRow, RuntimeStateRows};
use crate::backend_runtime::{
constants::{BYOK_LOCAL_LEASE_ACTIVE_PURPOSE, BYOK_LOCAL_LEASE_PURPOSE},
error::napi_error,
types::RuntimeByokLocalLeaseRecord,
use super::{
BYOK_LOCAL_LEASE_ACTIVE_PURPOSE, BYOK_LOCAL_LEASE_PURPOSE, Result, RuntimeByokLocalLeaseRecord, RuntimeError,
dto::{RuntimeStateInsertPayload, RuntimeStatePayloadRow, RuntimeStateRows},
};
pub(super) async fn get(rows: &RuntimeStateRows, lease_id: String) -> Result<Option<RuntimeByokLocalLeaseRecord>> {
@@ -19,7 +15,7 @@ pub(super) async fn create(
ttl_ms: i64,
) -> Result<RuntimeByokLocalLeaseRecord> {
if ttl_ms <= 0 {
return Err(napi_error("BYOK local lease ttl must be positive"));
return Err(RuntimeError::invalid_input("BYOK local lease ttl must be positive"));
}
let mut tx = rows.begin("RuntimeState BYOK local lease").await?;
@@ -27,7 +23,7 @@ pub(super) async fn create(
.bind(&active_key)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("RuntimeState BYOK local lease active lock failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState BYOK local lease active lock failed", err))?;
if let Some(active) = rows
.active_payload_with_expires_for_update_in_tx(
@@ -43,11 +39,9 @@ pub(super) async fn create(
None => None,
};
if let Some(lease) = existing_lease {
tx.commit().await.map_err(|err| {
napi_error(format!(
"RuntimeState BYOK local lease transaction commit failed: {err}"
))
})?;
tx.commit()
.await
.map_err(|err| RuntimeError::database("RuntimeState BYOK local lease transaction commit failed", err))?;
return Ok(lease);
}
@@ -89,11 +83,9 @@ pub(super) async fn create(
)
.await?;
tx.commit().await.map_err(|err| {
napi_error(format!(
"RuntimeState BYOK local lease transaction commit failed: {err}"
))
})?;
tx.commit()
.await
.map_err(|err| RuntimeError::database("RuntimeState BYOK local lease transaction commit failed", err))?;
Ok(RuntimeByokLocalLeaseRecord {
lease_id,
packages/backend/native/src/backend_runtime/runtime_state/dto.rs → packages/backend/native/src/runtime/backend_runtime/runtime_state/dto.rs
+19 -18
View File
@@ -1,7 +1,8 @@
use napi::Result;
use sqlx::{PgPool, Row};
use crate::backend_runtime::{error::napi_error, token_hash};
use super::{RuntimeError, RuntimeResult, token_hash};
type Result<T> = RuntimeResult<T>;
pub(super) struct RuntimeStatePayloadRow {
pub(super) payload: serde_json::Value,
@@ -42,7 +43,7 @@ impl RuntimeStateRows {
.pool
.begin()
.await
.map_err(|err| napi_error(format!("{context} transaction failed: {err}")))
.map_err(|err| RuntimeError::database(format!("{context} transaction failed"), err))
}
pub(super) async fn insert_payload(
@@ -67,7 +68,7 @@ impl RuntimeStateRows {
.bind(ttl_ms as f64)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(())
}
@@ -95,7 +96,7 @@ impl RuntimeStateRows {
.bind(ttl_ms as f64)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?
.map_err(|err| RuntimeError::database(context, err))?
.rows_affected()
== 1;
@@ -131,7 +132,7 @@ impl RuntimeStateRows {
.bind(ttl_ms as f64)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(())
}
@@ -156,7 +157,7 @@ impl RuntimeStateRows {
.bind(token_hash(token))
.fetch_optional(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(row.map(|row| row.get::<serde_json::Value, _>("payload")))
}
@@ -181,7 +182,7 @@ impl RuntimeStateRows {
.bind(token_hash(token))
.fetch_optional(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(row.map(payload_row))
}
@@ -208,7 +209,7 @@ impl RuntimeStateRows {
.bind(token_hash(token))
.fetch_optional(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(row.map(|row| row.get::<serde_json::Value, _>("payload")))
}
@@ -235,7 +236,7 @@ impl RuntimeStateRows {
.bind(token_hash(token))
.fetch_optional(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(row.map(payload_row))
}
@@ -262,7 +263,7 @@ impl RuntimeStateRows {
.bind(token_hash(token))
.fetch_optional(&mut **tx)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(row.map(payload_row))
}
@@ -288,7 +289,7 @@ impl RuntimeStateRows {
.bind(token_hash(token))
.fetch_optional(&mut **tx)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(row.map(|row| RuntimeStateLockedRow {
payload: row.get("payload"),
@@ -316,7 +317,7 @@ impl RuntimeStateRows {
.bind(input.ttl_ms as f64)
.fetch_one(&mut **tx)
.await
.map_err(|err| napi_error(format!("{} failed: {err}", input.context)))?;
.map_err(|err| RuntimeError::database(input.context, err))?;
Ok(row.get::<i64, _>("expires_at_ms"))
}
@@ -348,7 +349,7 @@ impl RuntimeStateRows {
.bind(input.ttl_ms as f64)
.fetch_optional(&mut **tx)
.await
.map_err(|err| napi_error(format!("{} failed: {err}", input.context)))?;
.map_err(|err| RuntimeError::database(input.context, err))?;
Ok(row.map(|row| row.get::<i64, _>("expires_at_ms")))
}
@@ -375,7 +376,7 @@ impl RuntimeStateRows {
.bind(attempts)
.execute(&mut **tx)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(())
}
@@ -392,7 +393,7 @@ impl RuntimeStateRows {
.bind(token_hash(token))
.execute(&mut **tx)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(())
}
@@ -413,7 +414,7 @@ impl RuntimeStateRows {
.bind(limit)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(result.rows_affected() as i64)
}
@@ -440,7 +441,7 @@ impl RuntimeStateRows {
.bind(limit)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("{context} failed: {err}")))?;
.map_err(|err| RuntimeError::database(context, err))?;
Ok(result.rows_affected() as i64)
}
packages/backend/native/src/backend_runtime/runtime_state/invite_link.rs → packages/backend/native/src/runtime/backend_runtime/runtime_state/invite_link.rs
+28 -38
View File
@@ -1,10 +1,7 @@
use napi::Result;
use super::dto::{RuntimeStateInsertPayload, RuntimeStatePayloadRow, RuntimeStateRows};
use crate::backend_runtime::{
constants::{WORKSPACE_INVITE_LINK_ID_PURPOSE, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE},
error::napi_error,
types::RuntimeWorkspaceInviteLinkRecord,
use super::{
Result, RuntimeError, RuntimeWorkspaceInviteLinkRecord, WORKSPACE_INVITE_LINK_ID_PURPOSE,
WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE,
dto::{RuntimeStateInsertPayload, RuntimeStatePayloadRow, RuntimeStateRows},
};
pub(super) async fn get_by_workspace(
@@ -29,7 +26,9 @@ pub(super) async fn create(
ttl_ms: i64,
) -> Result<RuntimeWorkspaceInviteLinkRecord> {
if ttl_ms <= 0 {
return Err(napi_error("workspace invite link ttl must be positive"));
return Err(RuntimeError::invalid_input(
"workspace invite link ttl must be positive",
));
}
let mut tx = rows.begin("RuntimeState workspace invite link").await?;
@@ -37,16 +36,14 @@ pub(super) async fn create(
.bind(&workspace_id)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("RuntimeState workspace invite link active lock failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState workspace invite link active lock failed", err))?;
if let Some(existing) =
get_by_key_in_tx(rows, &mut tx, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, &workspace_id).await?
{
tx.commit().await.map_err(|err| {
napi_error(format!(
"RuntimeState workspace invite link transaction commit failed: {err}"
))
})?;
tx.commit()
.await
.map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?;
return Ok(existing);
}
@@ -71,12 +68,11 @@ pub(super) async fn create(
.await?
else {
let existing = get_by_key_in_tx(rows, &mut tx, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, &workspace_id).await?;
tx.commit().await.map_err(|err| {
napi_error(format!(
"RuntimeState workspace invite link transaction commit failed: {err}"
))
})?;
return existing.ok_or_else(|| napi_error("RuntimeState workspace invite link active conflict missing row"));
tx.commit()
.await
.map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?;
return existing
.ok_or_else(|| RuntimeError::invalid_state("RuntimeState workspace invite link active conflict missing row"));
};
rows
.insert_payload_returning_expires_in_tx(
@@ -92,11 +88,9 @@ pub(super) async fn create(
)
.await?;
tx.commit().await.map_err(|err| {
napi_error(format!(
"RuntimeState workspace invite link transaction commit failed: {err}"
))
})?;
tx.commit()
.await
.map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?;
Ok(RuntimeWorkspaceInviteLinkRecord {
workspace_id,
@@ -110,11 +104,9 @@ pub(super) async fn revoke(rows: &RuntimeStateRows, workspace_id: String) -> Res
let mut tx = rows.begin("RuntimeState workspace invite link").await?;
let existing = get_by_key_in_tx(rows, &mut tx, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, &workspace_id).await?;
let Some(existing) = existing else {
tx.commit().await.map_err(|err| {
napi_error(format!(
"RuntimeState workspace invite link transaction commit failed: {err}"
))
})?;
tx.commit()
.await
.map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?;
return Ok(false);
};
@@ -135,11 +127,9 @@ pub(super) async fn revoke(rows: &RuntimeStateRows, workspace_id: String) -> Res
)
.await?;
tx.commit().await.map_err(|err| {
napi_error(format!(
"RuntimeState workspace invite link transaction commit failed: {err}"
))
})?;
tx.commit()
.await
.map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?;
Ok(true)
}
@@ -175,19 +165,19 @@ fn record_from_row(row: RuntimeStatePayloadRow) -> Result<RuntimeWorkspaceInvite
.payload
.get("workspaceId")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| napi_error("RuntimeState workspace invite link payload missing workspaceId"))?
.ok_or_else(|| RuntimeError::invalid_state("RuntimeState workspace invite link payload missing workspaceId"))?
.to_string(),
invite_id: row
.payload
.get("inviteId")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| napi_error("RuntimeState workspace invite link payload missing inviteId"))?
.ok_or_else(|| RuntimeError::invalid_state("RuntimeState workspace invite link payload missing inviteId"))?
.to_string(),
inviter_user_id: row
.payload
.get("inviterUserId")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| napi_error("RuntimeState workspace invite link payload missing inviterUserId"))?
.ok_or_else(|| RuntimeError::invalid_state("RuntimeState workspace invite link payload missing inviterUserId"))?
.to_string(),
expires_at_ms: row.expires_at_ms,
})
packages/backend/native/src/backend_runtime/runtime_state/magic_link_otp.rs → packages/backend/native/src/runtime/backend_runtime/runtime_state/magic_link_otp.rs
+14 -20
View File
@@ -1,10 +1,6 @@
use napi::Result;
use super::dto::RuntimeStateRows;
use crate::backend_runtime::{
constants::{MAGIC_LINK_OTP_PURPOSE, MAX_MAGIC_LINK_OTP_ATTEMPTS},
error::napi_error,
types::RuntimeMagicLinkOtpConsumeResult,
use super::{
MAGIC_LINK_OTP_PURPOSE, MAX_MAGIC_LINK_OTP_ATTEMPTS, Result, RuntimeError, RuntimeMagicLinkOtpConsumeResult,
dto::RuntimeStateRows,
};
impl RuntimeMagicLinkOtpConsumeResult {
@@ -34,7 +30,7 @@ pub(super) async fn upsert(
ttl_ms: i64,
) -> Result<()> {
if ttl_ms <= 0 {
return Err(napi_error("magic link otp ttl must be positive"));
return Err(RuntimeError::invalid_input("magic link otp ttl must be positive"));
}
let payload = serde_json::json!({
@@ -73,11 +69,9 @@ pub(super) async fn consume(
.await?;
let Some(row) = row else {
tx.rollback().await.map_err(|err| {
napi_error(format!(
"RuntimeState magic link otp transaction rollback failed: {err}"
))
})?;
tx.rollback()
.await
.map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction rollback failed", err))?;
return Ok(RuntimeMagicLinkOtpConsumeResult::fail("not_found"));
};
@@ -96,7 +90,7 @@ pub(super) async fn consume(
.await?;
tx.commit()
.await
.map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?;
return Ok(RuntimeMagicLinkOtpConsumeResult::fail("expired"));
}
@@ -104,7 +98,7 @@ pub(super) async fn consume(
if stored_client_nonce.is_some() && stored_client_nonce != client_nonce.as_deref() {
tx.commit()
.await
.map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?;
return Ok(RuntimeMagicLinkOtpConsumeResult::fail("nonce_mismatch"));
}
@@ -119,7 +113,7 @@ pub(super) async fn consume(
.await?;
tx.commit()
.await
.map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?;
return Ok(RuntimeMagicLinkOtpConsumeResult::fail("locked"));
}
@@ -137,7 +131,7 @@ pub(super) async fn consume(
.await?;
tx.commit()
.await
.map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?;
return Ok(RuntimeMagicLinkOtpConsumeResult::fail("locked"));
}
@@ -153,14 +147,14 @@ pub(super) async fn consume(
tx.commit()
.await
.map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?;
return Ok(RuntimeMagicLinkOtpConsumeResult::fail("invalid_otp"));
}
let token = payload
.get("token")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| napi_error("RuntimeState magic link otp payload missing token"))?
.ok_or_else(|| RuntimeError::invalid_state("RuntimeState magic link otp payload missing token"))?
.to_string();
rows
.delete_by_key_in_tx(
@@ -172,7 +166,7 @@ pub(super) async fn consume(
.await?;
tx.commit()
.await
.map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?;
Ok(RuntimeMagicLinkOtpConsumeResult::ok(token))
}
packages/backend/native/src/backend_runtime/runtime_state/mod.rs → packages/backend/native/src/runtime/backend_runtime/runtime_state/mod.rs
+47 -24
View File
@@ -1,8 +1,10 @@
use napi::Result;
use super::{
BackendRuntime,
error::napi_error,
use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error};
pub(super) use super::{
constants::{
BYOK_LOCAL_LEASE_ACTIVE_PURPOSE, BYOK_LOCAL_LEASE_PURPOSE, MAGIC_LINK_OTP_PURPOSE, MAX_MAGIC_LINK_OTP_ATTEMPTS,
WORKSPACE_INVITE_LINK_ID_PURPOSE, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE,
},
token_hash,
types::{
RuntimeByokLocalLeaseRecord, RuntimeMagicLinkOtpConsumeResult, RuntimeVerificationTokenRecord,
RuntimeWorkspaceInviteLinkRecord,
@@ -18,6 +20,8 @@ mod store;
mod verification_token;
use store::RuntimeStateStore;
pub(super) type Result<T> = RuntimeResult<T>;
pub(super) fn auth_challenge_purpose(purpose: &str) -> String {
format!("auth_challenge:{purpose}")
}
@@ -35,27 +39,34 @@ impl BackendRuntime {
token: String,
payload: serde_json::Value,
ttl_ms: i64,
) -> Result<bool> {
) -> napi::Result<bool> {
if ttl_ms <= 0 {
return Err(napi_error("auth challenge ttl must be positive"));
}
RuntimeStateStore::new(self.pool().await?)
.create_auth_challenge(&purpose, &token, payload, ttl_ms)
.await
.map_err(napi::Error::from)
}
#[napi]
pub async fn get_auth_challenge(&self, purpose: String, token: String) -> Result<Option<serde_json::Value>> {
pub async fn get_auth_challenge(&self, purpose: String, token: String) -> napi::Result<Option<serde_json::Value>> {
RuntimeStateStore::new(self.pool().await?)
.get_auth_challenge(&purpose, &token)
.await
.map_err(napi::Error::from)
}
#[napi]
pub async fn consume_auth_challenge(&self, purpose: String, token: String) -> Result<Option<serde_json::Value>> {
pub async fn consume_auth_challenge(
&self,
purpose: String,
token: String,
) -> napi::Result<Option<serde_json::Value>> {
RuntimeStateStore::new(self.pool().await?)
.consume_auth_challenge(&purpose, &token)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -64,13 +75,14 @@ impl BackendRuntime {
token_type: i32,
credential: Option<String>,
ttl_ms: i64,
) -> Result<String> {
) -> napi::Result<String> {
if ttl_ms <= 0 {
return Err(napi_error("verification token ttl must be positive"));
}
RuntimeStateStore::new(self.pool().await?)
.create_verification_token(token_type, credential, ttl_ms)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -79,11 +91,12 @@ impl BackendRuntime {
token_type: i32,
token: String,
keep: Option<bool>,
) -> Result<Option<RuntimeVerificationTokenRecord>> {
) -> napi::Result<Option<RuntimeVerificationTokenRecord>> {
let keep = keep.unwrap_or(false);
RuntimeStateStore::new(self.pool().await?)
.get_verification_token(token_type, token, keep)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -93,21 +106,23 @@ impl BackendRuntime {
token: String,
credential: Option<String>,
keep: Option<bool>,
) -> Result<Option<RuntimeVerificationTokenRecord>> {
) -> napi::Result<Option<RuntimeVerificationTokenRecord>> {
let keep = keep.unwrap_or(false);
RuntimeStateStore::new(self.pool().await?)
.verify_verification_token(token_type, token, credential, keep)
.await
.map_err(napi::Error::from)
}
#[napi]
pub async fn cleanup_expired_verification_tokens(&self, limit: i64) -> Result<i64> {
pub async fn cleanup_expired_verification_tokens(&self, limit: i64) -> napi::Result<i64> {
if limit <= 0 {
return Err(napi_error("verification token cleanup limit must be positive"));
}
RuntimeStateStore::new(self.pool().await?)
.cleanup_expired_verification_tokens(limit)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -118,10 +133,11 @@ impl BackendRuntime {
token: String,
client_nonce: Option<String>,
ttl_ms: i64,
) -> Result<()> {
) -> napi::Result<()> {
RuntimeStateStore::new(self.pool().await?)
.upsert_magic_link_otp(email, otp_hash, token, client_nonce, ttl_ms)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -130,10 +146,11 @@ impl BackendRuntime {
email: String,
otp_hash: String,
client_nonce: Option<String>,
) -> Result<RuntimeMagicLinkOtpConsumeResult> {
) -> napi::Result<RuntimeMagicLinkOtpConsumeResult> {
RuntimeStateStore::new(self.pool().await?)
.consume_magic_link_otp(email, otp_hash, client_nonce)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -143,37 +160,41 @@ impl BackendRuntime {
invite_id: String,
inviter_user_id: String,
ttl_ms: i64,
) -> Result<RuntimeWorkspaceInviteLinkRecord> {
) -> napi::Result<RuntimeWorkspaceInviteLinkRecord> {
RuntimeStateStore::new(self.pool().await?)
.create_workspace_invite_link(workspace_id, invite_id, inviter_user_id, ttl_ms)
.await
.map_err(napi::Error::from)
}
#[napi]
pub async fn get_workspace_invite_link(
&self,
workspace_id: String,
) -> Result<Option<RuntimeWorkspaceInviteLinkRecord>> {
) -> napi::Result<Option<RuntimeWorkspaceInviteLinkRecord>> {
RuntimeStateStore::new(self.pool().await?)
.get_workspace_invite_link(workspace_id)
.await
.map_err(napi::Error::from)
}
#[napi]
pub async fn get_workspace_invite_link_by_id(
&self,
invite_id: String,
) -> Result<Option<RuntimeWorkspaceInviteLinkRecord>> {
) -> napi::Result<Option<RuntimeWorkspaceInviteLinkRecord>> {
RuntimeStateStore::new(self.pool().await?)
.get_workspace_invite_link_by_id(invite_id)
.await
.map_err(napi::Error::from)
}
#[napi]
pub async fn revoke_workspace_invite_link(&self, workspace_id: String) -> Result<bool> {
pub async fn revoke_workspace_invite_link(&self, workspace_id: String) -> napi::Result<bool> {
RuntimeStateStore::new(self.pool().await?)
.revoke_workspace_invite_link(workspace_id)
.await
.map_err(napi::Error::from)
}
#[napi]
@@ -183,35 +204,37 @@ impl BackendRuntime {
lease_id: String,
payload: serde_json::Value,
ttl_ms: i64,
) -> Result<RuntimeByokLocalLeaseRecord> {
) -> napi::Result<RuntimeByokLocalLeaseRecord> {
RuntimeStateStore::new(self.pool().await?)
.create_byok_local_lease(active_key, lease_id, payload, ttl_ms)
.await
.map_err(napi::Error::from)
}
#[napi]
pub async fn get_byok_local_lease(&self, lease_id: String) -> Result<Option<RuntimeByokLocalLeaseRecord>> {
pub async fn get_byok_local_lease(&self, lease_id: String) -> napi::Result<Option<RuntimeByokLocalLeaseRecord>> {
RuntimeStateStore::new(self.pool().await?)
.get_byok_local_lease(lease_id)
.await
.map_err(napi::Error::from)
}
#[napi]
pub async fn cleanup_expired_runtime_states(&self, limit: i64) -> Result<i64> {
pub async fn cleanup_expired_runtime_states(&self, limit: i64) -> napi::Result<i64> {
if limit <= 0 {
return Err(napi_error("runtime state cleanup limit must be positive"));
}
RuntimeStateStore::new(self.pool().await?)
.cleanup_expired_runtime_states(limit)
.await
.map_err(napi::Error::from)
}
}
#[cfg(test)]
mod tests {
use crate::backend_runtime::{
constants::{MAGIC_LINK_OTP_PURPOSE, WORKSPACE_INVITE_LINK_ID_PURPOSE, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE},
token_hash,
use super::{
MAGIC_LINK_OTP_PURPOSE, WORKSPACE_INVITE_LINK_ID_PURPOSE, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, token_hash,
};
#[test]
packages/backend/native/src/backend_runtime/runtime_state/store.rs → packages/backend/native/src/runtime/backend_runtime/runtime_state/store.rs
+4 -5
View File
@@ -1,10 +1,9 @@
use napi::Result;
use sqlx::PgPool;
use super::{auth_challenge, byok_local_lease, dto::RuntimeStateRows, invite_link, magic_link_otp, verification_token};
use crate::backend_runtime::types::{
RuntimeByokLocalLeaseRecord, RuntimeMagicLinkOtpConsumeResult, RuntimeVerificationTokenRecord,
RuntimeWorkspaceInviteLinkRecord,
use super::{
Result, RuntimeByokLocalLeaseRecord, RuntimeMagicLinkOtpConsumeResult, RuntimeVerificationTokenRecord,
RuntimeWorkspaceInviteLinkRecord, auth_challenge, byok_local_lease, dto::RuntimeStateRows, invite_link,
magic_link_otp, verification_token,
};
pub(super) struct RuntimeStateStore {
packages/backend/native/src/backend_runtime/runtime_state/verification_token.rs → packages/backend/native/src/runtime/backend_runtime/runtime_state/verification_token.rs
+3 -4
View File
@@ -1,12 +1,11 @@
use napi::Result;
use sqlx::{PgPool, Row};
use uuid::Uuid;
use super::{
Result, RuntimeError, RuntimeVerificationTokenRecord,
dto::{RuntimeStatePayloadRow, RuntimeStateRows},
verification_token_purpose,
token_hash, verification_token_purpose,
};
use crate::backend_runtime::{error::napi_error, token_hash, types::RuntimeVerificationTokenRecord};
pub(super) async fn create(
rows: &RuntimeStateRows,
@@ -64,7 +63,7 @@ pub(super) async fn verify(
} else {
consume_payload_with_credential(rows.pool(), &purpose, &token, credential.as_deref()).await
}
.map_err(|err| napi_error(format!("RuntimeState verification token verify failed: {err}")))?;
.map_err(|err| RuntimeError::database("RuntimeState verification token verify failed", err))?;
Ok(row.map(|row| record_from_row(token_type, token, row)))
}
packages/backend/native/src/backend_runtime/sql/upsert_workspace_admin_stats.sql → packages/backend/native/src/runtime/backend_runtime/sql/upsert_workspace_admin_stats.sql
View File
packages/backend/native/src/backend_runtime/tests.rs → packages/backend/native/src/runtime/backend_runtime/tests.rs
+6 -5
View File
@@ -1,6 +1,10 @@
use anyhow::{Context, Result as AnyResult, anyhow};
use super::{runtime_state::*, *};
use super::{
super::migrations::{RUNTIME_MIGRATIONS, migrate_runtime_tables},
runtime_state::*,
*,
};
static PG_TEST_LOCK: std::sync::OnceLock<tokio::sync::Mutex<()>> = std::sync::OnceLock::new();
const TEST_VERIFICATION_TOKEN_TYPE: i32 = 99_999;
@@ -70,10 +74,7 @@ async fn runtime_from_database_url() -> AnyResult<Option<BackendRuntime>> {
.context("cleanup runtime_leases for backend runtime tests")?;
Ok(Some(BackendRuntime {
config: std::sync::RwLock::new(RuntimeConfig {
database_url,
storage: None,
}),
config: std::sync::RwLock::new(BackendRuntimeConfig { database_url }),
pool: Mutex::new(Some(pool)),
}))
}
packages/backend/native/src/backend_runtime/workspace_stats.rs → packages/backend/native/src/runtime/backend_runtime/workspace_stats.rs
+52 -49
View File
@@ -1,11 +1,10 @@
use napi::Result;
use sqlx::{FromRow, PgPool, Postgres, Row, Transaction};
use tokio::time::{Duration as TokioDuration, sleep};
use super::{
BackendRuntime,
BackendRuntime, RuntimeError, RuntimeResult,
constants::{WORKSPACE_STATS_LEASE_KEY, WORKSPACE_STATS_LOCK_NAMESPACE, WORKSPACE_STATS_REFRESH_LOCK_KEY},
error::napi_error,
napi_error,
types::{
CoordinationLeaseGrant, RuntimeWorkspaceStatsDailyRecalibrationResult, RuntimeWorkspaceStatsRecalibrationResult,
RuntimeWorkspaceStatsRefreshResult, RuntimeWorkspaceStatsSnapshotResult,
@@ -22,13 +21,13 @@ impl BackendRuntime {
batch_limit: i64,
owner: String,
lease_ttl_ms: i64,
) -> Result<RuntimeWorkspaceStatsRefreshResult> {
) -> napi::Result<RuntimeWorkspaceStatsRefreshResult> {
if batch_limit <= 0 {
return Err(napi_error("workspace stats dirty refresh limit must be positive"));
}
let Some(lease) = self
.acquire_coordination_lease(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms)
.acquire_coordination_lease_inner(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms)
.await?
else {
return Ok(RuntimeWorkspaceStatsRefreshResult {
@@ -46,7 +45,7 @@ impl BackendRuntime {
.await;
release_workspace_stats_lease(self, lease).await?;
result
Ok(result?)
}
#[napi]
@@ -56,13 +55,13 @@ impl BackendRuntime {
batch_limit: i64,
owner: String,
lease_ttl_ms: i64,
) -> Result<RuntimeWorkspaceStatsRecalibrationResult> {
) -> napi::Result<RuntimeWorkspaceStatsRecalibrationResult> {
if batch_limit <= 0 {
return Err(napi_error("workspace stats recalibration limit must be positive"));
}
let Some(lease) = self
.acquire_coordination_lease(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms)
.acquire_coordination_lease_inner(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms)
.await?
else {
return Ok(RuntimeWorkspaceStatsRecalibrationResult {
@@ -80,7 +79,7 @@ impl BackendRuntime {
.await;
release_workspace_stats_lease(self, lease).await?;
result
Ok(result?)
}
#[napi]
@@ -88,9 +87,9 @@ impl BackendRuntime {
&self,
owner: String,
lease_ttl_ms: i64,
) -> Result<RuntimeWorkspaceStatsSnapshotResult> {
) -> napi::Result<RuntimeWorkspaceStatsSnapshotResult> {
let Some(lease) = self
.acquire_coordination_lease(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms)
.acquire_coordination_lease_inner(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms)
.await?
else {
return Ok(RuntimeWorkspaceStatsSnapshotResult {
@@ -107,7 +106,7 @@ impl BackendRuntime {
.await;
release_workspace_stats_lease(self, lease).await?;
result
Ok(result?)
}
#[napi]
@@ -118,7 +117,7 @@ impl BackendRuntime {
lease_ttl_ms: i64,
lock_retry_times: i64,
lock_retry_delay_ms: i64,
) -> Result<RuntimeWorkspaceStatsDailyRecalibrationResult> {
) -> napi::Result<RuntimeWorkspaceStatsDailyRecalibrationResult> {
if batch_limit <= 0 {
return Err(napi_error("workspace stats daily recalibration limit must be positive"));
}
@@ -150,7 +149,7 @@ impl BackendRuntime {
});
};
let result = async {
let result: RuntimeResult<RuntimeWorkspaceStatsDailyRecalibrationResult> = async {
let store = WorkspaceStatsStore::new(self.pool().await?);
let mut processed = 0;
let mut last_sid = 0;
@@ -195,7 +194,7 @@ impl BackendRuntime {
.await;
release_workspace_stats_lease(self, lease).await?;
result
Ok(result?)
}
}
@@ -214,16 +213,16 @@ impl WorkspaceStatsStore {
Self { pool }
}
async fn refresh_dirty(&self, batch_limit: i64) -> Result<RuntimeWorkspaceStatsRefreshResult> {
async fn refresh_dirty(&self, batch_limit: i64) -> RuntimeResult<RuntimeWorkspaceStatsRefreshResult> {
let mut tx = self
.pool
.begin()
.await
.map_err(|err| napi_error(format!("WorkspaceStats dirty refresh transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats dirty refresh transaction failed", err))?;
if !try_transaction_lock(&mut tx).await? {
tx.commit()
.await
.map_err(|err| napi_error(format!("WorkspaceStats dirty refresh commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats dirty refresh commit failed", err))?;
return Ok(RuntimeWorkspaceStatsRefreshResult {
processed: 0,
backlog: 0,
@@ -236,7 +235,7 @@ impl WorkspaceStatsStore {
if dirty.is_empty() {
tx.commit()
.await
.map_err(|err| napi_error(format!("WorkspaceStats dirty refresh commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats dirty refresh commit failed", err))?;
return Ok(RuntimeWorkspaceStatsRefreshResult {
processed: 0,
backlog,
@@ -248,7 +247,7 @@ impl WorkspaceStatsStore {
clear_dirty(&mut tx, &dirty).await?;
tx.commit()
.await
.map_err(|err| napi_error(format!("WorkspaceStats dirty refresh commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats dirty refresh commit failed", err))?;
Ok(RuntimeWorkspaceStatsRefreshResult {
processed: dirty.len() as i64,
@@ -257,16 +256,20 @@ impl WorkspaceStatsStore {
})
}
async fn recalibrate(&self, last_sid: i64, batch_limit: i64) -> Result<RuntimeWorkspaceStatsRecalibrationResult> {
async fn recalibrate(
&self,
last_sid: i64,
batch_limit: i64,
) -> RuntimeResult<RuntimeWorkspaceStatsRecalibrationResult> {
let mut tx = self
.pool
.begin()
.await
.map_err(|err| napi_error(format!("WorkspaceStats recalibration transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats recalibration transaction failed", err))?;
if !try_transaction_lock(&mut tx).await? {
tx.commit()
.await
.map_err(|err| napi_error(format!("WorkspaceStats recalibration commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats recalibration commit failed", err))?;
return Ok(RuntimeWorkspaceStatsRecalibrationResult {
processed: 0,
last_sid,
@@ -278,7 +281,7 @@ impl WorkspaceStatsStore {
if workspaces.is_empty() {
tx.commit()
.await
.map_err(|err| napi_error(format!("WorkspaceStats recalibration commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats recalibration commit failed", err))?;
return Ok(RuntimeWorkspaceStatsRecalibrationResult {
processed: 0,
last_sid,
@@ -297,7 +300,7 @@ impl WorkspaceStatsStore {
upsert_stats(&mut tx, &ids).await?;
tx.commit()
.await
.map_err(|err| napi_error(format!("WorkspaceStats recalibration commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats recalibration commit failed", err))?;
Ok(RuntimeWorkspaceStatsRecalibrationResult {
processed: ids.len() as i64,
@@ -306,16 +309,16 @@ impl WorkspaceStatsStore {
})
}
async fn write_daily_snapshot(&self) -> Result<RuntimeWorkspaceStatsSnapshotResult> {
async fn write_daily_snapshot(&self) -> RuntimeResult<RuntimeWorkspaceStatsSnapshotResult> {
let mut tx = self
.pool
.begin()
.await
.map_err(|err| napi_error(format!("WorkspaceStats daily snapshot transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats daily snapshot transaction failed", err))?;
if !try_transaction_lock(&mut tx).await? {
tx.commit()
.await
.map_err(|err| napi_error(format!("WorkspaceStats daily snapshot commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats daily snapshot commit failed", err))?;
return Ok(RuntimeWorkspaceStatsSnapshotResult {
snapshotted: 0,
skipped: true,
@@ -324,7 +327,7 @@ impl WorkspaceStatsStore {
let snapshotted = write_daily_snapshot(&mut tx).await?;
tx.commit()
.await
.map_err(|err| napi_error(format!("WorkspaceStats daily snapshot commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats daily snapshot commit failed", err))?;
Ok(RuntimeWorkspaceStatsSnapshotResult {
snapshotted,
@@ -333,9 +336,9 @@ impl WorkspaceStatsStore {
}
}
async fn release_workspace_stats_lease(runtime: &BackendRuntime, lease: CoordinationLeaseGrant) -> Result<()> {
async fn release_workspace_stats_lease(runtime: &BackendRuntime, lease: CoordinationLeaseGrant) -> RuntimeResult<()> {
let _ = runtime
.release_coordination_lease(lease.key, lease.owner, lease.fencing_token)
.release_coordination_lease_inner(lease.key, lease.owner, lease.fencing_token)
.await?;
Ok(())
}
@@ -346,10 +349,10 @@ async fn acquire_workspace_stats_lease_with_retry(
lease_ttl_ms: i64,
retry_times: i64,
retry_delay_ms: i64,
) -> Result<Option<CoordinationLeaseGrant>> {
) -> RuntimeResult<Option<CoordinationLeaseGrant>> {
for attempt in 0..retry_times {
let lease = runtime
.acquire_coordination_lease(WORKSPACE_STATS_LEASE_KEY.to_string(), owner.clone(), lease_ttl_ms)
.acquire_coordination_lease_inner(WORKSPACE_STATS_LEASE_KEY.to_string(), owner.clone(), lease_ttl_ms)
.await?;
if lease.is_some() {
return Ok(lease);
@@ -367,11 +370,11 @@ async fn retry_workspace_stats_operation<T, F, Fut>(
retry_times: i64,
retry_delay_ms: i64,
mut operation: F,
) -> Result<T>
) -> RuntimeResult<T>
where
T: WorkspaceStatsSkippable,
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
Fut: std::future::Future<Output = RuntimeResult<T>>,
{
for attempt in 0..retry_times {
let result = operation().await?;
@@ -403,7 +406,7 @@ impl WorkspaceStatsSkippable for RuntimeWorkspaceStatsSnapshotResult {
}
}
async fn try_transaction_lock(tx: &mut Transaction<'_, Postgres>) -> Result<bool> {
async fn try_transaction_lock(tx: &mut Transaction<'_, Postgres>) -> RuntimeResult<bool> {
let row = sqlx::query(
r#"
SELECT pg_try_advisory_xact_lock(($1::bigint << 32) + $2::bigint) AS locked
@@ -413,12 +416,12 @@ async fn try_transaction_lock(tx: &mut Transaction<'_, Postgres>) -> Result<bool
.bind(WORKSPACE_STATS_REFRESH_LOCK_KEY)
.fetch_one(&mut **tx)
.await
.map_err(|err| napi_error(format!("WorkspaceStats transaction lock failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats transaction lock failed", err))?;
Ok(row.get::<bool, _>("locked"))
}
async fn load_dirty(tx: &mut Transaction<'_, Postgres>, limit: i64) -> Result<Vec<String>> {
async fn load_dirty(tx: &mut Transaction<'_, Postgres>, limit: i64) -> RuntimeResult<Vec<String>> {
let rows = sqlx::query(
r#"
SELECT workspace_id
@@ -431,20 +434,20 @@ async fn load_dirty(tx: &mut Transaction<'_, Postgres>, limit: i64) -> Result<Ve
.bind(limit)
.fetch_all(&mut **tx)
.await
.map_err(|err| napi_error(format!("WorkspaceStats load dirty workspaces failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats load dirty workspaces failed", err))?;
Ok(rows.into_iter().map(|row| row.get("workspace_id")).collect())
}
async fn count_dirty(tx: &mut Transaction<'_, Postgres>) -> Result<i64> {
async fn count_dirty(tx: &mut Transaction<'_, Postgres>) -> RuntimeResult<i64> {
let row = sqlx::query("SELECT COUNT(*) AS total FROM workspace_admin_stats_dirty")
.fetch_one(&mut **tx)
.await
.map_err(|err| napi_error(format!("WorkspaceStats count dirty workspaces failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats count dirty workspaces failed", err))?;
Ok(row.get::<i64, _>("total"))
}
async fn clear_dirty(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String]) -> Result<()> {
async fn clear_dirty(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String]) -> RuntimeResult<()> {
sqlx::query(
r#"
DELETE FROM workspace_admin_stats_dirty
@@ -454,11 +457,11 @@ async fn clear_dirty(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String
.bind(workspace_ids)
.execute(&mut **tx)
.await
.map_err(|err| napi_error(format!("WorkspaceStats clear dirty workspaces failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats clear dirty workspaces failed", err))?;
Ok(())
}
async fn upsert_stats(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String]) -> Result<()> {
async fn upsert_stats(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String]) -> RuntimeResult<()> {
if workspace_ids.is_empty() {
return Ok(());
}
@@ -467,7 +470,7 @@ async fn upsert_stats(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[Strin
.bind(workspace_ids)
.execute(&mut **tx)
.await
.map_err(|err| napi_error(format!("WorkspaceStats upsert stats failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats upsert stats failed", err))?;
Ok(())
}
@@ -475,7 +478,7 @@ async fn fetch_workspace_batch(
tx: &mut Transaction<'_, Postgres>,
last_sid: i64,
limit: i64,
) -> Result<Vec<WorkspaceSid>> {
) -> RuntimeResult<Vec<WorkspaceSid>> {
sqlx::query_as::<_, WorkspaceSid>(
r#"
SELECT id, sid
@@ -489,10 +492,10 @@ async fn fetch_workspace_batch(
.bind(limit)
.fetch_all(&mut **tx)
.await
.map_err(|err| napi_error(format!("WorkspaceStats fetch workspace batch failed: {err}")))
.map_err(|err| RuntimeError::database("WorkspaceStats fetch workspace batch failed", err))
}
async fn write_daily_snapshot(tx: &mut Transaction<'_, Postgres>) -> Result<i64> {
async fn write_daily_snapshot(tx: &mut Transaction<'_, Postgres>) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
INSERT INTO workspace_admin_stats_daily (
@@ -521,7 +524,7 @@ async fn write_daily_snapshot(tx: &mut Transaction<'_, Postgres>) -> Result<i64>
)
.execute(&mut **tx)
.await
.map_err(|err| napi_error(format!("WorkspaceStats daily snapshot failed: {err}")))?;
.map_err(|err| RuntimeError::database("WorkspaceStats daily snapshot failed", err))?;
Ok(result.rows_affected() as i64)
}
packages/backend/native/src/backend_runtime/config.rs → packages/backend/native/src/runtime/config.rs
+21 -78
View File
@@ -1,43 +1,35 @@
use std::{
collections::HashMap,
env, fs,
path::{Path, PathBuf},
};
use napi::Result;
use serde::Deserialize;
use serde_json::{Map, Value};
use serde_json::Map;
use sqlx::{PgPool, Row};
use super::{
error::napi_error,
object_storage::{ObjectStorageConfig, StorageProviderConfig},
};
use super::{RuntimeError, RuntimeResult};
#[derive(Clone, Debug)]
pub(super) struct RuntimeConfig {
pub(super) database_url: String,
pub(super) storage: Option<ObjectStorageConfig>,
pub(crate) struct BackendRuntimeConfig {
pub(crate) database_url: String,
}
impl RuntimeConfig {
pub(super) fn from_config_files() -> Result<Self> {
impl BackendRuntimeConfig {
pub(crate) fn from_config_files() -> RuntimeResult<Self> {
let app_config = app_config_from_config_files()?;
let database_url = database_url_from_env()
.or(app_config.database_url())
.unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string());
let storage = ObjectStorageConfig::from_provider_config(app_config.blob_storage_provider_config()?)?;
Ok(Self { database_url, storage })
Ok(Self { database_url })
}
pub(super) async fn with_db_overrides(&self, pool: &PgPool) -> Result<Self> {
pub(crate) async fn with_db_overrides(&self, pool: &PgPool) -> RuntimeResult<Self> {
let mut app_config = app_config_from_config_files()?;
app_config.apply_file_config(load_app_config_overrides_from_db(pool).await?);
Ok(Self {
// The DB override is loaded after this connection already exists, so it
// must not rewrite the active datasource URL.
database_url: self.database_url.clone(),
storage: ObjectStorageConfig::from_provider_config(app_config.blob_storage_provider_config()?)?,
})
}
}
@@ -45,8 +37,6 @@ impl RuntimeConfig {
#[derive(Debug, Default, Deserialize)]
struct AppConfigFile {
db: Option<DbConfigFile>,
#[serde(default)]
storages: Option<HashMap<String, Value>>,
}
#[derive(Debug, Default, Deserialize)]
@@ -63,16 +53,6 @@ impl AppConfigFile {
.and_then(|db| db.datasource_url.clone())
.and_then(non_empty_string)
}
fn blob_storage_provider_config(&self) -> Result<Option<StorageProviderConfig>> {
self
.storages
.as_ref()
.and_then(|storages| storages.get("blob.storage").cloned())
.map(serde_json::from_value)
.transpose()
.map_err(|err| napi_error(format!("invalid blob storage config: {err}")))
}
}
fn database_url_from_env() -> Option<String> {
@@ -83,16 +63,15 @@ fn non_empty_string(value: String) -> Option<String> {
if value.trim().is_empty() { None } else { Some(value) }
}
fn app_config_from_config_files() -> Result<AppConfigFile> {
fn app_config_from_config_files() -> RuntimeResult<AppConfigFile> {
let mut merged = AppConfigFile::default();
for path in config_json_paths() {
if !path.exists() {
continue;
}
let raw = fs::read_to_string(&path)
.map_err(|err| napi_error(format!("failed to read config file {}: {err}", path.display())))?;
let config: AppConfigFile = serde_json::from_str(&raw)
.map_err(|err| napi_error(format!("failed to parse config file {}: {err}", path.display())))?;
let raw = fs::read_to_string(&path).map_err(|err| RuntimeError::io("failed to read config file", err))?;
let config: AppConfigFile =
serde_json::from_str(&raw).map_err(|err| RuntimeError::json("failed to parse config file", err))?;
merged.apply_file_config(config);
}
@@ -104,19 +83,14 @@ impl AppConfigFile {
if config.db.is_some() {
self.db = config.db;
}
if let Some(storages) = config.storages
&& !storages.is_empty()
{
self.storages.get_or_insert_with(HashMap::new).extend(storages);
}
}
}
async fn load_app_config_overrides_from_db(pool: &PgPool) -> Result<AppConfigFile> {
async fn load_app_config_overrides_from_db(pool: &PgPool) -> RuntimeResult<AppConfigFile> {
let rows = match sqlx::query("SELECT id, value FROM app_configs").fetch_all(pool).await {
Ok(rows) => rows,
Err(sqlx::Error::Database(err)) if err.code().as_deref() == Some("42P01") => return Ok(AppConfigFile::default()),
Err(err) => return Err(napi_error(format!("failed to load app config overrides: {err}"))),
Err(err) => return Err(RuntimeError::database("failed to load app config overrides", err)),
};
app_config_from_flat_overrides(rows.into_iter().map(|row| {
@@ -126,7 +100,7 @@ async fn load_app_config_overrides_from_db(pool: &PgPool) -> Result<AppConfigFil
}))
}
fn app_config_from_flat_overrides<I, S>(rows: I) -> Result<AppConfigFile>
fn app_config_from_flat_overrides<I, S>(rows: I) -> RuntimeResult<AppConfigFile>
where
I: IntoIterator<Item = (S, serde_json::Value)>,
S: AsRef<str>,
@@ -145,7 +119,7 @@ where
}
serde_json::from_value(serde_json::Value::Object(root))
.map_err(|err| napi_error(format!("invalid app config overrides: {err}")))
.map_err(|err| RuntimeError::json("invalid app config overrides", err))
}
pub(super) fn config_json_paths() -> Vec<PathBuf> {
@@ -208,50 +182,19 @@ mod tests {
}
#[test]
fn parses_blob_storage_app_config_value() {
fn ignores_storage_app_config_values() {
let app_config = app_config_from_flat_overrides([
(
"unknown.future.config",
serde_json::json!({
"shape": "ignored"
}),
),
(
"storages.blob.storage",
serde_json::json!({
"provider": "cloudflare-r2",
"bucket": "workspace-blobs-canary",
"config": {
"accountId": "account",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"usePresignedURL": {
"enabled": true
}
}
}),
),
(
"storages.avatar.publicPath",
serde_json::json!("https://avatar.affineassets.com/"),
serde_json::json!({"provider": "cloudflare-r2"}),
),
("db.datasourceUrl", serde_json::json!("postgresql://example/runtime")),
])
.unwrap();
let storage = app_config.blob_storage_provider_config().unwrap().unwrap();
let config = ObjectStorageConfig::from_provider_config(Some(storage))
.unwrap()
.unwrap();
let health = config.health();
assert!(health.configured);
assert_eq!(health.provider.as_deref(), Some("cloudflare-r2"));
assert_eq!(health.bucket.as_deref(), Some("workspace-blobs-canary"));
assert_eq!(
health.endpoint.as_deref(),
Some("https://account.r2.cloudflarestorage.com")
app_config.database_url().as_deref(),
Some("postgresql://example/runtime")
);
assert!(health.use_presigned_url);
}
}
packages/backend/native/src/runtime/error.rs
+126
View File
@@ -0,0 +1,126 @@
use napi::{Error, Status};
use super::storage_runtime::object_storage::error::ObjectStorageError;
pub(crate) type RuntimeResult<T> = std::result::Result<T, RuntimeError>;
#[derive(Debug, thiserror::Error)]
pub(crate) enum RuntimeError {
#[error("{0}")]
Config(String),
#[error("{0}")]
InvalidInput(String),
#[error("{0}")]
InvalidState(String),
#[error("{context}: {source}")]
Database {
context: String,
#[source]
source: sqlx::Error,
},
#[error("{context}: {source}")]
Io {
context: String,
#[source]
source: std::io::Error,
},
#[error("{context}: {source}")]
Json {
context: String,
#[source]
source: serde_json::Error,
},
#[error("{context}: {source}")]
Time {
context: String,
#[source]
source: std::time::SystemTimeError,
},
#[error(transparent)]
ObjectStorage(#[from] ObjectStorageError),
#[error("{0}")]
NapiBoundary(String),
}
impl RuntimeError {
pub(crate) fn config(message: impl Into<String>) -> Self {
Self::Config(message.into())
}
pub(crate) fn invalid_input(message: impl Into<String>) -> Self {
Self::InvalidInput(message.into())
}
pub(crate) fn invalid_state(message: impl Into<String>) -> Self {
Self::InvalidState(message.into())
}
pub(crate) fn database(context: impl Into<String>, source: sqlx::Error) -> Self {
Self::Database {
context: context.into(),
source,
}
}
pub(crate) fn io(context: impl Into<String>, source: std::io::Error) -> Self {
Self::Io {
context: context.into(),
source,
}
}
pub(crate) fn json(context: impl Into<String>, source: serde_json::Error) -> Self {
Self::Json {
context: context.into(),
source,
}
}
pub(crate) fn is_object_missing(&self) -> bool {
match self {
Self::ObjectStorage(error) => error.is_not_found(),
Self::Io { source, .. } => source.kind() == std::io::ErrorKind::NotFound,
Self::InvalidState(message)
| Self::InvalidInput(message)
| Self::Config(message)
| Self::NapiBoundary(message) => {
message.contains("NoSuchKey") || message.contains("NotFound") || message.contains("not found")
}
_ => false,
}
}
}
pub(crate) fn to_napi_error(error: RuntimeError) -> Error {
Error::new(Status::GenericFailure, error.to_string())
}
impl From<RuntimeError> for Error {
fn from(error: RuntimeError) -> Self {
to_napi_error(error)
}
}
impl From<ObjectStorageError> for Error {
fn from(error: ObjectStorageError) -> Self {
to_napi_error(RuntimeError::from(error))
}
}
impl From<Error> for RuntimeError {
fn from(error: Error) -> Self {
Self::NapiBoundary(error.to_string())
}
}
pub(crate) fn napi_error(message: impl Into<String>) -> Error {
Error::new(Status::GenericFailure, message.into())
}
packages/backend/native/src/runtime/migrations.rs
+20
View File
@@ -0,0 +1,20 @@
use sqlx::PgPool;
use super::{RuntimeError, RuntimeResult};
pub(crate) const RUNTIME_MIGRATIONS: &str = include_str!("sql/runtime_migrations.sql");
pub(crate) async fn migrate_runtime_tables(pool: &PgPool) -> RuntimeResult<()> {
for statement in RUNTIME_MIGRATIONS
.split(';')
.map(str::trim)
.filter(|statement| !statement.is_empty())
{
sqlx::query(statement)
.execute(pool)
.await
.map_err(|err| RuntimeError::database("Runtime migration failed", err))?;
}
Ok(())
}
packages/backend/native/src/runtime/mod.rs
+10
View File
@@ -0,0 +1,10 @@
pub mod backend_runtime;
pub mod storage_runtime;
pub(crate) mod config;
pub(crate) mod error;
pub(crate) mod migrations;
pub(crate) mod types;
pub(crate) use config::BackendRuntimeConfig;
pub(crate) use error::{RuntimeError, RuntimeResult, napi_error, to_napi_error};
packages/backend/native/src/backend_runtime/sql/runtime_migrations.sql → packages/backend/native/src/runtime/sql/runtime_migrations.sql
View File
packages/backend/native/src/runtime/storage_runtime/assetpack.rs
+358
View File
@@ -0,0 +1,358 @@
use std::{
io::{BufReader, Cursor},
path::PathBuf,
time::SystemTime,
};
use assetpack_core::{
Codec, FileHint, FileTransformConfig, Hash32, ObjectKind, Pipeline, PipelineConfig, SqliteStore, TransformRegistry,
TransformSelector, build_recipe, pack::ObjectRecord, parse_recipe_checked,
};
use sqlx::Row;
use super::{
FsStorageConfig, MAX_BLOB_SIZE, ObjectGetResult, ObjectListEntry, ObjectMetadata, ObjectPutMetadata, RuntimeError,
RuntimeResult, fs_bucket_path, normalize_storage_key, system_time_ms,
};
pub(super) async fn put(
config: &FsStorageConfig,
scope: &str,
key: &str,
body: Vec<u8>,
metadata: ObjectPutMetadata,
) -> RuntimeResult<ObjectMetadata> {
normalize_storage_key(key)?;
let metadata = metadata.complete_for_body(&body);
let content_length = metadata.content_length.unwrap_or(body.len() as i64);
if content_length != body.len() as i64 {
return Err(RuntimeError::invalid_input(
"Assetpack contentLength does not match body length",
));
}
if !(0..=MAX_BLOB_SIZE).contains(&content_length) {
return Err(RuntimeError::invalid_input(
"Assetpack contentLength exceeds supported blob size",
));
}
let store = open_store(config).await?;
let transform_config = FileTransformConfig::default();
let bucket_path = fs_bucket_path(config);
let selector = TransformSelector::new(
transform_config.clone(),
transform_config.resolved_temp_dir(&bucket_path),
assetpack_transform_precomp2::default_specs(),
);
let original_hash = Hash32::sha3_256(&body);
let hint = FileHint {
size: body.len() as u64,
extension: extension_from_key(key),
head: Some(body.iter().take(4096).copied().collect()),
};
let plan = Pipeline::new(PipelineConfig::default())
.run(body, &hint, original_hash, Some(&selector))
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack pipeline failed: {err}")))?;
let chunks = plan
.chunks
.iter()
.map(|chunk| (chunk.hash, chunk.raw_len))
.collect::<Vec<_>>();
let recipe = build_recipe(
plan.original_size,
&chunks,
plan.original_hash,
plan.transform_id,
plan.transform_version,
);
let recipe_hash = Hash32::sha3_256(&recipe);
let mut objects = Vec::with_capacity(plan.chunks.len() + 1);
for chunk in plan.chunks {
let Some(payload) = chunk.payload else {
return Err(RuntimeError::invalid_state(
"Assetpack pipeline unexpectedly discarded chunk payload",
));
};
objects.push(ObjectRecord {
hash: chunk.hash,
kind: ObjectKind::Chunk,
size: chunk.raw_len as u64,
codec: chunk.codec,
content: payload,
});
}
objects.push(ObjectRecord {
hash: recipe_hash,
kind: ObjectKind::Recipe,
size: recipe.len() as u64,
codec: Codec::Raw,
content: recipe,
});
let mut tx = store
.begin_write_tx()
.await
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack begin write failed: {err}")))?;
store
.put_objects_batch_tx(&mut tx, &objects)
.await
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack object write failed: {err}")))?;
store
.put_file_recipe_cache_batch_tx(&mut tx, &[(original_hash, recipe_hash)])
.await
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack recipe cache write failed: {err}")))?;
let object_metadata = ObjectMetadata {
content_type: metadata
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string()),
content_length,
last_modified_ms: system_time_ms(SystemTime::now())?,
checksum_crc32: metadata.checksum_crc32,
};
sqlx::query(
r#"
INSERT INTO storage_assetpack_blobs
(scope, key, recipe_hash, content_type, content_length, checksum_crc32, last_modified_ms)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
ON CONFLICT (scope, key)
DO UPDATE SET
recipe_hash = excluded.recipe_hash,
content_type = excluded.content_type,
content_length = excluded.content_length,
checksum_crc32 = excluded.checksum_crc32,
last_modified_ms = excluded.last_modified_ms
"#,
)
.bind(scope)
.bind(key)
.bind(recipe_hash.to_hex())
.bind(&object_metadata.content_type)
.bind(object_metadata.content_length)
.bind(&object_metadata.checksum_crc32)
.bind(object_metadata.last_modified_ms)
.execute(&mut *tx)
.await
.map_err(|err| RuntimeError::database("Assetpack manifest write failed", err))?;
tx.commit()
.await
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack commit failed: {err}")))?;
Ok(object_metadata)
}
pub(super) async fn head(config: &FsStorageConfig, scope: &str, key: &str) -> RuntimeResult<Option<ObjectMetadata>> {
normalize_storage_key(key)?;
let store = open_store(config).await?;
manifest_row(&store, scope, key)
.await
.map(|row| row.map(|row| row.metadata))
}
pub(super) async fn get(config: &FsStorageConfig, scope: &str, key: &str) -> RuntimeResult<Option<ObjectGetResult>> {
normalize_storage_key(key)?;
let store = open_store(config).await?;
let Some(row) = manifest_row(&store, scope, key).await? else {
return Ok(None);
};
let recipe_hash = Hash32::from_hex(&row.recipe_hash)
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack manifest recipe hash is invalid: {err}")))?;
let Some(recipe_object) = store
.get_object(&recipe_hash)
.await
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack recipe read failed: {err}")))?
else {
return Err(RuntimeError::invalid_state(format!(
"Assetpack recipe object is missing for {key}"
)));
};
let recipe = parse_recipe_checked(&recipe_object.content, &recipe_hash)
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack recipe parse failed: {err}")))?;
let mut stored_stream = Vec::with_capacity(recipe.stored_stream_size as usize);
for (chunk_hash, expected_len) in &recipe.chunks {
let Some(chunk) = store
.get_object(chunk_hash)
.await
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack chunk read failed: {err}")))?
else {
return Err(RuntimeError::invalid_state(format!(
"Assetpack chunk is missing for {key}: {chunk_hash}"
)));
};
if chunk.kind != ObjectKind::Chunk || chunk.size != *expected_len as u64 {
return Err(RuntimeError::invalid_state(format!(
"Assetpack chunk metadata mismatch for {key}: {chunk_hash}"
)));
}
stored_stream.extend_from_slice(&chunk.content);
}
let body = decode_stored_stream(recipe.transform_id, stored_stream)?;
if body.len() as u64 != recipe.original_file_size || Hash32::sha3_256(&body) != recipe.original_file_hash {
return Err(RuntimeError::invalid_state(format!(
"Assetpack reconstructed body failed integrity check for {key}"
)));
}
Ok(Some(ObjectGetResult {
body,
metadata: row.metadata,
}))
}
pub(super) async fn list(
config: &FsStorageConfig,
scope: &str,
prefix: Option<String>,
) -> RuntimeResult<Vec<ObjectListEntry>> {
let prefix = prefix
.map(|prefix| super::normalize_storage_prefix(&prefix))
.transpose()?
.unwrap_or_default();
let store = open_store(config).await?;
let rows = sqlx::query(
r#"
SELECT key, content_length, last_modified_ms
FROM storage_assetpack_blobs
WHERE scope = ?1 AND key LIKE ?2 ESCAPE '\'
ORDER BY key ASC
"#,
)
.bind(scope)
.bind(format!("{}%", escape_sqlite_like(&prefix)))
.fetch_all(store.pool())
.await
.map_err(|err| RuntimeError::database("Assetpack manifest list failed", err))?;
rows
.into_iter()
.map(|row| {
Ok(ObjectListEntry {
key: row.get("key"),
content_length: row.get::<i64, _>("content_length"),
last_modified_ms: row.get("last_modified_ms"),
})
})
.collect()
}
fn escape_sqlite_like(value: &str) -> String {
let mut escaped = String::with_capacity(value.len());
for ch in value.chars() {
match ch {
'%' | '_' | '\\' => {
escaped.push('\\');
escaped.push(ch);
}
_ => escaped.push(ch),
}
}
escaped
}
pub(super) async fn delete(config: &FsStorageConfig, scope: &str, key: &str) -> RuntimeResult<()> {
normalize_storage_key(key)?;
let store = open_store(config).await?;
sqlx::query("DELETE FROM storage_assetpack_blobs WHERE scope = ?1 AND key = ?2")
.bind(scope)
.bind(key)
.execute(store.pool())
.await
.map_err(|err| RuntimeError::database("Assetpack manifest delete failed", err))?;
Ok(())
}
async fn open_store(config: &FsStorageConfig) -> RuntimeResult<SqliteStore> {
let store = SqliteStore::open(store_path(config))
.await
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack store open failed: {err}")))?;
ensure_manifest_schema(&store).await?;
Ok(store)
}
fn store_path(config: &FsStorageConfig) -> PathBuf {
fs_bucket_path(config).join("assetpack.sqlite")
}
fn extension_from_key(key: &str) -> Option<String> {
key
.rsplit_once('.')
.and_then(|(_, extension)| (!extension.is_empty()).then(|| extension.to_ascii_lowercase()))
}
struct ManifestRow {
recipe_hash: String,
metadata: ObjectMetadata,
}
async fn ensure_manifest_schema(store: &SqliteStore) -> RuntimeResult<()> {
sqlx::query(
r#"
CREATE TABLE IF NOT EXISTS storage_assetpack_blobs (
scope TEXT NOT NULL,
key TEXT NOT NULL,
recipe_hash TEXT NOT NULL,
content_type TEXT NOT NULL,
content_length INTEGER NOT NULL,
checksum_crc32 TEXT,
last_modified_ms INTEGER NOT NULL,
PRIMARY KEY (scope, key)
)
"#,
)
.execute(store.pool())
.await
.map_err(|err| RuntimeError::database("Assetpack manifest schema create failed", err))?;
sqlx::query(
"CREATE INDEX IF NOT EXISTS storage_assetpack_blobs_scope_prefix_idx ON storage_assetpack_blobs (scope, key)",
)
.execute(store.pool())
.await
.map_err(|err| RuntimeError::database("Assetpack manifest index create failed", err))?;
Ok(())
}
async fn manifest_row(store: &SqliteStore, scope: &str, key: &str) -> RuntimeResult<Option<ManifestRow>> {
let row = sqlx::query(
r#"
SELECT recipe_hash, content_type, content_length, checksum_crc32, last_modified_ms
FROM storage_assetpack_blobs
WHERE scope = ?1 AND key = ?2
"#,
)
.bind(scope)
.bind(key)
.fetch_optional(store.pool())
.await
.map_err(|err| RuntimeError::database("Assetpack manifest read failed", err))?;
row
.map(|row| {
Ok(ManifestRow {
recipe_hash: row.get("recipe_hash"),
metadata: ObjectMetadata {
content_type: row.get("content_type"),
content_length: row.get("content_length"),
checksum_crc32: row.get("checksum_crc32"),
last_modified_ms: row.get("last_modified_ms"),
},
})
})
.transpose()
}
fn decode_stored_stream(transform_id: u16, stored_stream: Vec<u8>) -> RuntimeResult<Vec<u8>> {
let transform_config = FileTransformConfig::default();
let registry = TransformRegistry::new(&transform_config, assetpack_transform_precomp2::default_specs());
let transform = registry
.get(transform_id)
.ok_or_else(|| RuntimeError::invalid_state(format!("Assetpack transform is not registered: {transform_id}")))?;
let mut out = Vec::new();
transform
.decode(&mut BufReader::new(Cursor::new(stored_stream)), &mut out)
.map_err(|err| RuntimeError::invalid_state(format!("Assetpack transform decode failed: {err}")))?;
Ok(out)
}
packages/backend/native/src/backend_runtime/blob_cleanup.rs → packages/backend/native/src/runtime/storage_runtime/blob_cleanup.rs
+44 -81
View File
@@ -1,52 +1,11 @@
use chrono::{DateTime, Duration, Utc};
use napi::Result;
use sqlx::{FromRow, PgPool};
use super::{
BackendRuntime,
error::napi_error,
types::{RuntimeBlobCleanupExecuteResult, RuntimeBlobCleanupPlanResult},
RuntimeBlobCleanupExecuteResult, RuntimeBlobCleanupPlanResult, RuntimeError, RuntimeResult, StorageRuntime,
napi_error,
};
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn blob_cleanup_plan_result_keeps_run_id_for_execute() {
let result = RuntimeBlobCleanupPlanResult {
run_id: Some("00000000-0000-0000-0000-000000000000".to_string()),
scanned_blobs: 1,
candidates_marked: 1,
protected_by_doc_refs: 0,
protected_by_metadata: 0,
protected_by_other_refs: 0,
next_cursor: None,
};
assert!(result.run_id.is_some());
assert_eq!(result.candidates_marked, 1);
}
#[test]
fn blob_cleanup_execute_result_tracks_skipped_and_failed_counts() {
let result = RuntimeBlobCleanupExecuteResult {
scanned_candidates: 3,
deleted_objects: 1,
deleted_metadata: 1,
skipped_still_referenced: 1,
failed: 1,
workspace_ids: vec!["workspace".to_string()],
};
assert_eq!(result.scanned_candidates, 3);
assert_eq!(
result.skipped_still_referenced + result.failed + result.deleted_objects,
3
);
}
}
#[derive(FromRow)]
struct BlobCandidateRow {
workspace_id: String,
@@ -66,7 +25,7 @@ fn push_workspace_once(workspace_ids: &mut Vec<String>, workspace_id: &str) {
}
}
async fn checkpoint_completed(pool: &PgPool, kind: &str, scope: &str) -> Result<bool> {
async fn checkpoint_completed(pool: &PgPool, kind: &str, scope: &str) -> RuntimeResult<bool> {
sqlx::query_scalar::<_, bool>(
"SELECT EXISTS(SELECT 1 FROM blob_reconciliation_checkpoints WHERE kind = $1 AND scope = $2 AND status = \
'completed')",
@@ -75,10 +34,10 @@ async fn checkpoint_completed(pool: &PgPool, kind: &str, scope: &str) -> Result<
.bind(scope)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup checkpoint check failed: {err}")))
.map_err(|err| RuntimeError::database("Blob cleanup checkpoint check failed", err))
}
async fn projection_is_stale(pool: &PgPool, workspace_id: &str) -> Result<bool> {
async fn projection_is_stale(pool: &PgPool, workspace_id: &str) -> RuntimeResult<bool> {
let checkpoint_fresh = checkpoint_completed(pool, "doc_blob_refs", workspace_id).await?;
let has_stale_rows = sqlx::query_scalar::<_, bool>(
"SELECT EXISTS(SELECT 1 FROM doc_blob_refs WHERE workspace_id = $1 AND status <> 'fresh')",
@@ -86,11 +45,11 @@ async fn projection_is_stale(pool: &PgPool, workspace_id: &str) -> Result<bool>
.bind(workspace_id)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup projection freshness check failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob cleanup projection freshness check failed", err))?;
Ok(!checkpoint_fresh || has_stale_rows)
}
async fn stale_projection_workspaces(pool: &PgPool, workspace_id: &str) -> Result<Vec<String>> {
async fn stale_projection_workspaces(pool: &PgPool, workspace_id: &str) -> RuntimeResult<Vec<String>> {
if projection_is_stale(pool, workspace_id).await? {
Ok(vec![workspace_id.to_string()])
} else {
@@ -98,11 +57,11 @@ async fn stale_projection_workspaces(pool: &PgPool, workspace_id: &str) -> Resul
}
}
async fn metadata_backfill_is_complete(pool: &PgPool, workspace_id: &str) -> Result<bool> {
async fn metadata_backfill_is_complete(pool: &PgPool, workspace_id: &str) -> RuntimeResult<bool> {
checkpoint_completed(pool, "blob_metadata_backfill", workspace_id).await
}
async fn has_doc_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result<bool> {
async fn has_doc_ref(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeResult<bool> {
sqlx::query_scalar::<_, bool>(
"SELECT EXISTS(SELECT 1 FROM doc_blob_refs WHERE workspace_id = $1 AND blob_key = $2 AND status = 'fresh')",
)
@@ -110,10 +69,10 @@ async fn has_doc_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result<boo
.bind(key)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup doc ref check failed: {err}")))
.map_err(|err| RuntimeError::database("Blob cleanup doc ref check failed", err))
}
async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result<bool> {
async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeResult<bool> {
let required_ref = sqlx::query_scalar::<_, bool>(
r#"
SELECT EXISTS(SELECT 1 FROM workspaces WHERE id = $1 AND avatar_key = $2)
@@ -136,7 +95,7 @@ async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result<b
.bind(key)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup protected ref check failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob cleanup protected ref check failed", err))?;
if required_ref {
return Ok(true);
}
@@ -148,7 +107,7 @@ async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result<b
.bind(key)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup workspace file ref check failed: {err}")))?
.map_err(|err| RuntimeError::database("Blob cleanup workspace file ref check failed", err))?
{
return Ok(true);
}
@@ -160,19 +119,19 @@ async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result<b
.bind(key)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup workspace blob embedding ref check failed: {err}")))?
.map_err(|err| RuntimeError::database("Blob cleanup workspace blob embedding ref check failed", err))?
{
return Ok(true);
}
Ok(false)
}
async fn table_exists(pool: &PgPool, table: &str) -> Result<bool> {
async fn table_exists(pool: &PgPool, table: &str) -> RuntimeResult<bool> {
sqlx::query_scalar::<_, bool>("SELECT to_regclass($1) IS NOT NULL")
.bind(format!("public.{table}"))
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup table existence check failed: {err}")))
.map_err(|err| RuntimeError::database("Blob cleanup table existence check failed", err))
}
async fn load_completed_blobs(
@@ -180,7 +139,7 @@ async fn load_completed_blobs(
workspace_id: &str,
after_key: Option<&str>,
limit: i64,
) -> Result<Vec<BlobCandidateRow>> {
) -> RuntimeResult<Vec<BlobCandidateRow>> {
sqlx::query_as::<_, BlobCandidateRow>(
r#"
SELECT workspace_id, key, size
@@ -198,17 +157,17 @@ async fn load_completed_blobs(
.bind(limit)
.fetch_all(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup load completed blobs failed: {err}")))
.map_err(|err| RuntimeError::database("Blob cleanup load completed blobs failed", err))
}
async fn load_plan_cursor(pool: &PgPool, workspace_id: &str) -> Result<Option<String>> {
async fn load_plan_cursor(pool: &PgPool, workspace_id: &str) -> RuntimeResult<Option<String>> {
let row = sqlx::query_as::<_, (String, serde_json::Value)>(
"SELECT status, cursor FROM blob_reconciliation_checkpoints WHERE kind = 'blob_cleanup_plan' AND scope = $1",
)
.bind(workspace_id)
.fetch_optional(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup plan checkpoint load failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob cleanup plan checkpoint load failed", err))?;
let Some((status, cursor)) = row else {
return Ok(None);
};
@@ -228,7 +187,7 @@ async fn upsert_plan_checkpoint(
workspace_id: &str,
last_blob_key: Option<&str>,
completed: bool,
) -> Result<()> {
) -> RuntimeResult<()> {
let status = if completed { "completed" } else { "running" };
sqlx::query(
r#"
@@ -250,11 +209,11 @@ async fn upsert_plan_checkpoint(
.bind(completed)
.execute(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup plan checkpoint write failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob cleanup plan checkpoint write failed", err))?;
Ok(())
}
async fn create_run(pool: &PgPool, workspace_id: &str) -> Result<String> {
async fn create_run(pool: &PgPool, workspace_id: &str) -> RuntimeResult<String> {
sqlx::query_scalar::<_, String>(
r#"
INSERT INTO blob_reconciliation_runs (kind, mode, status, workspace_id)
@@ -265,7 +224,7 @@ async fn create_run(pool: &PgPool, workspace_id: &str) -> Result<String> {
.bind(workspace_id)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup create run failed: {err}")))
.map_err(|err| RuntimeError::database("Blob cleanup create run failed", err))
}
async fn finish_run(
@@ -274,14 +233,14 @@ async fn finish_run(
workspace_id: &str,
result: &RuntimeBlobCleanupPlanResult,
stale_projection_workspaces: Vec<String>,
) -> Result<()> {
) -> RuntimeResult<()> {
let candidate_bytes = sqlx::query_scalar::<_, Option<i64>>(
"SELECT SUM(object_size)::bigint FROM blob_cleanup_candidates WHERE run_id = $1::uuid AND status = 'marked'",
)
.bind(run_id)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup candidate bytes audit failed: {err}")))?
.map_err(|err| RuntimeError::database("Blob cleanup candidate bytes audit failed", err))?
.unwrap_or(0);
sqlx::query(
r#"
@@ -309,7 +268,7 @@ async fn finish_run(
}))
.execute(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup finish run failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob cleanup finish run failed", err))?;
Ok(())
}
@@ -321,7 +280,7 @@ async fn mark_candidate_status(
status: &str,
evidence: serde_json::Value,
error: Option<&str>,
) -> Result<()> {
) -> RuntimeResult<()> {
sqlx::query(
r#"
UPDATE blob_cleanup_candidates
@@ -340,11 +299,15 @@ async fn mark_candidate_status(
.bind(run_id)
.execute(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup mark candidate status failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob cleanup mark candidate status failed", err))?;
Ok(())
}
async fn finish_execute_run(pool: &PgPool, run_id: &str, result: &RuntimeBlobCleanupExecuteResult) -> Result<()> {
async fn finish_execute_run(
pool: &PgPool,
run_id: &str,
result: &RuntimeBlobCleanupExecuteResult,
) -> RuntimeResult<()> {
sqlx::query(
r#"
UPDATE blob_reconciliation_runs
@@ -369,7 +332,7 @@ async fn finish_execute_run(pool: &PgPool, run_id: &str, result: &RuntimeBlobCle
}))
.execute(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup execute run finish failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob cleanup execute run finish failed", err))?;
Ok(())
}
@@ -379,7 +342,7 @@ async fn mark_candidate(
row: &BlobCandidateRow,
object_size: i64,
object_last_modified: DateTime<Utc>,
) -> Result<i64> {
) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
INSERT INTO blob_cleanup_candidates
@@ -404,11 +367,11 @@ async fn mark_candidate(
.bind(serde_json::json!({ "metadataSize": row.size }))
.execute(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup mark candidate failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob cleanup mark candidate failed", err))?;
Ok(result.rows_affected() as i64)
}
async fn load_marked_candidates(pool: &PgPool, run_id: &str, limit: i64) -> Result<Vec<MarkedCandidateRow>> {
async fn load_marked_candidates(pool: &PgPool, run_id: &str, limit: i64) -> RuntimeResult<Vec<MarkedCandidateRow>> {
sqlx::query_as::<_, MarkedCandidateRow>(
r#"
SELECT workspace_id, blob_key
@@ -422,18 +385,18 @@ async fn load_marked_candidates(pool: &PgPool, run_id: &str, limit: i64) -> Resu
.bind(limit)
.fetch_all(pool)
.await
.map_err(|err| napi_error(format!("Blob cleanup load marked candidates failed: {err}")))
.map_err(|err| RuntimeError::database("Blob cleanup load marked candidates failed", err))
}
#[napi_derive::napi]
impl BackendRuntime {
impl StorageRuntime {
#[napi]
pub async fn plan_unreferenced_workspace_blobs(
&self,
workspace_id: String,
grace_period_days: i64,
limit: i64,
) -> Result<RuntimeBlobCleanupPlanResult> {
) -> napi::Result<RuntimeBlobCleanupPlanResult> {
if limit <= 0 {
return Err(napi_error("blob cleanup plan limit must be positive"));
}
@@ -484,7 +447,7 @@ impl BackendRuntime {
continue;
};
let last_modified = DateTime::<Utc>::from_timestamp_millis(metadata.last_modified_ms)
.ok_or_else(|| napi_error("blob cleanup object last modified is invalid"))?;
.ok_or_else(|| RuntimeError::invalid_state("blob cleanup object last modified is invalid"))?;
if metadata.content_length != row.size as i64 || last_modified > min_last_modified {
result.protected_by_metadata += 1;
continue;
@@ -506,7 +469,7 @@ impl BackendRuntime {
run_id: String,
grace_period_days: i64,
limit: i64,
) -> Result<RuntimeBlobCleanupExecuteResult> {
) -> napi::Result<RuntimeBlobCleanupExecuteResult> {
if limit <= 0 {
return Err(napi_error("blob cleanup execute limit must be positive"));
}
@@ -566,7 +529,7 @@ impl BackendRuntime {
};
if let Some(metadata) = metadata {
let last_modified = DateTime::<Utc>::from_timestamp_millis(metadata.last_modified_ms)
.ok_or_else(|| napi_error("blob cleanup execute object last modified is invalid"))?;
.ok_or_else(|| RuntimeError::invalid_state("blob cleanup execute object last modified is invalid"))?;
if last_modified > min_last_modified {
result.skipped_still_referenced += 1;
mark_candidate_status(
packages/backend/native/src/backend_runtime/blob_reclaimer.rs → packages/backend/native/src/runtime/storage_runtime/blob_reclaimer.rs
+15 -23
View File
@@ -2,7 +2,7 @@ use chrono::{DateTime, Utc};
use napi::Result;
use sqlx::{FromRow, PgPool};
use super::{BackendRuntime, error::napi_error, types::RuntimeBlobCleanupResult};
use super::{RuntimeBlobCleanupResult, RuntimeError, RuntimeResult, StorageRuntime, napi_error};
#[derive(FromRow)]
struct BlobRow {
@@ -20,7 +20,7 @@ impl BlobReclaimerStore {
Self { pool }
}
async fn load_expired_pending(&self, cutoff: DateTime<Utc>, limit: i64) -> Result<Vec<BlobRow>> {
async fn load_expired_pending(&self, cutoff: DateTime<Utc>, limit: i64) -> RuntimeResult<Vec<BlobRow>> {
sqlx::query_as::<_, BlobRow>(
r#"
SELECT workspace_id, key, upload_id
@@ -36,10 +36,10 @@ impl BlobReclaimerStore {
.bind(limit)
.fetch_all(&self.pool)
.await
.map_err(|err| napi_error(format!("BlobReclaimer load pending blobs failed: {err}")))
.map_err(|err| RuntimeError::database("BlobReclaimer load pending blobs failed", err))
}
async fn load_deleted(&self, workspace_id: &str, limit: i64) -> Result<Vec<BlobRow>> {
async fn load_deleted(&self, workspace_id: &str, limit: i64) -> RuntimeResult<Vec<BlobRow>> {
sqlx::query_as::<_, BlobRow>(
r#"
SELECT workspace_id, key, upload_id
@@ -54,10 +54,10 @@ impl BlobReclaimerStore {
.bind(limit)
.fetch_all(&self.pool)
.await
.map_err(|err| napi_error(format!("BlobReclaimer load deleted blobs failed: {err}")))
.map_err(|err| RuntimeError::database("BlobReclaimer load deleted blobs failed", err))
}
async fn delete_pending_metadata(&self, workspace_id: &str, key: &str) -> Result<i64> {
async fn delete_pending_metadata(&self, workspace_id: &str, key: &str) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
DELETE FROM blobs
@@ -70,11 +70,11 @@ impl BlobReclaimerStore {
.bind(key)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("BlobReclaimer delete pending blob metadata failed: {err}")))?;
.map_err(|err| RuntimeError::database("BlobReclaimer delete pending blob metadata failed", err))?;
Ok(result.rows_affected() as i64)
}
async fn delete_released_metadata(&self, workspace_id: &str, key: &str) -> Result<i64> {
async fn delete_released_metadata(&self, workspace_id: &str, key: &str) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
DELETE FROM blobs
@@ -86,31 +86,23 @@ impl BlobReclaimerStore {
.bind(key)
.execute(&self.pool)
.await
.map_err(|err| napi_error(format!("BlobReclaimer delete blob metadata failed: {err}")))?;
.map_err(|err| RuntimeError::database("BlobReclaimer delete blob metadata failed", err))?;
Ok(result.rows_affected() as i64)
}
}
fn object_missing_error(err: &napi::Error) -> bool {
let message = err.to_string();
message.contains("NoSuchKey")
|| message.contains("NoSuchUpload")
|| message.contains("NotFound")
|| message.contains("not found")
}
async fn delete_object_idempotent(runtime: &BackendRuntime, key: &str) -> Result<()> {
async fn delete_object_idempotent(runtime: &StorageRuntime, key: &str) -> RuntimeResult<()> {
match runtime.object_storage_delete_object(key).await {
Ok(()) => Ok(()),
Err(err) if object_missing_error(&err) => Ok(()),
Err(err) if err.is_object_missing() => Ok(()),
Err(err) => Err(err),
}
}
async fn abort_upload_idempotent(runtime: &BackendRuntime, key: &str, upload_id: &str) -> Result<()> {
async fn abort_upload_idempotent(runtime: &StorageRuntime, key: &str, upload_id: &str) -> RuntimeResult<()> {
match runtime.object_storage_abort_upload(key, upload_id).await {
Ok(()) => Ok(()),
Err(err) if object_missing_error(&err) => Ok(()),
Err(err) if err.is_object_missing() => Ok(()),
Err(err) => Err(err),
}
}
@@ -122,7 +114,7 @@ fn push_workspace_once(workspace_ids: &mut Vec<String>, workspace_id: &str) {
}
#[napi_derive::napi]
impl BackendRuntime {
impl StorageRuntime {
#[napi]
pub async fn cleanup_expired_pending_blobs(&self, cutoff_ms: i64, limit: i64) -> Result<RuntimeBlobCleanupResult> {
if limit <= 0 {
@@ -130,7 +122,7 @@ impl BackendRuntime {
}
let cutoff = DateTime::<Utc>::from_timestamp_millis(cutoff_ms)
.ok_or_else(|| napi_error("pending blob cleanup cutoff is invalid"))?;
.ok_or_else(|| RuntimeError::invalid_input("pending blob cleanup cutoff is invalid"))?;
let store = BlobReclaimerStore::new(self.pool().await?);
let rows = store.load_expired_pending(cutoff, limit).await?;
packages/backend/native/src/backend_runtime/blob_reconciliation.rs → packages/backend/native/src/runtime/storage_runtime/blob_reconciliation.rs
+17 -20
View File
@@ -1,28 +1,25 @@
use chrono::{DateTime, Utc};
use napi::Result;
use sqlx::{FromRow, PgPool};
use super::{
BackendRuntime,
error::napi_error,
types::{RuntimeBlobMetadataBackfillResult, RuntimeObjectMetadata},
RuntimeBlobMetadataBackfillResult, RuntimeError, RuntimeObjectMetadata, RuntimeResult, StorageRuntime, napi_error,
};
async fn workspace_exists(pool: &PgPool, workspace_id: &str) -> Result<bool> {
async fn workspace_exists(pool: &PgPool, workspace_id: &str) -> RuntimeResult<bool> {
sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM workspaces WHERE id = $1)")
.bind(workspace_id)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob metadata backfill workspace check failed: {err}")))
.map_err(|err| RuntimeError::database("Blob metadata backfill workspace check failed", err))
}
async fn blob_exists(pool: &PgPool, workspace_id: &str, key: &str) -> Result<bool> {
async fn blob_exists(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeResult<bool> {
sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM blobs WHERE workspace_id = $1 AND key = $2)")
.bind(workspace_id)
.bind(key)
.fetch_one(pool)
.await
.map_err(|err| napi_error(format!("Blob metadata backfill blob check failed: {err}")))
.map_err(|err| RuntimeError::database("Blob metadata backfill blob check failed", err))
}
async fn upsert_blob_metadata(
@@ -30,9 +27,9 @@ async fn upsert_blob_metadata(
workspace_id: &str,
key: &str,
metadata: RuntimeObjectMetadata,
) -> Result<i64> {
) -> RuntimeResult<i64> {
let last_modified = DateTime::<Utc>::from_timestamp_millis(metadata.last_modified_ms)
.ok_or_else(|| napi_error("Blob metadata backfill object last modified is invalid"))?;
.ok_or_else(|| RuntimeError::invalid_state("Blob metadata backfill object last modified is invalid"))?;
let result = sqlx::query(
r#"
INSERT INTO blobs (workspace_id, key, size, mime, status, upload_id, created_at, deleted_at)
@@ -53,7 +50,7 @@ async fn upsert_blob_metadata(
.bind(last_modified)
.execute(pool)
.await
.map_err(|err| napi_error(format!("Blob metadata backfill upsert failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob metadata backfill upsert failed", err))?;
Ok(result.rows_affected() as i64)
}
@@ -86,14 +83,14 @@ impl BackfillCheckpoint {
}
}
async fn load_checkpoint(pool: &PgPool, scope: &str) -> Result<Option<BackfillCheckpoint>> {
async fn load_checkpoint(pool: &PgPool, scope: &str) -> RuntimeResult<Option<BackfillCheckpoint>> {
sqlx::query_as::<_, BackfillCheckpoint>(
"SELECT last_key, cursor FROM blob_reconciliation_checkpoints WHERE kind = 'blob_metadata_backfill' AND scope = $1",
)
.bind(scope)
.fetch_optional(pool)
.await
.map_err(|err| napi_error(format!("Blob metadata backfill checkpoint load failed: {err}")))
.map_err(|err| RuntimeError::database("Blob metadata backfill checkpoint load failed", err))
}
async fn upsert_checkpoint(
@@ -102,7 +99,7 @@ async fn upsert_checkpoint(
last_key: Option<&str>,
continuation_token: Option<&str>,
completed: bool,
) -> Result<()> {
) -> RuntimeResult<()> {
let status = if completed { "completed" } else { "running" };
sqlx::query(
r#"
@@ -131,7 +128,7 @@ async fn upsert_checkpoint(
}))
.execute(pool)
.await
.map_err(|err| napi_error(format!("Blob metadata backfill checkpoint write failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob metadata backfill checkpoint write failed", err))?;
Ok(())
}
@@ -163,18 +160,18 @@ fn push_workspace_once(workspace_ids: &mut Vec<String>, workspace_id: &str) {
}
}
fn checked_list_page_limit(limit: i64) -> Result<i32> {
i32::try_from(limit).map_err(|_| napi_error("blob metadata backfill limit exceeds i32::MAX"))
fn checked_list_page_limit(limit: i64) -> RuntimeResult<i32> {
i32::try_from(limit).map_err(|_| RuntimeError::invalid_input("blob metadata backfill limit exceeds i32::MAX"))
}
#[napi_derive::napi]
impl BackendRuntime {
impl StorageRuntime {
#[napi]
pub async fn backfill_missing_blob_metadata(
&self,
workspace_id: Option<String>,
limit: i64,
) -> Result<RuntimeBlobMetadataBackfillResult> {
) -> napi::Result<RuntimeBlobMetadataBackfillResult> {
if limit <= 0 {
return Err(napi_error("blob metadata backfill limit must be positive"));
}
@@ -269,7 +266,7 @@ impl BackendRuntime {
}))
.execute(&pool)
.await
.map_err(|err| napi_error(format!("Blob metadata backfill run record failed: {err}")))?;
.map_err(|err| RuntimeError::database("Blob metadata backfill run record failed", err))?;
Ok(result)
}
packages/backend/native/src/backend_runtime/doc_blob_refs.rs → packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs
+73 -62
View File
@@ -1,10 +1,9 @@
use affine_common::doc_parser;
use chrono::{DateTime, Utc};
use napi::Result;
use sqlx::{FromRow, PgPool};
use y_octo::Doc;
use super::{BackendRuntime, error::napi_error, types::RuntimeDocBlobRefsResult};
use super::{RuntimeDocBlobRefsResult, RuntimeError, RuntimeResult, StorageRuntime, napi_error};
const PARSER_VERSION: i32 = 1;
@@ -28,7 +27,7 @@ struct ExtractedRef {
flavour: String,
}
async fn load_snapshot(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result<Option<SnapshotRow>> {
async fn load_snapshot(pool: &PgPool, workspace_id: &str, doc_id: &str) -> RuntimeResult<Option<SnapshotRow>> {
sqlx::query_as::<_, SnapshotRow>(
r#"
SELECT workspace_id, guid AS doc_id, blob, updated_at
@@ -40,10 +39,10 @@ async fn load_snapshot(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Resul
.bind(doc_id)
.fetch_optional(pool)
.await
.map_err(|err| napi_error(format!("Doc blob refs load snapshot failed: {err}")))
.map_err(|err| RuntimeError::database("Doc blob refs load snapshot failed", err))
}
async fn load_updates(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result<Vec<UpdateRow>> {
async fn load_updates(pool: &PgPool, workspace_id: &str, doc_id: &str) -> RuntimeResult<Vec<UpdateRow>> {
sqlx::query_as::<_, UpdateRow>(
r#"
SELECT blob, created_at
@@ -56,22 +55,22 @@ async fn load_updates(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result
.bind(doc_id)
.fetch_all(pool)
.await
.map_err(|err| napi_error(format!("Doc blob refs load updates failed: {err}")))
.map_err(|err| RuntimeError::database("Doc blob refs load updates failed", err))
}
fn apply_doc_updates(updates: impl IntoIterator<Item = Vec<u8>>) -> Result<Vec<u8>> {
fn apply_doc_updates(updates: impl IntoIterator<Item = Vec<u8>>) -> RuntimeResult<Vec<u8>> {
let mut doc = Doc::default();
for update in updates {
doc
.apply_update_from_binary_v1(&update)
.map_err(|err| napi_error(format!("Doc blob refs merge failed: {err}")))?;
.map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs merge failed: {err}")))?;
}
doc
.encode_update_v1()
.map_err(|err| napi_error(format!("Doc blob refs encode failed: {err}")))
.map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs encode failed: {err}")))
}
async fn load_current_doc(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result<Option<SnapshotRow>> {
async fn load_current_doc(pool: &PgPool, workspace_id: &str, doc_id: &str) -> RuntimeResult<Option<SnapshotRow>> {
let snapshot = load_snapshot(pool, workspace_id, doc_id).await?;
let updates = load_updates(pool, workspace_id, doc_id).await?;
if snapshot.is_none() && updates.is_empty() {
@@ -99,12 +98,12 @@ async fn load_current_doc(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Re
}))
}
async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> Result<Vec<String>> {
async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeResult<Vec<String>> {
let Some(root) = load_current_doc(pool, workspace_id, workspace_id).await? else {
return Ok(Vec::new());
};
let ids = doc_parser::get_doc_ids_from_binary(root.blob, false)
.map_err(|err| napi_error(format!("Doc blob refs root doc parse failed: {err}")))?;
.map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs root doc parse failed: {err}")))?;
let mut ids = ids;
ids.sort();
Ok(ids)
@@ -114,7 +113,7 @@ async fn upsert_projection_checkpoint(
pool: &PgPool,
workspace_id: &str,
result: &RuntimeDocBlobRefsResult,
) -> Result<()> {
) -> RuntimeResult<()> {
let completed = result.next_cursor.is_none();
let status = if completed && result.failed_docs == 0 {
"completed"
@@ -145,11 +144,11 @@ async fn upsert_projection_checkpoint(
}))
.execute(pool)
.await
.map_err(|err| napi_error(format!("Doc blob refs checkpoint write failed: {err}")))?;
.map_err(|err| RuntimeError::database("Doc blob refs checkpoint write failed", err))?;
Ok(())
}
async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str, error: &str) -> Result<()> {
async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str, error: &str) -> RuntimeResult<()> {
sqlx::query(
r#"
INSERT INTO blob_reconciliation_checkpoints
@@ -170,18 +169,18 @@ async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str,
}))
.execute(pool)
.await
.map_err(|err| napi_error(format!("Doc blob refs failure checkpoint write failed: {err}")))?;
.map_err(|err| RuntimeError::database("Doc blob refs failure checkpoint write failed", err))?;
Ok(())
}
async fn load_projection_cursor(pool: &PgPool, workspace_id: &str) -> Result<Option<String>> {
async fn load_projection_cursor(pool: &PgPool, workspace_id: &str) -> RuntimeResult<Option<String>> {
let cursor = sqlx::query_scalar::<_, serde_json::Value>(
"SELECT cursor FROM blob_reconciliation_checkpoints WHERE kind = 'doc_blob_refs' AND scope = $1",
)
.bind(workspace_id)
.fetch_optional(pool)
.await
.map_err(|err| napi_error(format!("Doc blob refs checkpoint load failed: {err}")))?;
.map_err(|err| RuntimeError::database("Doc blob refs checkpoint load failed", err))?;
Ok(cursor.and_then(|cursor| {
cursor
.get("lastDocId")
@@ -190,7 +189,7 @@ async fn load_projection_cursor(pool: &PgPool, workspace_id: &str) -> Result<Opt
}))
}
async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_ids: &[String]) -> Result<i64> {
async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_ids: &[String]) -> RuntimeResult<i64> {
let result = sqlx::query(
r#"
DELETE FROM doc_blob_refs
@@ -202,13 +201,13 @@ async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_i
.bind(current_doc_ids)
.execute(pool)
.await
.map_err(|err| napi_error(format!("Doc blob refs purge removed docs failed: {err}")))?;
.map_err(|err| RuntimeError::database("Doc blob refs purge removed docs failed", err))?;
Ok(result.rows_affected() as i64)
}
fn extract_refs(snapshot: &SnapshotRow) -> Result<Vec<ExtractedRef>> {
fn extract_refs(snapshot: &SnapshotRow) -> RuntimeResult<Vec<ExtractedRef>> {
let parsed = doc_parser::parse_doc_from_binary(snapshot.blob.clone(), snapshot.doc_id.clone())
.map_err(|err| napi_error(format!("Doc blob refs parse failed: {err}")))?;
.map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs parse failed: {err}")))?;
let mut refs = Vec::new();
for block in parsed.blocks {
let Some(blob_keys) = block.blob else {
@@ -251,18 +250,18 @@ mod tests {
}
}
async fn replace_doc_refs(pool: &PgPool, snapshot: &SnapshotRow, refs: Vec<ExtractedRef>) -> Result<(i64, i64)> {
async fn replace_doc_refs(pool: &PgPool, snapshot: &SnapshotRow, refs: Vec<ExtractedRef>) -> RuntimeResult<(i64, i64)> {
let mut tx = pool
.begin()
.await
.map_err(|err| napi_error(format!("Doc blob refs transaction failed: {err}")))?;
.map_err(|err| RuntimeError::database("Doc blob refs transaction failed", err))?;
let deleted = sqlx::query("DELETE FROM doc_blob_refs WHERE workspace_id = $1 AND doc_id = $2")
.bind(&snapshot.workspace_id)
.bind(&snapshot.doc_id)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("Doc blob refs delete failed: {err}")))?
.map_err(|err| RuntimeError::database("Doc blob refs delete failed", err))?
.rows_affected() as i64;
let mut written = 0;
@@ -290,18 +289,18 @@ async fn replace_doc_refs(pool: &PgPool, snapshot: &SnapshotRow, refs: Vec<Extra
.bind(PARSER_VERSION)
.execute(&mut *tx)
.await
.map_err(|err| napi_error(format!("Doc blob refs insert failed: {err}")))?
.map_err(|err| RuntimeError::database("Doc blob refs insert failed", err))?
.rows_affected() as i64;
written += affected;
}
tx.commit()
.await
.map_err(|err| napi_error(format!("Doc blob refs transaction commit failed: {err}")))?;
.map_err(|err| RuntimeError::database("Doc blob refs transaction commit failed", err))?;
Ok((written, deleted))
}
async fn mark_doc_failed(pool: &PgPool, workspace_id: &str, doc_id: &str, error: &str) -> Result<()> {
async fn mark_doc_failed(pool: &PgPool, workspace_id: &str, doc_id: &str, error: &str) -> RuntimeResult<()> {
sqlx::query(
r#"
INSERT INTO doc_blob_refs
@@ -319,44 +318,56 @@ async fn mark_doc_failed(pool: &PgPool, workspace_id: &str, doc_id: &str, error:
.bind(error)
.execute(pool)
.await
.map_err(|err| napi_error(format!("Doc blob refs mark failure failed: {err}")))?;
.map_err(|err| RuntimeError::database("Doc blob refs mark failure failed", err))?;
Ok(())
}
#[napi_derive::napi]
impl BackendRuntime {
#[napi]
pub async fn rebuild_doc_blob_refs(&self, workspace_id: String, doc_id: String) -> Result<RuntimeDocBlobRefsResult> {
let pool = self.pool().await?;
let mut result = RuntimeDocBlobRefsResult {
scanned_docs: 1,
parsed_docs: 0,
refs_written: 0,
refs_deleted: 0,
failed_docs: 0,
next_cursor: None,
};
async fn rebuild_doc_blob_refs_inner(
runtime: &StorageRuntime,
workspace_id: String,
doc_id: String,
) -> RuntimeResult<RuntimeDocBlobRefsResult> {
let pool = runtime.pool().await?;
let mut result = RuntimeDocBlobRefsResult {
scanned_docs: 1,
parsed_docs: 0,
refs_written: 0,
refs_deleted: 0,
failed_docs: 0,
next_cursor: None,
};
let Some(snapshot) = load_current_doc(&pool, &workspace_id, &doc_id).await? else {
result.failed_docs = 1;
mark_doc_failed(&pool, &workspace_id, &doc_id, "snapshot_missing").await?;
return Ok(result);
};
let Some(snapshot) = load_current_doc(&pool, &workspace_id, &doc_id).await? else {
result.failed_docs = 1;
mark_doc_failed(&pool, &workspace_id, &doc_id, "snapshot_missing").await?;
return Ok(result);
};
match extract_refs(&snapshot) {
Ok(refs) => {
let (written, deleted) = replace_doc_refs(&pool, &snapshot, refs).await?;
result.parsed_docs = 1;
result.refs_written = written;
result.refs_deleted = deleted;
}
Err(err) => {
result.failed_docs = 1;
mark_doc_failed(&pool, &workspace_id, &doc_id, &err.to_string()).await?;
}
match extract_refs(&snapshot) {
Ok(refs) => {
let (written, deleted) = replace_doc_refs(&pool, &snapshot, refs).await?;
result.parsed_docs = 1;
result.refs_written = written;
result.refs_deleted = deleted;
}
Err(err) => {
result.failed_docs = 1;
mark_doc_failed(&pool, &workspace_id, &doc_id, &err.to_string()).await?;
}
}
Ok(result)
Ok(result)
}
#[napi_derive::napi]
impl StorageRuntime {
#[napi]
pub async fn rebuild_doc_blob_refs(
&self,
workspace_id: String,
doc_id: String,
) -> napi::Result<RuntimeDocBlobRefsResult> {
Ok(rebuild_doc_blob_refs_inner(self, workspace_id, doc_id).await?)
}
#[napi]
@@ -364,7 +375,7 @@ impl BackendRuntime {
&self,
workspace_id: String,
limit: i64,
) -> Result<RuntimeDocBlobRefsResult> {
) -> napi::Result<RuntimeDocBlobRefsResult> {
if limit <= 0 {
return Err(napi_error("doc blob refs rebuild limit must be positive"));
}
@@ -374,7 +385,7 @@ impl BackendRuntime {
Ok(doc_ids) => doc_ids,
Err(err) => {
upsert_projection_failure_checkpoint(&pool, &workspace_id, &err.to_string()).await?;
return Err(err);
return Err(err.into());
}
};
let cursor = load_projection_cursor(&pool, &workspace_id).await?;
@@ -396,7 +407,7 @@ impl BackendRuntime {
let mut last_doc_id = None;
for doc_id in doc_ids.into_iter().take(limit as usize) {
last_doc_id = Some(doc_id.clone());
let result = self.rebuild_doc_blob_refs(workspace_id.clone(), doc_id).await?;
let result = rebuild_doc_blob_refs_inner(self, workspace_id.clone(), doc_id).await?;
total.scanned_docs += result.scanned_docs;
total.parsed_docs += result.parsed_docs;
total.refs_written += result.refs_written;
packages/backend/native/src/runtime/storage_runtime/mod.rs
+1786
View File
File diff suppressed because it is too large Load Diff
packages/backend/native/src/runtime/storage_runtime/object_storage/client.rs
+850
View File
@@ -0,0 +1,850 @@
use std::{
collections::HashMap,
future::Future,
pin::Pin,
time::{Duration, SystemTime},
};
use chrono::{DateTime, FixedOffset};
use reqwest::{
Client as ReqwestClient, Method, StatusCode,
header::{CONTENT_LENGTH, CONTENT_TYPE, ETAG, HeaderMap, HeaderName, HeaderValue, LAST_MODIFIED},
};
use rusty_s3::{
Bucket, Credentials,
actions::{
AbortMultipartUpload, CompleteMultipartUpload, CreateMultipartUpload, DeleteObject, GetObject, HeadObject,
ListObjectsV2, ListParts, PutObject, S3Action, UploadPart,
},
};
use url::Url;
use super::{
error::{ObjectStorageError, ObjectStorageResult},
types::{
MultipartUploadInitResult, MultipartUploadPart, ObjectGetResult, ObjectListEntry, ObjectListPage, ObjectMetadata,
ObjectPutMetadata, PresignedObjectRequest, completed_multipart_parts, trim_etag,
},
};
const DEFAULT_REQUEST_TIMEOUT_MS: u64 = 30_000;
const MAX_MULTIPART_PART_NUMBER: i32 = 10_000;
const MAX_RESPONSE_BODY_BYTES: usize = i32::MAX as usize;
type StorageHttpFuture<'a> = Pin<Box<dyn Future<Output = ObjectStorageResult<StorageHttpResponse>> + Send + 'a>>;
#[derive(Clone)]
struct StorageHttpRequest {
method: Method,
url: Url,
headers: HashMap<String, String>,
body: Option<Vec<u8>>,
max_response_body_bytes: usize,
}
struct StorageHttpResponse {
status: StatusCode,
headers: HeaderMap,
body: Vec<u8>,
}
trait StorageHttpClient: Clone + Send + Sync + 'static {
fn execute(&self, request: StorageHttpRequest) -> StorageHttpFuture<'_>;
}
#[derive(Clone)]
struct ReqwestStorageHttpClient {
client: ReqwestClient,
}
impl ReqwestStorageHttpClient {
fn new(request_timeout_ms: Option<u64>) -> ObjectStorageResult<Self> {
let builder = ReqwestClient::builder().timeout(Duration::from_millis(
request_timeout_ms.unwrap_or(DEFAULT_REQUEST_TIMEOUT_MS),
));
Ok(Self {
client: builder.build().map_err(ObjectStorageError::HttpClientBuild)?,
})
}
}
impl StorageHttpClient for ReqwestStorageHttpClient {
fn execute(&self, request: StorageHttpRequest) -> StorageHttpFuture<'_> {
Box::pin(async move {
let mut builder = self.client.request(request.method, request.url);
for (key, value) in request.headers {
let name =
HeaderName::from_bytes(key.as_bytes()).map_err(|err| ObjectStorageError::InvalidHeader(err.to_string()))?;
let value = HeaderValue::from_str(&value).map_err(|err| ObjectStorageError::InvalidHeader(err.to_string()))?;
builder = builder.header(name, value);
}
if let Some(body) = request.body {
builder = builder.body(body);
}
let mut response = builder.send().await.map_err(ObjectStorageError::HttpRequest)?;
let status = response.status();
let headers = response.headers().clone();
if response
.content_length()
.is_some_and(|length| length > request.max_response_body_bytes as u64)
{
return Err(ObjectStorageError::BodyTooLarge {
limit: request.max_response_body_bytes,
});
}
let mut body = Vec::new();
while let Some(chunk) = response.chunk().await.map_err(ObjectStorageError::HttpRequest)? {
if body.len() + chunk.len() > request.max_response_body_bytes {
return Err(ObjectStorageError::BodyTooLarge {
limit: request.max_response_body_bytes,
});
}
body.extend_from_slice(&chunk);
}
Ok(StorageHttpResponse { status, headers, body })
})
}
}
#[derive(Clone)]
pub(crate) struct ObjectStorageClient {
bucket: Bucket,
credentials: Credentials,
http: ReqwestStorageHttpClient,
presign_expires_in_seconds: u64,
presign_sign_content_type_for_put: bool,
}
impl ObjectStorageClient {
pub(crate) fn new(
bucket: Bucket,
credentials: Credentials,
request_timeout_ms: Option<u64>,
presign_expires_in_seconds: u64,
presign_sign_content_type_for_put: bool,
) -> ObjectStorageResult<Self> {
Ok(Self {
bucket,
credentials,
http: ReqwestStorageHttpClient::new(request_timeout_ms)?,
presign_expires_in_seconds,
presign_sign_content_type_for_put,
})
}
pub(crate) async fn put(
&self,
key: &str,
body: Vec<u8>,
metadata: ObjectPutMetadata,
) -> ObjectStorageResult<ObjectMetadata> {
let metadata = metadata.complete_for_body(&body);
let object_metadata = metadata.clone().into_object_metadata(
crate::utils::system_time_millis(SystemTime::now())
.map(|millis| millis as i64)
.map_err(|err| ObjectStorageError::InvalidInput(format!("system time before unix epoch: {err}")))?,
);
let mut headers = HashMap::from([
("content-type".to_string(), object_metadata.content_type.clone()),
("content-length".to_string(), object_metadata.content_length.to_string()),
]);
if let Some(checksum) = object_metadata.checksum_crc32.clone() {
headers.insert("x-amz-checksum-crc32".to_string(), checksum);
}
let mut action = PutObject::new(&self.bucket, Some(&self.credentials), key);
insert_action_headers(&mut action, &headers);
let response = self
.http
.execute(StorageHttpRequest {
method: Method::PUT,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers,
body: Some(body),
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| operation_error(format!("ObjectStorage put failed for {key}"), source))?;
ensure_success_status(&response, &format!("ObjectStorage put failed for {key}"))?;
Ok(object_metadata)
}
pub(crate) async fn presign_put(
&self,
key: &str,
metadata: ObjectPutMetadata,
) -> ObjectStorageResult<PresignedObjectRequest> {
let content_type = metadata
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string());
let mut headers = HashMap::new();
headers.insert("Content-Type".to_string(), content_type.clone());
if let Some(content_length) = metadata.content_length {
headers.insert("Content-Length".to_string(), content_length.to_string());
}
let mut action = PutObject::new(&self.bucket, Some(&self.credentials), key);
if self.presign_sign_content_type_for_put {
action.headers_mut().insert("content-type", content_type);
}
if let Some(content_length) = metadata.content_length {
action
.headers_mut()
.insert("content-length", content_length.to_string());
}
Ok(PresignedObjectRequest {
url: action.sign(expires_in(self.presign_expires_in_seconds)).to_string(),
headers,
expires_at_ms: expires_at_ms(self.presign_expires_in_seconds)?,
})
}
pub(crate) async fn presign_get(&self, key: &str) -> ObjectStorageResult<PresignedObjectRequest> {
let action = GetObject::new(&self.bucket, Some(&self.credentials), key);
Ok(PresignedObjectRequest {
url: action.sign(expires_in(self.presign_expires_in_seconds)).to_string(),
headers: HashMap::new(),
expires_at_ms: expires_at_ms(self.presign_expires_in_seconds)?,
})
}
pub(crate) async fn create_multipart_upload(
&self,
key: &str,
metadata: ObjectPutMetadata,
) -> ObjectStorageResult<Option<MultipartUploadInitResult>> {
let mut action = CreateMultipartUpload::new(&self.bucket, Some(&self.credentials), key);
if let Some(content_type) = metadata.content_type {
action.headers_mut().insert("content-type", content_type.clone());
let headers = HashMap::from([("content-type".to_string(), content_type)]);
return self.create_multipart_upload_with_headers(key, action, headers).await;
}
self
.create_multipart_upload_with_headers(key, action, HashMap::new())
.await
}
async fn create_multipart_upload_with_headers(
&self,
key: &str,
action: CreateMultipartUpload<'_>,
headers: HashMap<String, String>,
) -> ObjectStorageResult<Option<MultipartUploadInitResult>> {
let response = self
.http
.execute(StorageHttpRequest {
method: Method::POST,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers,
body: None,
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| {
operation_error(
format!("ObjectStorage create multipart upload failed for {key}"),
source,
)
})?;
let body = ensure_success_text(
response,
format!("ObjectStorage create multipart upload failed for {key}"),
)?;
let parsed = CreateMultipartUpload::parse_response(&body).map_err(|source| ObjectStorageError::InvalidXml {
context: format!("ObjectStorage parse multipart upload response failed for {key}"),
source,
})?;
Ok(Some(MultipartUploadInitResult {
upload_id: parsed.upload_id().to_string(),
expires_at_ms: expires_at_ms(self.presign_expires_in_seconds)?,
}))
}
pub(crate) async fn presign_upload_part(
&self,
key: &str,
upload_id: &str,
part_number: i32,
) -> ObjectStorageResult<PresignedObjectRequest> {
let part_number = checked_part_number(part_number)?;
let action = UploadPart::new(&self.bucket, Some(&self.credentials), key, part_number, upload_id);
Ok(PresignedObjectRequest {
url: action.sign(expires_in(self.presign_expires_in_seconds)).to_string(),
headers: HashMap::new(),
expires_at_ms: expires_at_ms(self.presign_expires_in_seconds)?,
})
}
pub(crate) async fn upload_part(
&self,
key: &str,
upload_id: &str,
part_number: i32,
body: Vec<u8>,
content_length: Option<i64>,
) -> ObjectStorageResult<Option<String>> {
let part_number = checked_part_number(part_number)?;
let mut action = UploadPart::new(&self.bucket, Some(&self.credentials), key, part_number, upload_id);
let mut headers = HashMap::new();
if let Some(content_length) = content_length {
action
.headers_mut()
.insert("content-length", content_length.to_string());
headers.insert("content-length".to_string(), content_length.to_string());
}
let response = self
.http
.execute(StorageHttpRequest {
method: Method::PUT,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers,
body: Some(body),
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| operation_error(format!("ObjectStorage upload multipart part failed for {key}"), source))?;
ensure_success_status(
&response,
&format!("ObjectStorage upload multipart part failed for {key}"),
)?;
Ok(response_header(&response.headers, ETAG).as_deref().map(trim_etag))
}
pub(crate) async fn list_multipart_upload_parts(
&self,
key: &str,
upload_id: &str,
) -> ObjectStorageResult<Vec<MultipartUploadPart>> {
let mut parts = Vec::new();
let mut marker = None;
loop {
let mut action = ListParts::new(&self.bucket, Some(&self.credentials), key, upload_id);
if let Some(marker) = marker {
action.set_part_number_marker(marker);
}
let response = self
.http
.execute(StorageHttpRequest {
method: Method::GET,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers: HashMap::new(),
body: None,
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| {
operation_error(
format!("ObjectStorage list multipart upload parts failed for {key}"),
source,
)
})?;
if response.status == StatusCode::NOT_FOUND && is_not_found_body(&response.body) {
return Ok(Vec::new());
}
let body = ensure_success_text(
response,
format!("ObjectStorage list multipart upload parts failed for {key}"),
)?;
let parsed = ListParts::parse_response(&body).map_err(|source| ObjectStorageError::InvalidXml {
context: format!("ObjectStorage parse multipart parts failed for {key}"),
source,
})?;
parts.extend(parsed.parts.into_iter().map(|part| MultipartUploadPart {
part_number: i32::from(part.number),
etag: trim_etag(&part.etag),
}));
let Some(next_marker) = parsed.next_part_number_marker else {
break;
};
marker = Some(next_marker);
}
Ok(parts)
}
pub(crate) async fn complete_multipart_upload(
&self,
key: &str,
upload_id: &str,
parts: Vec<MultipartUploadPart>,
) -> ObjectStorageResult<()> {
let ordered_parts = completed_multipart_parts(parts);
validate_completed_parts(&ordered_parts)?;
let etags = ordered_parts.iter().map(|part| part.etag.as_str());
let action = CompleteMultipartUpload::new(&self.bucket, Some(&self.credentials), key, upload_id, etags);
let url = action.sign(expires_in(self.presign_expires_in_seconds));
let body = complete_multipart_body(&ordered_parts);
let response = self
.http
.execute(StorageHttpRequest {
method: Method::POST,
url,
headers: HashMap::from([("content-type".to_string(), "application/xml".to_string())]),
body: Some(body.into_bytes()),
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| {
operation_error(
format!("ObjectStorage complete multipart upload failed for {key}"),
source,
)
})?;
ensure_success_status(
&response,
&format!("ObjectStorage complete multipart upload failed for {key}"),
)?;
Ok(())
}
pub(crate) async fn abort_multipart_upload(&self, key: &str, upload_id: &str) -> ObjectStorageResult<()> {
let action = AbortMultipartUpload::new(&self.bucket, Some(&self.credentials), key, upload_id);
let response = self
.http
.execute(StorageHttpRequest {
method: Method::DELETE,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers: HashMap::new(),
body: None,
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| operation_error(format!("ObjectStorage abort multipart upload failed for {key}"), source))?;
ensure_success_status(
&response,
&format!("ObjectStorage abort multipart upload failed for {key}"),
)?;
Ok(())
}
pub(crate) async fn head(&self, key: &str) -> ObjectStorageResult<Option<ObjectMetadata>> {
let action = HeadObject::new(&self.bucket, Some(&self.credentials), key);
let response = self
.http
.execute(StorageHttpRequest {
method: Method::HEAD,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers: HashMap::new(),
body: None,
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| operation_error(format!("ObjectStorage head failed for {key}"), source))?;
if response.status == StatusCode::NOT_FOUND {
let get_action = GetObject::new(&self.bucket, Some(&self.credentials), key);
let get_response = self
.http
.execute(StorageHttpRequest {
method: Method::GET,
url: get_action.sign(expires_in(self.presign_expires_in_seconds)),
headers: HashMap::new(),
body: None,
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| operation_error(format!("ObjectStorage head missing check failed for {key}"), source))?;
if get_response.status == StatusCode::NOT_FOUND && is_not_found_body(&get_response.body) {
return Ok(None);
}
ensure_success_status(
&get_response,
&format!("ObjectStorage head missing check failed for {key}"),
)?;
return Ok(Some(metadata_from_headers(&get_response.headers)));
}
ensure_success_status(&response, &format!("ObjectStorage head failed for {key}"))?;
Ok(Some(metadata_from_headers(&response.headers)))
}
pub(crate) async fn get(&self, key: &str) -> ObjectStorageResult<Option<ObjectGetResult>> {
let action = GetObject::new(&self.bucket, Some(&self.credentials), key);
let response = self
.http
.execute(StorageHttpRequest {
method: Method::GET,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers: HashMap::new(),
body: None,
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| operation_error(format!("ObjectStorage get failed for {key}"), source))?;
if response.status == StatusCode::NOT_FOUND && is_not_found_body(&response.body) {
return Ok(None);
}
ensure_success_status(&response, &format!("ObjectStorage get failed for {key}"))?;
let metadata = metadata_from_headers(&response.headers);
Ok(Some(ObjectGetResult {
body: response.body,
metadata,
}))
}
pub(crate) async fn list(&self, prefix: Option<String>) -> ObjectStorageResult<Vec<ObjectListEntry>> {
let mut entries = Vec::new();
let mut token = None;
loop {
let page = self.list_page(prefix.clone(), token, None, 1000).await?;
entries.extend(page.entries);
if let Some(next_token) = page.next_continuation_token {
token = Some(next_token);
} else {
break;
}
}
Ok(entries)
}
pub(crate) async fn list_page(
&self,
prefix: Option<String>,
continuation_token: Option<String>,
start_after: Option<String>,
max_keys: i32,
) -> ObjectStorageResult<ObjectListPage> {
let max_keys = usize::try_from(max_keys)
.map_err(|_| ObjectStorageError::InvalidInput("maxKeys must be positive".to_string()))?;
let mut action = ListObjectsV2::new(&self.bucket, Some(&self.credentials));
action.with_max_keys(max_keys);
if let Some(prefix) = &prefix {
action.with_prefix(prefix.clone());
}
if let Some(continuation_token) = &continuation_token {
action.with_continuation_token(continuation_token.clone());
} else if let Some(start_after) = &start_after {
action.with_start_after(start_after.clone());
}
let response = self
.http
.execute(StorageHttpRequest {
method: Method::GET,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers: HashMap::new(),
body: None,
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| operation_error("ObjectStorage list page failed", source))?;
let body = ensure_success_text(response, "ObjectStorage list page failed".to_string())?;
let parsed = ListObjectsV2::parse_response(&body).map_err(|source| ObjectStorageError::InvalidXml {
context: "ObjectStorage parse list response failed".to_string(),
source,
})?;
Ok(ObjectListPage {
entries: parsed
.contents
.into_iter()
.map(|object| ObjectListEntry {
key: object.key,
content_length: i64::try_from(object.size).unwrap_or(i64::MAX),
last_modified_ms: parse_rfc3339_ms(&object.last_modified),
})
.collect(),
next_continuation_token: parsed.next_continuation_token,
})
}
pub(crate) async fn delete(&self, key: &str) -> ObjectStorageResult<()> {
let action = DeleteObject::new(&self.bucket, Some(&self.credentials), key);
let response = self
.http
.execute(StorageHttpRequest {
method: Method::DELETE,
url: action.sign(expires_in(self.presign_expires_in_seconds)),
headers: HashMap::new(),
body: None,
max_response_body_bytes: MAX_RESPONSE_BODY_BYTES,
})
.await
.map_err(|source| operation_error(format!("ObjectStorage delete failed for {key}"), source))?;
ensure_success_status(&response, &format!("ObjectStorage delete failed for {key}"))?;
Ok(())
}
}
fn insert_action_headers<'a, T: S3Action<'a>>(action: &mut T, headers: &HashMap<String, String>) {
for (key, value) in headers {
action.headers_mut().insert(key.clone(), value.clone());
}
}
fn operation_error(context: impl Into<String>, source: ObjectStorageError) -> ObjectStorageError {
ObjectStorageError::Operation {
context: context.into(),
source: Box::new(source),
}
}
fn ensure_success_text(response: StorageHttpResponse, context: String) -> ObjectStorageResult<String> {
ensure_success_status(&response, &context)?;
String::from_utf8(response.body).map_err(|source| ObjectStorageError::InvalidUtf8 { context, source })
}
fn ensure_success_status(response: &StorageHttpResponse, context: &str) -> ObjectStorageResult<()> {
if response.status.is_success() {
return Ok(());
}
let body = String::from_utf8_lossy(&response.body);
Err(ObjectStorageError::HttpStatus {
context: context.to_string(),
status: response.status,
body: body.to_string(),
})
}
fn is_not_found_body(body: &[u8]) -> bool {
let body = String::from_utf8_lossy(body);
body.contains("<Code>NoSuchKey</Code>")
|| body.contains("<Code>NotFound</Code>")
|| body.contains("<Code>NoSuchUpload</Code>")
}
fn metadata_from_headers(headers: &HeaderMap) -> ObjectMetadata {
ObjectMetadata {
content_type: response_header(headers, CONTENT_TYPE).unwrap_or_else(|| "application/octet-stream".to_string()),
content_length: response_header(headers, CONTENT_LENGTH)
.and_then(|value| value.parse::<i64>().ok())
.unwrap_or(0),
last_modified_ms: response_header(headers, LAST_MODIFIED)
.and_then(|value| DateTime::<FixedOffset>::parse_from_rfc2822(&value).ok())
.map(|value| value.timestamp_millis())
.unwrap_or(0),
checksum_crc32: response_header_name(headers, "x-amz-checksum-crc32"),
}
}
fn response_header(headers: &HeaderMap, name: HeaderName) -> Option<String> {
headers
.get(name)
.and_then(|value| value.to_str().ok())
.map(ToString::to_string)
}
fn response_header_name(headers: &HeaderMap, name: &str) -> Option<String> {
headers
.get(name)
.and_then(|value| value.to_str().ok())
.map(ToString::to_string)
}
fn checked_part_number(part_number: i32) -> ObjectStorageResult<u16> {
if !(1..=MAX_MULTIPART_PART_NUMBER).contains(&part_number) {
return Err(ObjectStorageError::InvalidInput(
"multipart part number must be between 1 and 10000".to_string(),
));
}
Ok(part_number as u16)
}
fn validate_completed_parts(parts: &[MultipartUploadPart]) -> ObjectStorageResult<()> {
for part in parts {
checked_part_number(part.part_number)?;
if part.etag.is_empty() {
return Err(ObjectStorageError::InvalidInput(
"multipart part etag is required".to_string(),
));
}
}
Ok(())
}
fn complete_multipart_body(parts: &[MultipartUploadPart]) -> String {
let mut body = String::from("<CompleteMultipartUpload>");
for part in parts {
body.push_str("<Part><ETag>");
body.push_str(&xml_escape(&part.etag));
body.push_str("</ETag><PartNumber>");
body.push_str(&part.part_number.to_string());
body.push_str("</PartNumber></Part>");
}
body.push_str("</CompleteMultipartUpload>");
body
}
fn xml_escape(value: &str) -> String {
value.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;")
}
fn expires_in(seconds: u64) -> Duration {
Duration::from_secs(seconds)
}
fn expires_at_ms(expires_in_seconds: u64) -> ObjectStorageResult<i64> {
let expires_at = SystemTime::now()
.checked_add(Duration::from_secs(expires_in_seconds))
.ok_or_else(|| ObjectStorageError::InvalidInput("presign expiration overflow".to_string()))?;
crate::utils::system_time_millis(expires_at)
.map(|millis| millis as i64)
.map_err(|err| ObjectStorageError::InvalidInput(format!("system time before unix epoch: {err}")))
}
fn parse_rfc3339_ms(value: &str) -> i64 {
DateTime::parse_from_rfc3339(value)
.map(|value| value.timestamp_millis())
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use reqwest::header::HeaderValue;
use super::*;
#[test]
fn metadata_from_headers_uses_s3_defaults_and_checksum() {
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain"));
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("42"));
headers.insert(LAST_MODIFIED, HeaderValue::from_static("Wed, 21 Oct 2015 07:28:00 GMT"));
headers.insert("x-amz-checksum-crc32", HeaderValue::from_static("checksum"));
let metadata = metadata_from_headers(&headers);
assert_eq!(metadata.content_type, "text/plain");
assert_eq!(metadata.content_length, 42);
assert_eq!(metadata.last_modified_ms, 1_445_412_480_000);
assert_eq!(metadata.checksum_crc32.as_deref(), Some("checksum"));
let defaults = metadata_from_headers(&HeaderMap::new());
assert_eq!(defaults.content_type, "application/octet-stream");
assert_eq!(defaults.content_length, 0);
assert_eq!(defaults.last_modified_ms, 0);
assert!(defaults.checksum_crc32.is_none());
}
#[test]
fn not_found_body_accepts_object_missing_codes_only() {
for body in [
"<Error><Code>NoSuchKey</Code></Error>",
"<Error><Code>NotFound</Code></Error>",
"<Error><Code>NoSuchUpload</Code></Error>",
] {
assert!(is_not_found_body(body.as_bytes()), "{body}");
}
assert!(!is_not_found_body(b""));
assert!(!is_not_found_body(b"<Error><Code>AccessDenied</Code></Error>"));
}
#[test]
fn list_parts_xml_handles_array_single_part_and_pagination() {
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>test</Bucket>
<Key>key</Key>
<UploadId>upload-id</UploadId>
<PartNumberMarker>0</PartNumberMarker>
<NextPartNumberMarker>3</NextPartNumberMarker>
<MaxParts>2</MaxParts>
<IsTruncated>true</IsTruncated>
<Part>
<PartNumber>1</PartNumber>
<LastModified>2010-11-10T20:48:34.000Z</LastModified>
<ETag>"etag-1"</ETag>
<Size>10485760</Size>
</Part>
<Part>
<PartNumber>2</PartNumber>
<LastModified>2010-11-10T20:48:33.000Z</LastModified>
<ETag>etag-2</ETag>
<Size>10485760</Size>
</Part>
</ListPartsResult>"#;
let parsed = ListParts::parse_response(xml).unwrap();
let parts = parsed
.parts
.into_iter()
.map(|part| MultipartUploadPart {
part_number: i32::from(part.number),
etag: trim_etag(&part.etag),
})
.collect::<Vec<_>>();
assert_eq!(
parts,
vec![
MultipartUploadPart {
part_number: 1,
etag: "etag-1".to_string()
},
MultipartUploadPart {
part_number: 2,
etag: "etag-2".to_string()
}
]
);
assert_eq!(parsed.next_part_number_marker, Some(3));
let single = r#"<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>test</Bucket>
<Key>key</Key>
<UploadId>upload-id</UploadId>
<MaxParts>1</MaxParts>
<IsTruncated>false</IsTruncated>
<Part>
<PartNumber>5</PartNumber>
<LastModified>2010-11-10T20:48:34.000Z</LastModified>
<ETag>"etag-5"</ETag>
<Size>10485760</Size>
</Part>
</ListPartsResult>"#;
let parsed = ListParts::parse_response(single).unwrap();
assert_eq!(parsed.parts.len(), 1);
assert_eq!(parsed.parts[0].number, 5);
assert_eq!(trim_etag(&parsed.parts[0].etag), "etag-5");
assert_eq!(parsed.next_part_number_marker, None);
}
#[test]
fn complete_multipart_body_orders_and_escapes_parts() {
let mut parts = completed_multipart_parts(vec![
MultipartUploadPart {
part_number: 2,
etag: "b&c".to_string(),
},
MultipartUploadPart {
part_number: 1,
etag: "a<tag>".to_string(),
},
]);
validate_completed_parts(&parts).unwrap();
let body = complete_multipart_body(&parts);
assert_eq!(
body,
"<CompleteMultipartUpload><Part><ETag>a&lt;tag&gt;</ETag><PartNumber>1</PartNumber></Part><Part><ETag>b&amp;c</\
ETag><PartNumber>2</PartNumber></Part></CompleteMultipartUpload>"
);
parts[0].etag.clear();
assert!(validate_completed_parts(&parts).is_err());
assert!(
validate_completed_parts(&[MultipartUploadPart {
part_number: -1,
etag: "etag".to_string(),
}])
.is_err()
);
assert!(
validate_completed_parts(&[MultipartUploadPart {
part_number: 0,
etag: "etag".to_string(),
}])
.is_err()
);
assert!(
validate_completed_parts(&[MultipartUploadPart {
part_number: 10_001,
etag: "etag".to_string(),
}])
.is_err()
);
}
#[test]
fn parse_rfc3339_ms_returns_zero_for_invalid_values() {
assert_eq!(parse_rfc3339_ms("2024-01-02T03:04:05Z"), 1_704_164_645_000);
assert_eq!(parse_rfc3339_ms("not a date"), 0);
}
}
packages/backend/native/src/backend_runtime/object_storage/config.rs → packages/backend/native/src/runtime/storage_runtime/object_storage/config.rs
+74 -89
View File
@@ -1,27 +1,29 @@
use aws_sdk_s3::config::{
BehaviorVersion, Credentials, Region, RequestChecksumCalculation, ResponseChecksumValidation, timeout::TimeoutConfig,
};
use napi::Result;
use rusty_s3::{Bucket, Credentials, UrlStyle};
use serde::Deserialize;
use url::Url;
use super::{client::ObjectStorageClient, types::StorageProviderConfig};
use crate::backend_runtime::{error::napi_error, types::RuntimeObjectStorageHealth};
use super::{
client::ObjectStorageClient,
error::{ObjectStorageError, ObjectStorageResult},
types::StorageProviderConfig,
};
#[derive(Clone, Debug)]
pub(in crate::backend_runtime) struct ObjectStorageConfig {
pub(super) provider: String,
pub(super) bucket: String,
pub(super) endpoint: Option<String>,
pub(super) region: Option<String>,
pub(super) access_key_id: Option<String>,
pub(super) secret_access_key: Option<String>,
pub(super) session_token: Option<String>,
pub(super) force_path_style: bool,
pub(super) request_timeout_ms: Option<u64>,
pub(super) min_part_size: Option<u64>,
pub(super) presign_expires_in_seconds: Option<u64>,
pub(super) presign_sign_content_type_for_put: Option<bool>,
pub(super) use_presigned_url: bool,
pub(crate) struct ObjectStorageConfig {
pub(crate) provider: String,
pub(crate) bucket: String,
pub(crate) endpoint: Option<String>,
pub(crate) region: Option<String>,
pub(crate) access_key_id: Option<String>,
pub(crate) secret_access_key: Option<String>,
pub(crate) session_token: Option<String>,
pub(crate) force_path_style: bool,
pub(crate) request_timeout_ms: Option<u64>,
pub(crate) min_part_size: Option<u64>,
pub(crate) presign_expires_in_seconds: Option<u64>,
pub(crate) presign_sign_content_type_for_put: Option<bool>,
pub(crate) use_presigned_url: bool,
pub(crate) proxy_upload: bool,
}
#[derive(Debug, Deserialize)]
@@ -68,14 +70,15 @@ struct S3PresignConfigFile {
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct UsePresignedUrlConfigFile {
enabled: bool,
url_prefix: Option<String>,
sign_key: Option<String>,
}
impl ObjectStorageConfig {
pub(in crate::backend_runtime) fn from_provider_config(
storage: Option<StorageProviderConfig>,
) -> Result<Option<Self>> {
pub(crate) fn from_provider_config(storage: Option<StorageProviderConfig>) -> ObjectStorageResult<Option<Self>> {
let Some(storage) = storage else {
return Ok(None);
};
@@ -84,18 +87,18 @@ impl ObjectStorageConfig {
"aws-s3" => Self::from_s3_config(storage),
"cloudflare-r2" => Self::from_r2_config(storage),
"fs" => Ok(None),
provider => Err(napi_error(format!(
"unsupported blob storage provider for BackendRuntime: {provider}"
provider => Err(ObjectStorageError::Config(format!(
"unsupported blob storage provider for StorageRuntime: {provider}"
))),
}
}
pub(super) fn from_s3_config(storage: StorageProviderConfig) -> Result<Option<Self>> {
pub(crate) fn from_s3_config(storage: StorageProviderConfig) -> ObjectStorageResult<Option<Self>> {
let config: S3ConfigFile = serde_json::from_value(storage.config)
.map_err(|err| napi_error(format!("invalid aws-s3 blob storage config: {err}")))?;
.map_err(|err| ObjectStorageError::Config(format!("invalid aws-s3 blob storage config: {err}")))?;
let region = config
.region
.ok_or_else(|| napi_error("aws-s3 blob storage config requires region"))?;
.ok_or_else(|| ObjectStorageError::Config("aws-s3 blob storage config requires region".to_string()))?;
let endpoint = config.endpoint.or_else(|| Some(resolve_s3_endpoint(&region)));
let credentials = config.credentials.unwrap_or_default();
@@ -113,17 +116,29 @@ impl ObjectStorageConfig {
presign_expires_in_seconds: config.presign.as_ref().and_then(|v| v.expires_in_seconds),
presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put),
use_presigned_url: config.use_presigned_url.map(|v| v.enabled).unwrap_or(false),
proxy_upload: false,
}))
}
pub(super) fn from_r2_config(storage: StorageProviderConfig) -> Result<Option<Self>> {
pub(crate) fn from_r2_config(storage: StorageProviderConfig) -> ObjectStorageResult<Option<Self>> {
let config: R2ConfigFile = serde_json::from_value(storage.config)
.map_err(|err| napi_error(format!("invalid cloudflare-r2 blob storage config: {err}")))?;
.map_err(|err| ObjectStorageError::Config(format!("invalid cloudflare-r2 blob storage config: {err}")))?;
let account = match config.jurisdiction {
Some(jurisdiction) => format!("{}.{}", config.account_id, jurisdiction),
None => config.account_id,
};
let credentials = config.credentials.unwrap_or_default();
let (use_presigned_url, proxy_upload) = config
.use_presigned_url
.map(|value| {
(
value.enabled,
value.enabled
&& value.url_prefix.as_ref().is_some_and(|prefix| !prefix.is_empty())
&& value.sign_key.as_ref().is_some_and(|key| !key.is_empty()),
)
})
.unwrap_or((false, false));
Ok(Some(Self {
provider: storage.provider,
@@ -138,81 +153,51 @@ impl ObjectStorageConfig {
min_part_size: config.min_part_size,
presign_expires_in_seconds: config.presign.as_ref().and_then(|v| v.expires_in_seconds),
presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put),
use_presigned_url: config.use_presigned_url.map(|v| v.enabled).unwrap_or(false),
use_presigned_url,
proxy_upload,
}))
}
pub(super) fn build_client(&self) -> Result<ObjectStorageClient> {
pub(crate) fn build_client(&self) -> ObjectStorageResult<ObjectStorageClient> {
let region = self
.region
.clone()
.ok_or_else(|| napi_error("object storage region is required"))?;
.ok_or_else(|| ObjectStorageError::Config("object storage region is required".to_string()))?;
let access_key_id = self
.access_key_id
.clone()
.ok_or_else(|| napi_error("object storage accessKeyId is required"))?;
.ok_or_else(|| ObjectStorageError::Config("object storage accessKeyId is required".to_string()))?;
let secret_access_key = self
.secret_access_key
.clone()
.ok_or_else(|| napi_error("object storage secretAccessKey is required"))?;
.ok_or_else(|| ObjectStorageError::Config("object storage secretAccessKey is required".to_string()))?;
let credentials = Credentials::new(
access_key_id,
secret_access_key,
self.session_token.clone(),
None,
"affine-server-config-json",
);
let mut builder = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(region))
.credentials_provider(credentials)
.force_path_style(self.force_path_style)
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.response_checksum_validation(ResponseChecksumValidation::WhenRequired);
if let Some(endpoint) = &self.endpoint {
builder = builder.endpoint_url(endpoint);
}
if let Some(request_timeout_ms) = self.request_timeout_ms {
builder = builder.timeout_config(
TimeoutConfig::builder()
.operation_timeout(std::time::Duration::from_millis(request_timeout_ms))
.build(),
);
}
Ok(ObjectStorageClient::new(
builder.build(),
let endpoint = self.endpoint.clone().unwrap_or_else(|| resolve_s3_endpoint(&region));
let endpoint = Url::parse(&endpoint)
.map_err(|err| ObjectStorageError::Config(format!("object storage endpoint is invalid: {err}")))?;
let bucket = Bucket::new(
endpoint,
if self.force_path_style {
UrlStyle::Path
} else {
UrlStyle::VirtualHost
},
self.bucket.clone(),
region,
)
.map_err(|err| ObjectStorageError::Config(format!("object storage bucket url is invalid: {err}")))?;
let credentials = match self.session_token.as_ref().filter(|token| !token.is_empty()) {
Some(session_token) => Credentials::new_with_token(access_key_id, secret_access_key, session_token.clone()),
None => Credentials::new(access_key_id, secret_access_key),
};
ObjectStorageClient::new(
bucket,
credentials,
self.request_timeout_ms,
self.presign_expires_in_seconds.unwrap_or(60),
self.presign_sign_content_type_for_put.unwrap_or(true),
))
}
pub(in crate::backend_runtime) fn health(&self) -> RuntimeObjectStorageHealth {
let client_buildable = self
.build_client()
.map(|client| client.non_destructive_health())
.unwrap_or(false);
RuntimeObjectStorageHealth {
configured: true,
provider: Some(self.provider.clone()),
bucket: Some(self.bucket.clone()),
endpoint: self.endpoint.clone(),
region: self.region.clone(),
has_credentials: self.access_key_id.is_some()
&& self.secret_access_key.is_some()
&& self.session_token.as_ref().map(|v| !v.is_empty()).unwrap_or(true),
force_path_style: self.force_path_style,
request_timeout_ms: self.request_timeout_ms.map(|v| v as i64),
min_part_size: self.min_part_size.map(|v| v as i64),
presign_expires_in_seconds: self.presign_expires_in_seconds.map(|v| v as i64),
presign_sign_content_type_for_put: self.presign_sign_content_type_for_put,
use_presigned_url: self.use_presigned_url,
client_buildable,
}
)
}
}
packages/backend/native/src/runtime/storage_runtime/object_storage/error.rs
+56
View File
@@ -0,0 +1,56 @@
use reqwest::StatusCode;
#[derive(Debug, thiserror::Error)]
pub(crate) enum ObjectStorageError {
#[error("ObjectStorage config error: {0}")]
Config(String),
#[error("{context}: {source}")]
Operation {
context: String,
#[source]
source: Box<ObjectStorageError>,
},
#[error("ObjectStorage http client build failed: {0}")]
HttpClientBuild(#[source] reqwest::Error),
#[error("ObjectStorage http request failed: {0}")]
HttpRequest(#[source] reqwest::Error),
#[error("ObjectStorage invalid http header: {0}")]
InvalidHeader(String),
#[error("ObjectStorage response body exceeds {limit} bytes")]
BodyTooLarge { limit: usize },
#[error("{context}: status={status} body={body}")]
HttpStatus {
context: String,
status: StatusCode,
body: String,
},
#[error("{context}: invalid utf8 response: {source}")]
InvalidUtf8 {
context: String,
#[source]
source: std::string::FromUtf8Error,
},
#[error("{context}: invalid xml response: {source}")]
InvalidXml {
context: String,
#[source]
source: instant_xml::Error,
},
#[error("ObjectStorage invalid input: {0}")]
InvalidInput(String),
}
impl ObjectStorageError {
pub(crate) fn is_not_found(&self) -> bool {
match self {
Self::Operation { source, .. } => source.is_not_found(),
Self::HttpStatus { status, body, .. } => {
*status == StatusCode::NOT_FOUND
&& (body.contains("NoSuchKey") || body.contains("NoSuchUpload") || body.contains("NotFound"))
}
_ => false,
}
}
}
pub(crate) type ObjectStorageResult<T> = std::result::Result<T, ObjectStorageError>;
packages/backend/native/src/runtime/storage_runtime/object_storage/mod.rs
+9
View File
@@ -0,0 +1,9 @@
pub(crate) mod client;
pub(crate) mod config;
pub(crate) mod error;
#[cfg(test)]
mod tests;
pub(crate) mod types;
pub(crate) use config::ObjectStorageConfig;
pub(crate) use types::StorageProviderConfig;
packages/backend/native/src/runtime/storage_runtime/object_storage/tests.rs
+367
View File
@@ -0,0 +1,367 @@
use reqwest::StatusCode;
use super::{
config::ObjectStorageConfig,
error::ObjectStorageError,
types::{MultipartUploadPart, ObjectPutMetadata, StorageProviderConfig, completed_multipart_parts, trim_etag},
};
fn storage_config(provider: &str, config: serde_json::Value) -> StorageProviderConfig {
StorageProviderConfig {
provider: provider.to_string(),
bucket: "test-bucket".to_string(),
config,
}
}
#[test]
fn resolves_r2_config_from_config_json_shape() {
let storage = StorageProviderConfig {
provider: "cloudflare-r2".to_string(),
bucket: "workspace-blobs".to_string(),
config: serde_json::json!({
"accountId": "account",
"jurisdiction": "eu",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"usePresignedURL": {
"enabled": true
}
}),
};
let config = ObjectStorageConfig::from_r2_config(storage).unwrap().unwrap();
assert_eq!(config.provider, "cloudflare-r2");
assert_eq!(config.bucket, "workspace-blobs");
assert_eq!(
config.endpoint.as_deref(),
Some("https://account.eu.r2.cloudflarestorage.com")
);
assert_eq!(config.region.as_deref(), Some("auto"));
assert!(config.force_path_style);
assert!(config.use_presigned_url);
assert!(!config.proxy_upload);
assert_eq!(config.access_key_id.as_deref(), Some("key"));
}
#[test]
fn resolves_r2_endpoint_cases_from_config_json_shape() {
for (case, config, expected_endpoint) in [
(
"default account endpoint",
serde_json::json!({
"accountId": "account",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
}
}),
Some("https://account.r2.cloudflarestorage.com"),
),
(
"explicit null jurisdiction",
serde_json::json!({
"accountId": "account",
"jurisdiction": null,
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
}
}),
Some("https://account.r2.cloudflarestorage.com"),
),
(
"eu jurisdiction",
serde_json::json!({
"accountId": "account",
"jurisdiction": "eu",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
}
}),
Some("https://account.eu.r2.cloudflarestorage.com"),
),
] {
let config = ObjectStorageConfig::from_r2_config(storage_config("cloudflare-r2", config))
.unwrap()
.unwrap();
assert_eq!(config.endpoint.as_deref(), expected_endpoint, "{case}");
assert!(config.force_path_style, "{case}");
}
assert!(
ObjectStorageConfig::from_r2_config(storage_config(
"cloudflare-r2",
serde_json::json!({
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
}
})
))
.is_err()
);
}
#[test]
fn object_storage_not_found_requires_object_error_code() {
let bucket_or_route_missing = ObjectStorageError::HttpStatus {
context: "head failed".to_string(),
status: StatusCode::NOT_FOUND,
body: String::new(),
};
let object_missing = ObjectStorageError::HttpStatus {
context: "get failed".to_string(),
status: StatusCode::NOT_FOUND,
body: "<Error><Code>NoSuchKey</Code></Error>".to_string(),
};
let upload_missing = ObjectStorageError::HttpStatus {
context: "abort failed".to_string(),
status: StatusCode::NOT_FOUND,
body: "<Error><Code>NoSuchUpload</Code></Error>".to_string(),
};
assert!(!bucket_or_route_missing.is_not_found());
assert!(object_missing.is_not_found());
assert!(upload_missing.is_not_found());
}
#[test]
fn resolves_r2_proxy_upload_capability_from_config_json_shape() {
let storage = StorageProviderConfig {
provider: "cloudflare-r2".to_string(),
bucket: "workspace-blobs".to_string(),
config: serde_json::json!({
"accountId": "account",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"usePresignedURL": {
"enabled": true,
"urlPrefix": "https://cdn.example.com",
"signKey": "secret"
}
}),
};
let config = ObjectStorageConfig::from_r2_config(storage).unwrap().unwrap();
assert!(config.use_presigned_url);
assert!(config.proxy_upload);
}
#[test]
fn resolves_s3_config_from_config_json_shape() {
let storage = StorageProviderConfig {
provider: "aws-s3".to_string(),
bucket: "workspace-blobs".to_string(),
config: serde_json::json!({
"region": "us-west-2",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret",
"sessionToken": "session"
},
"forcePathStyle": true,
"requestTimeoutMs": 1000,
"minPartSize": 1024,
"presign": {
"expiresInSeconds": 60,
"signContentTypeForPut": false
}
}),
};
let config = ObjectStorageConfig::from_s3_config(storage).unwrap().unwrap();
assert_eq!(config.provider, "aws-s3");
assert_eq!(config.endpoint.as_deref(), Some("https://s3.us-west-2.amazonaws.com"));
assert_eq!(config.session_token.as_deref(), Some("session"));
assert!(config.force_path_style);
assert_eq!(config.request_timeout_ms, Some(1000));
assert_eq!(config.min_part_size, Some(1024));
assert_eq!(config.presign_expires_in_seconds, Some(60));
assert_eq!(config.presign_sign_content_type_for_put, Some(false));
}
#[test]
fn resolves_s3_default_endpoint_cases_from_config_json_shape() {
for (region, expected_endpoint) in [
("us-east-1", "https://s3.amazonaws.com"),
("us-west-2", "https://s3.us-west-2.amazonaws.com"),
] {
let config = ObjectStorageConfig::from_s3_config(storage_config(
"aws-s3",
serde_json::json!({
"region": region,
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
}
}),
))
.unwrap()
.unwrap();
assert_eq!(config.endpoint.as_deref(), Some(expected_endpoint), "{region}");
}
}
#[tokio::test]
async fn object_storage_presign_put_returns_sigv4_url_and_headers() {
let storage = StorageProviderConfig {
provider: "aws-s3".to_string(),
bucket: "test-bucket".to_string(),
config: serde_json::json!({
"region": "us-east-1",
"endpoint": "https://s3.us-east-1.amazonaws.com",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"presign": {
"expiresInSeconds": 60
}
}),
};
let config = ObjectStorageConfig::from_s3_config(storage).unwrap().unwrap();
let Ok(Ok(client)) = std::panic::catch_unwind(|| config.build_client()) else {
eprintln!("skipping object storage presign test: S3 client cannot be built in this environment");
return;
};
let result = client
.presign_put(
"key",
ObjectPutMetadata {
content_type: Some("text/plain".to_string()),
..Default::default()
},
)
.await
.unwrap();
assert!(result.url.contains("X-Amz-Algorithm=AWS4-HMAC-SHA256"));
assert!(result.url.contains("X-Amz-SignedHeaders="));
assert_eq!(
result.headers.get("Content-Type").map(String::as_str),
Some("text/plain")
);
assert!(result.expires_at_ms > 0);
}
#[tokio::test]
async fn object_storage_presign_put_respects_content_length_and_signed_content_type_flag() {
let config = ObjectStorageConfig::from_s3_config(storage_config(
"aws-s3",
serde_json::json!({
"region": "us-east-1",
"endpoint": "https://s3.us-east-1.amazonaws.com",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"presign": {
"expiresInSeconds": 60,
"signContentTypeForPut": false
}
}),
))
.unwrap()
.unwrap();
let client = config.build_client().unwrap();
let result = client
.presign_put(
"key",
ObjectPutMetadata {
content_type: Some("text/plain".to_string()),
content_length: Some(42),
..Default::default()
},
)
.await
.unwrap();
assert_eq!(
result.headers.get("Content-Type").map(String::as_str),
Some("text/plain")
);
assert_eq!(result.headers.get("Content-Length").map(String::as_str), Some("42"));
assert!(!result.url.contains("content-type"));
assert!(result.url.contains("content-length"));
}
#[tokio::test]
async fn object_storage_presign_get_returns_sigv4_url_without_headers() {
let storage = StorageProviderConfig {
provider: "cloudflare-r2".to_string(),
bucket: "test-bucket".to_string(),
config: serde_json::json!({
"accountId": "account",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"presign": {
"expiresInSeconds": 60
}
}),
};
let config = ObjectStorageConfig::from_r2_config(storage).unwrap().unwrap();
let client = config.build_client().unwrap();
let result = client.presign_get("workspace/key").await.unwrap();
assert!(result.url.contains("X-Amz-Algorithm=AWS4-HMAC-SHA256"));
assert!(result.url.contains("X-Amz-SignedHeaders=host"));
assert!(result.url.contains("/test-bucket/workspace/key?"));
assert!(result.headers.is_empty());
assert!(result.expires_at_ms > 0);
}
#[tokio::test]
async fn object_storage_presign_upload_part_returns_sigv4_url() {
let config = ObjectStorageConfig::from_s3_config(storage_config(
"aws-s3",
serde_json::json!({
"region": "us-east-1",
"endpoint": "https://s3.us-east-1.amazonaws.com",
"credentials": {
"accessKeyId": "key",
"secretAccessKey": "secret"
},
"presign": {
"expiresInSeconds": 60
}
}),
))
.unwrap()
.unwrap();
let client = config.build_client().unwrap();
let result = client.presign_upload_part("key", "upload-1", 3).await.unwrap();
assert!(result.url.contains("X-Amz-Algorithm=AWS4-HMAC-SHA256"));
assert!(result.url.contains("partNumber=3"));
assert!(result.url.contains("uploadId=upload-1"));
assert!(result.headers.is_empty());
assert!(result.expires_at_ms > 0);
}
#[test]
fn object_storage_orders_completed_multipart_parts_and_trims_etags() {
let parts = completed_multipart_parts(vec![
MultipartUploadPart {
part_number: 2,
etag: trim_etag("\"b\""),
},
MultipartUploadPart {
part_number: 1,
etag: trim_etag("a"),
},
]);
assert_eq!(parts[0].part_number, 1);
assert_eq!(parts[0].etag, "a");
assert_eq!(parts[1].part_number, 2);
assert_eq!(parts[1].etag, "b");
}
packages/backend/native/src/runtime/storage_runtime/object_storage/types.rs
+182
View File
@@ -0,0 +1,182 @@
use std::collections::HashMap;
use serde::Deserialize;
use super::super::{
RuntimeError, RuntimeMultipartUploadInit, RuntimeMultipartUploadPart, RuntimeObjectGetResult, RuntimeObjectListEntry,
RuntimeObjectMetadata, RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest, RuntimeResult,
};
#[derive(Clone, Debug, Default)]
pub(crate) struct ObjectPutMetadata {
pub(crate) content_type: Option<String>,
pub(crate) content_length: Option<i64>,
pub(crate) checksum_crc32: Option<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct ObjectMetadata {
pub(crate) content_type: String,
pub(crate) content_length: i64,
pub(crate) last_modified_ms: i64,
pub(crate) checksum_crc32: Option<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct ObjectListEntry {
pub(crate) key: String,
pub(crate) content_length: i64,
pub(crate) last_modified_ms: i64,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct ObjectListPage {
pub(crate) entries: Vec<ObjectListEntry>,
pub(crate) next_continuation_token: Option<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct ObjectGetResult {
pub(crate) body: Vec<u8>,
pub(crate) metadata: ObjectMetadata,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct PresignedObjectRequest {
pub(crate) url: String,
pub(crate) headers: HashMap<String, String>,
pub(crate) expires_at_ms: i64,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct MultipartUploadInitResult {
pub(crate) upload_id: String,
pub(crate) expires_at_ms: i64,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct MultipartUploadPart {
pub(crate) part_number: i32,
pub(crate) etag: String,
}
#[derive(Clone, Debug, Deserialize)]
pub(crate) struct StorageProviderConfig {
pub(crate) provider: String,
pub(crate) bucket: String,
#[serde(default)]
pub(crate) config: serde_json::Value,
}
pub(crate) fn trim_etag(etag: &str) -> String {
etag.trim_matches('"').to_string()
}
pub(crate) fn completed_multipart_parts(mut parts: Vec<MultipartUploadPart>) -> Vec<MultipartUploadPart> {
parts.sort_by_key(|part| part.part_number);
parts
}
impl From<RuntimeObjectStoragePutOptions> for ObjectPutMetadata {
fn from(options: RuntimeObjectStoragePutOptions) -> Self {
Self {
content_type: options.content_type,
content_length: options.content_length,
checksum_crc32: options.checksum_crc32,
}
}
}
impl ObjectPutMetadata {
pub(crate) fn complete_for_body(mut self, body: &[u8]) -> Self {
self.content_length.get_or_insert(body.len() as i64);
self
.checksum_crc32
.get_or_insert_with(|| format!("{:x}", crc32fast::hash(body)));
self
.content_type
.get_or_insert_with(|| crate::file_type::get_mime(body));
self
}
pub(crate) fn into_object_metadata(self, last_modified_ms: i64) -> ObjectMetadata {
ObjectMetadata {
content_type: self
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string()),
content_length: self.content_length.unwrap_or(0),
last_modified_ms,
checksum_crc32: self.checksum_crc32,
}
}
}
impl From<ObjectMetadata> for RuntimeObjectMetadata {
fn from(metadata: ObjectMetadata) -> Self {
Self {
content_type: metadata.content_type,
content_length: metadata.content_length,
last_modified_ms: metadata.last_modified_ms,
checksum_crc32: metadata.checksum_crc32,
}
}
}
impl From<ObjectListEntry> for RuntimeObjectListEntry {
fn from(entry: ObjectListEntry) -> Self {
Self {
key: entry.key,
content_length: entry.content_length,
last_modified_ms: entry.last_modified_ms,
}
}
}
impl TryFrom<PresignedObjectRequest> for RuntimePresignedObjectRequest {
type Error = RuntimeError;
fn try_from(request: PresignedObjectRequest) -> RuntimeResult<Self> {
Ok(Self {
url: request.url,
headers_json: serde_json::to_string(&request.headers)
.map_err(|err| RuntimeError::json("ObjectStorage headers serialization failed", err))?,
expires_at_ms: request.expires_at_ms,
})
}
}
impl From<ObjectGetResult> for RuntimeObjectGetResult {
fn from(result: ObjectGetResult) -> Self {
Self {
body: result.body.into(),
metadata: result.metadata.into(),
}
}
}
impl From<MultipartUploadInitResult> for RuntimeMultipartUploadInit {
fn from(init: MultipartUploadInitResult) -> Self {
Self {
upload_id: init.upload_id,
expires_at_ms: init.expires_at_ms,
}
}
}
impl From<RuntimeMultipartUploadPart> for MultipartUploadPart {
fn from(part: RuntimeMultipartUploadPart) -> Self {
Self {
part_number: part.part_number,
etag: part.etag,
}
}
}
impl From<MultipartUploadPart> for RuntimeMultipartUploadPart {
fn from(part: MultipartUploadPart) -> Self {
Self {
part_number: part.part_number,
etag: part.etag,
}
}
}
packages/backend/native/src/backend_runtime/types.rs → packages/backend/native/src/runtime/types.rs
-18
View File
@@ -12,24 +12,6 @@ pub struct RuntimeVerificationTokenRecord {
pub struct BackendRuntimeHealth {
pub started: bool,
pub database_connected: bool,
pub object_storage_configured: bool,
}
#[napi_derive::napi(object)]
pub struct RuntimeObjectStorageHealth {
pub configured: bool,
pub provider: Option<String>,
pub bucket: Option<String>,
pub endpoint: Option<String>,
pub region: Option<String>,
pub has_credentials: bool,
pub force_path_style: bool,
pub request_timeout_ms: Option<i64>,
pub min_part_size: Option<i64>,
pub presign_expires_in_seconds: Option<i64>,
pub presign_sign_content_type_for_put: Option<bool>,
pub use_presigned_url: bool,
pub client_buildable: bool,
}
#[napi_derive::napi(object)]
packages/backend/native/src/utils.rs
+6
View File
@@ -1,3 +1,9 @@
use std::time::{SystemTime, SystemTimeError, UNIX_EPOCH};
pub(crate) fn system_time_millis(time: SystemTime) -> Result<u128, SystemTimeError> {
Ok(time.duration_since(UNIX_EPOCH)?.as_millis())
}
fn collapse_whitespace(s: &str) -> String {
let mut result = String::new();
let mut prev_was_whitespace = false;