fix(server): s3 metadata encode

This commit is contained in:
DarkSky
2026-07-02 00:27:17 +08:00
parent 8c68319094
commit e5d44b8ff2
3 changed files with 20 additions and 8 deletions
@@ -23,7 +23,7 @@ pub(crate) mod object_storage;
use self::object_storage::{ use self::object_storage::{
ObjectStorageConfig, StorageProviderConfig, ObjectStorageConfig, StorageProviderConfig,
types::{ObjectGetResult, ObjectListEntry, ObjectMetadata, ObjectPutMetadata}, types::{ObjectGetResult, ObjectListEntry, ObjectMetadata, ObjectPutMetadata, checksum_crc32_base64},
}; };
pub(super) use super::{ pub(super) use super::{
RuntimeError, RuntimeResult, RuntimeError, RuntimeResult,
@@ -1081,7 +1081,7 @@ fn fs_put(config: &FsStorageConfig, key: &str, body: Vec<u8>, metadata: ObjectPu
return Err(RuntimeError::invalid_input("StorageRuntime fs content length mismatch")); return Err(RuntimeError::invalid_input("StorageRuntime fs content length mismatch"));
} }
if let Some(checksum) = metadata.checksum_crc32.as_deref() { if let Some(checksum) = metadata.checksum_crc32.as_deref() {
let actual = format!("{:x}", crc32fast::hash(&body)); let actual = checksum_crc32_base64(&body);
if actual != checksum { if actual != checksum {
return Err(RuntimeError::invalid_input("StorageRuntime fs checksum mismatch")); return Err(RuntimeError::invalid_input("StorageRuntime fs checksum mismatch"));
} }
@@ -1439,7 +1439,7 @@ mod tests {
bucket: "bucket".to_string(), bucket: "bucket".to_string(),
}; };
let body = b"hello".to_vec(); let body = b"hello".to_vec();
let checksum = format!("{:x}", crc32fast::hash(&body)); let checksum = checksum_crc32_base64(&body);
fs_put( fs_put(
&config, &config,
@@ -1741,7 +1741,7 @@ mod tests {
ObjectPutMetadata { ObjectPutMetadata {
content_type: Some("text/plain".to_string()), content_type: Some("text/plain".to_string()),
content_length: Some(body.len() as i64), content_length: Some(body.len() as i64),
checksum_crc32: Some(format!("{:x}", crc32fast::hash(&body))), checksum_crc32: Some(checksum_crc32_base64(&body)),
}, },
) )
.await?; .await?;
@@ -3,7 +3,10 @@ use reqwest::StatusCode;
use super::{ use super::{
config::ObjectStorageConfig, config::ObjectStorageConfig,
error::ObjectStorageError, error::ObjectStorageError,
types::{MultipartUploadPart, ObjectPutMetadata, StorageProviderConfig, completed_multipart_parts, trim_etag}, types::{
MultipartUploadPart, ObjectPutMetadata, StorageProviderConfig, checksum_crc32_base64, completed_multipart_parts,
trim_etag,
},
}; };
fn storage_config(provider: &str, config: serde_json::Value) -> StorageProviderConfig { fn storage_config(provider: &str, config: serde_json::Value) -> StorageProviderConfig {
@@ -365,3 +368,9 @@ fn object_storage_orders_completed_multipart_parts_and_trims_etags() {
assert_eq!(parts[1].part_number, 2); assert_eq!(parts[1].part_number, 2);
assert_eq!(parts[1].etag, "b"); assert_eq!(parts[1].etag, "b");
} }
#[test]
fn object_storage_crc32_checksum_uses_s3_base64_format() {
assert_eq!(checksum_crc32_base64(b"hello"), "NhCmhg==");
assert_ne!(checksum_crc32_base64(b"hello"), "3610a686");
}
@@ -1,5 +1,6 @@
use std::collections::HashMap; use std::collections::HashMap;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use serde::Deserialize; use serde::Deserialize;
use super::super::{ use super::super::{
@@ -90,9 +91,7 @@ impl From<RuntimeObjectStoragePutOptions> for ObjectPutMetadata {
impl ObjectPutMetadata { impl ObjectPutMetadata {
pub(crate) fn complete_for_body(mut self, body: &[u8]) -> Self { pub(crate) fn complete_for_body(mut self, body: &[u8]) -> Self {
self.content_length.get_or_insert(body.len() as i64); self.content_length.get_or_insert(body.len() as i64);
self self.checksum_crc32.get_or_insert_with(|| checksum_crc32_base64(body));
.checksum_crc32
.get_or_insert_with(|| format!("{:x}", crc32fast::hash(body)));
self self
.content_type .content_type
.get_or_insert_with(|| crate::file_type::get_mime(body)); .get_or_insert_with(|| crate::file_type::get_mime(body));
@@ -111,6 +110,10 @@ impl ObjectPutMetadata {
} }
} }
pub(crate) fn checksum_crc32_base64(body: &[u8]) -> String {
STANDARD.encode(crc32fast::hash(body).to_be_bytes())
}
impl From<ObjectMetadata> for RuntimeObjectMetadata { impl From<ObjectMetadata> for RuntimeObjectMetadata {
fn from(metadata: ObjectMetadata) -> Self { fn from(metadata: ObjectMetadata) -> Self {
Self { Self {