mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-08-10 13:38:49 +08:00
fix(server): s3 metadata encode
This commit is contained in:
@@ -23,7 +23,7 @@ pub(crate) mod object_storage;
|
|||||||
|
|
||||||
use self::object_storage::{
|
use self::object_storage::{
|
||||||
ObjectStorageConfig, StorageProviderConfig,
|
ObjectStorageConfig, StorageProviderConfig,
|
||||||
types::{ObjectGetResult, ObjectListEntry, ObjectMetadata, ObjectPutMetadata},
|
types::{ObjectGetResult, ObjectListEntry, ObjectMetadata, ObjectPutMetadata, checksum_crc32_base64},
|
||||||
};
|
};
|
||||||
pub(super) use super::{
|
pub(super) use super::{
|
||||||
RuntimeError, RuntimeResult,
|
RuntimeError, RuntimeResult,
|
||||||
@@ -1081,7 +1081,7 @@ fn fs_put(config: &FsStorageConfig, key: &str, body: Vec<u8>, metadata: ObjectPu
|
|||||||
return Err(RuntimeError::invalid_input("StorageRuntime fs content length mismatch"));
|
return Err(RuntimeError::invalid_input("StorageRuntime fs content length mismatch"));
|
||||||
}
|
}
|
||||||
if let Some(checksum) = metadata.checksum_crc32.as_deref() {
|
if let Some(checksum) = metadata.checksum_crc32.as_deref() {
|
||||||
let actual = format!("{:x}", crc32fast::hash(&body));
|
let actual = checksum_crc32_base64(&body);
|
||||||
if actual != checksum {
|
if actual != checksum {
|
||||||
return Err(RuntimeError::invalid_input("StorageRuntime fs checksum mismatch"));
|
return Err(RuntimeError::invalid_input("StorageRuntime fs checksum mismatch"));
|
||||||
}
|
}
|
||||||
@@ -1439,7 +1439,7 @@ mod tests {
|
|||||||
bucket: "bucket".to_string(),
|
bucket: "bucket".to_string(),
|
||||||
};
|
};
|
||||||
let body = b"hello".to_vec();
|
let body = b"hello".to_vec();
|
||||||
let checksum = format!("{:x}", crc32fast::hash(&body));
|
let checksum = checksum_crc32_base64(&body);
|
||||||
|
|
||||||
fs_put(
|
fs_put(
|
||||||
&config,
|
&config,
|
||||||
@@ -1741,7 +1741,7 @@ mod tests {
|
|||||||
ObjectPutMetadata {
|
ObjectPutMetadata {
|
||||||
content_type: Some("text/plain".to_string()),
|
content_type: Some("text/plain".to_string()),
|
||||||
content_length: Some(body.len() as i64),
|
content_length: Some(body.len() as i64),
|
||||||
checksum_crc32: Some(format!("{:x}", crc32fast::hash(&body))),
|
checksum_crc32: Some(checksum_crc32_base64(&body)),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|||||||
@@ -3,7 +3,10 @@ use reqwest::StatusCode;
|
|||||||
use super::{
|
use super::{
|
||||||
config::ObjectStorageConfig,
|
config::ObjectStorageConfig,
|
||||||
error::ObjectStorageError,
|
error::ObjectStorageError,
|
||||||
types::{MultipartUploadPart, ObjectPutMetadata, StorageProviderConfig, completed_multipart_parts, trim_etag},
|
types::{
|
||||||
|
MultipartUploadPart, ObjectPutMetadata, StorageProviderConfig, checksum_crc32_base64, completed_multipart_parts,
|
||||||
|
trim_etag,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
fn storage_config(provider: &str, config: serde_json::Value) -> StorageProviderConfig {
|
fn storage_config(provider: &str, config: serde_json::Value) -> StorageProviderConfig {
|
||||||
@@ -365,3 +368,9 @@ fn object_storage_orders_completed_multipart_parts_and_trims_etags() {
|
|||||||
assert_eq!(parts[1].part_number, 2);
|
assert_eq!(parts[1].part_number, 2);
|
||||||
assert_eq!(parts[1].etag, "b");
|
assert_eq!(parts[1].etag, "b");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn object_storage_crc32_checksum_uses_s3_base64_format() {
|
||||||
|
assert_eq!(checksum_crc32_base64(b"hello"), "NhCmhg==");
|
||||||
|
assert_ne!(checksum_crc32_base64(b"hello"), "3610a686");
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
|
||||||
use super::super::{
|
use super::super::{
|
||||||
@@ -90,9 +91,7 @@ impl From<RuntimeObjectStoragePutOptions> for ObjectPutMetadata {
|
|||||||
impl ObjectPutMetadata {
|
impl ObjectPutMetadata {
|
||||||
pub(crate) fn complete_for_body(mut self, body: &[u8]) -> Self {
|
pub(crate) fn complete_for_body(mut self, body: &[u8]) -> Self {
|
||||||
self.content_length.get_or_insert(body.len() as i64);
|
self.content_length.get_or_insert(body.len() as i64);
|
||||||
self
|
self.checksum_crc32.get_or_insert_with(|| checksum_crc32_base64(body));
|
||||||
.checksum_crc32
|
|
||||||
.get_or_insert_with(|| format!("{:x}", crc32fast::hash(body)));
|
|
||||||
self
|
self
|
||||||
.content_type
|
.content_type
|
||||||
.get_or_insert_with(|| crate::file_type::get_mime(body));
|
.get_or_insert_with(|| crate::file_type::get_mime(body));
|
||||||
@@ -111,6 +110,10 @@ impl ObjectPutMetadata {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn checksum_crc32_base64(body: &[u8]) -> String {
|
||||||
|
STANDARD.encode(crc32fast::hash(body).to_be_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
impl From<ObjectMetadata> for RuntimeObjectMetadata {
|
impl From<ObjectMetadata> for RuntimeObjectMetadata {
|
||||||
fn from(metadata: ObjectMetadata) -> Self {
|
fn from(metadata: ObjectMetadata) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|||||||
Reference in New Issue
Block a user