diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index a72973aa20..bdc7be1399 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -505,6 +505,7 @@ "jurisdiction": { "type": "string", "enum": [ + "default", "eu" ], "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." @@ -530,6 +531,28 @@ } } } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "assetpack" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } } ], "default": { @@ -703,6 +726,7 @@ "jurisdiction": { "type": "string", "enum": [ + "default", "eu" ], "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." @@ -728,6 +752,28 @@ } } } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "assetpack" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } } ], "default": { @@ -1344,6 +1390,7 @@ "jurisdiction": { "type": "string", "enum": [ + "default", "eu" ], "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." @@ -1369,6 +1416,28 @@ } } } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "assetpack" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } } ], "default": { diff --git a/Cargo.lock b/Cargo.lock index 5a021d9a80..367e9b5dbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,7 @@ dependencies = [ "assert-json-diff", "chrono", "criterion", + "hex", "nanoid", "napi", "pulldown-cmark", @@ -196,15 +197,18 @@ dependencies = [ "aes-gcm", "affine_common", "anyhow", - "aws-sdk-s3", + "assetpack-core", + "assetpack-transform-precomp2", "base64", "chrono", + "crc32fast", "doc_extractor", "file-format", "hex", "homedir", "image", "infer", + "instant-xml", "jsonschema", "libwebp-sys", "little_exif", @@ -219,13 +223,17 @@ dependencies = [ "p256", "rand 0.9.4", "rayon", + "reqwest", + "rusty-s3", "safefetch", "schemars", "serde", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "sha3", "sqlx", + "tempfile", + "thiserror 2.0.18", "tiktoken-rs", "tokio", "url", @@ -419,15 +427,6 @@ dependencies = [ "derive_arbitrary", ] -[[package]] -name = "arc-swap" -version = "1.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" -dependencies = [ - "rustversion", -] - [[package]] name = "arrayref" version = "0.3.9" @@ -492,6 +491,39 @@ dependencies = [ "serde_json", ] +[[package]] +name = "assetpack-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72bb7294f3c191cc25839285896f7973ce4470dca2a412a95ad39956c99326f5" +dependencies = [ + "brotli", + "chrono", + "fastcdc", + "futures-core", + "futures-util", + "hex", + "serde", + "sha3", + "sqlx", + "tempfile", + "thiserror 2.0.18", + "tokio", + "zstd", +] + +[[package]] +name = "assetpack-transform-precomp2" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780badf66e57dd350c0b2778966c383a871b3df6873d4e6ca12174344761819c" +dependencies = [ + "assetpack-core", + "lzma-rust2", + "precomp2", + "zstd", +] + [[package]] name = "async-compat" version = "0.2.5" @@ -571,18 +603,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "aws-credential-types" -version = "1.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "zeroize", -] - [[package]] name = "aws-lc-rs" version = "1.16.3" @@ -605,328 +625,6 @@ dependencies = [ "fs_extra", ] -[[package]] -name = "aws-runtime" -version = "1.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "bytes-utils", - "fastrand", - "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", - "http-body 1.0.1", - "percent-encoding", - "pin-project-lite", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-s3" -version = "1.137.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd7213994e2ff9382ff100403b78c30d1b74cdfcd8fa9d0d1dc3a94a5c4874" -dependencies = [ - "arc-swap", - "aws-credential-types", - "aws-runtime", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "fastrand", - "hex", - "hmac 0.13.0", - "http 0.2.12", - "http 1.4.0", - "http-body 1.0.1", - "lru", - "percent-encoding", - "regex-lite", - "sha2 0.11.0", - "tracing", - "url", -] - -[[package]] -name = "aws-sigv4" -version = "1.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" -dependencies = [ - "aws-credential-types", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "crypto-bigint", - "form_urlencoded", - "hex", - "hmac 0.13.0", - "http 0.2.12", - "http 1.4.0", - "p256", - "percent-encoding", - "sha2 0.11.0", - "subtle", - "time", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-async" -version = "1.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.64.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e8e65f4f81fcccdeb6c3eca2af17ac21d421a1786a26a394aecf421d616d3a" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc-fast", - "hex", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "md-5 0.11.0", - "pin-project-lite", - "sha1 0.11.0", - "sha2 0.11.0", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.60.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78d8391e65fcea47c586a22e1a41f173b38615b112b2c6b7a44e80cec3e6b706" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.63.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http-client" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "h2 0.3.27", - "h2 0.4.13", - "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper 1.9.0", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.9", - "hyper-util", - "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.37", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls 0.26.4", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.62.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-observability" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" -dependencies = [ - "aws-smithy-runtime-api", -] - -[[package]] -name = "aws-smithy-runtime" -version = "1.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-client", - "aws-smithy-observability", - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "pin-project-lite", - "pin-utils", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api-macros", - "aws-smithy-types", - "bytes", - "http 0.2.12", - "http 1.4.0", - "pin-project-lite", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-runtime-api-macros" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "aws-smithy-schema" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "http 1.4.0", -] - -[[package]] -name = "aws-smithy-types" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" -dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "itoa", - "num-integer", - "pin-project-lite", - "pin-utils", - "ryu", - "serde", - "time", - "tokio", - "tokio-util", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.60.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "1.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", - "rustc_version", - "tracing", -] - [[package]] name = "az" version = "1.3.0" @@ -1074,6 +772,30 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bitcode" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6ed1b54d8dc333e7be604d00fa9262f4635485ffea923647b6521a5fff045d" +dependencies = [ + "arrayvec", + "bitcode_derive", + "bytemuck", + "glam", + "serde", +] + +[[package]] +name = "bitcode_derive" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238b90427dfad9da4a9abd60f3ec1cdee6b80454bde49ed37f1781dd8e9dc7f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -1253,13 +975,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] -name = "bytes-utils" -version = "0.1.4" +name = "bzip2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ - "bytes", - "either", + "libbz2-rs-sys", +] + +[[package]] +name = "cabac" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606873b80c9b424f9b6ed0cd03da139edc26d325eb9b31969ebf27c2bdd98f68" +dependencies = [ + "bytemuck", + "byteorder", ] [[package]] @@ -1814,6 +1545,16 @@ dependencies = [ "windows 0.54.0", ] +[[package]] +name = "cpu-time" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e393a7668fe1fad3075085b86c781883000b4ede868f43627b34a87c8b7ded" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "cpubits" version = "0.1.1" @@ -1853,16 +1594,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" -[[package]] -name = "crc-fast" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" -dependencies = [ - "digest 0.10.7", - "spin 0.10.0", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -2097,6 +1828,26 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" +[[package]] +name = "default-boxed" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48cc7cb0b3bcd8acee5eb26e45f8f4b597c5e57a67deb86e3d1d4a1e57a1c31" +dependencies = [ + "default-boxed-derive", +] + +[[package]] +name = "default-boxed-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94bbd6a078996f7e3af0e053377eb5577dee801eae2f22af2cd1e58fa33a0f5e" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "der" version = "0.7.10" @@ -2517,6 +2268,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd2e7510819d6fbf51a5545c8f922716ecfb14df168a3242f7d33e0239efe6a1" +[[package]] +name = "fastcdc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf51ceb43e96afbfe4dd5c6f6082af5dfd60e220820b8123792d61963f2ce6bc" + [[package]] name = "fastrand" version = "2.3.0" @@ -2607,7 +2364,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin 0.9.8", + "spin", ] [[package]] @@ -2943,6 +2700,32 @@ dependencies = [ "weezl 0.1.12", ] +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "glam" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f22fb22f065b308be0d8724e3706c7fa3fc2a6c7d6899df4cad7860e7a75436" + [[package]] name = "glidesort" version = "0.1.2" @@ -2983,25 +2766,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.13" @@ -3013,7 +2777,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http", "indexmap", "slab", "tokio", @@ -3194,21 +2958,21 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "h2 0.4.13", + "h2", "hickory-proto", - "http 1.4.0", + "http", "idna", "ipnet", "jni 0.22.4", "rand 0.10.1", - "rustls 0.23.37", + "rustls", "thiserror 2.0.18", "tinyvec", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tracing", "url", - "webpki-roots", + "webpki-roots 1.0.6", ] [[package]] @@ -3250,14 +3014,14 @@ dependencies = [ "parking_lot", "rand 0.10.1", "resolv-conf", - "rustls 0.23.37", + "rustls", "smallvec", "system-configuration", "thiserror 2.0.18", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tracing", - "webpki-roots", + "webpki-roots 1.0.6", ] [[package]] @@ -3322,17 +3086,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.4.0" @@ -3343,17 +3096,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -3361,7 +3103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http", ] [[package]] @@ -3372,8 +3114,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -3383,12 +3125,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "hybrid-array" version = "0.4.12" @@ -3398,30 +3134,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.9.0" @@ -3432,9 +3144,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.13", - "http 1.4.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "httparse", "itoa", "pin-project-lite", @@ -3443,34 +3155,18 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.0", - "hyper 1.9.0", + "http", + "hyper", "hyper-util", - "rustls 0.23.37", - "rustls-native-certs", + "rustls", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tower-service", ] @@ -3484,14 +3180,14 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "hyper 1.9.0", + "http", + "http-body", + "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "tokio", "tower-service", "tracing", @@ -3941,6 +3637,29 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "instant-xml" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8863a17b9487acadbfc6a54f1215b67695dcc56d760c69cc08a16ad5e8fd5d0e" +dependencies = [ + "instant-xml-macros", + "thiserror 2.0.18", + "xmlparser", +] + +[[package]] +name = "instant-xml-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44127a3a387c070ef0656a6ce53dd0e616cf8d6cf5b159aa478cfd49e1c166e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "io-surface" version = "0.16.1" @@ -3959,7 +3678,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ - "socket2 0.6.3", + "socket2", "widestring", "windows-registry", "windows-result 0.4.1", @@ -4214,7 +3933,7 @@ dependencies = [ "regex", "regex-syntax", "reqwest", - "rustls 0.23.37", + "rustls", "serde", "serde_json", "unicode-general-category", @@ -4232,11 +3951,12 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.6" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" dependencies = [ - "cpufeatures 0.2.17", + "cfg-if", + "cpufeatures 0.3.0", ] [[package]] @@ -4279,7 +3999,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin", ] [[package]] @@ -4303,6 +4023,29 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lepton_jpeg" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "540c312fa0f528f4cad6c7dbffb0b29d39f34026e133a4c79d23d95f2863e46e" +dependencies = [ + "bytemuck", + "byteorder", + "cpu-time", + "default-boxed", + "flate2", + "git-version", + "log", + "thread-priority", + "wide", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + [[package]] name = "libc" version = "0.2.183" @@ -4394,13 +4137,12 @@ dependencies = [ [[package]] name = "libwebp-sys" -version = "0.14.2" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375ca3fbd6d89769361c5d505c9da676eb4128ee471b9fd763144d377a2d30e6" +checksum = "54cd30df7c7165ce74a456e4ca9732c603e8dc5e60784558c1c6dc047f876733" dependencies = [ "cc", "glob", - "pkg-config", ] [[package]] @@ -4530,6 +4272,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rust2" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce716bf1a316f47a280fc76295f6495b5bea4752bca01c3b3885e101b1c23c02" +dependencies = [ + "sha2 0.11.0", +] + [[package]] name = "mac" version = "0.1.1" @@ -5532,12 +5283,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pinyin" version = "0.11.0" @@ -5718,6 +5463,24 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "precomp2" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d9879cdb87b5bd4d8be61fa457c8dca65004270dda0f70d2348db3298d7c7fe" +dependencies = [ + "base64", + "bzip2", + "crc32fast", + "lepton_jpeg", + "memchr", + "nom 7.1.3", + "preflate-rs", + "thiserror 2.0.18", + "tokio", + "webp", +] + [[package]] name = "precomputed-hash" version = "0.1.1" @@ -5735,6 +5498,19 @@ dependencies = [ "num-traits", ] +[[package]] +name = "preflate-rs" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2e9d0ac8c6cd9bb134681e11b5a32c7c56b89c5958cf785775080def38537c" +dependencies = [ + "bitcode", + "byteorder", + "cabac", + "default-boxed", + "deranged", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -5899,8 +5675,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.37", - "socket2 0.6.3", + "rustls", + "socket2", "thiserror 2.0.18", "tokio", "tracing", @@ -5920,7 +5696,7 @@ dependencies = [ "rand 0.9.4", "ring", "rustc-hash 2.1.1", - "rustls 0.23.37", + "rustls", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -5938,7 +5714,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.59.0", ] @@ -6201,12 +5977,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "regex-lite" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" - [[package]] name = "regex-syntax" version = "0.8.10" @@ -6224,26 +5994,26 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.13", - "http 1.4.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.9.0", - "hyper-rustls 0.27.9", + "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.37", + "rustls", "rustls-pki-types", "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tower", "tower-http", "tower-service", @@ -6391,18 +6161,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.23.37" @@ -6414,7 +6172,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.13", + "rustls-webpki", "subtle", "zeroize", ] @@ -6452,10 +6210,10 @@ dependencies = [ "jni 0.21.1", "log", "once_cell", - "rustls 0.23.37", + "rustls", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki 0.103.13", + "rustls-webpki", "security-framework", "security-framework-sys", "webpki-root-certs", @@ -6468,16 +6226,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustls-webpki" version = "0.103.13" @@ -6496,6 +6244,25 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-s3" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0d23aa8ac3b44d4cfb1e4b3611e6f3776debfb3f7701c4ea9f2252a701403" +dependencies = [ + "base64", + "hmac 0.13.0", + "instant-xml", + "jiff", + "md-5 0.11.0", + "percent-encoding", + "serde", + "serde_json", + "sha2 0.11.0", + "url", + "zeroize", +] + [[package]] name = "rustybuzz" version = "0.20.1" @@ -6520,6 +6287,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629516c85c29fe757770fa03f2074cf1eac43d44c02a3de9fc2ef7b0e207dfdd" +dependencies = [ + "bytemuck", +] + [[package]] name = "safefetch" version = "0.1.0" @@ -6530,10 +6306,10 @@ dependencies = [ "hickory-resolver", "image", "reqwest", - "rustls 0.23.37", + "rustls", "tokio", "url", - "webpki-roots", + "webpki-roots 1.0.6", ] [[package]] @@ -6626,16 +6402,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sec1" version = "0.7.3" @@ -6783,17 +6549,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - [[package]] name = "sha2" version = "0.10.9" @@ -6818,11 +6573,11 @@ dependencies = [ [[package]] name = "sha3" -version = "0.10.8" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ - "digest 0.10.7", + "digest 0.11.3", "keccak", ] @@ -6964,16 +6719,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -6993,12 +6738,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "spin" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" - [[package]] name = "spki" version = "0.7.3" @@ -7046,6 +6785,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", + "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -7055,6 +6795,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "webpki-roots 0.26.11", ] [[package]] @@ -7128,7 +6869,7 @@ dependencies = [ "rand 0.8.6", "rsa", "serde", - "sha1 0.10.6", + "sha1", "sha2 0.10.9", "smallvec", "sqlx-core", @@ -7734,6 +7475,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "thread-priority" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe075d7053dae61ac5413a34ea7d4913b6e6207844fd726bdd858b37ff72bf5" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "libc", + "log", + "rustversion", + "winapi", +] + [[package]] name = "thread_local" version = "1.1.9" @@ -7873,7 +7628,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] @@ -7889,23 +7644,13 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.37", + "rustls", "tokio", ] @@ -8037,8 +7782,8 @@ dependencies = [ "bitflags 2.11.0", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "iri-string", "pin-project-lite", "tower", @@ -8889,11 +8634,11 @@ dependencies = [ "flate2", "log", "percent-encoding", - "rustls 0.23.37", + "rustls", "rustls-pki-types", "ureq-proto", "utf8-zero", - "webpki-roots", + "webpki-roots 1.0.6", ] [[package]] @@ -8903,7 +8648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" dependencies = [ "base64", - "http 1.4.0", + "http", "httparse", "log", ] @@ -9165,7 +8910,7 @@ version = "0.51.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb321403ce594274827657a908e13d1d9918aa02257b8bf8391949d9764023ff" dependencies = [ - "spin 0.9.8", + "spin", "wasmi_collections", "wasmi_core", "wasmi_ir", @@ -9237,6 +8982,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webp" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c071456adef4aca59bf6a583c46b90ff5eb0b4f758fc347cea81290288f37ce1" +dependencies = [ + "image", + "libwebp-sys", +] + [[package]] name = "webpki-root-certs" version = "1.0.7" @@ -9246,6 +9001,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.6", +] + [[package]] name = "webpki-roots" version = "1.0.6" @@ -9286,12 +9050,38 @@ dependencies = [ "wasite", ] +[[package]] +name = "wide" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13ca908d26e4786149c48efcf6c0ea09ab0e06d1fe3c17dc1b4b0f1ca4a7e788" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "widestring" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -9301,6 +9091,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" version = "0.54.0" diff --git a/Cargo.toml b/Cargo.toml index 6825ce9386..e96fcec7a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,17 +17,12 @@ resolver = "3" aes-gcm = "0.10" affine_common = { path = "./packages/common/native" } affine_nbstore = { path = "./packages/frontend/native/nbstore" } - ahash = "0.8" anyhow = "1" - arbitrary = { version = "1.3", features = ["derive"] } assert-json-diff = "2.0" base64 = "0.22.1" base64-simd = "0.8" - bitvec = "1.0" block2 = "0.6" - byteorder = "1.5" chrono = "0.4" - clap = { version = "4.4", features = ["derive"] } core-foundation = "0.10" coreaudio-rs = "0.12" cpal = "0.15" @@ -48,14 +43,10 @@ resolver = "3" "webp", ] } infer = { version = "0.19.0" } - lasso = { version = "0.7", features = ["multi-threaded"] } - lib0 = { version = "0.16", features = ["lib0-serde"] } libc = "0.2" - libwebp-sys = "0.14.2" little_exif = "0.6.23" llm_adapter = { version = "0.2", default-features = false } llm_runtime = { version = "0.2", default-features = false } - log = "0.4" lru = "0.16" matroska = "0.30" memory-indexer = "0.3.1" @@ -72,33 +63,22 @@ resolver = "3" ] } napi-build = { version = "2" } napi-derive = { version = "3.4" } - nom = "8" - notify = { version = "8", features = ["serde"] } objc2 = "0.6" objc2-foundation = "0.3" ogg = "0.9" once_cell = "1" - ordered-float = "5" p256 = { version = "0.13", features = ["ecdsa", "pem"] } - parking_lot = "0.12" - phf = { version = "0.11", features = ["macros"] } - proptest = "1.3" - proptest-derive = "0.5" pulldown-cmark = "0.13" rand = "0.9" - rand_chacha = "0.9" - rand_distr = "0.5" rayon = "1.10" - regex = "1.10" rubato = "0.16" safefetch = "0.1.0" schemars = "0.8" screencapturekit = "0.3" serde = "1" serde_json = "1" - sha2 = "0.10" - sha3 = "0.10" - smol_str = "0.3" + sha2 = "0.11" + sha3 = "0.11" sqlx = { version = "0.8", default-features = false, features = [ "chrono", "macros", @@ -136,8 +116,6 @@ resolver = "3" ] } windows-core = { version = "0.61" } y-octo = "0.0.3" - y-sync = { version = "0.4" } - yrs = "0.23.0" [profile.dev.package.sqlx-macros] opt-level = 3 diff --git a/packages/backend/native/Cargo.toml b/packages/backend/native/Cargo.toml index 4ec61fec72..8aacf54ca1 100644 --- a/packages/backend/native/Cargo.toml +++ b/packages/backend/native/Cargo.toml @@ -16,17 +16,20 @@ affine_common = { workspace = true, features = [ "ydoc-loader", ] } anyhow = { workspace = true } -aws-sdk-s3 = "1.137.0" +assetpack-core = "0.1.0" +assetpack-transform-precomp2 = "0.1.0" base64 = { workspace = true } chrono = { workspace = true } +crc32fast = "1.5.0" doc_extractor = { workspace = true } file-format = { workspace = true } hex = { workspace = true } homedir = { workspace = true } image = { workspace = true } infer = { workspace = true } +instant-xml = "0.7.5" jsonschema = "0.46" -libwebp-sys = { workspace = true } +libwebp-sys = "0.9.6" little_exif = { workspace = true } llm_adapter = { workspace = true, features = ["schema", "ureq-client"] } llm_runtime = { workspace = true, features = ["schema", "ureq-client"] } @@ -36,6 +39,10 @@ napi = { workspace = true, features = ["async", "serde-json"] } napi-derive = { workspace = true } p256 = { workspace = true } rand = { workspace = true } +reqwest = { version = "0.13.4", default-features = false, features = [ + "rustls", +] } +rusty-s3 = "0.10.0" safefetch = { workspace = true } schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } @@ -50,6 +57,7 @@ sqlx = { workspace = true, default-features = false, features = [ "postgres", "runtime-tokio", ] } +thiserror.workspace = true tiktoken-rs = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "sync"] } url = { workspace = true } @@ -64,8 +72,9 @@ mimalloc = { workspace = true } mimalloc = { workspace = true, features = ["local_dynamic_tls"] } [dev-dependencies] -rayon = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +rayon = { workspace = true } +tempfile = "3.27.0" +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [build-dependencies] napi-build = { workspace = true } diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index 32de3166ed..925149d887 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -5,18 +5,9 @@ declare const _default: typeof import('./index') export default _default export declare class BackendRuntime { - planUnreferencedWorkspaceBlobs(workspaceId: string, gracePeriodDays: number, limit: number): Promise - executeBlobCleanupCandidates(runId: string, gracePeriodDays: number, limit: number): Promise - completeBlobUpload(workspaceId: string, key: string, expectedSize: number, expectedMime: string): Promise - completeFsBlobUpload(root: string, bucket: string, workspaceId: string, key: string, expectedSize: number, expectedMime: string): Promise - cleanupExpiredPendingBlobs(cutoffMs: number, limit: number): Promise - releaseDeletedBlobs(workspaceId: string, limit: number): Promise - backfillMissingBlobMetadata(workspaceId: string | undefined | null, limit: number): Promise acquireCoordinationLease(key: string, owner: string, ttlMs: number): Promise releaseCoordinationLease(key: string, owner: string, fencingToken: bigint | number): Promise renewCoordinationLease(key: string, owner: string, fencingToken: bigint | number, ttlMs: number): Promise - rebuildDocBlobRefs(workspaceId: string, docId: string): Promise - rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number): Promise /** * Merge pending doc updates with y-octo and persist the merged snapshot. * @@ -36,18 +27,6 @@ export declare class BackendRuntime { cleanupExpiredRuntimeGates(limit: number): Promise cleanupExpiredUserSessions(limit: number): Promise cleanupExpiredSnapshotHistories(limit: number): Promise - objectStorageHealth(): RuntimeObjectStorageHealth - objectStoragePut(key: string, body: Buffer, metadata?: RuntimeObjectStoragePutOptions | undefined | null): Promise - objectStoragePresignPut(key: string, metadata?: RuntimeObjectStoragePutOptions | undefined | null): Promise - objectStorageCreateMultipartUpload(key: string, metadata?: RuntimeObjectStoragePutOptions | undefined | null): Promise - objectStoragePresignUploadPart(key: string, uploadId: string, partNumber: number): Promise - objectStorageListMultipartUploadParts(key: string, uploadId: string): Promise> - objectStorageCompleteMultipartUpload(key: string, uploadId: string, parts: Array): Promise - objectStorageAbortMultipartUpload(key: string, uploadId: string): Promise - objectStorageHead(key: string): Promise - objectStorageGet(key: string): Promise - objectStorageList(prefix?: string | undefined | null): Promise> - objectStorageDelete(key: string): Promise createAuthChallenge(purpose: string, token: string, payload: any, ttlMs: number): Promise getAuthChallenge(purpose: string, token: string): Promise consumeAuthChallenge(purpose: string, token: string): Promise @@ -79,6 +58,36 @@ export declare class LlmStreamHandle { abort(): void } +export declare class StorageRuntime { + planUnreferencedWorkspaceBlobs(workspaceId: string, gracePeriodDays: number, limit: number): Promise + executeBlobCleanupCandidates(runId: string, gracePeriodDays: number, limit: number): Promise + cleanupExpiredPendingBlobs(cutoffMs: number, limit: number): Promise + releaseDeletedBlobs(workspaceId: string, limit: number): Promise + backfillMissingBlobMetadata(workspaceId: string | undefined | null, limit: number): Promise + rebuildDocBlobRefs(workspaceId: string, docId: string): Promise + rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number): Promise + constructor() + start(): Promise + stop(): Promise + runMigrations(): Promise + health(): Promise + providerCapabilities(scope: string): Promise + putObject(scope: string, key: string, body: Buffer, metadata?: RuntimeObjectStoragePutOptions | undefined | null): Promise + headObject(scope: string, key: string): Promise + getObject(scope: string, key: string): Promise + listObjects(scope: string, prefix?: string | undefined | null): Promise> + deleteObject(scope: string, key: string): Promise + presignPut(scope: string, key: string, metadata?: RuntimeObjectStoragePutOptions | undefined | null): Promise + presignGet(scope: string, key: string): Promise + createMultipartUpload(scope: string, key: string, metadata?: RuntimeObjectStoragePutOptions | undefined | null): Promise + presignUploadPart(scope: string, key: string, uploadId: string, partNumber: number): Promise + proxyUploadPart(scope: string, key: string, uploadId: string, partNumber: number, body: Buffer, contentLength?: number | undefined | null): Promise + listMultipartUploadParts(scope: string, key: string, uploadId: string): Promise | null> + completeMultipartUpload(scope: string, key: string, uploadId: string, parts: Array): Promise + abortMultipartUpload(scope: string, key: string, uploadId: string): Promise + completeWorkspaceBlobUpload(workspaceId: string, key: string, expectedSize: number, expectedMime: string): Promise +} + export declare class Tokenizer { count(content: string, allowedSpecial?: Array | undefined | null): number } @@ -152,7 +161,6 @@ export interface AssertSafeUrlRequest { export interface BackendRuntimeHealth { started: boolean databaseConnected: boolean - objectStorageConfigured: boolean } export declare function buildPublicRootDoc(rootDocBin: Buffer, docMetas: Array): Buffer @@ -514,12 +522,6 @@ export declare function llmValidateJsonSchema(schema: any, value: any): any */ export declare function mergeUpdatesInApplyWay(updates: Array): Buffer -/** - * Check whether a Yjs update binary can be decoded without applying it to a - * document state. - */ -export declare function validateDocUpdate(update: Buffer): Promise - export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise export interface ModelConditionsContract { @@ -945,22 +947,6 @@ export interface RuntimeObjectMetadata { checksumCrc32?: string } -export interface RuntimeObjectStorageHealth { - configured: boolean - provider?: string - bucket?: string - endpoint?: string - region?: string - hasCredentials: boolean - forcePathStyle: boolean - requestTimeoutMs?: number - minPartSize?: number - presignExpiresInSeconds?: number - presignSignContentTypeForPut?: boolean - usePresignedUrl: boolean - clientBuildable: boolean -} - export interface RuntimeObjectStoragePutOptions { contentType?: string contentLength?: number @@ -1043,6 +1029,28 @@ export interface SafeFetchResponse { body: Buffer } +export interface StorageProviderCapabilities { + put: boolean + get: boolean + head: boolean + list: boolean + delete: boolean + presignPut: boolean + presignGet: boolean + multipartDirect: boolean + proxyUpload: boolean + assetpack: boolean + serverMediatedOnly: boolean +} + +export interface StorageRuntimeHealth { + started: boolean + databaseConnected: boolean + providerConfigured: boolean + provider?: string + bucket?: string +} + export interface ToolContract { name: string description?: string @@ -1109,4 +1117,10 @@ export declare function updateLicenseSeats(request: LicenseSeatsRequest): Promis */ export declare function updateRootDocMetaTitle(rootDocBin: Buffer, docId: string, title: string): Buffer +/** + * Check whether a Yjs update binary can be decoded without applying it to a + * document state. + */ +export declare function validateDocUpdate(update: Buffer): Promise + export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise diff --git a/packages/backend/native/src/backend_runtime/blob_complete.rs b/packages/backend/native/src/backend_runtime/blob_complete.rs deleted file mode 100644 index 28fe0334ac..0000000000 --- a/packages/backend/native/src/backend_runtime/blob_complete.rs +++ /dev/null @@ -1,296 +0,0 @@ -use std::{ - fs, - path::{Path, PathBuf}, -}; - -use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use napi::Result; -use serde::Deserialize; -use sha2::{Digest, Sha256}; - -use super::{BackendRuntime, error::napi_error, types::RuntimeBlobCompleteResult}; - -const MAX_BLOB_SIZE: i64 = i32::MAX as i64; - -fn object_missing_error(err: &napi::Error) -> bool { - let message = err.to_string(); - message.contains("NoSuchKey") || message.contains("NotFound") || message.contains("not found") -} - -fn blob_complete_failure(reason: &str) -> RuntimeBlobCompleteResult { - RuntimeBlobCompleteResult { - ok: false, - reason: Some(reason.to_string()), - content_type: None, - content_length: None, - last_modified_ms: None, - } -} - -fn blob_complete_success( - content_type: String, - content_length: i64, - last_modified_ms: i64, -) -> RuntimeBlobCompleteResult { - RuntimeBlobCompleteResult { - ok: true, - reason: None, - content_type: Some(content_type), - content_length: Some(content_length), - last_modified_ms: Some(last_modified_ms), - } -} - -fn normalize_base64_url_key(key: &str) -> &str { - key.trim_end_matches('=') -} - -fn sha256_base64_url(body: &[u8]) -> String { - URL_SAFE_NO_PAD.encode(Sha256::digest(body)) -} - -fn sha256_base64_url_matches(body: &[u8], key: &str) -> bool { - sha256_base64_url(body) == normalize_base64_url_key(key) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct FsBlobMetadata { - content_type: String, - content_length: i64, - last_modified: i64, -} - -fn normalize_storage_key(key: &str) -> Result> { - let normalized = key.replace('\\', "/"); - let segments = normalized.split('/').map(ToString::to_string).collect::>(); - - if normalized.is_empty() - || normalized.starts_with('/') - || segments - .iter() - .any(|segment| segment.is_empty() || segment == "." || segment == "..") - { - return Err(napi_error(format!("Invalid storage key: {key}"))); - } - - Ok(segments) -} - -fn fs_bucket_path(root: &str, bucket: &str) -> PathBuf { - if let Some(stripped) = root.strip_prefix("~/") - && let Ok(Some(home)) = homedir::my_home() - { - return home.join(stripped).join(bucket); - } - - Path::new(root).join(bucket) -} - -fn fs_object_path(root: &str, bucket: &str, key: &str) -> Result { - let mut path = fs_bucket_path(root, bucket); - for segment in normalize_storage_key(key)? { - path.push(segment); - } - Ok(path) -} - -fn read_fs_metadata(path: &Path) -> Result> { - let metadata_path = PathBuf::from(format!("{}.metadata.json", path.display())); - let raw = match fs::read_to_string(metadata_path) { - Ok(raw) => raw, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(err) => { - return Err(napi_error(format!("BlobComplete read fs metadata failed: {err}"))); - } - }; - - serde_json::from_str(&raw).map(Some).map_err(|err| { - napi_error(format!( - "BlobComplete parse fs metadata failed for {}: {err}", - path.display() - )) - }) -} - -async fn upsert_completed_blob( - runtime: &BackendRuntime, - workspace_id: &str, - key: &str, - mime: &str, - size: i64, -) -> Result<()> { - if !(0..=MAX_BLOB_SIZE).contains(&size) { - return Err(napi_error("BlobComplete size exceeds limit")); - } - let size = i32::try_from(size).map_err(|_| napi_error("BlobComplete size exceeds limit"))?; - - sqlx::query( - r#" - INSERT INTO blobs (workspace_id, key, mime, size, status, upload_id) - VALUES ($1, $2, $3, $4, 'completed', NULL) - ON CONFLICT (workspace_id, key) - DO UPDATE SET - mime = EXCLUDED.mime, - size = EXCLUDED.size, - status = EXCLUDED.status, - upload_id = NULL - "#, - ) - .bind(workspace_id) - .bind(key) - .bind(mime) - .bind(size) - .execute(&runtime.pool().await?) - .await - .map_err(|err| napi_error(format!("BlobComplete upsert metadata failed: {err}")))?; - - Ok(()) -} - -#[napi_derive::napi] -impl BackendRuntime { - #[napi] - pub async fn complete_blob_upload( - &self, - workspace_id: String, - key: String, - expected_size: i64, - expected_mime: String, - ) -> Result { - if !(0..=MAX_BLOB_SIZE).contains(&expected_size) { - return Ok(blob_complete_failure("size_too_large")); - } - - let object_key = format!("{workspace_id}/{key}"); - let object = match self.object_storage_get(object_key.clone()).await { - Ok(Some(object)) => object, - Ok(None) => return Ok(blob_complete_failure("not_found")), - Err(err) if object_missing_error(&err) => return Ok(blob_complete_failure("not_found")), - Err(err) => return Err(err), - }; - - if !(0..=MAX_BLOB_SIZE).contains(&object.metadata.content_length) { - match self.object_storage_delete(object_key).await { - Ok(()) => {} - Err(err) if object_missing_error(&err) => {} - Err(err) => return Err(err), - } - return Ok(blob_complete_failure("size_too_large")); - } - if object.metadata.content_length != expected_size { - return Ok(blob_complete_failure("size_mismatch")); - } - - if !expected_mime.is_empty() && object.metadata.content_type != expected_mime { - return Ok(blob_complete_failure("mime_mismatch")); - } - - if !sha256_base64_url_matches(&object.body, &key) { - match self.object_storage_delete(object_key).await { - Ok(()) => {} - Err(err) if object_missing_error(&err) => {} - Err(err) => return Err(err), - } - return Ok(blob_complete_failure("checksum_mismatch")); - } - - upsert_completed_blob( - self, - &workspace_id, - &key, - &object.metadata.content_type, - object.metadata.content_length, - ) - .await?; - - Ok(blob_complete_success( - object.metadata.content_type, - object.metadata.content_length, - object.metadata.last_modified_ms, - )) - } - - #[napi] - pub async fn complete_fs_blob_upload( - &self, - root: String, - bucket: String, - workspace_id: String, - key: String, - expected_size: i64, - expected_mime: String, - ) -> Result { - if !(0..=MAX_BLOB_SIZE).contains(&expected_size) { - return Ok(blob_complete_failure("size_too_large")); - } - - let storage_key = format!("{workspace_id}/{key}"); - let path = fs_object_path(&root, &bucket, &storage_key)?; - let metadata = match read_fs_metadata(&path)? { - Some(metadata) => metadata, - None => return Ok(blob_complete_failure("not_found")), - }; - - if !(0..=MAX_BLOB_SIZE).contains(&metadata.content_length) { - let _ = fs::remove_file(&path); - let _ = fs::remove_file(PathBuf::from(format!("{}.metadata.json", path.display()))); - return Ok(blob_complete_failure("size_too_large")); - } - if metadata.content_length != expected_size { - return Ok(blob_complete_failure("size_mismatch")); - } - - if !expected_mime.is_empty() && metadata.content_type != expected_mime { - return Ok(blob_complete_failure("mime_mismatch")); - } - - let body = match fs::read(&path) { - Ok(body) => body, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(blob_complete_failure("not_found")), - Err(err) => return Err(napi_error(format!("BlobComplete read fs object failed: {err}"))), - }; - - if !sha256_base64_url_matches(&body, &key) { - let _ = fs::remove_file(&path); - let _ = fs::remove_file(PathBuf::from(format!("{}.metadata.json", path.display()))); - return Ok(blob_complete_failure("checksum_mismatch")); - } - - upsert_completed_blob( - self, - &workspace_id, - &key, - &metadata.content_type, - metadata.content_length, - ) - .await?; - - Ok(blob_complete_success( - metadata.content_type, - metadata.content_length, - metadata.last_modified, - )) - } -} - -#[cfg(test)] -mod tests { - use super::{sha256_base64_url, sha256_base64_url_matches}; - - #[test] - fn sha256_base64_url_omits_padding() { - assert_eq!( - sha256_base64_url(b"hello"), - "LPJNul-wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ" - ); - } - - #[test] - fn sha256_base64_url_matches_legacy_padding() { - assert!(sha256_base64_url_matches( - b"hello", - "LPJNul-wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ=" - )); - } -} diff --git a/packages/backend/native/src/backend_runtime/error.rs b/packages/backend/native/src/backend_runtime/error.rs deleted file mode 100644 index fd0a38d1b4..0000000000 --- a/packages/backend/native/src/backend_runtime/error.rs +++ /dev/null @@ -1,5 +0,0 @@ -use napi::{Error, Status}; - -pub(super) fn napi_error(message: impl Into) -> Error { - Error::new(Status::GenericFailure, message.into()) -} diff --git a/packages/backend/native/src/backend_runtime/object_storage/client.rs b/packages/backend/native/src/backend_runtime/object_storage/client.rs deleted file mode 100644 index 300c19b852..0000000000 --- a/packages/backend/native/src/backend_runtime/object_storage/client.rs +++ /dev/null @@ -1,389 +0,0 @@ -use std::{ - collections::HashMap, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use aws_sdk_s3::{ - Client as S3Client, presigning::PresigningConfig, primitives::ByteStream, types::CompletedMultipartUpload, -}; -use napi::Result; - -use super::types::{ - MultipartUploadInitResult, MultipartUploadPart, ObjectGetResult, ObjectListEntry, ObjectListPage, ObjectMetadata, - ObjectPutMetadata, PresignedObjectRequest, completed_multipart_parts, trim_etag, -}; -use crate::backend_runtime::error::napi_error; - -#[derive(Clone)] -pub(super) struct ObjectStorageClient { - client: S3Client, - bucket: String, - presign_expires_in_seconds: u64, - presign_sign_content_type_for_put: bool, -} - -impl ObjectStorageClient { - pub(super) fn new( - config: aws_sdk_s3::Config, - bucket: String, - presign_expires_in_seconds: u64, - presign_sign_content_type_for_put: bool, - ) -> Self { - Self { - client: S3Client::from_conf(config), - bucket, - presign_expires_in_seconds, - presign_sign_content_type_for_put, - } - } - - pub(super) fn non_destructive_health(&self) -> bool { - let _ = &self.client; - !self.bucket.is_empty() - } - - pub(super) async fn put(&self, key: &str, body: Vec, metadata: ObjectPutMetadata) -> Result<()> { - let content_length = metadata.content_length.unwrap_or(body.len() as i64); - let content_type = metadata - .content_type - .unwrap_or_else(|| "application/octet-stream".to_string()); - - let mut request = self - .client - .put_object() - .bucket(&self.bucket) - .key(key) - .body(ByteStream::from(body)) - .content_type(content_type) - .content_length(content_length); - - if let Some(checksum) = metadata.checksum_crc32 { - request = request.checksum_crc32(checksum); - } - - request - .send() - .await - .map_err(|err| napi_error(format!("ObjectStorage put failed for {key}: {err:?}")))?; - Ok(()) - } - - pub(super) async fn presign_put(&self, key: &str, metadata: ObjectPutMetadata) -> Result { - let content_type = metadata - .content_type - .unwrap_or_else(|| "application/octet-stream".to_string()); - let expires_at_ms = expires_at_ms(self.presign_expires_in_seconds)?; - let config = PresigningConfig::expires_in(Duration::from_secs(self.presign_expires_in_seconds)) - .map_err(|err| napi_error(format!("ObjectStorage presign config failed: {err}")))?; - - let mut request = self.client.put_object().bucket(&self.bucket).key(key); - if self.presign_sign_content_type_for_put { - request = request.content_type(content_type.clone()); - } - if let Some(content_length) = metadata.content_length { - request = request.content_length(content_length); - } - - let presigned = request - .presigned(config) - .await - .map_err(|err| napi_error(format!("ObjectStorage presign put failed for {key}: {err}")))?; - let mut headers = presigned_headers(&presigned); - headers.insert("Content-Type".to_string(), content_type); - - Ok(PresignedObjectRequest { - url: presigned.uri().to_string(), - headers, - expires_at_ms, - }) - } - - pub(super) async fn create_multipart_upload( - &self, - key: &str, - metadata: ObjectPutMetadata, - ) -> Result> { - let content_type = metadata - .content_type - .unwrap_or_else(|| "application/octet-stream".to_string()); - let result = self - .client - .create_multipart_upload() - .bucket(&self.bucket) - .key(key) - .content_type(content_type) - .send() - .await - .map_err(|err| { - napi_error(format!( - "ObjectStorage create multipart upload failed for {key}: {err:?}" - )) - })?; - - let expires_at_ms = expires_at_ms(self.presign_expires_in_seconds)?; - Ok(result.upload_id.map(|upload_id| MultipartUploadInitResult { - upload_id, - expires_at_ms, - })) - } - - pub(super) async fn presign_upload_part( - &self, - key: &str, - upload_id: &str, - part_number: i32, - ) -> Result { - let expires_at_ms = expires_at_ms(self.presign_expires_in_seconds)?; - let config = PresigningConfig::expires_in(Duration::from_secs(self.presign_expires_in_seconds)) - .map_err(|err| napi_error(format!("ObjectStorage presign config failed: {err}")))?; - let presigned = self - .client - .upload_part() - .bucket(&self.bucket) - .key(key) - .upload_id(upload_id) - .part_number(part_number) - .presigned(config) - .await - .map_err(|err| napi_error(format!("ObjectStorage presign upload part failed for {key}: {err}")))?; - - Ok(PresignedObjectRequest { - url: presigned.uri().to_string(), - headers: presigned_headers(&presigned), - expires_at_ms, - }) - } - - pub(super) async fn list_multipart_upload_parts( - &self, - key: &str, - upload_id: &str, - ) -> Result> { - let result = self - .client - .list_parts() - .bucket(&self.bucket) - .key(key) - .upload_id(upload_id) - .send() - .await - .map_err(|err| { - napi_error(format!( - "ObjectStorage list multipart upload parts failed for {key}: {err}" - )) - })?; - - Ok( - result - .parts() - .iter() - .filter_map(|part| { - Some(MultipartUploadPart { - part_number: part.part_number?, - etag: trim_etag(part.e_tag.as_deref().unwrap_or_default()), - }) - }) - .collect(), - ) - } - - pub(super) async fn complete_multipart_upload( - &self, - key: &str, - upload_id: &str, - parts: Vec, - ) -> Result<()> { - let ordered_parts = completed_multipart_parts(parts); - self - .client - .complete_multipart_upload() - .bucket(&self.bucket) - .key(key) - .upload_id(upload_id) - .multipart_upload( - CompletedMultipartUpload::builder() - .set_parts(Some(ordered_parts)) - .build(), - ) - .send() - .await - .map_err(|err| { - napi_error(format!( - "ObjectStorage complete multipart upload failed for {key}: {err}" - )) - })?; - Ok(()) - } - - pub(super) async fn abort_multipart_upload(&self, key: &str, upload_id: &str) -> Result<()> { - self - .client - .abort_multipart_upload() - .bucket(&self.bucket) - .key(key) - .upload_id(upload_id) - .send() - .await - .map_err(|err| { - napi_error(format!( - "ObjectStorage abort multipart upload failed for {key}: {err:?}" - )) - })?; - Ok(()) - } - - pub(super) async fn head(&self, key: &str) -> Result> { - let result = match self.client.head_object().bucket(&self.bucket).key(key).send().await { - Ok(result) => result, - Err(err) if is_not_found_error(&err) => return Ok(None), - Err(err) => return Err(napi_error(format!("ObjectStorage head failed for {key}: {err:?}"))), - }; - - Ok(Some(ObjectMetadata { - content_type: result - .content_type - .unwrap_or_else(|| "application/octet-stream".to_string()), - content_length: result.content_length.unwrap_or(0), - last_modified_ms: optional_datetime_ms(result.last_modified), - checksum_crc32: result.checksum_crc32, - })) - } - - pub(super) async fn get(&self, key: &str) -> Result> { - let result = match self.client.get_object().bucket(&self.bucket).key(key).send().await { - Ok(result) => result, - Err(err) if is_not_found_error(&err) => return Ok(None), - Err(err) => return Err(napi_error(format!("ObjectStorage get failed for {key}: {err:?}"))), - }; - let metadata = ObjectMetadata { - content_type: result - .content_type - .unwrap_or_else(|| "application/octet-stream".to_string()), - content_length: result.content_length.unwrap_or(0), - last_modified_ms: optional_datetime_ms(result.last_modified), - checksum_crc32: result.checksum_crc32, - }; - let body = result - .body - .collect() - .await - .map_err(|err| napi_error(format!("ObjectStorage read body failed for {key}: {err}")))? - .into_bytes() - .to_vec(); - - Ok(Some(ObjectGetResult { body, metadata })) - } - - pub(super) async fn list(&self, prefix: Option) -> Result> { - let mut entries = Vec::new(); - let mut token = None; - loop { - let mut request = self.client.list_objects_v2().bucket(&self.bucket); - if let Some(prefix) = &prefix { - request = request.prefix(prefix); - } - if let Some(next_token) = token { - request = request.continuation_token(next_token); - } - let result = request - .send() - .await - .map_err(|err| napi_error(format!("ObjectStorage list failed: {err:?}")))?; - - entries.extend(result.contents().iter().filter_map(|object| { - Some(ObjectListEntry { - key: object.key.as_ref()?.clone(), - content_length: object.size.unwrap_or(0), - last_modified_ms: optional_datetime_ms(object.last_modified), - }) - })); - - if result.is_truncated.unwrap_or(false) { - token = result.next_continuation_token; - } else { - break; - } - } - - Ok(entries) - } - - pub(super) async fn list_page( - &self, - prefix: Option, - continuation_token: Option, - start_after: Option, - max_keys: i32, - ) -> Result { - let mut request = self.client.list_objects_v2().bucket(&self.bucket).max_keys(max_keys); - if let Some(prefix) = prefix { - request = request.prefix(prefix); - } - if let Some(continuation_token) = continuation_token { - request = request.continuation_token(continuation_token); - } else if let Some(start_after) = start_after { - request = request.start_after(start_after); - } - let result = request - .send() - .await - .map_err(|err| napi_error(format!("ObjectStorage list page failed: {err:?}")))?; - - Ok(ObjectListPage { - entries: result - .contents() - .iter() - .filter_map(|object| { - Some(ObjectListEntry { - key: object.key.as_ref()?.clone(), - content_length: object.size.unwrap_or(0), - last_modified_ms: optional_datetime_ms(object.last_modified), - }) - }) - .collect(), - next_continuation_token: result.next_continuation_token, - }) - } - - pub(super) async fn delete(&self, key: &str) -> Result<()> { - self - .client - .delete_object() - .bucket(&self.bucket) - .key(key) - .send() - .await - .map_err(|err| napi_error(format!("ObjectStorage delete failed for {key}: {err:?}")))?; - Ok(()) - } -} - -fn is_not_found_error(error: &impl std::fmt::Debug) -> bool { - let message = format!("{error:?}"); - message.contains("NoSuchKey") || (message.contains("NotFound") && !message.contains("NoSuchBucket")) -} - -fn expires_at_ms(expires_in_seconds: u64) -> Result { - let expires_at = SystemTime::now() - .checked_add(Duration::from_secs(expires_in_seconds)) - .ok_or_else(|| napi_error("ObjectStorage presign expiration overflow"))?; - system_time_ms(expires_at) -} - -fn system_time_ms(time: SystemTime) -> Result { - let duration = time - .duration_since(UNIX_EPOCH) - .map_err(|err| napi_error(format!("system time before unix epoch: {err}")))?; - Ok(duration.as_millis() as i64) -} - -fn optional_datetime_ms(time: Option) -> i64 { - time.and_then(|value| value.to_millis().ok()).unwrap_or(0) -} - -fn presigned_headers(request: &aws_sdk_s3::presigning::PresignedRequest) -> HashMap { - request - .headers() - .map(|(key, value)| (key.to_string(), value.to_string())) - .collect() -} diff --git a/packages/backend/native/src/backend_runtime/object_storage/mod.rs b/packages/backend/native/src/backend_runtime/object_storage/mod.rs deleted file mode 100644 index 4918c68efd..0000000000 --- a/packages/backend/native/src/backend_runtime/object_storage/mod.rs +++ /dev/null @@ -1,197 +0,0 @@ -mod client; -mod config; -#[cfg(test)] -mod tests; -mod types; - -use client::ObjectStorageClient; -pub(super) use config::ObjectStorageConfig; -use napi::{Result, bindgen_prelude::Buffer}; -use types::ObjectListPage; -pub(super) use types::StorageProviderConfig; - -use super::{ - BackendRuntime, - types::{ - RuntimeMultipartUploadInit, RuntimeMultipartUploadPart, RuntimeObjectGetResult, RuntimeObjectListEntry, - RuntimeObjectMetadata, RuntimeObjectStorageHealth, RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest, - }, -}; - -#[napi_derive::napi] -impl BackendRuntime { - fn object_storage_client(&self) -> Result { - let storage = self - .config()? - .storage - .ok_or_else(|| super::error::napi_error("ObjectStorageClient is not configured"))?; - storage.build_client() - } - - pub(super) async fn object_storage_delete_object(&self, key: &str) -> Result<()> { - self.object_storage_client()?.delete(key).await - } - - pub(super) async fn object_storage_list_page( - &self, - prefix: Option, - continuation_token: Option, - start_after: Option, - max_keys: i32, - ) -> Result { - self - .object_storage_client()? - .list_page(prefix, continuation_token, start_after, max_keys) - .await - } - - pub(super) async fn object_storage_abort_upload(&self, key: &str, upload_id: &str) -> Result<()> { - self - .object_storage_client()? - .abort_multipart_upload(key, upload_id) - .await - } - - #[napi] - pub fn object_storage_health(&self) -> RuntimeObjectStorageHealth { - match self.config().ok().and_then(|config| config.storage) { - Some(storage) => storage.health(), - None => RuntimeObjectStorageHealth { - configured: false, - provider: None, - bucket: None, - endpoint: None, - region: None, - has_credentials: false, - force_path_style: false, - request_timeout_ms: None, - min_part_size: None, - presign_expires_in_seconds: None, - presign_sign_content_type_for_put: None, - use_presigned_url: false, - client_buildable: false, - }, - } - } - - #[napi] - pub async fn object_storage_put( - &self, - key: String, - body: Buffer, - metadata: Option, - ) -> Result<()> { - self - .object_storage_client()? - .put(&key, body.to_vec(), metadata.map(Into::into).unwrap_or_default()) - .await - } - - #[napi] - pub async fn object_storage_presign_put( - &self, - key: String, - metadata: Option, - ) -> Result { - self - .object_storage_client()? - .presign_put(&key, metadata.map(Into::into).unwrap_or_default()) - .await? - .try_into() - } - - #[napi] - pub async fn object_storage_create_multipart_upload( - &self, - key: String, - metadata: Option, - ) -> Result> { - Ok( - self - .object_storage_client()? - .create_multipart_upload(&key, metadata.map(Into::into).unwrap_or_default()) - .await? - .map(Into::into), - ) - } - - #[napi] - pub async fn object_storage_presign_upload_part( - &self, - key: String, - upload_id: String, - part_number: i32, - ) -> Result { - self - .object_storage_client()? - .presign_upload_part(&key, &upload_id, part_number) - .await? - .try_into() - } - - #[napi] - pub async fn object_storage_list_multipart_upload_parts( - &self, - key: String, - upload_id: String, - ) -> Result> { - Ok( - self - .object_storage_client()? - .list_multipart_upload_parts(&key, &upload_id) - .await? - .into_iter() - .map(Into::into) - .collect(), - ) - } - - #[napi] - pub async fn object_storage_complete_multipart_upload( - &self, - key: String, - upload_id: String, - parts: Vec, - ) -> Result<()> { - self - .object_storage_client()? - .complete_multipart_upload(&key, &upload_id, parts.into_iter().map(Into::into).collect()) - .await - } - - #[napi] - pub async fn object_storage_abort_multipart_upload(&self, key: String, upload_id: String) -> Result<()> { - self - .object_storage_client()? - .abort_multipart_upload(&key, &upload_id) - .await - } - - #[napi] - pub async fn object_storage_head(&self, key: String) -> Result> { - Ok(self.object_storage_client()?.head(&key).await?.map(Into::into)) - } - - #[napi] - pub async fn object_storage_get(&self, key: String) -> Result> { - Ok(self.object_storage_client()?.get(&key).await?.map(Into::into)) - } - - #[napi] - pub async fn object_storage_list(&self, prefix: Option) -> Result> { - Ok( - self - .object_storage_client()? - .list(prefix) - .await? - .into_iter() - .map(Into::into) - .collect(), - ) - } - - #[napi] - pub async fn object_storage_delete(&self, key: String) -> Result<()> { - self.object_storage_client()?.delete(&key).await - } -} diff --git a/packages/backend/native/src/backend_runtime/object_storage/types.rs b/packages/backend/native/src/backend_runtime/object_storage/types.rs deleted file mode 100644 index f16715f39b..0000000000 --- a/packages/backend/native/src/backend_runtime/object_storage/types.rs +++ /dev/null @@ -1,171 +0,0 @@ -use std::collections::HashMap; - -use aws_sdk_s3::types::CompletedPart; -use napi::Result; -use serde::Deserialize; - -use crate::backend_runtime::{ - error::napi_error, - types::{ - RuntimeMultipartUploadInit, RuntimeMultipartUploadPart, RuntimeObjectGetResult, RuntimeObjectListEntry, - RuntimeObjectMetadata, RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest, - }, -}; - -#[derive(Clone, Debug, Default)] -pub(super) struct ObjectPutMetadata { - pub(super) content_type: Option, - pub(super) content_length: Option, - pub(super) checksum_crc32: Option, -} - -#[derive(Clone, Debug, PartialEq)] -pub(super) struct ObjectMetadata { - pub(super) content_type: String, - pub(super) content_length: i64, - pub(super) last_modified_ms: i64, - pub(super) checksum_crc32: Option, -} - -#[derive(Clone, Debug, PartialEq)] -pub(in crate::backend_runtime) struct ObjectListEntry { - pub(in crate::backend_runtime) key: String, - pub(in crate::backend_runtime) content_length: i64, - pub(in crate::backend_runtime) last_modified_ms: i64, -} - -#[derive(Clone, Debug, PartialEq)] -pub(in crate::backend_runtime) struct ObjectListPage { - pub(in crate::backend_runtime) entries: Vec, - pub(in crate::backend_runtime) next_continuation_token: Option, -} - -#[derive(Clone, Debug, PartialEq)] -pub(super) struct ObjectGetResult { - pub(super) body: Vec, - pub(super) metadata: ObjectMetadata, -} - -#[derive(Clone, Debug, PartialEq)] -pub(super) struct PresignedObjectRequest { - pub(super) url: String, - pub(super) headers: HashMap, - pub(super) expires_at_ms: i64, -} - -#[derive(Clone, Debug, PartialEq)] -pub(super) struct MultipartUploadInitResult { - pub(super) upload_id: String, - pub(super) expires_at_ms: i64, -} - -#[derive(Clone, Debug, PartialEq)] -pub(super) struct MultipartUploadPart { - pub(super) part_number: i32, - pub(super) etag: String, -} - -#[derive(Clone, Debug, Deserialize)] -pub(in crate::backend_runtime) struct StorageProviderConfig { - pub(super) provider: String, - pub(super) bucket: String, - #[serde(default)] - pub(super) config: serde_json::Value, -} - -pub(super) fn trim_etag(etag: &str) -> String { - etag.trim_matches('"').to_string() -} - -pub(super) fn completed_multipart_parts(mut parts: Vec) -> Vec { - parts.sort_by_key(|part| part.part_number); - parts - .into_iter() - .map(|part| { - CompletedPart::builder() - .part_number(part.part_number) - .e_tag(part.etag) - .build() - }) - .collect() -} - -impl From for ObjectPutMetadata { - fn from(options: RuntimeObjectStoragePutOptions) -> Self { - Self { - content_type: options.content_type, - content_length: options.content_length, - checksum_crc32: options.checksum_crc32, - } - } -} - -impl From for RuntimeObjectMetadata { - fn from(metadata: ObjectMetadata) -> Self { - Self { - content_type: metadata.content_type, - content_length: metadata.content_length, - last_modified_ms: metadata.last_modified_ms, - checksum_crc32: metadata.checksum_crc32, - } - } -} - -impl From for RuntimeObjectListEntry { - fn from(entry: ObjectListEntry) -> Self { - Self { - key: entry.key, - content_length: entry.content_length, - last_modified_ms: entry.last_modified_ms, - } - } -} - -impl TryFrom for RuntimePresignedObjectRequest { - type Error = napi::Error; - - fn try_from(request: PresignedObjectRequest) -> Result { - Ok(Self { - url: request.url, - headers_json: serde_json::to_string(&request.headers) - .map_err(|err| napi_error(format!("ObjectStorage headers serialization failed: {err}")))?, - expires_at_ms: request.expires_at_ms, - }) - } -} - -impl From for RuntimeObjectGetResult { - fn from(result: ObjectGetResult) -> Self { - Self { - body: result.body.into(), - metadata: result.metadata.into(), - } - } -} - -impl From for RuntimeMultipartUploadInit { - fn from(init: MultipartUploadInitResult) -> Self { - Self { - upload_id: init.upload_id, - expires_at_ms: init.expires_at_ms, - } - } -} - -impl From for MultipartUploadPart { - fn from(part: RuntimeMultipartUploadPart) -> Self { - Self { - part_number: part.part_number, - etag: part.etag, - } - } -} - -impl From for RuntimeMultipartUploadPart { - fn from(part: MultipartUploadPart) -> Self { - Self { - part_number: part.part_number, - etag: part.etag, - } - } -} diff --git a/packages/backend/native/src/lib.rs b/packages/backend/native/src/lib.rs index b9c8c9fc2c..c17852971f 100644 --- a/packages/backend/native/src/lib.rs +++ b/packages/backend/native/src/lib.rs @@ -2,7 +2,6 @@ mod utils; -pub mod backend_runtime; pub mod doc; pub mod doc_loader; pub mod entitlement; @@ -13,6 +12,7 @@ pub mod image; pub mod license; pub mod llm; pub mod permission; +pub mod runtime; pub mod safe_fetch; pub mod tiktoken; diff --git a/packages/backend/native/src/license.rs b/packages/backend/native/src/license.rs index f82782ca15..ae87f31a48 100644 --- a/packages/backend/native/src/license.rs +++ b/packages/backend/native/src/license.rs @@ -1,7 +1,7 @@ use std::{ collections::HashMap, sync::{Mutex, OnceLock}, - time::{Duration, SystemTime, UNIX_EPOCH}, + time::{Duration, SystemTime}, }; use anyhow::{Context, Result as AnyResult, bail}; @@ -458,10 +458,7 @@ fn parse_future_end_at(value: &serde_json::Value) -> AnyResult { } fn now_millis() -> f64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as f64 + crate::utils::system_time_millis(SystemTime::now()).unwrap_or_default() as f64 } fn affine_pro_ech_config() -> AnyResult> { diff --git a/packages/backend/native/src/backend_runtime/constants.rs b/packages/backend/native/src/runtime/backend_runtime/constants.rs similarity index 88% rename from packages/backend/native/src/backend_runtime/constants.rs rename to packages/backend/native/src/runtime/backend_runtime/constants.rs index fa790d66ee..29c3f474e1 100644 --- a/packages/backend/native/src/backend_runtime/constants.rs +++ b/packages/backend/native/src/runtime/backend_runtime/constants.rs @@ -7,4 +7,3 @@ pub(super) const WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE: &str = "workspace_invi pub(super) const WORKSPACE_STATS_LEASE_KEY: &str = "workspace:admin-stats:refresh"; pub(super) const WORKSPACE_STATS_LOCK_NAMESPACE: i64 = 97_301; pub(super) const WORKSPACE_STATS_REFRESH_LOCK_KEY: i64 = 1; -pub(super) const RUNTIME_MIGRATIONS: &str = include_str!("sql/runtime_migrations.sql"); diff --git a/packages/backend/native/src/backend_runtime/coordination_lease.rs b/packages/backend/native/src/runtime/backend_runtime/coordination_lease.rs similarity index 74% rename from packages/backend/native/src/backend_runtime/coordination_lease.rs rename to packages/backend/native/src/runtime/backend_runtime/coordination_lease.rs index b048a05e92..5287167f7e 100644 --- a/packages/backend/native/src/backend_runtime/coordination_lease.rs +++ b/packages/backend/native/src/runtime/backend_runtime/coordination_lease.rs @@ -1,7 +1,7 @@ use napi::Result; use sqlx::{FromRow, PgPool}; -use super::{BackendRuntime, error::napi_error, types::CoordinationLeaseGrant}; +use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error, types::CoordinationLeaseGrant}; #[derive(FromRow)] struct LeaseGrantRow { @@ -17,7 +17,7 @@ impl CoordinationLeaseStore { Self { pool } } - async fn acquire(&self, key: String, owner: String, ttl_ms: i64) -> Result> { + async fn acquire(&self, key: String, owner: String, ttl_ms: i64) -> RuntimeResult> { let row = sqlx::query_as::<_, LeaseGrantRow>( r#" INSERT INTO runtime_leases (key, owner, fencing_token, expires_at) @@ -36,7 +36,7 @@ impl CoordinationLeaseStore { .bind(ttl_ms as f64) .fetch_optional(&self.pool) .await - .map_err(|err| napi_error(format!("CoordinationLease acquire failed: {err}")))?; + .map_err(|err| RuntimeError::database("CoordinationLease acquire failed", err))?; Ok(row.map(|row| CoordinationLeaseGrant { key, @@ -45,7 +45,7 @@ impl CoordinationLeaseStore { })) } - async fn release(&self, key: &str, owner: &str, fencing_token: i64) -> Result { + async fn release(&self, key: &str, owner: &str, fencing_token: i64) -> RuntimeResult { let result = sqlx::query( r#" DELETE FROM runtime_leases @@ -57,12 +57,12 @@ impl CoordinationLeaseStore { .bind(fencing_token) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("CoordinationLease release failed: {err}")))?; + .map_err(|err| RuntimeError::database("CoordinationLease release failed", err))?; Ok(result.rows_affected() == 1) } - async fn renew(&self, key: &str, owner: &str, fencing_token: i64, ttl_ms: i64) -> Result { + async fn renew(&self, key: &str, owner: &str, fencing_token: i64, ttl_ms: i64) -> RuntimeResult { let result = sqlx::query( r#" UPDATE runtime_leases @@ -80,7 +80,7 @@ impl CoordinationLeaseStore { .bind(ttl_ms as f64) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("CoordinationLease renew failed: {err}")))?; + .map_err(|err| RuntimeError::database("CoordinationLease renew failed", err))?; Ok(result.rows_affected() == 1) } @@ -88,6 +88,28 @@ impl CoordinationLeaseStore { #[napi_derive::napi] impl BackendRuntime { + pub(crate) async fn acquire_coordination_lease_inner( + &self, + key: String, + owner: String, + ttl_ms: i64, + ) -> RuntimeResult> { + CoordinationLeaseStore::new(self.pool().await?) + .acquire(key, owner, ttl_ms) + .await + } + + pub(crate) async fn release_coordination_lease_inner( + &self, + key: String, + owner: String, + fencing_token: i64, + ) -> RuntimeResult { + CoordinationLeaseStore::new(self.pool().await?) + .release(&key, &owner, fencing_token) + .await + } + #[napi] pub async fn acquire_coordination_lease( &self, @@ -102,9 +124,10 @@ impl BackendRuntime { return Err(napi_error("coordination lease owner is required")); } - CoordinationLeaseStore::new(self.pool().await?) - .acquire(key, owner, ttl_ms) + self + .acquire_coordination_lease_inner(key, owner, ttl_ms) .await + .map_err(napi::Error::from) } #[napi] @@ -114,9 +137,10 @@ impl BackendRuntime { owner: String, #[napi(ts_arg_type = "bigint | number")] fencing_token: i64, ) -> Result { - CoordinationLeaseStore::new(self.pool().await?) - .release(&key, &owner, fencing_token) + self + .release_coordination_lease_inner(key, owner, fencing_token) .await + .map_err(napi::Error::from) } #[napi] @@ -134,5 +158,6 @@ impl BackendRuntime { CoordinationLeaseStore::new(self.pool().await?) .renew(&key, &owner, fencing_token, ttl_ms) .await + .map_err(napi::Error::from) } } diff --git a/packages/backend/native/src/backend_runtime/doc_compactor.rs b/packages/backend/native/src/runtime/backend_runtime/doc_compactor.rs similarity index 80% rename from packages/backend/native/src/backend_runtime/doc_compactor.rs rename to packages/backend/native/src/runtime/backend_runtime/doc_compactor.rs index 5827af0ad9..bd36cabce8 100644 --- a/packages/backend/native/src/backend_runtime/doc_compactor.rs +++ b/packages/backend/native/src/runtime/backend_runtime/doc_compactor.rs @@ -1,9 +1,8 @@ use chrono::{DateTime, Duration, Utc}; -use napi::Result; use sqlx::{FromRow, PgPool, Postgres, Row, Transaction}; use y_octo::Doc; -use super::{BackendRuntime, error::napi_error, types::RuntimeDocCompactionResult}; +use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error, types::RuntimeDocCompactionResult}; #[derive(FromRow)] struct SnapshotRow { @@ -35,7 +34,7 @@ impl DocCompactorStore { batch_limit: i64, history_min_interval_ms: i64, history_max_age_seconds: i64, - ) -> Result<(i64, bool)> { + ) -> RuntimeResult<(i64, bool)> { compact_doc( self.pool.clone(), workspace_id, @@ -52,31 +51,32 @@ fn is_empty_doc(bin: &[u8]) -> bool { bin.is_empty() || (bin.len() == 1 && bin[0] == 0) || (bin.len() == 2 && bin[0] == 0 && bin[1] == 0) } -fn apply_updates(updates: impl IntoIterator>) -> Result> { +fn apply_updates(updates: impl IntoIterator>) -> RuntimeResult> { let mut doc = Doc::default(); for update in updates { doc .apply_update_from_binary_v1(&update) - .map_err(|err| napi_error(format!("DocCompactor merge failed: {err}")))?; + .map_err(|err| RuntimeError::invalid_state(format!("DocCompactor merge failed: {err}")))?; } doc .encode_update_v1() - .map_err(|err| napi_error(format!("DocCompactor encode failed: {err}"))) + .map_err(|err| RuntimeError::invalid_state(format!("DocCompactor encode failed: {err}"))) } -fn checked_milliseconds(value: i64, field: &str) -> Result { - Duration::try_milliseconds(value).ok_or_else(|| napi_error(format!("DocCompactor {field} is too large"))) +fn checked_milliseconds(value: i64, field: &str) -> RuntimeResult { + Duration::try_milliseconds(value) + .ok_or_else(|| RuntimeError::invalid_input(format!("DocCompactor {field} is too large"))) } -fn checked_seconds(value: i64, field: &str) -> Result { - Duration::try_seconds(value).ok_or_else(|| napi_error(format!("DocCompactor {field} is too large"))) +fn checked_seconds(value: i64, field: &str) -> RuntimeResult { + Duration::try_seconds(value).ok_or_else(|| RuntimeError::invalid_input(format!("DocCompactor {field} is too large"))) } async fn load_snapshot( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, doc_id: &str, -) -> Result> { +) -> RuntimeResult> { sqlx::query_as::<_, SnapshotRow>( r#" SELECT blob, updated_at, updated_by @@ -89,7 +89,7 @@ async fn load_snapshot( .bind(doc_id) .fetch_optional(&mut **tx) .await - .map_err(|err| napi_error(format!("DocCompactor load snapshot failed: {err}"))) + .map_err(|err| RuntimeError::database("DocCompactor load snapshot failed", err)) } async fn load_updates( @@ -97,7 +97,7 @@ async fn load_updates( workspace_id: &str, doc_id: &str, batch_limit: i64, -) -> Result> { +) -> RuntimeResult> { sqlx::query_as::<_, UpdateRow>( r#" SELECT blob, created_at, created_by @@ -113,7 +113,7 @@ async fn load_updates( .bind(batch_limit) .fetch_all(&mut **tx) .await - .map_err(|err| napi_error(format!("DocCompactor load updates failed: {err}"))) + .map_err(|err| RuntimeError::database("DocCompactor load updates failed", err)) } async fn upsert_snapshot( @@ -123,7 +123,7 @@ async fn upsert_snapshot( blob: &[u8], timestamp: DateTime, editor: Option<&str>, -) -> Result { +) -> RuntimeResult { if is_empty_doc(blob) { return Ok(false); } @@ -154,7 +154,7 @@ async fn upsert_snapshot( .bind(editor) .fetch_optional(&mut **tx) .await - .map_err(|err| napi_error(format!("DocCompactor upsert snapshot failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocCompactor upsert snapshot failed", err))?; Ok(row.is_some()) } @@ -165,7 +165,7 @@ async fn should_create_history( workspace_id: &str, doc_id: &str, history_min_interval_ms: i64, -) -> Result { +) -> RuntimeResult { if is_empty_doc(&snapshot.blob) { return Ok(false); } @@ -183,7 +183,7 @@ async fn should_create_history( .bind(doc_id) .fetch_optional(&mut **tx) .await - .map_err(|err| napi_error(format!("DocCompactor load latest history failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocCompactor load latest history failed", err))?; let Some(row) = row else { return Ok(true); @@ -198,7 +198,7 @@ async fn should_create_history( let threshold = snapshot .updated_at .checked_sub_signed(min_interval) - .ok_or_else(|| napi_error("DocCompactor history interval is out of range"))?; + .ok_or_else(|| RuntimeError::invalid_input("DocCompactor history interval is out of range"))?; Ok(last_timestamp < threshold) } @@ -209,7 +209,7 @@ async fn create_history( doc_id: &str, snapshot: &SnapshotRow, max_age_seconds: i64, -) -> Result { +) -> RuntimeResult { if max_age_seconds <= 0 { return Ok(false); } @@ -217,7 +217,7 @@ async fn create_history( let max_age = checked_seconds(max_age_seconds, "history max age")?; let expired_at = Utc::now() .checked_add_signed(max_age) - .ok_or_else(|| napi_error("DocCompactor history max age is out of range"))?; + .ok_or_else(|| RuntimeError::invalid_input("DocCompactor history max age is out of range"))?; sqlx::query( r#" INSERT INTO snapshot_histories @@ -236,7 +236,7 @@ async fn create_history( .bind(snapshot.updated_by.as_deref()) .execute(&mut **tx) .await - .map_err(|err| napi_error(format!("DocCompactor create history failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocCompactor create history failed", err))?; Ok(true) } @@ -246,7 +246,7 @@ async fn delete_updates( workspace_id: &str, doc_id: &str, timestamps: &[DateTime], -) -> Result { +) -> RuntimeResult { let result = sqlx::query( r#" DELETE FROM updates @@ -260,7 +260,7 @@ async fn delete_updates( .bind(timestamps) .execute(&mut **tx) .await - .map_err(|err| napi_error(format!("DocCompactor delete updates failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocCompactor delete updates failed", err))?; Ok(result.rows_affected() as i64) } @@ -272,18 +272,18 @@ async fn compact_doc( batch_limit: i64, history_min_interval_ms: i64, history_max_age_seconds: i64, -) -> Result<(i64, bool)> { +) -> RuntimeResult<(i64, bool)> { let mut tx = pool .begin() .await - .map_err(|err| napi_error(format!("DocCompactor begin transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocCompactor begin transaction failed", err))?; let snapshot = load_snapshot(&mut tx, workspace_id, doc_id).await?; let updates = load_updates(&mut tx, workspace_id, doc_id, batch_limit).await?; if updates.is_empty() { tx.commit() .await - .map_err(|err| napi_error(format!("DocCompactor commit transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocCompactor commit transaction failed", err))?; return Ok((0, false)); } @@ -323,7 +323,7 @@ async fn compact_doc( tx.commit() .await - .map_err(|err| napi_error(format!("DocCompactor commit transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocCompactor commit transaction failed", err))?; Ok((deleted, history_created)) } @@ -350,7 +350,7 @@ impl BackendRuntime { history_max_age_seconds: i64, owner: String, lease_ttl_ms: i64, - ) -> Result { + ) -> napi::Result { if batch_limit <= 0 { return Err(napi_error("doc compactor batch limit must be positive")); } @@ -365,7 +365,7 @@ impl BackendRuntime { let max_age = checked_seconds(history_max_age_seconds, "history max age")?; Utc::now() .checked_add_signed(max_age) - .ok_or_else(|| napi_error("DocCompactor history max age is out of range"))?; + .ok_or_else(|| RuntimeError::invalid_input("DocCompactor history max age is out of range"))?; } let lease_key = format!("doc:update:{workspace_id}:{doc_id}"); @@ -394,7 +394,7 @@ impl BackendRuntime { .release_coordination_lease(lease.key, lease.owner, lease.fencing_token) .await?; if !released { - return Err(napi_error("DocCompactor failed to release coordination lease")); + return Err(RuntimeError::invalid_state("DocCompactor failed to release coordination lease").into()); } let (updates_merged, history_created) = result?; diff --git a/packages/backend/native/src/backend_runtime/doc_storage.rs b/packages/backend/native/src/runtime/backend_runtime/doc_storage.rs similarity index 74% rename from packages/backend/native/src/backend_runtime/doc_storage.rs rename to packages/backend/native/src/runtime/backend_runtime/doc_storage.rs index 0f2f61a8e6..c8fc8ad755 100644 --- a/packages/backend/native/src/backend_runtime/doc_storage.rs +++ b/packages/backend/native/src/runtime/backend_runtime/doc_storage.rs @@ -1,14 +1,18 @@ use chrono::{DateTime, Duration, Utc}; -use napi::{Result, bindgen_prelude::Buffer}; +use napi::bindgen_prelude::Buffer; use sqlx::{PgPool, Row}; -use super::{BackendRuntime, error::napi_error, types::RuntimeDocHistoryInput}; +use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error, types::RuntimeDocHistoryInput}; fn is_empty_doc(bin: &[u8]) -> bool { bin.is_empty() || (bin.len() == 1 && bin[0] == 0) || (bin.len() == 2 && bin[0] == 0 && bin[1] == 0) } -async fn latest_history_timestamp(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result>> { +async fn latest_history_timestamp( + pool: &PgPool, + workspace_id: &str, + doc_id: &str, +) -> RuntimeResult>> { sqlx::query( r#" SELECT timestamp @@ -23,7 +27,7 @@ async fn latest_history_timestamp(pool: &PgPool, workspace_id: &str, doc_id: &st .fetch_optional(pool) .await .map(|row| row.map(|row| row.get("timestamp"))) - .map_err(|err| napi_error(format!("DocStorage load latest history failed: {err}"))) + .map_err(|err| RuntimeError::database("DocStorage load latest history failed", err)) } #[napi_derive::napi] @@ -36,13 +40,13 @@ impl BackendRuntime { blob: Buffer, timestamp_ms: i64, editor_id: Option, - ) -> Result { + ) -> napi::Result { if is_empty_doc(blob.as_ref()) { return Ok(false); } let timestamp = DateTime::::from_timestamp_millis(timestamp_ms) - .ok_or_else(|| napi_error(format!("Invalid doc snapshot timestamp: {timestamp_ms}")))?; + .ok_or_else(|| RuntimeError::invalid_input(format!("Invalid doc snapshot timestamp: {timestamp_ms}")))?; let pool = self.pool().await?; let row = sqlx::query( r#" @@ -70,13 +74,13 @@ impl BackendRuntime { .bind(editor_id.as_deref()) .fetch_optional(&pool) .await - .map_err(|err| napi_error(format!("DocStorage upsert snapshot failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocStorage upsert snapshot failed", err))?; Ok(row.is_some()) } #[napi] - pub async fn create_doc_history(&self, input: RuntimeDocHistoryInput) -> Result { + pub async fn create_doc_history(&self, input: RuntimeDocHistoryInput) -> napi::Result { if input.history_min_interval_ms < 0 { return Err(napi_error("doc history interval must be non-negative")); } @@ -85,7 +89,7 @@ impl BackendRuntime { } let timestamp = DateTime::::from_timestamp_millis(input.timestamp_ms) - .ok_or_else(|| napi_error(format!("Invalid doc history timestamp: {}", input.timestamp_ms)))?; + .ok_or_else(|| RuntimeError::invalid_input(format!("Invalid doc history timestamp: {}", input.timestamp_ms)))?; let pool = self.pool().await?; let should_create = match latest_history_timestamp(&pool, &input.workspace_id, &input.doc_id).await? { None => true, @@ -118,41 +122,41 @@ impl BackendRuntime { .bind(input.editor_id.as_deref()) .execute(&pool) .await - .map_err(|err| napi_error(format!("DocStorage create history failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocStorage create history failed", err))?; Ok(true) } #[napi] - pub async fn delete_doc_storage(&self, workspace_id: String, doc_id: String) -> Result<()> { + pub async fn delete_doc_storage(&self, workspace_id: String, doc_id: String) -> napi::Result<()> { let pool = self.pool().await?; let mut tx = pool .begin() .await - .map_err(|err| napi_error(format!("DocStorage delete begin transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocStorage delete begin transaction failed", err))?; sqlx::query("DELETE FROM snapshots WHERE workspace_id = $1 AND guid = $2") .bind(&workspace_id) .bind(&doc_id) .execute(&mut *tx) .await - .map_err(|err| napi_error(format!("DocStorage delete snapshot failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocStorage delete snapshot failed", err))?; sqlx::query("DELETE FROM updates WHERE workspace_id = $1 AND guid = $2") .bind(&workspace_id) .bind(&doc_id) .execute(&mut *tx) .await - .map_err(|err| napi_error(format!("DocStorage delete updates failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocStorage delete updates failed", err))?; sqlx::query("DELETE FROM snapshot_histories WHERE workspace_id = $1 AND guid = $2") .bind(&workspace_id) .bind(&doc_id) .execute(&mut *tx) .await - .map_err(|err| napi_error(format!("DocStorage delete histories failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocStorage delete histories failed", err))?; tx.commit() .await - .map_err(|err| napi_error(format!("DocStorage delete commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("DocStorage delete commit failed", err))?; Ok(()) } } diff --git a/packages/backend/native/src/backend_runtime/gate.rs b/packages/backend/native/src/runtime/backend_runtime/gate.rs similarity index 66% rename from packages/backend/native/src/backend_runtime/gate.rs rename to packages/backend/native/src/runtime/backend_runtime/gate.rs index 8686183131..f738045851 100644 --- a/packages/backend/native/src/backend_runtime/gate.rs +++ b/packages/backend/native/src/runtime/backend_runtime/gate.rs @@ -1,7 +1,7 @@ use napi::Result; use sqlx::PgPool; -use super::{BackendRuntime, error::napi_error}; +use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error}; struct RuntimeGateStore { pool: PgPool, @@ -12,18 +12,18 @@ impl RuntimeGateStore { Self { pool } } - async fn put_if_absent(&self, key: &str, ttl_ms: i64) -> Result { + async fn put_if_absent(&self, key: &str, ttl_ms: i64) -> RuntimeResult { let mut tx = self .pool .begin() .await - .map_err(|err| napi_error(format!("RuntimeGate transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeGate transaction failed", err))?; sqlx::query("DELETE FROM runtime_gates WHERE key = $1 AND expires_at <= CURRENT_TIMESTAMP") .bind(key) .execute(&mut *tx) .await - .map_err(|err| napi_error(format!("RuntimeGate expired cleanup failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeGate expired cleanup failed", err))?; let inserted = sqlx::query( r#" @@ -36,18 +36,18 @@ impl RuntimeGateStore { .bind(ttl_ms as f64) .execute(&mut *tx) .await - .map_err(|err| napi_error(format!("RuntimeGate put_if_absent failed: {err}")))? + .map_err(|err| RuntimeError::database("RuntimeGate put_if_absent failed", err))? .rows_affected() == 1; tx.commit() .await - .map_err(|err| napi_error(format!("RuntimeGate transaction commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeGate transaction commit failed", err))?; Ok(inserted) } - async fn cleanup_expired(&self, limit: i64) -> Result { + async fn cleanup_expired(&self, limit: i64) -> RuntimeResult { let result = sqlx::query( r#" DELETE FROM runtime_gates @@ -62,7 +62,7 @@ impl RuntimeGateStore { .bind(limit) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("RuntimeGate cleanup failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeGate cleanup failed", err))?; Ok(result.rows_affected() as i64) } @@ -78,6 +78,7 @@ impl BackendRuntime { RuntimeGateStore::new(self.pool().await?) .put_if_absent(&key, ttl_ms) .await + .map_err(napi::Error::from) } #[napi] @@ -85,6 +86,9 @@ impl BackendRuntime { if limit <= 0 { return Err(napi_error("runtime gate cleanup limit must be positive")); } - RuntimeGateStore::new(self.pool().await?).cleanup_expired(limit).await + RuntimeGateStore::new(self.pool().await?) + .cleanup_expired(limit) + .await + .map_err(napi::Error::from) } } diff --git a/packages/backend/native/src/backend_runtime/housekeeping.rs b/packages/backend/native/src/runtime/backend_runtime/housekeeping.rs similarity index 82% rename from packages/backend/native/src/backend_runtime/housekeeping.rs rename to packages/backend/native/src/runtime/backend_runtime/housekeeping.rs index f89dc360cc..980ef6bf47 100644 --- a/packages/backend/native/src/backend_runtime/housekeeping.rs +++ b/packages/backend/native/src/runtime/backend_runtime/housekeeping.rs @@ -1,7 +1,7 @@ use napi::Result; use sqlx::PgPool; -use super::{BackendRuntime, error::napi_error}; +use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error}; struct HousekeepingStore { pool: PgPool, @@ -12,7 +12,7 @@ impl HousekeepingStore { Self { pool } } - async fn cleanup_expired_user_sessions(&self, limit: i64) -> Result { + async fn cleanup_expired_user_sessions(&self, limit: i64) -> RuntimeResult { let result = sqlx::query( r#" DELETE FROM user_sessions @@ -27,12 +27,12 @@ impl HousekeepingStore { .bind(limit) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("Housekeeping user sessions cleanup failed: {err}")))?; + .map_err(|err| RuntimeError::database("Housekeeping user sessions cleanup failed", err))?; Ok(result.rows_affected() as i64) } - async fn cleanup_expired_snapshot_histories(&self, limit: i64) -> Result { + async fn cleanup_expired_snapshot_histories(&self, limit: i64) -> RuntimeResult { let result = sqlx::query( r#" DELETE FROM snapshot_histories @@ -48,7 +48,7 @@ impl HousekeepingStore { .bind(limit) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("Housekeeping snapshot histories cleanup failed: {err}")))?; + .map_err(|err| RuntimeError::database("Housekeeping snapshot histories cleanup failed", err))?; Ok(result.rows_affected() as i64) } @@ -65,6 +65,7 @@ impl BackendRuntime { HousekeepingStore::new(self.pool().await?) .cleanup_expired_user_sessions(limit) .await + .map_err(napi::Error::from) } #[napi] @@ -76,5 +77,6 @@ impl BackendRuntime { HousekeepingStore::new(self.pool().await?) .cleanup_expired_snapshot_histories(limit) .await + .map_err(napi::Error::from) } } diff --git a/packages/backend/native/src/backend_runtime/mod.rs b/packages/backend/native/src/runtime/backend_runtime/mod.rs similarity index 60% rename from packages/backend/native/src/backend_runtime/mod.rs rename to packages/backend/native/src/runtime/backend_runtime/mod.rs index 69c6ab83eb..7068b21db8 100644 --- a/packages/backend/native/src/backend_runtime/mod.rs +++ b/packages/backend/native/src/runtime/backend_runtime/mod.rs @@ -1,23 +1,13 @@ -mod blob_cleanup; -mod blob_complete; -mod blob_reclaimer; -mod blob_reconciliation; -mod config; mod constants; mod coordination_lease; -mod doc_blob_refs; mod doc_compactor; mod doc_storage; -mod error; mod gate; mod housekeeping; -mod object_storage; mod runtime_state; #[cfg(test)] mod tests; -mod types; mod workspace_stats; - use std::{sync::RwLock, time::Duration}; use napi::Result; @@ -25,7 +15,11 @@ use sha2::{Digest, Sha256}; use sqlx::{PgPool, Row, postgres::PgPoolOptions}; use tokio::sync::Mutex; -use self::{config::RuntimeConfig, constants::RUNTIME_MIGRATIONS, error::napi_error, types::BackendRuntimeHealth}; +use self::types::BackendRuntimeHealth; +pub(crate) use super::types; +use super::{ + BackendRuntimeConfig, RuntimeError, RuntimeResult, migrations::migrate_runtime_tables, napi_error, to_napi_error, +}; pub(super) fn token_hash(token: &str) -> String { hex::encode(Sha256::digest(token.as_bytes())) @@ -33,7 +27,7 @@ pub(super) fn token_hash(token: &str) -> String { #[napi_derive::napi] pub struct BackendRuntime { - config: RwLock, + config: RwLock, pool: Mutex>, } @@ -42,13 +36,17 @@ impl BackendRuntime { #[napi(constructor)] pub fn new() -> Result { Ok(Self { - config: RwLock::new(RuntimeConfig::from_config_files()?), + config: RwLock::new(BackendRuntimeConfig::from_config_files().map_err(to_napi_error)?), pool: Mutex::new(None), }) } #[napi] pub async fn start(&self) -> Result<()> { + self.start_inner().await.map_err(to_napi_error) + } + + async fn start_inner(&self) -> RuntimeResult<()> { let mut guard = self.pool.lock().await; if guard.is_some() { return Ok(()); @@ -60,12 +58,12 @@ impl BackendRuntime { .acquire_timeout(Duration::from_secs(5)) .connect(&database_url) .await - .map_err(|err| napi_error(format!("BackendRuntime failed to connect postgres: {err}")))?; + .map_err(|err| RuntimeError::database("BackendRuntime failed to connect postgres", err))?; sqlx::query("SELECT 1") .execute(&pool) .await - .map_err(|err| napi_error(format!("BackendRuntime postgres health check failed: {err}")))?; + .map_err(|err| RuntimeError::database("BackendRuntime postgres health check failed", err))?; let config = self.config()?.with_db_overrides(&pool).await?; self.update_config(config)?; @@ -98,54 +96,38 @@ impl BackendRuntime { Ok(BackendRuntimeHealth { started: pool.is_some(), database_connected, - object_storage_configured: self.config()?.storage.is_some(), }) } #[napi] pub async fn run_migrations(&self) -> Result<()> { let pool = self.pool().await?; - migrate_runtime_tables(&pool).await + migrate_runtime_tables(&pool).await.map_err(to_napi_error) } - async fn pool(&self) -> Result { + pub(crate) async fn pool(&self) -> RuntimeResult { self .pool .lock() .await .as_ref() .cloned() - .ok_or_else(|| napi_error("BackendRuntime must be started before using postgres operations")) + .ok_or_else(|| RuntimeError::invalid_state("BackendRuntime must be started before using postgres operations")) } - pub(in crate::backend_runtime) fn config(&self) -> Result { + pub(crate) fn config(&self) -> RuntimeResult { self .config .read() .map(|config| config.clone()) - .map_err(|_| napi_error("BackendRuntime config lock poisoned")) + .map_err(|_| RuntimeError::invalid_state("BackendRuntime config lock poisoned")) } - fn update_config(&self, config: RuntimeConfig) -> Result<()> { + fn update_config(&self, config: BackendRuntimeConfig) -> RuntimeResult<()> { *self .config .write() - .map_err(|_| napi_error("BackendRuntime config lock poisoned"))? = config; + .map_err(|_| RuntimeError::invalid_state("BackendRuntime config lock poisoned"))? = config; Ok(()) } } - -async fn migrate_runtime_tables(pool: &PgPool) -> Result<()> { - for statement in RUNTIME_MIGRATIONS - .split(';') - .map(str::trim) - .filter(|statement| !statement.is_empty()) - { - sqlx::query(statement) - .execute(pool) - .await - .map_err(|err| napi_error(format!("BackendRuntime migration failed: {err}")))?; - } - - Ok(()) -} diff --git a/packages/backend/native/src/backend_runtime/runtime_state/auth_challenge.rs b/packages/backend/native/src/runtime/backend_runtime/runtime_state/auth_challenge.rs similarity index 91% rename from packages/backend/native/src/backend_runtime/runtime_state/auth_challenge.rs rename to packages/backend/native/src/runtime/backend_runtime/runtime_state/auth_challenge.rs index 17b9063192..4058de1154 100644 --- a/packages/backend/native/src/backend_runtime/runtime_state/auth_challenge.rs +++ b/packages/backend/native/src/runtime/backend_runtime/runtime_state/auth_challenge.rs @@ -1,6 +1,4 @@ -use napi::Result; - -use super::{auth_challenge_purpose, dto::RuntimeStateRows}; +use super::{Result, auth_challenge_purpose, dto::RuntimeStateRows}; pub(super) async fn create( rows: &RuntimeStateRows, diff --git a/packages/backend/native/src/backend_runtime/runtime_state/byok_local_lease.rs b/packages/backend/native/src/runtime/backend_runtime/runtime_state/byok_local_lease.rs similarity index 80% rename from packages/backend/native/src/backend_runtime/runtime_state/byok_local_lease.rs rename to packages/backend/native/src/runtime/backend_runtime/runtime_state/byok_local_lease.rs index 57c00fca43..1318bd392a 100644 --- a/packages/backend/native/src/backend_runtime/runtime_state/byok_local_lease.rs +++ b/packages/backend/native/src/runtime/backend_runtime/runtime_state/byok_local_lease.rs @@ -1,10 +1,6 @@ -use napi::Result; - -use super::dto::{RuntimeStateInsertPayload, RuntimeStatePayloadRow, RuntimeStateRows}; -use crate::backend_runtime::{ - constants::{BYOK_LOCAL_LEASE_ACTIVE_PURPOSE, BYOK_LOCAL_LEASE_PURPOSE}, - error::napi_error, - types::RuntimeByokLocalLeaseRecord, +use super::{ + BYOK_LOCAL_LEASE_ACTIVE_PURPOSE, BYOK_LOCAL_LEASE_PURPOSE, Result, RuntimeByokLocalLeaseRecord, RuntimeError, + dto::{RuntimeStateInsertPayload, RuntimeStatePayloadRow, RuntimeStateRows}, }; pub(super) async fn get(rows: &RuntimeStateRows, lease_id: String) -> Result> { @@ -19,7 +15,7 @@ pub(super) async fn create( ttl_ms: i64, ) -> Result { if ttl_ms <= 0 { - return Err(napi_error("BYOK local lease ttl must be positive")); + return Err(RuntimeError::invalid_input("BYOK local lease ttl must be positive")); } let mut tx = rows.begin("RuntimeState BYOK local lease").await?; @@ -27,7 +23,7 @@ pub(super) async fn create( .bind(&active_key) .execute(&mut *tx) .await - .map_err(|err| napi_error(format!("RuntimeState BYOK local lease active lock failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState BYOK local lease active lock failed", err))?; if let Some(active) = rows .active_payload_with_expires_for_update_in_tx( @@ -43,11 +39,9 @@ pub(super) async fn create( None => None, }; if let Some(lease) = existing_lease { - tx.commit().await.map_err(|err| { - napi_error(format!( - "RuntimeState BYOK local lease transaction commit failed: {err}" - )) - })?; + tx.commit() + .await + .map_err(|err| RuntimeError::database("RuntimeState BYOK local lease transaction commit failed", err))?; return Ok(lease); } @@ -89,11 +83,9 @@ pub(super) async fn create( ) .await?; - tx.commit().await.map_err(|err| { - napi_error(format!( - "RuntimeState BYOK local lease transaction commit failed: {err}" - )) - })?; + tx.commit() + .await + .map_err(|err| RuntimeError::database("RuntimeState BYOK local lease transaction commit failed", err))?; Ok(RuntimeByokLocalLeaseRecord { lease_id, diff --git a/packages/backend/native/src/backend_runtime/runtime_state/dto.rs b/packages/backend/native/src/runtime/backend_runtime/runtime_state/dto.rs similarity index 90% rename from packages/backend/native/src/backend_runtime/runtime_state/dto.rs rename to packages/backend/native/src/runtime/backend_runtime/runtime_state/dto.rs index 35b4fefcc6..c30b266f8e 100644 --- a/packages/backend/native/src/backend_runtime/runtime_state/dto.rs +++ b/packages/backend/native/src/runtime/backend_runtime/runtime_state/dto.rs @@ -1,7 +1,8 @@ -use napi::Result; use sqlx::{PgPool, Row}; -use crate::backend_runtime::{error::napi_error, token_hash}; +use super::{RuntimeError, RuntimeResult, token_hash}; + +type Result = RuntimeResult; pub(super) struct RuntimeStatePayloadRow { pub(super) payload: serde_json::Value, @@ -42,7 +43,7 @@ impl RuntimeStateRows { .pool .begin() .await - .map_err(|err| napi_error(format!("{context} transaction failed: {err}"))) + .map_err(|err| RuntimeError::database(format!("{context} transaction failed"), err)) } pub(super) async fn insert_payload( @@ -67,7 +68,7 @@ impl RuntimeStateRows { .bind(ttl_ms as f64) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(()) } @@ -95,7 +96,7 @@ impl RuntimeStateRows { .bind(ttl_ms as f64) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))? + .map_err(|err| RuntimeError::database(context, err))? .rows_affected() == 1; @@ -131,7 +132,7 @@ impl RuntimeStateRows { .bind(ttl_ms as f64) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(()) } @@ -156,7 +157,7 @@ impl RuntimeStateRows { .bind(token_hash(token)) .fetch_optional(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(row.map(|row| row.get::("payload"))) } @@ -181,7 +182,7 @@ impl RuntimeStateRows { .bind(token_hash(token)) .fetch_optional(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(row.map(payload_row)) } @@ -208,7 +209,7 @@ impl RuntimeStateRows { .bind(token_hash(token)) .fetch_optional(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(row.map(|row| row.get::("payload"))) } @@ -235,7 +236,7 @@ impl RuntimeStateRows { .bind(token_hash(token)) .fetch_optional(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(row.map(payload_row)) } @@ -262,7 +263,7 @@ impl RuntimeStateRows { .bind(token_hash(token)) .fetch_optional(&mut **tx) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(row.map(payload_row)) } @@ -288,7 +289,7 @@ impl RuntimeStateRows { .bind(token_hash(token)) .fetch_optional(&mut **tx) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(row.map(|row| RuntimeStateLockedRow { payload: row.get("payload"), @@ -316,7 +317,7 @@ impl RuntimeStateRows { .bind(input.ttl_ms as f64) .fetch_one(&mut **tx) .await - .map_err(|err| napi_error(format!("{} failed: {err}", input.context)))?; + .map_err(|err| RuntimeError::database(input.context, err))?; Ok(row.get::("expires_at_ms")) } @@ -348,7 +349,7 @@ impl RuntimeStateRows { .bind(input.ttl_ms as f64) .fetch_optional(&mut **tx) .await - .map_err(|err| napi_error(format!("{} failed: {err}", input.context)))?; + .map_err(|err| RuntimeError::database(input.context, err))?; Ok(row.map(|row| row.get::("expires_at_ms"))) } @@ -375,7 +376,7 @@ impl RuntimeStateRows { .bind(attempts) .execute(&mut **tx) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(()) } @@ -392,7 +393,7 @@ impl RuntimeStateRows { .bind(token_hash(token)) .execute(&mut **tx) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(()) } @@ -413,7 +414,7 @@ impl RuntimeStateRows { .bind(limit) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(result.rows_affected() as i64) } @@ -440,7 +441,7 @@ impl RuntimeStateRows { .bind(limit) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("{context} failed: {err}")))?; + .map_err(|err| RuntimeError::database(context, err))?; Ok(result.rows_affected() as i64) } diff --git a/packages/backend/native/src/backend_runtime/runtime_state/invite_link.rs b/packages/backend/native/src/runtime/backend_runtime/runtime_state/invite_link.rs similarity index 71% rename from packages/backend/native/src/backend_runtime/runtime_state/invite_link.rs rename to packages/backend/native/src/runtime/backend_runtime/runtime_state/invite_link.rs index 54aba85e28..4a3f6c15e8 100644 --- a/packages/backend/native/src/backend_runtime/runtime_state/invite_link.rs +++ b/packages/backend/native/src/runtime/backend_runtime/runtime_state/invite_link.rs @@ -1,10 +1,7 @@ -use napi::Result; - -use super::dto::{RuntimeStateInsertPayload, RuntimeStatePayloadRow, RuntimeStateRows}; -use crate::backend_runtime::{ - constants::{WORKSPACE_INVITE_LINK_ID_PURPOSE, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE}, - error::napi_error, - types::RuntimeWorkspaceInviteLinkRecord, +use super::{ + Result, RuntimeError, RuntimeWorkspaceInviteLinkRecord, WORKSPACE_INVITE_LINK_ID_PURPOSE, + WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, + dto::{RuntimeStateInsertPayload, RuntimeStatePayloadRow, RuntimeStateRows}, }; pub(super) async fn get_by_workspace( @@ -29,7 +26,9 @@ pub(super) async fn create( ttl_ms: i64, ) -> Result { if ttl_ms <= 0 { - return Err(napi_error("workspace invite link ttl must be positive")); + return Err(RuntimeError::invalid_input( + "workspace invite link ttl must be positive", + )); } let mut tx = rows.begin("RuntimeState workspace invite link").await?; @@ -37,16 +36,14 @@ pub(super) async fn create( .bind(&workspace_id) .execute(&mut *tx) .await - .map_err(|err| napi_error(format!("RuntimeState workspace invite link active lock failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState workspace invite link active lock failed", err))?; if let Some(existing) = get_by_key_in_tx(rows, &mut tx, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, &workspace_id).await? { - tx.commit().await.map_err(|err| { - napi_error(format!( - "RuntimeState workspace invite link transaction commit failed: {err}" - )) - })?; + tx.commit() + .await + .map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?; return Ok(existing); } @@ -71,12 +68,11 @@ pub(super) async fn create( .await? else { let existing = get_by_key_in_tx(rows, &mut tx, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, &workspace_id).await?; - tx.commit().await.map_err(|err| { - napi_error(format!( - "RuntimeState workspace invite link transaction commit failed: {err}" - )) - })?; - return existing.ok_or_else(|| napi_error("RuntimeState workspace invite link active conflict missing row")); + tx.commit() + .await + .map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?; + return existing + .ok_or_else(|| RuntimeError::invalid_state("RuntimeState workspace invite link active conflict missing row")); }; rows .insert_payload_returning_expires_in_tx( @@ -92,11 +88,9 @@ pub(super) async fn create( ) .await?; - tx.commit().await.map_err(|err| { - napi_error(format!( - "RuntimeState workspace invite link transaction commit failed: {err}" - )) - })?; + tx.commit() + .await + .map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?; Ok(RuntimeWorkspaceInviteLinkRecord { workspace_id, @@ -110,11 +104,9 @@ pub(super) async fn revoke(rows: &RuntimeStateRows, workspace_id: String) -> Res let mut tx = rows.begin("RuntimeState workspace invite link").await?; let existing = get_by_key_in_tx(rows, &mut tx, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, &workspace_id).await?; let Some(existing) = existing else { - tx.commit().await.map_err(|err| { - napi_error(format!( - "RuntimeState workspace invite link transaction commit failed: {err}" - )) - })?; + tx.commit() + .await + .map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?; return Ok(false); }; @@ -135,11 +127,9 @@ pub(super) async fn revoke(rows: &RuntimeStateRows, workspace_id: String) -> Res ) .await?; - tx.commit().await.map_err(|err| { - napi_error(format!( - "RuntimeState workspace invite link transaction commit failed: {err}" - )) - })?; + tx.commit() + .await + .map_err(|err| RuntimeError::database("RuntimeState workspace invite link transaction commit failed", err))?; Ok(true) } @@ -175,19 +165,19 @@ fn record_from_row(row: RuntimeStatePayloadRow) -> Result Result<()> { if ttl_ms <= 0 { - return Err(napi_error("magic link otp ttl must be positive")); + return Err(RuntimeError::invalid_input("magic link otp ttl must be positive")); } let payload = serde_json::json!({ @@ -73,11 +69,9 @@ pub(super) async fn consume( .await?; let Some(row) = row else { - tx.rollback().await.map_err(|err| { - napi_error(format!( - "RuntimeState magic link otp transaction rollback failed: {err}" - )) - })?; + tx.rollback() + .await + .map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction rollback failed", err))?; return Ok(RuntimeMagicLinkOtpConsumeResult::fail("not_found")); }; @@ -96,7 +90,7 @@ pub(super) async fn consume( .await?; tx.commit() .await - .map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?; return Ok(RuntimeMagicLinkOtpConsumeResult::fail("expired")); } @@ -104,7 +98,7 @@ pub(super) async fn consume( if stored_client_nonce.is_some() && stored_client_nonce != client_nonce.as_deref() { tx.commit() .await - .map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?; return Ok(RuntimeMagicLinkOtpConsumeResult::fail("nonce_mismatch")); } @@ -119,7 +113,7 @@ pub(super) async fn consume( .await?; tx.commit() .await - .map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?; return Ok(RuntimeMagicLinkOtpConsumeResult::fail("locked")); } @@ -137,7 +131,7 @@ pub(super) async fn consume( .await?; tx.commit() .await - .map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?; return Ok(RuntimeMagicLinkOtpConsumeResult::fail("locked")); } @@ -153,14 +147,14 @@ pub(super) async fn consume( tx.commit() .await - .map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?; return Ok(RuntimeMagicLinkOtpConsumeResult::fail("invalid_otp")); } let token = payload .get("token") .and_then(serde_json::Value::as_str) - .ok_or_else(|| napi_error("RuntimeState magic link otp payload missing token"))? + .ok_or_else(|| RuntimeError::invalid_state("RuntimeState magic link otp payload missing token"))? .to_string(); rows .delete_by_key_in_tx( @@ -172,7 +166,7 @@ pub(super) async fn consume( .await?; tx.commit() .await - .map_err(|err| napi_error(format!("RuntimeState magic link otp transaction commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState magic link otp transaction commit failed", err))?; Ok(RuntimeMagicLinkOtpConsumeResult::ok(token)) } diff --git a/packages/backend/native/src/backend_runtime/runtime_state/mod.rs b/packages/backend/native/src/runtime/backend_runtime/runtime_state/mod.rs similarity index 74% rename from packages/backend/native/src/backend_runtime/runtime_state/mod.rs rename to packages/backend/native/src/runtime/backend_runtime/runtime_state/mod.rs index 39ed1f7610..bee1c15e7e 100644 --- a/packages/backend/native/src/backend_runtime/runtime_state/mod.rs +++ b/packages/backend/native/src/runtime/backend_runtime/runtime_state/mod.rs @@ -1,8 +1,10 @@ -use napi::Result; - -use super::{ - BackendRuntime, - error::napi_error, +use super::{BackendRuntime, RuntimeError, RuntimeResult, napi_error}; +pub(super) use super::{ + constants::{ + BYOK_LOCAL_LEASE_ACTIVE_PURPOSE, BYOK_LOCAL_LEASE_PURPOSE, MAGIC_LINK_OTP_PURPOSE, MAX_MAGIC_LINK_OTP_ATTEMPTS, + WORKSPACE_INVITE_LINK_ID_PURPOSE, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, + }, + token_hash, types::{ RuntimeByokLocalLeaseRecord, RuntimeMagicLinkOtpConsumeResult, RuntimeVerificationTokenRecord, RuntimeWorkspaceInviteLinkRecord, @@ -18,6 +20,8 @@ mod store; mod verification_token; use store::RuntimeStateStore; +pub(super) type Result = RuntimeResult; + pub(super) fn auth_challenge_purpose(purpose: &str) -> String { format!("auth_challenge:{purpose}") } @@ -35,27 +39,34 @@ impl BackendRuntime { token: String, payload: serde_json::Value, ttl_ms: i64, - ) -> Result { + ) -> napi::Result { if ttl_ms <= 0 { return Err(napi_error("auth challenge ttl must be positive")); } RuntimeStateStore::new(self.pool().await?) .create_auth_challenge(&purpose, &token, payload, ttl_ms) .await + .map_err(napi::Error::from) } #[napi] - pub async fn get_auth_challenge(&self, purpose: String, token: String) -> Result> { + pub async fn get_auth_challenge(&self, purpose: String, token: String) -> napi::Result> { RuntimeStateStore::new(self.pool().await?) .get_auth_challenge(&purpose, &token) .await + .map_err(napi::Error::from) } #[napi] - pub async fn consume_auth_challenge(&self, purpose: String, token: String) -> Result> { + pub async fn consume_auth_challenge( + &self, + purpose: String, + token: String, + ) -> napi::Result> { RuntimeStateStore::new(self.pool().await?) .consume_auth_challenge(&purpose, &token) .await + .map_err(napi::Error::from) } #[napi] @@ -64,13 +75,14 @@ impl BackendRuntime { token_type: i32, credential: Option, ttl_ms: i64, - ) -> Result { + ) -> napi::Result { if ttl_ms <= 0 { return Err(napi_error("verification token ttl must be positive")); } RuntimeStateStore::new(self.pool().await?) .create_verification_token(token_type, credential, ttl_ms) .await + .map_err(napi::Error::from) } #[napi] @@ -79,11 +91,12 @@ impl BackendRuntime { token_type: i32, token: String, keep: Option, - ) -> Result> { + ) -> napi::Result> { let keep = keep.unwrap_or(false); RuntimeStateStore::new(self.pool().await?) .get_verification_token(token_type, token, keep) .await + .map_err(napi::Error::from) } #[napi] @@ -93,21 +106,23 @@ impl BackendRuntime { token: String, credential: Option, keep: Option, - ) -> Result> { + ) -> napi::Result> { let keep = keep.unwrap_or(false); RuntimeStateStore::new(self.pool().await?) .verify_verification_token(token_type, token, credential, keep) .await + .map_err(napi::Error::from) } #[napi] - pub async fn cleanup_expired_verification_tokens(&self, limit: i64) -> Result { + pub async fn cleanup_expired_verification_tokens(&self, limit: i64) -> napi::Result { if limit <= 0 { return Err(napi_error("verification token cleanup limit must be positive")); } RuntimeStateStore::new(self.pool().await?) .cleanup_expired_verification_tokens(limit) .await + .map_err(napi::Error::from) } #[napi] @@ -118,10 +133,11 @@ impl BackendRuntime { token: String, client_nonce: Option, ttl_ms: i64, - ) -> Result<()> { + ) -> napi::Result<()> { RuntimeStateStore::new(self.pool().await?) .upsert_magic_link_otp(email, otp_hash, token, client_nonce, ttl_ms) .await + .map_err(napi::Error::from) } #[napi] @@ -130,10 +146,11 @@ impl BackendRuntime { email: String, otp_hash: String, client_nonce: Option, - ) -> Result { + ) -> napi::Result { RuntimeStateStore::new(self.pool().await?) .consume_magic_link_otp(email, otp_hash, client_nonce) .await + .map_err(napi::Error::from) } #[napi] @@ -143,37 +160,41 @@ impl BackendRuntime { invite_id: String, inviter_user_id: String, ttl_ms: i64, - ) -> Result { + ) -> napi::Result { RuntimeStateStore::new(self.pool().await?) .create_workspace_invite_link(workspace_id, invite_id, inviter_user_id, ttl_ms) .await + .map_err(napi::Error::from) } #[napi] pub async fn get_workspace_invite_link( &self, workspace_id: String, - ) -> Result> { + ) -> napi::Result> { RuntimeStateStore::new(self.pool().await?) .get_workspace_invite_link(workspace_id) .await + .map_err(napi::Error::from) } #[napi] pub async fn get_workspace_invite_link_by_id( &self, invite_id: String, - ) -> Result> { + ) -> napi::Result> { RuntimeStateStore::new(self.pool().await?) .get_workspace_invite_link_by_id(invite_id) .await + .map_err(napi::Error::from) } #[napi] - pub async fn revoke_workspace_invite_link(&self, workspace_id: String) -> Result { + pub async fn revoke_workspace_invite_link(&self, workspace_id: String) -> napi::Result { RuntimeStateStore::new(self.pool().await?) .revoke_workspace_invite_link(workspace_id) .await + .map_err(napi::Error::from) } #[napi] @@ -183,35 +204,37 @@ impl BackendRuntime { lease_id: String, payload: serde_json::Value, ttl_ms: i64, - ) -> Result { + ) -> napi::Result { RuntimeStateStore::new(self.pool().await?) .create_byok_local_lease(active_key, lease_id, payload, ttl_ms) .await + .map_err(napi::Error::from) } #[napi] - pub async fn get_byok_local_lease(&self, lease_id: String) -> Result> { + pub async fn get_byok_local_lease(&self, lease_id: String) -> napi::Result> { RuntimeStateStore::new(self.pool().await?) .get_byok_local_lease(lease_id) .await + .map_err(napi::Error::from) } #[napi] - pub async fn cleanup_expired_runtime_states(&self, limit: i64) -> Result { + pub async fn cleanup_expired_runtime_states(&self, limit: i64) -> napi::Result { if limit <= 0 { return Err(napi_error("runtime state cleanup limit must be positive")); } RuntimeStateStore::new(self.pool().await?) .cleanup_expired_runtime_states(limit) .await + .map_err(napi::Error::from) } } #[cfg(test)] mod tests { - use crate::backend_runtime::{ - constants::{MAGIC_LINK_OTP_PURPOSE, WORKSPACE_INVITE_LINK_ID_PURPOSE, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE}, - token_hash, + use super::{ + MAGIC_LINK_OTP_PURPOSE, WORKSPACE_INVITE_LINK_ID_PURPOSE, WORKSPACE_INVITE_LINK_WORKSPACE_PURPOSE, token_hash, }; #[test] diff --git a/packages/backend/native/src/backend_runtime/runtime_state/store.rs b/packages/backend/native/src/runtime/backend_runtime/runtime_state/store.rs similarity index 92% rename from packages/backend/native/src/backend_runtime/runtime_state/store.rs rename to packages/backend/native/src/runtime/backend_runtime/runtime_state/store.rs index 4d12a356b2..6eba25f63f 100644 --- a/packages/backend/native/src/backend_runtime/runtime_state/store.rs +++ b/packages/backend/native/src/runtime/backend_runtime/runtime_state/store.rs @@ -1,10 +1,9 @@ -use napi::Result; use sqlx::PgPool; -use super::{auth_challenge, byok_local_lease, dto::RuntimeStateRows, invite_link, magic_link_otp, verification_token}; -use crate::backend_runtime::types::{ - RuntimeByokLocalLeaseRecord, RuntimeMagicLinkOtpConsumeResult, RuntimeVerificationTokenRecord, - RuntimeWorkspaceInviteLinkRecord, +use super::{ + Result, RuntimeByokLocalLeaseRecord, RuntimeMagicLinkOtpConsumeResult, RuntimeVerificationTokenRecord, + RuntimeWorkspaceInviteLinkRecord, auth_challenge, byok_local_lease, dto::RuntimeStateRows, invite_link, + magic_link_otp, verification_token, }; pub(super) struct RuntimeStateStore { diff --git a/packages/backend/native/src/backend_runtime/runtime_state/verification_token.rs b/packages/backend/native/src/runtime/backend_runtime/runtime_state/verification_token.rs similarity index 93% rename from packages/backend/native/src/backend_runtime/runtime_state/verification_token.rs rename to packages/backend/native/src/runtime/backend_runtime/runtime_state/verification_token.rs index 6de836870b..d0f45fdee7 100644 --- a/packages/backend/native/src/backend_runtime/runtime_state/verification_token.rs +++ b/packages/backend/native/src/runtime/backend_runtime/runtime_state/verification_token.rs @@ -1,12 +1,11 @@ -use napi::Result; use sqlx::{PgPool, Row}; use uuid::Uuid; use super::{ + Result, RuntimeError, RuntimeVerificationTokenRecord, dto::{RuntimeStatePayloadRow, RuntimeStateRows}, - verification_token_purpose, + token_hash, verification_token_purpose, }; -use crate::backend_runtime::{error::napi_error, token_hash, types::RuntimeVerificationTokenRecord}; pub(super) async fn create( rows: &RuntimeStateRows, @@ -64,7 +63,7 @@ pub(super) async fn verify( } else { consume_payload_with_credential(rows.pool(), &purpose, &token, credential.as_deref()).await } - .map_err(|err| napi_error(format!("RuntimeState verification token verify failed: {err}")))?; + .map_err(|err| RuntimeError::database("RuntimeState verification token verify failed", err))?; Ok(row.map(|row| record_from_row(token_type, token, row))) } diff --git a/packages/backend/native/src/backend_runtime/sql/upsert_workspace_admin_stats.sql b/packages/backend/native/src/runtime/backend_runtime/sql/upsert_workspace_admin_stats.sql similarity index 100% rename from packages/backend/native/src/backend_runtime/sql/upsert_workspace_admin_stats.sql rename to packages/backend/native/src/runtime/backend_runtime/sql/upsert_workspace_admin_stats.sql diff --git a/packages/backend/native/src/backend_runtime/tests.rs b/packages/backend/native/src/runtime/backend_runtime/tests.rs similarity index 98% rename from packages/backend/native/src/backend_runtime/tests.rs rename to packages/backend/native/src/runtime/backend_runtime/tests.rs index a74cb0a185..63bb4a7e8e 100644 --- a/packages/backend/native/src/backend_runtime/tests.rs +++ b/packages/backend/native/src/runtime/backend_runtime/tests.rs @@ -1,6 +1,10 @@ use anyhow::{Context, Result as AnyResult, anyhow}; -use super::{runtime_state::*, *}; +use super::{ + super::migrations::{RUNTIME_MIGRATIONS, migrate_runtime_tables}, + runtime_state::*, + *, +}; static PG_TEST_LOCK: std::sync::OnceLock> = std::sync::OnceLock::new(); const TEST_VERIFICATION_TOKEN_TYPE: i32 = 99_999; @@ -70,10 +74,7 @@ async fn runtime_from_database_url() -> AnyResult> { .context("cleanup runtime_leases for backend runtime tests")?; Ok(Some(BackendRuntime { - config: std::sync::RwLock::new(RuntimeConfig { - database_url, - storage: None, - }), + config: std::sync::RwLock::new(BackendRuntimeConfig { database_url }), pool: Mutex::new(Some(pool)), })) } diff --git a/packages/backend/native/src/backend_runtime/workspace_stats.rs b/packages/backend/native/src/runtime/backend_runtime/workspace_stats.rs similarity index 75% rename from packages/backend/native/src/backend_runtime/workspace_stats.rs rename to packages/backend/native/src/runtime/backend_runtime/workspace_stats.rs index 70e749b00d..a6e757a6b3 100644 --- a/packages/backend/native/src/backend_runtime/workspace_stats.rs +++ b/packages/backend/native/src/runtime/backend_runtime/workspace_stats.rs @@ -1,11 +1,10 @@ -use napi::Result; use sqlx::{FromRow, PgPool, Postgres, Row, Transaction}; use tokio::time::{Duration as TokioDuration, sleep}; use super::{ - BackendRuntime, + BackendRuntime, RuntimeError, RuntimeResult, constants::{WORKSPACE_STATS_LEASE_KEY, WORKSPACE_STATS_LOCK_NAMESPACE, WORKSPACE_STATS_REFRESH_LOCK_KEY}, - error::napi_error, + napi_error, types::{ CoordinationLeaseGrant, RuntimeWorkspaceStatsDailyRecalibrationResult, RuntimeWorkspaceStatsRecalibrationResult, RuntimeWorkspaceStatsRefreshResult, RuntimeWorkspaceStatsSnapshotResult, @@ -22,13 +21,13 @@ impl BackendRuntime { batch_limit: i64, owner: String, lease_ttl_ms: i64, - ) -> Result { + ) -> napi::Result { if batch_limit <= 0 { return Err(napi_error("workspace stats dirty refresh limit must be positive")); } let Some(lease) = self - .acquire_coordination_lease(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms) + .acquire_coordination_lease_inner(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms) .await? else { return Ok(RuntimeWorkspaceStatsRefreshResult { @@ -46,7 +45,7 @@ impl BackendRuntime { .await; release_workspace_stats_lease(self, lease).await?; - result + Ok(result?) } #[napi] @@ -56,13 +55,13 @@ impl BackendRuntime { batch_limit: i64, owner: String, lease_ttl_ms: i64, - ) -> Result { + ) -> napi::Result { if batch_limit <= 0 { return Err(napi_error("workspace stats recalibration limit must be positive")); } let Some(lease) = self - .acquire_coordination_lease(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms) + .acquire_coordination_lease_inner(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms) .await? else { return Ok(RuntimeWorkspaceStatsRecalibrationResult { @@ -80,7 +79,7 @@ impl BackendRuntime { .await; release_workspace_stats_lease(self, lease).await?; - result + Ok(result?) } #[napi] @@ -88,9 +87,9 @@ impl BackendRuntime { &self, owner: String, lease_ttl_ms: i64, - ) -> Result { + ) -> napi::Result { let Some(lease) = self - .acquire_coordination_lease(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms) + .acquire_coordination_lease_inner(WORKSPACE_STATS_LEASE_KEY.to_string(), owner, lease_ttl_ms) .await? else { return Ok(RuntimeWorkspaceStatsSnapshotResult { @@ -107,7 +106,7 @@ impl BackendRuntime { .await; release_workspace_stats_lease(self, lease).await?; - result + Ok(result?) } #[napi] @@ -118,7 +117,7 @@ impl BackendRuntime { lease_ttl_ms: i64, lock_retry_times: i64, lock_retry_delay_ms: i64, - ) -> Result { + ) -> napi::Result { if batch_limit <= 0 { return Err(napi_error("workspace stats daily recalibration limit must be positive")); } @@ -150,7 +149,7 @@ impl BackendRuntime { }); }; - let result = async { + let result: RuntimeResult = async { let store = WorkspaceStatsStore::new(self.pool().await?); let mut processed = 0; let mut last_sid = 0; @@ -195,7 +194,7 @@ impl BackendRuntime { .await; release_workspace_stats_lease(self, lease).await?; - result + Ok(result?) } } @@ -214,16 +213,16 @@ impl WorkspaceStatsStore { Self { pool } } - async fn refresh_dirty(&self, batch_limit: i64) -> Result { + async fn refresh_dirty(&self, batch_limit: i64) -> RuntimeResult { let mut tx = self .pool .begin() .await - .map_err(|err| napi_error(format!("WorkspaceStats dirty refresh transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats dirty refresh transaction failed", err))?; if !try_transaction_lock(&mut tx).await? { tx.commit() .await - .map_err(|err| napi_error(format!("WorkspaceStats dirty refresh commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats dirty refresh commit failed", err))?; return Ok(RuntimeWorkspaceStatsRefreshResult { processed: 0, backlog: 0, @@ -236,7 +235,7 @@ impl WorkspaceStatsStore { if dirty.is_empty() { tx.commit() .await - .map_err(|err| napi_error(format!("WorkspaceStats dirty refresh commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats dirty refresh commit failed", err))?; return Ok(RuntimeWorkspaceStatsRefreshResult { processed: 0, backlog, @@ -248,7 +247,7 @@ impl WorkspaceStatsStore { clear_dirty(&mut tx, &dirty).await?; tx.commit() .await - .map_err(|err| napi_error(format!("WorkspaceStats dirty refresh commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats dirty refresh commit failed", err))?; Ok(RuntimeWorkspaceStatsRefreshResult { processed: dirty.len() as i64, @@ -257,16 +256,20 @@ impl WorkspaceStatsStore { }) } - async fn recalibrate(&self, last_sid: i64, batch_limit: i64) -> Result { + async fn recalibrate( + &self, + last_sid: i64, + batch_limit: i64, + ) -> RuntimeResult { let mut tx = self .pool .begin() .await - .map_err(|err| napi_error(format!("WorkspaceStats recalibration transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats recalibration transaction failed", err))?; if !try_transaction_lock(&mut tx).await? { tx.commit() .await - .map_err(|err| napi_error(format!("WorkspaceStats recalibration commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats recalibration commit failed", err))?; return Ok(RuntimeWorkspaceStatsRecalibrationResult { processed: 0, last_sid, @@ -278,7 +281,7 @@ impl WorkspaceStatsStore { if workspaces.is_empty() { tx.commit() .await - .map_err(|err| napi_error(format!("WorkspaceStats recalibration commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats recalibration commit failed", err))?; return Ok(RuntimeWorkspaceStatsRecalibrationResult { processed: 0, last_sid, @@ -297,7 +300,7 @@ impl WorkspaceStatsStore { upsert_stats(&mut tx, &ids).await?; tx.commit() .await - .map_err(|err| napi_error(format!("WorkspaceStats recalibration commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats recalibration commit failed", err))?; Ok(RuntimeWorkspaceStatsRecalibrationResult { processed: ids.len() as i64, @@ -306,16 +309,16 @@ impl WorkspaceStatsStore { }) } - async fn write_daily_snapshot(&self) -> Result { + async fn write_daily_snapshot(&self) -> RuntimeResult { let mut tx = self .pool .begin() .await - .map_err(|err| napi_error(format!("WorkspaceStats daily snapshot transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats daily snapshot transaction failed", err))?; if !try_transaction_lock(&mut tx).await? { tx.commit() .await - .map_err(|err| napi_error(format!("WorkspaceStats daily snapshot commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats daily snapshot commit failed", err))?; return Ok(RuntimeWorkspaceStatsSnapshotResult { snapshotted: 0, skipped: true, @@ -324,7 +327,7 @@ impl WorkspaceStatsStore { let snapshotted = write_daily_snapshot(&mut tx).await?; tx.commit() .await - .map_err(|err| napi_error(format!("WorkspaceStats daily snapshot commit failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats daily snapshot commit failed", err))?; Ok(RuntimeWorkspaceStatsSnapshotResult { snapshotted, @@ -333,9 +336,9 @@ impl WorkspaceStatsStore { } } -async fn release_workspace_stats_lease(runtime: &BackendRuntime, lease: CoordinationLeaseGrant) -> Result<()> { +async fn release_workspace_stats_lease(runtime: &BackendRuntime, lease: CoordinationLeaseGrant) -> RuntimeResult<()> { let _ = runtime - .release_coordination_lease(lease.key, lease.owner, lease.fencing_token) + .release_coordination_lease_inner(lease.key, lease.owner, lease.fencing_token) .await?; Ok(()) } @@ -346,10 +349,10 @@ async fn acquire_workspace_stats_lease_with_retry( lease_ttl_ms: i64, retry_times: i64, retry_delay_ms: i64, -) -> Result> { +) -> RuntimeResult> { for attempt in 0..retry_times { let lease = runtime - .acquire_coordination_lease(WORKSPACE_STATS_LEASE_KEY.to_string(), owner.clone(), lease_ttl_ms) + .acquire_coordination_lease_inner(WORKSPACE_STATS_LEASE_KEY.to_string(), owner.clone(), lease_ttl_ms) .await?; if lease.is_some() { return Ok(lease); @@ -367,11 +370,11 @@ async fn retry_workspace_stats_operation( retry_times: i64, retry_delay_ms: i64, mut operation: F, -) -> Result +) -> RuntimeResult where T: WorkspaceStatsSkippable, F: FnMut() -> Fut, - Fut: std::future::Future>, + Fut: std::future::Future>, { for attempt in 0..retry_times { let result = operation().await?; @@ -403,7 +406,7 @@ impl WorkspaceStatsSkippable for RuntimeWorkspaceStatsSnapshotResult { } } -async fn try_transaction_lock(tx: &mut Transaction<'_, Postgres>) -> Result { +async fn try_transaction_lock(tx: &mut Transaction<'_, Postgres>) -> RuntimeResult { let row = sqlx::query( r#" SELECT pg_try_advisory_xact_lock(($1::bigint << 32) + $2::bigint) AS locked @@ -413,12 +416,12 @@ async fn try_transaction_lock(tx: &mut Transaction<'_, Postgres>) -> Result("locked")) } -async fn load_dirty(tx: &mut Transaction<'_, Postgres>, limit: i64) -> Result> { +async fn load_dirty(tx: &mut Transaction<'_, Postgres>, limit: i64) -> RuntimeResult> { let rows = sqlx::query( r#" SELECT workspace_id @@ -431,20 +434,20 @@ async fn load_dirty(tx: &mut Transaction<'_, Postgres>, limit: i64) -> Result) -> Result { +async fn count_dirty(tx: &mut Transaction<'_, Postgres>) -> RuntimeResult { let row = sqlx::query("SELECT COUNT(*) AS total FROM workspace_admin_stats_dirty") .fetch_one(&mut **tx) .await - .map_err(|err| napi_error(format!("WorkspaceStats count dirty workspaces failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats count dirty workspaces failed", err))?; Ok(row.get::("total")) } -async fn clear_dirty(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String]) -> Result<()> { +async fn clear_dirty(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String]) -> RuntimeResult<()> { sqlx::query( r#" DELETE FROM workspace_admin_stats_dirty @@ -454,11 +457,11 @@ async fn clear_dirty(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String .bind(workspace_ids) .execute(&mut **tx) .await - .map_err(|err| napi_error(format!("WorkspaceStats clear dirty workspaces failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats clear dirty workspaces failed", err))?; Ok(()) } -async fn upsert_stats(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String]) -> Result<()> { +async fn upsert_stats(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[String]) -> RuntimeResult<()> { if workspace_ids.is_empty() { return Ok(()); } @@ -467,7 +470,7 @@ async fn upsert_stats(tx: &mut Transaction<'_, Postgres>, workspace_ids: &[Strin .bind(workspace_ids) .execute(&mut **tx) .await - .map_err(|err| napi_error(format!("WorkspaceStats upsert stats failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats upsert stats failed", err))?; Ok(()) } @@ -475,7 +478,7 @@ async fn fetch_workspace_batch( tx: &mut Transaction<'_, Postgres>, last_sid: i64, limit: i64, -) -> Result> { +) -> RuntimeResult> { sqlx::query_as::<_, WorkspaceSid>( r#" SELECT id, sid @@ -489,10 +492,10 @@ async fn fetch_workspace_batch( .bind(limit) .fetch_all(&mut **tx) .await - .map_err(|err| napi_error(format!("WorkspaceStats fetch workspace batch failed: {err}"))) + .map_err(|err| RuntimeError::database("WorkspaceStats fetch workspace batch failed", err)) } -async fn write_daily_snapshot(tx: &mut Transaction<'_, Postgres>) -> Result { +async fn write_daily_snapshot(tx: &mut Transaction<'_, Postgres>) -> RuntimeResult { let result = sqlx::query( r#" INSERT INTO workspace_admin_stats_daily ( @@ -521,7 +524,7 @@ async fn write_daily_snapshot(tx: &mut Transaction<'_, Postgres>) -> Result ) .execute(&mut **tx) .await - .map_err(|err| napi_error(format!("WorkspaceStats daily snapshot failed: {err}")))?; + .map_err(|err| RuntimeError::database("WorkspaceStats daily snapshot failed", err))?; Ok(result.rows_affected() as i64) } diff --git a/packages/backend/native/src/backend_runtime/config.rs b/packages/backend/native/src/runtime/config.rs similarity index 57% rename from packages/backend/native/src/backend_runtime/config.rs rename to packages/backend/native/src/runtime/config.rs index 15fa106382..047ff3f1a7 100644 --- a/packages/backend/native/src/backend_runtime/config.rs +++ b/packages/backend/native/src/runtime/config.rs @@ -1,43 +1,35 @@ use std::{ - collections::HashMap, env, fs, path::{Path, PathBuf}, }; -use napi::Result; use serde::Deserialize; -use serde_json::{Map, Value}; +use serde_json::Map; use sqlx::{PgPool, Row}; -use super::{ - error::napi_error, - object_storage::{ObjectStorageConfig, StorageProviderConfig}, -}; +use super::{RuntimeError, RuntimeResult}; #[derive(Clone, Debug)] -pub(super) struct RuntimeConfig { - pub(super) database_url: String, - pub(super) storage: Option, +pub(crate) struct BackendRuntimeConfig { + pub(crate) database_url: String, } -impl RuntimeConfig { - pub(super) fn from_config_files() -> Result { +impl BackendRuntimeConfig { + pub(crate) fn from_config_files() -> RuntimeResult { let app_config = app_config_from_config_files()?; let database_url = database_url_from_env() .or(app_config.database_url()) .unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string()); - let storage = ObjectStorageConfig::from_provider_config(app_config.blob_storage_provider_config()?)?; - Ok(Self { database_url, storage }) + Ok(Self { database_url }) } - pub(super) async fn with_db_overrides(&self, pool: &PgPool) -> Result { + pub(crate) async fn with_db_overrides(&self, pool: &PgPool) -> RuntimeResult { let mut app_config = app_config_from_config_files()?; app_config.apply_file_config(load_app_config_overrides_from_db(pool).await?); Ok(Self { // The DB override is loaded after this connection already exists, so it // must not rewrite the active datasource URL. database_url: self.database_url.clone(), - storage: ObjectStorageConfig::from_provider_config(app_config.blob_storage_provider_config()?)?, }) } } @@ -45,8 +37,6 @@ impl RuntimeConfig { #[derive(Debug, Default, Deserialize)] struct AppConfigFile { db: Option, - #[serde(default)] - storages: Option>, } #[derive(Debug, Default, Deserialize)] @@ -63,16 +53,6 @@ impl AppConfigFile { .and_then(|db| db.datasource_url.clone()) .and_then(non_empty_string) } - - fn blob_storage_provider_config(&self) -> Result> { - self - .storages - .as_ref() - .and_then(|storages| storages.get("blob.storage").cloned()) - .map(serde_json::from_value) - .transpose() - .map_err(|err| napi_error(format!("invalid blob storage config: {err}"))) - } } fn database_url_from_env() -> Option { @@ -83,16 +63,15 @@ fn non_empty_string(value: String) -> Option { if value.trim().is_empty() { None } else { Some(value) } } -fn app_config_from_config_files() -> Result { +fn app_config_from_config_files() -> RuntimeResult { let mut merged = AppConfigFile::default(); for path in config_json_paths() { if !path.exists() { continue; } - let raw = fs::read_to_string(&path) - .map_err(|err| napi_error(format!("failed to read config file {}: {err}", path.display())))?; - let config: AppConfigFile = serde_json::from_str(&raw) - .map_err(|err| napi_error(format!("failed to parse config file {}: {err}", path.display())))?; + let raw = fs::read_to_string(&path).map_err(|err| RuntimeError::io("failed to read config file", err))?; + let config: AppConfigFile = + serde_json::from_str(&raw).map_err(|err| RuntimeError::json("failed to parse config file", err))?; merged.apply_file_config(config); } @@ -104,19 +83,14 @@ impl AppConfigFile { if config.db.is_some() { self.db = config.db; } - if let Some(storages) = config.storages - && !storages.is_empty() - { - self.storages.get_or_insert_with(HashMap::new).extend(storages); - } } } -async fn load_app_config_overrides_from_db(pool: &PgPool) -> Result { +async fn load_app_config_overrides_from_db(pool: &PgPool) -> RuntimeResult { let rows = match sqlx::query("SELECT id, value FROM app_configs").fetch_all(pool).await { Ok(rows) => rows, Err(sqlx::Error::Database(err)) if err.code().as_deref() == Some("42P01") => return Ok(AppConfigFile::default()), - Err(err) => return Err(napi_error(format!("failed to load app config overrides: {err}"))), + Err(err) => return Err(RuntimeError::database("failed to load app config overrides", err)), }; app_config_from_flat_overrides(rows.into_iter().map(|row| { @@ -126,7 +100,7 @@ async fn load_app_config_overrides_from_db(pool: &PgPool) -> Result(rows: I) -> Result +fn app_config_from_flat_overrides(rows: I) -> RuntimeResult where I: IntoIterator, S: AsRef, @@ -145,7 +119,7 @@ where } serde_json::from_value(serde_json::Value::Object(root)) - .map_err(|err| napi_error(format!("invalid app config overrides: {err}"))) + .map_err(|err| RuntimeError::json("invalid app config overrides", err)) } pub(super) fn config_json_paths() -> Vec { @@ -208,50 +182,19 @@ mod tests { } #[test] - fn parses_blob_storage_app_config_value() { + fn ignores_storage_app_config_values() { let app_config = app_config_from_flat_overrides([ - ( - "unknown.future.config", - serde_json::json!({ - "shape": "ignored" - }), - ), ( "storages.blob.storage", - serde_json::json!({ - "provider": "cloudflare-r2", - "bucket": "workspace-blobs-canary", - "config": { - "accountId": "account", - "credentials": { - "accessKeyId": "key", - "secretAccessKey": "secret" - }, - "usePresignedURL": { - "enabled": true - } - } - }), - ), - ( - "storages.avatar.publicPath", - serde_json::json!("https://avatar.affineassets.com/"), + serde_json::json!({"provider": "cloudflare-r2"}), ), + ("db.datasourceUrl", serde_json::json!("postgresql://example/runtime")), ]) .unwrap(); - let storage = app_config.blob_storage_provider_config().unwrap().unwrap(); - let config = ObjectStorageConfig::from_provider_config(Some(storage)) - .unwrap() - .unwrap(); - let health = config.health(); - assert!(health.configured); - assert_eq!(health.provider.as_deref(), Some("cloudflare-r2")); - assert_eq!(health.bucket.as_deref(), Some("workspace-blobs-canary")); assert_eq!( - health.endpoint.as_deref(), - Some("https://account.r2.cloudflarestorage.com") + app_config.database_url().as_deref(), + Some("postgresql://example/runtime") ); - assert!(health.use_presigned_url); } } diff --git a/packages/backend/native/src/runtime/error.rs b/packages/backend/native/src/runtime/error.rs new file mode 100644 index 0000000000..55a3393832 --- /dev/null +++ b/packages/backend/native/src/runtime/error.rs @@ -0,0 +1,126 @@ +use napi::{Error, Status}; + +use super::storage_runtime::object_storage::error::ObjectStorageError; + +pub(crate) type RuntimeResult = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum RuntimeError { + #[error("{0}")] + Config(String), + + #[error("{0}")] + InvalidInput(String), + + #[error("{0}")] + InvalidState(String), + + #[error("{context}: {source}")] + Database { + context: String, + #[source] + source: sqlx::Error, + }, + + #[error("{context}: {source}")] + Io { + context: String, + #[source] + source: std::io::Error, + }, + + #[error("{context}: {source}")] + Json { + context: String, + #[source] + source: serde_json::Error, + }, + + #[error("{context}: {source}")] + Time { + context: String, + #[source] + source: std::time::SystemTimeError, + }, + + #[error(transparent)] + ObjectStorage(#[from] ObjectStorageError), + + #[error("{0}")] + NapiBoundary(String), +} + +impl RuntimeError { + pub(crate) fn config(message: impl Into) -> Self { + Self::Config(message.into()) + } + + pub(crate) fn invalid_input(message: impl Into) -> Self { + Self::InvalidInput(message.into()) + } + + pub(crate) fn invalid_state(message: impl Into) -> Self { + Self::InvalidState(message.into()) + } + + pub(crate) fn database(context: impl Into, source: sqlx::Error) -> Self { + Self::Database { + context: context.into(), + source, + } + } + + pub(crate) fn io(context: impl Into, source: std::io::Error) -> Self { + Self::Io { + context: context.into(), + source, + } + } + + pub(crate) fn json(context: impl Into, source: serde_json::Error) -> Self { + Self::Json { + context: context.into(), + source, + } + } + + pub(crate) fn is_object_missing(&self) -> bool { + match self { + Self::ObjectStorage(error) => error.is_not_found(), + Self::Io { source, .. } => source.kind() == std::io::ErrorKind::NotFound, + Self::InvalidState(message) + | Self::InvalidInput(message) + | Self::Config(message) + | Self::NapiBoundary(message) => { + message.contains("NoSuchKey") || message.contains("NotFound") || message.contains("not found") + } + _ => false, + } + } +} + +pub(crate) fn to_napi_error(error: RuntimeError) -> Error { + Error::new(Status::GenericFailure, error.to_string()) +} + +impl From for Error { + fn from(error: RuntimeError) -> Self { + to_napi_error(error) + } +} + +impl From for Error { + fn from(error: ObjectStorageError) -> Self { + to_napi_error(RuntimeError::from(error)) + } +} + +impl From for RuntimeError { + fn from(error: Error) -> Self { + Self::NapiBoundary(error.to_string()) + } +} + +pub(crate) fn napi_error(message: impl Into) -> Error { + Error::new(Status::GenericFailure, message.into()) +} diff --git a/packages/backend/native/src/runtime/migrations.rs b/packages/backend/native/src/runtime/migrations.rs new file mode 100644 index 0000000000..b49d687df2 --- /dev/null +++ b/packages/backend/native/src/runtime/migrations.rs @@ -0,0 +1,20 @@ +use sqlx::PgPool; + +use super::{RuntimeError, RuntimeResult}; + +pub(crate) const RUNTIME_MIGRATIONS: &str = include_str!("sql/runtime_migrations.sql"); + +pub(crate) async fn migrate_runtime_tables(pool: &PgPool) -> RuntimeResult<()> { + for statement in RUNTIME_MIGRATIONS + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + { + sqlx::query(statement) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("Runtime migration failed", err))?; + } + + Ok(()) +} diff --git a/packages/backend/native/src/runtime/mod.rs b/packages/backend/native/src/runtime/mod.rs new file mode 100644 index 0000000000..73a9fd8132 --- /dev/null +++ b/packages/backend/native/src/runtime/mod.rs @@ -0,0 +1,10 @@ +pub mod backend_runtime; +pub mod storage_runtime; + +pub(crate) mod config; +pub(crate) mod error; +pub(crate) mod migrations; +pub(crate) mod types; + +pub(crate) use config::BackendRuntimeConfig; +pub(crate) use error::{RuntimeError, RuntimeResult, napi_error, to_napi_error}; diff --git a/packages/backend/native/src/backend_runtime/sql/runtime_migrations.sql b/packages/backend/native/src/runtime/sql/runtime_migrations.sql similarity index 100% rename from packages/backend/native/src/backend_runtime/sql/runtime_migrations.sql rename to packages/backend/native/src/runtime/sql/runtime_migrations.sql diff --git a/packages/backend/native/src/runtime/storage_runtime/assetpack.rs b/packages/backend/native/src/runtime/storage_runtime/assetpack.rs new file mode 100644 index 0000000000..201f97c50b --- /dev/null +++ b/packages/backend/native/src/runtime/storage_runtime/assetpack.rs @@ -0,0 +1,344 @@ +use std::{ + io::{BufReader, Cursor}, + path::PathBuf, + time::SystemTime, +}; + +use assetpack_core::{ + Codec, FileHint, FileTransformConfig, Hash32, ObjectKind, Pipeline, PipelineConfig, SqliteStore, TransformRegistry, + TransformSelector, build_recipe, pack::ObjectRecord, parse_recipe_checked, +}; +use sqlx::Row; + +use super::{ + FsStorageConfig, MAX_BLOB_SIZE, ObjectGetResult, ObjectListEntry, ObjectMetadata, ObjectPutMetadata, RuntimeError, + RuntimeResult, fs_bucket_path, normalize_storage_key, system_time_ms, +}; + +pub(super) async fn put( + config: &FsStorageConfig, + scope: &str, + key: &str, + body: Vec, + metadata: ObjectPutMetadata, +) -> RuntimeResult { + normalize_storage_key(key)?; + let metadata = metadata.complete_for_body(&body); + let content_length = metadata.content_length.unwrap_or(body.len() as i64); + if content_length != body.len() as i64 { + return Err(RuntimeError::invalid_input( + "Assetpack contentLength does not match body length", + )); + } + if !(0..=MAX_BLOB_SIZE).contains(&content_length) { + return Err(RuntimeError::invalid_input( + "Assetpack contentLength exceeds supported blob size", + )); + } + + let store = open_store(config).await?; + let transform_config = FileTransformConfig::default(); + let bucket_path = fs_bucket_path(config); + let selector = TransformSelector::new( + transform_config.clone(), + transform_config.resolved_temp_dir(&bucket_path), + assetpack_transform_precomp2::default_specs(), + ); + let original_hash = Hash32::sha3_256(&body); + let hint = FileHint { + size: body.len() as u64, + extension: extension_from_key(key), + head: Some(body.iter().take(4096).copied().collect()), + }; + let plan = Pipeline::new(PipelineConfig::default()) + .run(body, &hint, original_hash, Some(&selector)) + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack pipeline failed: {err}")))?; + + let chunks = plan + .chunks + .iter() + .map(|chunk| (chunk.hash, chunk.raw_len)) + .collect::>(); + let recipe = build_recipe( + plan.original_size, + &chunks, + plan.original_hash, + plan.transform_id, + plan.transform_version, + ); + let recipe_hash = Hash32::sha3_256(&recipe); + + let mut objects = Vec::with_capacity(plan.chunks.len() + 1); + for chunk in plan.chunks { + let Some(payload) = chunk.payload else { + return Err(RuntimeError::invalid_state( + "Assetpack pipeline unexpectedly discarded chunk payload", + )); + }; + objects.push(ObjectRecord { + hash: chunk.hash, + kind: ObjectKind::Chunk, + size: chunk.raw_len as u64, + codec: chunk.codec, + content: payload, + }); + } + objects.push(ObjectRecord { + hash: recipe_hash, + kind: ObjectKind::Recipe, + size: recipe.len() as u64, + codec: Codec::Raw, + content: recipe, + }); + + let mut tx = store + .begin_write_tx() + .await + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack begin write failed: {err}")))?; + store + .put_objects_batch_tx(&mut tx, &objects) + .await + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack object write failed: {err}")))?; + store + .put_file_recipe_cache_batch_tx(&mut tx, &[(original_hash, recipe_hash)]) + .await + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack recipe cache write failed: {err}")))?; + let object_metadata = ObjectMetadata { + content_type: metadata + .content_type + .unwrap_or_else(|| "application/octet-stream".to_string()), + content_length, + last_modified_ms: system_time_ms(SystemTime::now())?, + checksum_crc32: metadata.checksum_crc32, + }; + + sqlx::query( + r#" + INSERT INTO storage_assetpack_blobs + (scope, key, recipe_hash, content_type, content_length, checksum_crc32, last_modified_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT (scope, key) + DO UPDATE SET + recipe_hash = excluded.recipe_hash, + content_type = excluded.content_type, + content_length = excluded.content_length, + checksum_crc32 = excluded.checksum_crc32, + last_modified_ms = excluded.last_modified_ms + "#, + ) + .bind(scope) + .bind(key) + .bind(recipe_hash.to_hex()) + .bind(&object_metadata.content_type) + .bind(object_metadata.content_length) + .bind(&object_metadata.checksum_crc32) + .bind(object_metadata.last_modified_ms) + .execute(&mut *tx) + .await + .map_err(|err| RuntimeError::database("Assetpack manifest write failed", err))?; + tx.commit() + .await + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack commit failed: {err}")))?; + + Ok(object_metadata) +} + +pub(super) async fn head(config: &FsStorageConfig, scope: &str, key: &str) -> RuntimeResult> { + normalize_storage_key(key)?; + let store = open_store(config).await?; + manifest_row(&store, scope, key) + .await + .map(|row| row.map(|row| row.metadata)) +} + +pub(super) async fn get(config: &FsStorageConfig, scope: &str, key: &str) -> RuntimeResult> { + normalize_storage_key(key)?; + let store = open_store(config).await?; + let Some(row) = manifest_row(&store, scope, key).await? else { + return Ok(None); + }; + let recipe_hash = Hash32::from_hex(&row.recipe_hash) + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack manifest recipe hash is invalid: {err}")))?; + let Some(recipe_object) = store + .get_object(&recipe_hash) + .await + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack recipe read failed: {err}")))? + else { + return Err(RuntimeError::invalid_state(format!( + "Assetpack recipe object is missing for {key}" + ))); + }; + let recipe = parse_recipe_checked(&recipe_object.content, &recipe_hash) + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack recipe parse failed: {err}")))?; + + let mut stored_stream = Vec::with_capacity(recipe.stored_stream_size as usize); + for (chunk_hash, expected_len) in &recipe.chunks { + let Some(chunk) = store + .get_object(chunk_hash) + .await + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack chunk read failed: {err}")))? + else { + return Err(RuntimeError::invalid_state(format!( + "Assetpack chunk is missing for {key}: {chunk_hash}" + ))); + }; + if chunk.kind != ObjectKind::Chunk || chunk.size != *expected_len as u64 { + return Err(RuntimeError::invalid_state(format!( + "Assetpack chunk metadata mismatch for {key}: {chunk_hash}" + ))); + } + stored_stream.extend_from_slice(&chunk.content); + } + + let body = decode_stored_stream(recipe.transform_id, stored_stream)?; + if body.len() as u64 != recipe.original_file_size || Hash32::sha3_256(&body) != recipe.original_file_hash { + return Err(RuntimeError::invalid_state(format!( + "Assetpack reconstructed body failed integrity check for {key}" + ))); + } + + Ok(Some(ObjectGetResult { + body, + metadata: row.metadata, + })) +} + +pub(super) async fn list( + config: &FsStorageConfig, + scope: &str, + prefix: Option, +) -> RuntimeResult> { + let prefix = prefix.unwrap_or_default(); + if !prefix.is_empty() { + super::normalize_storage_prefix(&prefix)?; + } + let store = open_store(config).await?; + let rows = sqlx::query( + r#" + SELECT key, content_length, last_modified_ms + FROM storage_assetpack_blobs + WHERE scope = ?1 AND key LIKE ?2 + ORDER BY key ASC + "#, + ) + .bind(scope) + .bind(format!("{prefix}%")) + .fetch_all(store.pool()) + .await + .map_err(|err| RuntimeError::database("Assetpack manifest list failed", err))?; + + rows + .into_iter() + .map(|row| { + Ok(ObjectListEntry { + key: row.get("key"), + content_length: row.get::("content_length"), + last_modified_ms: row.get("last_modified_ms"), + }) + }) + .collect() +} + +pub(super) async fn delete(config: &FsStorageConfig, scope: &str, key: &str) -> RuntimeResult<()> { + normalize_storage_key(key)?; + let store = open_store(config).await?; + sqlx::query("DELETE FROM storage_assetpack_blobs WHERE scope = ?1 AND key = ?2") + .bind(scope) + .bind(key) + .execute(store.pool()) + .await + .map_err(|err| RuntimeError::database("Assetpack manifest delete failed", err))?; + Ok(()) +} + +async fn open_store(config: &FsStorageConfig) -> RuntimeResult { + let store = SqliteStore::open(store_path(config)) + .await + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack store open failed: {err}")))?; + ensure_manifest_schema(&store).await?; + Ok(store) +} + +fn store_path(config: &FsStorageConfig) -> PathBuf { + fs_bucket_path(config).join("assetpack.sqlite") +} + +fn extension_from_key(key: &str) -> Option { + key + .rsplit_once('.') + .and_then(|(_, extension)| (!extension.is_empty()).then(|| extension.to_ascii_lowercase())) +} + +struct ManifestRow { + recipe_hash: String, + metadata: ObjectMetadata, +} + +async fn ensure_manifest_schema(store: &SqliteStore) -> RuntimeResult<()> { + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS storage_assetpack_blobs ( + scope TEXT NOT NULL, + key TEXT NOT NULL, + recipe_hash TEXT NOT NULL, + content_type TEXT NOT NULL, + content_length INTEGER NOT NULL, + checksum_crc32 TEXT, + last_modified_ms INTEGER NOT NULL, + PRIMARY KEY (scope, key) + ) + "#, + ) + .execute(store.pool()) + .await + .map_err(|err| RuntimeError::database("Assetpack manifest schema create failed", err))?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS storage_assetpack_blobs_scope_prefix_idx ON storage_assetpack_blobs (scope, key)", + ) + .execute(store.pool()) + .await + .map_err(|err| RuntimeError::database("Assetpack manifest index create failed", err))?; + Ok(()) +} + +async fn manifest_row(store: &SqliteStore, scope: &str, key: &str) -> RuntimeResult> { + let row = sqlx::query( + r#" + SELECT recipe_hash, content_type, content_length, checksum_crc32, last_modified_ms + FROM storage_assetpack_blobs + WHERE scope = ?1 AND key = ?2 + "#, + ) + .bind(scope) + .bind(key) + .fetch_optional(store.pool()) + .await + .map_err(|err| RuntimeError::database("Assetpack manifest read failed", err))?; + + row + .map(|row| { + Ok(ManifestRow { + recipe_hash: row.get("recipe_hash"), + metadata: ObjectMetadata { + content_type: row.get("content_type"), + content_length: row.get("content_length"), + checksum_crc32: row.get("checksum_crc32"), + last_modified_ms: row.get("last_modified_ms"), + }, + }) + }) + .transpose() +} + +fn decode_stored_stream(transform_id: u16, stored_stream: Vec) -> RuntimeResult> { + let transform_config = FileTransformConfig::default(); + let registry = TransformRegistry::new(&transform_config, assetpack_transform_precomp2::default_specs()); + let transform = registry + .get(transform_id) + .ok_or_else(|| RuntimeError::invalid_state(format!("Assetpack transform is not registered: {transform_id}")))?; + let mut out = Vec::new(); + transform + .decode(&mut BufReader::new(Cursor::new(stored_stream)), &mut out) + .map_err(|err| RuntimeError::invalid_state(format!("Assetpack transform decode failed: {err}")))?; + Ok(out) +} diff --git a/packages/backend/native/src/backend_runtime/blob_cleanup.rs b/packages/backend/native/src/runtime/storage_runtime/blob_cleanup.rs similarity index 86% rename from packages/backend/native/src/backend_runtime/blob_cleanup.rs rename to packages/backend/native/src/runtime/storage_runtime/blob_cleanup.rs index 203e077b56..9c6daf789b 100644 --- a/packages/backend/native/src/backend_runtime/blob_cleanup.rs +++ b/packages/backend/native/src/runtime/storage_runtime/blob_cleanup.rs @@ -1,11 +1,9 @@ use chrono::{DateTime, Duration, Utc}; -use napi::Result; use sqlx::{FromRow, PgPool}; use super::{ - BackendRuntime, - error::napi_error, - types::{RuntimeBlobCleanupExecuteResult, RuntimeBlobCleanupPlanResult}, + RuntimeBlobCleanupExecuteResult, RuntimeBlobCleanupPlanResult, RuntimeError, RuntimeResult, StorageRuntime, + napi_error, }; #[cfg(test)] @@ -66,7 +64,7 @@ fn push_workspace_once(workspace_ids: &mut Vec, workspace_id: &str) { } } -async fn checkpoint_completed(pool: &PgPool, kind: &str, scope: &str) -> Result { +async fn checkpoint_completed(pool: &PgPool, kind: &str, scope: &str) -> RuntimeResult { sqlx::query_scalar::<_, bool>( "SELECT EXISTS(SELECT 1 FROM blob_reconciliation_checkpoints WHERE kind = $1 AND scope = $2 AND status = \ 'completed')", @@ -75,10 +73,10 @@ async fn checkpoint_completed(pool: &PgPool, kind: &str, scope: &str) -> Result< .bind(scope) .fetch_one(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup checkpoint check failed: {err}"))) + .map_err(|err| RuntimeError::database("Blob cleanup checkpoint check failed", err)) } -async fn projection_is_stale(pool: &PgPool, workspace_id: &str) -> Result { +async fn projection_is_stale(pool: &PgPool, workspace_id: &str) -> RuntimeResult { let checkpoint_fresh = checkpoint_completed(pool, "doc_blob_refs", workspace_id).await?; let has_stale_rows = sqlx::query_scalar::<_, bool>( "SELECT EXISTS(SELECT 1 FROM doc_blob_refs WHERE workspace_id = $1 AND status <> 'fresh')", @@ -86,11 +84,11 @@ async fn projection_is_stale(pool: &PgPool, workspace_id: &str) -> Result .bind(workspace_id) .fetch_one(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup projection freshness check failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob cleanup projection freshness check failed", err))?; Ok(!checkpoint_fresh || has_stale_rows) } -async fn stale_projection_workspaces(pool: &PgPool, workspace_id: &str) -> Result> { +async fn stale_projection_workspaces(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { if projection_is_stale(pool, workspace_id).await? { Ok(vec![workspace_id.to_string()]) } else { @@ -98,11 +96,11 @@ async fn stale_projection_workspaces(pool: &PgPool, workspace_id: &str) -> Resul } } -async fn metadata_backfill_is_complete(pool: &PgPool, workspace_id: &str) -> Result { +async fn metadata_backfill_is_complete(pool: &PgPool, workspace_id: &str) -> RuntimeResult { checkpoint_completed(pool, "blob_metadata_backfill", workspace_id).await } -async fn has_doc_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result { +async fn has_doc_ref(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeResult { sqlx::query_scalar::<_, bool>( "SELECT EXISTS(SELECT 1 FROM doc_blob_refs WHERE workspace_id = $1 AND blob_key = $2 AND status = 'fresh')", ) @@ -110,10 +108,10 @@ async fn has_doc_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result Result { +async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeResult { let required_ref = sqlx::query_scalar::<_, bool>( r#" SELECT EXISTS(SELECT 1 FROM workspaces WHERE id = $1 AND avatar_key = $2) @@ -136,7 +134,7 @@ async fn has_other_ref(pool: &PgPool, workspace_id: &str, key: &str) -> Result Result Result Result { +async fn table_exists(pool: &PgPool, table: &str) -> RuntimeResult { sqlx::query_scalar::<_, bool>("SELECT to_regclass($1) IS NOT NULL") .bind(format!("public.{table}")) .fetch_one(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup table existence check failed: {err}"))) + .map_err(|err| RuntimeError::database("Blob cleanup table existence check failed", err)) } async fn load_completed_blobs( @@ -180,7 +178,7 @@ async fn load_completed_blobs( workspace_id: &str, after_key: Option<&str>, limit: i64, -) -> Result> { +) -> RuntimeResult> { sqlx::query_as::<_, BlobCandidateRow>( r#" SELECT workspace_id, key, size @@ -198,17 +196,17 @@ async fn load_completed_blobs( .bind(limit) .fetch_all(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup load completed blobs failed: {err}"))) + .map_err(|err| RuntimeError::database("Blob cleanup load completed blobs failed", err)) } -async fn load_plan_cursor(pool: &PgPool, workspace_id: &str) -> Result> { +async fn load_plan_cursor(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { let row = sqlx::query_as::<_, (String, serde_json::Value)>( "SELECT status, cursor FROM blob_reconciliation_checkpoints WHERE kind = 'blob_cleanup_plan' AND scope = $1", ) .bind(workspace_id) .fetch_optional(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup plan checkpoint load failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob cleanup plan checkpoint load failed", err))?; let Some((status, cursor)) = row else { return Ok(None); }; @@ -228,7 +226,7 @@ async fn upsert_plan_checkpoint( workspace_id: &str, last_blob_key: Option<&str>, completed: bool, -) -> Result<()> { +) -> RuntimeResult<()> { let status = if completed { "completed" } else { "running" }; sqlx::query( r#" @@ -250,11 +248,11 @@ async fn upsert_plan_checkpoint( .bind(completed) .execute(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup plan checkpoint write failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob cleanup plan checkpoint write failed", err))?; Ok(()) } -async fn create_run(pool: &PgPool, workspace_id: &str) -> Result { +async fn create_run(pool: &PgPool, workspace_id: &str) -> RuntimeResult { sqlx::query_scalar::<_, String>( r#" INSERT INTO blob_reconciliation_runs (kind, mode, status, workspace_id) @@ -265,7 +263,7 @@ async fn create_run(pool: &PgPool, workspace_id: &str) -> Result { .bind(workspace_id) .fetch_one(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup create run failed: {err}"))) + .map_err(|err| RuntimeError::database("Blob cleanup create run failed", err)) } async fn finish_run( @@ -274,14 +272,14 @@ async fn finish_run( workspace_id: &str, result: &RuntimeBlobCleanupPlanResult, stale_projection_workspaces: Vec, -) -> Result<()> { +) -> RuntimeResult<()> { let candidate_bytes = sqlx::query_scalar::<_, Option>( "SELECT SUM(object_size)::bigint FROM blob_cleanup_candidates WHERE run_id = $1::uuid AND status = 'marked'", ) .bind(run_id) .fetch_one(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup candidate bytes audit failed: {err}")))? + .map_err(|err| RuntimeError::database("Blob cleanup candidate bytes audit failed", err))? .unwrap_or(0); sqlx::query( r#" @@ -309,7 +307,7 @@ async fn finish_run( })) .execute(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup finish run failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob cleanup finish run failed", err))?; Ok(()) } @@ -321,7 +319,7 @@ async fn mark_candidate_status( status: &str, evidence: serde_json::Value, error: Option<&str>, -) -> Result<()> { +) -> RuntimeResult<()> { sqlx::query( r#" UPDATE blob_cleanup_candidates @@ -340,11 +338,15 @@ async fn mark_candidate_status( .bind(run_id) .execute(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup mark candidate status failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob cleanup mark candidate status failed", err))?; Ok(()) } -async fn finish_execute_run(pool: &PgPool, run_id: &str, result: &RuntimeBlobCleanupExecuteResult) -> Result<()> { +async fn finish_execute_run( + pool: &PgPool, + run_id: &str, + result: &RuntimeBlobCleanupExecuteResult, +) -> RuntimeResult<()> { sqlx::query( r#" UPDATE blob_reconciliation_runs @@ -369,7 +371,7 @@ async fn finish_execute_run(pool: &PgPool, run_id: &str, result: &RuntimeBlobCle })) .execute(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup execute run finish failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob cleanup execute run finish failed", err))?; Ok(()) } @@ -379,7 +381,7 @@ async fn mark_candidate( row: &BlobCandidateRow, object_size: i64, object_last_modified: DateTime, -) -> Result { +) -> RuntimeResult { let result = sqlx::query( r#" INSERT INTO blob_cleanup_candidates @@ -404,11 +406,11 @@ async fn mark_candidate( .bind(serde_json::json!({ "metadataSize": row.size })) .execute(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup mark candidate failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob cleanup mark candidate failed", err))?; Ok(result.rows_affected() as i64) } -async fn load_marked_candidates(pool: &PgPool, run_id: &str, limit: i64) -> Result> { +async fn load_marked_candidates(pool: &PgPool, run_id: &str, limit: i64) -> RuntimeResult> { sqlx::query_as::<_, MarkedCandidateRow>( r#" SELECT workspace_id, blob_key @@ -422,18 +424,18 @@ async fn load_marked_candidates(pool: &PgPool, run_id: &str, limit: i64) -> Resu .bind(limit) .fetch_all(pool) .await - .map_err(|err| napi_error(format!("Blob cleanup load marked candidates failed: {err}"))) + .map_err(|err| RuntimeError::database("Blob cleanup load marked candidates failed", err)) } #[napi_derive::napi] -impl BackendRuntime { +impl StorageRuntime { #[napi] pub async fn plan_unreferenced_workspace_blobs( &self, workspace_id: String, grace_period_days: i64, limit: i64, - ) -> Result { + ) -> napi::Result { if limit <= 0 { return Err(napi_error("blob cleanup plan limit must be positive")); } @@ -484,7 +486,7 @@ impl BackendRuntime { continue; }; let last_modified = DateTime::::from_timestamp_millis(metadata.last_modified_ms) - .ok_or_else(|| napi_error("blob cleanup object last modified is invalid"))?; + .ok_or_else(|| RuntimeError::invalid_state("blob cleanup object last modified is invalid"))?; if metadata.content_length != row.size as i64 || last_modified > min_last_modified { result.protected_by_metadata += 1; continue; @@ -506,7 +508,7 @@ impl BackendRuntime { run_id: String, grace_period_days: i64, limit: i64, - ) -> Result { + ) -> napi::Result { if limit <= 0 { return Err(napi_error("blob cleanup execute limit must be positive")); } @@ -566,7 +568,7 @@ impl BackendRuntime { }; if let Some(metadata) = metadata { let last_modified = DateTime::::from_timestamp_millis(metadata.last_modified_ms) - .ok_or_else(|| napi_error("blob cleanup execute object last modified is invalid"))?; + .ok_or_else(|| RuntimeError::invalid_state("blob cleanup execute object last modified is invalid"))?; if last_modified > min_last_modified { result.skipped_still_referenced += 1; mark_candidate_status( diff --git a/packages/backend/native/src/backend_runtime/blob_reclaimer.rs b/packages/backend/native/src/runtime/storage_runtime/blob_reclaimer.rs similarity index 78% rename from packages/backend/native/src/backend_runtime/blob_reclaimer.rs rename to packages/backend/native/src/runtime/storage_runtime/blob_reclaimer.rs index a4517aa9f2..190a4d4390 100644 --- a/packages/backend/native/src/backend_runtime/blob_reclaimer.rs +++ b/packages/backend/native/src/runtime/storage_runtime/blob_reclaimer.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; use napi::Result; use sqlx::{FromRow, PgPool}; -use super::{BackendRuntime, error::napi_error, types::RuntimeBlobCleanupResult}; +use super::{RuntimeBlobCleanupResult, RuntimeError, RuntimeResult, StorageRuntime, napi_error}; #[derive(FromRow)] struct BlobRow { @@ -20,7 +20,7 @@ impl BlobReclaimerStore { Self { pool } } - async fn load_expired_pending(&self, cutoff: DateTime, limit: i64) -> Result> { + async fn load_expired_pending(&self, cutoff: DateTime, limit: i64) -> RuntimeResult> { sqlx::query_as::<_, BlobRow>( r#" SELECT workspace_id, key, upload_id @@ -36,10 +36,10 @@ impl BlobReclaimerStore { .bind(limit) .fetch_all(&self.pool) .await - .map_err(|err| napi_error(format!("BlobReclaimer load pending blobs failed: {err}"))) + .map_err(|err| RuntimeError::database("BlobReclaimer load pending blobs failed", err)) } - async fn load_deleted(&self, workspace_id: &str, limit: i64) -> Result> { + async fn load_deleted(&self, workspace_id: &str, limit: i64) -> RuntimeResult> { sqlx::query_as::<_, BlobRow>( r#" SELECT workspace_id, key, upload_id @@ -54,10 +54,10 @@ impl BlobReclaimerStore { .bind(limit) .fetch_all(&self.pool) .await - .map_err(|err| napi_error(format!("BlobReclaimer load deleted blobs failed: {err}"))) + .map_err(|err| RuntimeError::database("BlobReclaimer load deleted blobs failed", err)) } - async fn delete_pending_metadata(&self, workspace_id: &str, key: &str) -> Result { + async fn delete_pending_metadata(&self, workspace_id: &str, key: &str) -> RuntimeResult { let result = sqlx::query( r#" DELETE FROM blobs @@ -70,11 +70,11 @@ impl BlobReclaimerStore { .bind(key) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("BlobReclaimer delete pending blob metadata failed: {err}")))?; + .map_err(|err| RuntimeError::database("BlobReclaimer delete pending blob metadata failed", err))?; Ok(result.rows_affected() as i64) } - async fn delete_released_metadata(&self, workspace_id: &str, key: &str) -> Result { + async fn delete_released_metadata(&self, workspace_id: &str, key: &str) -> RuntimeResult { let result = sqlx::query( r#" DELETE FROM blobs @@ -86,31 +86,23 @@ impl BlobReclaimerStore { .bind(key) .execute(&self.pool) .await - .map_err(|err| napi_error(format!("BlobReclaimer delete blob metadata failed: {err}")))?; + .map_err(|err| RuntimeError::database("BlobReclaimer delete blob metadata failed", err))?; Ok(result.rows_affected() as i64) } } -fn object_missing_error(err: &napi::Error) -> bool { - let message = err.to_string(); - message.contains("NoSuchKey") - || message.contains("NoSuchUpload") - || message.contains("NotFound") - || message.contains("not found") -} - -async fn delete_object_idempotent(runtime: &BackendRuntime, key: &str) -> Result<()> { +async fn delete_object_idempotent(runtime: &StorageRuntime, key: &str) -> RuntimeResult<()> { match runtime.object_storage_delete_object(key).await { Ok(()) => Ok(()), - Err(err) if object_missing_error(&err) => Ok(()), + Err(err) if err.is_object_missing() => Ok(()), Err(err) => Err(err), } } -async fn abort_upload_idempotent(runtime: &BackendRuntime, key: &str, upload_id: &str) -> Result<()> { +async fn abort_upload_idempotent(runtime: &StorageRuntime, key: &str, upload_id: &str) -> RuntimeResult<()> { match runtime.object_storage_abort_upload(key, upload_id).await { Ok(()) => Ok(()), - Err(err) if object_missing_error(&err) => Ok(()), + Err(err) if err.is_object_missing() => Ok(()), Err(err) => Err(err), } } @@ -122,7 +114,7 @@ fn push_workspace_once(workspace_ids: &mut Vec, workspace_id: &str) { } #[napi_derive::napi] -impl BackendRuntime { +impl StorageRuntime { #[napi] pub async fn cleanup_expired_pending_blobs(&self, cutoff_ms: i64, limit: i64) -> Result { if limit <= 0 { @@ -130,7 +122,7 @@ impl BackendRuntime { } let cutoff = DateTime::::from_timestamp_millis(cutoff_ms) - .ok_or_else(|| napi_error("pending blob cleanup cutoff is invalid"))?; + .ok_or_else(|| RuntimeError::invalid_input("pending blob cleanup cutoff is invalid"))?; let store = BlobReclaimerStore::new(self.pool().await?); let rows = store.load_expired_pending(cutoff, limit).await?; diff --git a/packages/backend/native/src/backend_runtime/blob_reconciliation.rs b/packages/backend/native/src/runtime/storage_runtime/blob_reconciliation.rs similarity index 85% rename from packages/backend/native/src/backend_runtime/blob_reconciliation.rs rename to packages/backend/native/src/runtime/storage_runtime/blob_reconciliation.rs index 93031e6610..dcf38488ff 100644 --- a/packages/backend/native/src/backend_runtime/blob_reconciliation.rs +++ b/packages/backend/native/src/runtime/storage_runtime/blob_reconciliation.rs @@ -1,28 +1,25 @@ use chrono::{DateTime, Utc}; -use napi::Result; use sqlx::{FromRow, PgPool}; use super::{ - BackendRuntime, - error::napi_error, - types::{RuntimeBlobMetadataBackfillResult, RuntimeObjectMetadata}, + RuntimeBlobMetadataBackfillResult, RuntimeError, RuntimeObjectMetadata, RuntimeResult, StorageRuntime, napi_error, }; -async fn workspace_exists(pool: &PgPool, workspace_id: &str) -> Result { +async fn workspace_exists(pool: &PgPool, workspace_id: &str) -> RuntimeResult { sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM workspaces WHERE id = $1)") .bind(workspace_id) .fetch_one(pool) .await - .map_err(|err| napi_error(format!("Blob metadata backfill workspace check failed: {err}"))) + .map_err(|err| RuntimeError::database("Blob metadata backfill workspace check failed", err)) } -async fn blob_exists(pool: &PgPool, workspace_id: &str, key: &str) -> Result { +async fn blob_exists(pool: &PgPool, workspace_id: &str, key: &str) -> RuntimeResult { sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM blobs WHERE workspace_id = $1 AND key = $2)") .bind(workspace_id) .bind(key) .fetch_one(pool) .await - .map_err(|err| napi_error(format!("Blob metadata backfill blob check failed: {err}"))) + .map_err(|err| RuntimeError::database("Blob metadata backfill blob check failed", err)) } async fn upsert_blob_metadata( @@ -30,9 +27,9 @@ async fn upsert_blob_metadata( workspace_id: &str, key: &str, metadata: RuntimeObjectMetadata, -) -> Result { +) -> RuntimeResult { let last_modified = DateTime::::from_timestamp_millis(metadata.last_modified_ms) - .ok_or_else(|| napi_error("Blob metadata backfill object last modified is invalid"))?; + .ok_or_else(|| RuntimeError::invalid_state("Blob metadata backfill object last modified is invalid"))?; let result = sqlx::query( r#" INSERT INTO blobs (workspace_id, key, size, mime, status, upload_id, created_at, deleted_at) @@ -53,7 +50,7 @@ async fn upsert_blob_metadata( .bind(last_modified) .execute(pool) .await - .map_err(|err| napi_error(format!("Blob metadata backfill upsert failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob metadata backfill upsert failed", err))?; Ok(result.rows_affected() as i64) } @@ -86,14 +83,14 @@ impl BackfillCheckpoint { } } -async fn load_checkpoint(pool: &PgPool, scope: &str) -> Result> { +async fn load_checkpoint(pool: &PgPool, scope: &str) -> RuntimeResult> { sqlx::query_as::<_, BackfillCheckpoint>( "SELECT last_key, cursor FROM blob_reconciliation_checkpoints WHERE kind = 'blob_metadata_backfill' AND scope = $1", ) .bind(scope) .fetch_optional(pool) .await - .map_err(|err| napi_error(format!("Blob metadata backfill checkpoint load failed: {err}"))) + .map_err(|err| RuntimeError::database("Blob metadata backfill checkpoint load failed", err)) } async fn upsert_checkpoint( @@ -102,7 +99,7 @@ async fn upsert_checkpoint( last_key: Option<&str>, continuation_token: Option<&str>, completed: bool, -) -> Result<()> { +) -> RuntimeResult<()> { let status = if completed { "completed" } else { "running" }; sqlx::query( r#" @@ -131,7 +128,7 @@ async fn upsert_checkpoint( })) .execute(pool) .await - .map_err(|err| napi_error(format!("Blob metadata backfill checkpoint write failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob metadata backfill checkpoint write failed", err))?; Ok(()) } @@ -163,18 +160,18 @@ fn push_workspace_once(workspace_ids: &mut Vec, workspace_id: &str) { } } -fn checked_list_page_limit(limit: i64) -> Result { - i32::try_from(limit).map_err(|_| napi_error("blob metadata backfill limit exceeds i32::MAX")) +fn checked_list_page_limit(limit: i64) -> RuntimeResult { + i32::try_from(limit).map_err(|_| RuntimeError::invalid_input("blob metadata backfill limit exceeds i32::MAX")) } #[napi_derive::napi] -impl BackendRuntime { +impl StorageRuntime { #[napi] pub async fn backfill_missing_blob_metadata( &self, workspace_id: Option, limit: i64, - ) -> Result { + ) -> napi::Result { if limit <= 0 { return Err(napi_error("blob metadata backfill limit must be positive")); } @@ -269,7 +266,7 @@ impl BackendRuntime { })) .execute(&pool) .await - .map_err(|err| napi_error(format!("Blob metadata backfill run record failed: {err}")))?; + .map_err(|err| RuntimeError::database("Blob metadata backfill run record failed", err))?; Ok(result) } diff --git a/packages/backend/native/src/backend_runtime/doc_blob_refs.rs b/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs similarity index 74% rename from packages/backend/native/src/backend_runtime/doc_blob_refs.rs rename to packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs index 7e77118aa2..2b223c1270 100644 --- a/packages/backend/native/src/backend_runtime/doc_blob_refs.rs +++ b/packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs @@ -1,10 +1,9 @@ use affine_common::doc_parser; use chrono::{DateTime, Utc}; -use napi::Result; use sqlx::{FromRow, PgPool}; use y_octo::Doc; -use super::{BackendRuntime, error::napi_error, types::RuntimeDocBlobRefsResult}; +use super::{RuntimeDocBlobRefsResult, RuntimeError, RuntimeResult, StorageRuntime, napi_error}; const PARSER_VERSION: i32 = 1; @@ -28,7 +27,7 @@ struct ExtractedRef { flavour: String, } -async fn load_snapshot(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result> { +async fn load_snapshot(pool: &PgPool, workspace_id: &str, doc_id: &str) -> RuntimeResult> { sqlx::query_as::<_, SnapshotRow>( r#" SELECT workspace_id, guid AS doc_id, blob, updated_at @@ -40,10 +39,10 @@ async fn load_snapshot(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Resul .bind(doc_id) .fetch_optional(pool) .await - .map_err(|err| napi_error(format!("Doc blob refs load snapshot failed: {err}"))) + .map_err(|err| RuntimeError::database("Doc blob refs load snapshot failed", err)) } -async fn load_updates(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result> { +async fn load_updates(pool: &PgPool, workspace_id: &str, doc_id: &str) -> RuntimeResult> { sqlx::query_as::<_, UpdateRow>( r#" SELECT blob, created_at @@ -56,22 +55,22 @@ async fn load_updates(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result .bind(doc_id) .fetch_all(pool) .await - .map_err(|err| napi_error(format!("Doc blob refs load updates failed: {err}"))) + .map_err(|err| RuntimeError::database("Doc blob refs load updates failed", err)) } -fn apply_doc_updates(updates: impl IntoIterator>) -> Result> { +fn apply_doc_updates(updates: impl IntoIterator>) -> RuntimeResult> { let mut doc = Doc::default(); for update in updates { doc .apply_update_from_binary_v1(&update) - .map_err(|err| napi_error(format!("Doc blob refs merge failed: {err}")))?; + .map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs merge failed: {err}")))?; } doc .encode_update_v1() - .map_err(|err| napi_error(format!("Doc blob refs encode failed: {err}"))) + .map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs encode failed: {err}"))) } -async fn load_current_doc(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Result> { +async fn load_current_doc(pool: &PgPool, workspace_id: &str, doc_id: &str) -> RuntimeResult> { let snapshot = load_snapshot(pool, workspace_id, doc_id).await?; let updates = load_updates(pool, workspace_id, doc_id).await?; if snapshot.is_none() && updates.is_empty() { @@ -99,12 +98,12 @@ async fn load_current_doc(pool: &PgPool, workspace_id: &str, doc_id: &str) -> Re })) } -async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> Result> { +async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { let Some(root) = load_current_doc(pool, workspace_id, workspace_id).await? else { return Ok(Vec::new()); }; let ids = doc_parser::get_doc_ids_from_binary(root.blob, false) - .map_err(|err| napi_error(format!("Doc blob refs root doc parse failed: {err}")))?; + .map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs root doc parse failed: {err}")))?; let mut ids = ids; ids.sort(); Ok(ids) @@ -114,7 +113,7 @@ async fn upsert_projection_checkpoint( pool: &PgPool, workspace_id: &str, result: &RuntimeDocBlobRefsResult, -) -> Result<()> { +) -> RuntimeResult<()> { let completed = result.next_cursor.is_none(); let status = if completed && result.failed_docs == 0 { "completed" @@ -145,11 +144,11 @@ async fn upsert_projection_checkpoint( })) .execute(pool) .await - .map_err(|err| napi_error(format!("Doc blob refs checkpoint write failed: {err}")))?; + .map_err(|err| RuntimeError::database("Doc blob refs checkpoint write failed", err))?; Ok(()) } -async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str, error: &str) -> Result<()> { +async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str, error: &str) -> RuntimeResult<()> { sqlx::query( r#" INSERT INTO blob_reconciliation_checkpoints @@ -170,18 +169,18 @@ async fn upsert_projection_failure_checkpoint(pool: &PgPool, workspace_id: &str, })) .execute(pool) .await - .map_err(|err| napi_error(format!("Doc blob refs failure checkpoint write failed: {err}")))?; + .map_err(|err| RuntimeError::database("Doc blob refs failure checkpoint write failed", err))?; Ok(()) } -async fn load_projection_cursor(pool: &PgPool, workspace_id: &str) -> Result> { +async fn load_projection_cursor(pool: &PgPool, workspace_id: &str) -> RuntimeResult> { let cursor = sqlx::query_scalar::<_, serde_json::Value>( "SELECT cursor FROM blob_reconciliation_checkpoints WHERE kind = 'doc_blob_refs' AND scope = $1", ) .bind(workspace_id) .fetch_optional(pool) .await - .map_err(|err| napi_error(format!("Doc blob refs checkpoint load failed: {err}")))?; + .map_err(|err| RuntimeError::database("Doc blob refs checkpoint load failed", err))?; Ok(cursor.and_then(|cursor| { cursor .get("lastDocId") @@ -190,7 +189,7 @@ async fn load_projection_cursor(pool: &PgPool, workspace_id: &str) -> Result Result { +async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_ids: &[String]) -> RuntimeResult { let result = sqlx::query( r#" DELETE FROM doc_blob_refs @@ -202,13 +201,13 @@ async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_i .bind(current_doc_ids) .execute(pool) .await - .map_err(|err| napi_error(format!("Doc blob refs purge removed docs failed: {err}")))?; + .map_err(|err| RuntimeError::database("Doc blob refs purge removed docs failed", err))?; Ok(result.rows_affected() as i64) } -fn extract_refs(snapshot: &SnapshotRow) -> Result> { +fn extract_refs(snapshot: &SnapshotRow) -> RuntimeResult> { let parsed = doc_parser::parse_doc_from_binary(snapshot.blob.clone(), snapshot.doc_id.clone()) - .map_err(|err| napi_error(format!("Doc blob refs parse failed: {err}")))?; + .map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs parse failed: {err}")))?; let mut refs = Vec::new(); for block in parsed.blocks { let Some(blob_keys) = block.blob else { @@ -251,18 +250,18 @@ mod tests { } } -async fn replace_doc_refs(pool: &PgPool, snapshot: &SnapshotRow, refs: Vec) -> Result<(i64, i64)> { +async fn replace_doc_refs(pool: &PgPool, snapshot: &SnapshotRow, refs: Vec) -> RuntimeResult<(i64, i64)> { let mut tx = pool .begin() .await - .map_err(|err| napi_error(format!("Doc blob refs transaction failed: {err}")))?; + .map_err(|err| RuntimeError::database("Doc blob refs transaction failed", err))?; let deleted = sqlx::query("DELETE FROM doc_blob_refs WHERE workspace_id = $1 AND doc_id = $2") .bind(&snapshot.workspace_id) .bind(&snapshot.doc_id) .execute(&mut *tx) .await - .map_err(|err| napi_error(format!("Doc blob refs delete failed: {err}")))? + .map_err(|err| RuntimeError::database("Doc blob refs delete failed", err))? .rows_affected() as i64; let mut written = 0; @@ -290,18 +289,18 @@ async fn replace_doc_refs(pool: &PgPool, snapshot: &SnapshotRow, refs: Vec Result<()> { +async fn mark_doc_failed(pool: &PgPool, workspace_id: &str, doc_id: &str, error: &str) -> RuntimeResult<()> { sqlx::query( r#" INSERT INTO doc_blob_refs @@ -319,44 +318,56 @@ async fn mark_doc_failed(pool: &PgPool, workspace_id: &str, doc_id: &str, error: .bind(error) .execute(pool) .await - .map_err(|err| napi_error(format!("Doc blob refs mark failure failed: {err}")))?; + .map_err(|err| RuntimeError::database("Doc blob refs mark failure failed", err))?; Ok(()) } -#[napi_derive::napi] -impl BackendRuntime { - #[napi] - pub async fn rebuild_doc_blob_refs(&self, workspace_id: String, doc_id: String) -> Result { - let pool = self.pool().await?; - let mut result = RuntimeDocBlobRefsResult { - scanned_docs: 1, - parsed_docs: 0, - refs_written: 0, - refs_deleted: 0, - failed_docs: 0, - next_cursor: None, - }; +async fn rebuild_doc_blob_refs_inner( + runtime: &StorageRuntime, + workspace_id: String, + doc_id: String, +) -> RuntimeResult { + let pool = runtime.pool().await?; + let mut result = RuntimeDocBlobRefsResult { + scanned_docs: 1, + parsed_docs: 0, + refs_written: 0, + refs_deleted: 0, + failed_docs: 0, + next_cursor: None, + }; - let Some(snapshot) = load_current_doc(&pool, &workspace_id, &doc_id).await? else { - result.failed_docs = 1; - mark_doc_failed(&pool, &workspace_id, &doc_id, "snapshot_missing").await?; - return Ok(result); - }; + let Some(snapshot) = load_current_doc(&pool, &workspace_id, &doc_id).await? else { + result.failed_docs = 1; + mark_doc_failed(&pool, &workspace_id, &doc_id, "snapshot_missing").await?; + return Ok(result); + }; - match extract_refs(&snapshot) { - Ok(refs) => { - let (written, deleted) = replace_doc_refs(&pool, &snapshot, refs).await?; - result.parsed_docs = 1; - result.refs_written = written; - result.refs_deleted = deleted; - } - Err(err) => { - result.failed_docs = 1; - mark_doc_failed(&pool, &workspace_id, &doc_id, &err.to_string()).await?; - } + match extract_refs(&snapshot) { + Ok(refs) => { + let (written, deleted) = replace_doc_refs(&pool, &snapshot, refs).await?; + result.parsed_docs = 1; + result.refs_written = written; + result.refs_deleted = deleted; } + Err(err) => { + result.failed_docs = 1; + mark_doc_failed(&pool, &workspace_id, &doc_id, &err.to_string()).await?; + } + } - Ok(result) + Ok(result) +} + +#[napi_derive::napi] +impl StorageRuntime { + #[napi] + pub async fn rebuild_doc_blob_refs( + &self, + workspace_id: String, + doc_id: String, + ) -> napi::Result { + Ok(rebuild_doc_blob_refs_inner(self, workspace_id, doc_id).await?) } #[napi] @@ -364,7 +375,7 @@ impl BackendRuntime { &self, workspace_id: String, limit: i64, - ) -> Result { + ) -> napi::Result { if limit <= 0 { return Err(napi_error("doc blob refs rebuild limit must be positive")); } @@ -374,7 +385,7 @@ impl BackendRuntime { Ok(doc_ids) => doc_ids, Err(err) => { upsert_projection_failure_checkpoint(&pool, &workspace_id, &err.to_string()).await?; - return Err(err); + return Err(err.into()); } }; let cursor = load_projection_cursor(&pool, &workspace_id).await?; @@ -396,7 +407,7 @@ impl BackendRuntime { let mut last_doc_id = None; for doc_id in doc_ids.into_iter().take(limit as usize) { last_doc_id = Some(doc_id.clone()); - let result = self.rebuild_doc_blob_refs(workspace_id.clone(), doc_id).await?; + let result = rebuild_doc_blob_refs_inner(self, workspace_id.clone(), doc_id).await?; total.scanned_docs += result.scanned_docs; total.parsed_docs += result.parsed_docs; total.refs_written += result.refs_written; diff --git a/packages/backend/native/src/runtime/storage_runtime/mod.rs b/packages/backend/native/src/runtime/storage_runtime/mod.rs new file mode 100644 index 0000000000..de3dd2f867 --- /dev/null +++ b/packages/backend/native/src/runtime/storage_runtime/mod.rs @@ -0,0 +1,1550 @@ +use std::{ + collections::HashMap, + env, fs, + path::{Path, PathBuf}, + sync::RwLock, + time::SystemTime, +}; + +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use napi::bindgen_prelude::Buffer; +use serde::Deserialize; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use sqlx::{PgPool, Row, postgres::PgPoolOptions}; +use tokio::sync::Mutex; + +mod assetpack; +mod blob_cleanup; +mod blob_reclaimer; +mod blob_reconciliation; +mod doc_blob_refs; +pub(crate) mod object_storage; + +use self::object_storage::{ + ObjectStorageConfig, StorageProviderConfig, + types::{ObjectGetResult, ObjectListEntry, ObjectMetadata, ObjectPutMetadata}, +}; +pub(super) use super::{ + RuntimeError, RuntimeResult, + migrations::migrate_runtime_tables, + napi_error, to_napi_error, + types::{ + RuntimeBlobCleanupExecuteResult, RuntimeBlobCleanupPlanResult, RuntimeBlobCleanupResult, RuntimeBlobCompleteResult, + RuntimeBlobMetadataBackfillResult, RuntimeDocBlobRefsResult, RuntimeMultipartUploadInit, + RuntimeMultipartUploadPart, RuntimeObjectGetResult, RuntimeObjectListEntry, RuntimeObjectMetadata, + RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest, + }, +}; + +const MAX_BLOB_SIZE: i64 = i32::MAX as i64; + +type Result = RuntimeResult; + +#[napi_derive::napi(object)] +pub struct StorageRuntimeHealth { + pub started: bool, + pub database_connected: bool, + pub provider_configured: bool, + pub provider: Option, + pub bucket: Option, +} + +#[napi_derive::napi(object)] +pub struct StorageProviderCapabilities { + pub put: bool, + pub get: bool, + pub head: bool, + pub list: bool, + pub delete: bool, + pub presign_put: bool, + pub presign_get: bool, + pub multipart_direct: bool, + pub proxy_upload: bool, + pub assetpack: bool, + pub server_mediated_only: bool, +} + +#[derive(Clone, Debug)] +enum StorageBackendConfig { + Fs(FsStorageConfig), + S3(ObjectStorageConfig), + Assetpack(FsStorageConfig), +} + +#[derive(Clone, Debug)] +struct FsStorageConfig { + provider: String, + root: String, + bucket: String, +} + +#[derive(Clone, Debug)] +struct StorageRuntimeConfig { + database_url: String, + backends: HashMap, +} + +#[derive(Debug, Default, Deserialize)] +struct AppConfigFile { + db: Option, + #[serde(default)] + storages: Option>, + copilot: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct DbConfigFile { + datasource_url: Option, +} + +#[derive(Debug, Deserialize)] +struct FsConfigFile { + path: String, +} + +#[derive(Debug, Default, Deserialize)] +struct CopilotConfigFile { + storage: Option, +} + +impl StorageRuntimeConfig { + fn from_config_files() -> RuntimeResult { + let app_config = app_config_from_config_files()?; + let database_url = database_url_from_env() + .or(app_config.database_url()) + .unwrap_or_else(|| "postgresql://localhost:5432/affine".to_string()); + let backends = app_config.storage_backends()?; + Ok(Self { database_url, backends }) + } + + async fn with_db_overrides(&self, pool: &PgPool) -> RuntimeResult { + let mut app_config = app_config_from_config_files()?; + app_config.apply_file_config(load_app_config_overrides_from_db(pool).await?); + Ok(Self { + database_url: self.database_url.clone(), + backends: app_config.storage_backends()?, + }) + } +} + +impl StorageBackendConfig { + fn from_provider_config(storage: Option) -> RuntimeResult> { + let Some(storage) = storage else { + return Ok(None); + }; + + match storage.provider.as_str() { + "fs" => { + let config: FsConfigFile = serde_json::from_value(storage.config) + .map_err(|err| RuntimeError::json("invalid fs blob storage config", err))?; + Ok(Some(Self::Fs(FsStorageConfig { + provider: storage.provider, + root: config.path, + bucket: storage.bucket, + }))) + } + "assetpack" => { + let config: FsConfigFile = serde_json::from_value(storage.config) + .map_err(|err| RuntimeError::json("invalid assetpack blob storage config", err))?; + Ok(Some(Self::Assetpack(FsStorageConfig { + provider: storage.provider, + root: config.path, + bucket: storage.bucket, + }))) + } + "aws-s3" | "cloudflare-r2" => ObjectStorageConfig::from_provider_config(Some(storage)) + .map(|v| v.map(Self::S3)) + .map_err(Into::into), + provider => Err(RuntimeError::config(format!( + "unsupported blob storage provider for StorageRuntime: {provider}" + ))), + } + } + + fn provider(&self) -> &str { + match self { + Self::Fs(config) | Self::Assetpack(config) => &config.provider, + Self::S3(config) => &config.provider, + } + } + + fn bucket(&self) -> &str { + match self { + Self::Fs(config) | Self::Assetpack(config) => &config.bucket, + Self::S3(config) => &config.bucket, + } + } + + fn capabilities(&self) -> StorageProviderCapabilities { + match self { + Self::Fs(_) => StorageProviderCapabilities { + put: true, + get: true, + head: true, + list: true, + delete: true, + presign_put: false, + presign_get: false, + multipart_direct: false, + proxy_upload: false, + assetpack: false, + server_mediated_only: true, + }, + Self::S3(config) => { + let _configured_min_part_size = config.min_part_size; + StorageProviderCapabilities { + put: true, + get: true, + head: true, + list: true, + delete: true, + presign_put: config.use_presigned_url, + presign_get: config.use_presigned_url, + multipart_direct: config.use_presigned_url, + proxy_upload: config.proxy_upload, + assetpack: false, + server_mediated_only: !config.use_presigned_url, + } + } + Self::Assetpack(_) => StorageProviderCapabilities { + put: true, + get: true, + head: true, + list: true, + delete: true, + presign_put: false, + presign_get: false, + multipart_direct: false, + proxy_upload: false, + assetpack: true, + server_mediated_only: true, + }, + } + } +} + +impl AppConfigFile { + fn database_url(&self) -> Option { + self + .db + .as_ref() + .and_then(|db| db.datasource_url.clone()) + .and_then(non_empty_string) + } + + fn storage_backends(&self) -> RuntimeResult> { + let mut backends = HashMap::new(); + if let Some(storage) = self.storage_provider_config("blob.storage")? + && let Some(backend) = StorageBackendConfig::from_provider_config(Some(storage))? + { + backends.insert("blob".to_string(), backend); + } + if let Some(storage) = self.storage_provider_config("avatar.storage")? + && let Some(backend) = StorageBackendConfig::from_provider_config(Some(storage))? + { + backends.insert("avatar".to_string(), backend); + } + if let Some(storage) = self.copilot.as_ref().and_then(|copilot| copilot.storage.clone()) + && let Some(backend) = StorageBackendConfig::from_provider_config(Some(storage))? + { + backends.insert("copilot".to_string(), backend); + } + Ok(backends) + } + + fn storage_provider_config(&self, key: &str) -> RuntimeResult> { + self + .storages + .as_ref() + .and_then(|storages| storages.get(key).cloned()) + .map(serde_json::from_value) + .transpose() + .map_err(|err| RuntimeError::json("invalid storage provider config", err)) + } + + fn apply_file_config(&mut self, config: AppConfigFile) { + if config.db.is_some() { + self.db = config.db; + } + if let Some(storages) = config.storages + && !storages.is_empty() + { + self.storages.get_or_insert_with(HashMap::new).extend(storages); + } + if config.copilot.is_some() { + self.copilot = config.copilot; + } + } +} + +#[napi_derive::napi] +pub struct StorageRuntime { + config: RwLock, + pool: Mutex>, +} + +#[napi_derive::napi] +impl StorageRuntime { + #[napi(constructor)] + pub fn new() -> napi::Result { + Ok(Self { + config: RwLock::new(StorageRuntimeConfig::from_config_files().map_err(to_napi_error)?), + pool: Mutex::new(None), + }) + } + + #[napi] + pub async fn start(&self) -> napi::Result<()> { + self.start_inner().await.map_err(to_napi_error) + } + + async fn start_inner(&self) -> RuntimeResult<()> { + let mut guard = self.pool.lock().await; + if guard.is_some() { + return Ok(()); + } + + let database_url = self.config()?.database_url; + let pool = PgPoolOptions::new() + .max_connections(5) + .acquire_timeout(std::time::Duration::from_secs(5)) + .connect(&database_url) + .await + .map_err(|err| RuntimeError::database("StorageRuntime failed to connect postgres", err))?; + + sqlx::query("SELECT 1") + .execute(&pool) + .await + .map_err(|err| RuntimeError::database("StorageRuntime postgres health check failed", err))?; + + let config = self.config()?.with_db_overrides(&pool).await?; + self.update_config(config)?; + *guard = Some(pool); + Ok(()) + } + + #[napi] + pub async fn stop(&self) -> napi::Result<()> { + let pool = self.pool.lock().await.take(); + if let Some(pool) = pool { + pool.close().await; + } + Ok(()) + } + + #[napi] + pub async fn run_migrations(&self) -> napi::Result<()> { + let pool = self.pool().await?; + migrate_runtime_tables(&pool).await.map_err(to_napi_error) + } + + #[napi] + pub async fn health(&self) -> napi::Result { + self.health_inner().await.map_err(to_napi_error) + } + + async fn health_inner(&self) -> RuntimeResult { + let pool = self.pool.lock().await.as_ref().cloned(); + let database_connected = match pool.as_ref() { + Some(pool) => sqlx::query("SELECT 1") + .fetch_one(pool) + .await + .map(|row| row.try_get::(0).unwrap_or(0) == 1) + .unwrap_or(false), + None => false, + }; + let backend = self.config()?.backends.get("blob").cloned(); + + Ok(StorageRuntimeHealth { + started: pool.is_some(), + database_connected, + provider_configured: !self.config()?.backends.is_empty(), + provider: backend.as_ref().map(|backend| backend.provider().to_string()), + bucket: backend.as_ref().map(|backend| backend.bucket().to_string()), + }) + } + + #[napi] + pub async fn provider_capabilities(&self, scope: String) -> napi::Result { + self + .backend_for_scope(&scope) + .map(|backend| backend.capabilities()) + .map_err(to_napi_error) + } + + #[napi] + pub async fn put_object( + &self, + _scope: String, + key: String, + body: Buffer, + metadata: Option, + ) -> napi::Result { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(config) => Ok( + fs_put( + &config, + &key, + body.to_vec(), + metadata.map(Into::into).unwrap_or_default(), + )? + .into(), + ), + StorageBackendConfig::Assetpack(config) => assetpack::put( + &config, + &_scope, + &key, + body.to_vec(), + metadata.map(Into::into).unwrap_or_default(), + ) + .await + .map(Into::into) + .map_err(napi::Error::from), + StorageBackendConfig::S3(config) => config + .build_client()? + .put(&key, body.to_vec(), metadata.map(Into::into).unwrap_or_default()) + .await + .map(Into::into) + .map_err(napi::Error::from), + } + } + + #[napi] + pub async fn head_object(&self, _scope: String, key: String) -> napi::Result> { + let metadata = match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(config) => fs_head(&config, &key)?, + StorageBackendConfig::Assetpack(config) => assetpack::head(&config, &_scope, &key).await?, + StorageBackendConfig::S3(config) => config.build_client()?.head(&key).await?, + }; + Ok(metadata.map(Into::into)) + } + + #[napi] + pub async fn get_object(&self, _scope: String, key: String) -> napi::Result> { + let object = match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(config) => fs_get(&config, &key)?, + StorageBackendConfig::Assetpack(config) => assetpack::get(&config, &_scope, &key).await?, + StorageBackendConfig::S3(config) => config.build_client()?.get(&key).await?, + }; + Ok(object.map(Into::into)) + } + + #[napi] + pub async fn list_objects( + &self, + _scope: String, + prefix: Option, + ) -> napi::Result> { + let entries = match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(config) => fs_list(&config, prefix)?, + StorageBackendConfig::Assetpack(config) => assetpack::list(&config, &_scope, prefix).await?, + StorageBackendConfig::S3(config) => config.build_client()?.list(prefix).await?, + }; + Ok(entries.into_iter().map(Into::into).collect()) + } + + #[napi] + pub async fn delete_object(&self, _scope: String, key: String) -> napi::Result<()> { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(config) => Ok(fs_delete(&config, &key)?), + StorageBackendConfig::Assetpack(config) => assetpack::delete(&config, &_scope, &key) + .await + .map_err(napi::Error::from), + StorageBackendConfig::S3(config) => config.build_client()?.delete(&key).await.map_err(Into::into), + } + } + + #[napi] + pub async fn presign_put( + &self, + _scope: String, + key: String, + metadata: Option, + ) -> napi::Result> { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(None), + StorageBackendConfig::S3(config) => Ok(Some( + config + .build_client()? + .presign_put(&key, metadata.map(Into::into).unwrap_or_default()) + .await + .map_err(napi::Error::from)? + .try_into()?, + )), + } + } + + #[napi] + pub async fn presign_get(&self, _scope: String, key: String) -> napi::Result> { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(None), + StorageBackendConfig::S3(config) => Ok(Some( + config + .build_client()? + .presign_get(&key) + .await + .map_err(napi::Error::from)? + .try_into()?, + )), + } + } + + #[napi] + pub async fn create_multipart_upload( + &self, + _scope: String, + key: String, + metadata: Option, + ) -> napi::Result> { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(None), + StorageBackendConfig::S3(config) => Ok( + config + .build_client()? + .create_multipart_upload(&key, metadata.map(Into::into).unwrap_or_default()) + .await + .map_err(napi::Error::from)? + .map(Into::into), + ), + } + } + + #[napi] + pub async fn presign_upload_part( + &self, + _scope: String, + key: String, + upload_id: String, + part_number: i32, + ) -> napi::Result> { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(None), + StorageBackendConfig::S3(config) => Ok(Some( + config + .build_client()? + .presign_upload_part(&key, &upload_id, part_number) + .await + .map_err(napi::Error::from)? + .try_into()?, + )), + } + } + + #[napi] + pub async fn proxy_upload_part( + &self, + _scope: String, + key: String, + upload_id: String, + part_number: i32, + body: Buffer, + content_length: Option, + ) -> napi::Result> { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(None), + StorageBackendConfig::S3(config) => config + .build_client()? + .upload_part(&key, &upload_id, part_number, body.to_vec(), content_length) + .await + .map_err(napi::Error::from), + } + } + + #[napi] + pub async fn list_multipart_upload_parts( + &self, + _scope: String, + key: String, + upload_id: String, + ) -> napi::Result>> { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(None), + StorageBackendConfig::S3(config) => Ok(Some( + config + .build_client()? + .list_multipart_upload_parts(&key, &upload_id) + .await + .map_err(napi::Error::from)? + .into_iter() + .map(Into::into) + .collect(), + )), + } + } + + #[napi] + pub async fn complete_multipart_upload( + &self, + _scope: String, + key: String, + upload_id: String, + parts: Vec, + ) -> napi::Result { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(false), + StorageBackendConfig::S3(config) => { + config + .build_client()? + .complete_multipart_upload(&key, &upload_id, parts.into_iter().map(Into::into).collect()) + .await + .map_err(napi::Error::from)?; + Ok(true) + } + } + } + + #[napi] + pub async fn abort_multipart_upload(&self, _scope: String, key: String, upload_id: String) -> napi::Result { + match self.backend_for_scope(&_scope)? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(false), + StorageBackendConfig::S3(config) => { + config + .build_client()? + .abort_multipart_upload(&key, &upload_id) + .await + .map_err(napi::Error::from)?; + Ok(true) + } + } + } + + #[napi] + pub async fn complete_workspace_blob_upload( + &self, + workspace_id: String, + key: String, + expected_size: i64, + expected_mime: String, + ) -> napi::Result { + match self.backend_for_scope("blob").map_err(napi::Error::from)? { + StorageBackendConfig::Fs(config) => self + .complete_fs_workspace_blob(config, workspace_id, key, expected_size, expected_mime) + .await + .map_err(napi::Error::from), + StorageBackendConfig::Assetpack(config) => self + .complete_assetpack_workspace_blob(config, workspace_id, key, expected_size, expected_mime) + .await + .map_err(napi::Error::from), + StorageBackendConfig::S3(_) => self + .complete_s3_workspace_blob(workspace_id, key, expected_size, expected_mime) + .await + .map_err(napi::Error::from), + } + } + + fn config(&self) -> Result { + self + .config + .read() + .map(|config| config.clone()) + .map_err(|_| RuntimeError::invalid_state("StorageRuntime config lock poisoned")) + } + + fn update_config(&self, config: StorageRuntimeConfig) -> Result<()> { + *self + .config + .write() + .map_err(|_| RuntimeError::invalid_state("StorageRuntime config lock poisoned"))? = config; + Ok(()) + } + + fn backend_for_scope(&self, scope: &str) -> Result { + self + .config()? + .backends + .get(scope) + .cloned() + .or_else(|| self.config().ok()?.backends.get("blob").cloned()) + .ok_or_else(|| RuntimeError::config(format!("StorageRuntime provider is not configured for scope {scope}"))) + } + + pub(crate) async fn object_storage_delete_object(&self, key: &str) -> Result<()> { + match self.backend_for_scope("blob")? { + StorageBackendConfig::Fs(config) => fs_delete(&config, key), + StorageBackendConfig::Assetpack(config) => assetpack::delete(&config, "blob", key).await, + StorageBackendConfig::S3(config) => config.build_client()?.delete(key).await.map_err(Into::into), + } + } + + pub(crate) async fn object_storage_abort_upload(&self, key: &str, upload_id: &str) -> Result<()> { + match self.backend_for_scope("blob")? { + StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(()), + StorageBackendConfig::S3(config) => config + .build_client()? + .abort_multipart_upload(key, upload_id) + .await + .map_err(Into::into), + } + } + + pub(crate) async fn object_storage_list_page( + &self, + prefix: Option, + continuation_token: Option, + start_after: Option, + max_keys: i32, + ) -> Result { + match self.backend_for_scope("blob")? { + StorageBackendConfig::Fs(config) => { + let mut entries = fs_list(&config, prefix)?; + if let Some(start_after) = start_after { + entries.retain(|entry| entry.key > start_after); + } + if continuation_token.is_some() { + return Err(RuntimeError::invalid_input( + "StorageRuntime fs list continuation token is not supported", + )); + } + let max_keys = usize::try_from(max_keys) + .map_err(|_| RuntimeError::invalid_input("StorageRuntime list maxKeys must be positive"))?; + entries.truncate(max_keys); + Ok(object_storage::types::ObjectListPage { + entries, + next_continuation_token: None, + }) + } + StorageBackendConfig::Assetpack(config) => { + let mut entries = assetpack::list(&config, "blob", prefix).await?; + if let Some(start_after) = start_after { + entries.retain(|entry| entry.key > start_after); + } + if continuation_token.is_some() { + return Err(RuntimeError::invalid_input( + "StorageRuntime assetpack list continuation token is not supported", + )); + } + let max_keys = usize::try_from(max_keys) + .map_err(|_| RuntimeError::invalid_input("StorageRuntime list maxKeys must be positive"))?; + entries.truncate(max_keys); + Ok(object_storage::types::ObjectListPage { + entries, + next_continuation_token: None, + }) + } + StorageBackendConfig::S3(config) => config + .build_client()? + .list_page(prefix, continuation_token, start_after, max_keys) + .await + .map_err(Into::into), + } + } + + pub(crate) async fn object_storage_head(&self, key: String) -> Result> { + let metadata = match self.backend_for_scope("blob")? { + StorageBackendConfig::Fs(config) => fs_head(&config, &key)?, + StorageBackendConfig::Assetpack(config) => assetpack::head(&config, "blob", &key).await?, + StorageBackendConfig::S3(config) => config.build_client()?.head(&key).await?, + }; + Ok(metadata.map(Into::into)) + } + + pub(crate) async fn object_storage_delete(&self, key: String) -> Result<()> { + self.object_storage_delete_object(&key).await + } + + async fn complete_fs_workspace_blob( + &self, + config: FsStorageConfig, + workspace_id: String, + key: String, + expected_size: i64, + expected_mime: String, + ) -> Result { + if !(0..=MAX_BLOB_SIZE).contains(&expected_size) { + return Ok(blob_complete_failure("size_too_large")); + } + + let storage_key = format!("{workspace_id}/{key}"); + let object = match fs_get(&config, &storage_key)? { + Some(object) => object, + None => return Ok(blob_complete_failure("not_found")), + }; + let metadata = object.metadata; + + if !(0..=MAX_BLOB_SIZE).contains(&metadata.content_length) { + let _ = fs_delete(&config, &storage_key); + return Ok(blob_complete_failure("size_too_large")); + } + if metadata.content_length != expected_size { + return Ok(blob_complete_failure("size_mismatch")); + } + if !expected_mime.is_empty() && metadata.content_type != expected_mime { + return Ok(blob_complete_failure("mime_mismatch")); + } + if !sha256_base64_url_matches(&object.body, &key) { + let _ = fs_delete(&config, &storage_key); + return Ok(blob_complete_failure("checksum_mismatch")); + } + + upsert_completed_blob( + &self.pool().await?, + &workspace_id, + &key, + &metadata.content_type, + metadata.content_length, + ) + .await?; + Ok(blob_complete_success( + metadata.content_type, + metadata.content_length, + metadata.last_modified_ms, + )) + } + + async fn complete_assetpack_workspace_blob( + &self, + config: FsStorageConfig, + workspace_id: String, + key: String, + expected_size: i64, + expected_mime: String, + ) -> Result { + if !(0..=MAX_BLOB_SIZE).contains(&expected_size) { + return Ok(blob_complete_failure("size_too_large")); + } + + let storage_key = format!("{workspace_id}/{key}"); + let object = match assetpack::get(&config, "blob", &storage_key).await? { + Some(object) => object, + None => return Ok(blob_complete_failure("not_found")), + }; + let metadata = object.metadata; + + if !(0..=MAX_BLOB_SIZE).contains(&metadata.content_length) { + let _ = assetpack::delete(&config, "blob", &storage_key).await; + return Ok(blob_complete_failure("size_too_large")); + } + if metadata.content_length != expected_size { + return Ok(blob_complete_failure("size_mismatch")); + } + if !expected_mime.is_empty() && metadata.content_type != expected_mime { + return Ok(blob_complete_failure("mime_mismatch")); + } + if !sha256_base64_url_matches(&object.body, &key) { + let _ = assetpack::delete(&config, "blob", &storage_key).await; + return Ok(blob_complete_failure("checksum_mismatch")); + } + + upsert_completed_blob( + &self.pool().await?, + &workspace_id, + &key, + &metadata.content_type, + metadata.content_length, + ) + .await?; + Ok(blob_complete_success( + metadata.content_type, + metadata.content_length, + metadata.last_modified_ms, + )) + } + + async fn complete_s3_workspace_blob( + &self, + workspace_id: String, + key: String, + expected_size: i64, + expected_mime: String, + ) -> Result { + if !(0..=MAX_BLOB_SIZE).contains(&expected_size) { + return Ok(blob_complete_failure("size_too_large")); + } + + let object_key = format!("{workspace_id}/{key}"); + let config = match self.backend_for_scope("blob")? { + StorageBackendConfig::S3(config) => config, + _ => return Err(RuntimeError::invalid_state("BlobComplete expected S3 backend")), + }; + let client = config.build_client()?; + let object = match client.get(&object_key).await.map_err(RuntimeError::from) { + Ok(Some(object)) => object, + Ok(None) => return Ok(blob_complete_failure("not_found")), + Err(err) if err.is_object_missing() => return Ok(blob_complete_failure("not_found")), + Err(err) => return Err(err), + }; + let metadata = object.metadata; + + if !(0..=MAX_BLOB_SIZE).contains(&metadata.content_length) { + match client.delete(&object_key).await.map_err(RuntimeError::from) { + Ok(()) => {} + Err(err) if err.is_object_missing() => {} + Err(err) => return Err(err), + } + return Ok(blob_complete_failure("size_too_large")); + } + if metadata.content_length != expected_size { + return Ok(blob_complete_failure("size_mismatch")); + } + if !expected_mime.is_empty() && metadata.content_type != expected_mime { + return Ok(blob_complete_failure("mime_mismatch")); + } + if !sha256_base64_url_matches(&object.body, &key) { + match client.delete(&object_key).await.map_err(RuntimeError::from) { + Ok(()) => {} + Err(err) if err.is_object_missing() => {} + Err(err) => return Err(err), + } + return Ok(blob_complete_failure("checksum_mismatch")); + } + + upsert_completed_blob( + &self.pool().await?, + &workspace_id, + &key, + &metadata.content_type, + metadata.content_length, + ) + .await?; + Ok(blob_complete_success( + metadata.content_type, + metadata.content_length, + metadata.last_modified_ms, + )) + } + + async fn pool(&self) -> Result { + self + .pool + .lock() + .await + .as_ref() + .cloned() + .ok_or_else(|| RuntimeError::invalid_state("StorageRuntime must be started before using postgres operations")) + } +} + +fn database_url_from_env() -> Option { + env::var("DATABASE_URL").ok().and_then(non_empty_string) +} + +fn non_empty_string(value: String) -> Option { + if value.trim().is_empty() { None } else { Some(value) } +} + +fn app_config_from_config_files() -> Result { + let mut merged = AppConfigFile::default(); + for path in config_json_paths() { + if !path.exists() { + continue; + } + let raw = fs::read_to_string(&path).map_err(|err| RuntimeError::io("failed to read config file", err))?; + let config: AppConfigFile = + serde_json::from_str(&raw).map_err(|err| RuntimeError::json("failed to parse config file", err))?; + merged.apply_file_config(config); + } + + Ok(merged) +} + +async fn load_app_config_overrides_from_db(pool: &PgPool) -> Result { + let rows = match sqlx::query("SELECT id, value FROM app_configs").fetch_all(pool).await { + Ok(rows) => rows, + Err(sqlx::Error::Database(err)) if err.code().as_deref() == Some("42P01") => return Ok(AppConfigFile::default()), + Err(err) => return Err(RuntimeError::database("failed to load app config overrides", err)), + }; + + app_config_from_flat_overrides(rows.into_iter().map(|row| { + let id: String = row.get("id"); + let value: serde_json::Value = row.get("value"); + (id, value) + })) +} + +fn app_config_from_flat_overrides(rows: I) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let mut root = Map::new(); + for (path, value) in rows { + let Some((module, key)) = path.as_ref().split_once('.') else { + continue; + }; + root + .entry(module.to_string()) + .or_insert_with(|| serde_json::Value::Object(Map::new())); + if let Some(serde_json::Value::Object(module_object)) = root.get_mut(module) { + module_object.insert(key.to_string(), value); + } + } + + serde_json::from_value(serde_json::Value::Object(root)) + .map_err(|err| RuntimeError::json("invalid app config overrides", err)) +} + +fn config_json_paths() -> Vec { + let mut paths = Vec::new(); + if let Ok(exe) = env::current_exe() + && let Some(dir) = exe.parent() + { + paths.push(config_in(dir)); + } + if let Ok(cwd) = env::current_dir() { + paths.push(config_in(&cwd)); + } + dedupe_paths(paths) +} + +fn config_in(dir: &Path) -> PathBuf { + dir.join("config.json") +} + +fn dedupe_paths(paths: Vec) -> Vec { + let mut deduped = Vec::new(); + for path in paths { + if !deduped.contains(&path) { + deduped.push(path); + } + } + deduped +} + +fn fs_bucket_path(config: &FsStorageConfig) -> PathBuf { + if let Some(stripped) = config.root.strip_prefix("~/") + && let Ok(Some(home)) = homedir::my_home() + { + return home.join(stripped).join(&config.bucket); + } + Path::new(&config.root).join(&config.bucket) +} + +fn normalize_storage_key(key: &str) -> Result> { + let normalized = key.replace('\\', "/"); + let segments = normalized.split('/').map(ToString::to_string).collect::>(); + if normalized.is_empty() + || normalized.starts_with('/') + || segments + .iter() + .any(|segment| segment.is_empty() || segment == "." || segment == "..") + { + return Err(RuntimeError::invalid_input(format!("Invalid storage key: {key}"))); + } + Ok(segments) +} + +fn normalize_storage_prefix(prefix: &str) -> Result { + let normalized = prefix.replace('\\', "/"); + if normalized.is_empty() { + return Ok(normalized); + } + if normalized.starts_with('/') { + return Err(RuntimeError::invalid_input(format!("Invalid storage prefix: {prefix}"))); + } + + let mut segments = normalized.split('/').collect::>(); + let last_segment = segments.pop(); + if last_segment.is_none() + || segments + .iter() + .any(|segment| segment.is_empty() || *segment == "." || *segment == "..") + || matches!(last_segment, Some(".") | Some("..")) + { + return Err(RuntimeError::invalid_input(format!("Invalid storage prefix: {prefix}"))); + } + + if matches!(last_segment, Some("")) { + return Ok(format!("{}/", segments.join("/"))); + } + + Ok(normalized) +} + +fn fs_object_path(config: &FsStorageConfig, key: &str) -> Result { + let mut path = fs_bucket_path(config); + for segment in normalize_storage_key(key)? { + path.push(segment); + } + Ok(path) +} + +fn fs_put(config: &FsStorageConfig, key: &str, body: Vec, metadata: ObjectPutMetadata) -> Result { + let path = fs_object_path(config, key)?; + let metadata = metadata.complete_for_body(&body); + if let Some(content_length) = metadata.content_length + && content_length != body.len() as i64 + { + return Err(RuntimeError::invalid_input("StorageRuntime fs content length mismatch")); + } + if let Some(checksum) = metadata.checksum_crc32.as_deref() { + let actual = format!("{:x}", crc32fast::hash(&body)); + if actual != checksum { + return Err(RuntimeError::invalid_input("StorageRuntime fs checksum mismatch")); + } + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|err| RuntimeError::io("StorageRuntime fs create dir failed", err))?; + } + fs::write(&path, &body).map_err(|err| RuntimeError::io("StorageRuntime fs write object failed", err))?; + let object_metadata = metadata.into_object_metadata(system_time_ms(SystemTime::now())?); + let metadata_json = serde_json::json!({ + "contentType": &object_metadata.content_type, + "contentLength": object_metadata.content_length, + "lastModified": object_metadata.last_modified_ms, + "checksumCRC32": &object_metadata.checksum_crc32, + }); + fs::write( + PathBuf::from(format!("{}.metadata.json", path.display())), + serde_json::to_vec(&metadata_json) + .map_err(|err| RuntimeError::json("StorageRuntime fs serialize metadata failed", err))?, + ) + .map_err(|err| RuntimeError::io("StorageRuntime fs write metadata failed", err))?; + Ok(object_metadata) +} + +fn fs_head(config: &FsStorageConfig, key: &str) -> Result> { + let path = fs_object_path(config, key)?; + read_fs_metadata(&path) +} + +fn fs_get(config: &FsStorageConfig, key: &str) -> Result> { + let path = fs_object_path(config, key)?; + let Some(metadata) = read_fs_metadata(&path)? else { + return Ok(None); + }; + let body = match fs::read(&path) { + Ok(body) => body, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(RuntimeError::io("StorageRuntime fs read object failed", err)), + }; + Ok(Some(ObjectGetResult { body, metadata })) +} + +fn fs_list(config: &FsStorageConfig, prefix: Option) -> Result> { + let root = fs_bucket_path(config); + let prefix = prefix.map(|prefix| normalize_storage_prefix(&prefix)).transpose()?; + let mut dir = root.clone(); + let mut name_prefix = prefix.as_deref(); + if let Some(prefix) = name_prefix + && !prefix.is_empty() + { + let parts = prefix.split('/').collect::>(); + if parts.len() > 1 { + for part in &parts[..parts.len() - 1] { + dir.push(part); + } + name_prefix = parts.last().copied(); + } + } + + let mut entries = Vec::new(); + collect_fs_entries(&root, &dir, name_prefix, &mut entries)?; + entries.sort_by(|a, b| a.key.cmp(&b.key)); + Ok(entries) +} + +fn collect_fs_entries( + root: &Path, + dir: &Path, + name_prefix: Option<&str>, + entries: &mut Vec, +) -> Result<()> { + let read_dir = match fs::read_dir(dir) { + Ok(read_dir) => read_dir, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(RuntimeError::io("StorageRuntime fs list failed", err)), + }; + + for entry in read_dir { + let entry = entry.map_err(|err| RuntimeError::io("StorageRuntime fs list entry failed", err))?; + let path = entry.path(); + let name = entry.file_name().to_string_lossy().to_string(); + if path.is_dir() { + if name_prefix.is_none_or(|prefix| name.starts_with(prefix)) { + collect_fs_entries(root, &path, None, entries)?; + } + } else if !name.ends_with(".metadata.json") && name_prefix.is_none_or(|prefix| name.starts_with(prefix)) { + let stat = entry + .metadata() + .map_err(|err| RuntimeError::io("StorageRuntime fs metadata failed", err))?; + let key = path + .strip_prefix(root) + .map_err(|err| RuntimeError::invalid_state(format!("StorageRuntime fs path trim failed: {err}")))? + .to_string_lossy() + .replace('\\', "/"); + entries.push(ObjectListEntry { + key, + content_length: stat.len() as i64, + last_modified_ms: stat + .modified() + .ok() + .and_then(|time| system_time_ms(time).ok()) + .unwrap_or(0), + }); + } + } + Ok(()) +} + +fn fs_delete(config: &FsStorageConfig, key: &str) -> Result<()> { + let path = fs_object_path(config, key)?; + match fs::remove_file(&path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(RuntimeError::io("StorageRuntime fs delete object failed", err)), + } + match fs::remove_file(PathBuf::from(format!("{}.metadata.json", path.display()))) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(RuntimeError::io("StorageRuntime fs delete metadata failed", err)), + } + Ok(()) +} + +fn read_fs_metadata(path: &Path) -> Result> { + let raw = match fs::read_to_string(PathBuf::from(format!("{}.metadata.json", path.display()))) { + Ok(raw) => raw, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(RuntimeError::io("StorageRuntime fs read metadata failed", err)), + }; + let metadata: FsBlobMetadata = + serde_json::from_str(&raw).map_err(|err| RuntimeError::json("StorageRuntime fs parse metadata failed", err))?; + Ok(Some(ObjectMetadata { + content_type: metadata.content_type, + content_length: metadata.content_length, + last_modified_ms: metadata.last_modified, + checksum_crc32: metadata.checksum_crc32, + })) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct FsBlobMetadata { + content_type: String, + content_length: i64, + last_modified: i64, + #[serde(rename = "checksumCRC32")] + checksum_crc32: Option, +} + +async fn upsert_completed_blob(pool: &PgPool, workspace_id: &str, key: &str, mime: &str, size: i64) -> Result<()> { + if !(0..=MAX_BLOB_SIZE).contains(&size) { + return Err(RuntimeError::invalid_input("BlobComplete size exceeds limit")); + } + let size = i32::try_from(size).map_err(|_| RuntimeError::invalid_input("BlobComplete size exceeds limit"))?; + + sqlx::query( + r#" + INSERT INTO blobs (workspace_id, key, mime, size, status, upload_id) + VALUES ($1, $2, $3, $4, 'completed', NULL) + ON CONFLICT (workspace_id, key) + DO UPDATE SET + mime = EXCLUDED.mime, + size = EXCLUDED.size, + status = EXCLUDED.status, + upload_id = NULL + "#, + ) + .bind(workspace_id) + .bind(key) + .bind(mime) + .bind(size) + .execute(pool) + .await + .map_err(|err| RuntimeError::database("BlobComplete upsert metadata failed", err))?; + + Ok(()) +} + +fn blob_complete_failure(reason: &str) -> RuntimeBlobCompleteResult { + RuntimeBlobCompleteResult { + ok: false, + reason: Some(reason.to_string()), + content_type: None, + content_length: None, + last_modified_ms: None, + } +} + +fn blob_complete_success( + content_type: String, + content_length: i64, + last_modified_ms: i64, +) -> RuntimeBlobCompleteResult { + RuntimeBlobCompleteResult { + ok: true, + reason: None, + content_type: Some(content_type), + content_length: Some(content_length), + last_modified_ms: Some(last_modified_ms), + } +} + +fn sha256_base64_url(body: &[u8]) -> String { + URL_SAFE_NO_PAD.encode(Sha256::digest(body)) +} + +fn sha256_base64_url_matches(body: &[u8], key: &str) -> bool { + sha256_base64_url(body) == key.trim_end_matches('=') +} + +fn system_time_ms(time: SystemTime) -> Result { + crate::utils::system_time_millis(time) + .map(|millis| millis as i64) + .map_err(|err| RuntimeError::Time { + context: "system time before unix epoch".to_string(), + source: err, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fs_key_normalization_rejects_traversal() { + for key in ["", "/a", "a//b", "a/./b", "a/../b", "..\\secret"] { + assert!(normalize_storage_key(key).is_err(), "{key}"); + } + assert_eq!(normalize_storage_key("workspace/blob").unwrap(), ["workspace", "blob"]); + assert_eq!(normalize_storage_prefix("workspace/").unwrap(), "workspace/"); + } + + #[test] + fn capabilities_are_explicit_for_server_mediated_provider() { + let capabilities = StorageBackendConfig::Fs(FsStorageConfig { + provider: "fs".to_string(), + root: "/tmp".to_string(), + bucket: "blob".to_string(), + }) + .capabilities(); + assert!(capabilities.put); + assert!(!capabilities.presign_put); + assert!(capabilities.server_mediated_only); + } + + #[test] + fn capabilities_enable_presign_get_for_presigned_s3_provider() { + let capabilities = StorageBackendConfig::S3(ObjectStorageConfig { + provider: "cloudflare-r2".to_string(), + bucket: "blob".to_string(), + endpoint: Some("https://account.r2.cloudflarestorage.com".to_string()), + region: Some("auto".to_string()), + access_key_id: Some("key".to_string()), + secret_access_key: Some("secret".to_string()), + session_token: None, + force_path_style: true, + request_timeout_ms: None, + min_part_size: None, + presign_expires_in_seconds: Some(60), + presign_sign_content_type_for_put: Some(true), + use_presigned_url: true, + proxy_upload: false, + }) + .capabilities(); + + assert!(capabilities.presign_put); + assert!(capabilities.presign_get); + assert!(capabilities.multipart_direct); + assert!(!capabilities.server_mediated_only); + } + + #[test] + fn capabilities_expose_r2_proxy_upload() { + let capabilities = StorageBackendConfig::S3(ObjectStorageConfig { + provider: "cloudflare-r2".to_string(), + bucket: "blob".to_string(), + endpoint: Some("https://account.r2.cloudflarestorage.com".to_string()), + region: Some("auto".to_string()), + access_key_id: Some("key".to_string()), + secret_access_key: Some("secret".to_string()), + session_token: None, + force_path_style: true, + request_timeout_ms: None, + min_part_size: None, + presign_expires_in_seconds: Some(60), + presign_sign_content_type_for_put: Some(true), + use_presigned_url: true, + proxy_upload: true, + }) + .capabilities(); + + assert!(capabilities.proxy_upload); + assert!(capabilities.presign_put); + assert!(capabilities.multipart_direct); + } + + #[test] + fn capabilities_are_explicit_for_assetpack_provider() { + let capabilities = StorageBackendConfig::Assetpack(FsStorageConfig { + provider: "assetpack".to_string(), + root: "/tmp".to_string(), + bucket: "blob".to_string(), + }) + .capabilities(); + + assert!(capabilities.put); + assert!(capabilities.get); + assert!(capabilities.assetpack); + assert!(!capabilities.presign_put); + assert!(!capabilities.multipart_direct); + assert!(capabilities.server_mediated_only); + } + + #[test] + fn assetpack_transform_specs_are_registered() { + let specs = assetpack_transform_precomp2::default_specs(); + let ids = specs.iter().map(|spec| spec.id).collect::>(); + + assert!(ids.contains(&assetpack_core::TRANSFORM_ID_PRECOMP2)); + assert!(ids.contains(&assetpack_core::TRANSFORM_ID_PRECOMP2_ZSTD)); + assert!(ids.contains(&assetpack_core::TRANSFORM_ID_PRECOMP2_LZMA)); + } + + #[test] + fn fs_backend_preserves_sidecar_metadata_format() { + let temp = tempfile::tempdir().unwrap(); + let config = FsStorageConfig { + provider: "fs".to_string(), + root: temp.path().to_string_lossy().to_string(), + bucket: "bucket".to_string(), + }; + let body = b"hello".to_vec(); + let checksum = format!("{:x}", crc32fast::hash(&body)); + + fs_put( + &config, + "workspace/blob", + body.clone(), + ObjectPutMetadata { + content_type: Some("text/plain".to_string()), + content_length: Some(body.len() as i64), + checksum_crc32: Some(checksum.clone()), + }, + ) + .unwrap(); + + let object_path = temp.path().join("bucket/workspace/blob"); + assert_eq!(fs::read(&object_path).unwrap(), body); + let sidecar: serde_json::Value = + serde_json::from_slice(&fs::read(temp.path().join("bucket/workspace/blob.metadata.json")).unwrap()).unwrap(); + assert_eq!(sidecar["contentType"], "text/plain"); + assert_eq!(sidecar["contentLength"], 5); + assert_eq!(sidecar["checksumCRC32"], checksum); + assert!(sidecar["lastModified"].as_i64().unwrap() > 0); + + let metadata = fs_head(&config, "workspace/blob").unwrap().unwrap(); + assert_eq!(metadata.content_type, "text/plain"); + assert_eq!(metadata.content_length, 5); + assert_eq!(metadata.checksum_crc32.as_deref(), Some(checksum.as_str())); + assert_eq!(fs_get(&config, "workspace/blob").unwrap().unwrap().body, body); + } + + #[test] + fn fs_backend_reads_existing_node_sidecar_and_lists_prefixes() { + let temp = tempfile::tempdir().unwrap(); + let config = FsStorageConfig { + provider: "fs".to_string(), + root: temp.path().to_string_lossy().to_string(), + bucket: "bucket".to_string(), + }; + let dir = temp.path().join("bucket/workspace"); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join("blob-a"), b"a").unwrap(); + fs::write( + dir.join("blob-a.metadata.json"), + r#"{"contentType":"text/plain","contentLength":1,"lastModified":123,"checksumCRC32":"e8b7be43"}"#, + ) + .unwrap(); + fs::create_dir_all(dir.join("nested")).unwrap(); + fs::write(dir.join("nested/blob-b"), b"b").unwrap(); + fs::write( + dir.join("nested/blob-b.metadata.json"), + r#"{"contentType":"text/plain","contentLength":1,"lastModified":124}"#, + ) + .unwrap(); + + let metadata = fs_head(&config, "workspace/blob-a").unwrap().unwrap(); + assert_eq!(metadata.last_modified_ms, 123); + assert_eq!(metadata.checksum_crc32.as_deref(), Some("e8b7be43")); + + let keys = fs_list(&config, Some("workspace/".to_string())) + .unwrap() + .into_iter() + .map(|entry| entry.key) + .collect::>(); + assert_eq!(keys, ["workspace/blob-a", "workspace/nested/blob-b"]); + } + + #[test] + fn fs_backend_rejects_metadata_mismatch() { + let temp = tempfile::tempdir().unwrap(); + let config = FsStorageConfig { + provider: "fs".to_string(), + root: temp.path().to_string_lossy().to_string(), + bucket: "bucket".to_string(), + }; + + assert!( + fs_put( + &config, + "workspace/blob", + b"hello".to_vec(), + ObjectPutMetadata { + content_type: None, + content_length: Some(10), + checksum_crc32: None, + }, + ) + .is_err() + ); + assert!( + fs_put( + &config, + "workspace/blob", + b"hello".to_vec(), + ObjectPutMetadata { + content_type: None, + content_length: None, + checksum_crc32: Some("wrong".to_string()), + }, + ) + .is_err() + ); + } + + #[tokio::test] + async fn assetpack_backend_roundtrips_manifest_and_body_in_assetpack_sqlite() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let config = FsStorageConfig { + provider: "assetpack".to_string(), + root: temp.path().to_string_lossy().to_string(), + bucket: "bucket".to_string(), + }; + let scope = format!("test_{}", uuid::Uuid::new_v4().simple()); + let key = "workspace/blob.txt"; + let body = b"assetpack body".repeat(512); + + assetpack::put( + &config, + &scope, + key, + body.clone(), + ObjectPutMetadata { + content_type: Some("text/plain".to_string()), + content_length: Some(body.len() as i64), + checksum_crc32: Some(format!("{:x}", crc32fast::hash(&body))), + }, + ) + .await?; + + let head = assetpack::head(&config, &scope, key).await?.unwrap(); + assert_eq!(head.content_type, "text/plain"); + assert_eq!(head.content_length, body.len() as i64); + + let object = assetpack::get(&config, &scope, key).await?.unwrap(); + assert_eq!(object.body, body); + assert_eq!( + assetpack::list(&config, &scope, Some("workspace/".to_string())) + .await? + .len(), + 1 + ); + + assetpack::delete(&config, &scope, key).await?; + assert!(assetpack::head(&config, &scope, key).await?.is_none()); + Ok(()) + } +} diff --git a/packages/backend/native/src/runtime/storage_runtime/object_storage/client.rs b/packages/backend/native/src/runtime/storage_runtime/object_storage/client.rs new file mode 100644 index 0000000000..48a051e8c4 --- /dev/null +++ b/packages/backend/native/src/runtime/storage_runtime/object_storage/client.rs @@ -0,0 +1,679 @@ +use std::{ + collections::HashMap, + future::Future, + pin::Pin, + time::{Duration, SystemTime}, +}; + +use chrono::{DateTime, FixedOffset}; +use reqwest::{ + Client as ReqwestClient, Method, StatusCode, + header::{CONTENT_LENGTH, CONTENT_TYPE, ETAG, HeaderMap, HeaderName, HeaderValue, LAST_MODIFIED}, +}; +use rusty_s3::{ + Bucket, Credentials, + actions::{ + AbortMultipartUpload, CompleteMultipartUpload, CreateMultipartUpload, DeleteObject, GetObject, HeadObject, + ListObjectsV2, ListParts, PutObject, S3Action, UploadPart, + }, +}; +use url::Url; + +use super::{ + error::{ObjectStorageError, ObjectStorageResult}, + types::{ + MultipartUploadInitResult, MultipartUploadPart, ObjectGetResult, ObjectListEntry, ObjectListPage, ObjectMetadata, + ObjectPutMetadata, PresignedObjectRequest, completed_multipart_parts, trim_etag, + }, +}; + +const MAX_RESPONSE_BODY_BYTES: usize = i32::MAX as usize; + +type StorageHttpFuture<'a> = Pin> + Send + 'a>>; + +#[derive(Clone)] +struct StorageHttpRequest { + method: Method, + url: Url, + headers: HashMap, + body: Option>, + max_response_body_bytes: usize, +} + +struct StorageHttpResponse { + status: StatusCode, + headers: HeaderMap, + body: Vec, +} + +trait StorageHttpClient: Clone + Send + Sync + 'static { + fn execute(&self, request: StorageHttpRequest) -> StorageHttpFuture<'_>; +} + +#[derive(Clone)] +struct ReqwestStorageHttpClient { + client: ReqwestClient, +} + +impl ReqwestStorageHttpClient { + fn new(request_timeout_ms: Option) -> ObjectStorageResult { + let mut builder = ReqwestClient::builder(); + if let Some(request_timeout_ms) = request_timeout_ms { + builder = builder.timeout(Duration::from_millis(request_timeout_ms)); + } + Ok(Self { + client: builder.build().map_err(ObjectStorageError::HttpClientBuild)?, + }) + } +} + +impl StorageHttpClient for ReqwestStorageHttpClient { + fn execute(&self, request: StorageHttpRequest) -> StorageHttpFuture<'_> { + Box::pin(async move { + let mut builder = self.client.request(request.method, request.url); + for (key, value) in request.headers { + let name = + HeaderName::from_bytes(key.as_bytes()).map_err(|err| ObjectStorageError::InvalidHeader(err.to_string()))?; + let value = HeaderValue::from_str(&value).map_err(|err| ObjectStorageError::InvalidHeader(err.to_string()))?; + builder = builder.header(name, value); + } + if let Some(body) = request.body { + builder = builder.body(body); + } + let mut response = builder.send().await.map_err(ObjectStorageError::HttpRequest)?; + let status = response.status(); + let headers = response.headers().clone(); + if response + .content_length() + .is_some_and(|length| length > request.max_response_body_bytes as u64) + { + return Err(ObjectStorageError::BodyTooLarge { + limit: request.max_response_body_bytes, + }); + } + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(ObjectStorageError::HttpRequest)? { + if body.len() + chunk.len() > request.max_response_body_bytes { + return Err(ObjectStorageError::BodyTooLarge { + limit: request.max_response_body_bytes, + }); + } + body.extend_from_slice(&chunk); + } + Ok(StorageHttpResponse { status, headers, body }) + }) + } +} + +#[derive(Clone)] +pub(crate) struct ObjectStorageClient { + bucket: Bucket, + credentials: Credentials, + http: ReqwestStorageHttpClient, + presign_expires_in_seconds: u64, + presign_sign_content_type_for_put: bool, +} + +impl ObjectStorageClient { + pub(crate) fn new( + bucket: Bucket, + credentials: Credentials, + request_timeout_ms: Option, + presign_expires_in_seconds: u64, + presign_sign_content_type_for_put: bool, + ) -> ObjectStorageResult { + Ok(Self { + bucket, + credentials, + http: ReqwestStorageHttpClient::new(request_timeout_ms)?, + presign_expires_in_seconds, + presign_sign_content_type_for_put, + }) + } + + pub(crate) async fn put( + &self, + key: &str, + body: Vec, + metadata: ObjectPutMetadata, + ) -> ObjectStorageResult { + let metadata = metadata.complete_for_body(&body); + let object_metadata = metadata.clone().into_object_metadata( + crate::utils::system_time_millis(SystemTime::now()) + .map(|millis| millis as i64) + .map_err(|err| ObjectStorageError::InvalidInput(format!("system time before unix epoch: {err}")))?, + ); + let mut headers = HashMap::from([ + ("content-type".to_string(), object_metadata.content_type.clone()), + ("content-length".to_string(), object_metadata.content_length.to_string()), + ]); + if let Some(checksum) = object_metadata.checksum_crc32.clone() { + headers.insert("x-amz-checksum-crc32".to_string(), checksum); + } + + let mut action = PutObject::new(&self.bucket, Some(&self.credentials), key); + insert_action_headers(&mut action, &headers); + let response = self + .http + .execute(StorageHttpRequest { + method: Method::PUT, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers, + body: Some(body), + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| operation_error(format!("ObjectStorage put failed for {key}"), source))?; + ensure_success_status(&response, &format!("ObjectStorage put failed for {key}"))?; + Ok(object_metadata) + } + + pub(crate) async fn presign_put( + &self, + key: &str, + metadata: ObjectPutMetadata, + ) -> ObjectStorageResult { + let content_type = metadata + .content_type + .unwrap_or_else(|| "application/octet-stream".to_string()); + let mut headers = HashMap::new(); + headers.insert("Content-Type".to_string(), content_type.clone()); + if let Some(content_length) = metadata.content_length { + headers.insert("Content-Length".to_string(), content_length.to_string()); + } + + let mut action = PutObject::new(&self.bucket, Some(&self.credentials), key); + if self.presign_sign_content_type_for_put { + action.headers_mut().insert("content-type", content_type); + } + if let Some(content_length) = metadata.content_length { + action + .headers_mut() + .insert("content-length", content_length.to_string()); + } + + Ok(PresignedObjectRequest { + url: action.sign(expires_in(self.presign_expires_in_seconds)).to_string(), + headers, + expires_at_ms: expires_at_ms(self.presign_expires_in_seconds)?, + }) + } + + pub(crate) async fn presign_get(&self, key: &str) -> ObjectStorageResult { + let action = GetObject::new(&self.bucket, Some(&self.credentials), key); + Ok(PresignedObjectRequest { + url: action.sign(expires_in(self.presign_expires_in_seconds)).to_string(), + headers: HashMap::new(), + expires_at_ms: expires_at_ms(self.presign_expires_in_seconds)?, + }) + } + + pub(crate) async fn create_multipart_upload( + &self, + key: &str, + metadata: ObjectPutMetadata, + ) -> ObjectStorageResult> { + let mut action = CreateMultipartUpload::new(&self.bucket, Some(&self.credentials), key); + if let Some(content_type) = metadata.content_type { + action.headers_mut().insert("content-type", content_type.clone()); + let headers = HashMap::from([("content-type".to_string(), content_type)]); + return self.create_multipart_upload_with_headers(key, action, headers).await; + } + self + .create_multipart_upload_with_headers(key, action, HashMap::new()) + .await + } + + async fn create_multipart_upload_with_headers( + &self, + key: &str, + action: CreateMultipartUpload<'_>, + headers: HashMap, + ) -> ObjectStorageResult> { + let response = self + .http + .execute(StorageHttpRequest { + method: Method::POST, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers, + body: None, + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| { + operation_error( + format!("ObjectStorage create multipart upload failed for {key}"), + source, + ) + })?; + let body = ensure_success_text( + response, + format!("ObjectStorage create multipart upload failed for {key}"), + )?; + let parsed = CreateMultipartUpload::parse_response(&body).map_err(|source| ObjectStorageError::InvalidXml { + context: format!("ObjectStorage parse multipart upload response failed for {key}"), + source, + })?; + Ok(Some(MultipartUploadInitResult { + upload_id: parsed.upload_id().to_string(), + expires_at_ms: expires_at_ms(self.presign_expires_in_seconds)?, + })) + } + + pub(crate) async fn presign_upload_part( + &self, + key: &str, + upload_id: &str, + part_number: i32, + ) -> ObjectStorageResult { + let part_number = checked_part_number(part_number)?; + let action = UploadPart::new(&self.bucket, Some(&self.credentials), key, part_number, upload_id); + Ok(PresignedObjectRequest { + url: action.sign(expires_in(self.presign_expires_in_seconds)).to_string(), + headers: HashMap::new(), + expires_at_ms: expires_at_ms(self.presign_expires_in_seconds)?, + }) + } + + pub(crate) async fn upload_part( + &self, + key: &str, + upload_id: &str, + part_number: i32, + body: Vec, + content_length: Option, + ) -> ObjectStorageResult> { + let part_number = checked_part_number(part_number)?; + let mut action = UploadPart::new(&self.bucket, Some(&self.credentials), key, part_number, upload_id); + let mut headers = HashMap::new(); + if let Some(content_length) = content_length { + action + .headers_mut() + .insert("content-length", content_length.to_string()); + headers.insert("content-length".to_string(), content_length.to_string()); + } + let response = self + .http + .execute(StorageHttpRequest { + method: Method::PUT, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers, + body: Some(body), + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| operation_error(format!("ObjectStorage upload multipart part failed for {key}"), source))?; + ensure_success_status( + &response, + &format!("ObjectStorage upload multipart part failed for {key}"), + )?; + Ok(response_header(&response.headers, ETAG).as_deref().map(trim_etag)) + } + + pub(crate) async fn list_multipart_upload_parts( + &self, + key: &str, + upload_id: &str, + ) -> ObjectStorageResult> { + let mut parts = Vec::new(); + let mut marker = None; + loop { + let mut action = ListParts::new(&self.bucket, Some(&self.credentials), key, upload_id); + if let Some(marker) = marker { + action.set_part_number_marker(marker); + } + let response = self + .http + .execute(StorageHttpRequest { + method: Method::GET, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers: HashMap::new(), + body: None, + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| { + operation_error( + format!("ObjectStorage list multipart upload parts failed for {key}"), + source, + ) + })?; + if response.status == StatusCode::NOT_FOUND && is_not_found_body(&response.body) { + return Ok(Vec::new()); + } + let body = ensure_success_text( + response, + format!("ObjectStorage list multipart upload parts failed for {key}"), + )?; + let parsed = ListParts::parse_response(&body).map_err(|source| ObjectStorageError::InvalidXml { + context: format!("ObjectStorage parse multipart parts failed for {key}"), + source, + })?; + parts.extend(parsed.parts.into_iter().map(|part| MultipartUploadPart { + part_number: i32::from(part.number), + etag: trim_etag(&part.etag), + })); + let Some(next_marker) = parsed.next_part_number_marker else { + break; + }; + marker = Some(next_marker); + } + Ok(parts) + } + + pub(crate) async fn complete_multipart_upload( + &self, + key: &str, + upload_id: &str, + parts: Vec, + ) -> ObjectStorageResult<()> { + let ordered_parts = completed_multipart_parts(parts); + validate_completed_parts(&ordered_parts)?; + let etags = ordered_parts.iter().map(|part| part.etag.as_str()); + let action = CompleteMultipartUpload::new(&self.bucket, Some(&self.credentials), key, upload_id, etags); + let url = action.sign(expires_in(self.presign_expires_in_seconds)); + let body = complete_multipart_body(&ordered_parts); + let response = self + .http + .execute(StorageHttpRequest { + method: Method::POST, + url, + headers: HashMap::from([("content-type".to_string(), "application/xml".to_string())]), + body: Some(body.into_bytes()), + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| { + operation_error( + format!("ObjectStorage complete multipart upload failed for {key}"), + source, + ) + })?; + ensure_success_status( + &response, + &format!("ObjectStorage complete multipart upload failed for {key}"), + )?; + Ok(()) + } + + pub(crate) async fn abort_multipart_upload(&self, key: &str, upload_id: &str) -> ObjectStorageResult<()> { + let action = AbortMultipartUpload::new(&self.bucket, Some(&self.credentials), key, upload_id); + let response = self + .http + .execute(StorageHttpRequest { + method: Method::DELETE, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers: HashMap::new(), + body: None, + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| operation_error(format!("ObjectStorage abort multipart upload failed for {key}"), source))?; + ensure_success_status( + &response, + &format!("ObjectStorage abort multipart upload failed for {key}"), + )?; + Ok(()) + } + + pub(crate) async fn head(&self, key: &str) -> ObjectStorageResult> { + let action = HeadObject::new(&self.bucket, Some(&self.credentials), key); + let response = self + .http + .execute(StorageHttpRequest { + method: Method::HEAD, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers: HashMap::new(), + body: None, + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| operation_error(format!("ObjectStorage head failed for {key}"), source))?; + if response.status == StatusCode::NOT_FOUND { + let get_action = GetObject::new(&self.bucket, Some(&self.credentials), key); + let get_response = self + .http + .execute(StorageHttpRequest { + method: Method::GET, + url: get_action.sign(expires_in(self.presign_expires_in_seconds)), + headers: HashMap::new(), + body: None, + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| operation_error(format!("ObjectStorage head missing check failed for {key}"), source))?; + if get_response.status == StatusCode::NOT_FOUND && is_not_found_body(&get_response.body) { + return Ok(None); + } + ensure_success_status( + &get_response, + &format!("ObjectStorage head missing check failed for {key}"), + )?; + return Ok(Some(metadata_from_headers(&get_response.headers))); + } + ensure_success_status(&response, &format!("ObjectStorage head failed for {key}"))?; + Ok(Some(metadata_from_headers(&response.headers))) + } + + pub(crate) async fn get(&self, key: &str) -> ObjectStorageResult> { + let action = GetObject::new(&self.bucket, Some(&self.credentials), key); + let response = self + .http + .execute(StorageHttpRequest { + method: Method::GET, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers: HashMap::new(), + body: None, + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| operation_error(format!("ObjectStorage get failed for {key}"), source))?; + if response.status == StatusCode::NOT_FOUND && is_not_found_body(&response.body) { + return Ok(None); + } + ensure_success_status(&response, &format!("ObjectStorage get failed for {key}"))?; + let metadata = metadata_from_headers(&response.headers); + Ok(Some(ObjectGetResult { + body: response.body, + metadata, + })) + } + + pub(crate) async fn list(&self, prefix: Option) -> ObjectStorageResult> { + let mut entries = Vec::new(); + let mut token = None; + loop { + let page = self.list_page(prefix.clone(), token, None, 1000).await?; + entries.extend(page.entries); + if let Some(next_token) = page.next_continuation_token { + token = Some(next_token); + } else { + break; + } + } + Ok(entries) + } + + pub(crate) async fn list_page( + &self, + prefix: Option, + continuation_token: Option, + start_after: Option, + max_keys: i32, + ) -> ObjectStorageResult { + let max_keys = usize::try_from(max_keys) + .map_err(|_| ObjectStorageError::InvalidInput("maxKeys must be positive".to_string()))?; + let mut action = ListObjectsV2::new(&self.bucket, Some(&self.credentials)); + action.with_max_keys(max_keys); + if let Some(prefix) = &prefix { + action.with_prefix(prefix.clone()); + } + if let Some(continuation_token) = &continuation_token { + action.with_continuation_token(continuation_token.clone()); + } else if let Some(start_after) = &start_after { + action.with_start_after(start_after.clone()); + } + let response = self + .http + .execute(StorageHttpRequest { + method: Method::GET, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers: HashMap::new(), + body: None, + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| operation_error("ObjectStorage list page failed", source))?; + let body = ensure_success_text(response, "ObjectStorage list page failed".to_string())?; + let parsed = ListObjectsV2::parse_response(&body).map_err(|source| ObjectStorageError::InvalidXml { + context: "ObjectStorage parse list response failed".to_string(), + source, + })?; + Ok(ObjectListPage { + entries: parsed + .contents + .into_iter() + .map(|object| ObjectListEntry { + key: object.key, + content_length: i64::try_from(object.size).unwrap_or(i64::MAX), + last_modified_ms: parse_rfc3339_ms(&object.last_modified), + }) + .collect(), + next_continuation_token: parsed.next_continuation_token, + }) + } + + pub(crate) async fn delete(&self, key: &str) -> ObjectStorageResult<()> { + let action = DeleteObject::new(&self.bucket, Some(&self.credentials), key); + let response = self + .http + .execute(StorageHttpRequest { + method: Method::DELETE, + url: action.sign(expires_in(self.presign_expires_in_seconds)), + headers: HashMap::new(), + body: None, + max_response_body_bytes: MAX_RESPONSE_BODY_BYTES, + }) + .await + .map_err(|source| operation_error(format!("ObjectStorage delete failed for {key}"), source))?; + ensure_success_status(&response, &format!("ObjectStorage delete failed for {key}"))?; + Ok(()) + } +} + +fn insert_action_headers<'a, T: S3Action<'a>>(action: &mut T, headers: &HashMap) { + for (key, value) in headers { + action.headers_mut().insert(key.clone(), value.clone()); + } +} + +fn operation_error(context: impl Into, source: ObjectStorageError) -> ObjectStorageError { + ObjectStorageError::Operation { + context: context.into(), + source: Box::new(source), + } +} + +fn ensure_success_text(response: StorageHttpResponse, context: String) -> ObjectStorageResult { + ensure_success_status(&response, &context)?; + String::from_utf8(response.body).map_err(|source| ObjectStorageError::InvalidUtf8 { context, source }) +} + +fn ensure_success_status(response: &StorageHttpResponse, context: &str) -> ObjectStorageResult<()> { + if response.status.is_success() { + return Ok(()); + } + let body = String::from_utf8_lossy(&response.body); + Err(ObjectStorageError::HttpStatus { + context: context.to_string(), + status: response.status, + body: body.to_string(), + }) +} + +fn is_not_found_body(body: &[u8]) -> bool { + let body = String::from_utf8_lossy(body); + body.contains("NoSuchKey") + || body.contains("NotFound") + || body.contains("NoSuchUpload") +} + +fn metadata_from_headers(headers: &HeaderMap) -> ObjectMetadata { + ObjectMetadata { + content_type: response_header(headers, CONTENT_TYPE).unwrap_or_else(|| "application/octet-stream".to_string()), + content_length: response_header(headers, CONTENT_LENGTH) + .and_then(|value| value.parse::().ok()) + .unwrap_or(0), + last_modified_ms: response_header(headers, LAST_MODIFIED) + .and_then(|value| DateTime::::parse_from_rfc2822(&value).ok()) + .map(|value| value.timestamp_millis()) + .unwrap_or(0), + checksum_crc32: response_header_name(headers, "x-amz-checksum-crc32"), + } +} + +fn response_header(headers: &HeaderMap, name: HeaderName) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(ToString::to_string) +} + +fn response_header_name(headers: &HeaderMap, name: &str) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(ToString::to_string) +} + +fn checked_part_number(part_number: i32) -> ObjectStorageResult { + u16::try_from(part_number).map_err(|_| ObjectStorageError::InvalidInput("part number must fit u16".to_string())) +} + +fn validate_completed_parts(parts: &[MultipartUploadPart]) -> ObjectStorageResult<()> { + for part in parts { + checked_part_number(part.part_number)?; + if part.etag.is_empty() { + return Err(ObjectStorageError::InvalidInput( + "multipart part etag is required".to_string(), + )); + } + } + Ok(()) +} + +fn complete_multipart_body(parts: &[MultipartUploadPart]) -> String { + let mut body = String::from(""); + for part in parts { + body.push_str(""); + body.push_str(&xml_escape(&part.etag)); + body.push_str(""); + body.push_str(&part.part_number.to_string()); + body.push_str(""); + } + body.push_str(""); + body +} + +fn xml_escape(value: &str) -> String { + value.replace('&', "&").replace('<', "<").replace('>', ">") +} + +fn expires_in(seconds: u64) -> Duration { + Duration::from_secs(seconds) +} + +fn expires_at_ms(expires_in_seconds: u64) -> ObjectStorageResult { + let expires_at = SystemTime::now() + .checked_add(Duration::from_secs(expires_in_seconds)) + .ok_or_else(|| ObjectStorageError::InvalidInput("presign expiration overflow".to_string()))?; + crate::utils::system_time_millis(expires_at) + .map(|millis| millis as i64) + .map_err(|err| ObjectStorageError::InvalidInput(format!("system time before unix epoch: {err}"))) +} + +fn parse_rfc3339_ms(value: &str) -> i64 { + DateTime::parse_from_rfc3339(value) + .map(|value| value.timestamp_millis()) + .unwrap_or(0) +} diff --git a/packages/backend/native/src/backend_runtime/object_storage/config.rs b/packages/backend/native/src/runtime/storage_runtime/object_storage/config.rs similarity index 51% rename from packages/backend/native/src/backend_runtime/object_storage/config.rs rename to packages/backend/native/src/runtime/storage_runtime/object_storage/config.rs index ae65328a82..3cd44923ef 100644 --- a/packages/backend/native/src/backend_runtime/object_storage/config.rs +++ b/packages/backend/native/src/runtime/storage_runtime/object_storage/config.rs @@ -1,27 +1,29 @@ -use aws_sdk_s3::config::{ - BehaviorVersion, Credentials, Region, RequestChecksumCalculation, ResponseChecksumValidation, timeout::TimeoutConfig, -}; -use napi::Result; +use rusty_s3::{Bucket, Credentials, UrlStyle}; use serde::Deserialize; +use url::Url; -use super::{client::ObjectStorageClient, types::StorageProviderConfig}; -use crate::backend_runtime::{error::napi_error, types::RuntimeObjectStorageHealth}; +use super::{ + client::ObjectStorageClient, + error::{ObjectStorageError, ObjectStorageResult}, + types::StorageProviderConfig, +}; #[derive(Clone, Debug)] -pub(in crate::backend_runtime) struct ObjectStorageConfig { - pub(super) provider: String, - pub(super) bucket: String, - pub(super) endpoint: Option, - pub(super) region: Option, - pub(super) access_key_id: Option, - pub(super) secret_access_key: Option, - pub(super) session_token: Option, - pub(super) force_path_style: bool, - pub(super) request_timeout_ms: Option, - pub(super) min_part_size: Option, - pub(super) presign_expires_in_seconds: Option, - pub(super) presign_sign_content_type_for_put: Option, - pub(super) use_presigned_url: bool, +pub(crate) struct ObjectStorageConfig { + pub(crate) provider: String, + pub(crate) bucket: String, + pub(crate) endpoint: Option, + pub(crate) region: Option, + pub(crate) access_key_id: Option, + pub(crate) secret_access_key: Option, + pub(crate) session_token: Option, + pub(crate) force_path_style: bool, + pub(crate) request_timeout_ms: Option, + pub(crate) min_part_size: Option, + pub(crate) presign_expires_in_seconds: Option, + pub(crate) presign_sign_content_type_for_put: Option, + pub(crate) use_presigned_url: bool, + pub(crate) proxy_upload: bool, } #[derive(Debug, Deserialize)] @@ -68,14 +70,15 @@ struct S3PresignConfigFile { } #[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] struct UsePresignedUrlConfigFile { enabled: bool, + url_prefix: Option, + sign_key: Option, } impl ObjectStorageConfig { - pub(in crate::backend_runtime) fn from_provider_config( - storage: Option, - ) -> Result> { + pub(crate) fn from_provider_config(storage: Option) -> ObjectStorageResult> { let Some(storage) = storage else { return Ok(None); }; @@ -84,18 +87,18 @@ impl ObjectStorageConfig { "aws-s3" => Self::from_s3_config(storage), "cloudflare-r2" => Self::from_r2_config(storage), "fs" => Ok(None), - provider => Err(napi_error(format!( - "unsupported blob storage provider for BackendRuntime: {provider}" + provider => Err(ObjectStorageError::Config(format!( + "unsupported blob storage provider for StorageRuntime: {provider}" ))), } } - pub(super) fn from_s3_config(storage: StorageProviderConfig) -> Result> { + pub(crate) fn from_s3_config(storage: StorageProviderConfig) -> ObjectStorageResult> { let config: S3ConfigFile = serde_json::from_value(storage.config) - .map_err(|err| napi_error(format!("invalid aws-s3 blob storage config: {err}")))?; + .map_err(|err| ObjectStorageError::Config(format!("invalid aws-s3 blob storage config: {err}")))?; let region = config .region - .ok_or_else(|| napi_error("aws-s3 blob storage config requires region"))?; + .ok_or_else(|| ObjectStorageError::Config("aws-s3 blob storage config requires region".to_string()))?; let endpoint = config.endpoint.or_else(|| Some(resolve_s3_endpoint(®ion))); let credentials = config.credentials.unwrap_or_default(); @@ -113,17 +116,29 @@ impl ObjectStorageConfig { presign_expires_in_seconds: config.presign.as_ref().and_then(|v| v.expires_in_seconds), presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put), use_presigned_url: config.use_presigned_url.map(|v| v.enabled).unwrap_or(false), + proxy_upload: false, })) } - pub(super) fn from_r2_config(storage: StorageProviderConfig) -> Result> { + pub(crate) fn from_r2_config(storage: StorageProviderConfig) -> ObjectStorageResult> { let config: R2ConfigFile = serde_json::from_value(storage.config) - .map_err(|err| napi_error(format!("invalid cloudflare-r2 blob storage config: {err}")))?; + .map_err(|err| ObjectStorageError::Config(format!("invalid cloudflare-r2 blob storage config: {err}")))?; let account = match config.jurisdiction { Some(jurisdiction) => format!("{}.{}", config.account_id, jurisdiction), None => config.account_id, }; let credentials = config.credentials.unwrap_or_default(); + let (use_presigned_url, proxy_upload) = config + .use_presigned_url + .map(|value| { + ( + value.enabled, + value.enabled + && value.url_prefix.as_ref().is_some_and(|prefix| !prefix.is_empty()) + && value.sign_key.as_ref().is_some_and(|key| !key.is_empty()), + ) + }) + .unwrap_or((false, false)); Ok(Some(Self { provider: storage.provider, @@ -138,81 +153,51 @@ impl ObjectStorageConfig { min_part_size: config.min_part_size, presign_expires_in_seconds: config.presign.as_ref().and_then(|v| v.expires_in_seconds), presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put), - use_presigned_url: config.use_presigned_url.map(|v| v.enabled).unwrap_or(false), + use_presigned_url, + proxy_upload, })) } - pub(super) fn build_client(&self) -> Result { + pub(crate) fn build_client(&self) -> ObjectStorageResult { let region = self .region .clone() - .ok_or_else(|| napi_error("object storage region is required"))?; + .ok_or_else(|| ObjectStorageError::Config("object storage region is required".to_string()))?; let access_key_id = self .access_key_id .clone() - .ok_or_else(|| napi_error("object storage accessKeyId is required"))?; + .ok_or_else(|| ObjectStorageError::Config("object storage accessKeyId is required".to_string()))?; let secret_access_key = self .secret_access_key .clone() - .ok_or_else(|| napi_error("object storage secretAccessKey is required"))?; + .ok_or_else(|| ObjectStorageError::Config("object storage secretAccessKey is required".to_string()))?; - let credentials = Credentials::new( - access_key_id, - secret_access_key, - self.session_token.clone(), - None, - "affine-server-config-json", - ); - let mut builder = aws_sdk_s3::Config::builder() - .behavior_version(BehaviorVersion::latest()) - .region(Region::new(region)) - .credentials_provider(credentials) - .force_path_style(self.force_path_style) - .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) - .response_checksum_validation(ResponseChecksumValidation::WhenRequired); - - if let Some(endpoint) = &self.endpoint { - builder = builder.endpoint_url(endpoint); - } - if let Some(request_timeout_ms) = self.request_timeout_ms { - builder = builder.timeout_config( - TimeoutConfig::builder() - .operation_timeout(std::time::Duration::from_millis(request_timeout_ms)) - .build(), - ); - } - - Ok(ObjectStorageClient::new( - builder.build(), + let endpoint = self.endpoint.clone().unwrap_or_else(|| resolve_s3_endpoint(®ion)); + let endpoint = Url::parse(&endpoint) + .map_err(|err| ObjectStorageError::Config(format!("object storage endpoint is invalid: {err}")))?; + let bucket = Bucket::new( + endpoint, + if self.force_path_style { + UrlStyle::Path + } else { + UrlStyle::VirtualHost + }, self.bucket.clone(), + region, + ) + .map_err(|err| ObjectStorageError::Config(format!("object storage bucket url is invalid: {err}")))?; + let credentials = match self.session_token.as_ref().filter(|token| !token.is_empty()) { + Some(session_token) => Credentials::new_with_token(access_key_id, secret_access_key, session_token.clone()), + None => Credentials::new(access_key_id, secret_access_key), + }; + + ObjectStorageClient::new( + bucket, + credentials, + self.request_timeout_ms, self.presign_expires_in_seconds.unwrap_or(60), self.presign_sign_content_type_for_put.unwrap_or(true), - )) - } - - pub(in crate::backend_runtime) fn health(&self) -> RuntimeObjectStorageHealth { - let client_buildable = self - .build_client() - .map(|client| client.non_destructive_health()) - .unwrap_or(false); - - RuntimeObjectStorageHealth { - configured: true, - provider: Some(self.provider.clone()), - bucket: Some(self.bucket.clone()), - endpoint: self.endpoint.clone(), - region: self.region.clone(), - has_credentials: self.access_key_id.is_some() - && self.secret_access_key.is_some() - && self.session_token.as_ref().map(|v| !v.is_empty()).unwrap_or(true), - force_path_style: self.force_path_style, - request_timeout_ms: self.request_timeout_ms.map(|v| v as i64), - min_part_size: self.min_part_size.map(|v| v as i64), - presign_expires_in_seconds: self.presign_expires_in_seconds.map(|v| v as i64), - presign_sign_content_type_for_put: self.presign_sign_content_type_for_put, - use_presigned_url: self.use_presigned_url, - client_buildable, - } + ) } } diff --git a/packages/backend/native/src/runtime/storage_runtime/object_storage/error.rs b/packages/backend/native/src/runtime/storage_runtime/object_storage/error.rs new file mode 100644 index 0000000000..232e2610d1 --- /dev/null +++ b/packages/backend/native/src/runtime/storage_runtime/object_storage/error.rs @@ -0,0 +1,55 @@ +use reqwest::StatusCode; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum ObjectStorageError { + #[error("ObjectStorage config error: {0}")] + Config(String), + #[error("{context}: {source}")] + Operation { + context: String, + #[source] + source: Box, + }, + #[error("ObjectStorage http client build failed: {0}")] + HttpClientBuild(#[source] reqwest::Error), + #[error("ObjectStorage http request failed: {0}")] + HttpRequest(#[source] reqwest::Error), + #[error("ObjectStorage invalid http header: {0}")] + InvalidHeader(String), + #[error("ObjectStorage response body exceeds {limit} bytes")] + BodyTooLarge { limit: usize }, + #[error("{context}: status={status} body={body}")] + HttpStatus { + context: String, + status: StatusCode, + body: String, + }, + #[error("{context}: invalid utf8 response: {source}")] + InvalidUtf8 { + context: String, + #[source] + source: std::string::FromUtf8Error, + }, + #[error("{context}: invalid xml response: {source}")] + InvalidXml { + context: String, + #[source] + source: instant_xml::Error, + }, + #[error("ObjectStorage invalid input: {0}")] + InvalidInput(String), +} + +impl ObjectStorageError { + pub(crate) fn is_not_found(&self) -> bool { + match self { + Self::Operation { source, .. } => source.is_not_found(), + Self::HttpStatus { status, body, .. } => { + *status == StatusCode::NOT_FOUND && (body.contains("NoSuchKey") || body.contains("NotFound")) + } + _ => false, + } + } +} + +pub(crate) type ObjectStorageResult = std::result::Result; diff --git a/packages/backend/native/src/runtime/storage_runtime/object_storage/mod.rs b/packages/backend/native/src/runtime/storage_runtime/object_storage/mod.rs new file mode 100644 index 0000000000..ce9f150884 --- /dev/null +++ b/packages/backend/native/src/runtime/storage_runtime/object_storage/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod client; +pub(crate) mod config; +pub(crate) mod error; +#[cfg(test)] +mod tests; +pub(crate) mod types; + +pub(crate) use config::ObjectStorageConfig; +pub(crate) use types::StorageProviderConfig; diff --git a/packages/backend/native/src/backend_runtime/object_storage/tests.rs b/packages/backend/native/src/runtime/storage_runtime/object_storage/tests.rs similarity index 61% rename from packages/backend/native/src/backend_runtime/object_storage/tests.rs rename to packages/backend/native/src/runtime/storage_runtime/object_storage/tests.rs index d39753709a..16278f988d 100644 --- a/packages/backend/native/src/backend_runtime/object_storage/tests.rs +++ b/packages/backend/native/src/runtime/storage_runtime/object_storage/tests.rs @@ -1,5 +1,8 @@ +use reqwest::StatusCode; + use super::{ config::ObjectStorageConfig, + error::ObjectStorageError, types::{MultipartUploadPart, ObjectPutMetadata, StorageProviderConfig, completed_multipart_parts, trim_etag}, }; @@ -31,9 +34,52 @@ fn resolves_r2_config_from_config_json_shape() { assert_eq!(config.region.as_deref(), Some("auto")); assert!(config.force_path_style); assert!(config.use_presigned_url); + assert!(!config.proxy_upload); assert_eq!(config.access_key_id.as_deref(), Some("key")); } +#[test] +fn object_storage_not_found_requires_object_error_code() { + let bucket_or_route_missing = ObjectStorageError::HttpStatus { + context: "head failed".to_string(), + status: StatusCode::NOT_FOUND, + body: String::new(), + }; + let object_missing = ObjectStorageError::HttpStatus { + context: "get failed".to_string(), + status: StatusCode::NOT_FOUND, + body: "NoSuchKey".to_string(), + }; + + assert!(!bucket_or_route_missing.is_not_found()); + assert!(object_missing.is_not_found()); +} + +#[test] +fn resolves_r2_proxy_upload_capability_from_config_json_shape() { + let storage = StorageProviderConfig { + provider: "cloudflare-r2".to_string(), + bucket: "workspace-blobs".to_string(), + config: serde_json::json!({ + "accountId": "account", + "credentials": { + "accessKeyId": "key", + "secretAccessKey": "secret" + }, + "usePresignedURL": { + "enabled": true, + "urlPrefix": "https://cdn.example.com", + "signKey": "secret" + } + }), + }; + + let config = ObjectStorageConfig::from_r2_config(storage).unwrap().unwrap(); + + assert!(config.use_presigned_url); + assert!(config.proxy_upload); +} + #[test] fn resolves_s3_config_from_config_json_shape() { let storage = StorageProviderConfig { @@ -109,6 +155,33 @@ async fn object_storage_presign_put_returns_sigv4_url_and_headers() { assert!(result.expires_at_ms > 0); } +#[tokio::test] +async fn object_storage_presign_get_returns_sigv4_url_without_headers() { + let storage = StorageProviderConfig { + provider: "cloudflare-r2".to_string(), + bucket: "test-bucket".to_string(), + config: serde_json::json!({ + "accountId": "account", + "credentials": { + "accessKeyId": "key", + "secretAccessKey": "secret" + }, + "presign": { + "expiresInSeconds": 60 + } + }), + }; + let config = ObjectStorageConfig::from_r2_config(storage).unwrap().unwrap(); + let client = config.build_client().unwrap(); + let result = client.presign_get("workspace/key").await.unwrap(); + + assert!(result.url.contains("X-Amz-Algorithm=AWS4-HMAC-SHA256")); + assert!(result.url.contains("X-Amz-SignedHeaders=host")); + assert!(result.url.contains("/test-bucket/workspace/key?")); + assert!(result.headers.is_empty()); + assert!(result.expires_at_ms > 0); +} + #[test] fn object_storage_orders_completed_multipart_parts_and_trims_etags() { let parts = completed_multipart_parts(vec![ @@ -122,8 +195,8 @@ fn object_storage_orders_completed_multipart_parts_and_trims_etags() { }, ]); - assert_eq!(parts[0].part_number, Some(1)); - assert_eq!(parts[0].e_tag.as_deref(), Some("a")); - assert_eq!(parts[1].part_number, Some(2)); - assert_eq!(parts[1].e_tag.as_deref(), Some("b")); + assert_eq!(parts[0].part_number, 1); + assert_eq!(parts[0].etag, "a"); + assert_eq!(parts[1].part_number, 2); + assert_eq!(parts[1].etag, "b"); } diff --git a/packages/backend/native/src/runtime/storage_runtime/object_storage/types.rs b/packages/backend/native/src/runtime/storage_runtime/object_storage/types.rs new file mode 100644 index 0000000000..b71b1575a4 --- /dev/null +++ b/packages/backend/native/src/runtime/storage_runtime/object_storage/types.rs @@ -0,0 +1,182 @@ +use std::collections::HashMap; + +use serde::Deserialize; + +use super::super::{ + RuntimeError, RuntimeMultipartUploadInit, RuntimeMultipartUploadPart, RuntimeObjectGetResult, RuntimeObjectListEntry, + RuntimeObjectMetadata, RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest, RuntimeResult, +}; + +#[derive(Clone, Debug, Default)] +pub(crate) struct ObjectPutMetadata { + pub(crate) content_type: Option, + pub(crate) content_length: Option, + pub(crate) checksum_crc32: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ObjectMetadata { + pub(crate) content_type: String, + pub(crate) content_length: i64, + pub(crate) last_modified_ms: i64, + pub(crate) checksum_crc32: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ObjectListEntry { + pub(crate) key: String, + pub(crate) content_length: i64, + pub(crate) last_modified_ms: i64, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ObjectListPage { + pub(crate) entries: Vec, + pub(crate) next_continuation_token: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ObjectGetResult { + pub(crate) body: Vec, + pub(crate) metadata: ObjectMetadata, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct PresignedObjectRequest { + pub(crate) url: String, + pub(crate) headers: HashMap, + pub(crate) expires_at_ms: i64, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct MultipartUploadInitResult { + pub(crate) upload_id: String, + pub(crate) expires_at_ms: i64, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct MultipartUploadPart { + pub(crate) part_number: i32, + pub(crate) etag: String, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct StorageProviderConfig { + pub(crate) provider: String, + pub(crate) bucket: String, + #[serde(default)] + pub(crate) config: serde_json::Value, +} + +pub(crate) fn trim_etag(etag: &str) -> String { + etag.trim_matches('"').to_string() +} + +pub(crate) fn completed_multipart_parts(mut parts: Vec) -> Vec { + parts.sort_by_key(|part| part.part_number); + parts +} + +impl From for ObjectPutMetadata { + fn from(options: RuntimeObjectStoragePutOptions) -> Self { + Self { + content_type: options.content_type, + content_length: options.content_length, + checksum_crc32: options.checksum_crc32, + } + } +} + +impl ObjectPutMetadata { + pub(crate) fn complete_for_body(mut self, body: &[u8]) -> Self { + self.content_length.get_or_insert(body.len() as i64); + self + .checksum_crc32 + .get_or_insert_with(|| format!("{:x}", crc32fast::hash(body))); + self + .content_type + .get_or_insert_with(|| crate::file_type::get_mime(body)); + self + } + + pub(crate) fn into_object_metadata(self, last_modified_ms: i64) -> ObjectMetadata { + ObjectMetadata { + content_type: self + .content_type + .unwrap_or_else(|| "application/octet-stream".to_string()), + content_length: self.content_length.unwrap_or(0), + last_modified_ms, + checksum_crc32: self.checksum_crc32, + } + } +} + +impl From for RuntimeObjectMetadata { + fn from(metadata: ObjectMetadata) -> Self { + Self { + content_type: metadata.content_type, + content_length: metadata.content_length, + last_modified_ms: metadata.last_modified_ms, + checksum_crc32: metadata.checksum_crc32, + } + } +} + +impl From for RuntimeObjectListEntry { + fn from(entry: ObjectListEntry) -> Self { + Self { + key: entry.key, + content_length: entry.content_length, + last_modified_ms: entry.last_modified_ms, + } + } +} + +impl TryFrom for RuntimePresignedObjectRequest { + type Error = RuntimeError; + + fn try_from(request: PresignedObjectRequest) -> RuntimeResult { + Ok(Self { + url: request.url, + headers_json: serde_json::to_string(&request.headers) + .map_err(|err| RuntimeError::json("ObjectStorage headers serialization failed", err))?, + expires_at_ms: request.expires_at_ms, + }) + } +} + +impl From for RuntimeObjectGetResult { + fn from(result: ObjectGetResult) -> Self { + Self { + body: result.body.into(), + metadata: result.metadata.into(), + } + } +} + +impl From for RuntimeMultipartUploadInit { + fn from(init: MultipartUploadInitResult) -> Self { + Self { + upload_id: init.upload_id, + expires_at_ms: init.expires_at_ms, + } + } +} + +impl From for MultipartUploadPart { + fn from(part: RuntimeMultipartUploadPart) -> Self { + Self { + part_number: part.part_number, + etag: part.etag, + } + } +} + +impl From for RuntimeMultipartUploadPart { + fn from(part: MultipartUploadPart) -> Self { + Self { + part_number: part.part_number, + etag: part.etag, + } + } +} diff --git a/packages/backend/native/src/backend_runtime/types.rs b/packages/backend/native/src/runtime/types.rs similarity index 89% rename from packages/backend/native/src/backend_runtime/types.rs rename to packages/backend/native/src/runtime/types.rs index 8e0bec64e8..23e73f7c59 100644 --- a/packages/backend/native/src/backend_runtime/types.rs +++ b/packages/backend/native/src/runtime/types.rs @@ -12,24 +12,6 @@ pub struct RuntimeVerificationTokenRecord { pub struct BackendRuntimeHealth { pub started: bool, pub database_connected: bool, - pub object_storage_configured: bool, -} - -#[napi_derive::napi(object)] -pub struct RuntimeObjectStorageHealth { - pub configured: bool, - pub provider: Option, - pub bucket: Option, - pub endpoint: Option, - pub region: Option, - pub has_credentials: bool, - pub force_path_style: bool, - pub request_timeout_ms: Option, - pub min_part_size: Option, - pub presign_expires_in_seconds: Option, - pub presign_sign_content_type_for_put: Option, - pub use_presigned_url: bool, - pub client_buildable: bool, } #[napi_derive::napi(object)] diff --git a/packages/backend/native/src/utils.rs b/packages/backend/native/src/utils.rs index 6940aa6253..b8525b0f60 100644 --- a/packages/backend/native/src/utils.rs +++ b/packages/backend/native/src/utils.rs @@ -1,3 +1,9 @@ +use std::time::{SystemTime, SystemTimeError, UNIX_EPOCH}; + +pub(crate) fn system_time_millis(time: SystemTime) -> Result { + Ok(time.duration_since(UNIX_EPOCH)?.as_millis()) +} + fn collapse_whitespace(s: &str) -> String { let mut result = String::new(); let mut prev_was_whitespace = false; diff --git a/packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts b/packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts index 402fa0d1ca..5a1e8c317f 100644 --- a/packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts @@ -1,4 +1,4 @@ -import { createHash } from 'node:crypto'; +import { createHash, createHmac } from 'node:crypto'; import { mock } from 'node:test'; import { @@ -6,22 +6,13 @@ import { ConfigFactory, PROXY_MULTIPART_PATH, PROXY_UPLOAD_PATH, - StorageProviderConfig, - StorageProviderFactory, - toBuffer, + type R2StorageConfig, + SIGNED_URL_EXPIRED, + type StorageProviderConfig, } from '../../../base'; -import { - R2StorageConfig, - R2StorageProvider, -} from '../../../base/storage/providers/r2'; -import { SIGNED_URL_EXPIRED } from '../../../base/storage/providers/utils'; import { EntitlementService } from '../../../core/entitlement'; -import { - CommentAttachmentStorage, - WorkspaceBlobStorage, -} from '../../../core/storage'; import { MULTIPART_THRESHOLD } from '../../../core/storage/constants'; -import { R2UploadController } from '../../../core/storage/r2-proxy'; +import { StorageRuntimeProvider } from '../../../core/storage-runtime'; import { SubscriptionPlan, SubscriptionRecurring, @@ -29,7 +20,7 @@ import { } from '../../../plugins/payment/types'; import { app, e2e, Mockers } from '../test'; -class MockR2Provider extends R2StorageProvider { +class MockStorageRuntime { createMultipartCalls = 0; putCalls: { key: string; @@ -46,45 +37,72 @@ class MockR2Provider extends R2StorageProvider { contentLength?: number; }[] = []; - constructor(config: R2StorageConfig, bucket: string) { - super(config, bucket); + async providerCapabilities() { + const storage = app.get(Config).storages.blob.storage; + const usePresignedURL = (storage.config as R2StorageConfig).usePresignedURL; + if (storage.provider !== 'cloudflare-r2') { + return { + put: true, + get: true, + head: true, + list: true, + delete: true, + presignPut: false, + presignGet: false, + multipartDirect: false, + proxyUpload: false, + assetpack: false, + serverMediatedOnly: true, + }; + } + return { + put: true, + get: true, + head: true, + list: true, + delete: true, + presignPut: true, + presignGet: false, + multipartDirect: true, + proxyUpload: !!usePresignedURL?.enabled, + assetpack: false, + serverMediatedOnly: false, + }; } - destroy() {} - - override async proxyPutObject( + async presignPut( + _scope: string, key: string, - body: any, - options: { contentType?: string; contentLength?: number } = {} + metadata: { contentType?: string; contentLength?: number } = {} ) { - this.putCalls.push({ - key, - body: await toBuffer(body), - contentType: options.contentType, - contentLength: options.contentLength, - }); + const storage = app.get(Config).storages.blob.storage; + const r2 = storage.config as R2StorageConfig; + if (!r2.usePresignedURL?.enabled) { + return { + url: 'https://test-bucket.r2.example.com/object?X-Amz-Algorithm=AWS4-HMAC-SHA256', + headers: {}, + expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000), + }; + } + const [workspaceId, blobKey] = key.split('/'); + return createProxyUrl( + PROXY_UPLOAD_PATH, + [ + workspaceId, + blobKey, + metadata.contentType ?? 'application/octet-stream', + metadata.contentLength, + ], + { + workspaceId, + key: blobKey, + contentType: metadata.contentType ?? 'application/octet-stream', + contentLength: metadata.contentLength, + } + ); } - override async proxyUploadPart( - key: string, - uploadId: string, - partNumber: number, - body: any, - options: { contentLength?: number } = {} - ) { - const etag = `etag-${partNumber}`; - this.partCalls.push({ - key, - uploadId, - partNumber, - etag, - body: await toBuffer(body), - contentLength: options.contentLength, - }); - return etag; - } - - override async createMultipartUpload() { + async createMultipartUpload() { this.createMultipartCalls += 1; return { uploadId: 'upload-id', @@ -92,7 +110,30 @@ class MockR2Provider extends R2StorageProvider { }; } - override async listMultipartUploadParts(key: string, uploadId: string) { + async presignUploadPart( + _scope: string, + key: string, + uploadId: string, + partNumber: number + ) { + const [workspaceId, blobKey] = key.split('/'); + return createProxyUrl( + PROXY_MULTIPART_PATH, + [workspaceId, blobKey, uploadId, partNumber], + { + workspaceId, + key: blobKey, + uploadId, + partNumber, + } + ); + } + + async listMultipartUploadParts( + _scope: string, + key: string, + uploadId: string + ) { const latest = new Map(); for (const part of this.partCalls) { if (part.key !== key || part.uploadId !== uploadId) { @@ -104,6 +145,45 @@ class MockR2Provider extends R2StorageProvider { .sort((left, right) => left[0] - right[0]) .map(([partNumber, etag]) => ({ partNumber, etag })); } + + async putObject( + _scope: string, + key: string, + body: Buffer, + options: { contentType?: string; contentLength?: number } = {} + ) { + this.putCalls.push({ + key, + body, + contentType: options.contentType, + contentLength: options.contentLength, + }); + return { + contentType: options.contentType ?? 'application/octet-stream', + contentLength: options.contentLength ?? body.length, + lastModified: new Date(), + }; + } + + async proxyUploadPart( + _scope: string, + key: string, + uploadId: string, + partNumber: number, + body: Buffer, + contentLength?: number + ) { + const etag = `etag-${partNumber}`; + this.partCalls.push({ + key, + uploadId, + partNumber, + etag, + body, + contentLength, + }); + return etag; + } } const baseR2Storage: StorageProviderConfig = { @@ -125,55 +205,40 @@ const baseR2Storage: StorageProviderConfig = { }; let defaultBlobStorage: StorageProviderConfig; -let provider: MockR2Provider | null = null; -let factoryCreateUnmocked: StorageProviderFactory['create']; +let runtime: MockStorageRuntime; e2e.before(() => { defaultBlobStorage = structuredClone(app.get(Config).storages.blob.storage); - const factory = app.get(StorageProviderFactory); - factoryCreateUnmocked = factory.create.bind(factory); }); e2e.beforeEach(async () => { - provider?.destroy(); - provider = null; - - const factory = app.get(StorageProviderFactory); - mock.method(factory, 'create', (config: StorageProviderConfig) => { - if (config.provider === 'cloudflare-r2') { - if (!provider) { - provider = new MockR2Provider( - config.config as R2StorageConfig, - config.bucket - ); - } - return provider; - } - return factoryCreateUnmocked(config); - }); + runtime = new MockStorageRuntime(); + const rt = app.get(StorageRuntimeProvider); + for (const method of [ + 'providerCapabilities', + 'presignPut', + 'createMultipartUpload', + 'presignUploadPart', + 'listMultipartUploadParts', + 'putObject', + 'proxyUploadPart', + ] as const) { + mock.method(rt, method, (...args: any[]) => + (runtime[method] as any)(...args) + ); + } await useR2Storage(); }); e2e.afterEach.always(async () => { await setBlobStorage(defaultBlobStorage); - provider?.destroy(); - provider = null; mock.reset(); }); async function setBlobStorage(storage: StorageProviderConfig) { - provider?.destroy(); - provider = null; const configFactory = app.get(ConfigFactory); configFactory.override({ storages: { blob: { storage } } }); - const blobStorage = app.get(WorkspaceBlobStorage); - await blobStorage.onConfigInit(); - const commentAttachmentStorage = app.get(CommentAttachmentStorage); - await commentAttachmentStorage.onConfigInit(); - const controller = app.get(R2UploadController); - // reset cached provider in controller - (controller as any).provider = null; } async function useR2Storage( @@ -193,11 +258,8 @@ async function useR2Storage( return storage; } -function getProvider(): MockR2Provider { - if (!provider) { - throw new Error('R2 provider is not initialized'); - } - return provider; +function getRuntime(): MockStorageRuntime { + return runtime; } async function createBlobUpload( @@ -285,7 +347,7 @@ async function gql( return res.body.data; } -e2e('should proxy single upload with valid signature', async t => { +e2e.serial('should proxy single upload with valid signature', async t => { const { workspace } = await setupWorkspace(); const buffer = Buffer.from('r2-proxy'); const key = sha256Base64urlWithPadding(buffer); @@ -309,7 +371,7 @@ e2e('should proxy single upload with valid signature', async t => { .send(buffer); t.is(res.status, 200); - const calls = getProvider().putCalls; + const calls = getRuntime().putCalls; t.is(calls.length, 1); t.is(calls[0].key, `${workspace.id}/${key}`); t.is(calls[0].contentType, 'text/plain'); @@ -317,7 +379,7 @@ e2e('should proxy single upload with valid signature', async t => { t.deepEqual(calls[0].body, buffer); }); -e2e('should proxy multipart upload and return etag', async t => { +e2e.serial('should proxy multipart upload and return etag', async t => { const { workspace } = await setupWorkspace(); const key = 'multipart-object'; const totalSize = MULTIPART_THRESHOLD + 1024; @@ -340,7 +402,7 @@ e2e('should proxy multipart upload and return etag', async t => { t.is(res.status, 200); t.is(res.get('etag'), 'etag-1'); - const calls = getProvider().partCalls; + const calls = getRuntime().partCalls; t.is(calls.length, 1); t.is(calls[0].key, `${workspace.id}/${key}`); t.is(calls[0].uploadId, 'upload-id'); @@ -349,34 +411,42 @@ e2e('should proxy multipart upload and return etag', async t => { t.deepEqual(calls[0].body, payload); }); -e2e('should resume multipart upload and return uploaded parts', async t => { - const { workspace } = await setupWorkspace(); - const key = 'multipart-resume'; - const totalSize = MULTIPART_THRESHOLD + 1024; +e2e.serial( + 'should resume multipart upload and return uploaded parts', + async t => { + const { workspace } = await setupWorkspace(); + const key = 'multipart-resume'; + const totalSize = MULTIPART_THRESHOLD + 1024; - const init1 = await createBlobUpload(workspace.id, key, totalSize, 'bin'); - t.is(init1.method, 'MULTIPART'); - t.is(init1.uploadId, 'upload-id'); - t.deepEqual(init1.uploadedParts, []); - t.is(getProvider().createMultipartCalls, 1); + const init1 = await createBlobUpload(workspace.id, key, totalSize, 'bin'); + t.is(init1.method, 'MULTIPART'); + t.is(init1.uploadId, 'upload-id'); + t.deepEqual(init1.uploadedParts, []); + t.is(getRuntime().createMultipartCalls, 1); - const part = await getBlobUploadPartUrl(workspace.id, key, init1.uploadId, 1); - const payload = Buffer.from('part-body'); - const partUrl = new URL(part.uploadUrl, app.url); - await app - .PUT(partUrl.pathname + partUrl.search) - .set('content-length', payload.length.toString()) - .send(payload) - .expect(200); + const part = await getBlobUploadPartUrl( + workspace.id, + key, + init1.uploadId, + 1 + ); + const payload = Buffer.from('part-body'); + const partUrl = new URL(part.uploadUrl, app.url); + await app + .PUT(partUrl.pathname + partUrl.search) + .set('content-length', payload.length.toString()) + .send(payload) + .expect(200); - const init2 = await createBlobUpload(workspace.id, key, totalSize, 'bin'); - t.is(init2.method, 'MULTIPART'); - t.is(init2.uploadId, 'upload-id'); - t.deepEqual(init2.uploadedParts, [{ partNumber: 1, etag: 'etag-1' }]); - t.is(getProvider().createMultipartCalls, 1); -}); + const init2 = await createBlobUpload(workspace.id, key, totalSize, 'bin'); + t.is(init2.method, 'MULTIPART'); + t.is(init2.uploadId, 'upload-id'); + t.deepEqual(init2.uploadedParts, [{ partNumber: 1, etag: 'etag-1' }]); + t.is(getRuntime().createMultipartCalls, 1); + } +); -e2e('should reject upload when token is invalid', async t => { +e2e.serial('should reject upload when token is invalid', async t => { const { workspace } = await setupWorkspace(); const buffer = Buffer.from('payload'); const init = await createBlobUpload( @@ -396,10 +466,10 @@ e2e('should reject upload when token is invalid', async t => { t.is(res.status, 400); t.is(res.body.message, 'Invalid upload token'); - t.is(getProvider().putCalls.length, 0); + t.is(getRuntime().putCalls.length, 0); }); -e2e('should reject upload when url is expired', async t => { +e2e.serial('should reject upload when url is expired', async t => { const { workspace } = await setupWorkspace(); const buffer = Buffer.from('expired'); const init = await createBlobUpload( @@ -422,10 +492,10 @@ e2e('should reject upload when url is expired', async t => { t.is(res.status, 400); t.is(res.body.message, 'Upload URL expired'); - t.is(getProvider().putCalls.length, 0); + t.is(getRuntime().putCalls.length, 0); }); -e2e( +e2e.serial( 'should fall back to direct presign when custom domain is disabled', async t => { await useR2Storage({ @@ -449,7 +519,7 @@ e2e( } ); -e2e( +e2e.serial( 'should still fallback to graphql when provider does not support presign', async t => { await setBlobStorage({ @@ -473,6 +543,40 @@ e2e( } ); +function createProxyUrl( + path: string, + canonicalFields: (string | number | undefined)[], + query: Record +) { + const signKey = ( + app.get(Config).storages.blob.storage.config as R2StorageConfig + ).usePresignedURL?.signKey; + if (!signKey) { + throw new Error('missing R2 proxy sign key'); + } + const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED; + const canonical = [ + path, + ...canonicalFields.map(field => + field === undefined ? '' : field.toString() + ), + exp.toString(), + ].join('\n'); + const token = createHmac('sha256', signKey) + .update(canonical) + .digest('base64'); + + const url = new URL(`http://localhost${path}`); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) { + url.searchParams.set(key, value.toString()); + } + } + url.searchParams.set('exp', exp.toString()); + url.searchParams.set('token', `${exp}-${token}`); + return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) }; +} + function sha256Base64urlWithPadding(buffer: Buffer) { return createHash('sha256') .update(buffer) diff --git a/packages/backend/server/src/__tests__/e2e/workspace/controller.spec.ts b/packages/backend/server/src/__tests__/e2e/workspace/controller.spec.ts index 09d57cdde3..2870d537a8 100644 --- a/packages/backend/server/src/__tests__/e2e/workspace/controller.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/workspace/controller.spec.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto'; +import { Readable } from 'node:stream'; import { mock } from 'node:test'; import { @@ -7,6 +8,8 @@ import { type StorageProviderConfig, } from '../../../base'; import { CommentAttachmentStorage } from '../../../core/storage'; +import { StorageRuntimeProvider } from '../../../core/storage-runtime'; +import { getMime } from '../../../native'; import { Mockers } from '../../mocks'; import { app, e2e } from '../test'; @@ -26,14 +29,68 @@ e2e.afterEach.always(() => { mock.reset(); }); +const objects = new Map< + string, + { + body: Buffer; + metadata?: { + contentLength?: number; + contentType?: string; + checksumCRC32?: string; + lastModified?: Date; + }; + } +>(); + +e2e.beforeEach(() => { + objects.clear(); + const rt = app.get(StorageRuntimeProvider); + mock.method( + rt, + 'putObject', + async ( + _scope: string, + key: string, + body: Buffer, + metadata?: { + contentLength?: number; + contentType?: string; + checksumCRC32?: string; + } + ) => { + const object = { + body, + metadata: { + ...metadata, + contentType: metadata?.contentType ?? getMime(body), + contentLength: metadata?.contentLength ?? body.length, + lastModified: new Date(), + }, + }; + objects.set(key, object); + return object.metadata; + } + ); + mock.method(rt, 'getObject', async (_scope: string, key: string) => { + const object = objects.get(key); + if (!object) { + return {}; + } + return { + body: Readable.from(object.body), + metadata: object.metadata, + }; + }); + mock.method(rt, 'presignGet', async () => undefined); +}); + async function useCommentAttachmentBlobStorage(storage: StorageProviderConfig) { app.get(ConfigFactory).override({ storages: { blob: { storage } } }); - await app.get(CommentAttachmentStorage).onConfigInit(); } // #region comment attachment -e2e( +e2e.serial( 'should get comment attachment not found when key is not exists', async t => { const { owner, workspace } = await createWorkspace(); @@ -50,7 +107,7 @@ e2e( } ); -e2e( +e2e.serial( 'should get comment attachment no permission when user is not member', async t => { const { workspace } = await createWorkspace(); @@ -117,7 +174,7 @@ e2e.serial('should get comment attachment body', async t => { } }); -e2e('should get comment attachment redirect url', async t => { +e2e.serial('should get comment attachment redirect url', async t => { const { owner, workspace } = await createWorkspace(); await app.login(owner); diff --git a/packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts b/packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts index 186a6e555e..bbe598e1f6 100644 --- a/packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts +++ b/packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts @@ -4,9 +4,9 @@ import ava, { TestFn } from 'ava'; import Sinon from 'sinon'; import { OneDay } from '../../base'; -import { BackendRuntimeProvider } from '../../core/backend-runtime'; import { StorageModule, WorkspaceBlobStorage } from '../../core/storage'; import { BlobUploadCleanupJob } from '../../core/storage/job'; +import { StorageRuntimeProvider } from '../../core/storage-runtime'; import { MockUser, MockWorkspace } from '../mocks'; import { createTestingModule, TestingModule } from '../utils'; @@ -28,7 +28,7 @@ test.before(async t => { imports: [ScheduleModule.forRoot(), StorageModule], tapModule: builder => { builder - .overrideProvider(BackendRuntimeProvider) + .overrideProvider(StorageRuntimeProvider) .useValue(t.context.runtime); }, }); diff --git a/packages/backend/server/src/__tests__/workspace/blobs.e2e.ts b/packages/backend/server/src/__tests__/workspace/blobs.e2e.ts index e75b6cec4e..13c81ea205 100644 --- a/packages/backend/server/src/__tests__/workspace/blobs.e2e.ts +++ b/packages/backend/server/src/__tests__/workspace/blobs.e2e.ts @@ -1,12 +1,15 @@ import { createHash } from 'node:crypto'; +import { Readable } from 'node:stream'; import test from 'ava'; import Sinon from 'sinon'; -import { Config, ConfigFactory, StorageProviderFactory } from '../../base'; +import { ConfigFactory } from '../../base'; import { QuotaStateService } from '../../core/quota/state'; import { WorkspaceBlobStorage } from '../../core/storage/wrappers/blob'; +import { StorageRuntimeProvider } from '../../core/storage-runtime'; import { BlobModel, WorkspaceFeatureModel } from '../../models'; +import { getMime } from '../../native'; import { collectAllBlobSizes, completeBlobUpload, @@ -32,9 +35,113 @@ const RESTRICTED_QUOTA = { let app: TestingApp; let model: WorkspaceFeatureModel; +const objects = new Map< + string, + { + body: Buffer; + metadata: { + contentType: string; + contentLength: number; + lastModified: Date; + }; + } +>(); +const storageRuntime = { + providerCapabilities: async () => ({ + put: true, + get: true, + head: true, + list: true, + delete: true, + presignPut: false, + presignGet: false, + multipartDirect: false, + proxyUpload: false, + assetpack: false, + serverMediatedOnly: true, + }), + putObject: async ( + _scope: string, + key: string, + body: Buffer, + metadata?: { contentType?: string; contentLength?: number } + ) => { + const object = { + body, + metadata: { + contentType: metadata?.contentType ?? getMime(body), + contentLength: metadata?.contentLength ?? body.length, + lastModified: new Date(), + }, + }; + objects.set(key, object); + return object.metadata; + }, + headObject: async (_scope: string, key: string) => { + return objects.get(key)?.metadata; + }, + getObject: async (_scope: string, key: string) => { + const object = objects.get(key); + return object + ? { body: Readable.from(object.body), metadata: object.metadata } + : {}; + }, + listObjects: async (_scope: string, prefix?: string) => { + return Array.from(objects.entries()) + .filter(([key]) => !prefix || key.startsWith(prefix)) + .map(([key, object]) => ({ key, ...object.metadata })); + }, + deleteObject: async (_scope: string, key: string) => { + objects.delete(key); + }, + presignPut: async () => undefined, + presignGet: async () => undefined, + createMultipartUpload: async () => undefined, + presignUploadPart: async () => undefined, + listMultipartUploadParts: async () => undefined, + completeMultipartUpload: async () => undefined, + completeWorkspaceBlobUpload: async ( + workspaceId: string, + key: string, + expected: { size: number; mime: string } + ) => { + const objectKey = `${workspaceId}/${key}`; + const object = objects.get(objectKey); + if (!object) { + return { ok: false, reason: 'not_found' }; + } + if (object.metadata.contentLength !== expected.size) { + return { ok: false, reason: 'size_mismatch' }; + } + if (object.metadata.contentType !== expected.mime) { + return { ok: false, reason: 'mime_mismatch' }; + } + if (sha256Base64urlWithPadding(object.body) !== key) { + return { ok: false, reason: 'checksum_mismatch' }; + } + await app.get(BlobModel).upsert({ + workspaceId, + key, + mime: object.metadata.contentType, + size: object.metadata.contentLength, + status: 'completed', + uploadId: null, + }); + return { + ok: true, + contentType: object.metadata.contentType, + contentLength: object.metadata.contentLength, + lastModifiedMs: object.metadata.lastModified.getTime(), + }; + }, +}; test.before(async () => { - app = await createTestingApp(); + app = await createTestingApp({ + tapModule: builder => { + builder.overrideProvider(StorageRuntimeProvider).useValue(storageRuntime); + }, + }); model = app.get(WorkspaceFeatureModel); app.get(ConfigFactory).override({ storages: { @@ -47,11 +154,11 @@ test.before(async () => { }, }, }); - await app.get(WorkspaceBlobStorage).onConfigInit(); }); test.beforeEach(async () => { await app.initTestingDB(); + objects.clear(); }); test.after.always(async () => { @@ -124,22 +231,19 @@ test('should keep partial blob metadata listing on DB path without storage scan' const workspace = await createWorkspace(app); const storage = app.get(WorkspaceBlobStorage); - const rawProvider = (storage as any).provider; - const listSpy = Sinon.spy(rawProvider, 'list'); + const rt = app.get(StorageRuntimeProvider); + const listSpy = Sinon.spy(rt, 'listObjects'); t.teardown(() => listSpy.restore()); const buffer1 = Buffer.from('with metadata'); const buffer2 = Buffer.from('without metadata'); const key1 = sha256Base64urlWithPadding(buffer1); const key2 = sha256Base64urlWithPadding(buffer2); - const config = app.get(Config); - const factory = app.get(StorageProviderFactory); - const provider = factory.create(config.storages.blob.storage); - await provider.put(`${workspace.id}/${key1}`, buffer1, { + await rt.putObject('blob', `${workspace.id}/${key1}`, buffer1, { contentType: 'text/plain', contentLength: buffer1.length, }); - await provider.put(`${workspace.id}/${key2}`, buffer2, { + await rt.putObject('blob', `${workspace.id}/${key2}`, buffer2, { contentType: 'text/plain', contentLength: buffer2.length, }); @@ -194,11 +298,9 @@ test('should complete pending blob upload', async t => { await createBlobUpload(app, workspace.id, key, buffer.length, mime); - const config = app.get(Config); - const factory = app.get(StorageProviderFactory); - const provider = factory.create(config.storages.blob.storage); + const rt = app.get(StorageRuntimeProvider); - await provider.put(`${workspace.id}/${key}`, buffer, { + await rt.putObject('blob', `${workspace.id}/${key}`, buffer, { contentType: mime, contentLength: buffer.length, }); @@ -225,11 +327,9 @@ test('should reject complete when blob key mismatched', async t => { const wrongKey = sha256Base64urlWithPadding(Buffer.from('other')); await createBlobUpload(app, workspace.id, wrongKey, buffer.length, mime); - const config = app.get(Config); - const factory = app.get(StorageProviderFactory); - const provider = factory.create(config.storages.blob.storage); + const rt = app.get(StorageRuntimeProvider); - await provider.put(`${workspace.id}/${wrongKey}`, buffer, { + await rt.putObject('blob', `${workspace.id}/${wrongKey}`, buffer, { contentType: mime, contentLength: buffer.length, }); @@ -265,9 +365,8 @@ test('should auto delete blobs when workspace is deleted', async t => { const blobs = await listBlobs(app, workspace.id); t.is(blobs.length, 2); - const storage = app.get(WorkspaceBlobStorage); - const rawProvider = (storage as any).provider; - const listSpy = Sinon.spy(rawProvider, 'list'); + const rt = app.get(StorageRuntimeProvider); + const listSpy = Sinon.spy(rt, 'listObjects'); t.teardown(() => listSpy.restore()); await deleteWorkspace(app, workspace.id); diff --git a/packages/backend/server/src/__tests__/workspace/controller.spec.ts b/packages/backend/server/src/__tests__/workspace/controller.spec.ts index c336b4fdf8..156d3aff54 100644 --- a/packages/backend/server/src/__tests__/workspace/controller.spec.ts +++ b/packages/backend/server/src/__tests__/workspace/controller.spec.ts @@ -212,8 +212,7 @@ test('should be able to get permission granted workspace', async t => { test('should return 404 if blob not found', async t => { const { app, storage } = t.context; - // @ts-expect-error mock - storage.get.resolves({ body: null }); + storage.get.resolves({ body: undefined }); const res = await app.GET('/api/workspaces/public/blobs/test'); t.is(res.status, HttpStatus.NOT_FOUND); diff --git a/packages/backend/server/src/app.module.ts b/packages/backend/server/src/app.module.ts index 0c38924eda..aeeb49fc13 100644 --- a/packages/backend/server/src/app.module.ts +++ b/packages/backend/server/src/app.module.ts @@ -25,7 +25,6 @@ import { MetricsModule } from './base/metrics'; import { MutexModule } from './base/mutex'; import { PrismaModule } from './base/prisma'; import { RedisModule } from './base/redis'; -import { StorageProviderModule } from './base/storage'; import { RateLimiterModule } from './base/throttler'; import { WebSocketModule } from './base/websocket'; import { AccessTokenModule } from './core/access-token'; @@ -47,6 +46,7 @@ import { RealtimeModule } from './core/realtime'; import { SelfhostModule } from './core/selfhost'; import { StaticFileModule } from './core/static-files'; import { StorageModule } from './core/storage'; +import { StorageRuntimeModule } from './core/storage-runtime'; import { SyncModule } from './core/sync'; import { TelemetryModule } from './core/telemetry'; import { UserModule } from './core/user'; @@ -114,7 +114,6 @@ export const FunctionalityModules = [ MutexModule, MetricsModule, RateLimiterModule, - StorageProviderModule, HelpersModule, ErrorModule, WebSocketModule, @@ -122,6 +121,7 @@ export const FunctionalityModules = [ RealtimeModule, ModelsModule, BackendRuntimeModule, + StorageRuntimeModule, ScheduleModule.forRoot(), MonitorModule, ]; diff --git a/packages/backend/server/src/base/error/errors.gen.ts b/packages/backend/server/src/base/error/errors.gen.ts index 497b92c4ec..4700509f17 100644 --- a/packages/backend/server/src/base/error/errors.gen.ts +++ b/packages/backend/server/src/base/error/errors.gen.ts @@ -1059,6 +1059,16 @@ export class UnsupportedClientVersion extends UserFriendlyError { super('action_forbidden', 'unsupported_client_version', message, args); } } +@ObjectType() +class UnsupportedServerVersionDataType { + @Field() requiredVersion!: string +} + +export class UnsupportedServerVersion extends UserFriendlyError { + constructor(args: UnsupportedServerVersionDataType, message?: string | ((args: UnsupportedServerVersionDataType) => string)) { + super('action_forbidden', 'unsupported_server_version', message, args); + } +} export class NotificationNotFound extends UserFriendlyError { constructor(message?: string) { @@ -1288,6 +1298,7 @@ export enum ErrorNames { INVALID_LICENSE_UPDATE_PARAMS, LICENSE_EXPIRED, UNSUPPORTED_CLIENT_VERSION, + UNSUPPORTED_SERVER_VERSION, NOTIFICATION_NOT_FOUND, MENTION_USER_DOC_ACCESS_DENIED, MENTION_USER_ONESELF_DENIED, @@ -1308,5 +1319,5 @@ registerEnumType(ErrorNames, { export const ErrorDataUnionType = createUnionType({ name: 'ErrorDataUnion', types: () => - [GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const, + [GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, UnsupportedServerVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const, }); diff --git a/packages/backend/server/src/base/index.ts b/packages/backend/server/src/base/index.ts index 1d62717476..ae37b8d8f3 100644 --- a/packages/backend/server/src/base/index.ts +++ b/packages/backend/server/src/base/index.ts @@ -30,11 +30,6 @@ export { Lock, Locker, Mutex, RequestMutex } from './mutex'; export * from './nestjs'; export { type PrismaTransaction } from './prisma'; export * from './storage'; -export { - autoMetadata, - type StorageProvider, - type StorageProviderConfig, - StorageProviderFactory, -} from './storage'; +export { type StorageProviderConfig } from './storage'; export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler'; export * from './utils'; diff --git a/packages/backend/server/src/base/storage/__tests__/fs.spec.ts b/packages/backend/server/src/base/storage/__tests__/fs.spec.ts deleted file mode 100644 index 9077adfd5e..0000000000 --- a/packages/backend/server/src/base/storage/__tests__/fs.spec.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { promises as fs } from 'node:fs'; -import { join } from 'node:path'; - -import test from 'ava'; -import { getStreamAsBuffer } from 'get-stream'; - -import { ListObjectsMetadata } from '../providers'; -import { FsStorageProvider } from '../providers/fs'; - -const config = { - path: join(process.cwd(), 'node_modules', '.cache/affine-test-storage'), -}; - -function createProvider() { - return new FsStorageProvider( - config, - 'test' + Math.random().toString(16).substring(2, 8) - ); -} - -function keys(list: ListObjectsMetadata[]) { - return list.map(i => i.key); -} - -async function randomPut( - provider: FsStorageProvider, - prefix = '' -): Promise { - const key = prefix + 'test-key-' + Math.random().toString(16).substring(2, 8); - const body = Buffer.from(key); - await provider.put(key, body); - return key; -} - -test.after.always(() => { - fs.rm(config.path, { recursive: true }).catch(console.error); -}); - -test('put & get', async t => { - const provider = createProvider(); - const key = 'testKey'; - const body = Buffer.from('testBody'); - await provider.put(key, body); - - const result = await provider.get(key); - - t.deepEqual(await getStreamAsBuffer(result.body!), body); - t.is(result.metadata?.contentLength, body.length); -}); - -test('list - one level', async t => { - const provider = createProvider(); - const list = await Promise.all( - Array.from({ length: 100 }).map(() => randomPut(provider)) - ); - list.sort(); - // random order, use set - const result = await provider.list(); - t.deepEqual(keys(result), list); - - const result2 = await provider.list('test-key'); - t.deepEqual(keys(result2), list); - - const result3 = await provider.list('testKey'); - t.is(result3.length, 0); -}); - -test('list recursively', async t => { - const provider = createProvider(); - - await Promise.all([ - Promise.all(Array.from({ length: 10 }).map(() => randomPut(provider))), - Promise.all( - Array.from({ length: 10 }).map(() => randomPut(provider, 'a/')) - ), - Promise.all( - Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/')) - ), - Promise.all( - Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/t/')) - ), - ]); - - const r1 = await provider.list(); - t.is(r1.length, 40); - - // contains all `a/xxx` and `a/b/xxx` and `a/b/c/xxx` - const r2 = await provider.list('a'); - t.is(r2.length, 30); - - // contains only `a/b/xxx` - const r3 = await provider.list('a/b'); - const r4 = await provider.list('a/b/'); - t.is(r3.length, 20); - t.deepEqual(r3, r4); - - // prefix is not ended with '/', it's open to all files and sub dirs - // contains all `a/b/t/xxx` and `a/b/t{xxxx}` - const r5 = await provider.list('a/b/t'); - - t.is(r5.length, 20); -}); - -test('delete', async t => { - const provider = createProvider(); - const key = 'testKey'; - const body = Buffer.from('testBody'); - await provider.put(key, body); - - await provider.delete(key); - - await t.throwsAsync(() => fs.access(join(config.path, provider.bucket, key))); -}); - -test('rejects unsafe object keys', async t => { - const provider = createProvider(); - - await t.throwsAsync(() => provider.put('../escape', Buffer.from('nope'))); - await t.throwsAsync(() => provider.get('nested/../escape')); - await t.throwsAsync(() => provider.head('./escape')); - t.throws(() => provider.delete('nested//escape')); -}); - -test('rejects unsafe list prefixes', async t => { - const provider = createProvider(); - - await t.throwsAsync(() => provider.list('../escape')); - await t.throwsAsync(() => provider.list('nested/../../escape')); - await t.throwsAsync(() => provider.list('/absolute')); -}); diff --git a/packages/backend/server/src/base/storage/__tests__/r2.spec.ts b/packages/backend/server/src/base/storage/__tests__/r2.spec.ts deleted file mode 100644 index d4a6b25f39..0000000000 --- a/packages/backend/server/src/base/storage/__tests__/r2.spec.ts +++ /dev/null @@ -1,82 +0,0 @@ -import test from 'ava'; - -import { R2StorageProvider } from '../providers/r2'; - -function endpointOf(provider: R2StorageProvider) { - return provider.endpointUrl; -} - -test('R2 provider should use account endpoint by default', t => { - const provider = new R2StorageProvider( - { - accountId: 'test-account', - region: 'auto', - credentials: { - accessKeyId: 'test', - secretAccessKey: 'test', - }, - }, - 'test-bucket' - ); - - t.is( - endpointOf(provider), - 'https://test-account.r2.cloudflarestorage.com/test-bucket' - ); -}); - -test('R2 provider should append jurisdiction suffix for EU buckets', t => { - const provider = new R2StorageProvider( - { - accountId: 'test-account', - jurisdiction: 'eu', - region: 'auto', - credentials: { - accessKeyId: 'test', - secretAccessKey: 'test', - }, - }, - 'test-bucket' - ); - - t.is( - endpointOf(provider), - 'https://test-account.eu.r2.cloudflarestorage.com/test-bucket' - ); -}); - -test('R2 provider should throw when accountId is missing', t => { - t.throws( - () => - new R2StorageProvider( - { - region: 'auto', - credentials: { - accessKeyId: 'test', - secretAccessKey: 'test', - }, - } as any, - 'test-bucket' - ) - ); -}); - -test('R2 provider should use default endpoint when jurisdiction is explicitly undefined', t => { - const provider = new R2StorageProvider( - { - accountId: 'test-account', - jurisdiction: undefined, - region: 'auto', - credentials: { - accessKeyId: 'test', - secretAccessKey: 'test', - }, - }, - 'test-bucket' - ); - - t.is( - endpointOf(provider), - 'https://test-account.r2.cloudflarestorage.com/test-bucket' - ); -}); diff --git a/packages/backend/server/src/base/storage/__tests__/s3-list-parts.spec.ts b/packages/backend/server/src/base/storage/__tests__/s3-list-parts.spec.ts deleted file mode 100644 index 407de50bc0..0000000000 --- a/packages/backend/server/src/base/storage/__tests__/s3-list-parts.spec.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { parseListPartsXml } from '@affine/s3-compat'; -import test from 'ava'; - -test('parseListPartsXml handles array parts and pagination', t => { - const xml = ` - - test - key - upload-id - 0 - 3 - true - - 1 - "etag-1" - - - 2 - etag-2 - -`; - - const result = parseListPartsXml(xml); - t.deepEqual(result.parts, [ - { partNumber: 1, etag: 'etag-1' }, - { partNumber: 2, etag: 'etag-2' }, - ]); - t.true(result.isTruncated); - t.is(result.nextPartNumberMarker, '3'); -}); - -test('parseListPartsXml handles single part', t => { - const xml = ` - - test - key - upload-id - false - - 5 - "etag-5" - -`; - - const result = parseListPartsXml(xml); - t.deepEqual(result.parts, [{ partNumber: 5, etag: 'etag-5' }]); - t.false(result.isTruncated); - t.is(result.nextPartNumberMarker, undefined); -}); diff --git a/packages/backend/server/src/base/storage/__tests__/s3.spec.ts b/packages/backend/server/src/base/storage/__tests__/s3.spec.ts deleted file mode 100644 index f596784f14..0000000000 --- a/packages/backend/server/src/base/storage/__tests__/s3.spec.ts +++ /dev/null @@ -1,90 +0,0 @@ -import test from 'ava'; - -import { S3StorageProvider } from '../providers/s3'; -import { SIGNED_URL_EXPIRED } from '../providers/utils'; - -const config = { - region: 'us-east-1', - endpoint: 'https://s3.us-east-1.amazonaws.com', - credentials: { - accessKeyId: 'test', - secretAccessKey: 'test', - }, -}; - -function createProvider() { - return new S3StorageProvider(config, 'test-bucket'); -} - -test('presignPut should return url and headers', async t => { - const provider = createProvider(); - const result = await provider.presignPut('key', { - contentType: 'text/plain', - }); - - t.truthy(result); - t.true(result!.url.length > 0); - t.true(result!.url.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256')); - t.true(result!.url.includes('X-Amz-SignedHeaders=')); - t.true(result!.url.includes('content-type')); - t.deepEqual(result!.headers, { 'Content-Type': 'text/plain' }); - const now = Date.now(); - t.true(result!.expiresAt.getTime() >= now + SIGNED_URL_EXPIRED * 1000 - 2000); - t.true(result!.expiresAt.getTime() <= now + SIGNED_URL_EXPIRED * 1000 + 2000); -}); - -test('presignUploadPart should return url', async t => { - const provider = createProvider(); - const result = await provider.presignUploadPart('key', 'upload-1', 3); - - t.truthy(result); - t.true(result!.url.length > 0); - t.true(result!.url.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256')); -}); - -test('createMultipartUpload should return uploadId', async t => { - const provider = createProvider(); - let receivedKey: string | undefined; - let receivedMeta: any; - (provider as any).client = { - createMultipartUpload: async (key: string, meta: any) => { - receivedKey = key; - receivedMeta = meta; - return { uploadId: 'upload-1' }; - }, - }; - - const now = Date.now(); - const result = await provider.createMultipartUpload('key', { - contentType: 'text/plain', - }); - - t.is(result?.uploadId, 'upload-1'); - t.true(result!.expiresAt.getTime() >= now + SIGNED_URL_EXPIRED * 1000 - 2000); - t.true(result!.expiresAt.getTime() <= now + SIGNED_URL_EXPIRED * 1000 + 2000); - t.is(receivedKey, 'key'); - t.is(receivedMeta.contentType, 'text/plain'); -}); - -test('completeMultipartUpload should order parts', async t => { - const provider = createProvider(); - let receivedParts: any; - (provider as any).client = { - completeMultipartUpload: async ( - _key: string, - _uploadId: string, - parts: any - ) => { - receivedParts = parts; - }, - }; - - await provider.completeMultipartUpload('key', 'upload-1', [ - { partNumber: 2, etag: 'b' }, - { partNumber: 1, etag: 'a' }, - ]); - t.deepEqual(receivedParts, [ - { partNumber: 1, etag: 'a' }, - { partNumber: 2, etag: 'b' }, - ]); -}); diff --git a/packages/backend/server/src/base/storage/factory.ts b/packages/backend/server/src/base/storage/factory.ts deleted file mode 100644 index 16b63dcfde..0000000000 --- a/packages/backend/server/src/base/storage/factory.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { - StorageProvider, - StorageProviderConfig, - StorageProviders, -} from './providers'; - -@Injectable() -export class StorageProviderFactory { - create(config: StorageProviderConfig): StorageProvider { - const Provider = StorageProviders[config.provider]; - - if (!Provider) { - throw new Error(`Unknown storage provider type: ${config.provider}`); - } - - return new Provider(config.config, config.bucket); - } -} diff --git a/packages/backend/server/src/base/storage/index.ts b/packages/backend/server/src/base/storage/index.ts index d0abd0d92e..1be57e9c5e 100644 --- a/packages/backend/server/src/base/storage/index.ts +++ b/packages/backend/server/src/base/storage/index.ts @@ -1,12 +1,3 @@ -import { Global, Module } from '@nestjs/common'; - -import { StorageProviderFactory } from './factory'; - -@Global() -@Module({ - providers: [StorageProviderFactory], - exports: [StorageProviderFactory], -}) -export class StorageProviderModule {} -export { StorageProviderFactory } from './factory'; export * from './providers'; +export type * from './types'; +export * from './utils'; diff --git a/packages/backend/server/src/base/storage/providers/fs.ts b/packages/backend/server/src/base/storage/providers/fs.ts deleted file mode 100644 index 6483931ed2..0000000000 --- a/packages/backend/server/src/base/storage/providers/fs.ts +++ /dev/null @@ -1,318 +0,0 @@ -import { - accessSync, - constants, - createReadStream, - Dirent, - mkdirSync, - readdirSync, - readFileSync, - rmSync, - statSync, - writeFileSync, -} from 'node:fs'; -import { homedir } from 'node:os'; -import { join, parse } from 'node:path'; -import { Readable } from 'node:stream'; - -import { Logger } from '@nestjs/common'; - -import { - BlobInputType, - GetObjectMetadata, - ListObjectsMetadata, - PutObjectMetadata, - StorageProvider, -} from './provider'; -import { autoMetadata, toBuffer } from './utils'; - -function normalizeStorageKey(key: string): string { - const normalized = key.replaceAll('\\', '/'); - const segments = normalized.split('/'); - - if ( - !normalized || - normalized.startsWith('/') || - segments.some(segment => !segment || segment === '.' || segment === '..') - ) { - throw new Error(`Invalid storage key: ${key}`); - } - - return segments.join('/'); -} - -function normalizeStoragePrefix(prefix: string): string { - const normalized = prefix.replaceAll('\\', '/'); - if (!normalized) { - return normalized; - } - if (normalized.startsWith('/')) { - throw new Error(`Invalid storage prefix: ${prefix}`); - } - - const segments = normalized.split('/'); - const lastSegment = segments.pop(); - - if ( - lastSegment === undefined || - segments.some(segment => !segment || segment === '.' || segment === '..') || - lastSegment === '.' || - lastSegment === '..' - ) { - throw new Error(`Invalid storage prefix: ${prefix}`); - } - - if (lastSegment === '') { - return `${segments.join('/')}/`; - } - - return [...segments, lastSegment].join('/'); -} - -export interface FsStorageConfig { - path: string; -} - -export class FsStorageProvider implements StorageProvider { - private readonly path: string; - private readonly logger: Logger; - - readonly type = 'fs'; - - constructor( - config: FsStorageConfig, - public readonly bucket: string - ) { - this.path = config.path.startsWith('~/') - ? join(homedir(), config.path.slice(2), bucket) - : join(config.path, bucket); - this.ensureAvailability(); - - this.logger = new Logger(`${FsStorageProvider.name}:${bucket}`); - } - - async put( - key: string, - body: BlobInputType, - metadata: PutObjectMetadata = {} - ): Promise { - key = normalizeStorageKey(key); - const blob = await toBuffer(body); - - // write object - this.writeObject(key, blob); - // write metadata - await this.writeMetadata(key, blob, metadata); - this.logger.verbose(`Object \`${key}\` put`); - } - - async head(key: string) { - key = normalizeStorageKey(key); - const metadata = this.readMetadata(key); - if (!metadata) { - this.logger.verbose(`Object \`${key}\` not found`); - return undefined; - } - return metadata; - } - - async get(key: string): Promise<{ - body?: Readable; - metadata?: GetObjectMetadata; - }> { - key = normalizeStorageKey(key); - - try { - const metadata = this.readMetadata(key); - const stream = this.readObject(this.join(key)); - this.logger.verbose(`Read object \`${key}\``); - return { - body: stream, - metadata, - }; - } catch (e) { - this.logger.error(`Failed to read object \`${key}\``, e); - return {}; - } - } - - async list(prefix?: string): Promise { - // prefix cases: - // - `undefined`: list all objects - // - `a/b`: list objects under dir `a` with prefix `b`, `b` might be a dir under `a` as well. - // - `a/b/` list objects under dir `a/b` - - // read dir recursively and filter out '.metadata.json' files - let dir = this.path; - if (prefix) { - prefix = normalizeStoragePrefix(prefix); - const parts = prefix.split(/[/\\]/); - // for prefix `a/b/c`, move `a/b` to dir and `c` to key prefix - if (parts.length > 1) { - dir = join(dir, ...parts.slice(0, -1)); - prefix = parts[parts.length - 1]; - } - } - - const results: ListObjectsMetadata[] = []; - async function getFiles(dir: string, prefix?: string): Promise { - try { - const entries: Dirent[] = readdirSync(dir, { withFileTypes: true }); - - for (const entry of entries) { - const res = join(dir, entry.name); - - if (entry.isDirectory()) { - if (!prefix || entry.name.startsWith(prefix)) { - await getFiles(res); - } - } else if ( - (!prefix || entry.name.startsWith(prefix)) && - !entry.name.endsWith('.metadata.json') - ) { - const stat = statSync(res); - results.push({ - key: res, - lastModified: stat.mtime, - contentLength: stat.size, - }); - } - } - } catch { - // failed to read dir, stop recursion - } - } - - await getFiles(dir, prefix); - - // trim path with `this.path` prefix - results.forEach(r => (r.key = r.key.slice(this.path.length + 1))); - - return results; - } - - delete(key: string): Promise { - key = normalizeStorageKey(key); - - try { - rmSync(this.join(key), { force: true }); - rmSync(this.join(`${key}.metadata.json`), { force: true }); - } catch (e) { - throw new Error(`Failed to delete object \`${key}\``, { - cause: e, - }); - } - - this.logger.verbose(`Object \`${key}\` deleted`); - - return Promise.resolve(); - } - - ensureAvailability() { - // check stats - const stats = statSync(this.path, { - throwIfNoEntry: false, - }); - - // not existing, create it - if (!stats) { - try { - mkdirSync(this.path, { recursive: true }); - } catch (e) { - throw new Error( - `Failed to create target directory for fs storage provider: ${this.path}`, - { - cause: e, - } - ); - } - } else if (stats.isDirectory()) { - // the target directory has already existed, check if it is readable & writable - try { - accessSync(this.path, constants.W_OK | constants.R_OK); - } catch (e) { - throw new Error( - `The target directory for fs storage provider has already existed, but it is not readable & writable: ${this.path}`, - { - cause: e, - } - ); - } - } else if (stats.isFile()) { - throw new Error( - `The target directory for fs storage provider is a file: ${this.path}` - ); - } - } - - private join(...paths: string[]) { - return join(this.path, ...paths); - } - - private readObject(file: string): Readable | undefined { - const state = statSync(file, { throwIfNoEntry: false }); - - if (state?.isFile()) { - return createReadStream(file); - } - - return undefined; - } - - private writeObject(key: string, blob: Buffer) { - const path = this.join(key); - mkdirSync(parse(path).dir, { recursive: true }); - writeFileSync(path, blob); - } - - private async writeMetadata( - key: string, - blob: Buffer, - raw: PutObjectMetadata - ) { - try { - const metadata = autoMetadata(blob, raw); - - if (raw.checksumCRC32 && metadata.checksumCRC32 !== raw.checksumCRC32) { - throw new Error( - 'The checksum of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.' - ); - } - - if (raw.contentLength && metadata.contentLength !== raw.contentLength) { - throw new Error( - 'The content length of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.' - ); - } - - writeFileSync( - this.join(`${key}.metadata.json`), - JSON.stringify({ - ...metadata, - lastModified: Date.now(), - }) - ); - } catch (e) { - this.logger.warn(`Failed to write metadata of object \`${key}\``, e); - } - } - - private readMetadata(key: string): GetObjectMetadata | undefined { - try { - const raw = JSON.parse( - readFileSync(this.join(`${key}.metadata.json`), { - encoding: 'utf-8', - }) - ); - - return { - ...raw, - lastModified: new Date(raw.lastModified), - expires: raw.expires ? new Date(raw.expires) : undefined, - }; - } catch (e) { - this.logger.warn(`Failed to read metadata of object \`${key}\``, e); - - return; - } - } -} diff --git a/packages/backend/server/src/base/storage/providers/index.ts b/packages/backend/server/src/base/storage/providers/index.ts index 5b2707bf3e..9f7c3fb16f 100644 --- a/packages/backend/server/src/base/storage/providers/index.ts +++ b/packages/backend/server/src/base/storage/providers/index.ts @@ -1,20 +1,45 @@ -import { Type } from '@nestjs/common'; - import { JSONSchema } from '../../config'; -import { FsStorageConfig, FsStorageProvider } from './fs'; -import { StorageProvider } from './provider'; -import { R2_JURISDICTIONS, R2StorageConfig, R2StorageProvider } from './r2'; -import { S3StorageConfig, S3StorageProvider } from './s3'; -export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2'; -export const StorageProviders: Record< - StorageProviderName, - Type -> = { - fs: FsStorageProvider, - 'aws-s3': S3StorageProvider, - 'cloudflare-r2': R2StorageProvider, -}; +export type StorageProviderName = + | 'fs' + | 'aws-s3' + | 'cloudflare-r2' + | 'assetpack'; + +export interface FsStorageConfig { + path: string; +} + +export type AssetpackStorageConfig = FsStorageConfig; + +export interface S3StorageConfig { + endpoint?: string; + region: string; + credentials?: { + accessKeyId?: string; + secretAccessKey?: string; + sessionToken?: string; + }; + forcePathStyle?: boolean; + requestTimeoutMs?: number; + minPartSize?: number; + presign?: { + expiresInSeconds?: number; + signContentTypeForPut?: boolean; + }; +} + +export const R2_JURISDICTIONS = ['default', 'eu'] as const; + +export interface R2StorageConfig extends Omit { + accountId: string; + jurisdiction?: (typeof R2_JURISDICTIONS)[number]; + usePresignedURL?: { + enabled: boolean; + urlPrefix?: string; + signKey?: string; + }; +} export type StorageProviderConfig = { bucket: string } & ( | { @@ -29,6 +54,10 @@ export type StorageProviderConfig = { bucket: string } & ( provider: 'cloudflare-r2'; config: R2StorageConfig; } + | { + provider: 'assetpack'; + config: AssetpackStorageConfig; + } ); const S3ConfigSchema: JSONSchema = { @@ -186,16 +215,35 @@ export const StorageJSONSchema: JSONSchema = { }, }, }, + { + type: 'object', + properties: { + provider: { + type: 'string', + enum: ['assetpack'], + }, + bucket: { + type: 'string', + }, + config: { + type: 'object', + properties: { + path: { + type: 'string', + }, + }, + }, + }, + }, ], }; -export type * from './provider'; +export type * from '../types'; export { applyAttachHeaders, - autoMetadata, PROXY_MULTIPART_PATH, PROXY_UPLOAD_PATH, sniffMime, STORAGE_PROXY_ROOT, toBuffer, -} from './utils'; +} from '../utils'; diff --git a/packages/backend/server/src/base/storage/providers/provider.ts b/packages/backend/server/src/base/storage/providers/provider.ts deleted file mode 100644 index 82ee5dbbc6..0000000000 --- a/packages/backend/server/src/base/storage/providers/provider.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { Readable } from 'node:stream'; - -export interface GetObjectMetadata { - /** - * @default 'application/octet-stream' - */ - contentType: string; - contentLength: number; - lastModified: Date; - checksumCRC32?: string; -} - -export interface PutObjectMetadata { - contentType?: string; - contentLength?: number; - checksumCRC32?: string; -} - -export interface ListObjectsMetadata { - key: string; - lastModified: Date; - contentLength: number; -} - -export type BlobInputType = Buffer | Readable | string; -export type BlobOutputType = Readable; - -export interface PresignedUpload { - url: string; - headers?: Record; - expiresAt: Date; -} - -export interface MultipartUploadInit { - uploadId: string; - expiresAt: Date; -} - -export interface MultipartUploadPart { - partNumber: number; - etag: string; -} - -export interface StorageProvider { - put( - key: string, - body: BlobInputType, - metadata?: PutObjectMetadata - ): Promise; - presignPut?( - key: string, - metadata?: PutObjectMetadata - ): Promise; - createMultipartUpload?( - key: string, - metadata?: PutObjectMetadata - ): Promise; - presignUploadPart?( - key: string, - uploadId: string, - partNumber: number - ): Promise; - listMultipartUploadParts?( - key: string, - uploadId: string - ): Promise; - completeMultipartUpload?( - key: string, - uploadId: string, - parts: MultipartUploadPart[] - ): Promise; - abortMultipartUpload?(key: string, uploadId: string): Promise; - head(key: string): Promise; - get( - key: string, - signedUrl?: boolean - ): Promise<{ - redirectUrl?: string; - body?: BlobOutputType; - metadata?: GetObjectMetadata; - }>; - list(prefix?: string): Promise; - delete(key: string): Promise; -} diff --git a/packages/backend/server/src/base/storage/providers/r2.ts b/packages/backend/server/src/base/storage/providers/r2.ts deleted file mode 100644 index 520c31598f..0000000000 --- a/packages/backend/server/src/base/storage/providers/r2.ts +++ /dev/null @@ -1,251 +0,0 @@ -import assert from 'node:assert'; -import { Readable } from 'node:stream'; - -import { Logger } from '@nestjs/common'; - -import { - GetObjectMetadata, - PresignedUpload, - PutObjectMetadata, -} from './provider'; -import { S3StorageConfig, S3StorageProvider } from './s3'; -import { - PROXY_MULTIPART_PATH, - PROXY_UPLOAD_PATH, - SIGNED_URL_EXPIRED, -} from './utils'; - -export const R2_JURISDICTIONS = ['eu'] as const; -type R2Jurisdiction = (typeof R2_JURISDICTIONS)[number]; - -export interface R2StorageConfig extends Omit< - S3StorageConfig, - 'endpoint' | 'forcePathStyle' -> { - accountId: string; - jurisdiction?: R2Jurisdiction; - usePresignedURL?: { - enabled: boolean; - urlPrefix?: string; - signKey?: string; - }; -} - -export class R2StorageProvider extends S3StorageProvider { - private readonly encoder = new TextEncoder(); - private readonly key: Uint8Array; - - constructor( - private readonly config: R2StorageConfig, - bucket: string - ) { - assert(config.accountId, 'accountId is required for R2 storage provider'); - const account = config.jurisdiction - ? `${config.accountId}.${config.jurisdiction}` - : config.accountId; - const endpoint = `https://${account}.r2.cloudflarestorage.com`; - - super( - { - ...config, - forcePathStyle: true, - endpoint, - }, - bucket - ); - this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`); - this.key = this.encoder.encode(config.usePresignedURL?.signKey ?? ''); - } - - private get shouldUseProxyUpload() { - const { usePresignedURL } = this.config; - return ( - !!usePresignedURL?.enabled && - !!usePresignedURL.signKey && - this.key.length > 0 - ); - } - - private parseWorkspaceKey(fullKey: string) { - const [workspaceId, ...rest] = fullKey.split('/'); - if (!workspaceId || rest.length !== 1) { - return null; - } - return { workspaceId, key: rest.join('/') }; - } - - private async signPayload(payload: string) { - const key = await crypto.subtle.importKey( - 'raw', - this.key, - { name: 'HMAC', hash: 'SHA-256' }, - false, - ['sign', 'verify'] - ); - const mac = await crypto.subtle.sign( - 'HMAC', - key, - this.encoder.encode(payload) - ); - - return Buffer.from(mac).toString('base64'); - } - - private async signUrl(url: URL): Promise { - const timestamp = Math.floor(Date.now() / 1000); - const base64Mac = await this.signPayload(`${url.pathname}${timestamp}`); - url.searchParams.set('sign', `${timestamp}-${base64Mac}`); - return url.toString(); - } - - private async createProxyUrl( - path: string, - canonicalFields: (string | number | undefined)[], - query: Record - ) { - const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED; - const canonical = [ - path, - ...canonicalFields.map(field => - field === undefined ? '' : field.toString() - ), - exp.toString(), - ].join('\n'); - const token = await this.signPayload(canonical); - - const url = new URL(`http://localhost${path}`); - for (const [key, value] of Object.entries(query)) { - if (value === undefined) continue; - url.searchParams.set(key, value.toString()); - } - url.searchParams.set('exp', exp.toString()); - url.searchParams.set('token', `${exp}-${token}`); - - return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) }; - } - - override async presignPut( - key: string, - metadata: PutObjectMetadata = {} - ): Promise { - if (!this.shouldUseProxyUpload) { - return super.presignPut(key, metadata); - } - - const parsed = this.parseWorkspaceKey(key); - if (!parsed) { - return super.presignPut(key, metadata); - } - - const contentType = metadata.contentType ?? 'application/octet-stream'; - const { url, expiresAt } = await this.createProxyUrl( - PROXY_UPLOAD_PATH, - [parsed.workspaceId, parsed.key, contentType, metadata.contentLength], - { - workspaceId: parsed.workspaceId, - key: parsed.key, - contentType, - contentLength: metadata.contentLength, - } - ); - - return { - url, - headers: { 'Content-Type': contentType }, - expiresAt, - }; - } - - override async presignUploadPart( - key: string, - uploadId: string, - partNumber: number - ): Promise { - if (!this.shouldUseProxyUpload) { - return super.presignUploadPart(key, uploadId, partNumber); - } - - const parsed = this.parseWorkspaceKey(key); - if (!parsed) { - return super.presignUploadPart(key, uploadId, partNumber); - } - - return this.createProxyUrl( - PROXY_MULTIPART_PATH, - [parsed.workspaceId, parsed.key, uploadId, partNumber], - { - workspaceId: parsed.workspaceId, - key: parsed.key, - uploadId, - partNumber, - } - ); - } - - async proxyPutObject( - key: string, - body: Readable | Buffer | Uint8Array | string, - options: { contentType?: string; contentLength?: number } = {} - ) { - return this.client.putObject(key, this.normalizeBody(body), { - contentType: options.contentType, - contentLength: options.contentLength, - }); - } - - async proxyUploadPart( - key: string, - uploadId: string, - partNumber: number, - body: Readable | Buffer | Uint8Array | string, - options: { contentLength?: number } = {} - ) { - const result = await this.client.uploadPart( - key, - uploadId, - partNumber, - this.normalizeBody(body), - { contentLength: options.contentLength } - ); - - return result.etag; - } - - private normalizeBody(body: Readable | Buffer | Uint8Array | string) { - // s3mini does not accept Node.js Readable directly. - // Convert it to Web ReadableStream for compatibility. - if (body instanceof Readable) { - return Readable.toWeb(body); - } else if (typeof body === 'string') { - return this.encoder.encode(body); - } - return body; - } - - override async get( - key: string, - signedUrl?: boolean - ): Promise<{ - body?: Readable; - metadata?: GetObjectMetadata; - redirectUrl?: string; - }> { - const { usePresignedURL: { enabled, urlPrefix } = {} } = this.config; - if (signedUrl && enabled && urlPrefix) { - const metadata = await this.head(key); - const url = await this.signUrl(new URL(`/${key}`, urlPrefix)); - if (metadata) { - return { - redirectUrl: url.toString(), - metadata, - }; - } - - // object not found - return {}; - } - - // fallback to s3 get - return super.get(key, signedUrl); - } -} diff --git a/packages/backend/server/src/base/storage/providers/s3.ts b/packages/backend/server/src/base/storage/providers/s3.ts deleted file mode 100644 index 1e99378223..0000000000 --- a/packages/backend/server/src/base/storage/providers/s3.ts +++ /dev/null @@ -1,363 +0,0 @@ -/* oxlint-disable @typescript-eslint/no-non-null-assertion */ -import { Readable } from 'node:stream'; - -import type { - S3CompatClient, - S3CompatConfig, - S3CompatCredentials, -} from '@affine/s3-compat'; -import { createS3CompatClient } from '@affine/s3-compat'; -import { Logger } from '@nestjs/common'; - -import { - BlobInputType, - GetObjectMetadata, - ListObjectsMetadata, - MultipartUploadInit, - MultipartUploadPart, - PresignedUpload, - PutObjectMetadata, - StorageProvider, -} from './provider'; -import { autoMetadata, SIGNED_URL_EXPIRED, toBuffer } from './utils'; - -export interface S3StorageConfig { - endpoint?: string; - region: string; - credentials: S3CompatCredentials; - forcePathStyle?: boolean; - requestTimeoutMs?: number; - minPartSize?: number; - presign?: { - expiresInSeconds?: number; - signContentTypeForPut?: boolean; - }; - usePresignedURL?: { - enabled: boolean; - }; -} - -function resolveEndpoint(config: S3StorageConfig) { - if (config.endpoint) { - return config.endpoint; - } - if (config.region === 'us-east-1') { - return 'https://s3.amazonaws.com'; - } - return `https://s3.${config.region}.amazonaws.com`; -} - -function joinPath(basePath: string, suffix: string) { - const trimmedBase = basePath.endsWith('/') ? basePath.slice(0, -1) : basePath; - const trimmedSuffix = suffix.startsWith('/') ? suffix.slice(1) : suffix; - if (!trimmedBase) { - return `/${trimmedSuffix}`; - } - if (!trimmedSuffix) { - return trimmedBase; - } - return `${trimmedBase}/${trimmedSuffix}`; -} - -function composeEndpointUrl(config: S3CompatConfig) { - const url = new URL(config.endpoint); - if (config.forcePathStyle) { - const firstSegment = url.pathname.split('/').find(Boolean); - if (firstSegment !== config.bucket) { - url.pathname = joinPath(url.pathname, config.bucket); - } - return url.toString(); - } - - const firstSegment = url.pathname.split('/').find(Boolean); - const hostHasBucket = url.hostname.startsWith(`${config.bucket}.`); - const pathHasBucket = firstSegment === config.bucket; - if (!hostHasBucket && !pathHasBucket) { - url.hostname = `${config.bucket}.${url.hostname}`; - } - return url.toString(); -} - -export class S3StorageProvider implements StorageProvider { - protected logger: Logger; - protected client: S3CompatClient; - private readonly usePresignedURL: boolean; - private readonly endpoint: string; - - get endpointUrl() { - return this.endpoint; - } - - constructor( - config: S3StorageConfig, - public readonly bucket: string - ) { - const { usePresignedURL, presign, credentials, ...clientConfig } = config; - - const compatConfig: S3CompatConfig = { - ...clientConfig, - endpoint: resolveEndpoint(config), - bucket, - requestTimeoutMs: clientConfig.requestTimeoutMs ?? 60_000, - presign: { - expiresInSeconds: presign?.expiresInSeconds ?? SIGNED_URL_EXPIRED, - signContentTypeForPut: presign?.signContentTypeForPut ?? true, - }, - }; - - this.endpoint = composeEndpointUrl(compatConfig); - this.client = createS3CompatClient(compatConfig, credentials); - this.usePresignedURL = usePresignedURL?.enabled ?? false; - this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`); - } - - async put( - key: string, - body: BlobInputType, - metadata: PutObjectMetadata = {} - ): Promise { - const blob = await toBuffer(body); - - metadata = autoMetadata(blob, metadata); - - try { - await this.client.putObject(key, blob, { - contentType: metadata.contentType, - contentLength: metadata.contentLength, - }); - - this.logger.verbose(`Object \`${key}\` put`); - } catch (e) { - this.logger.error( - `Failed to put object (${JSON.stringify({ - key, - bucket: this.bucket, - metadata, - })})` - ); - throw e; - } - } - - async presignPut( - key: string, - metadata: PutObjectMetadata = {} - ): Promise { - try { - const contentType = metadata.contentType ?? 'application/octet-stream'; - const result = await this.client.presignPutObject(key, { contentType }); - - return { - url: result.url, - headers: result.headers, - expiresAt: result.expiresAt, - }; - } catch (e) { - this.logger.error( - `Failed to presign put object (${JSON.stringify({ - key, - bucket: this.bucket, - metadata, - })}` - ); - throw e; - } - } - - async createMultipartUpload( - key: string, - metadata: PutObjectMetadata = {} - ): Promise { - try { - const contentType = metadata.contentType ?? 'application/octet-stream'; - const response = await this.client.createMultipartUpload(key, { - contentType, - }); - - if (!response.uploadId) { - return; - } - - return { - uploadId: response.uploadId, - expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000), - }; - } catch (e) { - this.logger.error( - `Failed to create multipart upload (${JSON.stringify({ - key, - bucket: this.bucket, - metadata, - })}` - ); - throw e; - } - } - - async presignUploadPart( - key: string, - uploadId: string, - partNumber: number - ): Promise { - try { - const result = await this.client.presignUploadPart( - key, - uploadId, - partNumber - ); - - return { - url: result.url, - expiresAt: result.expiresAt, - }; - } catch (e) { - this.logger.error( - `Failed to presign upload part (${JSON.stringify({ key, bucket: this.bucket, uploadId, partNumber })}` - ); - throw e; - } - } - - async listMultipartUploadParts( - key: string, - uploadId: string - ): Promise { - try { - return await this.client.listParts(key, uploadId); - } catch (e) { - this.logger.error(`Failed to list multipart upload parts for \`${key}\``); - throw e; - } - } - - async completeMultipartUpload( - key: string, - uploadId: string, - parts: MultipartUploadPart[] - ): Promise { - try { - const orderedParts = [...parts].sort( - (left, right) => left.partNumber - right.partNumber - ); - - await this.client.completeMultipartUpload(key, uploadId, orderedParts); - } catch (e) { - this.logger.error(`Failed to complete multipart upload for \`${key}\``); - throw e; - } - } - - async abortMultipartUpload(key: string, uploadId: string): Promise { - try { - await this.client.abortMultipartUpload(key, uploadId); - } catch (e) { - this.logger.error(`Failed to abort multipart upload for \`${key}\``); - throw e; - } - } - - async head(key: string) { - try { - const obj = await this.client.headObject(key); - if (!obj) { - this.logger.verbose(`Object \`${key}\` not found`); - return undefined; - } - - return { - contentType: obj.contentType ?? 'application/octet-stream', - contentLength: obj.contentLength ?? 0, - lastModified: obj.lastModified ?? new Date(0), - checksumCRC32: obj.checksumCRC32, - }; - } catch (e) { - this.logger.error(`Failed to head object \`${key}\``); - throw e; - } - } - - async get( - key: string, - signedUrl?: boolean - ): Promise<{ - body?: Readable; - metadata?: GetObjectMetadata; - redirectUrl?: string; - }> { - try { - if (this.usePresignedURL && signedUrl) { - const metadata = await this.head(key); - if (metadata) { - const result = await this.client.presignGetObject(key); - - return { - redirectUrl: result.url, - metadata, - }; - } - - // object not found - return {}; - } - - const obj = await this.client.getObjectResponse(key); - if (!obj || !obj.body) { - this.logger.verbose(`Object \`${key}\` not found`); - return {}; - } - - const contentType = obj.headers.get('content-type') ?? undefined; - const contentLengthHeader = obj.headers.get('content-length'); - const contentLength = contentLengthHeader - ? Number(contentLengthHeader) - : undefined; - const lastModifiedHeader = obj.headers.get('last-modified'); - const lastModified = lastModifiedHeader - ? new Date(lastModifiedHeader) - : undefined; - - this.logger.verbose(`Read object \`${key}\``); - return { - body: Readable.fromWeb(obj.body), - metadata: { - contentType: contentType ?? 'application/octet-stream', - contentLength: contentLength ?? 0, - lastModified: lastModified ?? new Date(0), - checksumCRC32: obj.headers.get('x-amz-checksum-crc32') ?? undefined, - }, - }; - } catch (e) { - this.logger.error(`Failed to read object \`${key}\``); - throw e; - } - } - - async list(prefix?: string): Promise { - try { - const result = await this.client.listObjectsV2(prefix); - - this.logger.verbose( - `List ${result.length} objects with prefix \`${prefix}\`` - ); - return result; - } catch (e) { - this.logger.error(`Failed to list objects with prefix \`${prefix}\``); - throw e; - } - } - - async delete(key: string): Promise { - try { - await this.client.deleteObject(key); - - this.logger.verbose(`Deleted object \`${key}\``); - } catch (e) { - this.logger.error(`Failed to delete object \`${key}\``, { - bucket: this.bucket, - key, - cause: e, - }); - throw e; - } - } -} diff --git a/packages/backend/server/src/base/storage/types.ts b/packages/backend/server/src/base/storage/types.ts new file mode 100644 index 0000000000..8c088fd1b0 --- /dev/null +++ b/packages/backend/server/src/base/storage/types.ts @@ -0,0 +1,32 @@ +import type { Readable } from 'node:stream'; + +export interface GetObjectMetadata { + /** + * @default 'application/octet-stream' + */ + contentType: string; + contentLength: number; + lastModified: Date; + checksumCRC32?: string; +} + +export interface PutObjectMetadata { + contentType?: string; + contentLength?: number; + checksumCRC32?: string; +} + +export interface ListObjectsMetadata { + key: string; + lastModified: Date; + contentLength: number; +} + +export type BlobInputType = Buffer | Readable | string; +export type BlobOutputType = Readable; + +export interface PresignedUpload { + url: string; + headers?: Record; + expiresAt: Date; +} diff --git a/packages/backend/server/src/base/storage/providers/utils.ts b/packages/backend/server/src/base/storage/utils.ts similarity index 75% rename from packages/backend/server/src/base/storage/providers/utils.ts rename to packages/backend/server/src/base/storage/utils.ts index cda601bc76..903b9f192d 100644 --- a/packages/backend/server/src/base/storage/providers/utils.ts +++ b/packages/backend/server/src/base/storage/utils.ts @@ -1,11 +1,10 @@ import { Readable } from 'node:stream'; -import { crc32 } from '@node-rs/crc32'; import type { Response } from 'express'; import { getStreamAsBuffer } from 'get-stream'; -import { getMime } from '../../../native'; -import { BlobInputType, PutObjectMetadata } from './provider'; +import { getMime } from '../../native'; +import type { BlobInputType } from './types'; export async function toBuffer(input: BlobInputType): Promise { return input instanceof Readable @@ -15,35 +14,6 @@ export async function toBuffer(input: BlobInputType): Promise { : Buffer.from(input as string); } -export function autoMetadata( - blob: Buffer, - raw: PutObjectMetadata = {} -): PutObjectMetadata { - const metadata = { - ...raw, - }; - - if (!metadata.contentLength) { - metadata.contentLength = blob.byteLength; - } - - try { - // checksum - if (!metadata.checksumCRC32) { - metadata.checksumCRC32 = crc32(blob).toString(16); - } - - // mime type - if (!metadata.contentType) { - metadata.contentType = getMime(blob); - } - } catch { - // noop - } - - return metadata; -} - const DANGEROUS_INLINE_MIME_PREFIXES = [ 'text/html', 'application/xhtml+xml', diff --git a/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts b/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts index 36b932dbb3..844767692a 100644 --- a/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts +++ b/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts @@ -12,7 +12,6 @@ test('backend-runtime provider starts once, runs migrations once, and reports he health: Sinon.stub().resolves({ started: true, databaseConnected: true, - objectStorageConfigured: true, }), }; (provider as any).runtime = runtime; @@ -25,7 +24,6 @@ test('backend-runtime provider starts once, runs migrations once, and reports he t.is(runtime.start.callCount, 2); t.is(runtime.runMigrations.callCount, 1); t.true(health.databaseConnected); - t.true(health.objectStorageConfigured); t.is(runtime.stop.callCount, 1); }); diff --git a/packages/backend/server/src/core/backend-runtime/index.ts b/packages/backend/server/src/core/backend-runtime/index.ts index c9245ee54c..19ecf7a3e5 100644 --- a/packages/backend/server/src/core/backend-runtime/index.ts +++ b/packages/backend/server/src/core/backend-runtime/index.ts @@ -1,17 +1,12 @@ import { Global, Module } from '@nestjs/common'; -import { BackendRuntimeBlobJob } from './blob-job'; import { BackendRuntimeHousekeepingJob } from './job'; import { BackendRuntimeProvider } from './provider'; @Global() @Module({ - providers: [ - BackendRuntimeProvider, - BackendRuntimeBlobJob, - BackendRuntimeHousekeepingJob, - ], - exports: [BackendRuntimeProvider, BackendRuntimeBlobJob], + providers: [BackendRuntimeProvider, BackendRuntimeHousekeepingJob], + exports: [BackendRuntimeProvider], }) export class BackendRuntimeModule {} diff --git a/packages/backend/server/src/core/backend-runtime/provider.ts b/packages/backend/server/src/core/backend-runtime/provider.ts index 30c85dc2c6..91c6f33235 100644 --- a/packages/backend/server/src/core/backend-runtime/provider.ts +++ b/packages/backend/server/src/core/backend-runtime/provider.ts @@ -30,9 +30,7 @@ export class BackendRuntimeProvider await this.runtime.start(); await this.runMigrationsOnce(); const health = await this.runtime.health(); - this.logger.log( - `backend runtime started: db=${health.databaseConnected} objectStorage=${health.objectStorageConfigured}` - ); + this.logger.log(`backend runtime started: db=${health.databaseConnected}`); } async stop() { @@ -44,18 +42,6 @@ export class BackendRuntimeProvider return await this.runtime.health(); } - async cleanupExpiredPendingBlobs(cutoffMs: number, limit: number) { - return await this.measured('cleanupExpiredPendingBlobs', rt => - rt.cleanupExpiredPendingBlobs(cutoffMs, limit) - ); - } - - async releaseDeletedBlobs(workspaceId: string, limit: number) { - return await this.measured('releaseDeletedBlobs', rt => - rt.releaseDeletedBlobs(workspaceId, limit) - ); - } - async cleanupExpiredSnapshotHistories(limit: number) { return await this.measured('cleanupExpiredSnapshotHistories', rt => rt.cleanupExpiredSnapshotHistories(limit) @@ -80,41 +66,6 @@ export class BackendRuntimeProvider ); } - async backfillMissingBlobMetadata( - workspaceId: string | null | undefined, - limit: number - ) { - return await this.measured('backfillMissingBlobMetadata', rt => - rt.backfillMissingBlobMetadata(workspaceId, limit) - ); - } - - async rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number) { - return await this.measured('rebuildWorkspaceDocBlobRefs', rt => - rt.rebuildWorkspaceDocBlobRefs(workspaceId, limit) - ); - } - - async planUnreferencedWorkspaceBlobs( - workspaceId: string, - gracePeriodDays: number, - limit: number - ) { - return await this.measured('planUnreferencedWorkspaceBlobs', rt => - rt.planUnreferencedWorkspaceBlobs(workspaceId, gracePeriodDays, limit) - ); - } - - async executeBlobCleanupCandidates( - runId: string, - gracePeriodDays: number, - limit: number - ) { - return await this.measured('executeBlobCleanupCandidates', rt => - rt.executeBlobCleanupCandidates(runId, gracePeriodDays, limit) - ); - } - private async measured( method: string, fn: (runtime: RuntimeInstance) => Promise diff --git a/packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts b/packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts new file mode 100644 index 0000000000..aeefd0a947 --- /dev/null +++ b/packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts @@ -0,0 +1,42 @@ +import test from 'ava'; +import Sinon from 'sinon'; + +import { StorageRuntimeProvider } from '../provider'; + +function createProvider() { + const provider = new StorageRuntimeProvider(); + const runtime = { + start: Sinon.stub().resolves(), + stop: Sinon.stub().resolves(), + runMigrations: Sinon.stub().resolves(), + health: Sinon.stub().resolves({ + started: true, + databaseConnected: true, + provider: 'fs', + }), + }; + (provider as any).runtime = runtime; + return { provider, runtime }; +} + +test('storage-runtime provider restarts on storage config changes', async t => { + const { provider, runtime } = createProvider(); + + await provider.start(); + await provider.onConfigChanged({ updates: { storages: {} } }); + + t.is(runtime.stop.callCount, 1); + t.is(runtime.start.callCount, 2); + t.is(runtime.runMigrations.callCount, 2); +}); + +test('storage-runtime provider ignores unrelated config changes', async t => { + const { provider, runtime } = createProvider(); + + await provider.start(); + await provider.onConfigChanged({ updates: { flags: {} } }); + + t.is(runtime.stop.callCount, 0); + t.is(runtime.start.callCount, 1); + t.is(runtime.runMigrations.callCount, 1); +}); diff --git a/packages/backend/server/src/core/storage-runtime/index.ts b/packages/backend/server/src/core/storage-runtime/index.ts new file mode 100644 index 0000000000..ed37d37b08 --- /dev/null +++ b/packages/backend/server/src/core/storage-runtime/index.ts @@ -0,0 +1,15 @@ +import { Global, Module } from '@nestjs/common'; + +import { StorageRuntimeProvider } from './provider'; + +@Global() +@Module({ + providers: [StorageRuntimeProvider], + exports: [StorageRuntimeProvider], +}) +export class StorageRuntimeModule {} + +export { + type StorageRuntimeGetObjectResult, + StorageRuntimeProvider, +} from './provider'; diff --git a/packages/backend/server/src/core/storage-runtime/provider.ts b/packages/backend/server/src/core/storage-runtime/provider.ts new file mode 100644 index 0000000000..905ae99d38 --- /dev/null +++ b/packages/backend/server/src/core/storage-runtime/provider.ts @@ -0,0 +1,350 @@ +import { Readable } from 'node:stream'; + +import { + Injectable, + Logger, + type OnApplicationBootstrap, + type OnApplicationShutdown, +} from '@nestjs/common'; + +import type { + BlobOutputType, + GetObjectMetadata, + ListObjectsMetadata, + PresignedUpload, + PutObjectMetadata, +} from '../../base'; +import { OnEvent } from '../../base'; +import { wrapCallMetric } from '../../base/metrics'; +import { + type RuntimeObjectGetResult, + type RuntimeObjectListEntry, + type RuntimeObjectMetadata, + type RuntimePresignedObjectRequest, + type StorageProviderCapabilities, + StorageRuntime, + type StorageRuntimeHealth, +} from '../../native'; + +type RuntimeInstance = InstanceType; + +@Injectable() +export class StorageRuntimeProvider + implements OnApplicationBootstrap, OnApplicationShutdown +{ + private readonly logger = new Logger(StorageRuntimeProvider.name); + private readonly runtime: RuntimeInstance = new StorageRuntime(); + private migrationsStarted = false; + + async onApplicationBootstrap() { + await this.start(); + } + + async onApplicationShutdown() { + await this.stop(); + } + + async start() { + await this.runtime.start(); + await this.runMigrationsOnce(); + const health = await this.runtime.health(); + this.logger.log( + `storage runtime started: db=${health.databaseConnected} provider=${health.provider ?? 'none'}` + ); + } + + async stop() { + await this.runtime.stop(); + this.logger.log('storage runtime stopped'); + } + + @OnEvent('config.init') + async onConfigInit() { + await this.start(); + } + + @OnEvent('config.changed') + async onConfigChanged({ updates }: Events['config.changed']) { + if (!('storages' in updates) && !('db' in updates)) { + return; + } + await this.restart(); + } + + async health(): Promise { + return await this.runtime.health(); + } + + async providerCapabilities( + scope: string + ): Promise { + return await this.measured('providerCapabilities', rt => + rt.providerCapabilities(scope) + ); + } + + async putObject( + scope: string, + key: string, + body: Buffer, + metadata?: PutObjectMetadata + ) { + const result = await this.measured('putObject', rt => + rt.putObject(scope, key, body, toRuntimeMetadata(metadata)) + ); + return fromRuntimeMetadata(result); + } + + async headObject(scope: string, key: string) { + const metadata = await this.measured('headObject', rt => + rt.headObject(scope, key) + ); + return metadata ? fromRuntimeMetadata(metadata) : undefined; + } + + async getObject( + scope: string, + key: string + ): Promise { + const result = await this.measured('getObject', rt => + rt.getObject(scope, key) + ); + return result ? fromRuntimeGetResult(result) : {}; + } + + async listObjects(scope: string, prefix?: string) { + const entries = await this.measured('listObjects', rt => + rt.listObjects(scope, prefix) + ); + return entries.map(fromRuntimeListEntry); + } + + async deleteObject(scope: string, key: string) { + await this.measured('deleteObject', rt => rt.deleteObject(scope, key)); + } + + async presignPut(scope: string, key: string, metadata?: PutObjectMetadata) { + const result = await this.measured('presignPut', rt => + rt.presignPut(scope, key, toRuntimeMetadata(metadata)) + ); + return result ? fromRuntimePresigned(result) : undefined; + } + + async presignGet(scope: string, key: string) { + const result = await this.measured('presignGet', rt => + rt.presignGet(scope, key) + ); + return result ? fromRuntimePresigned(result) : undefined; + } + + async createMultipartUpload( + scope: string, + key: string, + metadata?: PutObjectMetadata + ) { + const result = await this.measured('createMultipartUpload', rt => + rt.createMultipartUpload(scope, key, toRuntimeMetadata(metadata)) + ); + return result + ? { uploadId: result.uploadId, expiresAt: new Date(result.expiresAtMs) } + : undefined; + } + + async presignUploadPart( + scope: string, + key: string, + uploadId: string, + partNumber: number + ) { + const result = await this.measured('presignUploadPart', rt => + rt.presignUploadPart(scope, key, uploadId, partNumber) + ); + return result ? fromRuntimePresigned(result) : undefined; + } + + async listMultipartUploadParts(scope: string, key: string, uploadId: string) { + return ( + (await this.measured('listMultipartUploadParts', rt => + rt.listMultipartUploadParts(scope, key, uploadId) + )) ?? undefined + ); + } + + async proxyUploadPart( + scope: string, + key: string, + uploadId: string, + partNumber: number, + body: Buffer, + contentLength?: number + ) { + return ( + (await this.measured('proxyUploadPart', rt => + rt.proxyUploadPart( + scope, + key, + uploadId, + partNumber, + body, + contentLength + ) + )) ?? undefined + ); + } + + async completeMultipartUpload( + scope: string, + key: string, + uploadId: string, + parts: { partNumber: number; etag: string }[] + ) { + return await this.measured('completeMultipartUpload', rt => + rt.completeMultipartUpload(scope, key, uploadId, parts) + ); + } + + async abortMultipartUpload(scope: string, key: string, uploadId: string) { + return await this.measured('abortMultipartUpload', rt => + rt.abortMultipartUpload(scope, key, uploadId) + ); + } + + async completeWorkspaceBlobUpload( + workspaceId: string, + key: string, + expected: { size: number; mime: string } + ) { + return await this.measured('completeWorkspaceBlobUpload', rt => + rt.completeWorkspaceBlobUpload( + workspaceId, + key, + expected.size, + expected.mime + ) + ); + } + + async cleanupExpiredPendingBlobs(cutoffMs: number, limit: number) { + return await this.measured('cleanupExpiredPendingBlobs', rt => + rt.cleanupExpiredPendingBlobs(cutoffMs, limit) + ); + } + + async releaseDeletedBlobs(workspaceId: string, limit: number) { + return await this.measured('releaseDeletedBlobs', rt => + rt.releaseDeletedBlobs(workspaceId, limit) + ); + } + + async backfillMissingBlobMetadata( + workspaceId: string | null | undefined, + limit: number + ) { + return await this.measured('backfillMissingBlobMetadata', rt => + rt.backfillMissingBlobMetadata(workspaceId, limit) + ); + } + + async rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number) { + return await this.measured('rebuildWorkspaceDocBlobRefs', rt => + rt.rebuildWorkspaceDocBlobRefs(workspaceId, limit) + ); + } + + async planUnreferencedWorkspaceBlobs( + workspaceId: string, + gracePeriodDays: number, + limit: number + ) { + return await this.measured('planUnreferencedWorkspaceBlobs', rt => + rt.planUnreferencedWorkspaceBlobs(workspaceId, gracePeriodDays, limit) + ); + } + + async executeBlobCleanupCandidates( + runId: string, + gracePeriodDays: number, + limit: number + ) { + return await this.measured('executeBlobCleanupCandidates', rt => + rt.executeBlobCleanupCandidates(runId, gracePeriodDays, limit) + ); + } + + private async measured( + method: string, + fn: (runtime: RuntimeInstance) => Promise + ): Promise { + return await wrapCallMetric(() => fn(this.runtime), 'storage', 'runtime', { + method, + })(); + } + + private async runMigrationsOnce() { + if (this.migrationsStarted) { + return; + } + await this.runtime.runMigrations(); + this.migrationsStarted = true; + } + + private async restart() { + await this.runtime.stop(); + this.migrationsStarted = false; + await this.start(); + } +} + +function toRuntimeMetadata(metadata?: PutObjectMetadata) { + return metadata + ? { + contentType: metadata.contentType, + contentLength: metadata.contentLength, + checksumCrc32: metadata.checksumCRC32, + } + : undefined; +} + +function fromRuntimeMetadata( + metadata: RuntimeObjectMetadata +): GetObjectMetadata { + return { + contentType: metadata.contentType, + contentLength: metadata.contentLength, + lastModified: new Date(metadata.lastModifiedMs), + checksumCRC32: metadata.checksumCrc32, + }; +} + +function fromRuntimeGetResult(result: RuntimeObjectGetResult) { + return { + body: Readable.from(result.body), + metadata: fromRuntimeMetadata(result.metadata), + }; +} + +function fromRuntimeListEntry( + entry: RuntimeObjectListEntry +): ListObjectsMetadata { + return { + key: entry.key, + contentLength: entry.contentLength, + lastModified: new Date(entry.lastModifiedMs), + }; +} + +function fromRuntimePresigned( + request: RuntimePresignedObjectRequest +): PresignedUpload { + return { + url: request.url, + headers: JSON.parse(request.headersJson) as Record, + expiresAt: new Date(request.expiresAtMs), + }; +} + +export type StorageRuntimeGetObjectResult = { + redirectUrl?: string; + body?: BlobOutputType; + metadata?: GetObjectMetadata; +}; diff --git a/packages/backend/server/src/core/backend-runtime/__tests__/blob-job.spec.ts b/packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts similarity index 94% rename from packages/backend/server/src/core/backend-runtime/__tests__/blob-job.spec.ts rename to packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts index 740aa947f8..40d5c1297a 100644 --- a/packages/backend/server/src/core/backend-runtime/__tests__/blob-job.spec.ts +++ b/packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts @@ -1,7 +1,7 @@ import ava, { TestFn } from 'ava'; import Sinon from 'sinon'; -import { BackendRuntimeBlobJob } from '../blob-job'; +import { StorageBlobJob } from '../blob-job'; interface Context { runtime: { @@ -22,7 +22,7 @@ interface Context { findMany: Sinon.SinonStub; }; }; - job: BackendRuntimeBlobJob; + job: StorageBlobJob; } const test = ava as TestFn; @@ -31,7 +31,7 @@ test.beforeEach(t => { t.context.runtime = { health: Sinon.stub().resolves({ databaseConnected: true, - objectStorageConfigured: true, + providerConfigured: true, }), backfillMissingBlobMetadata: Sinon.stub(), rebuildWorkspaceDocBlobRefs: Sinon.stub(), @@ -49,7 +49,7 @@ test.beforeEach(t => { findMany: Sinon.stub(), }, }; - t.context.job = new BackendRuntimeBlobJob( + t.context.job = new StorageBlobJob( t.context.runtime as any, t.context.event as any, t.context.queue as any, @@ -103,7 +103,7 @@ for (const scenario of objectStorageRequiredCases) { test(`${scenario.name} skips when object storage is not configured`, async t => { t.context.runtime.health.resolves({ databaseConnected: true, - objectStorageConfigured: false, + providerConfigured: false, }); await scenario.run(t.context); diff --git a/packages/backend/server/src/core/storage/__tests__/comment-attachment.spec.ts b/packages/backend/server/src/core/storage/__tests__/comment-attachment.spec.ts index d99729fceb..e3f24410b5 100644 --- a/packages/backend/server/src/core/storage/__tests__/comment-attachment.spec.ts +++ b/packages/backend/server/src/core/storage/__tests__/comment-attachment.spec.ts @@ -5,17 +5,66 @@ import test from 'ava'; import { createModule } from '../../../__tests__/create-module'; import { Mockers } from '../../../__tests__/mocks'; +import { initTestingDB } from '../../../__tests__/utils'; import { Models } from '../../../models'; +import { getMime } from '../../../native'; +import { StorageRuntimeProvider } from '../../storage-runtime'; import { CommentAttachmentStorage, StorageModule } from '../index'; +const objects = new Map< + string, + { + body: Buffer; + metadata: { + contentType: string; + contentLength: number; + lastModified: Date; + }; + } +>(); +const storageRuntime = { + putObject: async ( + _scope: string, + key: string, + body: Buffer, + _metadata?: { contentType?: string; contentLength?: number } + ) => { + const object = { + body, + metadata: { + contentType: getMime(body), + contentLength: body.length, + lastModified: new Date(), + }, + }; + objects.set(key, object); + return object.metadata; + }, + headObject: async (_scope: string, key: string) => objects.get(key)?.metadata, + getObject: async (_scope: string, key: string) => { + const object = objects.get(key); + return object + ? { body: Readable.from(object.body), metadata: object.metadata } + : {}; + }, + deleteObject: async (_scope: string, key: string) => { + objects.delete(key); + }, + presignGet: async () => undefined, +}; + const module = await createModule({ imports: [StorageModule], + tapModule: builder => { + builder.overrideProvider(StorageRuntimeProvider).useValue(storageRuntime); + }, }); const storage = module.get(CommentAttachmentStorage); const models = module.get(Models); -test.before(async () => { - await storage.onConfigInit(); +test.beforeEach(async () => { + await initTestingDB(module); + objects.clear(); }); test.after.always(async () => { diff --git a/packages/backend/server/src/core/backend-runtime/blob-job.ts b/packages/backend/server/src/core/storage/blob-job.ts similarity index 95% rename from packages/backend/server/src/core/backend-runtime/blob-job.ts rename to packages/backend/server/src/core/storage/blob-job.ts index b097e715de..13cf8d74e2 100644 --- a/packages/backend/server/src/core/backend-runtime/blob-job.ts +++ b/packages/backend/server/src/core/storage/blob-job.ts @@ -3,8 +3,10 @@ import { Cron, CronExpression } from '@nestjs/schedule'; import { PrismaClient } from '@prisma/client'; import { EventBus, JobQueue, OnJob } from '../../base'; -import { BackendRuntimeProvider } from './provider'; +import { StorageRuntimeProvider } from '../storage-runtime'; +// Queue keys are persisted API; keep the legacy backendRuntime.* names while +// StorageBlobJob and StorageRuntimeProvider own the implementation. declare global { interface Jobs { 'backendRuntime.backfillMissingBlobMetadata': { @@ -45,11 +47,11 @@ declare global { } @Injectable() -export class BackendRuntimeBlobJob { - private readonly logger = new Logger(BackendRuntimeBlobJob.name); +export class StorageBlobJob { + private readonly logger = new Logger(StorageBlobJob.name); constructor( - private readonly rt: BackendRuntimeProvider, + private readonly rt: StorageRuntimeProvider, private readonly event: EventBus, private readonly queue: JobQueue, private readonly db: PrismaClient @@ -355,7 +357,7 @@ export class BackendRuntimeBlobJob { limit ); await Promise.all( - result.workspaceIds.map(workspaceId => + result.workspaceIds.map((workspaceId: string) => this.event.emitAsync('workspace.blobs.updated', { workspaceId }) ) ); @@ -375,7 +377,7 @@ export class BackendRuntimeBlobJob { limit ); await Promise.all( - result.workspaceIds.map(workspaceId => + result.workspaceIds.map((workspaceId: string) => this.event.emitAsync('workspace.blobs.updated', { workspaceId }) ) ); @@ -409,12 +411,12 @@ export class BackendRuntimeBlobJob { private async hasObjectStorage(operation: string) { const health = await this.rt.health(); - if (health.objectStorageConfigured) { + if (health.providerConfigured) { return true; } this.logger.warn( - `skip ${operation}: BackendRuntime object storage is not configured` + `skip ${operation}: StorageRuntime provider is not configured` ); return false; } diff --git a/packages/backend/server/src/core/storage/index.ts b/packages/backend/server/src/core/storage/index.ts index cbc1495b75..0b70816de7 100644 --- a/packages/backend/server/src/core/storage/index.ts +++ b/packages/backend/server/src/core/storage/index.ts @@ -2,7 +2,8 @@ import './config'; import { Module } from '@nestjs/common'; -import { BackendRuntimeModule } from '../backend-runtime'; +import { StorageRuntimeModule } from '../storage-runtime'; +import { StorageBlobJob } from './blob-job'; import { BlobUploadCleanupJob } from './job'; import { R2UploadController } from './r2-proxy'; import { @@ -12,16 +13,23 @@ import { } from './wrappers'; @Module({ - imports: [BackendRuntimeModule], + imports: [StorageRuntimeModule], controllers: [R2UploadController], providers: [ WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage, + StorageBlobJob, BlobUploadCleanupJob, ], - exports: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage], + exports: [ + WorkspaceBlobStorage, + AvatarStorage, + CommentAttachmentStorage, + StorageBlobJob, + ], }) export class StorageModule {} +export { StorageBlobJob } from './blob-job'; export { AvatarStorage, CommentAttachmentStorage, WorkspaceBlobStorage }; diff --git a/packages/backend/server/src/core/storage/job.ts b/packages/backend/server/src/core/storage/job.ts index 5e01520860..bdd3dedabb 100644 --- a/packages/backend/server/src/core/storage/job.ts +++ b/packages/backend/server/src/core/storage/job.ts @@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { EventBus, JobQueue, OneDay, OnJob } from '../../base'; -import { BackendRuntimeProvider } from '../backend-runtime'; +import { StorageRuntimeProvider } from '../storage-runtime'; declare global { interface Jobs { @@ -15,7 +15,7 @@ export class BlobUploadCleanupJob { private readonly logger = new Logger(BlobUploadCleanupJob.name); constructor( - private readonly rt: BackendRuntimeProvider, + private readonly rt: StorageRuntimeProvider, private readonly event: EventBus, private readonly queue: JobQueue ) {} diff --git a/packages/backend/server/src/core/storage/r2-proxy.ts b/packages/backend/server/src/core/storage/r2-proxy.ts index c93fff7ca3..c283e6b7e3 100644 --- a/packages/backend/server/src/core/storage/r2-proxy.ts +++ b/packages/backend/server/src/core/storage/r2-proxy.ts @@ -7,19 +7,16 @@ import { BlobInvalid, CallMetric, Config, - OnEvent, PROXY_MULTIPART_PATH, PROXY_UPLOAD_PATH, + type R2StorageConfig, STORAGE_PROXY_ROOT, - StorageProviderConfig, - StorageProviderFactory, + type StorageProviderConfig, + toBuffer, } from '../../base'; -import { - R2StorageConfig, - R2StorageProvider, -} from '../../base/storage/providers/r2'; import { Models } from '../../models'; import { Public } from '../auth/guard'; +import { StorageRuntimeProvider } from '../storage-runtime'; import { MULTIPART_PART_SIZE } from './constants'; type R2BlobStorageConfig = StorageProviderConfig & { @@ -37,21 +34,13 @@ type R2Config = { @Controller(STORAGE_PROXY_ROOT) export class R2UploadController { private readonly logger = new Logger(R2UploadController.name); - private provider: R2StorageProvider | null = null; constructor( private readonly config: Config, private readonly models: Models, - private readonly storageFactory: StorageProviderFactory + private readonly rt: StorageRuntimeProvider ) {} - @OnEvent('config.changed') - onConfigChanged(event: Events['config.changed']) { - if (event.updates.storages?.blob?.storage) { - this.provider = null; - } - } - private getR2Config(): R2Config { const storage = this.config.storages.blob.storage as StorageProviderConfig; if (storage.provider !== 'cloudflare-r2') { @@ -69,16 +58,6 @@ export class R2UploadController { return { storage: storage as R2BlobStorageConfig, signKey }; } - private getProvider(storage: R2BlobStorageConfig) { - if (!this.provider) { - const candidate = this.storageFactory.create(storage); - if (candidate instanceof R2StorageProvider) { - this.provider = candidate; - } - } - return this.provider; - } - private sign(canonical: string, signKey: string) { return createHmac('sha256', signKey).update(canonical).digest('base64'); } @@ -173,7 +152,7 @@ export class R2UploadController { @Put('upload') @CallMetric('controllers', 'r2_proxy_upload') async upload(@Req() req: Request, @Res() res: Response) { - const { storage, signKey } = this.getR2Config(); + const { signKey } = this.getR2Config(); const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId'); const key = this.expectString(req.query.key, 'key'); @@ -229,16 +208,16 @@ export class R2UploadController { throw new BlobInvalid('Mime type mismatch'); } - const provider = this.getProvider(storage); - if (!provider) { - throw new BlobInvalid('R2 provider is not available'); - } - try { - await provider.proxyPutObject(`${workspaceId}/${key}`, req, { - contentType: mime, - contentLength, - }); + await this.rt.putObject( + 'blob', + `${workspaceId}/${key}`, + await toBuffer(req), + { + contentType: mime, + contentLength, + } + ); } catch (error) { this.logger.error('Failed to proxy upload', error as Error); throw new BlobInvalid('Upload failed'); @@ -251,7 +230,7 @@ export class R2UploadController { @Put('multipart') @CallMetric('controllers', 'r2_proxy_multipart') async uploadPart(@Req() req: Request, @Res() res: Response) { - const { storage, signKey } = this.getR2Config(); + const { signKey } = this.getR2Config(); const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId'); const key = this.expectString(req.query.key, 'key'); @@ -305,18 +284,14 @@ export class R2UploadController { throw new BlobInvalid('Part size exceeds upload metadata'); } - const provider = this.getProvider(storage); - if (!provider) { - throw new BlobInvalid('R2 provider is not available'); - } - try { - const etag = await provider.proxyUploadPart( + const etag = await this.rt.proxyUploadPart( + 'blob', `${workspaceId}/${key}`, uploadId, partNumber, - req, - { contentLength } + await toBuffer(req), + contentLength ); if (etag) { res.setHeader('etag', etag); diff --git a/packages/backend/server/src/core/storage/wrappers/avatar.ts b/packages/backend/server/src/core/storage/wrappers/avatar.ts index 6a1c1f040c..a83359be70 100644 --- a/packages/backend/server/src/core/storage/wrappers/avatar.ts +++ b/packages/backend/server/src/core/storage/wrappers/avatar.ts @@ -1,21 +1,11 @@ import { Injectable } from '@nestjs/common'; -import type { - BlobInputType, - PutObjectMetadata, - StorageProvider, -} from '../../../base'; -import { - Config, - OnEvent, - StorageProviderFactory, - URLHelper, -} from '../../../base'; +import type { BlobInputType, PutObjectMetadata } from '../../../base'; +import { Config, OnEvent, toBuffer, URLHelper } from '../../../base'; +import { StorageRuntimeProvider } from '../../storage-runtime'; @Injectable() export class AvatarStorage { - private provider!: StorageProvider; - get config() { return this.AFFiNEConfig.storages.avatar; } @@ -23,23 +13,11 @@ export class AvatarStorage { constructor( private readonly AFFiNEConfig: Config, private readonly url: URLHelper, - private readonly storageFactory: StorageProviderFactory + private readonly rt: StorageRuntimeProvider ) {} - @OnEvent('config.init') - async onConfigInit() { - this.provider = this.storageFactory.create(this.config.storage); - } - - @OnEvent('config.changed') - async onConfigChanged(event: Events['config.changed']) { - if (event.updates.storages?.avatar?.storage) { - this.provider = this.storageFactory.create(this.config.storage); - } - } - async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) { - await this.provider.put(key, blob, metadata); + await this.rt.putObject('avatar', key, await toBuffer(blob), metadata); let link = this.config.publicPath + key; if (link.startsWith('/')) { @@ -50,11 +28,11 @@ export class AvatarStorage { } get(key: string) { - return this.provider.get(key); + return this.rt.getObject('avatar', key); } delete(link: string) { - return this.provider.delete(link.split('/').pop() as string); + return this.rt.deleteObject('avatar', link.split('/').pop() as string); } @OnEvent('user.deleted') diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts index dacee8fd3f..219b9c01bc 100644 --- a/packages/backend/server/src/core/storage/wrappers/blob.ts +++ b/packages/backend/server/src/core/storage/wrappers/blob.ts @@ -1,20 +1,24 @@ -import { createHash } from 'node:crypto'; +import { createHmac } from 'node:crypto'; import { Injectable, Logger } from '@nestjs/common'; import { - autoMetadata, + type BlobOutputType, Config, EventBus, type GetObjectMetadata, OnEvent, - PutObjectMetadata, - type StorageProvider, - StorageProviderFactory, + PROXY_MULTIPART_PATH, + PROXY_UPLOAD_PATH, + type PutObjectMetadata, + type R2StorageConfig, + SIGNED_URL_EXPIRED, + type StorageProviderConfig, URLHelper, } from '../../../base'; import { Models } from '../../../models'; -import { BackendRuntimeProvider } from '../../backend-runtime'; +import type { StorageProviderCapabilities } from '../../../native'; +import { StorageRuntimeProvider } from '../../storage-runtime'; declare global { interface Events { @@ -36,52 +40,70 @@ type BlobCompleteResult = | 'not_found' | 'size_mismatch' | 'mime_mismatch' - | 'checksum_mismatch'; + | 'checksum_mismatch' + | 'size_too_large'; }; +type BlobGetResult = { + redirectUrl?: string; + body?: BlobOutputType; + metadata?: GetObjectMetadata; +}; + @Injectable() export class WorkspaceBlobStorage { private readonly logger = new Logger(WorkspaceBlobStorage.name); - private provider!: StorageProvider; - - get config() { - return this.AFFiNEConfig.storages.blob; - } constructor( - private readonly AFFiNEConfig: Config, private readonly event: EventBus, - private readonly storageFactory: StorageProviderFactory, private readonly models: Models, private readonly url: URLHelper, - private readonly rt: BackendRuntimeProvider + private readonly rt: StorageRuntimeProvider, + private readonly config: Config ) {} - @OnEvent('config.init') - async onConfigInit() { - this.provider = this.storageFactory.create(this.config.storage); - } - - @OnEvent('config.changed') - async onConfigChanged(event: Events['config.changed']) { - if (event.updates.storages?.blob?.storage) { - this.provider = this.storageFactory.create(this.config.storage); - } - } - async put(workspaceId: string, key: string, blob: Buffer) { - const meta: PutObjectMetadata = autoMetadata(blob); - - await this.provider.put(`${workspaceId}/${key}`, blob, meta); + const metadata = await this.rt.putObject( + 'blob', + `${workspaceId}/${key}`, + blob + ); await this.upsert(workspaceId, key, { - contentType: meta.contentType ?? 'application/octet-stream', - contentLength: blob.length, - lastModified: new Date(), + contentType: metadata.contentType, + contentLength: metadata.contentLength, + lastModified: metadata.lastModified, }); } - async get(workspaceId: string, key: string, signedUrl?: boolean) { - return this.provider.get(`${workspaceId}/${key}`, signedUrl); + async capabilities(): Promise { + const capabilities = await this.rt.providerCapabilities('blob'); + if (!this.r2ProxyConfig()) { + return capabilities; + } + return { + ...capabilities, + presignPut: true, + multipartDirect: true, + proxyUpload: true, + serverMediatedOnly: false, + }; + } + + async get( + workspaceId: string, + key: string, + signedUrl?: boolean + ): Promise { + if (signedUrl) { + const presigned = await this.rt.presignGet( + 'blob', + `${workspaceId}/${key}` + ); + if (presigned) { + return { redirectUrl: presigned.url }; + } + } + return this.rt.getObject('blob', `${workspaceId}/${key}`); } async presignPut( @@ -89,7 +111,11 @@ export class WorkspaceBlobStorage { key: string, metadata?: PutObjectMetadata ) { - return this.provider.presignPut?.(`${workspaceId}/${key}`, metadata); + const proxy = this.r2ProxyConfig(); + if (proxy) { + return this.createProxyUploadUrl(workspaceId, key, metadata, proxy); + } + return this.rt.presignPut('blob', `${workspaceId}/${key}`, metadata); } async createMultipartUpload( @@ -97,7 +123,8 @@ export class WorkspaceBlobStorage { key: string, metadata?: PutObjectMetadata ) { - return this.provider.createMultipartUpload?.( + return this.rt.createMultipartUpload( + 'blob', `${workspaceId}/${key}`, metadata ); @@ -109,7 +136,18 @@ export class WorkspaceBlobStorage { uploadId: string, partNumber: number ) { - return this.provider.presignUploadPart?.( + const proxy = this.r2ProxyConfig(); + if (proxy) { + return this.createProxyMultipartUrl( + workspaceId, + key, + uploadId, + partNumber, + proxy + ); + } + return this.rt.presignUploadPart( + 'blob', `${workspaceId}/${key}`, uploadId, partNumber @@ -121,7 +159,8 @@ export class WorkspaceBlobStorage { key: string, uploadId: string ) { - return this.provider.listMultipartUploadParts?.( + return this.rt.listMultipartUploadParts( + 'blob', `${workspaceId}/${key}`, uploadId ); @@ -133,16 +172,12 @@ export class WorkspaceBlobStorage { uploadId: string, parts: { partNumber: number; etag: string }[] ) { - if (!this.provider.completeMultipartUpload) { - return false; - } - - await this.provider.completeMultipartUpload( + return await this.rt.completeMultipartUpload( + 'blob', `${workspaceId}/${key}`, uploadId, parts ); - return true; } async abortMultipartUpload( @@ -150,16 +185,15 @@ export class WorkspaceBlobStorage { key: string, uploadId: string ) { - if (!this.provider.abortMultipartUpload) { - return false; - } - - await this.provider.abortMultipartUpload(`${workspaceId}/${key}`, uploadId); - return true; + return await this.rt.abortMultipartUpload( + 'blob', + `${workspaceId}/${key}`, + uploadId + ); } async head(workspaceId: string, key: string) { - return this.provider.head(`${workspaceId}/${key}`); + return this.rt.headObject('blob', `${workspaceId}/${key}`); } async complete( @@ -167,57 +201,28 @@ export class WorkspaceBlobStorage { key: string, expected: { size: number; mime: string } ): Promise { - const metadata = await this.head(workspaceId, key); - if (!metadata) { - return { ok: false, reason: 'not_found' }; - } - - if (metadata.contentLength !== expected.size) { - return { ok: false, reason: 'size_mismatch' }; - } - - if (expected.mime && metadata.contentType !== expected.mime) { - return { ok: false, reason: 'mime_mismatch' }; - } - - const object = await this.provider.get(`${workspaceId}/${key}`); - if (!object.body) { - return { ok: false, reason: 'not_found' }; - } - - const checksum = createHash('sha256'); - try { - for await (const chunk of object.body) { - checksum.update(chunk as Buffer); - } - } catch (e) { - this.logger.error('failed to read blob for checksum verification', e); - return { ok: false, reason: 'checksum_mismatch' }; - } - - const base64 = checksum.digest('base64'); - const base64urlWithPadding = base64.replace(/\+/g, '-').replace(/\//g, '_'); - - if (base64urlWithPadding !== key) { - try { - await this.provider.delete(`${workspaceId}/${key}`); - } catch (e) { - // never throw - this.logger.error('failed to delete invalid blob', e); - } - return { ok: false, reason: 'checksum_mismatch' }; - } - - await this.models.blob.upsert({ + const result = await this.rt.completeWorkspaceBlobUpload( workspaceId, key, - mime: metadata.contentType, - size: metadata.contentLength, - status: 'completed', - uploadId: null, - }); - - return { ok: true, metadata }; + expected + ); + if (!result.ok) { + return { + ok: false, + reason: (result.reason ?? 'checksum_mismatch') as Exclude< + BlobCompleteResult, + { ok: true } + >['reason'], + }; + } + return { + ok: true, + metadata: { + contentType: result.contentType ?? 'application/octet-stream', + contentLength: result.contentLength ?? expected.size, + lastModified: new Date(result.lastModifiedMs ?? Date.now()), + }, + }; } async list(workspaceId: string) { @@ -226,7 +231,7 @@ export class WorkspaceBlobStorage { async delete(workspaceId: string, key: string, permanently = false) { if (permanently) { - await this.provider.delete(`${workspaceId}/${key}`); + await this.rt.deleteObject('blob', `${workspaceId}/${key}`); } await this.models.blob.delete(workspaceId, key, permanently); if (!permanently) { @@ -297,4 +302,96 @@ export class WorkspaceBlobStorage { }: Events['workspace.blob.delete']) { await this.delete(workspaceId, key, true); } + + private r2ProxyConfig() { + const storage = this.config.storages.blob.storage as StorageProviderConfig; + if (storage.provider !== 'cloudflare-r2') { + return; + } + const r2 = storage.config as R2StorageConfig; + const usePresignedURL = r2.usePresignedURL; + if ( + !usePresignedURL?.enabled || + !usePresignedURL.urlPrefix || + !usePresignedURL.signKey + ) { + return; + } + return { signKey: usePresignedURL.signKey }; + } + + private signProxy( + path: string, + canonicalFields: (string | number | undefined)[], + exp: number, + signKey: string + ) { + const canonical = [ + path, + ...canonicalFields.map(field => + field === undefined ? '' : field.toString() + ), + exp.toString(), + ].join('\n'); + return `${exp}-${createHmac('sha256', signKey).update(canonical).digest('base64')}`; + } + + private createProxyUploadUrl( + workspaceId: string, + key: string, + metadata: PutObjectMetadata | undefined, + proxy: { signKey: string } + ) { + const contentType = metadata?.contentType ?? 'application/octet-stream'; + const contentLength = metadata?.contentLength; + const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000); + const exp = Math.floor(expiresAt.getTime() / 1000); + const token = this.signProxy( + PROXY_UPLOAD_PATH, + [workspaceId, key, contentType, contentLength], + exp, + proxy.signKey + ); + return { + url: this.url.link(PROXY_UPLOAD_PATH, { + workspaceId, + key, + contentType, + contentLength, + exp, + token, + }), + headers: {}, + expiresAt, + }; + } + + private createProxyMultipartUrl( + workspaceId: string, + key: string, + uploadId: string, + partNumber: number, + proxy: { signKey: string } + ) { + const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000); + const exp = Math.floor(expiresAt.getTime() / 1000); + const token = this.signProxy( + PROXY_MULTIPART_PATH, + [workspaceId, key, uploadId, partNumber], + exp, + proxy.signKey + ); + return { + url: this.url.link(PROXY_MULTIPART_PATH, { + workspaceId, + key, + uploadId, + partNumber, + exp, + token, + }), + headers: {}, + expiresAt, + }; + } } diff --git a/packages/backend/server/src/core/storage/wrappers/comment-attachment.ts b/packages/backend/server/src/core/storage/wrappers/comment-attachment.ts index aa65d24184..3730501e5c 100644 --- a/packages/backend/server/src/core/storage/wrappers/comment-attachment.ts +++ b/packages/backend/server/src/core/storage/wrappers/comment-attachment.ts @@ -1,16 +1,11 @@ import { Injectable, Logger } from '@nestjs/common'; -import { - autoMetadata, - Config, - EventBus, - metrics, - OnEvent, - type StorageProvider, - StorageProviderFactory, - URLHelper, -} from '../../../base'; +import { EventBus, metrics, OnEvent, URLHelper } from '../../../base'; import { Models } from '../../../models'; +import { + type StorageRuntimeGetObjectResult, + StorageRuntimeProvider, +} from '../../storage-runtime'; declare global { interface Events { @@ -25,32 +20,14 @@ declare global { @Injectable() export class CommentAttachmentStorage { private readonly logger = new Logger(CommentAttachmentStorage.name); - private provider!: StorageProvider; - - get config() { - return this.AFFiNEConfig.storages.blob; - } constructor( - private readonly AFFiNEConfig: Config, private readonly event: EventBus, - private readonly storageFactory: StorageProviderFactory, private readonly models: Models, - private readonly url: URLHelper + private readonly url: URLHelper, + private readonly rt: StorageRuntimeProvider ) {} - @OnEvent('config.init') - async onConfigInit() { - this.provider = this.storageFactory.create(this.config.storage); - } - - @OnEvent('config.changed') - async onConfigChanged(event: Events['config.changed']) { - if (event.updates.storages?.blob?.storage) { - this.provider = this.storageFactory.create(this.config.storage); - } - } - private storageKey(workspaceId: string, docId: string, key: string) { return `comment-attachments/${workspaceId}/${docId}/${key}`; } @@ -63,15 +40,13 @@ export class CommentAttachmentStorage { blob: Buffer, userId: string ) { - const meta = autoMetadata(blob); - - await this.provider.put( + const metadata = await this.rt.putObject( + 'blob', this.storageKey(workspaceId, docId, key), - blob, - meta + blob ); - const mime = meta.contentType ?? 'application/octet-stream'; - const size = blob.length; + const mime = metadata.contentType; + const size = metadata.contentLength; await this.models.commentAttachment.upsert({ workspaceId, docId, @@ -94,15 +69,22 @@ export class CommentAttachmentStorage { docId: string, key: string, signedUrl?: boolean - ) { - return await this.provider.get( - this.storageKey(workspaceId, docId, key), - signedUrl - ); + ): Promise { + const storageKey = this.storageKey(workspaceId, docId, key); + if (signedUrl) { + const presigned = await this.rt.presignGet('blob', storageKey); + if (presigned) { + return { redirectUrl: presigned.url }; + } + } + return await this.rt.getObject('blob', storageKey); } async delete(workspaceId: string, docId: string, key: string) { - await this.provider.delete(this.storageKey(workspaceId, docId, key)); + await this.rt.deleteObject( + 'blob', + this.storageKey(workspaceId, docId, key) + ); await this.models.commentAttachment.delete(workspaceId, docId, key); this.logger.log( `deleted comment attachment ${workspaceId}/${docId}/${key}` diff --git a/packages/backend/server/src/core/workspaces/resolvers/blob.ts b/packages/backend/server/src/core/workspaces/resolvers/blob.ts index ca50957038..b6ca0a6f1b 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/blob.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/blob.ts @@ -248,11 +248,12 @@ export class WorkspaceBlobResolver { } const metadata = { contentType: mime, contentLength: size }; + const capabilities = await this.storage.capabilities(); let init: BlobUploadInit | null = null; let uploadIdForRecord: string | null = null; // try to resume multipart uploads - if (record && record.uploadId) { + if (capabilities.multipartDirect && record && record.uploadId) { const uploadedParts = await this.storage.listMultipartUploadParts( workspaceId, key, @@ -270,7 +271,7 @@ export class WorkspaceBlobResolver { } } - if (size >= MULTIPART_THRESHOLD) { + if (capabilities.multipartDirect && size >= MULTIPART_THRESHOLD) { const multipart = await this.storage.createMultipartUpload( workspaceId, key, @@ -289,7 +290,7 @@ export class WorkspaceBlobResolver { } } - if (!init) { + if (!init && capabilities.presignPut) { const presigned = await this.storage.presignPut( workspaceId, key, diff --git a/packages/backend/server/src/native.ts b/packages/backend/server/src/native.ts index 9dcdba821d..1cc5b8edaa 100644 --- a/packages/backend/server/src/native.ts +++ b/packages/backend/server/src/native.ts @@ -60,7 +60,6 @@ import serverNativeModule, { type RuntimeObjectGetResult, type RuntimeObjectListEntry, type RuntimeObjectMetadata, - type RuntimeObjectStorageHealth, type RuntimeObjectStoragePutOptions, type RuntimePresignedObjectRequest, type RuntimeVerificationTokenRecord, @@ -68,6 +67,8 @@ import serverNativeModule, { type RuntimeWorkspaceStatsDailyRecalibrationResult, type SafeFetchRequest, type SafeFetchResponse, + type StorageProviderCapabilities, + type StorageRuntimeHealth, type Tokenizer, } from '@affine/server-native'; @@ -109,7 +110,6 @@ export type { RuntimeObjectGetResult, RuntimeObjectListEntry, RuntimeObjectMetadata, - RuntimeObjectStorageHealth, RuntimeObjectStoragePutOptions, RuntimePresignedObjectRequest, RuntimeVerificationTokenRecord, @@ -117,6 +117,8 @@ export type { RuntimeWorkspaceStatsDailyRecalibrationResult, SafeFetchRequest, SafeFetchResponse, + StorageProviderCapabilities, + StorageRuntimeHealth, }; export type ActionEventType = @@ -274,6 +276,7 @@ export const AFFINE_PRO_PUBLIC_KEY = serverNativeModule.AFFINE_PRO_PUBLIC_KEY; export const AFFINE_PRO_LICENSE_AES_KEY = serverNativeModule.AFFINE_PRO_LICENSE_AES_KEY; export const BackendRuntime = serverNativeModule.BackendRuntime; +export const StorageRuntime = serverNativeModule.StorageRuntime; export type PermissionWorkspaceRole = 'external' | 'member' | 'admin' | 'owner'; export type PermissionDocRole = diff --git a/packages/backend/server/src/plugins/copilot/storage.ts b/packages/backend/server/src/plugins/copilot/storage.ts index 5213d0e5bb..2cb94b1d5f 100644 --- a/packages/backend/server/src/plugins/copilot/storage.ts +++ b/packages/backend/server/src/plugins/copilot/storage.ts @@ -6,43 +6,29 @@ import { type BlobInputType, BlobQuotaExceeded, CallMetric, - Config, type FileUpload, OneMB, - OnEvent, readBuffer, - type StorageProvider, - StorageProviderFactory, + toBuffer, URLHelper, } from '../../base'; import { QuotaService } from '../../core/quota'; +import { + type StorageRuntimeGetObjectResult, + StorageRuntimeProvider, +} from '../../core/storage-runtime'; import { fetchRemoteAttachment } from '../../native'; const REMOTE_BLOB_MAX_BYTES = 20 * OneMB; @Injectable() export class CopilotStorage { - public provider!: StorageProvider; - constructor( - private readonly config: Config, private readonly url: URLHelper, - private readonly storageFactory: StorageProviderFactory, + private readonly rt: StorageRuntimeProvider, private readonly quota: QuotaService ) {} - @OnEvent('config.init') - async onConfigInit() { - this.provider = this.storageFactory.create(this.config.copilot.storage); - } - - @OnEvent('config.changed') - async onConfigChanged(event: Events['config.changed']) { - if (event.updates?.copilot?.storage) { - this.provider = this.storageFactory.create(this.config.copilot.storage); - } - } - @CallMetric('ai', 'blob_put') async put( userId: string, @@ -52,10 +38,11 @@ export class CopilotStorage { mimeType = 'image/png' ) { const name = `${userId}/${workspaceId}/${key}`; - await this.provider.put(name, blob); + const buffer = await toBuffer(blob); + await this.rt.putObject('copilot', name, buffer); if (!env.prod) { // return image base64url for dev environment - return `data:${mimeType};base64,${blob.toString('base64')}`; + return `data:${mimeType};base64,${buffer.toString('base64')}`; } return this.url.link(`/api/copilot/blob/${name}`); } @@ -66,13 +53,20 @@ export class CopilotStorage { workspaceId: string, key: string, signedUrl?: boolean - ) { - return this.provider.get(`${userId}/${workspaceId}/${key}`, signedUrl); + ): Promise { + const name = `${userId}/${workspaceId}/${key}`; + if (signedUrl) { + const presigned = await this.rt.presignGet('copilot', name); + if (presigned) { + return { redirectUrl: presigned.url }; + } + } + return this.rt.getObject('copilot', name); } @CallMetric('ai', 'blob_delete') async delete(userId: string, workspaceId: string, key: string) { - await this.provider.delete(`${userId}/${workspaceId}/${key}`); + await this.rt.deleteObject('copilot', `${userId}/${workspaceId}/${key}`); } @CallMetric('ai', 'blob_upload') diff --git a/packages/common/native/Cargo.toml b/packages/common/native/Cargo.toml index fcb7517bbc..b4f52125c1 100644 --- a/packages/common/native/Cargo.toml +++ b/packages/common/native/Cargo.toml @@ -7,7 +7,7 @@ version = "0.1.0" [features] default = [] -hashcash = ["chrono", "sha3", "rand"] +hashcash = ["chrono", "hex", "sha3", "rand"] napi = ["dep:napi"] ydoc-loader = [ "assert-json-diff", @@ -22,6 +22,7 @@ ydoc-loader = [ [dependencies] assert-json-diff = { workspace = true, optional = true } chrono = { workspace = true, optional = true } +hex = { workspace = true, optional = true } nanoid = { workspace = true, optional = true } napi = { workspace = true, optional = true } pulldown-cmark = { workspace = true, optional = true } diff --git a/packages/common/native/src/hashcash.rs b/packages/common/native/src/hashcash.rs index 1c2354483c..b80fde9c63 100644 --- a/packages/common/native/src/hashcash.rs +++ b/packages/common/native/src/hashcash.rs @@ -40,7 +40,7 @@ impl Stamp { // check challenge let mut hasher = Sha3_256::new(); hasher.update(self.format().as_bytes()); - let result = format!("{:x}", hasher.finalize()); + let result = hex::encode(hasher.finalize()); result[..hex_digits] == String::from_utf8(vec![b'0'; hex_digits]).unwrap() } else { false @@ -77,7 +77,7 @@ impl Stamp { let zeros = String::from_utf8(vec![b'0'; hex_digits]).unwrap(); loop { hasher.update(format!("{challenge}:{counter:x}").as_bytes()); - let result = format!("{:x}", hasher.finalize_reset()); + let result = hex::encode(hasher.finalize_reset()); if result[..hex_digits] == zeros { break format!("{counter:x}"); };