From 95dbda24fc205ff075fd424487051c04818f722c Mon Sep 17 00:00:00 2001 From: Brooooooklyn Date: Mon, 21 Apr 2025 02:51:15 +0000 Subject: [PATCH] feat(y-octo): import y-octo monorepo (#11750) --- .github/workflows/build-test.yml | 140 ++ Cargo.lock | 452 ++++- Cargo.toml | 29 +- package.json | 1 + packages/common/y-octo/core/Cargo.toml | 95 ++ packages/common/y-octo/core/LICENSE | 9 + packages/common/y-octo/core/README.md | 100 ++ .../y-octo/core/benches/apply_benchmarks.rs | 34 + .../core/benches/array_ops_benchmarks.rs | 71 + .../y-octo/core/benches/codec_benchmarks.rs | 91 + .../y-octo/core/benches/map_ops_benchmarks.rs | 65 + .../core/benches/text_ops_benchmarks.rs | 50 + .../y-octo/core/benches/update_benchmarks.rs | 34 + .../common/y-octo/core/benches/utils/files.rs | 42 + .../common/y-octo/core/benches/utils/mod.rs | 3 + .../common/y-octo/core/src/codec/buffer.rs | 87 + .../common/y-octo/core/src/codec/integer.rs | 166 ++ packages/common/y-octo/core/src/codec/mod.rs | 9 + .../common/y-octo/core/src/codec/string.rs | 90 + .../common/y-octo/core/src/doc/awareness.rs | 253 +++ .../common/y-octo/core/src/doc/codec/any.rs | 716 ++++++++ .../y-octo/core/src/doc/codec/content.rs | 417 +++++ .../y-octo/core/src/doc/codec/delete_set.rs | 233 +++ .../common/y-octo/core/src/doc/codec/id.rs | 68 + .../y-octo/core/src/doc/codec/io/codec_v1.rs | 296 ++++ .../y-octo/core/src/doc/codec/io/mod.rs | 9 + .../y-octo/core/src/doc/codec/io/reader.rs | 30 + .../y-octo/core/src/doc/codec/io/writer.rs | 28 + .../common/y-octo/core/src/doc/codec/item.rs | 427 +++++ .../y-octo/core/src/doc/codec/item_flag.rs | 170 ++ .../common/y-octo/core/src/doc/codec/mod.rs | 25 + .../common/y-octo/core/src/doc/codec/refs.rs | 480 ++++++ .../y-octo/core/src/doc/codec/update.rs | 721 ++++++++ .../y-octo/core/src/doc/codec/utils/items.rs | 102 ++ .../y-octo/core/src/doc/codec/utils/mod.rs | 5 + .../common/y-octo/core/src/doc/common/mod.rs | 9 + .../y-octo/core/src/doc/common/range.rs | 481 ++++++ .../common/y-octo/core/src/doc/common/somr.rs | 525 ++++++ .../y-octo/core/src/doc/common/state.rs | 140 ++ .../common/y-octo/core/src/doc/document.rs | 656 ++++++++ packages/common/y-octo/core/src/doc/hasher.rs | 35 + .../common/y-octo/core/src/doc/history.rs | 327 ++++ packages/common/y-octo/core/src/doc/mod.rs | 33 + .../common/y-octo/core/src/doc/publisher.rs | 244 +++ packages/common/y-octo/core/src/doc/store.rs | 1355 +++++++++++++++ .../common/y-octo/core/src/doc/types/array.rs | 216 +++ .../core/src/doc/types/list/iterator.rs | 23 + .../y-octo/core/src/doc/types/list/mod.rs | 241 +++ .../core/src/doc/types/list/search_marker.rs | 340 ++++ .../common/y-octo/core/src/doc/types/map.rs | 326 ++++ .../common/y-octo/core/src/doc/types/mod.rs | 376 +++++ .../common/y-octo/core/src/doc/types/text.rs | 293 ++++ .../common/y-octo/core/src/doc/types/value.rs | 159 ++ .../common/y-octo/core/src/doc/types/xml.rs | 14 + packages/common/y-octo/core/src/doc/utils.rs | 28 + .../common/y-octo/core/src/fixtures/basic.bin | Bin 0 -> 5726 bytes .../y-octo/core/src/fixtures/database.bin | Bin 0 -> 3474 bytes .../edge-case-left-right-same-node.bin | Bin 0 -> 6673 bytes .../common/y-octo/core/src/fixtures/large.bin | Bin 0 -> 239535 bytes .../y-octo/core/src/fixtures/local_docs.json | 1 + .../y-octo/core/src/fixtures/with-subdoc.bin | Bin 0 -> 734 bytes packages/common/y-octo/core/src/lib.rs | 67 + .../y-octo/core/src/protocol/awareness.rs | 151 ++ .../common/y-octo/core/src/protocol/doc.rs | 103 ++ .../common/y-octo/core/src/protocol/mod.rs | 23 + .../y-octo/core/src/protocol/scanner.rs | 64 + .../common/y-octo/core/src/protocol/sync.rs | 165 ++ packages/common/y-octo/core/src/sync.rs | 32 + packages/common/y-octo/node/.gitignore | 2 + packages/common/y-octo/node/Cargo.toml | 20 + packages/common/y-octo/node/build.rs | 3 + packages/common/y-octo/node/index.d.ts | 48 + packages/common/y-octo/node/index.js | 377 +++++ packages/common/y-octo/node/package.json | 72 + .../common/y-octo/node/scripts/run-test.mts | 78 + packages/common/y-octo/node/src/array.rs | 160 ++ packages/common/y-octo/node/src/doc.rs | 176 ++ packages/common/y-octo/node/src/lib.rs | 17 + packages/common/y-octo/node/src/map.rs | 134 ++ packages/common/y-octo/node/src/text.rs | 82 + packages/common/y-octo/node/src/utils.rs | 117 ++ .../common/y-octo/node/tests/array.spec.mts | 62 + .../common/y-octo/node/tests/doc.spec.mts | 99 ++ .../common/y-octo/node/tests/map.spec.mts | 152 ++ .../common/y-octo/node/tests/text.spec.mts | 54 + packages/common/y-octo/node/tsconfig.json | 10 + packages/common/y-octo/utils/Cargo.toml | 71 + .../y-octo/utils/benches/apply_benchmarks.rs | 35 + .../utils/benches/array_ops_benchmarks.rs | 79 + .../y-octo/utils/benches/codec_benchmarks.rs | 89 + .../utils/benches/map_ops_benchmarks.rs | 79 + .../utils/benches/text_ops_benchmarks.rs | 54 + .../y-octo/utils/benches/update_benchmarks.rs | 33 + .../y-octo/utils/benches/utils/files.rs | 42 + .../common/y-octo/utils/benches/utils/mod.rs | 3 + .../y-octo/utils/bin/bench_result_render.rs | 134 ++ .../common/y-octo/utils/bin/doc_merger.rs | 100 ++ .../y-octo/utils/bin/memory_leak_test.rs | 79 + packages/common/y-octo/utils/fuzz/.gitignore | 4 + packages/common/y-octo/utils/fuzz/Cargo.lock | 1483 +++++++++++++++++ packages/common/y-octo/utils/fuzz/Cargo.toml | 88 + .../utils/fuzz/fuzz_targets/apply_update.rs | 51 + .../utils/fuzz/fuzz_targets/codec_doc_any.rs | 17 + .../fuzz/fuzz_targets/codec_doc_any_struct.rs | 43 + .../utils/fuzz/fuzz_targets/decode_bytes.rs | 11 + .../utils/fuzz/fuzz_targets/i32_decode.rs | 18 + .../utils/fuzz/fuzz_targets/i32_encode.rs | 17 + .../utils/fuzz/fuzz_targets/ins_del_text.rs | 34 + .../utils/fuzz/fuzz_targets/sync_message.rs | 20 + .../utils/fuzz/fuzz_targets/u64_decode.rs | 18 + .../utils/fuzz/fuzz_targets/u64_encode.rs | 17 + packages/common/y-octo/utils/src/codec.rs | 78 + packages/common/y-octo/utils/src/doc.rs | 21 + .../y-octo/utils/src/doc_operation/mod.rs | 5 + .../y-octo/utils/src/doc_operation/types.rs | 63 + .../utils/src/doc_operation/yrs_op/array.rs | 172 ++ .../utils/src/doc_operation/yrs_op/map.rs | 168 ++ .../utils/src/doc_operation/yrs_op/mod.rs | 193 +++ .../utils/src/doc_operation/yrs_op/text.rs | 180 ++ .../src/doc_operation/yrs_op/xml_element.rs | 45 + .../src/doc_operation/yrs_op/xml_fragment.rs | 45 + .../src/doc_operation/yrs_op/xml_text.rs | 62 + packages/common/y-octo/utils/src/lib.rs | 7 + tools/commitlint/.commitlintrc.json | 3 +- tools/utils/src/workspace.gen.ts | 6 + tsconfig.json | 1 + yarn.lock | 31 +- 127 files changed, 17319 insertions(+), 18 deletions(-) create mode 100644 packages/common/y-octo/core/Cargo.toml create mode 100644 packages/common/y-octo/core/LICENSE create mode 100644 packages/common/y-octo/core/README.md create mode 100644 packages/common/y-octo/core/benches/apply_benchmarks.rs create mode 100644 packages/common/y-octo/core/benches/array_ops_benchmarks.rs create mode 100644 packages/common/y-octo/core/benches/codec_benchmarks.rs create mode 100644 packages/common/y-octo/core/benches/map_ops_benchmarks.rs create mode 100644 packages/common/y-octo/core/benches/text_ops_benchmarks.rs create mode 100644 packages/common/y-octo/core/benches/update_benchmarks.rs create mode 100644 packages/common/y-octo/core/benches/utils/files.rs create mode 100644 packages/common/y-octo/core/benches/utils/mod.rs create mode 100644 packages/common/y-octo/core/src/codec/buffer.rs create mode 100644 packages/common/y-octo/core/src/codec/integer.rs create mode 100644 packages/common/y-octo/core/src/codec/mod.rs create mode 100644 packages/common/y-octo/core/src/codec/string.rs create mode 100644 packages/common/y-octo/core/src/doc/awareness.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/any.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/content.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/delete_set.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/id.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/io/codec_v1.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/io/mod.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/io/reader.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/io/writer.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/item.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/item_flag.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/mod.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/refs.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/update.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/utils/items.rs create mode 100644 packages/common/y-octo/core/src/doc/codec/utils/mod.rs create mode 100644 packages/common/y-octo/core/src/doc/common/mod.rs create mode 100644 packages/common/y-octo/core/src/doc/common/range.rs create mode 100644 packages/common/y-octo/core/src/doc/common/somr.rs create mode 100644 packages/common/y-octo/core/src/doc/common/state.rs create mode 100644 packages/common/y-octo/core/src/doc/document.rs create mode 100644 packages/common/y-octo/core/src/doc/hasher.rs create mode 100644 packages/common/y-octo/core/src/doc/history.rs create mode 100644 packages/common/y-octo/core/src/doc/mod.rs create mode 100644 packages/common/y-octo/core/src/doc/publisher.rs create mode 100644 packages/common/y-octo/core/src/doc/store.rs create mode 100644 packages/common/y-octo/core/src/doc/types/array.rs create mode 100644 packages/common/y-octo/core/src/doc/types/list/iterator.rs create mode 100644 packages/common/y-octo/core/src/doc/types/list/mod.rs create mode 100644 packages/common/y-octo/core/src/doc/types/list/search_marker.rs create mode 100644 packages/common/y-octo/core/src/doc/types/map.rs create mode 100644 packages/common/y-octo/core/src/doc/types/mod.rs create mode 100644 packages/common/y-octo/core/src/doc/types/text.rs create mode 100644 packages/common/y-octo/core/src/doc/types/value.rs create mode 100644 packages/common/y-octo/core/src/doc/types/xml.rs create mode 100644 packages/common/y-octo/core/src/doc/utils.rs create mode 100644 packages/common/y-octo/core/src/fixtures/basic.bin create mode 100644 packages/common/y-octo/core/src/fixtures/database.bin create mode 100644 packages/common/y-octo/core/src/fixtures/edge-case-left-right-same-node.bin create mode 100644 packages/common/y-octo/core/src/fixtures/large.bin create mode 100644 packages/common/y-octo/core/src/fixtures/local_docs.json create mode 100644 packages/common/y-octo/core/src/fixtures/with-subdoc.bin create mode 100644 packages/common/y-octo/core/src/lib.rs create mode 100644 packages/common/y-octo/core/src/protocol/awareness.rs create mode 100644 packages/common/y-octo/core/src/protocol/doc.rs create mode 100644 packages/common/y-octo/core/src/protocol/mod.rs create mode 100644 packages/common/y-octo/core/src/protocol/scanner.rs create mode 100644 packages/common/y-octo/core/src/protocol/sync.rs create mode 100644 packages/common/y-octo/core/src/sync.rs create mode 100644 packages/common/y-octo/node/.gitignore create mode 100644 packages/common/y-octo/node/Cargo.toml create mode 100644 packages/common/y-octo/node/build.rs create mode 100644 packages/common/y-octo/node/index.d.ts create mode 100644 packages/common/y-octo/node/index.js create mode 100644 packages/common/y-octo/node/package.json create mode 100755 packages/common/y-octo/node/scripts/run-test.mts create mode 100644 packages/common/y-octo/node/src/array.rs create mode 100644 packages/common/y-octo/node/src/doc.rs create mode 100644 packages/common/y-octo/node/src/lib.rs create mode 100644 packages/common/y-octo/node/src/map.rs create mode 100644 packages/common/y-octo/node/src/text.rs create mode 100644 packages/common/y-octo/node/src/utils.rs create mode 100644 packages/common/y-octo/node/tests/array.spec.mts create mode 100644 packages/common/y-octo/node/tests/doc.spec.mts create mode 100644 packages/common/y-octo/node/tests/map.spec.mts create mode 100644 packages/common/y-octo/node/tests/text.spec.mts create mode 100644 packages/common/y-octo/node/tsconfig.json create mode 100644 packages/common/y-octo/utils/Cargo.toml create mode 100644 packages/common/y-octo/utils/benches/apply_benchmarks.rs create mode 100644 packages/common/y-octo/utils/benches/array_ops_benchmarks.rs create mode 100644 packages/common/y-octo/utils/benches/codec_benchmarks.rs create mode 100644 packages/common/y-octo/utils/benches/map_ops_benchmarks.rs create mode 100644 packages/common/y-octo/utils/benches/text_ops_benchmarks.rs create mode 100644 packages/common/y-octo/utils/benches/update_benchmarks.rs create mode 100644 packages/common/y-octo/utils/benches/utils/files.rs create mode 100644 packages/common/y-octo/utils/benches/utils/mod.rs create mode 100644 packages/common/y-octo/utils/bin/bench_result_render.rs create mode 100644 packages/common/y-octo/utils/bin/doc_merger.rs create mode 100644 packages/common/y-octo/utils/bin/memory_leak_test.rs create mode 100644 packages/common/y-octo/utils/fuzz/.gitignore create mode 100644 packages/common/y-octo/utils/fuzz/Cargo.lock create mode 100644 packages/common/y-octo/utils/fuzz/Cargo.toml create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/apply_update.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/codec_doc_any.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/codec_doc_any_struct.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/decode_bytes.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/i32_decode.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/i32_encode.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/ins_del_text.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/sync_message.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/u64_decode.rs create mode 100644 packages/common/y-octo/utils/fuzz/fuzz_targets/u64_encode.rs create mode 100644 packages/common/y-octo/utils/src/codec.rs create mode 100644 packages/common/y-octo/utils/src/doc.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/mod.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/types.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/yrs_op/array.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/yrs_op/map.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/yrs_op/mod.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/yrs_op/text.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_element.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_fragment.rs create mode 100644 packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_text.rs create mode 100644 packages/common/y-octo/utils/src/lib.rs diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e972ed853b..008ed9051a 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -661,6 +661,142 @@ jobs: name: affine fail_ci_if_error: false + miri: + name: miri code check + runs-on: ubuntu-latest + needs: + - optimize_ci + if: needs.optimize_ci.outputs.skip == 'false' + env: + RUST_BACKTRACE: full + CARGO_TERM_COLOR: always + MIRIFLAGS: -Zmiri-backtrace=full -Zmiri-tree-borrows + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly + components: miri + - name: Install latest nextest release + uses: taiki-e/install-action@nextest + + - name: Miri Code Check + continue-on-error: true + run: | + cargo +nightly miri nextest run -p y-octo -j4 + + loom: + name: loom thread test + runs-on: ubuntu-latest + needs: + - optimize_ci + if: needs.optimize_ci.outputs.skip == 'false' + env: + RUSTFLAGS: --cfg loom + RUST_BACKTRACE: full + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + - name: Install latest nextest release + uses: taiki-e/install-action@nextest + + - name: Loom Thread Test + run: | + cargo nextest run -p y-octo --lib + + fuzzing: + name: fuzzing + runs-on: ubuntu-latest + needs: + - optimize_ci + if: needs.optimize_ci.outputs.skip == 'false' + env: + RUSTFLAGS: -D warnings + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly + + - name: fuzzing + working-directory: ./packages/common/y-octo/utils + run: | + cargo install cargo-fuzz + cargo +nightly fuzz run apply_update -- -max_total_time=30 + cargo +nightly fuzz run codec_doc_any_struct -- -max_total_time=30 + cargo +nightly fuzz run codec_doc_any -- -max_total_time=30 + cargo +nightly fuzz run decode_bytes -- -max_total_time=30 + cargo +nightly fuzz run i32_decode -- -max_total_time=30 + cargo +nightly fuzz run i32_encode -- -max_total_time=30 + cargo +nightly fuzz run ins_del_text -- -max_total_time=30 + cargo +nightly fuzz run sync_message -- -max_total_time=30 + cargo +nightly fuzz run u64_decode -- -max_total_time=30 + cargo +nightly fuzz run u64_encode -- -max_total_time=30 + cargo +nightly fuzz run apply_update -- -max_total_time=30 + + - name: upload fuzz artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: fuzz-artifact + path: packages/common/y-octo/utils/fuzz/artifacts/**/* + + y-octo-binding-test: + name: y-octo binding test on ${{ matrix.settings.target }} + runs-on: ${{ matrix.settings.os }} + strategy: + fail-fast: false + matrix: + settings: + - { target: 'x86_64-unknown-linux-gnu', os: 'ubuntu-latest' } + - { target: 'aarch64-unknown-linux-gnu', os: 'ubuntu-24.04-arm' } + - { target: 'x86_64-apple-darwin', os: 'macos-13' } + - { target: 'aarch64-apple-darwin', os: 'macos-latest' } + - { target: 'x86_64-pc-windows-msvc', os: 'windows-latest' } + - { target: 'aarch64-pc-windows-msvc', os: 'windows-11-arm' } + needs: + - optimize_ci + if: needs.optimize_ci.outputs.skip == 'false' + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: ./.github/actions/setup-node + with: + extra-flags: workspaces focus @affine-tools/cli @affine/monorepo @y-octo/node + electron-install: false + - name: Install rustup (Windows 11 ARM) + if: matrix.settings.os == 'windows-11-arm' + shell: pwsh + run: | + Invoke-WebRequest -Uri "https://static.rust-lang.org/rustup/dist/aarch64-pc-windows-msvc/rustup-init.exe" -OutFile rustup-init.exe + .\rustup-init.exe --default-toolchain none -y + "$env:USERPROFILE\.cargo\bin" | Out-File -Append -Encoding ascii $env:GITHUB_PATH + "CARGO_HOME=$env:USERPROFILE\.cargo" | Out-File -Append -Encoding ascii $env:GITHUB_ENV + - name: Install Rust (Windows 11 ARM) + if: matrix.settings.os == 'windows-11-arm' + shell: pwsh + run: | + rustup install stable + rustup target add ${{ matrix.settings.target }} + cargo --version + - name: Build Rust + uses: ./.github/actions/build-rust + with: + target: ${{ matrix.settings.target }} + package: '@y-octo/node' + - name: Run tests + run: yarn affine @y-octo/node test + rust-test: name: Run native tests runs-on: ubuntu-latest @@ -1185,6 +1321,10 @@ jobs: - build-server-native - build-electron-renderer - native-unit-test + - miri + - loom + - fuzzing + - y-octo-binding-test - server-test - server-e2e-test - rust-test diff --git a/Cargo.lock b/Cargo.lock index 3013fd7f74..b2c97855e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -229,6 +229,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anes" version = "0.2.0" @@ -300,12 +306,28 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-compat" version = "0.2.4" @@ -331,6 +353,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "atoi" version = "2.0.0" @@ -340,6 +373,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + [[package]] name = "auto_enums" version = "0.8.7" @@ -434,7 +473,16 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" dependencies = [ - "bit-vec", + "bit-vec 0.6.3", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", ] [[package]] @@ -443,6 +491,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -943,13 +997,49 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes 0.1.6", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "criterion2" version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b43b9cdbf592c78d882f2a3b9e6ebe8aedc749ef84915103a0248802ce2f6b3" dependencies = [ - "anes", + "anes 0.2.0", "bpaf", "cast", "ciborium", @@ -1270,7 +1360,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" dependencies = [ - "bit-set", + "bit-set 0.5.3", "regex-automata 0.4.9", "regex-syntax 0.8.5", ] @@ -1280,6 +1370,9 @@ name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +dependencies = [ + "getrandom 0.2.15", +] [[package]] name = "file-format" @@ -1482,6 +1575,19 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -1489,8 +1595,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1599,6 +1707,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" + [[package]] name = "hex" version = "0.4.3" @@ -1885,12 +1999,32 @@ dependencies = [ "libc", ] +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1974,6 +2108,17 @@ dependencies = [ "leak", ] +[[package]] +name = "lib0" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf23122cb1c970b77ea6030eac5e328669415b65d2ab245c99bfb110f9d62dc" +dependencies = [ + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "libc" version = "0.2.172" @@ -2108,7 +2253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" dependencies = [ "log", - "phf", + "phf 0.10.1", "phf_codegen", "string_cache", "string_cache_codegen", @@ -2484,12 +2629,13 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "ordered-float" -version = "4.6.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +checksum = "e2c1f9f56e534ac6a9b8a4600bdf0f530fb393b5f393e7b4d03489c3cf0c3f01" dependencies = [ "arbitrary", "num-traits", + "proptest", ] [[package]] @@ -2598,6 +2744,16 @@ dependencies = [ "phf_shared 0.10.0", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + [[package]] name = "phf_codegen" version = "0.10.0" @@ -2628,6 +2784,19 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "phf_shared" version = "0.10.0" @@ -2691,6 +2860,34 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "pom" version = "1.1.0" @@ -2745,6 +2942,37 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" +dependencies = [ + "bit-set 0.8.0", + "bit-vec 0.8.0", + "bitflags 2.9.0", + "lazy_static", + "num-traits", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_xorshift", + "regex-syntax 0.8.5", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "proptest-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "pulldown-cmark" version = "0.13.0" @@ -2756,6 +2984,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.40" @@ -2777,6 +3011,19 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -2799,6 +3046,16 @@ dependencies = [ "zerocopy 0.8.24", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -2819,6 +3076,15 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -2839,12 +3105,30 @@ dependencies = [ [[package]] name = "rand_distr" -version = "0.4.3" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.9.0", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", ] [[package]] @@ -3125,6 +3409,18 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.20" @@ -3329,6 +3625,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallstr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b1aefdf380735ff8ded0b15f31aab05daf1f70216c01c02a12926badd1df9d" +dependencies = [ + "smallvec", +] + [[package]] name = "smallvec" version = "1.15.0" @@ -4035,6 +4340,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.9.0" @@ -4333,6 +4648,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.8.1" @@ -4565,6 +4886,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -4575,6 +4905,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -4654,6 +4990,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.8" @@ -5108,26 +5454,73 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "y-octo" version = "0.0.1" -source = "git+https://github.com/y-crdt/y-octo.git?branch=main#1e705e4c9bb10dec5b7893c225fba2436a6e02f3" dependencies = [ "ahash", "arbitrary", + "assert-json-diff", "async-lock", "bitvec", "byteorder", + "criterion", "lasso", + "lib0", "log", "loom", "nanoid", - "nom 7.1.3", + "nom 8.0.0", "ordered-float", - "rand 0.8.5", - "rand_chacha 0.3.1", + "path-ext", + "proptest", + "proptest-derive", + "rand 0.9.0", + "rand_chacha 0.9.0", "rand_distr", "serde", "serde_json", "smol_str", "thiserror 2.0.12", + "yrs 0.23.0", +] + +[[package]] +name = "y-octo-node" +version = "0.0.1" +dependencies = [ + "anyhow", + "napi", + "napi-build", + "napi-derive", + "y-octo", +] + +[[package]] +name = "y-octo-utils" +version = "0.0.1" +dependencies = [ + "arbitrary", + "clap", + "criterion", + "lib0", + "path-ext", + "phf 0.11.3", + "proptest", + "proptest-derive", + "rand 0.9.0", + "rand_chacha 0.9.0", + "regex", + "y-octo", + "y-sync", + "yrs 0.23.0", +] + +[[package]] +name = "y-sync" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e3675a497cde881a71e7e5c2ae1d087dfc7733ddece9b24a9a61408e969d3b" +dependencies = [ + "thiserror 1.0.69", + "yrs 0.17.4", ] [[package]] @@ -5154,6 +5547,39 @@ dependencies = [ "synstructure", ] +[[package]] +name = "yrs" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4830316bfee4bec0044fe34a001cda783506d5c4c0852f8433c6041dfbfce51" +dependencies = [ + "atomic_refcell", + "rand 0.7.3", + "serde", + "serde_json", + "smallstr", + "smallvec", + "thiserror 1.0.69", +] + +[[package]] +name = "yrs" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0189b51d8ab1283e7c1f1f515c610875262e629cf258bec530da5cd4aa115d59" +dependencies = [ + "arc-swap", + "async-lock", + "async-trait", + "dashmap", + "fastrand", + "serde", + "serde_json", + "smallstr", + "smallvec", + "thiserror 1.0.69", +] + [[package]] name = "zerocopy" version = "0.7.35" diff --git a/Cargo.toml b/Cargo.toml index 5165645f4e..f1fc3edf55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,9 @@ members = [ "./packages/backend/native", "./packages/common/native", + "./packages/common/y-octo/core", + "./packages/common/y-octo/node", + "./packages/common/y-octo/utils", "./packages/frontend/mobile-native", "./packages/frontend/native", "./packages/frontend/native/nbstore", @@ -16,12 +19,20 @@ edition = "2024" [workspace.dependencies] affine_common = { path = "./packages/common/native" } affine_nbstore = { path = "./packages/frontend/native/nbstore" } +ahash = "0.8" anyhow = "1" +arbitrary = { version = "1.3", features = ["derive"] } +assert-json-diff = "2.0" +async-lock = { version = "3.4.0", features = ["loom"] } base64-simd = "0.8" +bitvec = "1.0" block2 = "0.6" +byteorder = "1.5" chrono = "0.4" +clap = { version = "4.4", features = ["derive"] } core-foundation = "0.10" coreaudio-rs = "0.12" +criterion = { version = "0.5", features = ["html_reports"] } criterion2 = { version = "3", default-features = false } dispatch2 = "0.2" docx-parser = { git = "https://github.com/toeverything/docx-parser" } @@ -29,26 +40,40 @@ dotenvy = "0.15" file-format = { version = "0.26", features = ["reader"] } homedir = "0.3" infer = { version = "0.19.0" } +lasso = { version = "0.7", features = ["multi-threaded"] } +lib0 = { version = "0.16", features = ["lib0-serde"] } libc = "0.2" +log = "0.4" +loom = { version = "0.7", features = ["checkpoint"] } mimalloc = "0.1" +nanoid = "0.4" napi = { version = "3.0.0-alpha.31", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] } napi-build = { version = "2" } napi-derive = { version = "3.0.0-alpha.28" } +nom = "8" notify = { version = "8", features = ["serde"] } objc2 = "0.6" objc2-foundation = "0.3" once_cell = "1" +ordered-float = "5" parking_lot = "0.12" path-ext = "0.1.1" pdf-extract = { git = "https://github.com/toeverything/pdf-extract", branch = "darksky/improve-font-decoding" } +phf = { version = "0.11", features = ["macros"] } +proptest = "1.3" +proptest-derive = "0.5" rand = "0.9" +rand_chacha = "0.9" +rand_distr = "0.5" rayon = "1.10" readability = { version = "0.3.0", default-features = false } +regex = "1.10" rubato = "0.16" screencapturekit = "0.3" serde = "1" serde_json = "1" sha3 = "0.10" +smol_str = "0.3" sqlx = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] } strum_macros = "0.27.0" symphonia = { version = "0.5", features = ["all", "opt-simd"] } @@ -72,7 +97,9 @@ uniffi = "0.29" url = { version = "2.5" } uuid = "1.8" v_htmlescape = "0.15" -y-octo = { git = "https://github.com/y-crdt/y-octo.git", branch = "main" } +y-octo = { path = "./packages/common/y-octo/core" } +y-sync = { version = "0.4" } +yrs = "0.23.0" [profile.dev.package.sqlx-macros] opt-level = 3 diff --git a/package.json b/package.json index 6790e829b3..565845beda 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ ".", "blocksuite/**/*", "packages/*/*", + "packages/common/y-octo/node", "packages/frontend/apps/*", "tools/*", "docs/reference", diff --git a/packages/common/y-octo/core/Cargo.toml b/packages/common/y-octo/core/Cargo.toml new file mode 100644 index 0000000000..6dc343e5c9 --- /dev/null +++ b/packages/common/y-octo/core/Cargo.toml @@ -0,0 +1,95 @@ +[package] +authors = [ + "DarkSky ", + "forehalo ", + "x1a0t <405028157@qq.com>", + "Brooklyn ", +] +description = "High-performance and thread-safe CRDT implementation compatible with Yjs" +edition = "2021" +homepage = "https://github.com/toeverything/y-octo" +include = ["src/**/*", "benches/**/*", "bin/**/*", "LICENSE", "README.md"] +keywords = ["collaboration", "crdt", "crdts", "yjs", "yata"] +license = "MIT" +name = "y-octo" +readme = "README.md" +repository = "https://github.com/toeverything/y-octo" +version = "0.0.1" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ahash = { workspace = true } +bitvec = { workspace = true } +byteorder = { workspace = true } +lasso = { workspace = true } +log = { workspace = true } +nanoid = { workspace = true } +nom = { workspace = true } +ordered-float = { workspace = true } +rand = { workspace = true } +rand_chacha = { workspace = true } +rand_distr = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +smol_str = { workspace = true } +thiserror = { workspace = true } + +[features] +bench = [] +debug = [] +large_refs = [] +serde_json = [] + +[target.'cfg(fuzzing)'.dependencies] +arbitrary = { workspace = true } +ordered-float = { workspace = true, features = ["arbitrary"] } + +[target.'cfg(loom)'.dependencies] +loom = { workspace = true } +# override the dev-dependencies feature +async-lock = { workspace = true } + +[dev-dependencies] +assert-json-diff = { workspace = true } +criterion = { workspace = true } +lib0 = { workspace = true } +ordered-float = { workspace = true, features = ["proptest"] } +path-ext = { workspace = true } +proptest = { workspace = true } +proptest-derive = { workspace = true } +yrs = { workspace = true } + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = [ + 'cfg(debug)', + 'cfg(fuzzing)', + 'cfg(loom)', +] } + +[[bench]] +harness = false +name = "array_ops_benchmarks" + +[[bench]] +harness = false +name = "codec_benchmarks" + +[[bench]] +harness = false +name = "map_ops_benchmarks" + +[[bench]] +harness = false +name = "text_ops_benchmarks" + +[[bench]] +harness = false +name = "apply_benchmarks" + +[[bench]] +harness = false +name = "update_benchmarks" + +[lib] +bench = true diff --git a/packages/common/y-octo/core/LICENSE b/packages/common/y-octo/core/LICENSE new file mode 100644 index 0000000000..9c5d15f21d --- /dev/null +++ b/packages/common/y-octo/core/LICENSE @@ -0,0 +1,9 @@ +The MIT License (MIT) + +Copyright (c) 2022-present TOEVERYTHING PTE. LTD. and its affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/packages/common/y-octo/core/README.md b/packages/common/y-octo/core/README.md new file mode 100644 index 0000000000..126b83231c --- /dev/null +++ b/packages/common/y-octo/core/README.md @@ -0,0 +1,100 @@ +# Y-Octo + +[![test](https://github.com/toeverything/y-octo/actions/workflows/y-octo.yml/badge.svg)](https://github.com/toeverything/y-octo/actions/workflows/y-octo.yml) +[![docs]](https://docs.rs/y-octo/latest/y_octo) +[![crates]](https://crates.io/crates/y-octo) +[![codecov]](https://codecov.io/gh/toeverything/y-octo) + +Y-Octo is a high-performance CRDT implementation compatible with [yjs]. + +## Introduction + +Y-Octo is a tiny, ultra-fast CRDT collaboration library built for all major platforms. Developers can use Y-Octo as the [Single source of truth](https://en.wikipedia.org/wiki/Single_source_of_truth) for their application state, naturally turning the application into a [local-first](https://www.inkandswitch.com/local-first/) collaborative app. + +Y-Octo also has interoperability and binary compatibility with [yjs]. Developers can use [yjs] to develop local-first web applications and collaborate with Y-Octo in native apps alongside web apps. + +## Who are using + + + +[AFFiNE](https://affine.pro) is using y-octo in production. There are [Electron](https://affine.pro/download) app and [Node.js server](https://github.com/toeverything/AFFiNE/tree/canary/packages/backend/native) using y-octo in production. + + + +[Mysc](https://www.mysc.app/) is using y-octo in the Rust server, and the iOS/Android client via the Swift/Kotlin bindings (Official bindings coming soon). + +## Features + +- ✅ Collaborative Text + - ✅ Read and write styled Unicode compatible data. + - 🚧 Add, modify and delete text styles. + - 🚧 Embedded JS data types and collaborative types. + - ✅ Collaborative types of thread-safe. +- Collaborative Array + - ✅ Add, modify, and delete basic JS data types. + - ✅ Recursively add, modify, and delete collaborative types. + - ✅ Collaborative types of thread-safe. + - 🚧 Recursive event subscription +- Collaborative Map + - ✅ Add, modify, and delete basic JS data types. + - ✅ Recursively add, modify, and delete collaborative types. + - ✅ Collaborative types of thread-safe. + - 🚧 Recursive event subscription +- 🚧 Collaborative Xml (Fragment / Element) +- ✅ Collaborative Doc Container + - ✅ YATA CRDT state apply/diff compatible with [yjs] + - ✅ State sync of thread-safe. + - ✅ Store all collaborative types and JS data types + - ✅ Update event subscription. + - 🚧 Sub Document. +- ✅ Yjs binary encoding + - ✅ Awareness encoding. + - ✅ Primitive type encoding. + - ✅ Sync Protocol encoding. + - ✅ Yjs update v1 encoding. + - 🚧 Yjs update v2 encoding. + +## Testing & Linting + +Put everything to the test! We've established various test suites, but we're continually striving to enhance our coverage： + +- Rust Tests + - Unit tests + - [Loom](https://docs.rs/loom/latest/loom/) multi-threading tests + - [Miri](https://github.com/rust-lang/miri) undefined behavior tests + - [Address Sanitizer](https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html) memory error detections + - [Fuzzing](https://github.com/rust-fuzz/cargo-fuzz) fuzzing tests +- Node Tests +- Smoke Tests +- Eslint, Clippy + +## Related projects + +- [OctoBase]: The open-source embedded database based on Y-Octo. +- [yjs]: Shared data types for building collaborative software in web. + +## Maintainers + +- [DarkSky](https://github.com/darkskygit) +- [liuyi](https://github.com/forehalo) +- [LongYinan](https://github.com/Brooooooklyn) + +## Why not [yrs](https://github.com/y-crdt/y-crdt/) + +See [Why we're not using yrs](./y-octo-utils/yrs-is-unsafe/README.md) + +## License + +Y-Octo are [MIT licensed]. + +[codecov]: https://codecov.io/gh/toeverything/y-octo/graph/badge.svg?token=9AQY5Q1BYH +[crates]: https://img.shields.io/crates/v/y-octo.svg +[docs]: https://img.shields.io/docsrs/y-octo.svg +[test]: https://github.com/toeverything/y-octo/actions/workflows/y-octo.yml/badge.svg +[yjs]: https://github.com/yjs/yjs +[Address Sanitizer]: https://github.com/toeverything/y-octo/actions/workflows/y-octo-asan.yml/badge.svg +[Memory Leak Detect]: https://github.com/toeverything/y-octo/actions/workflows/y-octo-memory-test.yml/badge.svg +[OctoBase]: https://github.com/toeverything/octobase +[BlockSuite]: https://github.com/toeverything/blocksuite +[AFFiNE]: https://github.com/toeverything/affine +[MIT licensed]: ./LICENSE diff --git a/packages/common/y-octo/core/benches/apply_benchmarks.rs b/packages/common/y-octo/core/benches/apply_benchmarks.rs new file mode 100644 index 0000000000..2409aefb04 --- /dev/null +++ b/packages/common/y-octo/core/benches/apply_benchmarks.rs @@ -0,0 +1,34 @@ +mod utils; + +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use path_ext::PathExt; +use utils::Files; + +fn apply(c: &mut Criterion) { + let files = Files::load(); + + let mut group = c.benchmark_group("apply"); + group.measurement_time(Duration::from_secs(15)); + + for file in &files.files { + group.throughput(Throughput::Bytes(file.content.len() as u64)); + group.bench_with_input( + BenchmarkId::new("apply with jwst", file.path.name_str()), + &file.content, + |b, content| { + b.iter(|| { + use y_octo::*; + let mut doc = Doc::new(); + doc.apply_update_from_binary_v1(content.clone()).unwrap() + }); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, apply); +criterion_main!(benches); diff --git a/packages/common/y-octo/core/benches/array_ops_benchmarks.rs b/packages/common/y-octo/core/benches/array_ops_benchmarks.rs new file mode 100644 index 0000000000..c931974e0d --- /dev/null +++ b/packages/common/y-octo/core/benches/array_ops_benchmarks.rs @@ -0,0 +1,71 @@ +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Criterion}; +use rand::{Rng, SeedableRng}; + +fn operations(c: &mut Criterion) { + let mut group = c.benchmark_group("ops/array"); + group.measurement_time(Duration::from_secs(15)); + + group.bench_function("jwst/insert", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); + + let idxs = (0..99) + .map(|_| rng.random_range(0..base_text.len() as u64)) + .collect::>(); + b.iter(|| { + use y_octo::*; + let doc = Doc::default(); + let mut array = doc.get_or_create_array("test").unwrap(); + for c in base_text.chars() { + array.push(c.to_string()).unwrap(); + } + for idx in &idxs { + array.insert(*idx, "test").unwrap(); + } + }); + }); + + group.bench_function("jwst/insert range", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); + + let idxs = (0..99) + .map(|_| rng.random_range(0..base_text.len() as u64)) + .collect::>(); + b.iter(|| { + use y_octo::*; + let doc = Doc::default(); + let mut array = doc.get_or_create_array("test").unwrap(); + for c in base_text.chars() { + array.push(c.to_string()).unwrap(); + } + for idx in &idxs { + array.insert(*idx, "test1").unwrap(); + array.insert(idx + 1, "test2").unwrap(); + } + }); + }); + + group.bench_function("jwst/remove", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + + b.iter(|| { + use y_octo::*; + let doc = Doc::default(); + let mut array = doc.get_or_create_array("test").unwrap(); + for c in base_text.chars() { + array.push(c.to_string()).unwrap(); + } + for idx in (0..base_text.len() as u64).rev() { + array.remove(idx, 1).unwrap(); + } + }); + }); + + group.finish(); +} + +criterion_group!(benches, operations); +criterion_main!(benches); diff --git a/packages/common/y-octo/core/benches/codec_benchmarks.rs b/packages/common/y-octo/core/benches/codec_benchmarks.rs new file mode 100644 index 0000000000..c4dea3ec54 --- /dev/null +++ b/packages/common/y-octo/core/benches/codec_benchmarks.rs @@ -0,0 +1,91 @@ +use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; +use y_octo::{read_var_i32, read_var_u64, write_var_i32, write_var_u64}; + +const BENCHMARK_SIZE: u32 = 100000; + +fn codec(c: &mut Criterion) { + let mut codec_group = c.benchmark_group("codec"); + codec_group.sampling_mode(SamplingMode::Flat); + + { + codec_group.bench_function("jwst encode var_int (32 bit)", |b| { + b.iter(|| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..(BENCHMARK_SIZE as i32) { + write_var_i32(&mut encoder, i).unwrap(); + } + }) + }); + codec_group.bench_function("jwst decode var_int (32 bit)", |b| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..(BENCHMARK_SIZE as i32) { + write_var_i32(&mut encoder, i).unwrap(); + } + + b.iter(|| { + let mut decoder = encoder.as_slice(); + for i in 0..(BENCHMARK_SIZE as i32) { + let (tail, num) = read_var_i32(decoder).unwrap(); + decoder = tail; + assert_eq!(num, i); + } + }) + }); + } + + { + codec_group.bench_function("jwst encode var_uint (32 bit)", |b| { + b.iter(|| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..BENCHMARK_SIZE { + write_var_u64(&mut encoder, i as u64).unwrap(); + } + }) + }); + codec_group.bench_function("jwst decode var_uint (32 bit)", |b| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..BENCHMARK_SIZE { + write_var_u64(&mut encoder, i as u64).unwrap(); + } + + b.iter(|| { + let mut decoder = encoder.as_slice(); + for i in 0..BENCHMARK_SIZE { + let (tail, num) = read_var_u64(decoder).unwrap(); + decoder = tail; + assert_eq!(num as u32, i); + } + }) + }); + } + + { + codec_group.bench_function("jwst encode var_uint (64 bit)", |b| { + b.iter(|| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..(BENCHMARK_SIZE as u64) { + write_var_u64(&mut encoder, i).unwrap(); + } + }) + }); + + codec_group.bench_function("jwst decode var_uint (64 bit)", |b| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..(BENCHMARK_SIZE as u64) { + write_var_u64(&mut encoder, i).unwrap(); + } + + b.iter(|| { + let mut decoder = encoder.as_slice(); + for i in 0..(BENCHMARK_SIZE as u64) { + let (tail, num) = read_var_u64(decoder).unwrap(); + decoder = tail; + assert_eq!(num, i); + } + }) + }); + } +} + +criterion_group!(benches, codec); +criterion_main!(benches); diff --git a/packages/common/y-octo/core/benches/map_ops_benchmarks.rs b/packages/common/y-octo/core/benches/map_ops_benchmarks.rs new file mode 100644 index 0000000000..6451223769 --- /dev/null +++ b/packages/common/y-octo/core/benches/map_ops_benchmarks.rs @@ -0,0 +1,65 @@ +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Criterion}; + +fn operations(c: &mut Criterion) { + let mut group = c.benchmark_group("ops/map"); + group.measurement_time(Duration::from_secs(15)); + + group.bench_function("jwst/insert", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" + .split(' ') + .collect::>(); + + b.iter(|| { + use y_octo::*; + let doc = Doc::default(); + let mut map = doc.get_or_create_map("test").unwrap(); + for (idx, key) in base_text.iter().enumerate() { + map.insert(key.to_string(), idx).unwrap(); + } + }); + }); + + group.bench_function("jwst/get", |b| { + use y_octo::*; + + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" + .split(' ') + .collect::>(); + let doc = Doc::default(); + let mut map = doc.get_or_create_map("test").unwrap(); + for (idx, key) in base_text.iter().enumerate() { + map.insert(key.to_string(), idx).unwrap(); + } + + b.iter(|| { + for key in &base_text { + map.get(key); + } + }); + }); + + group.bench_function("jwst/remove", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" + .split(' ') + .collect::>(); + + b.iter(|| { + use y_octo::*; + let doc = Doc::default(); + let mut map = doc.get_or_create_map("test").unwrap(); + for (idx, key) in base_text.iter().enumerate() { + map.insert(key.to_string(), idx).unwrap(); + } + for key in &base_text { + map.remove(key); + } + }); + }); + + group.finish(); +} + +criterion_group!(benches, operations); +criterion_main!(benches); diff --git a/packages/common/y-octo/core/benches/text_ops_benchmarks.rs b/packages/common/y-octo/core/benches/text_ops_benchmarks.rs new file mode 100644 index 0000000000..ed031d1b1a --- /dev/null +++ b/packages/common/y-octo/core/benches/text_ops_benchmarks.rs @@ -0,0 +1,50 @@ +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Criterion}; +use rand::{Rng, SeedableRng}; + +fn operations(c: &mut Criterion) { + let mut group = c.benchmark_group("ops/text"); + group.measurement_time(Duration::from_secs(15)); + + group.bench_function("jwst/insert", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); + + let idxs = (0..99) + .map(|_| rng.random_range(0..base_text.len() as u64)) + .collect::>(); + b.iter(|| { + use y_octo::*; + let doc = Doc::default(); + let mut text = doc.get_or_create_text("test").unwrap(); + + text.insert(0, base_text).unwrap(); + for idx in &idxs { + text.insert(*idx, "test").unwrap(); + } + }); + }); + + group.bench_function("jwst/remove", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + + b.iter(|| { + use y_octo::*; + let doc = Doc::default(); + let mut text = doc.get_or_create_text("test").unwrap(); + + text.insert(0, base_text).unwrap(); + text.insert(0, base_text).unwrap(); + text.insert(0, base_text).unwrap(); + for idx in (0..base_text.len() as u64).rev() { + text.remove(idx, 1).unwrap(); + } + }); + }); + + group.finish(); +} + +criterion_group!(benches, operations); +criterion_main!(benches); diff --git a/packages/common/y-octo/core/benches/update_benchmarks.rs b/packages/common/y-octo/core/benches/update_benchmarks.rs new file mode 100644 index 0000000000..e9820077c4 --- /dev/null +++ b/packages/common/y-octo/core/benches/update_benchmarks.rs @@ -0,0 +1,34 @@ +mod utils; + +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use path_ext::PathExt; +use utils::Files; + +fn update(c: &mut Criterion) { + let files = Files::load(); + + let mut group = c.benchmark_group("update"); + group.measurement_time(Duration::from_secs(15)); + + for file in &files.files { + group.throughput(Throughput::Bytes(file.content.len() as u64)); + group.bench_with_input( + BenchmarkId::new("parse with jwst", file.path.name_str()), + &file.content, + |b, content| { + b.iter(|| { + use y_octo::*; + let mut decoder = RawDecoder::new(content); + Update::read(&mut decoder).unwrap() + }); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, update); +criterion_main!(benches); diff --git a/packages/common/y-octo/core/benches/utils/files.rs b/packages/common/y-octo/core/benches/utils/files.rs new file mode 100644 index 0000000000..31374f3736 --- /dev/null +++ b/packages/common/y-octo/core/benches/utils/files.rs @@ -0,0 +1,42 @@ +use std::{ + fs::{read, read_dir}, + path::{Path, PathBuf}, +}; + +use path_ext::PathExt; + +pub struct File { + pub path: PathBuf, + pub content: Vec, +} + +const BASE: &str = "src/fixtures/"; + +impl File { + fn new(path: &Path) -> Self { + let content = read(path).unwrap(); + Self { + path: path.into(), + content, + } + } +} + +pub struct Files { + pub files: Vec, +} + +impl Files { + pub fn load() -> Self { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(BASE); + + let files = read_dir(path).unwrap(); + let files = files + .flatten() + .filter(|f| f.path().is_file() && f.path().ext_str() == "bin") + .map(|f| File::new(&f.path())) + .collect::>(); + + Self { files } + } +} diff --git a/packages/common/y-octo/core/benches/utils/mod.rs b/packages/common/y-octo/core/benches/utils/mod.rs new file mode 100644 index 0000000000..412eb3d0a6 --- /dev/null +++ b/packages/common/y-octo/core/benches/utils/mod.rs @@ -0,0 +1,3 @@ +mod files; + +pub use files::Files; diff --git a/packages/common/y-octo/core/src/codec/buffer.rs b/packages/common/y-octo/core/src/codec/buffer.rs new file mode 100644 index 0000000000..ed993a3312 --- /dev/null +++ b/packages/common/y-octo/core/src/codec/buffer.rs @@ -0,0 +1,87 @@ +use std::io::{Error, Write}; + +use nom::bytes::complete::take; + +use super::*; + +pub fn read_var_buffer(input: &[u8]) -> IResult<&[u8], &[u8]> { + let (tail, len) = read_var_u64(input)?; + let (tail, val) = take(len as usize)(tail)?; + Ok((tail, val)) +} + +pub fn write_var_buffer(buffer: &mut W, data: &[u8]) -> Result<(), Error> { + write_var_u64(buffer, data.len() as u64)?; + buffer.write_all(data)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use nom::{ + error::{Error, ErrorKind}, + AsBytes, Err, + }; + + use super::*; + + #[test] + fn test_read_var_buffer() { + // Test case 1: valid input, buffer length = 5 + let input = [0x05, 0x01, 0x02, 0x03, 0x04, 0x05]; + let expected_output = [0x01, 0x02, 0x03, 0x04, 0x05]; + let result = read_var_buffer(&input); + assert_eq!(result, Ok((&[][..], &expected_output[..]))); + + // Test case 2: truncated input, missing buffer + let input = [0x05, 0x01, 0x02, 0x03]; + let result = read_var_buffer(&input); + assert_eq!( + result, + Err(Err::Error(Error::new(&input[1..], ErrorKind::Eof))) + ); + + // Test case 3: invalid input + let input = [0xFF, 0x01, 0x02, 0x03]; + let result = read_var_buffer(&input); + assert_eq!( + result, + Err(Err::Error(Error::new(&input[2..], ErrorKind::Eof))) + ); + + // Test case 4: invalid var int encoding + let input = [0xFF, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]; + let result = read_var_buffer(&input); + assert_eq!( + result, + Err(Err::Error(Error::new(&input[7..], ErrorKind::Eof))) + ); + } + + #[test] + fn test_var_buf_codec() { + test_var_buf_enc_dec(&[]); + test_var_buf_enc_dec(&[0x01, 0x02, 0x03, 0x04, 0x05]); + test_var_buf_enc_dec(b"test_var_buf_enc_dec"); + + #[cfg(not(miri))] + { + use rand::{rng, Rng}; + let mut rng = rng(); + for _ in 0..100 { + test_var_buf_enc_dec(&{ + let mut bytes = vec![0u8; rng.random_range(0..u16::MAX as usize)]; + rng.fill(&mut bytes[..]); + bytes + }); + } + } + } + + fn test_var_buf_enc_dec(data: &[u8]) { + let mut buf = Vec::::new(); + write_var_buffer(&mut buf, data).unwrap(); + let result = read_var_buffer(buf.as_bytes()); + assert_eq!(result, Ok((&[][..], data))); + } +} diff --git a/packages/common/y-octo/core/src/codec/integer.rs b/packages/common/y-octo/core/src/codec/integer.rs new file mode 100644 index 0000000000..945620eea0 --- /dev/null +++ b/packages/common/y-octo/core/src/codec/integer.rs @@ -0,0 +1,166 @@ +use std::io::{Error, Write}; + +use byteorder::WriteBytesExt; +use nom::Needed; + +use super::*; + +pub fn read_var_u64(input: &[u8]) -> IResult<&[u8], u64> { + // parse the first byte + if let Some(next_byte) = input.first() { + let mut shift = 7; + let mut curr_byte = *next_byte; + let mut rest = &input[1..]; + + // same logic in loop, but enable early exit when dealing with small numbers + let mut num = (curr_byte & 0b0111_1111) as u64; + + // if the sign bit is set, we need more bits + while (curr_byte >> 7) & 0b1 != 0 { + if let Some(next_byte) = rest.first() { + curr_byte = *next_byte; + // add the remaining 7 bits to the number + num |= ((curr_byte & 0b0111_1111) as u64).wrapping_shl(shift); + shift += 7; + rest = &rest[1..]; + } else { + return Err(nom::Err::Incomplete(Needed::new(input.len() + 1))); + } + } + + Ok((rest, num)) + } else { + Err(nom::Err::Incomplete(Needed::new(1))) + } +} + +pub fn write_var_u64(buffer: &mut W, mut num: u64) -> Result<(), Error> { + // bit or 0b1000_0000 pre 7 bit if has more bits + while num >= 0b10000000 { + buffer.write_u8(num as u8 & 0b0111_1111 | 0b10000000)?; + num >>= 7; + } + + buffer.write_u8((num & 0b01111111) as u8)?; + + Ok(()) +} + +pub fn read_var_i32(input: &[u8]) -> IResult<&[u8], i32> { + // parse the first byte + if let Some(next_byte) = input.first() { + let mut shift = 6; + let mut curr_byte = *next_byte; + let mut rest: &[u8] = &input[1..]; + + // get the sign bit and the first 6 bits of the number + let sign_bit = (curr_byte >> 6) & 0b1; + let mut num = (curr_byte & 0b0011_1111) as i64; + + // if the sign bit is set, we need more bits + while (curr_byte >> 7) & 0b1 != 0 { + if let Some(next_byte) = rest.first() { + curr_byte = *next_byte; + // add the remaining 7 bits to the number + num |= ((curr_byte & 0b0111_1111) as i64).wrapping_shl(shift); + shift += 7; + rest = &rest[1..]; + } else { + return Err(nom::Err::Incomplete(Needed::new(input.len() + 1))); + } + } + + // negate the number if the sign bit is set + if sign_bit == 1 { + num = -num; + } + + Ok((rest, num as i32)) + } else { + Err(nom::Err::Incomplete(Needed::new(1))) + } +} + +pub fn write_var_i32(buffer: &mut W, num: i32) -> Result<(), Error> { + let mut num = num as i64; + let is_negative = num < 0; + if is_negative { + num = -num; + } + + buffer.write_u8( + // bit or 0b1000_0000 if has more bits + if num > 0b00111111 { 0b10000000 } else { 0 } + // bit or 0b0100_0000 if negative + | if is_negative { 0b0100_0000 } else { 0 } + // store last 6 bits + | num as u8 & 0b0011_1111, + )?; + num >>= 6; + while num > 0 { + buffer.write_u8( + // bit or 0b1000_0000 pre 7 bit if has more bits + if num > 0b01111111 { 0b10000000 } else { 0 } + // store last 7 bits + | num as u8 & 0b0111_1111, + )?; + num >>= 7; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + + use super::*; + + fn test_var_uint_enc_dec(num: u64) { + let mut buf = Vec::new(); + write_var_u64(&mut buf, num).unwrap(); + + let (rest, decoded_num) = read_var_u64(&buf).unwrap(); + assert_eq!(num, decoded_num); + assert_eq!(rest.len(), 0); + } + + fn test_var_int_enc_dec(num: i32) { + { + let mut buf = Vec::new(); + write_var_i32(&mut buf, num).unwrap(); + + let (rest, decoded_num) = read_var_i32(&buf).unwrap(); + assert_eq!(num, decoded_num); + assert_eq!(rest.len(), 0); + } + } + + #[test] + fn test_var_uint_codec() { + test_var_uint_enc_dec(0); + test_var_uint_enc_dec(1); + test_var_uint_enc_dec(127); + test_var_uint_enc_dec(0b1000_0000); + test_var_uint_enc_dec(0b1_0000_0000); + test_var_uint_enc_dec(0b1_1111_1111); + test_var_uint_enc_dec(0b10_0000_0000); + test_var_uint_enc_dec(0b11_1111_1111); + test_var_uint_enc_dec(0x7fff_ffff_ffff_ffff); + test_var_uint_enc_dec(u64::MAX); + } + + #[test] + fn test_var_int() { + test_var_int_enc_dec(0); + test_var_int_enc_dec(1); + test_var_int_enc_dec(-1); + test_var_int_enc_dec(63); + test_var_int_enc_dec(-63); + test_var_int_enc_dec(64); + test_var_int_enc_dec(-64); + test_var_int_enc_dec(i32::MAX); + test_var_int_enc_dec(i32::MIN); + test_var_int_enc_dec(((1 << 20) - 1) * 8); + test_var_int_enc_dec(-((1 << 20) - 1) * 8); + } +} diff --git a/packages/common/y-octo/core/src/codec/mod.rs b/packages/common/y-octo/core/src/codec/mod.rs new file mode 100644 index 0000000000..b7f08b98d6 --- /dev/null +++ b/packages/common/y-octo/core/src/codec/mod.rs @@ -0,0 +1,9 @@ +mod buffer; +mod integer; +mod string; + +pub use buffer::{read_var_buffer, write_var_buffer}; +pub use integer::{read_var_i32, read_var_u64, write_var_i32, write_var_u64}; +pub use string::{read_var_string, write_var_string}; + +use super::*; diff --git a/packages/common/y-octo/core/src/codec/string.rs b/packages/common/y-octo/core/src/codec/string.rs new file mode 100644 index 0000000000..f150076049 --- /dev/null +++ b/packages/common/y-octo/core/src/codec/string.rs @@ -0,0 +1,90 @@ +use std::io::{Error, Write}; + +use nom::{combinator::map_res, Parser}; + +use super::*; + +pub fn read_var_string(input: &[u8]) -> IResult<&[u8], String> { + map_res(read_var_buffer, |s| String::from_utf8(s.to_vec())).parse(input) +} + +pub fn write_var_string>(buffer: &mut W, input: S) -> Result<(), Error> { + let bytes = input.as_ref().as_bytes(); + write_var_buffer(buffer, bytes)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use nom::{ + error::{Error, ErrorKind}, + AsBytes, Err, + }; + + use super::*; + + #[test] + fn test_read_var_string() { + // Test case 1: valid input, string length = 5 + let input = [0x05, 0x68, 0x65, 0x6C, 0x6C, 0x6F]; + let expected_output = "hello".to_string(); + let result = read_var_string(&input); + assert_eq!(result, Ok((&[][..], expected_output))); + + // Test case 2: missing string length + let input = [0x68, 0x65, 0x6C, 0x6C, 0x6F]; + let result = read_var_string(&input); + assert_eq!( + result, + Err(Err::Error(Error::new(&input[1..], ErrorKind::Eof))) + ); + + // Test case 3: truncated input + let input = [0x05, 0x68, 0x65, 0x6C, 0x6C]; + let result = read_var_string(&input); + assert_eq!( + result, + Err(Err::Error(Error::new(&input[1..], ErrorKind::Eof))) + ); + + // Test case 4: invalid input + let input = [0xFF, 0x01, 0x02, 0x03, 0x04]; + let result = read_var_string(&input); + assert_eq!( + result, + Err(Err::Error(Error::new(&input[2..], ErrorKind::Eof))) + ); + + // Test case 5: invalid var int encoding + let input = [0xFF, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]; + let result = read_var_string(&input); + assert_eq!( + result, + Err(Err::Error(Error::new(&input[7..], ErrorKind::Eof))) + ); + + // Test case 6: invalid input, invalid UTF-8 encoding + let input = [0x05, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]; + let result = read_var_string(&input); + assert_eq!( + result, + Err(Err::Error(Error::new(&input[..], ErrorKind::MapRes))) + ); + } + + #[test] + fn test_var_str_codec() { + test_var_str_enc_dec("".to_string()); + test_var_str_enc_dec(" ".to_string()); + test_var_str_enc_dec("abcde".to_string()); + test_var_str_enc_dec("🃒🃓🃟☗🀥🀫∺∼≂≇⓵➎⓷➏‍".to_string()); + } + + fn test_var_str_enc_dec(input: String) { + let mut buf = Vec::::new(); + write_var_string(&mut buf, input.clone()).unwrap(); + let (rest, decoded_str) = read_var_string(buf.as_bytes()).unwrap(); + assert_eq!(decoded_str, input); + assert_eq!(rest.len(), 0); + } +} diff --git a/packages/common/y-octo/core/src/doc/awareness.rs b/packages/common/y-octo/core/src/doc/awareness.rs new file mode 100644 index 0000000000..b4c7c024cd --- /dev/null +++ b/packages/common/y-octo/core/src/doc/awareness.rs @@ -0,0 +1,253 @@ +use std::{cmp::max, collections::hash_map::Entry}; + +use super::*; +use crate::sync::Arc; + +pub type AwarenessCallback = Arc; + +pub struct Awareness { + awareness: AwarenessStates, + callback: Option, + local_id: u64, +} + +impl Awareness { + pub fn new(local_id: u64) -> Self { + Self { + awareness: AwarenessStates::new(), + callback: None, + local_id, + } + } + + pub fn on_update(&mut self, f: impl Fn(&Awareness, AwarenessEvent) + Send + Sync + 'static) { + self.callback = Some(Arc::new(f)); + } + + pub fn get_states(&self) -> &AwarenessStates { + &self.awareness + } + + pub fn get_local_state(&self) -> Option { + self + .awareness + .get(&self.local_id) + .map(|state| state.content.clone()) + } + + fn mut_local_state(&mut self) -> &mut AwarenessState { + self.awareness.entry(self.local_id).or_default() + } + + pub fn set_local_state(&mut self, content: String) { + self.mut_local_state().set_content(content); + if let Some(cb) = self.callback.as_ref() { + cb( + self, + AwarenessEventBuilder::new().update(self.local_id).build(), + ); + } + } + + pub fn clear_local_state(&mut self) { + self.mut_local_state().delete(); + if let Some(cb) = self.callback.as_ref() { + cb( + self, + AwarenessEventBuilder::new().remove(self.local_id).build(), + ); + } + } + + pub fn apply_update(&mut self, update: AwarenessStates) { + let mut event = AwarenessEventBuilder::new(); + + for (client_id, state) in update { + match self.awareness.entry(client_id) { + Entry::Occupied(mut entry) => { + let prev_state = entry.get_mut(); + if client_id == self.local_id { + // ignore remote update about local client and + // add clock to overwrite remote data + prev_state.set_clock(max(prev_state.clock, state.clock) + 1); + event.update(client_id); + continue; + } + + if prev_state.clock < state.clock { + if state.is_deleted() { + prev_state.delete(); + event.remove(client_id); + } else { + *prev_state = state; + event.update(client_id); + } + } + } + Entry::Vacant(entry) => { + entry.insert(state); + event.add(client_id); + } + } + } + + if let Some(cb) = self.callback.as_ref() { + cb(self, event.build()); + } + } +} + +pub struct AwarenessEvent { + added: Vec, + updated: Vec, + removed: Vec, +} + +impl AwarenessEvent { + pub fn get_updated(&self, states: &AwarenessStates) -> AwarenessStates { + states + .iter() + .filter(|(id, _)| { + self.added.contains(id) || self.updated.contains(id) || self.removed.contains(id) + }) + .map(|(id, state)| (*id, state.clone())) + .collect() + } +} + +struct AwarenessEventBuilder { + added: Vec, + updated: Vec, + removed: Vec, +} + +impl AwarenessEventBuilder { + fn new() -> Self { + Self { + added: Vec::new(), + updated: Vec::new(), + removed: Vec::new(), + } + } + + fn add(&mut self, client_id: u64) -> &mut Self { + self.added.push(client_id); + self + } + + fn update(&mut self, client_id: u64) -> &mut Self { + self.updated.push(client_id); + self + } + + fn remove(&mut self, client_id: u64) -> &mut Self { + self.removed.push(client_id); + self + } + + fn build(&mut self) -> AwarenessEvent { + AwarenessEvent { + added: self.added.clone(), + updated: self.updated.clone(), + removed: self.removed.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sync::{Mutex, MutexGuard}; + + #[test] + fn test_awareness() { + loom_model!({ + let mut awareness = Awareness::new(0); + + { + // init state + assert_eq!(awareness.local_id, 0); + assert_eq!(awareness.awareness.len(), 0); + } + + { + // local state + awareness.set_local_state("test".to_string()); + assert_eq!(awareness.get_local_state(), Some("test".to_string())); + awareness.clear_local_state(); + assert_eq!(awareness.get_local_state(), Some("null".to_string())); + } + + { + // apply remote update + let mut states = AwarenessStates::new(); + states.insert(0, AwarenessState::new(2, "test0".to_string())); + states.insert(1, AwarenessState::new(2, "test1".to_string())); + awareness.apply_update(states); + assert!(awareness.get_states().contains_key(&1)); + + // local state will not apply + assert_eq!( + awareness.get_states().get(&0).unwrap().content, + "null".to_string() + ); + assert_eq!( + awareness.get_states().get(&1).unwrap().content, + "test1".to_string() + ); + } + + { + // callback + let values: Arc>> = Arc::new(Mutex::new(Vec::new())); + let callback_values = Arc::clone(&values); + awareness.on_update(move |_, event| { + let mut values = callback_values.lock().unwrap(); + values.push(event); + }); + + let mut new_states = AwarenessStates::new(); + // exists in local awareness: update + new_states.insert(1, AwarenessState::new(3, "test update".to_string())); + // not exists in local awareness: add + new_states.insert(2, AwarenessState::new(1, "test update".to_string())); + // not exists in local awareness: add + new_states.insert(3, AwarenessState::new(1, "null".to_string())); + // not exists in local awareness: add + new_states.insert(4, AwarenessState::new(1, "test update".to_string())); + awareness.apply_update(new_states); + + let mut new_states = AwarenessStates::new(); + // exists in local awareness: delete + new_states.insert(4, AwarenessState::new(2, "null".to_string())); + awareness.apply_update(new_states); + + awareness.set_local_state("test".to_string()); + awareness.clear_local_state(); + + let values: MutexGuard> = values.lock().unwrap(); + assert_eq!(values.len(), 4); + let event = values.first().unwrap(); + + let mut added = event.added.clone(); + added.sort(); + assert_eq!(added, [2, 3, 4]); + assert_eq!(event.updated, [1]); + + assert_eq!( + event.get_updated(awareness.get_states()).get(&1).unwrap(), + &AwarenessState::new(3, "test update".to_string()) + ); + + let event = values.get(1).unwrap(); + assert_eq!(event.removed, [4]); + + let event = values.get(2).unwrap(); + assert_eq!(event.updated, [0]); + + let event = values.get(3).unwrap(); + assert_eq!(event.removed, [0]); + } + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/any.rs b/packages/common/y-octo/core/src/doc/codec/any.rs new file mode 100644 index 0000000000..2723518a9d --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/any.rs @@ -0,0 +1,716 @@ +use std::{ + fmt::{self, Display}, + ops::RangeInclusive, +}; + +use ordered_float::OrderedFloat; + +use super::*; + +const MAX_JS_INT: i64 = 0x001F_FFFF_FFFF_FFFF; +// The smallest int in js number. +const MIN_JS_INT: i64 = -MAX_JS_INT; +pub const JS_INT_RANGE: RangeInclusive = MIN_JS_INT..=MAX_JS_INT; + +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +pub enum Any { + Undefined, + Null, + Integer(i32), + Float32(OrderedFloat), + Float64(OrderedFloat), + BigInt64(i64), + False, + True, + String(String), + // FIXME: due to macro's overflow evaluating, we can't use proptest here + #[cfg_attr(test, proptest(skip))] + Object(HashMap), + #[cfg_attr(test, proptest(skip))] + Array(Vec), + Binary(Vec), +} + +impl CrdtRead for Any { + fn read(reader: &mut R) -> JwstCodecResult { + let index = reader.read_u8()?; + match 127u8.overflowing_sub(index).0 { + 0 => Ok(Any::Undefined), + 1 => Ok(Any::Null), + // in yjs implementation, flag 2 only save 32bit integer + 2 => Ok(Any::Integer(reader.read_var_i32()?)), // Integer + 3 => Ok(Any::Float32(reader.read_f32_be()?.into())), // Float32 + 4 => Ok(Any::Float64(reader.read_f64_be()?.into())), // Float64 + 5 => Ok(Any::BigInt64(reader.read_i64_be()?)), // BigInt64 + 6 => Ok(Any::False), + 7 => Ok(Any::True), + 8 => Ok(Any::String(reader.read_var_string()?)), // String + 9 => { + let len = reader.read_var_u64()?; + let object = (0..len) + .map(|_| Self::read_key_value(reader)) + .collect::, _>>()?; + + Ok(Any::Object(object.into_iter().collect())) + } // Object + 10 => { + let len = reader.read_var_u64()?; + let any = (0..len) + .map(|_| Self::read(reader)) + .collect::, _>>()?; + + Ok(Any::Array(any)) + } // Array + 11 => { + let binary = reader.read_var_buffer()?; + Ok(Any::Binary(binary.to_vec())) + } // Binary + _ => Ok(Any::Undefined), + } + } +} + +impl CrdtWrite for Any { + fn write(&self, writer: &mut W) -> JwstCodecResult { + match self { + Any::Undefined => writer.write_u8(127)?, + Any::Null => writer.write_u8(127 - 1)?, + Any::Integer(value) => { + writer.write_u8(127 - 2)?; + writer.write_var_i32(*value)?; + } + Any::Float32(value) => { + writer.write_u8(127 - 3)?; + writer.write_f32_be(value.into_inner())?; + } + Any::Float64(value) => { + writer.write_u8(127 - 4)?; + writer.write_f64_be(value.into_inner())?; + } + Any::BigInt64(value) => { + writer.write_u8(127 - 5)?; + writer.write_i64_be(*value)?; + } + Any::False => writer.write_u8(127 - 6)?, + Any::True => writer.write_u8(127 - 7)?, + Any::String(value) => { + writer.write_u8(127 - 8)?; + writer.write_var_string(value)?; + } + Any::Object(value) => { + writer.write_u8(127 - 9)?; + writer.write_var_u64(value.len() as u64)?; + for (key, value) in value { + Self::write_key_value(writer, key, value)?; + } + } + Any::Array(values) => { + writer.write_u8(127 - 10)?; + writer.write_var_u64(values.len() as u64)?; + for value in values { + value.write(writer)?; + } + } + Any::Binary(value) => { + writer.write_u8(127 - 11)?; + writer.write_var_buffer(value)?; + } + } + + Ok(()) + } +} + +impl Any { + fn read_key_value(reader: &mut R) -> JwstCodecResult<(String, Any)> { + let key = reader.read_var_string()?; + let value = Self::read(reader)?; + + Ok((key, value)) + } + + fn write_key_value(writer: &mut W, key: &str, value: &Any) -> JwstCodecResult { + writer.write_var_string(key)?; + value.write(writer)?; + + Ok(()) + } + + pub(crate) fn read_multiple(reader: &mut R) -> JwstCodecResult> { + let len = reader.read_var_u64()? as usize; + let mut vec = Vec::with_capacity(len); + for _ in 0..len { + vec.push(Any::read(reader)?); + } + + Ok(vec) + } + + pub(crate) fn write_multiple(writer: &mut W, any: &[Any]) -> JwstCodecResult { + writer.write_var_u64(any.len() as u64)?; + for value in any { + value.write(writer)?; + } + + Ok(()) + } +} + +macro_rules! impl_primitive_from { + (unsigned, $($ty: ty),*) => { + $( + impl From<$ty> for Any { + fn from(value: $ty) -> Self { + // INFO: i64::MAX > value > u64::MAX will cut down + // yjs binary does not consider the case that the int size exceeds i64 + let int: i64 = value as i64; + // handle the behavior same as yjs + if JS_INT_RANGE.contains(&int) { + if int <= i32::MAX as i64 { + Self::Integer(int as i32) + } else if int as f32 as i64 == int { + Self::Float32((int as f32).into()) + } else { + Self::Float64((int as f64).into()) + } + } else { + Self::BigInt64(int) + } + } + } + )* + }; + (signed, $($ty: ty),*) => { + $( + impl From<$ty> for Any { + fn from(value: $ty) -> Self { + let int: i64 = value.into(); + // handle the behavior same as yjs + if JS_INT_RANGE.contains(&int) { + if int <= i32::MAX as i64 { + Self::Integer(int as i32) + } else if int as f32 as i64 == int { + Self::Float32((int as f32).into()) + } else { + Self::Float64((int as f64).into()) + } + } else { + Self::BigInt64(int) + } + } + } + )* + }; + (string, $($ty: ty),*) => { + $( + impl From<$ty> for Any { + fn from(value: $ty) -> Self { + Self::String(value.into()) + } + } + )* + }; +} + +impl_primitive_from!(unsigned, u8, u16, u32, u64); +impl_primitive_from!(signed, i8, i16, i32, i64); +impl_primitive_from!(string, String, &str); + +impl From for Any { + fn from(value: usize) -> Self { + (value as u64).into() + } +} + +impl From for Any { + fn from(value: isize) -> Self { + (value as i64).into() + } +} + +impl From for Any { + fn from(value: f32) -> Self { + Self::Float32(value.into()) + } +} + +impl From for Any { + fn from(value: f64) -> Self { + if value.trunc() == value { + (value as i64).into() + } else if value as f32 as f64 == value { + Self::Float32((value as f32).into()) + } else { + Self::Float64(value.into()) + } + } +} + +impl From for Any { + fn from(value: bool) -> Self { + if value { + Self::True + } else { + Self::False + } + } +} + +impl TryFrom for String { + type Error = JwstCodecError; + + fn try_from(value: Any) -> Result { + match value { + Any::String(s) => Ok(s), + _ => Err(JwstCodecError::UnexpectedType("String")), + } + } +} + +impl TryFrom for HashMap { + type Error = JwstCodecError; + + fn try_from(value: Any) -> Result { + match value { + Any::Object(map) => Ok(map), + _ => Err(JwstCodecError::UnexpectedType("Object")), + } + } +} + +impl TryFrom for Vec { + type Error = JwstCodecError; + + fn try_from(value: Any) -> Result { + match value { + Any::Array(vec) => Ok(vec), + _ => Err(JwstCodecError::UnexpectedType("Array")), + } + } +} + +impl TryFrom for bool { + type Error = JwstCodecError; + + fn try_from(value: Any) -> Result { + match value { + Any::True => Ok(true), + Any::False => Ok(false), + _ => Err(JwstCodecError::UnexpectedType("Boolean")), + } + } +} + +impl FromIterator for Any { + fn from_iter>(iter: I) -> Self { + Self::Array(iter.into_iter().collect()) + } +} + +impl<'a> FromIterator<&'a Any> for Any { + fn from_iter>(iter: I) -> Self { + Self::Array(iter.into_iter().cloned().collect()) + } +} + +impl FromIterator<(String, Any)> for Any { + fn from_iter>(iter: I) -> Self { + let mut map = HashMap::new(); + map.extend(iter); + Self::Object(map) + } +} + +impl From> for Any { + fn from(value: HashMap) -> Self { + Self::Object(value) + } +} + +impl From> for Any { + fn from(value: Vec) -> Self { + Self::Binary(value) + } +} + +impl From<&[u8]> for Any { + fn from(value: &[u8]) -> Self { + Self::Binary(value.into()) + } +} + +// TODO: impl for Any::Undefined +impl> From> for Any { + fn from(value: Option) -> Self { + if let Some(val) = value { + val.into() + } else { + Any::Null + } + } +} + +#[cfg(feature = "serde_json")] +impl From for Any { + fn from(value: serde_json::Value) -> Self { + match value { + serde_json::Value::Null => Self::Null, + serde_json::Value::Bool(b) => { + if b { + Self::True + } else { + Self::False + } + } + serde_json::Value::Number(n) => { + if n.is_f64() { + Self::Float64(n.as_f64().unwrap().into()) + } else if n.is_i64() { + Self::Integer(n.as_i64().unwrap() as i32) + } else { + Self::Integer(n.as_u64().unwrap() as i32) + } + } + serde_json::Value::String(s) => Self::String(s), + serde_json::Value::Array(vec) => { + Self::Array(vec.into_iter().map(|v| v.into()).collect::>()) + } + serde_json::Value::Object(obj) => { + Self::Object(obj.into_iter().map(|(k, v)| (k, v.into())).collect()) + } + } + } +} + +impl<'de> serde::Deserialize<'de> for Any { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::{Error, MapAccess, SeqAccess, Visitor}; + struct ValueVisitor; + + impl<'de> Visitor<'de> for ValueVisitor { + type Value = Any; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("any valid JSON value") + } + + #[inline] + fn visit_bool(self, value: bool) -> Result { + Ok(if value { Any::True } else { Any::False }) + } + + #[inline] + fn visit_i64(self, value: i64) -> Result { + Ok(Any::BigInt64(value)) + } + + #[inline] + fn visit_u64(self, value: u64) -> Result { + Ok((value as i64).into()) + } + + #[inline] + fn visit_f64(self, value: f64) -> Result { + Ok(Any::Float64(OrderedFloat(value))) + } + + #[inline] + fn visit_str(self, value: &str) -> Result + where + E: Error, + { + self.visit_string(String::from(value)) + } + + #[inline] + fn visit_string(self, value: String) -> Result { + Ok(Any::String(value)) + } + + #[inline] + fn visit_none(self) -> Result { + Ok(Any::Null) + } + + #[inline] + fn visit_some(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + serde::Deserialize::deserialize(deserializer) + } + + #[inline] + fn visit_unit(self) -> Result { + Ok(Any::Null) + } + + #[inline] + fn visit_seq(self, mut visitor: V) -> Result + where + V: SeqAccess<'de>, + { + let mut vec = Vec::new(); + + while let Some(elem) = visitor.next_element()? { + vec.push(elem); + } + + Ok(Any::Array(vec)) + } + + fn visit_map(self, mut visitor: V) -> Result + where + V: MapAccess<'de>, + { + match visitor.next_key::()? { + Some(k) => { + let mut values = HashMap::new(); + + values.insert(k, visitor.next_value()?); + while let Some((key, value)) = visitor.next_entry()? { + values.insert(key, value); + } + + Ok(Any::Object(values)) + } + None => Ok(Any::Object(HashMap::new())), + } + } + } + + deserializer.deserialize_any(ValueVisitor) + } +} + +impl serde::Serialize for Any { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::{SerializeMap, SerializeSeq}; + + match self { + Any::Null => serializer.serialize_none(), + Any::Undefined => serializer.serialize_none(), + Any::True => serializer.serialize_bool(true), + Any::False => serializer.serialize_bool(false), + Any::Float32(value) => serializer.serialize_f32(value.0), + Any::Float64(value) => serializer.serialize_f64(value.0), + Any::Integer(value) => serializer.serialize_i32(*value), + Any::BigInt64(value) => serializer.serialize_i64(*value), + Any::String(value) => serializer.serialize_str(value.as_ref()), + Any::Array(values) => { + let mut seq = serializer.serialize_seq(Some(values.len()))?; + for value in values.iter() { + seq.serialize_element(value)?; + } + seq.end() + } + Any::Object(entries) => { + let mut map = serializer.serialize_map(Some(entries.len()))?; + for (key, value) in entries.iter() { + map.serialize_entry(key, value)?; + } + map.end() + } + Any::Binary(buf) => serializer.serialize_bytes(buf), + } + } +} + +impl Display for Any { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::True => write!(f, "true"), + Self::False => write!(f, "false"), + Self::String(s) => write!(f, "\"{}\"", s), + Self::Integer(i) => write!(f, "{}", i), + Self::Float32(v) => write!(f, "{}", v), + Self::Float64(v) => write!(f, "{}", v), + Self::BigInt64(v) => write!(f, "{}", v), + Self::Object(map) => { + write!(f, "{{")?; + for (i, (key, value)) in map.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}: {}", key, value)?; + } + write!(f, "}}") + } + Self::Array(vec) => { + write!(f, "[")?; + for (i, value) in vec.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}", value)?; + } + write!(f, "]") + } + Self::Binary(buf) => write!(f, "{:?}", buf), + Self::Undefined => write!(f, "undefined"), + Self::Null => write!(f, "null"), + } + } +} + +#[cfg(test)] +mod tests { + use proptest::{collection::vec, prelude::*}; + + use super::*; + + #[test] + fn test_any_codec() { + let any = Any::Object( + vec![ + ("name".to_string(), Any::String("Alice".to_string())), + ("age".to_string(), Any::Integer(25)), + ( + "contacts".to_string(), + Any::Array(vec![ + Any::Object( + vec![ + ("type".to_string(), Any::String("Mobile".to_string())), + ("number".to_string(), Any::String("1234567890".to_string())), + ] + .into_iter() + .collect(), + ), + Any::Object( + vec![ + ("type".to_string(), Any::String("Email".to_string())), + ( + "address".to_string(), + Any::String("alice@example.com".to_string()), + ), + ] + .into_iter() + .collect(), + ), + Any::Undefined, + ]), + ), + ( + "standard_data".to_string(), + Any::Array(vec![ + Any::Undefined, + Any::Null, + Any::Integer(114514), + Any::Float32(114.514.into()), + Any::Float64(115.514.into()), + Any::BigInt64(-1145141919810), + Any::False, + Any::True, + Any::Object( + vec![ + ("name".to_string(), Any::String("tadokoro".to_string())), + ("age".to_string(), Any::String("24".to_string())), + ("profession".to_string(), Any::String("student".to_string())), + ] + .into_iter() + .collect(), + ), + Any::Binary(vec![1, 2, 3, 4, 5]), + ]), + ), + ] + .into_iter() + .collect(), + ); + + let mut encoder = RawEncoder::default(); + any.write(&mut encoder).unwrap(); + let encoded = encoder.into_inner(); + + let mut decoder = RawDecoder::new(&encoded); + let decoded = Any::read(&mut decoder).unwrap(); + + assert_eq!(any, decoded); + } + + proptest! { + #[test] + #[cfg_attr(miri, ignore)] + fn test_random_any(any in vec(any::(), 0..100)) { + for any in &any { + let mut encoder = RawEncoder::default(); + any.write(&mut encoder).unwrap(); + let encoded = encoder.into_inner(); + + let mut decoder = RawDecoder::new(&encoded); + let decoded = Any::read(&mut decoder).unwrap(); + + assert_eq!(any, &decoded); + } + } + } + + #[test] + fn test_convert_to_any() { + let any: Vec = vec![ + 42u8.into(), + 42u16.into(), + 42u32.into(), + 42u64.into(), + 114.514f32.into(), + 1919.810f64.into(), + (-42i8).into(), + (-42i16).into(), + (-42i32).into(), + (-42i64).into(), + false.into(), + true.into(), + "JWST".to_string().into(), + "OctoBase".into(), + vec![1u8, 9, 1, 9].into(), + (&[8u8, 1, 0][..]).into(), + [Any::True, 42u8.into()].iter().collect(), + ]; + assert_eq!( + any, + vec![ + Any::Integer(42), + Any::Integer(42), + Any::Integer(42), + Any::Integer(42), + Any::Float32(114.514.into()), + Any::Float64(1919.810.into()), + Any::Integer(-42), + Any::Integer(-42), + Any::Integer(-42), + Any::Integer(-42), + Any::False, + Any::True, + Any::String("JWST".to_string()), + Any::String("OctoBase".to_string()), + Any::Binary(vec![1, 9, 1, 9]), + Any::Binary(vec![8, 1, 0]), + Any::Array(vec![Any::True, Any::Integer(42)]) + ] + ); + + assert_eq!( + vec![("key".to_string(), 10u64.into())] + .into_iter() + .collect::(), + Any::Object(HashMap::from_iter(vec![( + "key".to_string(), + Any::Integer(10) + )])) + ); + + let any: Any = 10u64.into(); + assert_eq!( + [any].iter().collect::(), + Any::Array(vec![Any::Integer(10)]) + ); + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/content.rs b/packages/common/y-octo/core/src/doc/codec/content.rs new file mode 100644 index 0000000000..2dec2203cb --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/content.rs @@ -0,0 +1,417 @@ +use super::*; + +#[derive(Clone)] +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +pub(crate) enum Content { + Deleted(u64), + Json(Vec>), + Binary(Vec), + String(String), + #[cfg_attr(test, proptest(skip))] + Embed(Any), + #[cfg_attr(test, proptest(skip))] + Format { + key: String, + value: Any, + }, + #[cfg_attr(test, proptest(skip))] + Type(YTypeRef), + Any(Vec), + Doc { + guid: String, + opts: Any, + }, +} + +unsafe impl Send for Content {} +unsafe impl Sync for Content {} + +impl From for Content { + fn from(value: Any) -> Self { + match value { + Any::Undefined + | Any::Null + | Any::Integer(_) + | Any::Float32(_) + | Any::Float64(_) + | Any::BigInt64(_) + | Any::False + | Any::True + | Any::String(_) + | Any::Object(_) => Content::Any(vec![value; 1]), + Any::Array(v) => Content::Any(v), + Any::Binary(b) => Content::Binary(b), + } + } +} + +impl PartialEq for Content { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Deleted(len1), Self::Deleted(len2)) => len1 == len2, + (Self::Json(vec1), Self::Json(vec2)) => vec1 == vec2, + (Self::Binary(vec1), Self::Binary(vec2)) => vec1 == vec2, + (Self::String(str1), Self::String(str2)) => str1 == str2, + (Self::Embed(json1), Self::Embed(json2)) => json1 == json2, + ( + Self::Format { + key: key1, + value: value1, + }, + Self::Format { + key: key2, + value: value2, + }, + ) => key1 == key2 && value1 == value2, + (Self::Any(any1), Self::Any(any2)) => any1 == any2, + (Self::Doc { guid: guid1, .. }, Self::Doc { guid: guid2, .. }) => guid1 == guid2, + (Self::Type(ty1), Self::Type(ty2)) => ty1 == ty2, + _ => false, + } + } +} + +impl std::fmt::Debug for Content { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Deleted(arg0) => f.debug_tuple("Deleted").field(arg0).finish(), + Self::Json(arg0) => f + .debug_tuple("JSON") + .field(&format!("Vec [len: {}]", arg0.len())) + .finish(), + Self::Binary(arg0) => f + .debug_tuple("Binary") + .field(&format!("Binary [len: {}]", arg0.len())) + .finish(), + Self::String(arg0) => f.debug_tuple("String").field(arg0).finish(), + Self::Embed(arg0) => f.debug_tuple("Embed").field(arg0).finish(), + Self::Format { key, value } => f + .debug_struct("Format") + .field("key", key) + .field("value", value) + .finish(), + Self::Type(arg0) => f + .debug_tuple("Type") + .field(&arg0.ty().unwrap().kind()) + .finish(), + Self::Any(arg0) => f.debug_tuple("Any").field(arg0).finish(), + Self::Doc { guid, opts } => f + .debug_struct("Doc") + .field("guid", guid) + .field("opts", opts) + .finish(), + } + } +} + +impl Content { + pub(crate) fn read(decoder: &mut R, tag_type: u8) -> JwstCodecResult { + match tag_type { + 1 => Ok(Self::Deleted(decoder.read_var_u64()?)), // Deleted + 2 => { + let len = decoder.read_var_u64()?; + let strings = (0..len) + .map(|_| { + decoder + .read_var_string() + .map(|s| (s != "undefined").then_some(s)) + }) + .collect::, _>>()?; + + Ok(Self::Json(strings)) + } // JSON + 3 => Ok(Self::Binary(decoder.read_var_buffer()?.to_vec())), // Binary + 4 => Ok(Self::String(decoder.read_var_string()?)), // String + 5 => { + let string = decoder.read_var_string()?; + let json = + serde_json::from_str(&string).map_err(|_| JwstCodecError::DamagedDocumentJson)?; + + Ok(Self::Embed(json)) + } // Embed + 6 => { + let key = decoder.read_var_string()?; + let value = decoder.read_var_string()?; + let value = + serde_json::from_str(&value).map_err(|_| JwstCodecError::DamagedDocumentJson)?; + + Ok(Self::Format { key, value }) + } // Format + 7 => { + let type_ref = decoder.read_var_u64()?; + let kind = YTypeKind::from(type_ref); + let tag_name = match kind { + YTypeKind::XMLElement | YTypeKind::XMLHook => Some(decoder.read_var_string()?), + YTypeKind::Unknown => { + return Err(JwstCodecError::IncompleteDocument(format!( + "Unknown y type: {type_ref}" + ))); + } + _ => None, + }; + + Ok(Self::Type(YTypeRef::new(kind, tag_name))) + } // YType + 8 => Ok(Self::Any(Any::read_multiple(decoder)?)), // Any + 9 => { + let guid = decoder.read_var_string()?; + let opts = Any::read(decoder)?; + Ok(Self::Doc { guid, opts }) + } // Doc + tag_type => Err(JwstCodecError::IncompleteDocument(format!( + "Unknown content type: {tag_type}" + ))), + } + } + + pub(crate) fn get_info(&self) -> u8 { + match self { + Self::Deleted(_) => 1, + Self::Json(_) => 2, + Self::Binary(_) => 3, + Self::String(_) => 4, + Self::Embed(_) => 5, + Self::Format { .. } => 6, + Self::Type(_) => 7, + Self::Any(_) => 8, + Self::Doc { .. } => 9, + } + } + + pub(crate) fn write(&self, encoder: &mut W) -> JwstCodecResult { + match self { + Self::Deleted(len) => { + encoder.write_var_u64(*len)?; + } + Self::Json(strings) => { + encoder.write_var_u64(strings.len() as u64)?; + for string in strings { + match string { + Some(string) => encoder.write_var_string(string)?, + None => encoder.write_var_string("undefined")?, + } + } + } + Self::Binary(buffer) => { + encoder.write_var_buffer(buffer)?; + } + Self::String(string) => { + encoder.write_var_string(string)?; + } + Self::Embed(val) => { + encoder.write_var_string( + serde_json::to_string(val).map_err(|_| JwstCodecError::DamagedDocumentJson)?, + )?; + } + Self::Format { key, value } => { + encoder.write_var_string(key)?; + encoder.write_var_string( + serde_json::to_string(value).map_err(|_| JwstCodecError::DamagedDocumentJson)?, + )?; + } + Self::Type(ty) => { + if let Some(ty) = ty.ty() { + let type_ref = u64::from(ty.kind()); + encoder.write_var_u64(type_ref)?; + + if matches!(ty.kind(), YTypeKind::XMLElement | YTypeKind::XMLHook) { + encoder.write_var_string(ty.name.as_ref().unwrap())?; + } + } + } + Self::Any(any) => { + Any::write_multiple(encoder, any)?; + } + Self::Doc { guid, opts } => { + encoder.write_var_string(guid)?; + opts.write(encoder)?; + } + } + Ok(()) + } + + pub fn clock_len(&self) -> u64 { + match self { + Self::Deleted(len) => *len, + Self::Json(strings) => strings.len() as u64, + // TODO: need a custom wrapper with length cached, this cost too much + Self::String(string) => string.chars().map(|c| c.len_utf16()).sum::() as u64, + Self::Any(any) => any.len() as u64, + Self::Binary(_) | Self::Embed(_) | Self::Format { .. } | Self::Type(_) | Self::Doc { .. } => { + 1 + } + } + } + + pub fn countable(&self) -> bool { + !matches!(self, Content::Format { .. } | Content::Deleted(_)) + } + + #[allow(dead_code)] + pub fn splittable(&self) -> bool { + matches!( + self, + Self::String { .. } | Self::Any { .. } | Self::Json { .. } + ) + } + + pub fn split(&self, diff: u64) -> JwstCodecResult<(Self, Self)> { + match self { + Self::String(str) => { + let (left, right) = Self::split_as_utf16_str(str.as_str(), diff); + Ok(( + Self::String(left.to_string()), + Self::String(right.to_string()), + )) + } + Self::Json(vec) => { + let (left, right) = vec.split_at(diff as usize); + Ok((Self::Json(left.to_owned()), Self::Json(right.to_owned()))) + } + Self::Any(vec) => { + let (left, right) = vec.split_at(diff as usize); + Ok((Self::Any(left.to_owned()), Self::Any(right.to_owned()))) + } + Self::Deleted(len) => { + let (left, right) = (diff, *len - diff); + + Ok((Self::Deleted(left), Self::Deleted(right))) + } + _ => Err(JwstCodecError::ContentSplitNotSupport(diff)), + } + } + + /// consider `offset` as a utf-16 encoded string offset + fn split_as_utf16_str(s: &str, offset: u64) -> (&str, &str) { + let mut utf_16_offset = 0; + let mut utf_8_offset = 0; + for ch in s.chars() { + utf_16_offset += ch.len_utf16(); + utf_8_offset += ch.len_utf8(); + if utf_16_offset as u64 >= offset { + break; + } + } + + s.split_at(utf_8_offset) + } +} + +#[cfg(test)] +mod tests { + use proptest::{collection::vec, prelude::*}; + + use super::*; + + fn content_round_trip(content: &Content) -> JwstCodecResult { + let mut writer = RawEncoder::default(); + writer.write_u8(content.get_info())?; + content.write(&mut writer)?; + let update = writer.into_inner(); + + let mut reader = RawDecoder::new(&update); + let tag_type = reader.read_u8()?; + assert_eq!(Content::read(&mut reader, tag_type)?, *content); + + Ok(()) + } + + #[test] + fn test_content() { + loom_model!({ + let contents = [ + Content::Deleted(42), + Content::Json(vec![ + None, + Some("test_1".to_string()), + Some("test_2".to_string()), + ]), + Content::Binary(vec![1, 2, 3]), + Content::String("hello".to_string()), + Content::Embed(Any::True), + Content::Format { + key: "key".to_string(), + value: Any::Integer(42), + }, + Content::Type(YTypeRef::new(YTypeKind::Array, None)), + Content::Type(YTypeRef::new(YTypeKind::Map, None)), + Content::Type(YTypeRef::new(YTypeKind::Text, None)), + Content::Type(YTypeRef::new( + YTypeKind::XMLElement, + Some("test".to_string()), + )), + Content::Type(YTypeRef::new(YTypeKind::XMLFragment, None)), + Content::Type(YTypeRef::new(YTypeKind::XMLHook, Some("test".to_string()))), + Content::Type(YTypeRef::new(YTypeKind::XMLText, None)), + Content::Any(vec![Any::BigInt64(42), Any::String("Test Any".to_string())]), + Content::Doc { + guid: "my_guid".to_string(), + opts: Any::BigInt64(42), + }, + ]; + + for content in &contents { + content_round_trip(content).unwrap(); + } + }); + } + + #[test] + fn test_content_split() { + let contents = [ + Content::String("hello".to_string()), + Content::Json(vec![ + None, + Some("test_1".to_string()), + Some("test_2".to_string()), + ]), + Content::Any(vec![Any::BigInt64(42), Any::String("Test Any".to_string())]), + Content::Binary(vec![]), + ]; + + { + let (left, right) = contents[0].split(1).unwrap(); + assert!(contents[0].splittable()); + assert_eq!(left, Content::String("h".to_string())); + assert_eq!(right, Content::String("ello".to_string())); + } + + { + let (left, right) = contents[1].split(1).unwrap(); + assert!(contents[1].splittable()); + assert_eq!(left, Content::Json(vec![None])); + assert_eq!( + right, + Content::Json(vec![Some("test_1".to_string()), Some("test_2".to_string())]) + ); + } + + { + let (left, right) = contents[2].split(1).unwrap(); + assert!(contents[2].splittable()); + assert_eq!(left, Content::Any(vec![Any::BigInt64(42)])); + assert_eq!( + right, + Content::Any(vec![Any::String("Test Any".to_string())]) + ); + } + + { + assert!(!contents[3].splittable()); + assert_eq!( + contents[3].split(2), + Err(JwstCodecError::ContentSplitNotSupport(2)) + ); + } + } + + proptest! { + #[test] + #[cfg_attr(miri, ignore)] + fn test_random_content(contents in vec(any::(), 0..10)) { + for content in &contents { + content_round_trip(content).unwrap(); + } + } + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/delete_set.rs b/packages/common/y-octo/core/src/doc/codec/delete_set.rs new file mode 100644 index 0000000000..f3f3324570 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/delete_set.rs @@ -0,0 +1,233 @@ +use std::{ + collections::{hash_map::Entry, VecDeque}, + ops::{Deref, DerefMut, Range}, +}; + +use super::*; +use crate::doc::OrderRange; + +impl CrdtRead for Range { + fn read(decoder: &mut R) -> JwstCodecResult { + let clock = decoder.read_var_u64()?; + let len = decoder.read_var_u64()?; + Ok(clock..clock + len) + } +} + +impl CrdtWrite for Range { + fn write(&self, encoder: &mut W) -> JwstCodecResult { + encoder.write_var_u64(self.start)?; + encoder.write_var_u64(self.end - self.start)?; + Ok(()) + } +} + +impl CrdtRead for OrderRange { + fn read(decoder: &mut R) -> JwstCodecResult { + let num_of_deletes = decoder.read_var_u64()? as usize; + if num_of_deletes == 1 { + Ok(OrderRange::Range(Range::::read(decoder)?)) + } else { + let mut deletes = VecDeque::with_capacity(num_of_deletes); + + for _ in 0..num_of_deletes { + deletes.push_back(Range::::read(decoder)?); + } + + Ok(OrderRange::Fragment(deletes)) + } + } +} + +impl CrdtWrite for OrderRange { + fn write(&self, encoder: &mut W) -> JwstCodecResult { + match self { + OrderRange::Range(range) => { + encoder.write_var_u64(1)?; + range.write(encoder)?; + } + OrderRange::Fragment(ranges) => { + encoder.write_var_u64(ranges.len() as u64)?; + for range in ranges { + range.write(encoder)?; + } + } + } + + Ok(()) + } +} + +#[derive(Debug, Default, Clone, PartialEq)] +pub struct DeleteSet(pub ClientMap); + +impl Deref for DeleteSet { + type Target = ClientMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From<[(Client, Vec>); N]> for DeleteSet { + fn from(value: [(Client, Vec>); N]) -> Self { + let mut map = ClientMap::with_capacity(N); + for (client, ranges) in value { + map.insert(client, ranges.into()); + } + Self(map) + } +} + +impl DerefMut for DeleteSet { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl DeleteSet { + pub fn add(&mut self, client: Client, from: Clock, len: Clock) { + self.add_range(client, from..from + len); + } + + pub fn add_range(&mut self, client: Client, range: Range) { + match self.0.entry(client) { + Entry::Occupied(e) => { + let r = e.into_mut(); + if r.is_empty() { + *r = range.into(); + } else { + r.push(range); + } + } + Entry::Vacant(e) => { + e.insert(range.into()); + } + } + } + + pub fn batch_add_ranges(&mut self, client: Client, ranges: Vec>) { + match self.0.entry(client) { + Entry::Occupied(e) => { + e.into_mut().extend(ranges); + } + Entry::Vacant(e) => { + e.insert(ranges.into()); + } + } + } + + pub fn merge(&mut self, other: &Self) { + for (client, range) in &other.0 { + match self.0.entry(*client) { + Entry::Occupied(e) => { + e.into_mut().merge(range.clone()); + } + Entry::Vacant(e) => { + e.insert(range.clone()); + } + } + } + } +} + +impl CrdtRead for DeleteSet { + fn read(decoder: &mut R) -> JwstCodecResult { + let num_of_clients = decoder.read_var_u64()? as usize; + // See: [HASHMAP_SAFE_CAPACITY] + let mut map = ClientMap::with_capacity(num_of_clients.min(HASHMAP_SAFE_CAPACITY)); + + for _ in 0..num_of_clients { + let client = decoder.read_var_u64()?; + let deletes = OrderRange::read(decoder)?; + map.insert(client, deletes); + } + + map.shrink_to_fit(); + Ok(DeleteSet(map)) + } +} + +impl CrdtWrite for DeleteSet { + fn write(&self, encoder: &mut W) -> JwstCodecResult { + encoder.write_var_u64(self.len() as u64)?; + let mut clients = self.keys().copied().collect::>(); + + // Descending + clients.sort_by(|a, b| b.cmp(a)); + + for client in clients { + encoder.write_var_u64(client)?; + self.get(&client).unwrap().write(encoder)?; + } + + Ok(()) + } +} + +#[cfg(test)] +#[allow(clippy::single_range_in_vec_init)] +mod tests { + use super::*; + + #[test] + fn test_delete_set_add() { + let delete_set = DeleteSet::from([ + (1, vec![0..10, 20..30]), + (2, vec![0..5, 10..20]), + (3, vec![15..20, 30..35]), + (4, vec![0..10]), + ]); + + { + let mut delete_set = delete_set.clone(); + delete_set.add(1, 5, 25); + assert_eq!(delete_set.get(&1), Some(&OrderRange::Range(0..30))); + } + + { + let mut delete_set = delete_set; + delete_set.add(1, 5, 10); + assert_eq!( + delete_set.get(&1), + Some(&OrderRange::from(vec![0..15, 20..30])) + ); + } + } + + #[test] + fn test_delete_set_batch_push() { + let delete_set = DeleteSet::from([ + (1, vec![0..10, 20..30]), + (2, vec![0..5, 10..20]), + (3, vec![15..20, 30..35]), + (4, vec![0..10]), + ]); + + { + let mut delete_set = delete_set.clone(); + delete_set.batch_add_ranges(1, vec![0..5, 10..20]); + assert_eq!(delete_set.get(&1), Some(&OrderRange::Range(0..30))); + } + + { + let mut delete_set = delete_set; + delete_set.batch_add_ranges(1, vec![40..50, 10..20]); + assert_eq!( + delete_set.get(&1), + Some(&OrderRange::from(vec![0..30, 40..50])) + ); + } + } + + #[test] + fn test_encode_decode() { + let delete_set = DeleteSet::from([(1, vec![0..10, 20..30]), (2, vec![0..5, 10..20])]); + let mut encoder = RawEncoder::default(); + delete_set.write(&mut encoder).unwrap(); + let update = encoder.into_inner(); + let mut decoder = RawDecoder::new(&update); + let decoded = DeleteSet::read(&mut decoder).unwrap(); + assert_eq!(delete_set, decoded); + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/id.rs b/packages/common/y-octo/core/src/doc/codec/id.rs new file mode 100644 index 0000000000..e51a4c3d19 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/id.rs @@ -0,0 +1,68 @@ +use std::{ + fmt::Display, + hash::Hash, + ops::{Add, Sub}, +}; + +pub type Client = u64; +pub type Clock = u64; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] +#[cfg_attr(fuzzing, derive(arbitrary::Arbitrary))] +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +pub struct Id { + pub client: Client, + pub clock: Clock, +} + +impl Id { + pub fn new(client: Client, clock: Clock) -> Self { + Self { client, clock } + } +} + +impl From<(Client, Clock)> for Id { + fn from((client, clock): (Client, Clock)) -> Self { + Id::new(client, clock) + } +} + +impl Sub for Id { + type Output = Id; + + fn sub(self, rhs: Clock) -> Self::Output { + (self.client, self.clock - rhs).into() + } +} + +impl Add for Id { + type Output = Id; + + fn add(self, rhs: Clock) -> Self::Output { + (self.client, self.clock + rhs).into() + } +} + +impl Display for Id { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({}, {})", self.client, self.clock) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic_id_operation() { + let id_with_different_client_1 = Id::new(1, 1); + let id_with_different_client_2 = Id::new(2, 1); + + assert_ne!(id_with_different_client_1, id_with_different_client_2); + assert_eq!(Id::new(1, 1), Id::new(1, 1)); + + let clock = 2; + assert_eq!(Id::new(1, 1) + clock, (1, 3).into()); + assert_eq!(Id::new(1, 3) - clock, (1, 1).into()); + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/io/codec_v1.rs b/packages/common/y-octo/core/src/doc/codec/io/codec_v1.rs new file mode 100644 index 0000000000..726def1be5 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/io/codec_v1.rs @@ -0,0 +1,296 @@ +use std::io::Cursor; + +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; + +use super::*; + +#[inline] +pub fn read_with_cursor(buffer: &mut Cursor<&[u8]>, f: F) -> JwstCodecResult +where + F: FnOnce(&[u8]) -> IResult<&[u8], T>, +{ + // TODO: use remaining_slice() instead after it is stabilized + let input = buffer.get_ref(); + let rest_pos = buffer.position().min(input.len() as u64) as usize; + let input = &input[rest_pos..]; + + let (tail, result) = f(input).map_err(|e| e.map_input(|u| u.len()))?; + + buffer.set_position((rest_pos + input.len() - tail.len()) as u64); + Ok(result) +} + +// compatible with ydoc v1 +#[derive(Clone)] +pub struct RawDecoder<'b> { + pub(super) buffer: Cursor<&'b [u8]>, +} + +impl<'b> RawDecoder<'b> { + pub fn new(buffer: &'b [u8]) -> Self { + Self { + buffer: Cursor::new(buffer), + } + } + + pub fn rest_ref(&self) -> &[u8] { + let pos = self.buffer.position(); + let buf = self.buffer.get_ref(); + + if pos == 0 { + buf + } else { + &buf[(pos as usize).min(buf.len())..] + } + } + + pub fn drain(self) -> &'b [u8] { + let pos = self.buffer.position() as usize; + let buf = self.buffer.into_inner(); + + if pos == 0 { + buf + } else { + &buf[pos..] + } + } +} + +impl CrdtReader for RawDecoder<'_> { + fn is_empty(&self) -> bool { + self.buffer.position() >= self.buffer.get_ref().len() as u64 + } + + fn len(&self) -> u64 { + self.buffer.get_ref().len() as u64 - self.buffer.position() + } + + fn read_var_u64(&mut self) -> JwstCodecResult { + read_with_cursor(&mut self.buffer, read_var_u64) + } + + fn read_var_i32(&mut self) -> JwstCodecResult { + read_with_cursor(&mut self.buffer, read_var_i32) + } + + fn read_var_string(&mut self) -> JwstCodecResult { + read_with_cursor(&mut self.buffer, read_var_string) + } + + fn read_var_buffer(&mut self) -> JwstCodecResult> { + read_with_cursor(&mut self.buffer, |i| { + read_var_buffer(i).map(|(tail, val)| (tail, val.to_vec())) + }) + } + + fn read_u8(&mut self) -> JwstCodecResult { + self.buffer.read_u8().map_err(reader::map_read_error) + } + + fn read_f32_be(&mut self) -> JwstCodecResult { + self + .buffer + .read_f32::() + .map_err(reader::map_read_error) + } + + fn read_f64_be(&mut self) -> JwstCodecResult { + self + .buffer + .read_f64::() + .map_err(reader::map_read_error) + } + + fn read_i64_be(&mut self) -> JwstCodecResult { + self + .buffer + .read_i64::() + .map_err(reader::map_read_error) + } + + #[inline(always)] + fn read_info(&mut self) -> JwstCodecResult { + self.read_u8() + } + + #[inline(always)] + fn read_item_id(&mut self) -> JwstCodecResult { + let client = self.read_var_u64()?; + let clock = self.read_var_u64()?; + Ok(Id::new(client, clock)) + } +} + +// compatible with ydoc v1 +#[derive(Default)] +pub struct RawEncoder { + buffer: Cursor>, +} + +impl RawEncoder { + pub fn into_inner(self) -> Vec { + self.buffer.into_inner() + } +} + +impl CrdtWriter for RawEncoder { + fn write_var_u64(&mut self, num: u64) -> JwstCodecResult { + write_var_u64(&mut self.buffer, num).map_err(writer::map_write_error) + } + fn write_var_i32(&mut self, num: i32) -> JwstCodecResult { + write_var_i32(&mut self.buffer, num).map_err(writer::map_write_error) + } + fn write_var_string>(&mut self, s: S) -> JwstCodecResult { + write_var_string(&mut self.buffer, s).map_err(writer::map_write_error) + } + fn write_var_buffer(&mut self, buf: &[u8]) -> JwstCodecResult { + write_var_buffer(&mut self.buffer, buf).map_err(writer::map_write_error) + } + fn write_u8(&mut self, num: u8) -> JwstCodecResult { + self.buffer.write_u8(num).map_err(writer::map_write_error)?; + Ok(()) + } + fn write_f32_be(&mut self, num: f32) -> JwstCodecResult { + self + .buffer + .write_f32::(num) + .map_err(writer::map_write_error) + } + fn write_f64_be(&mut self, num: f64) -> JwstCodecResult { + self + .buffer + .write_f64::(num) + .map_err(writer::map_write_error) + } + fn write_i64_be(&mut self, num: i64) -> JwstCodecResult { + self + .buffer + .write_i64::(num) + .map_err(writer::map_write_error) + } + + #[inline(always)] + fn write_info(&mut self, num: u8) -> JwstCodecResult { + self.write_u8(num) + } + + #[inline(always)] + fn write_item_id(&mut self, id: &Id) -> JwstCodecResult { + self.write_var_u64(id.client)?; + self.write_var_u64(id.clock)?; + Ok(()) + } +} + +#[cfg(test)] +#[allow(clippy::approx_constant)] +mod tests { + use super::*; + + #[test] + fn test_crdt_reader() { + { + let mut reader = RawDecoder::new(&[0xf2, 0x5]); + assert_eq!(reader.read_var_u64().unwrap(), 754); + } + { + let mut reader = RawDecoder::new(&[0x5, b'h', b'e', b'l', b'l', b'o']); + + assert_eq!(reader.clone().read_var_string().unwrap(), "hello"); + assert_eq!( + reader.clone().read_var_buffer().unwrap().as_slice(), + b"hello" + ); + + assert_eq!(reader.read_u8().unwrap(), 5); + assert_eq!(reader.read_u8().unwrap(), b'h'); + assert_eq!(reader.read_u8().unwrap(), b'e'); + assert_eq!(reader.read_u8().unwrap(), b'l'); + assert_eq!(reader.read_u8().unwrap(), b'l'); + assert_eq!(reader.read_u8().unwrap(), b'o'); + } + { + let mut reader = RawDecoder::new(&[0x40, 0x49, 0x0f, 0xdb]); + assert_eq!(reader.read_f32_be().unwrap(), 3.1415927); + } + { + let mut reader = RawDecoder::new(&[0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18]); + assert_eq!(reader.read_f64_be().unwrap(), 3.141592653589793); + } + { + let mut reader = RawDecoder::new(&[0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); + assert_eq!(reader.read_i64_be().unwrap(), i64::MAX); + } + { + let mut reader = RawDecoder::new(&[0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + assert_eq!(reader.read_i64_be().unwrap(), i64::MIN); + } + } + + #[test] + fn test_crdt_writer() { + { + let mut writer = RawEncoder::default(); + writer.write_var_u64(754).unwrap(); + assert_eq!(writer.into_inner(), vec![0xf2, 0x5]); + } + { + let ret = vec![0x5, b'h', b'e', b'l', b'l', b'o']; + let mut writer = RawEncoder::default(); + writer.write_var_string("hello").unwrap(); + assert_eq!(writer.into_inner(), ret); + + let mut writer = RawEncoder::default(); + writer.write_var_buffer(b"hello").unwrap(); + assert_eq!(writer.into_inner(), ret); + + let mut writer = RawEncoder::default(); + writer.write_u8(5).unwrap(); + writer.write_u8(b'h').unwrap(); + writer.write_u8(b'e').unwrap(); + writer.write_u8(b'l').unwrap(); + writer.write_u8(b'l').unwrap(); + writer.write_u8(b'o').unwrap(); + assert_eq!(writer.into_inner(), ret); + } + { + let mut writer = RawEncoder::default(); + writer.write_f32_be(3.1415927).unwrap(); + assert_eq!(writer.into_inner(), vec![0x40, 0x49, 0x0f, 0xdb]); + } + { + let mut writer = RawEncoder::default(); + writer.write_f64_be(3.141592653589793).unwrap(); + assert_eq!( + writer.into_inner(), + vec![0x40, 0x09, 0x21, 0xfb, 0x54, 0x44, 0x2d, 0x18] + ); + } + { + let mut writer = RawEncoder::default(); + writer.write_i64_be(i64::MAX).unwrap(); + assert_eq!( + writer.into_inner(), + vec![0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff] + ); + } + { + let mut writer = RawEncoder::default(); + writer.write_i64_be(i64::MIN).unwrap(); + assert_eq!( + writer.into_inner(), + vec![0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + { + let mut writer = RawEncoder::default(); + writer.write_info(0x80).unwrap(); + assert_eq!(writer.into_inner(), vec![0x80]); + } + { + let mut writer = RawEncoder::default(); + writer.write_item_id(&Id::new(1, 2)).unwrap(); + assert_eq!(writer.into_inner(), vec![0x1, 0x2]); + } + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/io/mod.rs b/packages/common/y-octo/core/src/doc/codec/io/mod.rs new file mode 100644 index 0000000000..33cbf975e3 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/io/mod.rs @@ -0,0 +1,9 @@ +mod codec_v1; +mod reader; +mod writer; + +pub use codec_v1::{RawDecoder, RawEncoder}; +pub use reader::{CrdtRead, CrdtReader}; +pub use writer::{CrdtWrite, CrdtWriter}; + +use super::*; diff --git a/packages/common/y-octo/core/src/doc/codec/io/reader.rs b/packages/common/y-octo/core/src/doc/codec/io/reader.rs new file mode 100644 index 0000000000..4407af89f3 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/io/reader.rs @@ -0,0 +1,30 @@ +use std::io::Error; + +use super::*; + +#[inline] +pub fn map_read_error(e: Error) -> JwstCodecError { + JwstCodecError::IncompleteDocument(e.to_string()) +} + +pub trait CrdtReader { + fn is_empty(&self) -> bool; + fn len(&self) -> u64; + fn read_var_u64(&mut self) -> JwstCodecResult; + fn read_var_i32(&mut self) -> JwstCodecResult; + fn read_var_string(&mut self) -> JwstCodecResult; + fn read_var_buffer(&mut self) -> JwstCodecResult>; + fn read_u8(&mut self) -> JwstCodecResult; + fn read_f32_be(&mut self) -> JwstCodecResult; + fn read_f64_be(&mut self) -> JwstCodecResult; + fn read_i64_be(&mut self) -> JwstCodecResult; + + fn read_info(&mut self) -> JwstCodecResult; + fn read_item_id(&mut self) -> JwstCodecResult; +} + +pub trait CrdtRead { + fn read(reader: &mut R) -> JwstCodecResult + where + Self: Sized; +} diff --git a/packages/common/y-octo/core/src/doc/codec/io/writer.rs b/packages/common/y-octo/core/src/doc/codec/io/writer.rs new file mode 100644 index 0000000000..16f71c545b --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/io/writer.rs @@ -0,0 +1,28 @@ +use std::io::Error; + +use super::*; + +#[inline] +pub fn map_write_error(e: Error) -> JwstCodecError { + JwstCodecError::InvalidWriteBuffer(e.to_string()) +} + +pub trait CrdtWriter { + fn write_var_u64(&mut self, num: u64) -> JwstCodecResult; + fn write_var_i32(&mut self, num: i32) -> JwstCodecResult; + fn write_var_string>(&mut self, s: S) -> JwstCodecResult; + fn write_var_buffer(&mut self, buf: &[u8]) -> JwstCodecResult; + fn write_u8(&mut self, num: u8) -> JwstCodecResult; + fn write_f32_be(&mut self, num: f32) -> JwstCodecResult; + fn write_f64_be(&mut self, num: f64) -> JwstCodecResult; + fn write_i64_be(&mut self, num: i64) -> JwstCodecResult; + + fn write_info(&mut self, num: u8) -> JwstCodecResult; + fn write_item_id(&mut self, id: &Id) -> JwstCodecResult; +} + +pub trait CrdtWrite { + fn write(&self, writer: &mut W) -> JwstCodecResult + where + Self: Sized; +} diff --git a/packages/common/y-octo/core/src/doc/codec/item.rs b/packages/common/y-octo/core/src/doc/codec/item.rs new file mode 100644 index 0000000000..90b91297d5 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/item.rs @@ -0,0 +1,427 @@ +use super::*; + +#[derive(Debug, Clone)] +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +pub(crate) enum Parent { + #[cfg_attr(test, proptest(skip))] + Type(YTypeRef), + #[cfg_attr(test, proptest(value = "Parent::String(SmolStr::default())"))] + String(SmolStr), + Id(Id), +} + +#[derive(Clone)] +#[cfg_attr(all(test, not(loom)), derive(proptest_derive::Arbitrary))] +pub(crate) struct Item { + pub id: Id, + pub origin_left_id: Option, + pub origin_right_id: Option, + #[cfg_attr(all(test, not(loom)), proptest(value = "Somr::none()"))] + pub left: ItemRef, + #[cfg_attr(all(test, not(loom)), proptest(value = "Somr::none()"))] + pub right: ItemRef, + pub parent: Option, + #[cfg_attr(all(test, not(loom)), proptest(value = "Option::::None"))] + pub parent_sub: Option, + pub content: Content, + #[cfg_attr(all(test, not(loom)), proptest(value = "ItemFlag::default()"))] + pub flags: ItemFlag, +} + +// make all Item readonly +pub(crate) type ItemRef = Somr; + +impl PartialEq for Item { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl std::fmt::Debug for Item { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut dbg = f.debug_struct("Item"); + dbg + .field("id", &self.id) + .field("origin_left_id", &self.origin_left_id) + .field("origin_right_id", &self.origin_right_id); + + if let Some(left) = self.left.get() { + dbg.field("left", &left.id); + } + + if let Some(right) = self.right.get() { + dbg.field("right", &right.id); + } + + dbg + .field( + "parent", + &self.parent.as_ref().map(|p| match p { + Parent::Type(_) => "[Type]".to_string(), + Parent::String(name) => format!("Parent({name})"), + Parent::Id(id) => format!("({}, {})", id.client, id.clock), + }), + ) + .field("parent_sub", &self.parent_sub) + .field("content", &self.content) + .field("flags", &self.flags) + .finish() + } +} + +impl std::fmt::Display for Item { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Item{}: [{:?}]", self.id, self.content) + } +} + +impl Default for Item { + fn default() -> Self { + Self { + id: Id::default(), + origin_left_id: None, + origin_right_id: None, + left: Somr::none(), + right: Somr::none(), + parent: None, + parent_sub: None, + content: Content::Deleted(0), + flags: ItemFlag::from(0), + } + } +} + +impl Item { + pub fn new( + id: Id, + content: Content, + left: Somr, + right: Somr, + parent: Option, + parent_sub: Option, + ) -> Self { + let flags = ItemFlag::from(if content.countable() { + item_flags::ITEM_COUNTABLE + } else { + 0 + }); + + Self { + id, + origin_left_id: left.get().map(|left| left.last_id()), + left, + origin_right_id: right.get().map(|right| right.id), + right, + parent, + parent_sub, + content, + flags, + } + } + + // find a note that has parent info + // in crdt tree, not all node has parent info + // so we need to check left and right node if they have parent info + pub fn find_node_with_parent_info(&self) -> Option { + if self.parent.is_some() { + return Some(self.clone()); + } else if let Some(item) = self.left.get() { + if item.parent.is_none() { + if let Some(item) = item.right.get() { + return Some(item.clone()); + } + } else { + return Some(item.clone()); + } + } else if let Some(item) = self.right.get() { + return Some(item.clone()); + } + None + } + + pub fn len(&self) -> u64 { + self.content.clock_len() + } + + pub fn deleted(&self) -> bool { + self.flags.deleted() + } + + pub fn delete(&self) -> bool { + if self.deleted() { + return false; + } + + self.flags.set_deleted(); + + true + } + + pub fn countable(&self) -> bool { + self.flags.countable() + } + + pub fn keep(&self) -> bool { + self.flags.keep() + } + + pub fn indexable(&self) -> bool { + self.countable() && !self.deleted() + } + + pub fn last_id(&self) -> Id { + let Id { client, clock } = self.id; + + Id::new(client, clock + self.len() - 1) + } + + pub fn split_at(&self, offset: u64) -> JwstCodecResult<(Self, Self)> { + debug_assert!(offset > 0 && self.len() > 1 && offset < self.len()); + let id = self.id; + let right_id = Id::new(id.client, id.clock + offset); + let (left_content, right_content) = self.content.split(offset)?; + + let left_item = Item::new( + id, + left_content, + // let caller connect left <-> node <-> right + Somr::none(), + Somr::none(), + self.parent.clone(), + self.parent_sub.clone(), + ); + + let right_item = Item::new( + right_id, + right_content, + // let caller connect left <-> node <-> right + Somr::none(), + Somr::none(), + self.parent.clone(), + self.parent_sub.clone(), + ); + + if left_item.deleted() { + left_item.flags.set_deleted(); + } + if left_item.keep() { + left_item.flags.set_keep(); + } + + Ok((left_item, right_item)) + } + + fn get_info(&self) -> u8 { + let mut info = self.content.get_info(); + + if self.origin_left_id.is_some() { + info |= item_flags::ITEM_HAS_LEFT_ID; + } + if self.origin_right_id.is_some() { + info |= item_flags::ITEM_HAS_RIGHT_ID; + } + if self.parent_sub.is_some() { + info |= item_flags::ITEM_HAS_PARENT_SUB; + } + + info + } + + pub fn is_valid(&self) -> bool { + let has_id = self.origin_left_id.is_some() || self.origin_right_id.is_some(); + !has_id && self.parent.is_some() || has_id && self.parent.is_none() && self.parent_sub.is_none() + } + + pub fn read( + decoder: &mut R, + id: Id, + info: u8, + first_5_bit: u8, + ) -> JwstCodecResult { + let flags: ItemFlag = info.into(); + let has_left_id = flags.check(item_flags::ITEM_HAS_LEFT_ID); + let has_right_id = flags.check(item_flags::ITEM_HAS_RIGHT_ID); + let has_parent_sub = flags.check(item_flags::ITEM_HAS_PARENT_SUB); + let has_not_sibling = flags.not(item_flags::ITEM_HAS_SIBLING); + + // NOTE: read order must keep the same as the order in yjs + // TODO: this data structure design will break the cpu OOE, need to be optimized + let item = Self { + id, + origin_left_id: if has_left_id { + Some(decoder.read_item_id()?) + } else { + None + }, + origin_right_id: if has_right_id { + Some(decoder.read_item_id()?) + } else { + None + }, + parent: { + if has_not_sibling { + let has_parent = decoder.read_var_u64()? == 1; + Some(if has_parent { + Parent::String(SmolStr::new(decoder.read_var_string()?)) + } else { + Parent::Id(decoder.read_item_id()?) + }) + } else { + None + } + }, + parent_sub: if has_not_sibling && has_parent_sub { + Some(SmolStr::new(decoder.read_var_string()?)) + } else { + None + }, + content: { + // tag must not GC or Skip, this must process in parse_struct + debug_assert_ne!(first_5_bit, 0); + debug_assert_ne!(first_5_bit, 10); + Content::read(decoder, first_5_bit)? + }, + left: Somr::none(), + right: Somr::none(), + flags: ItemFlag::from(0), + }; + + if item.content.countable() { + item.flags.set_countable(); + } + + if matches!(item.content, Content::Deleted(_)) { + item.flags.set_deleted(); + } + + debug_assert!(item.is_valid()); + + Ok(item) + } + + pub fn write(&self, encoder: &mut W) -> JwstCodecResult { + let info = self.get_info(); + let has_not_sibling = info & item_flags::ITEM_HAS_SIBLING == 0; + + encoder.write_info(info)?; + + if let Some(left_id) = self.origin_left_id { + encoder.write_item_id(&left_id)?; + } + if let Some(right_id) = self.origin_right_id { + encoder.write_item_id(&right_id)?; + } + + if has_not_sibling { + if let Some(parent) = &self.parent { + match parent { + Parent::String(s) => { + encoder.write_var_u64(1)?; + encoder.write_var_string(s)?; + } + Parent::Id(id) => { + encoder.write_var_u64(0)?; + encoder.write_item_id(id)?; + } + Parent::Type(ty) => { + if let Some(ty) = ty.ty() { + if let Some(item) = ty.item.get() { + encoder.write_var_u64(0)?; + encoder.write_item_id(&item.id)?; + } else if let Some(name) = &ty.root_name { + encoder.write_var_u64(1)?; + encoder.write_var_string(name)?; + } + } + } + } + } else { + // if item delete, it must not exists in crdt state tree + debug_assert!(!self.deleted()); + return Err(JwstCodecError::ParentNotFound); + } + + if let Some(parent_sub) = &self.parent_sub { + encoder.write_var_string(parent_sub)?; + } + } + + self.content.write(encoder)?; + + Ok(()) + } +} + +#[allow(dead_code)] +#[cfg(any(debug, test))] +impl Item { + pub fn print_left(&self) { + let mut ret = vec![format!("Self{}: [{:?}]", self.id, self.content)]; + let mut left: Somr = self.left.clone(); + + while let Some(item) = left.get() { + ret.push(format!("{item}")); + left = item.left.clone(); + } + ret.reverse(); + + println!("{}", ret.join(" <- ")); + } + + pub fn print_right(&self) { + let mut ret = vec![format!("Self{}: [{:?}]", self.id, self.content)]; + let mut right = self.right.clone(); + + while let Some(item) = right.get() { + ret.push(format!("{item}")); + right = item.right.clone(); + } + + println!("{}", ret.join(" -> ")); + } +} + +#[cfg(test)] +mod tests { + #[cfg(not(loom))] + use proptest::{collection::vec, prelude::*}; + + #[cfg(not(loom))] + use super::*; + + #[cfg(not(loom))] + fn item_round_trip(item: &mut Item) -> JwstCodecResult { + if !item.is_valid() { + return Ok(()); + } + + if item.content.countable() { + item.flags.set_countable(); + } + + let mut encoder = RawEncoder::default(); + item.write(&mut encoder)?; + + let update = encoder.into_inner(); + let mut decoder = RawDecoder::new(&update); + + let info = decoder.read_info()?; + let first_5_bit = info & 0b11111; + let decoded_item = Item::read(&mut decoder, item.id, info, first_5_bit)?; + + assert_eq!(item, &decoded_item); + + Ok(()) + } + + #[cfg(not(loom))] + proptest! { + #[test] + #[cfg_attr(miri, ignore)] + fn test_random_content(mut items in vec(any::(), 0..10)) { + for item in &mut items { + item_round_trip(item).unwrap(); + } + } + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/item_flag.rs b/packages/common/y-octo/core/src/doc/codec/item_flag.rs new file mode 100644 index 0000000000..472066f746 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/item_flag.rs @@ -0,0 +1,170 @@ +use std::sync::atomic::{AtomicU8, Ordering}; + +#[rustfmt::skip] +#[allow(dead_code)] +pub mod item_flags { + pub const ITEM_KEEP : u8 = 0b0000_0001; + pub const ITEM_COUNTABLE : u8 = 0b0000_0010; + pub const ITEM_DELETED : u8 = 0b0000_0100; + pub const ITEM_MARKED : u8 = 0b0000_1000; + pub const ITEM_HAS_PARENT_SUB : u8 = 0b0010_0000; + pub const ITEM_HAS_RIGHT_ID : u8 = 0b0100_0000; + pub const ITEM_HAS_LEFT_ID : u8 = 0b1000_0000; + pub const ITEM_HAS_SIBLING : u8 = 0b1100_0000; +} + +#[derive(Debug)] +pub struct ItemFlag(pub(self) AtomicU8); + +impl Default for ItemFlag { + fn default() -> Self { + Self(AtomicU8::new(0)) + } +} + +impl Clone for ItemFlag { + fn clone(&self) -> Self { + Self(AtomicU8::new(self.0.load(Ordering::Acquire))) + } +} + +impl From for ItemFlag { + fn from(flags: u8) -> Self { + Self(AtomicU8::new(flags)) + } +} + +#[allow(dead_code)] +impl ItemFlag { + #[inline(always)] + pub fn set(&self, flag: u8) { + self.0.fetch_or(flag, Ordering::SeqCst); + } + + #[inline(always)] + pub fn clear(&self, flag: u8) { + self.0.fetch_and(!flag, Ordering::SeqCst); + } + + #[inline(always)] + pub fn check(&self, flag: u8) -> bool { + self.0.load(Ordering::Acquire) & flag == flag + } + + #[inline(always)] + pub fn not(&self, flag: u8) -> bool { + self.0.load(Ordering::Acquire) & flag == 0 + } + + #[inline(always)] + pub fn keep(&self) -> bool { + self.check(item_flags::ITEM_KEEP) + } + + #[inline(always)] + pub fn set_keep(&self) { + self.set(item_flags::ITEM_KEEP); + } + + #[inline(always)] + pub fn clear_keep(&self) { + self.clear(item_flags::ITEM_KEEP); + } + + #[inline(always)] + pub fn countable(&self) -> bool { + self.check(item_flags::ITEM_COUNTABLE) + } + + #[inline(always)] + pub fn set_countable(&self) { + self.set(item_flags::ITEM_COUNTABLE); + } + + #[inline(always)] + pub fn clear_countable(&self) { + self.clear(item_flags::ITEM_COUNTABLE); + } + + #[inline(always)] + pub fn deleted(&self) -> bool { + self.check(item_flags::ITEM_DELETED) + } + + #[inline(always)] + pub fn set_deleted(&self) { + self.set(item_flags::ITEM_DELETED); + } + + #[inline(always)] + pub fn clear_deleted(&self) { + self.clear(item_flags::ITEM_DELETED); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_flag_set_and_clear() { + { + let flag = super::ItemFlag::default(); + assert!(!flag.keep()); + flag.set_keep(); + assert!(flag.keep()); + flag.clear_keep(); + assert!(!flag.keep()); + assert_eq!( + flag.0.load(Ordering::SeqCst), + ItemFlag::default().0.load(Ordering::SeqCst) + ); + } + + { + let flag = super::ItemFlag::default(); + assert!(!flag.countable()); + flag.set_countable(); + assert!(flag.countable()); + flag.clear_countable(); + assert!(!flag.countable()); + assert_eq!( + flag.0.load(Ordering::SeqCst), + ItemFlag::default().0.load(Ordering::SeqCst) + ); + } + + { + let flag = super::ItemFlag::default(); + assert!(!flag.deleted()); + flag.set_deleted(); + assert!(flag.deleted()); + flag.clear_deleted(); + assert!(!flag.deleted()); + assert_eq!( + flag.0.load(Ordering::SeqCst), + ItemFlag::default().0.load(Ordering::SeqCst) + ); + } + + { + let flag = super::ItemFlag::default(); + flag.set_keep(); + flag.set_countable(); + flag.set_deleted(); + assert!(flag.keep()); + assert!(flag.countable()); + assert!(flag.deleted()); + flag.clear_keep(); + flag.clear_countable(); + flag.clear_deleted(); + assert!(!flag.keep()); + assert!(!flag.countable()); + assert!(!flag.deleted()); + assert_eq!( + flag.0.load(Ordering::SeqCst), + ItemFlag::default().0.load(Ordering::SeqCst) + ); + } + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/mod.rs b/packages/common/y-octo/core/src/doc/codec/mod.rs new file mode 100644 index 0000000000..8c08854018 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/mod.rs @@ -0,0 +1,25 @@ +mod any; +mod content; +mod delete_set; +mod id; +mod io; +mod item; +mod item_flag; +mod refs; +mod update; +#[cfg(test)] +mod utils; + +pub use any::Any; +pub(crate) use content::Content; +pub use delete_set::DeleteSet; +pub use id::{Client, Clock, Id}; +pub use io::{CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, RawDecoder, RawEncoder}; +pub(crate) use item::{Item, ItemRef, Parent}; +pub(crate) use item_flag::{item_flags, ItemFlag}; +pub(crate) use refs::Node; +pub use update::Update; +#[cfg(test)] +pub(crate) use utils::*; + +use super::*; diff --git a/packages/common/y-octo/core/src/doc/codec/refs.rs b/packages/common/y-octo/core/src/doc/codec/refs.rs new file mode 100644 index 0000000000..4f65e4ac38 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/refs.rs @@ -0,0 +1,480 @@ +use super::*; + +// make fields Copy + Clone without much effort +#[derive(Debug, Clone)] +#[cfg_attr(all(test, not(loom)), derive(proptest_derive::Arbitrary))] +pub(crate) enum Node { + GC(Box), + Skip(Box), + Item(ItemRef), +} + +/// Simple representation of id and len struct used by GC and Skip node. +#[derive(Debug, Clone)] +#[cfg_attr(all(test, not(loom)), derive(proptest_derive::Arbitrary))] +pub(crate) struct NodeLen { + pub id: Id, + pub len: u64, +} + +impl CrdtWrite for Node { + fn write(&self, writer: &mut W) -> JwstCodecResult { + match self { + Node::GC(item) => { + writer.write_info(0)?; + writer.write_var_u64(item.len) + } + Node::Skip(item) => { + writer.write_info(10)?; + writer.write_var_u64(item.len) + } + Node::Item(item) => item.get().unwrap().write(writer), + } + } +} + +impl PartialEq for Node { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Node::GC(left), Node::GC(right)) => left.id == right.id, + (Node::Skip(left), Node::Skip(right)) => left.id == right.id, + (Node::Item(item1), Node::Item(item2)) => item1.get() == item2.get(), + _ => false, + } + } +} + +impl Eq for Node { + fn assert_receiver_is_total_eq(&self) {} +} + +impl From for Node { + fn from(value: Item) -> Self { + Self::Item(Somr::new(value)) + } +} + +impl Node { + pub fn new_skip(id: Id, len: u64) -> Self { + Self::Skip(Box::new(NodeLen { id, len })) + } + + pub fn new_gc(id: Id, len: u64) -> Self { + Self::GC(Box::new(NodeLen { id, len })) + } + + pub fn read(decoder: &mut R, id: Id) -> JwstCodecResult { + let info = decoder.read_info()?; + let first_5_bit = info & 0b11111; + + match first_5_bit { + 0 => { + let len = decoder.read_var_u64()?; + Ok(Node::new_gc(id, len)) + } + 10 => { + let len = decoder.read_var_u64()?; + Ok(Node::new_skip(id, len)) + } + _ => { + let item = Somr::new(Item::read(decoder, id, info, first_5_bit)?); + + if let Content::Type(ty) = &item.get().unwrap().content { + if let Some(mut ty) = ty.ty_mut() { + ty.item = item.clone(); + } + } + + Ok(Node::Item(item)) + } + } + } + + pub fn id(&self) -> Id { + match self { + Node::GC(item) => item.id, + Node::Skip(item) => item.id, + Node::Item(item) => unsafe { item.get_unchecked() }.id, + } + } + + pub fn client(&self) -> Client { + self.id().client + } + + pub fn clock(&self) -> Clock { + self.id().clock + } + + pub fn len(&self) -> u64 { + match self { + Self::GC(item) => item.len, + Self::Skip(item) => item.len, + Self::Item(item) => unsafe { item.get_unchecked() }.len(), + } + } + + pub fn is_gc(&self) -> bool { + matches!(self, Self::GC { .. }) + } + + pub fn is_skip(&self) -> bool { + matches!(self, Self::Skip { .. }) + } + + pub fn is_item(&self) -> bool { + matches!(self, Self::Item(_)) + } + + pub fn as_item(&self) -> Somr { + if let Self::Item(item) = self { + item.clone() + } else { + Somr::none() + } + } + + pub fn left(&self) -> Option { + if let Node::Item(item) = self { + item.get().map(|item| Node::Item(item.left.clone())) + } else { + None + } + } + + pub fn right(&self) -> Option { + if let Node::Item(item) = self { + item.get().map(|item| Node::Item(item.right.clone())) + } else { + None + } + } + + pub fn head(&self) -> Self { + let mut cur = self.clone(); + + while let Some(left) = cur.left() { + if left.is_item() { + cur = left + } else { + break; + } + } + + cur + } + + #[allow(dead_code)] + pub fn tail(&self) -> Self { + let mut cur = self.clone(); + + while let Some(right) = cur.right() { + if right.is_item() { + cur = right + } else { + break; + } + } + + cur + } + + pub fn flags(&self) -> ItemFlag { + if let Node::Item(item) = self { + item.get().unwrap().flags.clone() + } else { + // deleted + ItemFlag::from(4) + } + } + + pub fn last_id(&self) -> Option { + if let Node::Item(item) = self { + item.get().map(|item| item.last_id()) + } else { + None + } + } + + pub fn split_at(&self, offset: u64) -> JwstCodecResult<(Self, Self)> { + if let Self::Item(item) = self { + let item = item.get().unwrap(); + debug_assert!(offset > 0 && item.len() > 1 && offset < item.len()); + let id = item.id; + let right_id = Id::new(id.client, id.clock + offset); + let (left_content, right_content) = item.content.split(offset)?; + + let left_item = Somr::new(Item::new( + id, + left_content, + // let caller connect left <-> node <-> right + Somr::none(), + Somr::none(), + item.parent.clone(), + item.parent_sub.clone(), + )); + + let right_item = Somr::new(Item::new( + right_id, + right_content, + // let caller connect left <-> node <-> right + Somr::none(), + Somr::none(), + item.parent.clone(), + item.parent_sub.clone(), + )); + + Ok((Self::Item(left_item), Self::Item(right_item))) + } else { + Err(JwstCodecError::ItemSplitNotSupport) + } + } + + #[inline] + #[allow(dead_code)] + pub fn countable(&self) -> bool { + self.flags().countable() + } + + #[inline] + pub fn deleted(&self) -> bool { + self.flags().deleted() + } + + pub fn merge(&mut self, right: Self) -> bool { + match (self, right) { + (Node::GC(left), Node::GC(right)) => { + left.len += right.len; + } + (Node::Skip(left), Node::Skip(right)) => { + left.len += right.len; + } + (Node::Item(lref), Node::Item(rref)) => { + let mut litem = unsafe { lref.get_mut_unchecked() }; + let mut ritem = unsafe { rref.get_mut_unchecked() }; + let llen = litem.len(); + + if litem.id.client != ritem.id.client + // not same delete status + || litem.deleted() != ritem.deleted() + // not clock continuous + || litem.id.clock + litem.len() != ritem.id.clock + // not insertion continuous + || Some(litem.last_id()) != ritem.origin_left_id + // not insertion continuous + || litem.origin_right_id != ritem.origin_right_id + // not runtime continuous + || litem.right != rref + { + return false; + } + + match (&mut litem.content, &mut ritem.content) { + (Content::Deleted(l), Content::Deleted(r)) => { + *l += *r; + } + (Content::Json(l), Content::Json(r)) => { + l.extend(r.drain(0..)); + } + (Content::String(l), Content::String(r)) => { + *l += r; + } + (Content::Any(l), Content::Any(r)) => { + l.extend(r.drain(0..)); + } + _ => { + return false; + } + } + + if let Some(Parent::Type(p)) = &litem.parent { + if let Some(parent) = p.ty_mut() { + if let Some(markers) = &parent.markers { + markers.replace_marker(rref.clone(), lref.clone(), -(llen as i64)); + } + } + } + + if ritem.keep() { + litem.flags.set_keep() + } + + litem.right = ritem.right.clone(); + unsafe { + if litem.right.is_some() { + litem.right.get_mut_unchecked().left = lref.clone(); + } + } + } + _ => { + return false; + } + } + + true + } +} + +impl From> for Somr { + fn from(value: Option) -> Self { + match value { + Some(n) => n.as_item(), + None => Somr::none(), + } + } +} + +impl From<&Option> for Somr { + fn from(value: &Option) -> Self { + match value { + Some(n) => n.as_item(), + None => Somr::none(), + } + } +} + +impl From> for Somr { + fn from(value: Option<&Node>) -> Self { + match value { + Some(n) => n.as_item(), + None => Somr::none(), + } + } +} + +#[cfg(test)] +mod tests { + #[cfg(not(loom))] + use proptest::{collection::vec, prelude::*}; + + use super::{utils::ItemBuilder, *}; + + #[test] + fn test_struct_info() { + loom_model!({ + { + let struct_info = Node::new_gc(Id::new(1, 0), 10); + assert_eq!(struct_info.len(), 10); + assert_eq!(struct_info.client(), 1); + assert_eq!(struct_info.clock(), 0); + } + + { + let struct_info = Node::new_skip(Id::new(2, 0), 20); + assert_eq!(struct_info.len(), 20); + assert_eq!(struct_info.client(), 2); + assert_eq!(struct_info.clock(), 0); + } + + { + let item = ItemBuilder::new() + .id((3, 0).into()) + .left_id(None) + .right_id(None) + .parent(Some(Parent::String(SmolStr::new_inline("parent")))) + .parent_sub(None) + .content(Content::String(String::from("content"))) + .build(); + let struct_info = Node::Item(Somr::new(item)); + + assert_eq!(struct_info.len(), 7); + assert_eq!(struct_info.client(), 3); + assert_eq!(struct_info.clock(), 0); + } + }); + } + + #[test] + fn test_read_write_struct_info() { + loom_model!({ + let has_not_parent_id_and_has_parent = Node::Item(Somr::new( + ItemBuilder::new() + .id((0, 0).into()) + .left_id(None) + .right_id(None) + .parent(Some(Parent::String(SmolStr::new_inline("parent")))) + .parent_sub(None) + .content(Content::String(String::from("content"))) + .build(), + )); + + let has_not_parent_id_and_has_parent_with_key = Node::Item(Somr::new( + ItemBuilder::new() + .id((0, 0).into()) + .left_id(None) + .right_id(None) + .parent(Some(Parent::String(SmolStr::new_inline("parent")))) + .parent_sub(Some(SmolStr::new_inline("parent_sub"))) + .content(Content::String(String::from("content"))) + .build(), + )); + + let has_parent_id = Node::Item(Somr::new( + ItemBuilder::new() + .id((0, 0).into()) + .left_id(Some((1, 2).into())) + .right_id(Some((2, 5).into())) + .parent(None) + .parent_sub(None) + .content(Content::String(String::from("content"))) + .build(), + )); + + let struct_infos = vec![ + Node::new_gc((0, 0).into(), 42), + Node::new_skip((0, 0).into(), 314), + has_not_parent_id_and_has_parent, + has_not_parent_id_and_has_parent_with_key, + has_parent_id, + ]; + + for info in struct_infos { + let mut encoder = RawEncoder::default(); + info.write(&mut encoder).unwrap(); + + let update = encoder.into_inner(); + let mut decoder = RawDecoder::new(&update); + let decoded = Node::read(&mut decoder, info.id()).unwrap(); + + assert_eq!(info, decoded); + } + }); + } + + #[cfg(not(loom))] + fn struct_info_round_trip(info: &mut Node) -> JwstCodecResult { + if let Node::Item(item) = info { + if let Some(item) = item.get_mut() { + if !item.is_valid() { + return Ok(()); + } + + if item.content.countable() { + item.flags.set_countable(); + } + } + } + let mut encoder = RawEncoder::default(); + info.write(&mut encoder)?; + + let ret = encoder.into_inner(); + let mut decoder = RawDecoder::new(&ret); + + let decoded = Node::read(&mut decoder, info.id())?; + + assert_eq!(info, &decoded); + + Ok(()) + } + + #[cfg(not(loom))] + proptest! { + #[test] + #[cfg_attr(miri, ignore)] + fn test_random_struct_info(mut infos in vec(any::(), 0..10)) { + for info in &mut infos { + struct_info_round_trip(info).unwrap(); + } + } + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/update.rs b/packages/common/y-octo/core/src/doc/codec/update.rs new file mode 100644 index 0000000000..9bc13ceaf1 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/update.rs @@ -0,0 +1,721 @@ +use std::{collections::VecDeque, ops::Range}; + +use super::*; +use crate::doc::StateVector; + +#[derive(Debug, Default, Clone)] +pub struct Update { + pub(crate) structs: ClientMap>, + pub(crate) delete_set: DeleteSet, + + /// all unapplicable items that we can't integrate into doc + /// any item with inconsistent id clock or missing dependency will be put + /// here + pub(crate) pending_structs: ClientMap>, + /// missing state vector after applying updates + pub(crate) missing_state: StateVector, + /// all unapplicable delete set + pub(crate) pending_delete_set: DeleteSet, +} + +impl CrdtRead for Update { + fn read(decoder: &mut R) -> JwstCodecResult { + let num_of_clients = decoder.read_var_u64()? as usize; + + // See: [HASHMAP_SAFE_CAPACITY] + let mut map = ClientMap::with_capacity(num_of_clients.min(HASHMAP_SAFE_CAPACITY)); + for _ in 0..num_of_clients { + let num_of_structs = decoder.read_var_u64()? as usize; + let client = decoder.read_var_u64()?; + let mut clock = decoder.read_var_u64()?; + + // same reason as above + let mut structs = VecDeque::with_capacity(num_of_structs.min(HASHMAP_SAFE_CAPACITY)); + + for _ in 0..num_of_structs { + let struct_info = Node::read(decoder, Id::new(client, clock))?; + clock += struct_info.len(); + structs.push_back(struct_info); + } + + structs.shrink_to_fit(); + map.insert(client, structs); + } + + map.shrink_to_fit(); + + let delete_set = DeleteSet::read(decoder)?; + + if !decoder.is_empty() { + return Err(JwstCodecError::UpdateNotFullyConsumed( + decoder.len() as usize + )); + } + + Ok(Update { + structs: map, + delete_set, + ..Update::default() + }) + } +} + +impl CrdtWrite for Update { + fn write(&self, encoder: &mut W) -> JwstCodecResult { + encoder.write_var_u64(self.structs.len() as u64)?; + + let mut clients = self.structs.keys().copied().collect::>(); + + // Descending + clients.sort_by(|a, b| b.cmp(a)); + + for client in clients { + let structs = self.structs.get(&client).unwrap(); + + encoder.write_var_u64(structs.len() as u64)?; + encoder.write_var_u64(client)?; + encoder.write_var_u64(structs.front().map(|s| s.clock()).unwrap_or(0))?; + + for struct_info in structs { + struct_info.write(encoder)?; + } + } + + self.delete_set.write(encoder)?; + + Ok(()) + } +} + +impl Update { + // decode from ydoc v1 + pub fn decode_v1>(buffer: T) -> JwstCodecResult { + Update::read(&mut RawDecoder::new(buffer.as_ref())) + } + + pub fn encode_v1(&self) -> JwstCodecResult> { + let mut encoder = RawEncoder::default(); + self.write(&mut encoder)?; + Ok(encoder.into_inner()) + } + + pub(crate) fn iter(&mut self, state: StateVector) -> UpdateIterator { + UpdateIterator::new(self, state) + } + + pub fn delete_set_iter(&mut self, state: StateVector) -> DeleteSetIterator { + DeleteSetIterator::new(self, state) + } + + // take all pending structs and delete set to [self] update struct + pub fn drain_pending_state(&mut self) { + debug_assert!(self.is_empty()); + + std::mem::swap(&mut self.pending_structs, &mut self.structs); + std::mem::swap(&mut self.pending_delete_set, &mut self.delete_set); + } + + pub fn merge>(updates: I) -> Update { + let mut merged = Update::default(); + + Self::merge_into(&mut merged, updates); + + merged + } + + pub fn merge_into>(target: &mut Update, updates: I) { + for update in updates { + target.delete_set.merge(&update.delete_set); + + for (client, structs) in update.structs { + let iter = structs.into_iter().filter(|p| !p.is_skip()); + if let Some(merged_structs) = target.structs.get_mut(&client) { + merged_structs.extend(iter); + } else { + target.structs.insert(client, iter.collect()); + } + } + } + + for structs in target.structs.values_mut() { + structs.make_contiguous().sort_by_key(|s| s.id().clock); + + // insert [Node::Skip] if structs[index].id().clock + structs[index].len() < + // structs[index + 1].id().clock + let mut index = 0; + let mut merged_index = vec![]; + while index < structs.len() - 1 { + let cur = &structs[index]; + let next = &structs[index + 1]; + + let clock_end = cur.id().clock + cur.len(); + let next_clock = next.id().clock; + + if next_clock > clock_end { + structs.insert( + index + 1, + Node::new_skip((cur.id().client, clock_end).into(), next_clock - clock_end), + ); + index += 1; + } else if cur.id().clock == next_clock { + if cur.deleted() == next.deleted() + && cur.last_id() == next.last_id() + && cur.left() == next.left() + && cur.right() == next.right() + { + // merge two nodes, mark the index + merged_index.push(index + 1); + } else { + debug!("merge failed: {:?} {:?}", cur, next) + } + } + + index += 1; + } + + { + // prune the merged nodes + let mut new_structs = VecDeque::with_capacity(structs.len() - merged_index.len()); + let mut next_remove_idx = 0; + for (idx, val) in structs.drain(..).enumerate() { + if next_remove_idx < merged_index.len() && idx == merged_index[next_remove_idx] { + next_remove_idx += 1; + } else { + new_structs.push_back(val); + } + } + structs.extend(new_structs); + } + } + } + + pub fn is_content_empty(&self) -> bool { + self.structs.is_empty() + } + + pub fn is_empty(&self) -> bool { + self.structs.is_empty() && self.delete_set.is_empty() + } + + pub fn is_pending_empty(&self) -> bool { + self.pending_structs.is_empty() && self.pending_delete_set.is_empty() + } +} + +pub(crate) struct UpdateIterator<'a> { + update: &'a mut Update, + + // --- local iterator state --- + /// current state vector from store + state: StateVector, + /// all client ids sorted ascending + client_ids: Vec, + /// current id of client of the updates we're processing + cur_client_id: Option, + /// stack of previous iterating item with higher priority than updates in + /// next iteration + stack: Vec, +} + +impl<'a> UpdateIterator<'a> { + pub fn new(update: &'a mut Update, state: StateVector) -> Self { + let mut client_ids = update.structs.keys().cloned().collect::>(); + client_ids.sort(); + let cur_client_id = client_ids.pop(); + + UpdateIterator { + update, + state, + client_ids, + cur_client_id, + stack: Vec::new(), + } + } + + /// iterate the client ids until we find the next client with left updates + /// that can be consumed + /// + /// note: + /// firstly we will check current client id as well to ensure current + /// updates queue is not empty yet + fn next_client(&mut self) -> Option { + while let Some(client_id) = self.cur_client_id { + match self.update.structs.get(&client_id) { + Some(refs) if !refs.is_empty() => { + self.cur_client_id.replace(client_id); + return self.cur_client_id; + } + _ => { + self.update.structs.remove(&client_id); + self.cur_client_id = self.client_ids.pop(); + } + } + } + + None + } + + /// update the missing state vector + /// tell it the smallest clock that missed. + fn update_missing_state(&mut self, client: Client, clock: Clock) { + self.update.missing_state.set_min(client, clock); + } + + /// any time we can't apply an update during the iteration, + /// we should put all items in pending stack to rest structs + fn add_stack_to_rest(&mut self) { + for s in self.stack.drain(..) { + let client = s.id().client; + let unapplicable_items = self.update.structs.remove(&client); + if let Some(mut items) = unapplicable_items { + items.push_front(s); + self.update.pending_structs.insert(client, items); + } else { + self.update.pending_structs.insert(client, [s].into()); + } + self.client_ids.retain(|&c| c != client); + } + } + + /// tell if current update's dependencies(left, right, parent) has already + /// been consumed and recorded and return the client of them if not. + fn get_missing_dep(&self, struct_info: &Node) -> Option { + if let Some(item) = struct_info.as_item().get() { + let id = item.id; + if let Some(left) = &item.origin_left_id { + if left.client != id.client && left.clock >= self.state.get(&left.client) { + return Some(left.client); + } + } + + if let Some(right) = &item.origin_right_id { + if right.client != id.client && right.clock >= self.state.get(&right.client) { + return Some(right.client); + } + } + + if let Some(parent) = &item.parent { + match parent { + Parent::Id(parent_id) + if parent_id.client != id.client + && parent_id.clock >= self.state.get(&parent_id.client) => + { + return Some(parent_id.client); + } + _ => {} + } + } + } + + None + } + + fn next_candidate(&mut self) -> Option { + let mut cur = None; + + if !self.stack.is_empty() { + cur.replace(self.stack.pop().unwrap()); + } else if let Some(client) = self.next_client() { + // Safety: + // client index of updates and update length are both checked in next_client + // safe to use unwrap + cur.replace( + self + .update + .structs + .get_mut(&client) + .unwrap() + .pop_front() + .unwrap(), + ); + } + + cur + } +} + +impl Iterator for UpdateIterator<'_> { + type Item = (Node, u64); + + fn next(&mut self) -> Option { + // fetch the first candidate from stack or updates + let mut cur = self.next_candidate(); + + while let Some(cur_update) = cur.take() { + let id = cur_update.id(); + if cur_update.is_skip() { + cur = self.next_candidate(); + continue; + } else if !self.state.contains(&id) { + // missing local state of same client + // can't apply the continuous updates from same client + // push into the stack and put tell all the items in stack are unapplicable + self.stack.push(cur_update); + self.update_missing_state(id.client, id.clock - 1); + self.add_stack_to_rest(); + } else { + let id = cur_update.id(); + let dep = self.get_missing_dep(&cur_update); + // some dependency is missing, we need to turn to iterate the dependency first. + if let Some(dep) = dep { + self.stack.push(cur_update); + + match self.update.structs.get_mut(&dep) { + Some(updates) if !updates.is_empty() => { + // iterate the dependency client first + cur.replace(updates.pop_front().unwrap()); + continue; + } + // but the dependency update is drained + // need to move all stack item to unapplicable store + _ => { + self.update_missing_state(dep, self.state.get(&dep)); + self.add_stack_to_rest(); + } + } + } else { + // we finally find the first applicable update + let local_state = self.state.get(&id.client); + // we've already check the local state is greater or equal to current update's + // clock so offset here will never be negative + let offset = local_state - id.clock; + if offset == 0 || offset < cur_update.len() { + self.state.set_max(id.client, id.clock + cur_update.len()); + return Some((cur_update, offset)); + } + } + } + + cur = self.next_candidate(); + } + + // we all done + None + } +} + +pub struct DeleteSetIterator<'a> { + update: &'a mut Update, + /// current state vector from store + state: StateVector, +} + +impl<'a> DeleteSetIterator<'a> { + pub fn new(update: &'a mut Update, state: StateVector) -> Self { + DeleteSetIterator { update, state } + } +} + +impl Iterator for DeleteSetIterator<'_> { + type Item = (Client, Range); + + fn next(&mut self) -> Option { + while let Some(client) = self.update.delete_set.keys().next().cloned() { + let deletes = self.update.delete_set.get_mut(&client).unwrap(); + let local_state = self.state.get(&client); + + while let Some(range) = deletes.pop() { + let start = range.start; + let end = range.end; + + if start < local_state { + if local_state < end { + // partially state missing + // [start..end) + // ^ local_state in between + // // split + // [start..local_state) [local_state..end) + // ^^^^^ unapplicable + self + .update + .pending_delete_set + .add(client, local_state, end - local_state); + + return Some((client, start..local_state)); + } + + return Some((client, range)); + } else { + // all state missing + self + .update + .pending_delete_set + .add(client, start, end - start); + } + } + + self.update.delete_set.remove(&client); + } + + None + } +} + +#[cfg(test)] +mod tests { + use std::{num::ParseIntError, path::PathBuf}; + + use serde::Deserialize; + + use super::*; + use crate::doc::common::OrderRange; + + fn struct_item(id: (Client, Clock), len: usize) -> Node { + Node::Item(Somr::new( + ItemBuilder::new() + .id(id.into()) + .content(Content::String("c".repeat(len))) + .build(), + )) + } + + fn parse_doc_update(input: Vec) -> JwstCodecResult { + Update::decode_v1(input) + } + + #[test] + #[cfg_attr(any(miri, loom), ignore)] + fn test_parse_doc() { + let docs = [ + (include_bytes!("../../fixtures/basic.bin").to_vec(), 1, 188), + ( + include_bytes!("../../fixtures/database.bin").to_vec(), + 1, + 149, + ), + (include_bytes!("../../fixtures/large.bin").to_vec(), 1, 9036), + ( + include_bytes!("../../fixtures/with-subdoc.bin").to_vec(), + 2, + 30, + ), + ( + include_bytes!("../../fixtures/edge-case-left-right-same-node.bin").to_vec(), + 2, + 243, + ), + ]; + + for (doc, clients, structs) in docs { + let update = parse_doc_update(doc).unwrap(); + + assert_eq!(update.structs.len(), clients); + assert_eq!( + update.structs.iter().map(|s| s.1.len()).sum::(), + structs + ); + } + } + + fn decode_hex(s: &str) -> Result, ParseIntError> { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16)) + .collect() + } + + #[allow(dead_code)] + #[derive(Deserialize, Debug)] + struct Data { + id: u64, + workspace: String, + timestamp: String, + blob: String, + } + + #[ignore = "just for local data test"] + #[test] + fn test_parse_local_doc() { + let json = + serde_json::from_slice::>(include_bytes!("../../fixtures/local_docs.json")) + .unwrap(); + + for ws in json { + let data = &ws.blob[5..=(ws.blob.len() - 2)]; + if let Ok(data) = decode_hex(data) { + match parse_doc_update(data.clone()) { + Ok(update) => { + println!( + "workspace: {}, global structs: {}, total structs: {}", + ws.workspace, + update.structs.len(), + update.structs.iter().map(|s| s.1.len()).sum::() + ); + } + Err(_e) => { + std::fs::write( + PathBuf::from("./src/fixtures/invalid").join(format!("{}.ydoc", ws.workspace)), + data, + ) + .unwrap(); + println!("doc error: {}", ws.workspace); + } + } + } else { + println!("error origin data: {}", ws.workspace); + } + } + } + + #[test] + fn test_update_iterator() { + loom_model!({ + let mut update = Update { + structs: ClientMap::from_iter([ + ( + 0, + VecDeque::from([ + struct_item((0, 0), 1), + struct_item((0, 1), 1), + Node::new_skip((0, 2).into(), 1), + ]), + ), + ( + 1, + VecDeque::from([ + struct_item((1, 0), 1), + Node::Item(Somr::new( + ItemBuilder::new() + .id((1, 1).into()) + .left_id(Some((0, 1).into())) + .content(Content::String("c".repeat(2))) + .build(), + )), + ]), + ), + ]), + ..Update::default() + }; + + let mut iter = update.iter(StateVector::default()); + assert_eq!(iter.next().unwrap().0.id(), (1, 0).into()); + assert_eq!(iter.next().unwrap().0.id(), (0, 0).into()); + assert_eq!(iter.next().unwrap().0.id(), (0, 1).into()); + assert_eq!(iter.next().unwrap().0.id(), (1, 1).into()); + assert_eq!(iter.next(), None); + }); + } + + #[test] + fn test_update_iterator_with_missing_state() { + loom_model!({ + let mut update = Update { + // an item with higher sequence id than local state + structs: ClientMap::from_iter([(0, VecDeque::from([struct_item((0, 4), 1)]))]), + ..Update::default() + }; + + let mut iter = update.iter(StateVector::from([(0, 3)])); + assert_eq!(iter.next(), None); + assert!(!update.pending_structs.is_empty()); + assert_eq!( + update + .pending_structs + .get_mut(&0) + .unwrap() + .pop_front() + .unwrap() + .id(), + (0, 4).into() + ); + assert!(!update.missing_state.is_empty()); + assert_eq!(update.missing_state.get(&0), 3); + }); + } + + #[test] + fn test_delete_set_iterator() { + let mut update = Update { + delete_set: DeleteSet::from([(0, vec![(0..2), (3..5)])]), + ..Update::default() + }; + + let mut iter = update.delete_set_iter(StateVector::from([(0, 10)])); + assert_eq!(iter.next().unwrap(), (0, 0..2)); + assert_eq!(iter.next().unwrap(), (0, 3..5)); + assert_eq!(iter.next(), None); + } + + #[test] + fn test_delete_set_with_missing_state() { + let mut update = Update { + delete_set: DeleteSet::from([(0, vec![(3..5), (7..12), (13..15)])]), + ..Update::default() + }; + + let mut iter = update.delete_set_iter(StateVector::from([(0, 10)])); + assert_eq!(iter.next().unwrap(), (0, 3..5)); + assert_eq!(iter.next().unwrap(), (0, 7..10)); + assert_eq!(iter.next(), None); + + assert!(!update.pending_delete_set.is_empty()); + assert_eq!( + update.pending_delete_set.get(&0).unwrap(), + &OrderRange::from(vec![(10..12), (13..15)]) + ); + } + + #[test] + fn should_add_skip_when_clock_not_continuous() { + loom_model!({ + let update = Update { + structs: ClientMap::from_iter([( + 0, + VecDeque::from([ + struct_item((0, 0), 1), + struct_item((0, 1), 1), + struct_item((0, 10), 1), + Node::new_gc((0, 20).into(), 10), + ]), + )]), + ..Default::default() + }; + + let merged = Update::merge([update]); + + assert_eq!( + merged.structs.get(&0).unwrap(), + &VecDeque::from([ + struct_item((0, 0), 1), + struct_item((0, 1), 1), + Node::new_skip((0, 2).into(), 8), + struct_item((0, 10), 1), + Node::new_skip((0, 11).into(), 9), + Node::new_gc((0, 20).into(), 10), + ]) + ); + }); + } + + #[test] + fn merged_update_should_not_be_released_in_next_turn() { + loom_model!({ + let update = Update { + structs: ClientMap::from_iter([( + 0, + VecDeque::from([ + struct_item((0, 0), 1), + struct_item((0, 1), 1), + struct_item((0, 10), 1), + Node::new_gc((0, 20).into(), 10), + ]), + )]), + ..Default::default() + }; + + let merged = Update::merge([update]); + + let update2 = Update { + structs: ClientMap::from_iter([( + 0, + VecDeque::from([struct_item((0, 30), 1), Node::new_gc((0, 32).into(), 1)]), + )]), + ..Default::default() + }; + + let merged2 = Update::merge([update2, merged]); + + assert_eq!(merged2.structs.get(&0).unwrap().len(), 9); + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/utils/items.rs b/packages/common/y-octo/core/src/doc/codec/utils/items.rs new file mode 100644 index 0000000000..1dd1f0113f --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/utils/items.rs @@ -0,0 +1,102 @@ +use super::*; + +pub(crate) struct ItemBuilder { + item: Item, +} + +#[allow(dead_code)] +impl ItemBuilder { + pub fn new() -> ItemBuilder { + Self { + item: Item::default(), + } + } + + pub fn id(mut self, id: Id) -> ItemBuilder { + self.item.id = id; + self + } + + pub fn left(mut self, left: Somr) -> ItemBuilder { + if let Some(l) = left.get() { + self.item.origin_left_id = Some(l.last_id()); + self.item.left = left; + } + self + } + + pub fn right(mut self, right: Somr) -> ItemBuilder { + if let Some(r) = right.get() { + self.item.origin_right_id = Some(r.id); + self.item.right = right; + } + self + } + + pub fn left_id(mut self, left_id: Option) -> ItemBuilder { + self.item.origin_left_id = left_id; + self + } + + pub fn right_id(mut self, right_id: Option) -> ItemBuilder { + self.item.origin_right_id = right_id; + self + } + + pub fn parent(mut self, parent: Option) -> ItemBuilder { + self.item.parent = parent; + self + } + + #[allow(dead_code)] + pub fn parent_sub(mut self, parent_sub: Option) -> ItemBuilder { + self.item.parent_sub = parent_sub; + self + } + + pub fn content(mut self, content: Content) -> ItemBuilder { + self.item.content = content; + self + } + + pub fn flags(mut self, flags: ItemFlag) -> ItemBuilder { + self.item.flags = flags; + self + } + + pub fn build(self) -> Item { + if self.item.content.countable() { + self.item.flags.set(item_flags::ITEM_COUNTABLE); + } + + self.item + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_item_builder() { + loom_model!({ + let item = ItemBuilder::new() + .id(Id::new(0, 1)) + .left_id(Some(Id::new(2, 3))) + .right_id(Some(Id::new(4, 5))) + .parent(Some(Parent::String("test".into()))) + .content(Content::Any(vec![Any::String("Hello".into())])) + .build(); + + assert_eq!(item.id, Id::new(0, 1)); + assert_eq!(item.origin_left_id, Some(Id::new(2, 3))); + assert_eq!(item.origin_right_id, Some(Id::new(4, 5))); + assert!(matches!(item.parent, Some(Parent::String(text)) if text == "test")); + assert_eq!(item.parent_sub, None); + assert_eq!( + item.content, + Content::Any(vec![Any::String("Hello".into())]) + ); + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/codec/utils/mod.rs b/packages/common/y-octo/core/src/doc/codec/utils/mod.rs new file mode 100644 index 0000000000..a0bab9e3d6 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/codec/utils/mod.rs @@ -0,0 +1,5 @@ +mod items; + +pub(crate) use items::*; + +use super::*; diff --git a/packages/common/y-octo/core/src/doc/common/mod.rs b/packages/common/y-octo/core/src/doc/common/mod.rs new file mode 100644 index 0000000000..27f9a561d0 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/common/mod.rs @@ -0,0 +1,9 @@ +mod range; +mod somr; +mod state; + +pub use range::*; +pub use somr::*; +pub use state::*; + +use super::*; diff --git a/packages/common/y-octo/core/src/doc/common/range.rs b/packages/common/y-octo/core/src/doc/common/range.rs new file mode 100644 index 0000000000..ec822d0909 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/common/range.rs @@ -0,0 +1,481 @@ +use std::{collections::VecDeque, mem, ops::Range}; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum OrderRange { + Range(Range), + Fragment(VecDeque>), +} + +impl Default for OrderRange { + fn default() -> Self { + Self::Range(0..0) + } +} + +impl From> for OrderRange { + fn from(range: Range) -> Self { + Self::Range(range) + } +} + +impl From>> for OrderRange { + fn from(value: Vec>) -> Self { + Self::Fragment(value.into_iter().collect()) + } +} + +impl From>> for OrderRange { + fn from(value: VecDeque>) -> Self { + Self::Fragment(value) + } +} + +#[inline] +fn is_continuous_range(lhs: &Range, rhs: &Range) -> bool { + lhs.end >= rhs.start && lhs.start <= rhs.end +} + +impl OrderRange { + pub fn ranges_len(&self) -> usize { + match self { + OrderRange::Range(_) => 1, + OrderRange::Fragment(ranges) => ranges.len(), + } + } + + pub fn is_empty(&self) -> bool { + match self { + OrderRange::Range(range) => range.is_empty(), + OrderRange::Fragment(vec) => vec.is_empty(), + } + } + + pub fn contains(&self, clock: u64) -> bool { + match self { + OrderRange::Range(range) => range.contains(&clock), + OrderRange::Fragment(ranges) => ranges.iter().any(|r| r.contains(&clock)), + } + } + + fn check_range_covered(old_vec: &[Range], new_vec: &[Range]) -> bool { + let mut old_iter = old_vec.iter(); + let mut next_old = old_iter.next(); + let mut new_iter = new_vec.iter().peekable(); + let mut next_new = new_iter.next(); + 'new_loop: while let Some(new_range) = next_new { + while let Some(old_range) = next_old { + if old_range.start < new_range.start || old_range.end > new_range.end { + if new_iter.peek().is_some() { + next_new = new_iter.next(); + continue 'new_loop; + } else { + return false; + } + } + next_old = old_iter.next(); + if let Some(next_old) = &next_old { + if next_old.start > new_range.end { + continue; + } + } + } + next_new = new_iter.next(); + } + true + } + + /// diff_range returns the difference between the old range and the new + /// range. current range must be covered by the new range + pub fn diff_range(&self, new_range: &OrderRange) -> Vec> { + let old_vec = self.clone().into_iter().collect::>(); + let new_vec = new_range.clone().into_iter().collect::>(); + + if !Self::check_range_covered(&old_vec, &new_vec) { + return Vec::new(); + } + + let mut diffs = Vec::new(); + let mut old_idx = 0; + + for new_range in &new_vec { + let mut overlap_ranges = Vec::new(); + while old_idx < old_vec.len() && old_vec[old_idx].start <= new_range.end { + overlap_ranges.push(old_vec[old_idx].clone()); + old_idx += 1; + } + + if overlap_ranges.is_empty() { + diffs.push(new_range.clone()); + } else { + let mut last_end = overlap_ranges[0].start; + if last_end > new_range.start { + diffs.push(new_range.start..last_end); + } + + for overlap in &overlap_ranges { + if overlap.start > last_end { + diffs.push(last_end..overlap.start); + } + last_end = overlap.end; + } + + if new_range.end > last_end { + diffs.push(last_end..new_range.end); + } + } + } + + diffs + } + + /// Push new range to current one. + /// Range will be merged if overlap exists or turned into fragment if it's + /// not continuous. + pub fn push(&mut self, range: Range) { + match self { + OrderRange::Range(r) => { + if r.start == r.end { + *self = range.into(); + } else if is_continuous_range(r, &range) { + r.end = r.end.max(range.end); + r.start = r.start.min(range.start); + } else { + *self = OrderRange::Fragment(if r.start < range.start { + VecDeque::from([r.clone(), range]) + } else { + VecDeque::from([range, r.clone()]) + }); + } + } + OrderRange::Fragment(ranges) => { + if ranges.is_empty() { + *self = OrderRange::Range(range); + } else { + OrderRange::push_inner(ranges, range); + self.make_single(); + } + } + } + } + + pub fn pop(&mut self) -> Option> { + if self.is_empty() { + None + } else { + match self { + OrderRange::Range(range) => Some(mem::replace(range, 0..0)), + OrderRange::Fragment(list) => list.pop_front(), + } + } + } + + pub fn merge(&mut self, other: Self) { + self.extend(&other); + } + + fn make_fragment(&mut self) { + if let OrderRange::Range(range) = self { + *self = OrderRange::Fragment(if range.is_empty() { + VecDeque::new() + } else { + VecDeque::from([range.clone()]) + }); + } + } + + fn make_single(&mut self) { + if let OrderRange::Fragment(ranges) = self { + if ranges.len() == 1 { + *self = OrderRange::Range(ranges[0].clone()); + } + } + } + + /// Merge all available ranges list into one. + pub fn squash(&mut self) { + // merge all available ranges + if let OrderRange::Fragment(ranges) = self { + if ranges.is_empty() { + *self = OrderRange::Range(0..0); + return; + } + + let mut changed = false; + let mut merged = VecDeque::with_capacity(ranges.len()); + let mut cur = ranges[0].clone(); + + for next in ranges.iter().skip(1) { + if is_continuous_range(&cur, next) { + cur.start = cur.start.min(next.start); + cur.end = cur.end.max(next.end); + changed = true; + } else { + merged.push_back(cur); + cur = next.clone(); + } + } + merged.push_back(cur); + + if merged.len() == 1 { + *self = OrderRange::Range(merged[0].clone()); + } else if changed { + mem::swap(ranges, &mut merged); + } + } + } + + fn push_inner(list: &mut VecDeque>, range: Range) { + if list.is_empty() { + list.push_back(range); + } else { + let search_result = list.binary_search_by(|r| { + if is_continuous_range(r, &range) { + std::cmp::Ordering::Equal + } else if r.end < range.start { + std::cmp::Ordering::Less + } else { + std::cmp::Ordering::Greater + } + }); + + match search_result { + Ok(idx) => { + let old = &mut list[idx]; + list[idx] = old.start.min(range.start)..old.end.max(range.end); + Self::squash_around(list, idx); + } + Err(idx) => { + list.insert(idx, range); + Self::squash_around(list, idx); + } + } + } + } + + fn squash_around(list: &mut VecDeque>, idx: usize) { + if idx > 0 { + let prev = &list[idx - 1]; + let cur = &list[idx]; + if is_continuous_range(prev, cur) { + list[idx - 1] = prev.start.min(cur.start)..prev.end.max(cur.end); + list.remove(idx); + } + } + + if idx < list.len() - 1 { + let next = &list[idx + 1]; + let cur = &list[idx]; + if is_continuous_range(cur, next) { + list[idx] = cur.start.min(next.start)..cur.end.max(next.end); + list.remove(idx + 1); + } + } + } +} + +impl<'a> IntoIterator for &'a OrderRange { + type Item = Range; + type IntoIter = OrderRangeIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + OrderRangeIter { + range: self, + idx: 0, + } + } +} + +impl Extend> for OrderRange { + fn extend>>(&mut self, other: T) { + self.make_fragment(); + match self { + OrderRange::Fragment(ranges) => { + for range in other { + OrderRange::push_inner(ranges, range); + } + + self.make_single(); + } + _ => unreachable!(), + } + } +} + +pub struct OrderRangeIter<'a> { + range: &'a OrderRange, + idx: usize, +} + +impl Iterator for OrderRangeIter<'_> { + type Item = Range; + + fn next(&mut self) -> Option { + match self.range { + OrderRange::Range(range) => { + if self.idx == 0 { + self.idx += 1; + Some(range.clone()) + } else { + None + } + } + OrderRange::Fragment(ranges) => { + if self.idx < ranges.len() { + let range = ranges[self.idx].clone(); + self.idx += 1; + Some(range) + } else { + None + } + } + } + } +} + +#[cfg(test)] +#[allow(clippy::single_range_in_vec_init)] +mod tests { + use super::OrderRange; + #[test] + fn test_range_push() { + let mut range: OrderRange = (0..10).into(); + + range.push(5..15); + assert_eq!(range, OrderRange::Range(0..15)); + + // turn to fragment + range.push(20..30); + assert_eq!(range, OrderRange::from(vec![(0..15), (20..30)])); + + // auto merge + range.push(15..16); + assert_eq!(range, OrderRange::from(vec![(0..16), (20..30)])); + + // squash + range.push(16..20); + assert_eq!(range, OrderRange::Range(0..30)); + } + + #[test] + fn test_range_pop() { + let mut range: OrderRange = vec![(0..10), (20..30)].into(); + assert_eq!(range.pop(), Some(0..10)); + + let mut range: OrderRange = (0..10).into(); + assert_eq!(range.pop(), Some(0..10)); + assert!(range.is_empty()); + assert_eq!(range.pop(), None); + } + + #[test] + fn test_ranges_squash() { + let mut range = OrderRange::from(vec![(0..10), (20..30)]); + + // do nothing + range.squash(); + assert_eq!(range, OrderRange::from(vec![(0..10), (20..30)])); + + // merged into list + range = OrderRange::from(vec![(0..10), (10..20), (30..40)]); + range.squash(); + assert_eq!(range, OrderRange::from(vec![(0..20), (30..40)])); + + // turn to range + range = OrderRange::from(vec![(0..10), (10..20), (20..30)]); + range.squash(); + assert_eq!(range, OrderRange::Range(0..30)); + } + + #[test] + fn test_range_covered() { + assert!(!OrderRange::check_range_covered(&[0..1], &[2..3])); + assert!(OrderRange::check_range_covered(&[0..1], &[0..3])); + assert!(!OrderRange::check_range_covered(&[0..1], &[1..3])); + assert!(OrderRange::check_range_covered(&[0..1], &[0..3])); + assert!(OrderRange::check_range_covered(&[1..2], &[0..3])); + assert!(OrderRange::check_range_covered(&[1..2, 2..3], &[0..3])); + assert!(!OrderRange::check_range_covered( + &[1..2, 2..3, 3..4], + &[0..3] + )); + assert!(OrderRange::check_range_covered( + &[0..1, 2..3], + &[0..2, 2..4] + )); + assert!(OrderRange::check_range_covered( + &[0..1, 2..3, 3..4], + &[0..2, 2..4] + ),); + } + + #[test] + fn test_range_diff() { + { + let old = OrderRange::Range(0..1); + let new = OrderRange::Range(2..3); + let ranges = old.diff_range(&new); + assert_eq!(ranges, vec![]); + } + + { + let old = OrderRange::Range(0..10); + let new = OrderRange::Range(0..11); + let ranges = old.diff_range(&new); + assert_eq!(ranges, vec![(10..11)]); + } + + { + let old: OrderRange = vec![(0..10), (20..30)].into(); + let new: OrderRange = vec![(0..15), (20..30)].into(); + let ranges = old.diff_range(&new); + assert_eq!(ranges, vec![(10..15)]); + } + + { + let old: OrderRange = vec![(0..3), (5..7), (8..10), (16..18), (21..23)].into(); + let new: OrderRange = vec![(0..12), (15..23)].into(); + let ranges = old.diff_range(&new); + assert_eq!(ranges, vec![(3..5), (7..8), (10..12), (15..16), (18..21)]); + } + + { + let old: OrderRange = vec![(1..6), (8..12)].into(); + let new: OrderRange = vec![(0..12), (15..23), (24..28)].into(); + let ranges = old.diff_range(&new); + assert_eq!(ranges, vec![(0..1), (6..8), (15..23), (24..28)]); + } + } + + #[test] + fn test_range_extend() { + let mut range: OrderRange = (0..10).into(); + range.merge((20..30).into()); + assert_eq!(range, OrderRange::from(vec![(0..10), (20..30)])); + + let mut range: OrderRange = (0..10).into(); + range.merge(vec![(10..15), (20..30)].into()); + assert_eq!(range, OrderRange::from(vec![(0..15), (20..30)])); + + let mut range: OrderRange = vec![(0..10), (20..30)].into(); + range.merge((10..20).into()); + assert_eq!(range, OrderRange::Range(0..30)); + + let mut range: OrderRange = vec![(0..10), (20..30)].into(); + range.merge(vec![(10..20), (30..40)].into()); + assert_eq!(range, OrderRange::Range(0..40)); + } + + #[test] + fn iter() { + let range: OrderRange = vec![(0..10), (20..30)].into(); + + assert_eq!( + range.into_iter().collect::>(), + vec![(0..10), (20..30)] + ); + + let range: OrderRange = OrderRange::Range(0..10); + + assert_eq!(range.into_iter().collect::>(), vec![(0..10)]); + } +} diff --git a/packages/common/y-octo/core/src/doc/common/somr.rs b/packages/common/y-octo/core/src/doc/common/somr.rs new file mode 100644 index 0000000000..7deefcf8b3 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/common/somr.rs @@ -0,0 +1,525 @@ +use std::{ + cell::UnsafeCell, + fmt::{self, Write}, + hash::{Hash, Hasher}, + marker::PhantomData, + mem, + ops::{Deref, DerefMut}, + ptr::NonNull, +}; + +use crate::sync::Ordering; +const DANGLING_PTR: usize = usize::MAX; +#[inline] +fn is_dangling(ptr: NonNull) -> bool { + ptr.as_ptr() as usize == DANGLING_PTR +} + +/// Heap data with single owner but multiple refs with dangling checking at +/// runtime. +pub(crate) enum Somr { + Owned(Owned), + Ref(Ref), +} + +#[repr(transparent)] +pub(crate) struct Owned(NonNull>); +#[repr(transparent)] +pub(crate) struct Ref(NonNull>); + +#[cfg(feature = "large_refs")] +type RefAtomicType = crate::sync::AtomicU32; +#[cfg(feature = "large_refs")] +type RefPrimitiveType = u32; + +#[cfg(not(feature = "large_refs"))] +type RefAtomicType = crate::sync::AtomicU16; +#[cfg(not(feature = "large_refs"))] +type RefPrimitiveType = u16; + +pub(crate) struct SomrInner { + data: Option>, + /// increase the size when we really meet the the scenario with refs more + /// then u16::MAX(65535) times + refs: RefAtomicType, + _marker: PhantomData>, +} + +pub(crate) struct InnerRefMut<'a, T> { + inner: NonNull, + _marker: PhantomData<&'a mut T>, +} + +impl Deref for InnerRefMut<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.inner.as_ptr() } + } +} + +impl DerefMut for InnerRefMut<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.inner.as_ptr() } + } +} + +unsafe impl Send for Somr {} +unsafe impl Sync for Somr {} + +impl Default for Somr { + fn default() -> Self { + Self::none() + } +} + +impl Somr { + pub fn new(data: T) -> Self { + let inner = Box::new(SomrInner { + data: Some(UnsafeCell::new(data)), + refs: RefAtomicType::new(1), + _marker: PhantomData, + }); + + Self::Owned(Owned(Box::leak(inner).into())) + } + + pub fn none() -> Self { + Self::Ref(Ref(NonNull::new(DANGLING_PTR as *mut _).unwrap())) + } +} + +impl SomrInner { + fn data_ref(&self) -> Option<&T> { + self.data.as_ref().map(|x| unsafe { &*x.get() }) + } + + fn data_mut(&self) -> Option> { + self.data.as_ref().map(|x| InnerRefMut { + inner: unsafe { NonNull::new_unchecked(x.get()) }, + _marker: PhantomData, + }) + } +} + +impl Somr { + #[inline] + pub fn is_none(&self) -> bool { + self.dangling() || self.inner().data_ref().is_none() + } + + #[inline] + pub fn is_some(&self) -> bool { + !self.dangling() && self.inner().data_ref().is_some() + } + + pub fn get(&self) -> Option<&T> { + if self.dangling() { + return None; + } + + self.inner().data_ref() + } + + pub unsafe fn get_unchecked(&self) -> &T { + if self.dangling() { + panic!("Try to visit Somr data that has already been dropped.") + } + + match &self.inner().data_ref() { + Some(data) => data, + None => { + panic!("Try to unwrap on None") + } + } + } + + #[allow(unused)] + pub fn get_mut(&mut self) -> Option<&mut T> { + if !self.is_owned() || self.dangling() { + return None; + } + + let inner = self.inner_mut(); + inner.data.as_mut().map(|x| x.get_mut()) + } + + #[allow(unused)] + pub unsafe fn get_mut_from_ref(&self) -> Option> { + if !self.is_owned() || self.dangling() { + return None; + } + + let inner = self.inner_mut(); + inner.data_mut() + } + + pub unsafe fn get_mut_unchecked(&self) -> InnerRefMut<'_, T> { + if self.dangling() { + panic!("Try to visit Somr data that has already been dropped.") + } + + match self.inner_mut().data_mut() { + Some(data) => data, + None => { + panic!("Try to unwrap on None") + } + } + } + + #[inline] + pub fn is_owned(&self) -> bool { + matches!(self, Self::Owned(_)) + } + + pub fn swap_take(&mut self) -> Self { + debug_assert!(self.is_owned()); + + let mut r = self.clone(); + + mem::swap(self, &mut r); + + r + } + + #[inline] + fn inner(&self) -> &SomrInner { + debug_assert!(!self.dangling()); + unsafe { self.ptr().as_ref() } + } + + #[inline] + #[allow(clippy::mut_from_ref)] + fn inner_mut(&self) -> &mut SomrInner { + debug_assert!(!self.dangling()); + unsafe { self.ptr().as_mut() } + } + + #[inline] + pub fn ptr(&self) -> NonNull> { + match self { + Somr::Owned(ptr) => ptr.0, + Somr::Ref(ptr) => ptr.0, + } + } + + #[inline] + pub fn ptr_eq(&self, other: &Self) -> bool { + self.ptr().as_ptr() as usize == other.ptr().as_ptr() as usize + } + + #[inline] + fn dangling(&self) -> bool { + is_dangling(self.ptr()) + } +} + +impl Clone for Somr { + fn clone(&self) -> Self { + if self.dangling() { + return Self::none(); + } + + let inner = unsafe { &*self.ptr().as_ptr() }; + + let old_size = inner.refs.fetch_add(1, Ordering::Relaxed); + + if old_size == RefPrimitiveType::MAX { + panic!("Too many refs on Somr, maybe we need to increase the limitation now.") + } + + Self::Ref(Ref(self.ptr())) + } +} + +impl Drop for Owned { + fn drop(&mut self) { + let inner = unsafe { &mut *self.0.as_ptr() }; + + // ensure all reads are finished + // See [Arc::Drop] + inner.refs.load(Ordering::Acquire); + + inner.data.take(); + drop(Ref(self.0)); + } +} + +impl Drop for Ref { + fn drop(&mut self) { + if is_dangling(self.0) { + return; + } + + let rc = unsafe { &(*self.0.as_ptr()).refs }; + + // no other refs + if rc.fetch_sub(1, Ordering::Release) == 1 { + // ensure all reads are finished + // See [Arc::Drop] + rc.load(Ordering::Acquire); + + drop(unsafe { Box::from_raw(self.0.as_ptr()) }); + } + } +} + +impl From for Somr { + fn from(value: T) -> Self { + Somr::new(value) + } +} + +impl From>> for Somr { + fn from(value: Option>) -> Self { + match value { + Some(somr) => somr, + None => Somr::none(), + } + } +} + +pub trait FlattenGet { + #[allow(dead_code)] + fn flatten_get(&self) -> Option<&T>; +} + +impl FlattenGet for Option> { + fn flatten_get(&self) -> Option<&T> { + self.as_ref().and_then(|data| data.get()) + } +} + +impl PartialEq for Somr { + fn eq(&self, other: &Self) -> bool { + self.ptr() == other.ptr() + || !self.dangling() && !other.dangling() && self.inner() == other.inner() + } +} + +impl PartialEq for SomrInner { + fn eq(&self, other: &Self) -> bool { + self.data_ref() == other.data_ref() + } +} + +impl Eq for Somr { + fn assert_receiver_is_total_eq(&self) {} +} + +impl PartialOrd for Somr { + fn partial_cmp(&self, other: &Self) -> Option { + match (self.get(), other.get()) { + (Some(a), Some(b)) => a.partial_cmp(b), + _ => None, + } + } +} + +impl Hash for Somr { + fn hash(&self, state: &mut H) { + self.ptr().hash(state) + } +} + +impl fmt::Debug for Somr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_owned() { + f.write_str("Owned(")?; + } else { + f.write_str("Ref(")?; + } + + if let Some(value) = self.get() { + fmt::Debug::fmt(value, f)?; + } else { + f.write_str("None")?; + } + + f.write_char(')') + } +} + +impl fmt::Display for Somr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_owned() { + f.write_str("Owned(")?; + } else { + f.write_str("Ref(")?; + } + + if let Some(value) = self.get() { + fmt::Display::fmt(value, f)?; + } else { + f.write_str("None")?; + } + + f.write_char(')') + } +} + +impl fmt::Pointer for Somr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Pointer::fmt(&(self.get().unwrap() as *const T), f) + } +} + +#[cfg(all(test, not(loom)))] +impl proptest::arbitrary::Arbitrary for Somr { + type Parameters = T::Parameters; + type Strategy = proptest::strategy::MapInto; + + fn arbitrary_with(args: Self::Parameters) -> Self::Strategy { + proptest::strategy::Strategy::prop_map_into(proptest::arbitrary::any_with::(args)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::loom_model; + + #[test] + fn basic_example() { + loom_model!({ + let five = Somr::new(5); + assert_eq!(five.get(), Some(&5)); + + let five_ref = five.clone(); + assert!(!five_ref.is_owned()); + assert_eq!(five_ref.get(), Some(&5)); + assert_eq!( + five_ref.ptr().as_ptr() as usize, + five.ptr().as_ptr() as usize + ); + + drop(five); + // owner released + assert_eq!(five_ref.get(), None); + }); + } + + #[test] + fn complex_struct() { + loom_model!({ + struct T { + a: usize, + b: String, + } + + let t1 = Somr::new(T { + a: 1, + b: "hello".to_owned(), + }); + + assert_eq!(t1.get().unwrap().a, 1); + assert_eq!(t1.get().unwrap().b.as_str(), "hello"); + + let t2 = t1.clone(); + assert!(!t2.is_owned()); + assert_eq!(t2.ptr().as_ptr() as usize, t1.ptr().as_ptr() as usize); + assert_eq!(t2.get().unwrap().a, 1); + assert_eq!(t2.get().unwrap().b.as_str(), "hello"); + + drop(t1); + + assert!(t2.get().is_none()); + }); + } + + #[test] + fn acquire_mut_ref() { + loom_model!({ + let mut five = Somr::new(5); + + *five.get_mut().unwrap() += 1; + assert_eq!(five.get(), Some(&6)); + + let five_ref = five.clone(); + + // only owner can mut ref + assert!(five_ref.get().is_some()); + assert!(unsafe { five_ref.get_mut_from_ref() }.is_none()); + + drop(five); + }); + } + + #[test] + fn comparison() { + loom_model!({ + let five = Somr::new(5); + let five_ref = five.clone(); + let another_five = Somr::new(5); + let six = Somr::new(6); + + assert_eq!(five, five_ref); + assert_eq!(five, another_five); + assert_eq!(five.ptr().as_ptr(), five_ref.ptr().as_ptr()); + assert_ne!(five.ptr().as_ptr(), another_five.ptr().as_ptr()); + + assert!(six > five); + assert!(six > five_ref); + + assert_eq!(five_ref.partial_cmp(&six), Some(std::cmp::Ordering::Less)); + drop(five); + assert_eq!(five_ref.partial_cmp(&six), None); + }); + } + + #[test] + fn represent_none() { + loom_model!({ + let none = Somr::::none(); + + assert!(!none.is_owned()); + assert!(none.is_none()); + assert!(none.get().is_none()); + }); + } + + #[test] + fn drop_ref_without_affecting_owner() { + loom_model!({ + let five = Somr::new(5); + let five_ref = five.clone(); + + assert_eq!(five.get().unwrap(), &5); + assert_eq!(five_ref.get().unwrap(), &5); + + drop(five_ref); + + assert_eq!(five.get().unwrap(), &5); + }); + } + + #[test] + fn swap_take() { + loom_model!({ + let mut five = Somr::new(5); + let owned = five.swap_take(); + + assert_eq!(owned.get().unwrap(), &5); + assert_eq!(five.get().unwrap(), &5); + + assert!(owned.is_owned()); + assert!(!five.is_owned()); + }); + } + + // This is UB if we didn't use `UnsafeCell` in `Somr` + #[test] + fn test_inner_mut() { + loom_model!({ + let five = Somr::new(5); + fn add(a: &Somr, b: &Somr) { + unsafe { a.get_mut_from_ref() } + .map(|mut x| *x += *b.get().unwrap()) + .unwrap(); + } + + add(&five, &five); + assert_eq!(five.get().copied().unwrap(), 10); + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/common/state.rs b/packages/common/y-octo/core/src/doc/common/state.rs new file mode 100644 index 0000000000..b7e7b54b05 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/common/state.rs @@ -0,0 +1,140 @@ +use std::ops::{Deref, DerefMut}; + +use super::{ + Client, ClientMap, Clock, CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, HashMapExt, Id, + JwstCodecResult, HASHMAP_SAFE_CAPACITY, +}; + +#[derive(Default, Debug, PartialEq, Clone)] +pub struct StateVector(ClientMap); + +impl StateVector { + pub fn set_max(&mut self, client: Client, clock: Clock) { + self + .entry(client) + .and_modify(|m_clock| { + if *m_clock < clock { + *m_clock = clock; + } + }) + .or_insert(clock); + } + + pub fn get(&self, client: &Client) -> Clock { + *self.0.get(client).unwrap_or(&0) + } + + pub fn contains(&self, id: &Id) -> bool { + id.clock <= self.get(&id.client) + } + + pub fn set_min(&mut self, client: Client, clock: Clock) { + self + .entry(client) + .and_modify(|m_clock| { + if *m_clock > clock { + *m_clock = clock; + } + }) + .or_insert(clock); + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + pub fn merge_with(&mut self, other: &Self) { + for (client, clock) in other.iter() { + self.set_min(*client, *clock); + } + } +} + +impl Deref for StateVector { + type Target = ClientMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for StateVector { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From<[(Client, Clock); N]> for StateVector { + fn from(value: [(Client, Clock); N]) -> Self { + let mut map = ClientMap::with_capacity(N); + + for (client, clock) in value { + map.insert(client, clock); + } + + Self(map) + } +} + +impl CrdtRead for StateVector { + fn read(decoder: &mut R) -> JwstCodecResult { + let len = decoder.read_var_u64()? as usize; + + // See: [HASHMAP_SAFE_CAPACITY] + let mut map = ClientMap::with_capacity(len.min(HASHMAP_SAFE_CAPACITY)); + for _ in 0..len { + let client = decoder.read_var_u64()?; + let clock = decoder.read_var_u64()?; + map.insert(client, clock); + } + + map.shrink_to_fit(); + Ok(Self(map)) + } +} + +impl CrdtWrite for StateVector { + fn write(&self, encoder: &mut W) -> JwstCodecResult { + encoder.write_var_u64(self.len() as u64)?; + + for (client, clock) in self.iter() { + encoder.write_var_u64(*client)?; + encoder.write_var_u64(*clock)?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_state_vector_basic() { + let mut state_vector = StateVector::from([(1, 1), (2, 2), (3, 3)]); + assert_eq!(state_vector.len(), 3); + assert_eq!(state_vector.get(&1), 1); + + state_vector.set_min(1, 0); + assert_eq!(state_vector.get(&1), 0); + + state_vector.set_max(1, 4); + assert_eq!(state_vector.get(&1), 4); + + // set inexistent client + state_vector.set_max(4, 1); + assert_eq!(state_vector.get(&4), 1); + + // same client with larger clock + assert!(!state_vector.contains(&(1, 5).into())); + } + + #[test] + fn test_state_vector_merge() { + let mut state_vector = StateVector::from([(1, 1), (2, 2), (3, 3)]); + let other_state_vector = StateVector::from([(1, 5), (2, 6), (3, 7)]); + state_vector.merge_with(&other_state_vector); + assert_eq!(state_vector, StateVector::from([(3, 3), (1, 1), (2, 2)])); + } +} diff --git a/packages/common/y-octo/core/src/doc/document.rs b/packages/common/y-octo/core/src/doc/document.rs new file mode 100644 index 0000000000..6988988abe --- /dev/null +++ b/packages/common/y-octo/core/src/doc/document.rs @@ -0,0 +1,656 @@ +use super::{history::StoreHistory, publisher::DocPublisher, store::StoreRef, *}; +use crate::sync::{Arc, RwLock}; + +#[cfg(feature = "debug")] +#[derive(Debug, Clone)] +pub struct DocStoreStatus { + pub nodes: usize, + pub delete_sets: usize, + pub types: usize, + pub dangling_types: usize, + pub pending_nodes: usize, +} + +/// [DocOptions] used to create a new [Doc] +/// +/// ``` +/// use y_octo::DocOptions; +/// +/// let doc = DocOptions::new() +/// .with_client_id(1) +/// .with_guid("guid".into()) +/// .auto_gc(true) +/// .build(); +/// +/// assert_eq!(doc.guid(), "guid") +/// ``` +#[derive(Clone, Debug)] +pub struct DocOptions { + pub guid: String, + pub client_id: u64, + pub gc: bool, +} + +impl Default for DocOptions { + fn default() -> Self { + if cfg!(any(test, feature = "bench")) { + Self { + client_id: 1, + guid: "test".into(), + gc: true, + } + } else { + /// It tends to generate small numbers. + /// Since the client id will be included in all crdt items, the + /// small client helps to reduce the binary size. + /// + /// NOTE: The probability of 36% of the random number generated by + /// this function is greater than [u32::MAX] + fn prefer_small_random() -> u64 { + use rand::{distr::Distribution, rng}; + use rand_distr::Exp; + + let scale_factor = u16::MAX as f64; + let v: f64 = Exp::new(1.0 / scale_factor) + .map(|exp| exp.sample(&mut rng())) + .unwrap_or_else(|_| rand::random()); + + (v * scale_factor) as u64 + } + + Self { + client_id: prefer_small_random(), + guid: nanoid::nanoid!(), + gc: true, + } + } + } +} + +impl DocOptions { + pub fn new() -> Self { + Self::default() + } + + pub fn with_client_id(mut self, client_id: u64) -> Self { + self.client_id = client_id; + self + } + + pub fn with_guid(mut self, guid: String) -> Self { + self.guid = guid; + self + } + + pub fn auto_gc(mut self, gc: bool) -> Self { + self.gc = gc; + self + } + + pub fn build(self) -> Doc { + Doc::with_options(self) + } +} + +impl From for Any { + fn from(value: DocOptions) -> Self { + Any::Object(HashMap::from_iter([ + ("gc".into(), value.gc.into()), + ("guid".into(), value.guid.into()), + ])) + } +} + +impl TryFrom for DocOptions { + type Error = JwstCodecError; + + fn try_from(value: Any) -> Result { + match value { + Any::Object(map) => { + let mut options = DocOptions::default(); + for (key, value) in map { + match key.as_str() { + "gc" => { + options.gc = bool::try_from(value)?; + } + "guid" => { + options.guid = String::try_from(value)?; + } + _ => {} + } + } + + Ok(options) + } + _ => Err(JwstCodecError::UnexpectedType("Object")), + } + } +} + +#[derive(Debug, Clone)] +pub struct Doc { + client_id: u64, + opts: DocOptions, + + pub(crate) store: StoreRef, + pub publisher: Arc, +} + +unsafe impl Send for Doc {} +unsafe impl Sync for Doc {} + +impl Default for Doc { + fn default() -> Self { + Doc::new() + } +} + +impl PartialEq for Doc { + fn eq(&self, other: &Self) -> bool { + self.client_id == other.client_id + } +} + +impl Doc { + pub fn new() -> Self { + Self::with_options(DocOptions::default()) + } + + pub fn with_options(options: DocOptions) -> Self { + let store = Arc::new(RwLock::new(DocStore::with_client(options.client_id))); + let publisher = Arc::new(DocPublisher::new(store.clone())); + + Self { + client_id: options.client_id, + opts: options, + store, + publisher, + } + } + + pub fn with_client(client_id: u64) -> Self { + DocOptions::new().with_client_id(client_id).build() + } + + pub fn client(&self) -> Client { + self.client_id + } + + pub fn clients(&self) -> Vec { + self.store.read().unwrap().clients() + } + + pub fn history(&self) -> StoreHistory { + let history = StoreHistory::new(&self.store); + history.resolve(); + history + } + + #[cfg(feature = "debug")] + pub fn store_status(&self) -> DocStoreStatus { + let store = self.store.read().unwrap(); + + DocStoreStatus { + nodes: store.total_nodes(), + delete_sets: store.total_delete_sets(), + types: store.total_types(), + dangling_types: store.total_dangling_types(), + pending_nodes: store.total_pending_nodes(), + } + } + + pub fn options(&self) -> &DocOptions { + &self.opts + } + + pub fn guid(&self) -> &str { + self.opts.guid.as_str() + } + + // TODO: + // provide a better way instead of `_v1` methods + // when implementing `v2` binary format + pub fn try_from_binary_v1>(binary: T) -> JwstCodecResult { + Self::try_from_binary_v1_with_options(binary, DocOptions::default()) + } + + pub fn try_from_binary_v1_with_options>( + binary: T, + options: DocOptions, + ) -> JwstCodecResult { + let mut doc = Doc::with_options(options); + doc.apply_update_from_binary_v1(binary)?; + Ok(doc) + } + + pub fn apply_update_from_binary_v1>(&mut self, binary: T) -> JwstCodecResult { + let mut decoder = RawDecoder::new(binary.as_ref()); + let update = Update::read(&mut decoder)?; + self.apply_update(update) + } + + pub fn apply_update(&mut self, mut update: Update) -> JwstCodecResult { + let mut store = self.store.write().unwrap(); + let mut retry = false; + loop { + for (mut s, offset) in update.iter(store.get_state_vector()) { + if let Node::Item(item) = &mut s { + debug_assert!(item.is_owned()); + let mut item = unsafe { item.get_mut_unchecked() }; + store.repair(&mut item, self.store.clone())?; + } + store.integrate(s, offset, None)?; + } + + for (client, range) in update.delete_set_iter(store.get_state_vector()) { + store.delete_range(client, range)?; + } + + if let Some(mut pending_update) = store.pending.take() { + if pending_update + .missing_state + .iter() + .any(|(client, clock)| *clock < store.get_state(*client)) + { + // new update has been applied to the doc, need to re-integrate + retry = true; + } + + for (client, range) in pending_update.delete_set_iter(store.get_state_vector()) { + store.delete_range(client, range)?; + } + + if update.is_pending_empty() { + update = pending_update; + } else { + // drain all pending state to pending update for later iteration + update.drain_pending_state(); + Update::merge_into(&mut update, [pending_update]); + } + } else { + // no pending update at store + + // no pending update in current iteration + // thank god, all clean + if update.is_pending_empty() { + break; + } else { + // need to turn all pending state into update for later iteration + update.drain_pending_state(); + retry = false; + }; + } + + // can't integrate any more, save the pending update + if !retry { + if !update.is_empty() { + store.pending.replace(update); + } + break; + } + } + + Ok(()) + } + + pub fn keys(&self) -> Vec { + let store = self.store.read().unwrap(); + store.types.keys().cloned().collect() + } + + pub fn get_or_create_text>(&self, name: S) -> JwstCodecResult { + YTypeBuilder::new(self.store.clone()) + .with_kind(YTypeKind::Text) + .set_name(name.as_ref().to_string()) + .build() + } + + pub fn create_text(&self) -> JwstCodecResult { + YTypeBuilder::new(self.store.clone()) + .with_kind(YTypeKind::Text) + .build() + } + + pub fn get_or_create_array>(&self, str: S) -> JwstCodecResult { + YTypeBuilder::new(self.store.clone()) + .with_kind(YTypeKind::Array) + .set_name(str.as_ref().to_string()) + .build() + } + + pub fn create_array(&self) -> JwstCodecResult { + YTypeBuilder::new(self.store.clone()) + .with_kind(YTypeKind::Array) + .build() + } + + pub fn get_or_create_map>(&self, str: S) -> JwstCodecResult { + YTypeBuilder::new(self.store.clone()) + .with_kind(YTypeKind::Map) + .set_name(str.as_ref().to_string()) + .build() + } + + pub fn create_map(&self) -> JwstCodecResult { + YTypeBuilder::new(self.store.clone()) + .with_kind(YTypeKind::Map) + .build() + } + + pub fn get_map(&self, str: &str) -> JwstCodecResult { + YTypeBuilder::new(self.store.clone()) + .with_kind(YTypeKind::Map) + .set_name(str.to_string()) + .build_exists() + } + + pub fn encode_update_v1(&self) -> JwstCodecResult> { + self.encode_state_as_update_v1(&StateVector::default()) + } + + pub fn encode_state_as_update_v1(&self, sv: &StateVector) -> JwstCodecResult> { + let update = self.encode_state_as_update(sv)?; + + let mut encoder = RawEncoder::default(); + update.write(&mut encoder)?; + Ok(encoder.into_inner()) + } + + pub fn encode_update(&self) -> JwstCodecResult { + self.encode_state_as_update(&StateVector::default()) + } + + pub fn encode_state_as_update(&self, sv: &StateVector) -> JwstCodecResult { + self.store.read().unwrap().diff_state_vector(sv, true) + } + + pub fn get_state_vector(&self) -> StateVector { + self.store.read().unwrap().get_state_vector() + } + + pub fn subscribe(&self, cb: impl Fn(&[u8], &[History]) + Sync + Send + 'static) { + self.publisher.subscribe(cb); + } + + pub fn unsubscribe_all(&self) { + self.publisher.unsubscribe_all(); + } + + pub fn subscribe_count(&self) -> usize { + self.publisher.count() + } + + pub fn gc(&self) -> JwstCodecResult<()> { + self.store.write().unwrap().optimize() + } +} + +#[cfg(test)] +mod tests { + use yrs::{types::ToJson, updates::decoder::Decode, Array, Map, Options, Transact}; + + use super::*; + use crate::sync::{AtomicU8, Ordering}; + + #[test] + fn test_encode_state_as_update() { + let yrs_options_left = Options::default(); + let yrs_options_right = Options::default(); + + loom_model!({ + let (binary, binary_new) = if cfg!(miri) { + let doc = Doc::new(); + + let mut map = doc.get_or_create_map("abc").unwrap(); + map.insert("a".to_string(), 1).unwrap(); + let binary = doc.encode_update_v1().unwrap(); + + let doc_new = Doc::new(); + let mut array = doc_new.get_or_create_array("array").unwrap(); + array.insert(0, "array_value").unwrap(); + let binary_new = doc.encode_update_v1().unwrap(); + + (binary, binary_new) + } else { + let yrs_doc = yrs::Doc::with_options(yrs_options_left.clone()); + + let map = yrs_doc.get_or_insert_map("abc"); + let mut trx = yrs_doc.transact_mut(); + map.insert(&mut trx, "a", 1); + let binary = trx.encode_update_v1(); + + let yrs_doc_new = yrs::Doc::with_options(yrs_options_right.clone()); + let array = yrs_doc_new.get_or_insert_array("array"); + let mut trx = yrs_doc_new.transact_mut(); + array.insert(&mut trx, 0, "array_value"); + let binary_new = trx.encode_update_v1(); + + (binary, binary_new) + }; + + let mut doc = Doc::try_from_binary_v1(binary).unwrap(); + let mut doc_new = Doc::try_from_binary_v1(binary_new).unwrap(); + + let diff_update = doc_new + .encode_state_as_update_v1(&doc.get_state_vector()) + .unwrap(); + + let diff_update_reverse = doc + .encode_state_as_update_v1(&doc_new.get_state_vector()) + .unwrap(); + + doc.apply_update_from_binary_v1(diff_update).unwrap(); + doc_new + .apply_update_from_binary_v1(diff_update_reverse) + .unwrap(); + + assert_eq!( + doc.encode_update_v1().unwrap(), + doc_new.encode_update_v1().unwrap() + ); + }); + } + + #[test] + #[cfg_attr(any(miri, loom), ignore)] + fn test_array_create() { + let yrs_options = yrs::Options::default(); + + let json = serde_json::json!([42.0, -42.0, true, false, "hello", "world", [1.0]]); + + { + let doc = yrs::Doc::with_options(yrs_options.clone()); + let array = doc.get_or_insert_array("abc"); + let mut trx = doc.transact_mut(); + array.insert(&mut trx, 0, 42); + array.insert(&mut trx, 1, -42); + array.insert(&mut trx, 2, true); + array.insert(&mut trx, 3, false); + array.insert(&mut trx, 4, "hello"); + array.insert(&mut trx, 5, "world"); + + let sub_array = yrs::ArrayPrelim::default(); + let sub_array = array.insert(&mut trx, 6, sub_array); + sub_array.insert(&mut trx, 0, 1); + + drop(trx); + let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict) + .numeric_mode(assert_json_diff::NumericMode::AssumeFloat); + assert_json_diff::assert_json_matches!(array.to_json(&doc.transact()), json, config); + }; + + let binary = { + let doc = Doc::new(); + let mut array = doc.get_or_create_array("abc").unwrap(); + array.insert(0, 42).unwrap(); + array.insert(1, -42).unwrap(); + array.insert(2, true).unwrap(); + array.insert(3, false).unwrap(); + array.insert(4, "hello").unwrap(); + array.insert(5, "world").unwrap(); + + let mut sub_array = doc.create_array().unwrap(); + array.insert(6, sub_array.clone()).unwrap(); + // FIXME: array need insert first to compatible with yrs + sub_array.insert(0, 1).unwrap(); + + doc.encode_update_v1().unwrap() + }; + + let ydoc = yrs::Doc::with_options(yrs_options); + let array = ydoc.get_or_insert_array("abc"); + let mut trx = ydoc.transact_mut(); + trx + .apply_update(yrs::Update::decode_v1(&binary).unwrap()) + .unwrap(); + + let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict) + .numeric_mode(assert_json_diff::NumericMode::AssumeFloat); + assert_json_diff::assert_json_matches!(array.to_json(&trx), json, config); + + let mut doc = Doc::new(); + let array = doc.get_or_create_array("abc").unwrap(); + doc.apply_update_from_binary_v1(binary).unwrap(); + + let list = array.iter().collect::>(); + + assert!(list.len() == 7); + assert!(matches!(list[6], Value::Array(_))); + } + + #[test] + #[ignore = "inaccurate timing on ci, need for more accurate timing testing"] + fn test_subscribe() { + loom_model!({ + let doc = Doc::default(); + let doc_clone = doc.clone(); + + let count = Arc::new(AtomicU8::new(0)); + let count_clone1 = count.clone(); + let count_clone2 = count.clone(); + doc.subscribe(move |_, _| { + count_clone1.fetch_add(1, Ordering::SeqCst); + }); + + doc_clone.subscribe(move |_, _| { + count_clone2.fetch_add(1, Ordering::SeqCst); + }); + + doc_clone + .get_or_create_array("abc") + .unwrap() + .insert(0, 42) + .unwrap(); + + // wait observer, cycle once every 100mm + std::thread::sleep(std::time::Duration::from_millis(200)); + + assert_eq!(count.load(Ordering::SeqCst), 2); + }); + } + + #[test] + fn test_repeated_applied_pending_update() { + // generate a pending update + // update: [1, 1, 1, 0, 39, 1, 4, 116, 101, 115, 116, 3, 109, 97, 112, 1, 0] + // update: [1, 1, 1, 1, 40, 0, 1, 0, 11, 115, 117, 98, 95, 109, 97, 112, 95, + // 107, 101, 121, 1, 119, 13, 115, 117, 98, 95, 109, 97, 112, 95, 118, 97, 108, + // 117, 101, 0] + // { + // let doc1 = Doc::default(); + + // doc1.subscribe(|update| { + // println!("update: {:?}", update); + // }); + + // let mut map = doc1.get_or_create_map("test").unwrap(); + // std::thread::sleep(std::time::Duration::from_millis(500)); + + // let mut sub_map = doc1.create_map().unwrap(); + // map.insert("map", sub_map.clone()).unwrap(); + // std::thread::sleep(std::time::Duration::from_millis(500)); + + // sub_map.insert("sub_map_key", "sub_map_value").unwrap(); + // std::thread::sleep(std::time::Duration::from_millis(500)); + // } + + loom_model!({ + let mut doc = Doc::default(); + + doc + .apply_update_from_binary_v1(vec![ + 1, 1, 1, 1, 40, 0, 1, 0, 11, 115, 117, 98, 95, 109, 97, 112, 95, 107, 101, 121, 1, 119, + 13, 115, 117, 98, 95, 109, 97, 112, 95, 118, 97, 108, 117, 101, 0, + ]) + .unwrap(); + + let pending_size = doc + .store + .read() + .unwrap() + .pending + .as_ref() + .unwrap() + .structs + .iter() + .map(|s| s.1.len()) + .sum::(); + doc + .apply_update_from_binary_v1(vec![ + 1, 1, 1, 1, 40, 0, 1, 0, 11, 115, 117, 98, 95, 109, 97, 112, 95, 107, 101, 121, 1, 119, + 13, 115, 117, 98, 95, 109, 97, 112, 95, 118, 97, 108, 117, 101, 0, + ]) + .unwrap(); + + // pending nodes should not grow up after apply same pending update + assert_eq!( + pending_size, + doc + .store + .read() + .unwrap() + .pending + .as_ref() + .unwrap() + .structs + .iter() + .map(|s| s.1.len()) + .sum::() + ); + }); + } + + #[test] + fn test_update_from_vec_ref() { + loom_model!({ + let doc = Doc::new(); + + let mut text = doc.get_or_create_text("text").unwrap(); + text.insert(0, "hello world").unwrap(); + + let update = doc.encode_update_v1().unwrap(); + + let doc = Doc::try_from_binary_v1(update).unwrap(); + let text = doc.get_or_create_text("text").unwrap(); + + assert_eq!(&text.to_string(), "hello world"); + }); + } + + #[test] + #[cfg_attr(any(miri, loom), ignore)] + fn test_apply_update() { + let updates = [ + include_bytes!("../fixtures/basic.bin").to_vec(), + include_bytes!("../fixtures/database.bin").to_vec(), + include_bytes!("../fixtures/large.bin").to_vec(), + include_bytes!("../fixtures/with-subdoc.bin").to_vec(), + include_bytes!("../fixtures/edge-case-left-right-same-node.bin").to_vec(), + ]; + + for update in updates { + let mut doc = Doc::new(); + doc.apply_update_from_binary_v1(&update).unwrap(); + } + } +} diff --git a/packages/common/y-octo/core/src/doc/hasher.rs b/packages/common/y-octo/core/src/doc/hasher.rs new file mode 100644 index 0000000000..3a932f191c --- /dev/null +++ b/packages/common/y-octo/core/src/doc/hasher.rs @@ -0,0 +1,35 @@ +use std::{ + collections::HashMap, + hash::{BuildHasher, Hasher}, +}; + +use super::Client; + +#[derive(Default)] +pub struct ClientHasher(Client); + +impl Hasher for ClientHasher { + fn finish(&self) -> u64 { + self.0 + } + + fn write(&mut self, _: &[u8]) {} + + fn write_u64(&mut self, i: u64) { + self.0 = i + } +} + +#[derive(Default, Clone)] +pub struct ClientHasherBuilder; + +impl BuildHasher for ClientHasherBuilder { + type Hasher = ClientHasher; + + fn build_hasher(&self) -> Self::Hasher { + ClientHasher::default() + } +} + +// use ClientID as key +pub type ClientMap = HashMap; diff --git a/packages/common/y-octo/core/src/doc/history.rs b/packages/common/y-octo/core/src/doc/history.rs new file mode 100644 index 0000000000..f0b947fbfd --- /dev/null +++ b/packages/common/y-octo/core/src/doc/history.rs @@ -0,0 +1,327 @@ +use std::{collections::VecDeque, sync::Arc}; + +use serde::{Deserialize, Serialize}; + +use super::{store::StoreRef, *}; +use crate::sync::RwLock; + +enum ParentNode { + Root(String), + Node(Somr), + Unknown, +} + +#[derive(Clone, Default)] +pub struct HistoryOptions { + pub client: Option, + /// Only available when client is set + pub skip: Option, + /// Only available when client is set + pub limit: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct StoreHistory { + store: StoreRef, + parents: Arc>>>, +} + +impl StoreHistory { + pub(crate) fn new(store: &StoreRef) -> Self { + Self { + store: store.clone(), + ..Default::default() + } + } + + pub fn resolve(&self) { + let store = self.store.read().unwrap(); + self.resolve_with_store(&store); + } + + pub(crate) fn resolve_with_store(&self, store: &DocStore) { + let mut parents = self.parents.write().unwrap(); + + for node in store.items.values().flat_map(|items| items.iter()) { + let node = node.as_item(); + if let Some(item) = node.get() { + parents + .entry(item.id) + .and_modify(|e| { + if *e != node { + *e = node.clone(); + } + }) + .or_insert(node.clone()); + } + } + } + + pub fn parse_update(&self, update: &Update) -> Vec { + let store_items = SortedNodes::new(update.structs.iter().collect::>()) + .filter_map(|n| n.as_item().get().cloned()) + .collect::>(); + + // make items as reference + let mut store_items = store_items.iter().collect::>(); + store_items.sort_by(|a, b| a.id.clock.cmp(&b.id.clock)); + + self.parse_items(store_items) + } + + pub fn parse_delete_sets( + &self, + old_sets: &ClientMap, + new_sets: &ClientMap, + ) -> Vec { + let store = self.store.read().unwrap(); + let deleted_items = new_sets + .iter() + .filter_map(|(id, new_range)| { + // diff range if old range exists, or use new range + let range = old_sets + .get(id) + .map(|r| r.diff_range(new_range).into()) + .unwrap_or(new_range.clone()); + (!range.is_empty()).then_some((id, range)) + }) + .filter_map(|(client, range)| { + // check items contains in deleted range + store.items.get(client).map(move |items| { + items + .iter() + .filter(move |i| range.contains(i.clock())) + .filter_map(|i| i.as_item().get().cloned()) + }) + }) + .flatten() + .collect(); + + self.parse_deleted_items(deleted_items) + } + + pub fn parse_store(&self, options: HistoryOptions) -> Vec { + let store_items = { + let client = options + .client + .as_ref() + .and_then(|client| client.ne(&0).then_some(client)); + let store = self.store.read().unwrap(); + let mut sort_iter: Box> = Box::new( + SortedNodes::new(if let Some(client) = client { + store + .items + .get(client) + .map(|i| vec![(client, i)]) + .unwrap_or_default() + } else { + store.items.iter().collect::>() + }) + .filter_map(|n| n.as_item().get().cloned()), + ); + if client.is_some() { + // skip and limit only available when client is set + if let Some(skip) = options.skip { + sort_iter = Box::new(sort_iter.skip(skip)); + } + if let Some(limit) = options.limit { + sort_iter = Box::new(sort_iter.take(limit)); + } + } + + sort_iter.collect::>() + }; + + // make items as reference + let mut store_items = store_items.iter().collect::>(); + store_items.sort_by(|a, b| a.id.clock.cmp(&b.id.clock)); + + self.parse_items(store_items) + } + + fn parse_items(&self, store_items: Vec<&Item>) -> Vec { + let parents = self.parents.read().unwrap(); + let mut histories = vec![]; + + for item in store_items { + if item.deleted() { + continue; + } + + histories.push(History { + id: item.id.to_string(), + parent: Self::parse_path(item, &parents), + content: Value::from(&item.content).to_string(), + action: HistoryAction::Update, + }) + } + + histories + } + + fn parse_deleted_items(&self, deleted_items: Vec) -> Vec { + let parents = self.parents.read().unwrap(); + let mut histories = vec![]; + + for item in deleted_items { + histories.push(History { + id: item.id.to_string(), + parent: Self::parse_path(&item, &parents), + content: Value::from(&item.content).to_string(), + action: HistoryAction::Delete, + }) + } + + histories + } + + fn parse_path(item: &Item, parents: &HashMap>) -> Vec { + let mut path = Vec::new(); + let mut cur = item.clone(); + + while let Some(node) = cur.find_node_with_parent_info() { + path.push(Self::get_node_name(&node)); + + match Self::get_parent(parents, &node.parent) { + ParentNode::Root(name) => { + path.push(name); + break; + } + ParentNode::Node(parent) => { + if let Some(parent) = parent.get() { + cur = parent.clone(); + } else { + break; + } + } + ParentNode::Unknown => { + break; + } + } + } + + path.reverse(); + path + } + + fn get_node_name(item: &Item) -> String { + if let Some(name) = item.parent_sub.clone() { + name.to_string() + } else { + let mut curr = item.clone(); + let mut idx = 0; + + while let Some(item) = curr.left.get() { + curr = item.clone(); + idx += 1; + } + + idx.to_string() + } + } + + fn get_parent(parents: &HashMap>, parent: &Option) -> ParentNode { + match parent { + None => ParentNode::Unknown, + Some(Parent::Type(ptr)) => ptr + .ty() + .and_then(|ty| { + ty.item + .get() + .and_then(|i| parents.get(&i.id).map(|p| ParentNode::Node(p.clone()))) + .or(ty.root_name.clone().map(ParentNode::Root)) + }) + .unwrap_or(ParentNode::Unknown), + Some(Parent::String(name)) => ParentNode::Root(name.to_string()), + Some(Parent::Id(id)) => parents + .get(id) + .map(|p| ParentNode::Node(p.clone())) + .unwrap_or(ParentNode::Unknown), + } + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub enum HistoryAction { + Insert, + Update, + Delete, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub struct History { + pub id: String, + pub parent: Vec, + pub content: String, + pub action: HistoryAction, +} + +pub(crate) struct SortedNodes<'a> { + nodes: Vec<(&'a Client, &'a VecDeque)>, + current: Option>, +} + +impl<'a> SortedNodes<'a> { + pub fn new(mut nodes: Vec<(&'a Client, &'a VecDeque)>) -> Self { + nodes.sort_by(|a, b| b.0.cmp(a.0)); + let current = nodes.pop().map(|(_, v)| v.clone()); + Self { nodes, current } + } +} + +impl Iterator for SortedNodes<'_> { + type Item = Node; + + fn next(&mut self) -> Option { + if let Some(current) = self.current.as_mut() { + if let Some(node) = current.pop_back() { + return Some(node); + } + } + + if let Some((_, nodes)) = self.nodes.pop() { + self.current = Some(nodes.clone()); + self.next() + } else { + None + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn parse_history_client_test() { + loom_model!({ + let doc = Doc::default(); + let mut map = doc.get_or_create_map("map").unwrap(); + let mut sub_map = doc.create_map().unwrap(); + map.insert("sub_map".to_string(), sub_map.clone()).unwrap(); + sub_map.insert("key".to_string(), "value").unwrap(); + + assert_eq!(doc.clients()[0], doc.client()); + }); + } + + #[test] + fn parse_history_test() { + loom_model!({ + let doc = Doc::default(); + let mut map = doc.get_or_create_map("map").unwrap(); + let mut sub_map = doc.create_map().unwrap(); + map.insert("sub_map".to_string(), sub_map.clone()).unwrap(); + sub_map.insert("key".to_string(), "value").unwrap(); + + let history = StoreHistory::new(&doc.store); + + let update = doc.encode_update().unwrap(); + + assert_eq!( + history.parse_store(Default::default()), + history.parse_update(&update,) + ); + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/mod.rs b/packages/common/y-octo/core/src/doc/mod.rs new file mode 100644 index 0000000000..608f654274 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/mod.rs @@ -0,0 +1,33 @@ +mod awareness; +mod codec; +mod common; +mod document; +mod hasher; +mod history; +mod publisher; +mod store; +mod types; +mod utils; + +pub use ahash::{HashMap, HashMapExt, HashSet, HashSetExt}; +pub use awareness::{Awareness, AwarenessEvent}; +pub use codec::*; +pub use common::*; +pub use document::{Doc, DocOptions}; +pub use hasher::ClientMap; +pub use history::{History, HistoryOptions, StoreHistory}; +use smol_str::SmolStr; +pub(crate) use store::DocStore; +pub use types::*; +pub use utils::*; + +use super::*; + +/// NOTE: +/// - We do not use [HashMap::with_capacity(num_of_clients)] directly here +/// because we don't trust the input data. +/// - For instance, what if the first u64 was somehow set a very big value? +/// - A pre-allocated HashMap with a big capacity may cause OOM. +/// - A kinda safer approach is give it a max capacity of 1024 at first +/// allocation, and then let std makes the growth as need. +pub const HASHMAP_SAFE_CAPACITY: usize = 1 << 10; diff --git a/packages/common/y-octo/core/src/doc/publisher.rs b/packages/common/y-octo/core/src/doc/publisher.rs new file mode 100644 index 0000000000..3912d2b1f6 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/publisher.rs @@ -0,0 +1,244 @@ +use std::{ + thread::{current, sleep, spawn}, + time::Duration, +}; + +use log::{debug, trace}; + +use super::{history::StoreHistory, store::StoreRef, *}; +use crate::sync::{Arc, AtomicBool, Mutex, Ordering, RwLock}; + +pub type DocSubscriber = Box; + +const OBSERVE_INTERVAL: u64 = 100; + +pub struct DocPublisher { + store: StoreRef, + history: StoreHistory, + subscribers: Arc>>, + observer: Arc>>>, + observing: Arc, +} + +impl DocPublisher { + pub(crate) fn new(store: StoreRef) -> Self { + let subscribers = Arc::new(RwLock::new(Vec::::new())); + let history = StoreHistory::new(&store); + history.resolve(); + + let publisher = Self { + store, + history, + subscribers, + observer: Arc::default(), + observing: Arc::new(AtomicBool::new(false)), + }; + + if cfg!(not(any(feature = "bench", fuzzing, loom, miri))) { + publisher.start(); + } + + publisher + } + + pub fn start(&self) { + let mut observer = self.observer.lock().unwrap(); + let observing = self.observing.clone(); + let store = self.store.clone(); + let history = self.history.clone(); + if observer.is_none() { + let thread_subscribers = self.subscribers.clone(); + observing.store(true, Ordering::Release); + debug!("start observing"); + let thread = spawn(move || { + let mut last_update = store.read().unwrap().get_state_vector(); + let mut last_deletes = store.read().unwrap().delete_set.clone(); + loop { + sleep(Duration::from_millis(OBSERVE_INTERVAL)); + if !observing.load(Ordering::Acquire) { + debug!("stop observing"); + break; + } + + let subscribers = thread_subscribers.read().unwrap(); + if subscribers.is_empty() { + continue; + } + + let store = store.read().unwrap(); + + let update = store.get_state_vector(); + let deletes = store.delete_set.clone(); + if update != last_update || deletes != last_deletes { + trace!( + "update: {:?}, last_update: {:?}, {:?}", + update, + last_update, + current().id(), + ); + trace!( + "deletes: {:?}, last_deletes: {:?}, {:?}", + deletes, + last_deletes, + current().id(), + ); + + history.resolve_with_store(&store); + let (binary, history) = match store.diff_state_vector(&last_update, false) { + Ok(update) => { + drop(store); + + let history = history + .parse_update(&update) + .into_iter() + .chain(history.parse_delete_sets(&last_deletes, &deletes)) + .collect::>(); + + let mut encoder = RawEncoder::default(); + if let Err(e) = update.write(&mut encoder) { + warn!("Failed to encode document: {}", e); + continue; + } + (encoder.into_inner(), history) + } + Err(e) => { + warn!("Failed to diff document: {}", e); + continue; + } + }; + + last_update = update; + last_deletes = deletes; + + for cb in subscribers.iter() { + use std::panic::{catch_unwind, AssertUnwindSafe}; + // catch panic if callback throw + catch_unwind(AssertUnwindSafe(|| { + cb(&binary, &history); + })) + .unwrap_or_else(|e| { + warn!("Failed to call subscriber: {:?}", e); + }); + } + } else { + drop(store); + } + } + }); + observer.replace(thread); + } else { + debug!("already observing"); + } + } + + pub fn stop(&self) { + let mut observer = self.observer.lock().unwrap(); + if let Some(observer) = observer.take() { + self.observing.store(false, Ordering::Release); + observer.join().unwrap(); + } + } + + pub(crate) fn count(&self) -> usize { + self.subscribers.read().unwrap().len() + } + + pub(crate) fn subscribe(&self, subscriber: impl Fn(&[u8], &[History]) + Send + Sync + 'static) { + self.subscribers.write().unwrap().push(Box::new(subscriber)); + } + + pub(crate) fn unsubscribe_all(&self) { + self.subscribers.write().unwrap().clear(); + } +} + +impl std::fmt::Debug for DocPublisher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DocPublisher").finish() + } +} + +impl Drop for DocPublisher { + fn drop(&mut self) { + self.stop(); + self.unsubscribe_all(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sync::AtomicUsize; + + #[test] + fn test_parse_update_history() { + loom_model!({ + let doc = Doc::default(); + + let ret = [ + vec![vec!["(1, 0)", "test.key1", "val1"]], + vec![ + vec!["(1, 1)", "test.key2", "val2"], + vec!["(1, 2)", "test.key3", "val3"], + ], + vec![ + vec!["(1, 3)", "array.0", "val1"], + vec!["(1, 4)", "array.1", "val2"], + vec!["(1, 5)", "array.2", "val3"], + ], + ]; + + let cycle = Arc::new(AtomicUsize::new(0)); + + // update: 24 + // history change by (1, 0) at test.key1: val1 + // update: 43 + // history change by (1, 1) at test.key2: val2 + // history change by (1, 2) at test.key3: val3 + // update: 40 + // history change by (1, 3) at array.0: val1 + // history change by (1, 4) at array.1: val2 + // history change by (1, 5) at array.2: val3 + doc.subscribe(move |u, history| { + println!("update: {}", u.len()); + let cycle = cycle.fetch_add(1, Ordering::SeqCst); + + let ret = ret[cycle].clone(); + for (i, h) in history.iter().enumerate() { + println!( + "history change by {} at {}: {}", + h.id, + h.parent.join("."), + h.content + ); + // lost first update by unknown reason in asan test, skip it if asan enabled + if option_env!("ASAN_OPTIONS").is_none() { + let ret = &ret[i]; + assert_eq!(h.id, ret[0]); + assert_eq!(h.parent.join("."), ret[1]); + assert_eq!(h.content, ret[2]); + } + } + }); + sleep(Duration::from_millis(500)); + + let mut map = doc.get_or_create_map("test").unwrap(); + map.insert("key1".to_string(), "val1").unwrap(); + + sleep(Duration::from_millis(500)); + + map.insert("key2".to_string(), "val2").unwrap(); + map.insert("key3".to_string(), "val3").unwrap(); + sleep(Duration::from_millis(500)); + + let mut array = doc.get_or_create_array("array").unwrap(); + array.push("val1").unwrap(); + array.push("val2").unwrap(); + array.push("val3").unwrap(); + + sleep(Duration::from_millis(500)); + + doc.publisher.stop(); + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/store.rs b/packages/common/y-octo/core/src/doc/store.rs new file mode 100644 index 0000000000..8ae493a17a --- /dev/null +++ b/packages/common/y-octo/core/src/doc/store.rs @@ -0,0 +1,1355 @@ +use std::{ + collections::{hash_map::Entry, VecDeque}, + mem, + ops::{Deref, Range}, +}; + +use super::*; +use crate::{ + doc::StateVector, + sync::{Arc, RwLock, RwLockWriteGuard, Weak}, +}; + +unsafe impl Send for DocStore {} +unsafe impl Sync for DocStore {} + +#[derive(Default, Debug)] +pub(crate) struct DocStore { + client: Client, + pub items: ClientMap>, + pub delete_set: DeleteSet, + + // following fields are only used in memory + pub types: HashMap, + // types created from this store but with no names, + // we store it here to keep the ownership inside store without being released. + pub dangling_types: HashMap, + pub pending: Option, + pub last_optimized_state: StateVector, +} + +pub(crate) type StoreRef = Arc>; +pub(crate) type WeakStoreRef = Weak>; + +impl PartialEq for DocStore { + fn eq(&self, other: &Self) -> bool { + self.client == other.client + } +} + +impl DocStore { + pub fn with_client(client: Client) -> Self { + Self { + client, + ..Default::default() + } + } + + pub fn client(&self) -> Client { + self.client + } + + pub fn clients(&self) -> Vec { + self.items.keys().cloned().collect() + } + + #[cfg(feature = "debug")] + pub fn total_nodes(&self) -> usize { + self.items.values().map(|v| v.len()).sum() + } + + #[cfg(feature = "debug")] + pub fn total_delete_sets(&self) -> usize { + self + .delete_set + .values() + .map(|v| match v { + OrderRange::Range(_) => 1, + OrderRange::Fragment(f) => f.len(), + }) + .sum() + } + + #[cfg(feature = "debug")] + pub fn total_types(&self) -> usize { + self.types.len() + } + + #[cfg(feature = "debug")] + pub fn total_dangling_types(&self) -> usize { + self.dangling_types.len() + } + + #[cfg(feature = "debug")] + pub fn total_pending_nodes(&self) -> usize { + self.pending.as_ref().map(|p| p.structs.len()).unwrap_or(0) + } + + pub fn get_state(&self, client: Client) -> Clock { + if let Some(structs) = self.items.get(&client) { + if let Some(last_struct) = structs.back() { + last_struct.clock() + last_struct.len() + } else { + warn!("client {} has no struct info", client); + 0 + } + } else { + 0 + } + } + + pub fn get_state_vector(&self) -> StateVector { + Self::items_as_state_vector(&self.items) + } + + fn items_as_state_vector(items: &ClientMap>) -> StateVector { + let mut state = StateVector::default(); + for (client, structs) in items.iter() { + if let Some(last_struct) = structs.back() { + state.insert(*client, last_struct.clock() + last_struct.len()); + } else { + warn!("client {} has no struct info", client); + } + } + state + } + + pub fn add_node(&mut self, item: Node) -> JwstCodecResult { + let client_id = item.client(); + match self.items.entry(client_id) { + Entry::Occupied(mut entry) => { + let structs = entry.get_mut(); + if let Some(last_struct) = structs.back() { + let expect = last_struct.clock() + last_struct.len(); + let actually = item.clock(); + if expect != actually { + return Err(JwstCodecError::StructClockInvalid { expect, actually }); + } + } else { + warn!("client {} has no struct info", client_id); + } + structs.push_back(item); + } + Entry::Vacant(entry) => { + entry.insert(VecDeque::from([item])); + } + } + + Ok(()) + } + + /// binary search struct info on a sorted array + pub fn get_node_index(items: &VecDeque, clock: Clock) -> Option { + let mut left = 0; + let mut right = items.len() - 1; + let middle = &items[right]; + let middle_clock = middle.clock(); + if middle_clock == clock { + return Some(right); + } + let mut middle_index = (clock / (middle_clock + middle.len() - 1)) as usize * right; + while left <= right { + let middle = &items[middle_index]; + let middle_clock = middle.clock(); + if middle_clock <= clock { + if clock < middle_clock + middle.len() { + return Some(middle_index); + } + left = middle_index + 1; + } else { + right = middle_index - 1; + } + middle_index = (left + right) / 2; + } + None + } + + pub fn create_item( + &self, + content: Content, + left: Somr, + right: Somr, + parent: Option, + parent_sub: Option, + ) -> ItemRef { + let id = (self.client(), self.get_state(self.client())).into(); + let item = Somr::new(Item::new(id, content, left, right, parent, parent_sub)); + + if let Content::Type(ty) = &item.get().unwrap().content { + if let Some(mut ty) = ty.ty_mut() { + ty.item = item.clone(); + } + } + + item + } + + pub fn get_node>(&self, id: I) -> Option { + self.get_node_with_idx(id).map(|(item, _)| item) + } + + pub fn get_node_with_idx>(&self, id: I) -> Option<(Node, usize)> { + let id = id.into(); + if let Some(items) = self.items.get(&id.client) { + if let Some(index) = Self::get_node_index(items, id.clock) { + return items.get(index).map(|item| (item.clone(), index)); + } + } + + None + } + + pub fn split_node>(&mut self, id: I, diff: u64) -> JwstCodecResult<(Node, Node)> { + debug_assert!(diff > 0); + + let id = id.into(); + + if let Some(items) = self.items.get_mut(&id.client) { + if let Some(idx) = Self::get_node_index(items, id.clock) { + return Self::split_node_at(items, idx, diff); + } + } + + Err(JwstCodecError::StructSequenceNotExists(id.client)) + } + + pub fn split_node_at( + items: &mut VecDeque, + idx: usize, + diff: u64, + ) -> JwstCodecResult<(Node, Node)> { + debug_assert!(diff > 0); + + let node = items.get(idx).unwrap().clone(); + debug_assert!(node.is_item()); + if let Node::Item(item_ref) = &node { + let item = item_ref.get().unwrap(); + + let (left, right) = item.split_at(diff)?; + + let left_ref = Somr::new(left); + let right_ref = Somr::new(right); + + // SAFETY: + // we make sure store is the only entry of mutating an item, + // and we already hold mutable reference of store, so it's safe to do so + unsafe { + let mut left_item = item_ref.get_mut_unchecked(); + let mut right_item = right_ref.get_mut_unchecked(); + left_item.content = left_ref.get_unchecked().content.clone(); + + // we had the correct left/right content + // now build the references + let right_right_ref = left_item.right.clone(); + right_item.left = if right_right_ref.is_some() { + let mut right_right = right_right_ref.get_mut_unchecked(); + mem::replace(&mut right_right.left, right_ref.clone()) + } else { + item_ref.clone() + }; + right_item.right = mem::replace(&mut left_item.right, right_ref.clone()); + right_item.origin_left_id = Some(left_item.last_id()); + right_item.origin_right_id = left_item.origin_right_id; + }; + + let right = Node::Item(right_ref); + let right_ref = right.clone(); + items.insert(idx + 1, right); + Ok((node, right_ref)) + } else { + Err(JwstCodecError::ItemSplitNotSupport) + } + } + + pub fn split_at_and_get_right>(&mut self, id: I) -> JwstCodecResult { + let id = id.into(); + if let Some(items) = self.items.get_mut(&id.client) { + if let Some(index) = Self::get_node_index(items, id.clock) { + let item = items.get(index).unwrap().clone(); + let offset = id.clock - item.clock(); + if offset > 0 && item.is_item() { + let (_, right) = Self::split_node_at(items, index, offset)?; + return Ok(right); + } else { + return Ok(item); + } + } + } + + Err(JwstCodecError::StructSequenceNotExists(id.client)) + } + + pub fn split_at_and_get_left>(&mut self, id: I) -> JwstCodecResult { + let id = id.into(); + if let Some(items) = self.items.get_mut(&id.client) { + if let Some(index) = Self::get_node_index(items, id.clock) { + let item = items.get(index).unwrap().clone(); + let offset = id.clock - item.clock(); + if offset != item.len() - 1 && !item.is_gc() { + let (left, _) = Self::split_node_at(items, index, offset + 1)?; + return Ok(left); + } else { + return Ok(item); + } + } + } + + Err(JwstCodecError::StructSequenceNotExists(id.client)) + } + + // TODO: use function in code + #[allow(dead_code)] + pub fn self_check(&self) -> JwstCodecResult { + for structs in self.items.values() { + for i in 1..structs.len() { + let l = &structs[i - 1]; + let r = &structs[i]; + if l.clock() + l.len() != r.clock() { + return Err(JwstCodecError::StructSequenceInvalid { + client_id: l.client(), + clock: l.clock(), + }); + } + } + } + + Ok(()) + } + + // only for creating named type + pub fn get_or_create_type(&mut self, store_ref: &StoreRef, name: &str) -> YTypeRef { + match self.types.entry(name.to_string()) { + Entry::Occupied(e) => e.get().clone(), + Entry::Vacant(e) => { + let mut inner = YType::new(YTypeKind::Unknown, None); + inner.root_name = Some(name.to_string()); + + let ty = YTypeRef { + store: Arc::downgrade(store_ref), + inner: Somr::new(RwLock::new(inner)), + }; + let ty_ref = ty.clone(); + e.insert(ty); + ty_ref + } + } + } + + /// A repair for an item do such things: + /// - split left if needed (insert in between a splitable item) + /// - split right if needed (insert in between a splitable item) + /// - recover parent to [Parent::Type] + /// - [Parent::String] for root level named type (e.g + /// `doc.get_or_create_text("content")`) + /// - [Parent::Id] for type as item (e.g `doc.create_text()`) + /// - [None] means borrow left.parent or right.parent + pub fn repair(&mut self, item: &mut Item, store_ref: StoreRef) -> JwstCodecResult { + if let Some(left_id) = item.origin_left_id { + if let Node::Item(left_ref) = self.split_at_and_get_left(left_id)? { + item.origin_left_id = left_ref.get().map(|left| left.last_id()); + item.left = left_ref; + } else { + item.origin_left_id = None; + } + } + + if let Some(right_id) = item.origin_right_id { + if let Node::Item(right_ref) = self.split_at_and_get_right(right_id)? { + item.origin_right_id = right_ref.get().map(|right| right.id); + item.right = right_ref; + } else { + item.origin_right_id = None; + } + } + + match &item.parent { + // root level named type + // doc.get_or_create_text("content"); + // ^^^^^^^ Parent::String("content") + Some(Parent::String(str)) => { + let ty = self.get_or_create_type(&store_ref, str); + item.parent.replace(Parent::Type(ty)); + } + // type as item + // let text = doc.create_text(); + // let mut map = doc.get_or_create_map("content"); + // map.insert("p1", text); + // + // Item { id: (1, 0), content: Content::Type(_) } + // ^^ Parent::Id((1, 0)) + Some(Parent::Id(parent_id)) => { + match self.get_node(*parent_id) { + Some(Node::Item(parent_item)) => { + if let Content::Type(ty) = &parent_item.get().unwrap().content { + item.parent.replace(Parent::Type(ty.clone())); + } else { + // invalid parent, take it. + item.parent.take(); + // return Err(JwstCodecError::InvalidParent); + } + } + _ => { + // GC & Skip are not valid parent, take it. + item.parent.take(); + } + } + } + // no item.parent, borrow left.parent or right.parent + None => { + if let Some(left) = ItemRef::from(item.left.clone()).get() { + item.parent = left.parent.clone(); + item.parent_sub = left.parent_sub.clone(); + } else if let Some(right) = ItemRef::from(item.right.clone()).get() { + item.parent = right.parent.clone(); + item.parent_sub = right.parent_sub.clone(); + } + } + _ => {} + }; + + // assign store in ytype to ensure store exists if a ytype not has any children + if let Content::Type(ty) = &mut item.content { + ty.store = Arc::downgrade(&store_ref); + + // we keep ty owner in dangling_types so the delete of any type will not make it + // dropped + if ty.inner.is_owned() { + let owned_inner = ty.inner.swap_take(); + self.dangling_types.insert( + ty.inner.ptr().as_ptr() as usize, + YTypeRef { + store: ty.store.clone(), + inner: owned_inner, + }, + ); + } else { + return Err(JwstCodecError::InvalidParent); + } + } + + Ok(()) + } + + pub fn integrate( + &mut self, + mut node: Node, + offset: u64, + parent: Option<&mut YType>, + ) -> JwstCodecResult { + match &mut node { + Node::Item(item_owner_ref) => { + assert!( + item_owner_ref.is_owned(), + "Required a owned Item type but got an shared reference" + ); + + // SAFETY: + // before we integrate struct into store, + // the struct => Arc is owned reference actually, + // no one else refer to such item yet, we can safely mutable refer to it now. + let this = &mut *unsafe { item_owner_ref.get_mut_unchecked() }; + + if offset > 0 { + this.id.clock += offset; + if let Node::Item(left_ref) = + self.split_at_and_get_left(Id::new(this.id.client, this.id.clock - 1))? + { + this.origin_left_id = left_ref.get().map(|left| left.last_id()); + this.left = left_ref; + } + this.content = this.content.split(offset)?.1; + } + + if let Some(Parent::Type(ty)) = &this.parent { + let mut parent_lock: Option> = None; + let parent = if let Some(p) = parent { + p + } else if let Some(ty) = ty.ty_mut() { + parent_lock = Some(ty); + parent_lock.as_deref_mut().unwrap() + } else { + return Ok(()); + }; + + let mut left = this.left.clone(); + let mut right = this.right.clone(); + + let right_is_null_or_has_left = match right.get() { + None => true, + Some(r) => r.left.is_some(), + }; + + let left_has_other_right_than_self = match left.get() { + Some(left) => left.right != right, + _ => false, + }; + + // conflicts + if left.is_none() && right_is_null_or_has_left || left_has_other_right_than_self { + // set the first conflicting item + let mut conflict = if let Some(left) = left.get() { + left.right.clone() + } else if let Some(parent_sub) = &this.parent_sub { + parent.map.get(parent_sub).cloned().unwrap_or(Somr::none()) + } else { + parent.start.clone() + }; + + let mut conflicting_items = HashSet::new(); + let mut items_before_origin = HashSet::new(); + + while conflict.is_some() { + if conflict == right { + break; + } + if let Some(conflict_item) = conflict.get() { + let conflict_id = conflict_item.id; + + items_before_origin.insert(conflict_id); + conflicting_items.insert(conflict_id); + + if this.origin_left_id == conflict_item.origin_left_id { + // case 1 + if conflict_id.client < this.id.client { + left = conflict.clone(); + conflicting_items.clear(); + } else if this.origin_right_id == conflict_item.origin_right_id { + // `this` and `c` are conflicting and point to the same + // integration points. The id decides which item comes first. + // Since `this` is to the left of `c`, we can break here. + break; + } + } else if let Some(conflict_item_left) = conflict_item.origin_left_id { + if items_before_origin.contains(&conflict_item_left) + && !conflicting_items.contains(&conflict_item_left) + { + left = conflict.clone(); + conflicting_items.clear(); + } + } else { + break; + } + + conflict = conflict_item.right.clone(); + } else { + break; + } + } + } + + // reconnect left/right + // has left, connect left <-> self <-> left.right + if left.is_some() { + unsafe { + // SAFETY: we get store write lock, no way the left get dropped by owner + let mut left = left.get_mut_unchecked(); + right = left.right.clone(); + left.right = item_owner_ref.clone(); + } + this.left = left.clone(); + } else { + // no left, parent.start = this + right = if let Some(parent_sub) = &this.parent_sub { + parent + .map + .get(parent_sub) + .map(|n| Node::Item(n.clone()).head()) + .into() + } else { + mem::replace(&mut parent.start, item_owner_ref.clone()) + }; + this.left = Somr::none(); + } + + // has right, connect + if right.is_some() { + unsafe { + // SAFETY: we get store write lock, no way the left get dropped by owner + let mut right = right.get_mut_unchecked(); + right.left = item_owner_ref.clone(); + } + } else { + // no right, parent.start = this, delete this.left + if let Some(parent_sub) = &this.parent_sub { + parent + .map + .insert(parent_sub.clone(), item_owner_ref.clone()); + + if let Some(left) = this.left.get() { + self.delete_item(left, Some(parent)); + } + } + } + this.right = right.clone(); + + let parent_deleted = parent + .item + .get() + .map(|item| item.deleted()) + .unwrap_or(false); + + // should delete + if parent_deleted || this.parent_sub.is_some() && this.right.is_some() { + self.delete_node(&Node::Item(item_owner_ref.clone()), Some(parent)); + } else { + // adjust parent length + if this.parent_sub.is_none() { + parent.len += this.len(); + } + } + + parent_lock.take(); + } else { + // if parent not exists, integrate GC node instead + // don't delete it because it may referenced by other nodes + // if all nodes that reference it are deleted, it will merged into one gc node + node = Node::new_gc(node.id(), node.len()); + } + } + Node::GC(item) => { + if offset > 0 { + item.id.clock += offset; + item.len -= offset; + } + } + Node::Skip(_) => { + // skip ignored + } + } + self.add_node(node) + } + + pub fn delete_item(&mut self, item: &Item, parent: Option<&mut YType>) { + let mut pending_delete_sets = HashMap::new(); + Self::delete_item_inner(&mut pending_delete_sets, item, parent); + for (client, ranges) in pending_delete_sets { + self.delete_set.batch_add_ranges(client, ranges); + } + } + + fn delete_item_inner( + delete_set: &mut HashMap>>, + item: &Item, + parent: Option<&mut YType>, + ) { + // 1. mark item as deleted, if item is gced, return + if !item.delete() { + return; + } + + // 2. add it to delete set + let range = item.id.clock..item.id.clock + item.len(); + delete_set + .entry(item.id.client) + .and_modify(|v| v.push(range.clone())) + .or_insert(vec![range]); + + // 3. adjust parent length + if item.parent_sub.is_none() && item.countable() { + if let Some(parent) = parent { + if parent.len != 0 { + parent.len -= item.len(); + } + } else if let Some(Parent::Type(ty)) = &item.parent { + ty.ty_mut().unwrap().len -= item.len(); + } + } + + match &item.content { + Content::Type(ty) => { + // 4. delete all children + if let Some(mut ty) = ty.ty_mut() { + // items in ty are all refs, not owned + let mut item_ref = ty.start.clone(); + while let Some(item) = item_ref.get() { + if !item.deleted() { + Self::delete_item_inner(delete_set, item, Some(&mut ty)); + } + + item_ref = item.right.clone(); + } + + let map_values = ty.map.values().cloned().collect::>(); + for item in map_values { + if let Some(item) = item.get() { + if !item.deleted() { + Self::delete_item_inner(delete_set, item, Some(&mut ty)); + } + } + } + } + } + Content::Doc { .. } => { + // TODO: remove subdoc + } + _ => {} + } + } + + pub fn delete_node(&mut self, struct_info: &Node, parent: Option<&mut YType>) { + if let Some(item) = struct_info.as_item().get() { + self.delete_item(item, parent); + } + } + + pub fn delete_range(&mut self, client: u64, range: Range) -> JwstCodecResult { + let start = range.start; + let end = range.end; + + if let Some(items) = self.items.get_mut(&client) { + if let Some(mut idx) = DocStore::get_node_index(items, start) { + { + // id.clock <= range.start < id.end + // need to split the item and delete the right part + // -----item----- + // ^start + let node = &items[idx]; + let id = node.id(); + + if !node.deleted() && id.clock < start { + DocStore::split_node_at(items, idx, start - id.clock)?; + idx += 1; + } + }; + + let mut pending_delete_sets = HashMap::new(); + while idx < items.len() { + let node = items[idx].clone(); + let id = node.id(); + + if id.clock < end { + if !node.deleted() { + if let Some(item) = node.as_item().get() { + // need to split the item + // -----item----- + // ^end + if end < id.clock + node.len() { + DocStore::split_node_at(items, idx, end - id.clock)?; + } + + Self::delete_item_inner(&mut pending_delete_sets, item, None); + } + } + } else { + break; + } + + idx += 1; + } + for (client, ranges) in pending_delete_sets { + self.delete_set.batch_add_ranges(client, ranges); + } + } + } + + Ok(()) + } + + fn diff_state_vectors( + local_state_vector: &StateVector, + remote_state_vector: &StateVector, + ) -> Vec<(Client, Clock)> { + let mut diff = Vec::new(); + + for (client, &remote_clock) in remote_state_vector.iter() { + let local_clock = local_state_vector.get(client); + if local_clock > remote_clock { + diff.push((*client, remote_clock)); + } + } + + for (client, _) in local_state_vector.iter() { + if remote_state_vector.get(client) == 0 { + diff.push((*client, 0)); + } + } + + diff + } + + pub fn diff_state_vector(&self, sv: &StateVector, with_pending: bool) -> JwstCodecResult { + let update_structs = Self::diff_structs(&self.items, sv)?; + + let mut update = Update { + structs: update_structs, + delete_set: Self::generate_delete_set(&self.items), + ..Update::default() + }; + + if with_pending { + if let Some(pending) = &self.pending { + Update::merge_into(&mut update, [pending.clone()]) + } + } + + Ok(update) + } + + fn diff_structs( + map: &ClientMap>, + sv: &StateVector, + ) -> JwstCodecResult>> { + let local_state_vector = Self::items_as_state_vector(map); + let diff = Self::diff_state_vectors(&local_state_vector, sv); + let mut update_structs = ClientMap::new(); + + for (client, clock) in diff { + // We have made sure that the client is in the local state vector in + // diff_state_vectors() + if let Some(items) = map.get(&client) { + if items.is_empty() { + continue; + } + + update_structs.insert(client, VecDeque::new()); + let vec_struct_info = update_structs.get_mut(&client).unwrap(); + + // the smallest clock in items may exceed the clock + let clock = items.front().unwrap().id().clock.max(clock); + if let Some(index) = Self::get_node_index(items, clock) { + let first_block = items.get(index).unwrap(); + let offset = first_block.clock() - clock; + if offset != 0 { + vec_struct_info.push_back(first_block.clone().split_at(offset)?.1); + } else { + vec_struct_info.push_back(first_block.clone()); + } + + for item in items.iter().skip(index + 1) { + vec_struct_info.push_back(item.clone()); + } + } + } + } + + Ok(update_structs) + } + + fn generate_delete_set(refs: &ClientMap>) -> DeleteSet { + let mut delete_set = DeleteSet::default(); + + for (client, nodes) in refs { + nodes + .iter() + .filter(|n| n.deleted()) + .for_each(|n| delete_set.add(*client, n.clock(), n.len())); + } + + delete_set + } + + /// Optimize the memory usage of store + pub fn optimize(&mut self) -> JwstCodecResult { + // 1. gc delete set + self.gc_delete_set()?; + // 2. merge delete set (in our delete set impl, which is based on `OrderRange` + // has already have auto-merge functionality), pass + // 3. merge same content siblings, e.g contentString + ContentString + self.make_continuous(); + Ok(()) + } + + fn gc_delete_set(&mut self) -> JwstCodecResult<()> { + for (client, deletes) in self.delete_set.deref() { + for range in deletes { + let start = range.start; + let end = range.end; + let items = self.items.get_mut(client).unwrap(); + if let Some(mut idx) = Self::get_node_index(items, start) { + while idx < items.len() { + if let Node::Item(item) = items[idx].clone() { + let item = unsafe { item.get_unchecked() }; + + if end <= item.id.clock { + break; + } + + if !item.keep() { + let parent_gced = matches!(&item.parent, Some(p) if { + if let Parent::Type(ty) = p { + if let Some(ty) = ty.ty() { + (ty.start.is_none() && ty.map.is_empty()) || ty.item.get().map(|item|item.deleted()).unwrap_or(false) + } else { + false + } + } else { + false + } + }); + Self::gc_item(items, idx, parent_gced)?; + } + } + + idx += 1; + } + } + } + } + + Ok(()) + } + + fn gc_item(items: &mut VecDeque, idx: usize, replace: bool) -> JwstCodecResult { + if let Node::Item(item_ref) = items[idx].clone() { + let item = unsafe { item_ref.get_unchecked() }; + + // if replace=true we don't check if the item deleted, + // because the parent already delete but children may not delete + if !replace && !item.deleted() { + return Err(JwstCodecError::Unexpected); + } + + Self::gc_content(&item.content)?; + + if replace { + let _ = mem::replace(&mut items[idx], Node::new_gc(item.id, item.len())); + } else { + let mut item = unsafe { item_ref.get_mut_unchecked() }; + item.content = Content::Deleted(item.len()); + item.flags.clear_countable(); + debug_assert!(!item.flags.countable()); + } + } + + Ok(()) + } + + fn gc_content(content: &Content) -> JwstCodecResult { + if let Content::Type(ty) = content { + if let Some(mut ty) = ty.ty_mut() { + ty.start = Somr::none(); + ty.map.clear(); + } + } + + Ok(()) + } + + fn make_continuous(&mut self) { + let state = self.get_state_vector(); + + for (client, state) in state.iter() { + let before_state = self.last_optimized_state.get(client); + if before_state == *state { + continue; + } + + let nodes = self.items.get_mut(client).unwrap(); + let first_change = Self::get_node_index(nodes, before_state) + .unwrap_or(1) + .max(1); + let mut idx = nodes.len() - 1; + + while idx > 0 && idx >= first_change { + idx = idx.saturating_sub(Self::merge_with_lefts(nodes, idx) + 1); + } + } + + self.last_optimized_state = state; + } + + fn merge_with_lefts(nodes: &mut VecDeque, idx: usize) -> usize { + let mut pos = idx; + loop { + if pos == 0 { + break; + } + + let right = nodes.get(pos).unwrap().clone(); + let left = nodes.get_mut(pos - 1).unwrap(); + + if !left.merge(right) { + break; + } + + pos -= 1; + } + nodes.drain(pos + 1..=idx); + + // return the index of processed items + idx - pos + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_state() { + loom_model!({ + let doc_store = DocStore::with_client(1); + let state = doc_store.get_state(1); + assert_eq!(state, 0); + }); + + loom_model!({ + let mut doc_store = DocStore::with_client(1); + + let client_id = 1; + + let struct_info1 = Node::new_gc(Id::new(1, 1), 5); + let struct_info2 = Node::new_skip(Id::new(1, 6), 7); + + doc_store.items.insert( + client_id, + VecDeque::from([struct_info1, struct_info2.clone()]), + ); + + let state = doc_store.get_state(client_id); + + assert_eq!(state, struct_info2.clock() + struct_info2.len()); + + assert!(doc_store.self_check().is_ok()); + }); + } + + #[test] + fn test_get_state_vector() { + loom_model!({ + let doc_store = DocStore::with_client(1); + let state_map = doc_store.get_state_vector(); + assert!(state_map.is_empty()); + }); + + loom_model!({ + let mut doc_store = DocStore::with_client(1); + + let client1 = 1; + let struct_info1 = Node::new_gc((1, 0).into(), 5); + + let client2 = 2; + let struct_info2 = Node::new_gc((2, 0).into(), 6); + let struct_info3 = Node::new_skip((2, 6).into(), 1); + + doc_store + .items + .insert(client1, VecDeque::from([struct_info1.clone()])); + doc_store.items.insert( + client2, + VecDeque::from([struct_info2, struct_info3.clone()]), + ); + + let state_map = doc_store.get_state_vector(); + + assert_eq!( + state_map.get(&client1), + struct_info1.clock() + struct_info1.len() + ); + assert_eq!( + state_map.get(&client2), + struct_info3.clock() + struct_info3.len() + ); + + assert!(doc_store.self_check().is_ok()); + }); + } + + #[test] + fn test_add_item() { + loom_model!({ + let mut doc_store = DocStore::with_client(1); + + let struct_info1 = Node::new_gc(Id::new(1, 0), 5); + let struct_info2 = Node::new_skip(Id::new(1, 5), 1); + let struct_info3_err = Node::new_skip(Id::new(1, 5), 1); + let struct_info3 = Node::new_skip(Id::new(1, 6), 1); + + assert!(doc_store.add_node(struct_info1.clone()).is_ok()); + assert!(doc_store.add_node(struct_info2).is_ok()); + assert_eq!( + doc_store.add_node(struct_info3_err), + Err(JwstCodecError::StructClockInvalid { + expect: 6, + actually: 5 + }) + ); + assert!(doc_store.add_node(struct_info3.clone()).is_ok()); + assert_eq!( + doc_store.get_state(struct_info1.client()), + struct_info3.clock() + struct_info3.len() + ); + }); + } + + #[test] + fn test_get_item() { + loom_model!({ + let mut doc_store = DocStore::with_client(1); + let struct_info = Node::new_gc(Id::new(1, 0), 10); + doc_store.add_node(struct_info.clone()).unwrap(); + + assert_eq!(doc_store.get_node(Id::new(1, 9)), Some(struct_info)); + }); + + loom_model!({ + let mut doc_store = DocStore::with_client(1); + let struct_info1 = Node::new_gc(Id::new(1, 0), 10); + let struct_info2 = Node::new_gc(Id::new(1, 10), 20); + doc_store.add_node(struct_info1).unwrap(); + doc_store.add_node(struct_info2.clone()).unwrap(); + + assert_eq!(doc_store.get_node(Id::new(1, 25)), Some(struct_info2)); + }); + + loom_model!({ + let doc_store = DocStore::with_client(1); + + assert_eq!(doc_store.get_node(Id::new(1, 0)), None); + }); + + loom_model!({ + let mut doc_store = DocStore::with_client(1); + let struct_info1 = Node::new_gc(Id::new(1, 0), 10); + let struct_info2 = Node::new_gc(Id::new(1, 10), 20); + doc_store.add_node(struct_info1).unwrap(); + doc_store.add_node(struct_info2).unwrap(); + + assert_eq!(doc_store.get_node(Id::new(1, 35)), None); + }); + } + + #[test] + fn test_split_node_at() { + loom_model!({ + let node = Node::Item(Somr::new( + ItemBuilder::new() + .id((1, 0).into()) + .content(Content::String(String::from("octo"))) + .build(), + )); + let mut list = VecDeque::from([node.clone()]); + + let (left, right) = DocStore::split_node_at(&mut list, 0, 2).unwrap(); + + assert_eq!( + node.as_item().ptr().as_ptr() as usize, + left.as_item().ptr().as_ptr() as usize + ); + assert_eq!(node.len(), 2); + assert_eq!(left.len(), 2); + assert_eq!(right.len(), 2); + assert_eq!(left.right(), Some(right.clone())); + assert_eq!(right.left(), Some(left)); + }); + } + + #[test] + fn test_split_and_get() { + loom_model!({ + let mut doc_store = DocStore::with_client(1); + let struct_info1 = Node::Item(Somr::new( + ItemBuilder::new() + .id((1, 0).into()) + .content(Content::String(String::from("octo"))) + .build(), + )); + + let struct_info2 = Node::Item(Somr::new( + ItemBuilder::new() + .id((1, struct_info1.len()).into()) + .left_id(Some(struct_info1.id())) + .content(Content::String(String::from("base"))) + .build(), + )); + let s1_ref = struct_info1.clone(); + doc_store.add_node(struct_info1).unwrap(); + doc_store.add_node(struct_info2).unwrap(); + + let s1 = doc_store.get_node(Id::new(1, 0)).unwrap(); + assert_eq!(s1, s1_ref); + let left = doc_store.split_at_and_get_left((1, 1)).unwrap(); + assert_eq!(left.len(), 2); // octo => oc_to + + // s1 used to be (1, 4), but it actually ref of first item in store, so now it + // should be (1, 2) + assert_eq!( + s1, left, + "doc internal mutation should not modify the pointer" + ); + let right = doc_store.split_at_and_get_right((1, 5)).unwrap(); + assert_eq!(right.len(), 3); // base => b_ase + }); + } + + #[test] + fn should_replace_gc_item_with_content_deleted() { + loom_model!({ + let item1 = ItemBuilder::new() + .id((1, 0).into()) + .content(Content::String(String::from("octo"))) + .build(); + let item2 = ItemBuilder::new() + .id((1, 4).into()) + .content(Content::String(String::from("base"))) + .build(); + + item1.delete(); + + let mut store = DocStore::with_client(1); + + store.add_node(Node::Item(Somr::new(item1))).unwrap(); + store.add_node(Node::Item(Somr::new(item2))).unwrap(); + store.delete_set.add_range(1, 0..4); + + store.gc_delete_set().unwrap(); + + assert_eq!( + &store + .get_node((1, 0)) + .unwrap() + .as_item() + .get() + .unwrap() + .content, + &Content::Deleted(4) + ); + }); + } + + #[test] + fn should_gc_type_items() { + loom_model!({ + let doc = DocOptions::new().with_client_id(1).build(); + + let mut arr = doc.get_or_create_array("arr").unwrap(); + let mut text = doc.create_text().unwrap(); + + arr.insert(0, Value::from(text.clone())).unwrap(); + + text.insert(0, "hello world").unwrap(); + text.remove(5, 6).unwrap(); + + arr.remove(0, 1).unwrap(); + let mut store = doc.store.write().unwrap(); + store.gc_delete_set().unwrap(); + + assert_eq!(arr.len(), 0); + assert_eq!( + &store + .get_node((1, 0)) + .unwrap() + .as_item() + .get() + .unwrap() + .content, + &Content::Deleted(1) + ); + + assert_eq!( + store.get_node((1, 1)).unwrap(), // "hello" GCd + Node::new_gc((1, 1).into(), 5) + ); + + assert_eq!( + store.get_node((1, 7)).unwrap(), // " world" GCd + Node::new_gc((1, 6).into(), 6) + ); + }); + } + + #[test] + fn should_gc_multi_client_ds() { + loom_model!({ + let bin = { + let doc = DocOptions::new().with_client_id(1).build(); + let mut pages = doc.get_or_create_map("pages").unwrap(); + let page1 = doc.create_text().unwrap(); + let mut page1_ref = page1.clone(); + pages + .insert("page1".to_string(), Value::from(page1)) + .unwrap(); + page1_ref.insert(0, "hello").unwrap(); + doc.encode_update_v1().unwrap() + }; + + let mut doc = DocOptions::new().with_client_id(2).build(); + doc.apply_update_from_binary_v1(bin).unwrap(); + let mut pages = doc.get_or_create_map("pages").unwrap(); + if let Value::Text(mut page1) = pages.get("page1").unwrap() { + page1.insert(5, " world").unwrap(); + } + + pages.remove("page1"); + + let mut store = doc.store.write().unwrap(); + store.gc_delete_set().unwrap(); + + assert_eq!( + &store + .get_node((1, 0)) + .unwrap() + .as_item() + .get() + .unwrap() + .content, + &Content::Deleted(1) + ); + + assert_eq!( + store.get_node((1, 1)).unwrap(), // "hello" GCd + Node::new_gc((1, 1).into(), 5) + ); + + assert_eq!( + store.get_node((2, 0)).unwrap(), // " world" GCd + Node::new_gc((2, 0).into(), 6) + ); + }); + } + + #[test] + fn should_merge_same_sibling_items() { + loom_model!({ + let mut store = DocStore::with_client(1); + store.items.insert( + 1, + VecDeque::from([ + Node::new_gc((1, 0).into(), 2), + Node::new_gc((1, 2).into(), 2), + Node::new_skip((1, 4).into(), 2), + Node::Item(Somr::new( + ItemBuilder::new() + .id((1, 6).into()) + .content(Content::String(String::from("hello"))) + .build(), + )), + // actually not mergable, due to runtime continuous check + // will cover it in [test_merge_same_sibling_items2] + Node::Item(Somr::new( + ItemBuilder::new() + .id((1, 11).into()) + .content(Content::String(String::from("world"))) + .left_id(Some((1, 11).into())) + .build(), + )), + ]), + ); + + store.make_continuous(); + + assert_eq!(store.items.get(&1).unwrap().len(), 4); + }); + } + + #[test] + fn test_merge_same_sibling_items2() { + loom_model!({ + let doc = Doc::new(); + + let mut text = doc.get_or_create_text("text").unwrap(); + text.insert(0, "a").unwrap(); + text.insert(1, "b").unwrap(); + text.insert(2, "c").unwrap(); + text.insert(3, ", hello").unwrap(); + + assert_eq!(text.to_string(), "abc, hello"); + + let mut store = doc.store.write().unwrap(); + assert_eq!(store.items.get(&1).unwrap().len(), 4); + store.make_continuous(); + assert_eq!(store.items.get(&1).unwrap().len(), 1); + assert_eq!(text.to_string(), "abc, hello"); + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/types/array.rs b/packages/common/y-octo/core/src/doc/types/array.rs new file mode 100644 index 0000000000..6581895a20 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/array.rs @@ -0,0 +1,216 @@ +use super::*; + +impl_type!(Array); + +impl ListType for Array {} + +pub struct ArrayIter<'a>(ListIterator<'a>); + +impl Iterator for ArrayIter<'_> { + type Item = Value; + + fn next(&mut self) -> Option { + for item in self.0.by_ref() { + if let Some(item) = item.get() { + if item.countable() { + return Some(Value::from(&item.content)); + } + } + } + + None + } +} + +impl Array { + #[inline] + pub fn len(&self) -> u64 { + self.content_len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn get(&self, index: u64) -> Option { + let (item, offset) = self.get_item_at(index)?; + + if let Some(item) = item.get() { + // TODO: rewrite to content.read(&mut [Any]) + return match &item.content { + Content::Any(any) => return any.get(offset as usize).map(|any| Value::Any(any.clone())), + _ => Some(Value::from(&item.content)), + }; + } + + None + } + + pub fn iter(&self) -> ArrayIter { + ArrayIter(self.iter_item()) + } + + pub fn push>(&mut self, val: V) -> JwstCodecResult { + self.insert(self.len(), val) + } + + pub fn insert>(&mut self, idx: u64, val: V) -> JwstCodecResult { + self.insert_at(idx, val.into().into()) + } + + pub fn remove(&mut self, idx: u64, len: u64) -> JwstCodecResult { + self.remove_at(idx, len) + } +} + +impl serde::Serialize for Array { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeSeq; + + let mut seq = serializer.serialize_seq(Some(self.len() as usize))?; + + for item in self.iter() { + seq.serialize_element(&item)?; + } + seq.end() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_yarray_insert() { + let options = DocOptions::default(); + + loom_model!({ + let doc = Doc::with_options(options.clone()); + let mut array = doc.get_or_create_array("abc").unwrap(); + + array.insert(0, " ").unwrap(); + array.insert(0, "Hello").unwrap(); + array.insert(2, "World").unwrap(); + + assert_eq!( + array.get(0).unwrap(), + Value::Any(Any::String("Hello".into())) + ); + assert_eq!(array.get(1).unwrap(), Value::Any(Any::String(" ".into()))); + assert_eq!( + array.get(2).unwrap(), + Value::Any(Any::String("World".into())) + ); + }); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_ytext_equal() { + use yrs::{Options, Text, Transact}; + let options = DocOptions::default(); + let yrs_options = Options::default(); + + loom_model!({ + let doc = yrs::Doc::with_options(yrs_options.clone()); + let array = doc.get_or_insert_text("abc"); + + let mut trx = doc.transact_mut(); + array.insert(&mut trx, 0, " "); + array.insert(&mut trx, 0, "Hello"); + array.insert(&mut trx, 6, "World"); + array.insert(&mut trx, 11, "!"); + let buffer = trx.encode_update_v1(); + + let mut decoder = RawDecoder::new(&buffer); + let update = Update::read(&mut decoder).unwrap(); + + let mut doc = Doc::with_options(options.clone()); + doc.apply_update(update).unwrap(); + let array = doc.get_or_create_array("abc").unwrap(); + + assert_eq!( + array.get(0).unwrap(), + Value::Any(Any::String("Hello".into())) + ); + assert_eq!(array.get(5).unwrap(), Value::Any(Any::String(" ".into()))); + assert_eq!( + array.get(6).unwrap(), + Value::Any(Any::String("World".into())) + ); + assert_eq!(array.get(11).unwrap(), Value::Any(Any::String("!".into()))); + }); + + let options = DocOptions::default(); + let yrs_options = Options::default(); + + loom_model!({ + let doc = yrs::Doc::with_options(yrs_options.clone()); + let array = doc.get_or_insert_text("abc"); + + let mut trx = doc.transact_mut(); + array.insert(&mut trx, 0, "Hello"); + array.insert(&mut trx, 5, " "); + array.insert(&mut trx, 6, "World"); + array.insert(&mut trx, 11, "!"); + let buffer = trx.encode_update_v1(); + + let mut decoder = RawDecoder::new(&buffer); + let update = Update::read(&mut decoder).unwrap(); + + let mut doc = Doc::with_options(options.clone()); + doc.apply_update(update).unwrap(); + let array = doc.get_or_create_array("abc").unwrap(); + + assert_eq!( + array.get(0).unwrap(), + Value::Any(Any::String("Hello".into())) + ); + assert_eq!(array.get(5).unwrap(), Value::Any(Any::String(" ".into()))); + assert_eq!( + array.get(6).unwrap(), + Value::Any(Any::String("World".into())) + ); + assert_eq!(array.get(11).unwrap(), Value::Any(Any::String("!".into()))); + }); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_yrs_array_decode() { + use yrs::{Array, Transact}; + + loom_model!({ + let update = { + let doc = yrs::Doc::new(); + let array = doc.get_or_insert_array("abc"); + let mut trx = doc.transact_mut(); + + array.insert(&mut trx, 0, "hello"); + array.insert(&mut trx, 1, "world"); + array.insert(&mut trx, 1, " "); + + trx.encode_update_v1() + }; + let doc = Doc::try_from_binary_v1_with_options( + update.clone(), + DocOptions { + guid: String::from("1"), + client_id: 1, + gc: true, + }, + ) + .unwrap(); + let arr = doc.get_or_create_array("abc").unwrap(); + + assert_eq!( + arr.get(2).unwrap(), + Value::Any(Any::String("world".to_string())) + ) + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/types/list/iterator.rs b/packages/common/y-octo/core/src/doc/types/list/iterator.rs new file mode 100644 index 0000000000..72e217601f --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/list/iterator.rs @@ -0,0 +1,23 @@ +use super::*; + +pub(crate) struct ListIterator<'a> { + pub(super) _lock: RwLockReadGuard<'a, YType>, + pub(super) cur: Somr, +} + +impl Iterator for ListIterator<'_> { + type Item = Somr; + + fn next(&mut self) -> Option { + while let Some(item) = self.cur.clone().get() { + let cur = std::mem::replace(&mut self.cur, item.right.clone()); + if item.deleted() { + continue; + } + + return Some(cur); + } + + None + } +} diff --git a/packages/common/y-octo/core/src/doc/types/list/mod.rs b/packages/common/y-octo/core/src/doc/types/list/mod.rs new file mode 100644 index 0000000000..9052b39ce0 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/list/mod.rs @@ -0,0 +1,241 @@ +mod iterator; +mod search_marker; + +pub(crate) use iterator::ListIterator; +pub(crate) use search_marker::MarkerList; + +use super::*; + +pub(crate) struct ItemPosition { + pub parent: YTypeRef, + pub left: ItemRef, + pub right: ItemRef, + pub index: u64, + pub offset: u64, +} + +impl ItemPosition { + pub fn forward(&mut self) { + if let Some(right) = self.right.get() { + if !right.deleted() { + self.index += right.len(); + } + + self.left = self.right.clone(); + self.right = right.right.clone(); + } else { + // FAIL + } + } + + /// we found a position cursor point in between a splitable item, + /// we need to split the item by the offset. + /// + /// before: + /// --------------------------------- + /// ^left ^right + /// ^offset + /// after: + /// --------------------------------- + /// ^left ^right + pub fn normalize(&mut self, store: &mut DocStore) -> JwstCodecResult { + if self.offset > 0 { + debug_assert!(self.left.is_some()); + if let Some(left) = self.left.get() { + let (left, right) = store.split_node(left.id, self.offset)?; + self.left = left.as_item(); + self.right = right.as_item(); + self.index += self.offset; + self.offset = 0; + } + } + + Ok(()) + } +} + +pub(crate) trait ListType: AsInner { + #[inline(always)] + fn content_len(&self) -> u64 { + self.as_inner().ty().unwrap().len + } + + fn iter_item(&self) -> ListIterator { + let inner = self.as_inner().ty().unwrap(); + ListIterator { + cur: inner.start.clone(), + _lock: inner, + } + } + + fn find_pos(&self, inner: &YType, index: u64) -> Option { + let mut remaining = index; + let start = inner.start.clone(); + + let mut pos = ItemPosition { + parent: self.as_inner().clone(), + left: Somr::none(), + right: start, + index: 0, + offset: 0, + }; + + if pos.right.is_none() { + return Some(pos); + } + + if let Some(markers) = &inner.markers { + if let Some(marker) = markers.find_marker(inner, index) { + if marker.index > remaining { + remaining = 0 + } else { + remaining -= marker.index; + } + pos.index = marker.index; + pos.left = marker + .ptr + .get() + .map(|ptr| ptr.left.clone()) + .unwrap_or_default(); + pos.right = marker.ptr; + } + }; + + while remaining > 0 { + if let Some(item) = pos.right.get() { + if !item.deleted() { + let content_len = item.len(); + if remaining < content_len { + pos.offset = remaining; + remaining = 0; + } else { + pos.index += content_len; + remaining -= content_len; + } + } + + pos.left = pos.right.clone(); + pos.right = item.right.clone(); + } else { + return None; + } + } + + Some(pos) + } + + fn insert_at(&mut self, index: u64, content: Content) -> JwstCodecResult { + if index > self.content_len() { + return Err(JwstCodecError::IndexOutOfBound(index)); + } + + if let Some((mut store, mut ty)) = self.as_inner().write() { + if let Some(mut pos) = self.find_pos(&ty, index) { + pos.normalize(&mut store)?; + Self::insert_after(&mut ty, &mut store, pos, content)?; + } + } else { + return Err(JwstCodecError::DocReleased); + } + + Ok(()) + } + + fn insert_after( + ty: &mut YType, + store: &mut DocStore, + pos: ItemPosition, + content: Content, + ) -> JwstCodecResult { + if let Some(markers) = &ty.markers { + markers.update_marker_changes(pos.index, content.clock_len() as i64); + } + + let item = store.create_item( + content, + pos.left.clone(), + pos.right.clone(), + Some(Parent::Type(pos.parent)), + None, + ); + + store.integrate(Node::Item(item), 0, Some(ty))?; + + Ok(()) + } + + fn get_item_at(&self, index: u64) -> Option<(Somr, u64)> { + if index >= self.content_len() { + return None; + } + + let ty = self.as_inner().ty().unwrap(); + + if let Some(pos) = self.find_pos(&ty, index) { + if pos.offset == 0 { + return Some((pos.right, 0)); + } else { + return Some((pos.left, pos.offset)); + } + } + + None + } + + fn remove_at(&mut self, idx: u64, len: u64) -> JwstCodecResult { + if len == 0 { + return Ok(()); + } + + if idx >= self.content_len() { + return Err(JwstCodecError::IndexOutOfBound(idx)); + } + + if let Some((mut store, mut ty)) = self.as_inner().write() { + if let Some(pos) = self.find_pos(&ty, idx) { + Self::remove_after(&mut ty, &mut store, pos, len)?; + } + } else { + return Err(JwstCodecError::DocReleased); + } + + Ok(()) + } + + fn remove_after( + ty: &mut YType, + store: &mut DocStore, + mut pos: ItemPosition, + len: u64, + ) -> JwstCodecResult { + pos.normalize(store)?; + + let mut remaining = len; + + while remaining > 0 { + if let Some(item) = pos.right.get() { + if !item.deleted() { + let content_len = item.len(); + if remaining < content_len { + store.split_node(item.id, remaining)?; + remaining = 0; + } else { + remaining -= content_len; + } + + store.delete_item(item, Some(ty)); + } + + pos.forward(); + } else { + break; + } + } + + if let Some(markers) = &ty.markers { + markers.update_marker_changes(pos.index, -((len - remaining) as i64)); + } + + Ok(()) + } +} diff --git a/packages/common/y-octo/core/src/doc/types/list/search_marker.rs b/packages/common/y-octo/core/src/doc/types/list/search_marker.rs new file mode 100644 index 0000000000..5fc9a1797c --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/list/search_marker.rs @@ -0,0 +1,340 @@ +use std::{ + cell::RefCell, + cmp::max, + collections::VecDeque, + ops::{Deref, DerefMut}, +}; + +use super::*; + +const MAX_SEARCH_MARKER: usize = 80; + +#[derive(Clone, Debug)] +pub(crate) struct SearchMarker { + pub ptr: Somr, + pub index: u64, +} + +impl SearchMarker { + fn new(ptr: Somr, index: u64) -> Self { + SearchMarker { ptr, index } + } + + fn overwrite_marker(&mut self, ptr: Somr, index: u64) { + self.ptr = ptr; + self.index = index; + } +} + +unsafe impl Sync for MarkerList {} + +/// in yjs, a timestamp field is used to sort markers and the oldest marker is +/// deleted once the limit is reached. this was designed for optimization +/// purposes for v8. In Rust, we can simply use a [VecDeque] and trust the +/// compiler to optimize. the [VecDeque] can naturally maintain the insertion +/// order, allowing us to know which marker is the oldest without using an extra +/// timestamp field. +/// +/// NOTE: +/// A [MarkerList] is always belonging to a [YType], +/// which means whenever [MakerList] is used, we actually have a [YType] +/// instance behind [RwLock] guard already, so it's safe to make the list +/// internal mutable. +#[derive(Debug)] +pub(crate) struct MarkerList(RefCell>); + +impl Deref for MarkerList { + type Target = RefCell>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for MarkerList { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Default for MarkerList { + fn default() -> Self { + Self::new() + } +} + +impl MarkerList { + pub fn new() -> Self { + MarkerList(RefCell::new(VecDeque::new())) + } + + // mark pos and push to the end of the linked list + fn mark_position( + list: &mut VecDeque, + ptr: Somr, + index: u64, + ) -> Option { + if list.len() >= MAX_SEARCH_MARKER { + let mut oldest_marker = list.pop_front().unwrap(); + oldest_marker.overwrite_marker(ptr, index); + list.push_back(oldest_marker); + } else { + let marker = SearchMarker::new(ptr, index); + list.push_back(marker); + } + list.back().cloned() + } + + // update mark position if the index is within the range of the marker + pub fn update_marker_changes(&self, index: u64, len: i64) { + let mut list = self.borrow_mut(); + + for marker in list.iter_mut() { + if len > 0 { + while let Some(ptr) = marker.ptr.get() { + if !ptr.indexable() { + let left_ref = ptr.left.clone(); + if let Some(left) = left_ref.get() { + if left.indexable() { + marker.index -= left.len(); + } + marker.ptr = left_ref; + } else { + // remove marker + marker.index = 0; + break; + } + } else { + break; + } + } + } + + if marker.ptr.is_some() && (index < marker.index || (len > 0 && index == marker.index)) { + marker.index = max(index as i64, marker.index as i64 + len) as u64; + } + } + + list.retain(|marker| marker.index > 0); + } + + // find and return the marker that is closest to the index + pub fn find_marker(&self, parent: &YType, index: u64) -> Option { + if parent.start.is_none() || index == 0 { + return None; + } + + let mut list = self.borrow_mut(); + + let marker = list + .iter_mut() + .min_by_key(|m| (index as i64 - m.index as i64).abs()); + + let mut marker_index = marker.as_ref().map(|m| m.index).unwrap_or(0); + + let mut item_ptr = marker + .as_ref() + .map(|m| m.ptr.clone()) + .unwrap_or_else(|| parent.start.clone()); + + // TODO: this logic here is a bit messy + // i think it can be implemented with more streamlined code, and then optimized + { + // iterate to the right if possible + while let Some(item) = item_ptr.clone().get() { + if marker_index >= index { + break; + } + + let right_ref: ItemRef = item.right.clone(); + if right_ref.is_some() { + if item.indexable() { + if index < marker_index + item.len() { + break; + } + + marker_index += item.len(); + } + item_ptr = right_ref; + } else { + break; + } + } + + // iterate to the left if necessary (might be that marker_index > index) + while let Some(item) = item_ptr.clone().get() { + if marker_index <= index { + break; + } + + let left_ref: ItemRef = item.left.clone(); + if let Some(left) = left_ref.get() { + if left.indexable() { + marker_index -= left.len(); + } + item_ptr = left_ref; + } else { + break; + } + } + + // we want to make sure that item_ptr can't be merged with left, because that + // would screw up everything in that case just return what we have + // (it is most likely the best marker anyway) iterate to left until + // item_ptr can't be merged with left + while let Some(item) = item_ptr.clone().get() { + let left_ref: ItemRef = item.left.clone(); + if let Some(left) = left_ref.get() { + if left.id.client == item.id.client && left.id.clock + left.len() == item.id.clock { + if left.indexable() { + marker_index -= left.len(); + } + item_ptr = left_ref; + continue; + } + break; + } else { + break; + } + } + } + + match marker { + Some(marker) + if (marker.index as f64 - marker_index as f64).abs() + < parent.len as f64 / MAX_SEARCH_MARKER as f64 => + { + // adjust existing marker + marker.overwrite_marker(item_ptr, marker_index); + Some(marker.clone()) + } + _ => { + // create new marker + Self::mark_position(&mut list, item_ptr, marker_index) + } + } + } + + #[allow(dead_code)] + pub fn get_last_marker(&self) -> Option { + self.borrow().back().cloned() + } + + pub fn replace_marker(&self, raw: Somr, new: Somr, len_shift: i64) { + let mut list = self.borrow_mut(); + + for marker in list.iter_mut() { + if marker.ptr == raw { + marker.ptr = new.clone(); + marker.index = ((marker.index as i64) + len_shift) as u64; + } + } + } +} + +#[cfg(test)] +mod tests { + #[cfg(not(loom))] + use rand::{Rng, SeedableRng}; + #[cfg(not(loom))] + use rand_chacha::ChaCha20Rng; + use yrs::{Array, Options, Transact}; + + use super::*; + + #[test] + fn test_marker_list() { + let options = DocOptions::default(); + let yrs_options = Options::default(); + + loom_model!({ + let (client_id, buffer) = if cfg!(miri) { + let doc = Doc::with_options(options.clone()); + let mut array = doc.get_or_create_array("abc").unwrap(); + + array.insert(0, " ").unwrap(); + array.insert(0, "Hello").unwrap(); + array.insert(2, "World").unwrap(); + + (doc.client(), doc.encode_update_v1().unwrap()) + } else { + let doc = yrs::Doc::with_options(yrs_options.clone()); + let array = doc.get_or_insert_array("abc"); + + let mut trx = doc.transact_mut(); + array.insert(&mut trx, 0, " "); + array.insert(&mut trx, 0, "Hello"); + array.insert(&mut trx, 2, "World"); + + (doc.client_id(), trx.encode_update_v1()) + }; + + let mut decoder = RawDecoder::new(&buffer); + let update = Update::read(&mut decoder).unwrap(); + + let mut doc = Doc::with_options(options.clone()); + doc.apply_update(update).unwrap(); + let array = doc.get_or_create_array("abc").unwrap(); + + let marker_list = MarkerList::new(); + + let marker = marker_list.find_marker(&array.0.ty().unwrap(), 8).unwrap(); + + assert_eq!(marker.index, 2); + assert_eq!( + marker.ptr, + doc + .store + .read() + .unwrap() + .get_node(Id::new(client_id, 2)) + .unwrap() + .as_item() + ); + }); + } + + #[test] + fn test_search_marker_flaky() { + let options = DocOptions::default(); + + loom_model!({ + let doc = Doc::with_options(options.clone()); + let mut text = doc.get_or_create_text("test").unwrap(); + text.insert(0, "0").unwrap(); + text.insert(1, "1").unwrap(); + text.insert(0, "0").unwrap(); + }); + } + + #[cfg(not(loom))] + fn search_with_seed(seed: u64) { + let rand = ChaCha20Rng::seed_from_u64(seed); + let iteration = 20; + + let doc = Doc::with_client(1); + let mut text = doc.get_or_create_text("test").unwrap(); + text.insert(0, "This is a string with length 32.").unwrap(); + let mut len = text.len(); + + for i in 0..iteration { + let mut rand: ChaCha20Rng = rand.clone(); + let pos = rand.random_range(0..text.len()); + let str = format!("hello {i}"); + len += str.len() as u64; + text.insert(pos, str).unwrap(); + } + + assert_eq!(text.len(), len); + assert_eq!(text.to_string().len() as u64, len); + } + + #[test] + #[cfg(not(loom))] + fn test_marker_list_with_seed() { + search_with_seed(785590655803394607); + search_with_seed(12958877733367615); + search_with_seed(71776330571528794); + search_with_seed(2207805473582911); + } +} diff --git a/packages/common/y-octo/core/src/doc/types/map.rs b/packages/common/y-octo/core/src/doc/types/map.rs new file mode 100644 index 0000000000..47188e572d --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/map.rs @@ -0,0 +1,326 @@ +use std::{collections::hash_map::Iter, rc::Rc}; + +use super::*; +use crate::{ + doc::{AsInner, Node, Parent, YTypeRef}, + impl_type, JwstCodecResult, +}; + +impl_type!(Map); + +pub(crate) trait MapType: AsInner { + fn _insert>(&mut self, key: String, value: V) -> JwstCodecResult { + if let Some((mut store, mut ty)) = self.as_inner().write() { + let left = ty.map.get(&SmolStr::new(&key)).cloned(); + + let item = store.create_item( + value.into().into(), + left.unwrap_or(Somr::none()), + Somr::none(), + Some(Parent::Type(self.as_inner().clone())), + Some(SmolStr::new(key)), + ); + store.integrate(Node::Item(item), 0, Some(&mut ty))?; + } + + Ok(()) + } + + fn _get(&self, key: &str) -> Option { + self.as_inner().ty().and_then(|ty| { + ty.map.get(key).and_then(|item| { + if let Some(item) = item.get() { + if item.deleted() { + return None; + } + + Some(Value::from(&item.content)) + } else { + None + } + }) + }) + } + + fn _contains_key(&self, key: &str) -> bool { + if let Some(ty) = self.as_inner().ty() { + ty.map + .get(key) + .and_then(|item| item.get()) + .is_some_and(|item| !item.deleted()) + } else { + false + } + } + + fn _remove(&mut self, key: &str) { + if let Some((mut store, mut ty)) = self.as_inner().write() { + if let Some(item) = ty.map.get(key).cloned() { + if let Some(item) = item.get() { + store.delete_item(item, Some(&mut ty)); + } + } + } + } + + fn _len(&self) -> u64 { + self._keys().count() as u64 + } + + fn _iter(&self) -> EntriesInnerIterator { + let ty = self.as_inner().ty(); + + if let Some(ty) = ty { + let ty = Rc::new(ty); + + EntriesInnerIterator { + iter: Some(unsafe { &*Rc::as_ptr(&ty) }.map.iter()), + _lock: Some(ty), + } + } else { + EntriesInnerIterator { + _lock: None, + iter: None, + } + } + } + + fn _keys(&self) -> KeysIterator { + KeysIterator(self._iter()) + } + + fn _values(&self) -> ValuesIterator { + ValuesIterator(self._iter()) + } + + fn _entries(&self) -> EntriesIterator { + EntriesIterator(self._iter()) + } +} + +pub(crate) struct EntriesInnerIterator<'a> { + _lock: Option>>, + iter: Option>, +} + +pub struct KeysIterator<'a>(EntriesInnerIterator<'a>); +pub struct ValuesIterator<'a>(EntriesInnerIterator<'a>); +pub struct EntriesIterator<'a>(EntriesInnerIterator<'a>); + +impl<'a> Iterator for EntriesInnerIterator<'a> { + type Item = (&'a str, &'a Item); + + fn next(&mut self) -> Option { + if let Some(iter) = &mut self.iter { + for (k, v) in iter { + if let Some(item) = v.get() { + if !item.deleted() { + return Some((k.as_str(), item)); + } + } + } + + None + } else { + None + } + } +} + +impl<'a> Iterator for KeysIterator<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option { + self.0.next().map(|(k, _)| k) + } +} + +impl Iterator for ValuesIterator<'_> { + type Item = Value; + + fn next(&mut self) -> Option { + self.0.next().map(|(_, v)| Value::from(&v.content)) + } +} + +impl<'a> Iterator for EntriesIterator<'a> { + type Item = (&'a str, Value); + + fn next(&mut self) -> Option { + self.0.next().map(|(k, v)| (k, Value::from(&v.content))) + } +} + +impl MapType for Map {} + +impl Map { + #[inline(always)] + pub fn insert>(&mut self, key: String, value: V) -> JwstCodecResult { + self._insert(key, value) + } + + #[inline(always)] + pub fn get(&self, key: &str) -> Option { + self._get(key) + } + + #[inline(always)] + pub fn contains_key(&self, key: &str) -> bool { + self._contains_key(key) + } + + #[inline(always)] + pub fn remove(&mut self, key: &str) { + self._remove(key) + } + + #[inline(always)] + pub fn len(&self) -> u64 { + self._len() + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline(always)] + pub fn iter(&self) -> EntriesIterator { + self._entries() + } + + #[inline(always)] + pub fn entries(&self) -> EntriesIterator { + self._entries() + } + + #[inline(always)] + pub fn keys(&self) -> KeysIterator { + self._keys() + } + + #[inline(always)] + pub fn values(&self) -> ValuesIterator { + self._values() + } +} + +impl serde::Serialize for Map { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::SerializeMap; + + let mut map = serializer.serialize_map(Some(self.len() as usize))?; + for (key, value) in self.iter() { + map.serialize_entry(&key, &value)?; + } + map.end() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{loom_model, Any, Doc}; + + #[test] + fn test_map_basic() { + loom_model!({ + let doc = Doc::new(); + let mut map = doc.get_or_create_map("map").unwrap(); + map.insert("1".to_string(), "value").unwrap(); + assert_eq!( + map.get("1").unwrap(), + Value::Any(Any::String("value".to_string())) + ); + assert!(!map.contains_key("nonexistent_key")); + assert_eq!(map.len(), 1); + assert!(map.contains_key("1")); + map.remove("1"); + assert!(!map.contains_key("1")); + assert_eq!(map.len(), 0); + }); + } + + #[test] + fn test_map_equal() { + loom_model!({ + let doc = Doc::new(); + let mut map = doc.get_or_create_map("map").unwrap(); + map.insert("1".to_string(), "value").unwrap(); + map.insert("2".to_string(), false).unwrap(); + + let binary = doc.encode_update_v1().unwrap(); + let new_doc = Doc::try_from_binary_v1(binary).unwrap(); + let map = new_doc.get_or_create_map("map").unwrap(); + assert_eq!( + map.get("1").unwrap(), + Value::Any(Any::String("value".to_string())) + ); + assert_eq!(map.get("2").unwrap(), Value::Any(Any::False)); + assert_eq!(map.len(), 2); + }); + } + + #[test] + fn test_map_renew_value() { + loom_model!({ + let doc = Doc::new(); + let mut map = doc.get_or_create_map("map").unwrap(); + map.insert("1".to_string(), "value").unwrap(); + map.insert("1".to_string(), "value2").unwrap(); + assert_eq!( + map.get("1").unwrap(), + Value::Any(Any::String("value2".to_string())) + ); + assert_eq!(map.len(), 1); + }); + } + + #[test] + fn test_map_re_encode() { + loom_model!({ + let binary = { + let doc = Doc::new(); + let mut map = doc.get_or_create_map("map").unwrap(); + map.insert("1".to_string(), "value1").unwrap(); + map.insert("2".to_string(), "value2").unwrap(); + doc.encode_update_v1().unwrap() + }; + + { + let doc = Doc::try_from_binary_v1(binary).unwrap(); + let map = doc.get_or_create_map("map").unwrap(); + assert_eq!( + map.get("1").unwrap(), + Value::Any(Any::String("value1".to_string())) + ); + assert_eq!( + map.get("2").unwrap(), + Value::Any(Any::String("value2".to_string())) + ); + } + }); + } + + #[test] + fn test_map_iter() { + loom_model!({ + let doc = Doc::new(); + let mut map = doc.get_or_create_map("map").unwrap(); + map.insert("1".to_string(), "value1").unwrap(); + map.insert("2".to_string(), "value2").unwrap(); + let mut vec = map.entries().collect::>(); + + // hashmap iteration is in random order instead of insert order + vec.sort_by(|a, b| a.0.cmp(b.0)); + + assert_eq!( + vec, + vec![ + ("1", Value::Any(Any::String("value1".to_string()))), + ("2", Value::Any(Any::String("value2".to_string()))) + ] + ) + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/types/mod.rs b/packages/common/y-octo/core/src/doc/types/mod.rs new file mode 100644 index 0000000000..ccfcf24b02 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/mod.rs @@ -0,0 +1,376 @@ +mod array; +mod list; +mod map; +mod text; +mod value; +mod xml; + +use std::{collections::hash_map::Entry, sync::Weak}; + +pub use array::*; +use list::*; +pub use map::*; +pub use text::*; +pub use value::*; +pub use xml::*; + +use super::{ + store::{StoreRef, WeakStoreRef}, + *, +}; +use crate::{ + sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, + Item, JwstCodecError, JwstCodecResult, +}; + +#[derive(Debug, Default)] +pub(crate) struct YType { + pub start: Somr, + pub item: Somr, + pub map: HashMap>, + pub len: u64, + /// The tag name of XMLElement and XMLHook type + pub name: Option, + /// The name of the type that directly belongs the store. + pub root_name: Option, + kind: YTypeKind, + pub markers: Option, +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct YTypeRef { + pub store: WeakStoreRef, + pub inner: Somr>, +} + +impl PartialEq for YType { + fn eq(&self, other: &Self) -> bool { + self.root_name == other.root_name + || (self.start.is_some() && self.start == other.start) + || self.map == other.map + } +} + +impl PartialEq for YTypeRef { + fn eq(&self, other: &Self) -> bool { + self.inner.ptr_eq(&other.inner) + || match (self.ty(), other.ty()) { + (Some(l), Some(r)) => *l == *r, + (None, None) => true, + _ => false, + } + } +} + +impl YType { + pub fn new(kind: YTypeKind, tag_name: Option) -> Self { + YType { + kind, + name: tag_name, + ..YType::default() + } + } + + pub fn kind(&self) -> YTypeKind { + self.kind + } + + pub fn set_kind(&mut self, kind: YTypeKind) -> JwstCodecResult { + std::debug_assert!(kind != YTypeKind::Unknown); + + if self.kind() != kind { + if self.kind == YTypeKind::Unknown { + self.kind = kind; + } else { + return Err(JwstCodecError::TypeCastError(kind.as_str())); + } + } + + Ok(()) + } +} + +impl YTypeRef { + pub fn new(kind: YTypeKind, tag_name: Option) -> Self { + Self { + inner: Somr::new(RwLock::new(YType::new(kind, tag_name))), + store: Weak::new(), + } + } + + pub fn ty(&self) -> Option> { + self.inner.get().and_then(|ty| ty.read().ok()) + } + + pub fn ty_mut(&self) -> Option> { + self.inner.get().and_then(|ty| ty.write().ok()) + } + + #[allow(dead_code)] + pub fn store<'a>(&self) -> Option> { + if let Some(store) = self.store.upgrade() { + let ptr = unsafe { &*Arc::as_ptr(&store) }; + + Some(ptr.read().unwrap()) + } else { + None + } + } + + pub fn store_mut<'a>(&self) -> Option> { + if let Some(store) = self.store.upgrade() { + let ptr = unsafe { &*Arc::as_ptr(&store) }; + + Some(ptr.write().unwrap()) + } else { + None + } + } + + #[allow(dead_code)] + pub fn read(&self) -> Option<(RwLockReadGuard, RwLockReadGuard)> { + self + .store() + .and_then(|store| self.ty().map(|ty| (store, ty))) + } + + pub fn write(&self) -> Option<(RwLockWriteGuard, RwLockWriteGuard)> { + self + .store_mut() + .and_then(|store| self.ty_mut().map(|ty| (store, ty))) + } +} + +pub(crate) struct YTypeBuilder { + store: StoreRef, + /// The tag name of XMLElement and XMLHook type + name: Option, + /// The name of the type that directly belongs the store. + root_name: Option, + kind: YTypeKind, +} + +impl YTypeBuilder { + pub fn new(store: StoreRef) -> Self { + Self { + store, + name: None, + root_name: None, + kind: YTypeKind::Unknown, + } + } + + pub fn with_kind(mut self, kind: YTypeKind) -> Self { + self.kind = kind; + + self + } + + pub fn set_name(mut self, name: String) -> Self { + self.root_name = Some(name); + + self + } + + #[allow(dead_code)] + pub fn set_tag_name(mut self, tag_name: String) -> Self { + self.name = Some(tag_name); + + self + } + + pub fn build_exists>(self) -> JwstCodecResult { + let store = self.store.read().unwrap(); + let ty = if let Some(root_name) = self.root_name { + match store.types.get(&root_name) { + Some(ty) => ty.clone(), + None => { + return Err(JwstCodecError::RootStructNotFound(root_name)); + } + } + } else { + return Err(JwstCodecError::TypeCastError("root_name is not set")); + }; + + drop(store); + + T::try_from(ty) + } + + pub fn build>(self) -> JwstCodecResult { + let mut store = self.store.write().unwrap(); + let ty = if let Some(root_name) = self.root_name { + match store.types.entry(root_name.clone()) { + Entry::Occupied(e) => e.get().clone(), + Entry::Vacant(e) => { + let inner = Somr::new(RwLock::new(YType { + kind: self.kind, + name: self.name, + root_name: Some(root_name), + markers: Self::markers(self.kind), + ..Default::default() + })); + + let ty = YTypeRef { + store: Arc::downgrade(&self.store), + inner, + }; + + let ty_ref = ty.clone(); + e.insert(ty); + + ty_ref + } + } + } else { + let inner = Somr::new(RwLock::new(YType { + kind: self.kind, + name: self.name, + root_name: self.root_name.clone(), + markers: Self::markers(self.kind), + ..Default::default() + })); + + let ty = YTypeRef { + store: Arc::downgrade(&self.store), + inner, + }; + + let ty_ref = ty.clone(); + + store + .dangling_types + .insert(ty.inner.ptr().as_ptr() as usize, ty); + + ty_ref + }; + + drop(store); + + T::try_from(ty) + } + + fn markers(kind: YTypeKind) -> Option { + match kind { + YTypeKind::Map => None, + _ => Some(MarkerList::new()), + } + } +} + +#[macro_export(local_inner_macros)] +macro_rules! impl_variants { + ({$($name: ident: $codec_ref: literal),*}) => { + #[derive(Debug, Clone, Copy, PartialEq, Default)] + pub enum YTypeKind { + $($name,)* + #[default] + Unknown, + } + + impl YTypeKind { + pub fn as_str(&self) -> &'static str { + match self { + $(YTypeKind::$name => std::stringify!($name),)* + YTypeKind::Unknown => "Unknown", + } + } + } + + impl From for YTypeKind { + fn from(value: u64) -> Self { + match value { + $($codec_ref => YTypeKind::$name,)* + _ => YTypeKind::Unknown, + } + } + } + + + impl From for u64 { + fn from(value: YTypeKind) -> Self { + std::debug_assert!(value != YTypeKind::Unknown); + match value { + $(YTypeKind::$name => $codec_ref,)* + _ => std::unreachable!(), + } + } + } + }; +} + +pub(crate) trait AsInner { + type Inner; + fn as_inner(&self) -> &Self::Inner; +} + +#[macro_export(local_inner_macros)] +macro_rules! impl_type { + ($name: ident) => { + #[derive(Debug, Clone, PartialEq)] + pub struct $name(pub(crate) super::YTypeRef); + unsafe impl Sync for $name {} + unsafe impl Send for $name {} + + impl $name { + pub(crate) fn new(inner: super::YTypeRef) -> Self { + Self(inner) + } + } + + impl super::AsInner for $name { + type Inner = super::YTypeRef; + + #[inline(always)] + fn as_inner(&self) -> &Self::Inner { + &self.0 + } + } + + impl TryFrom for $name { + type Error = $crate::JwstCodecError; + + fn try_from(value: super::YTypeRef) -> Result { + if let Some((_, mut inner)) = value.write() { + match inner.kind { + super::YTypeKind::$name => Ok($name::new(value.clone())), + super::YTypeKind::Unknown => { + inner.set_kind(super::YTypeKind::$name)?; + Ok($name::new(value.clone())) + } + _ => Err($crate::JwstCodecError::TypeCastError(std::stringify!( + $name + ))), + } + } else { + Err($crate::JwstCodecError::TypeCastError(std::stringify!( + $name + ))) + } + } + } + + impl $name { + pub(crate) fn from_unchecked(value: super::YTypeRef) -> Self { + $name::new(value.clone()) + } + } + + impl From<$name> for super::Value { + fn from(value: $name) -> Self { + Self::$name(value) + } + } + }; +} + +impl_variants!({ + Array: 0, + Map: 1, + Text: 2, + XMLElement: 3, + XMLFragment: 4, + XMLHook: 5, + XMLText: 6 + // Doc: 9? +}); diff --git a/packages/common/y-octo/core/src/doc/types/text.rs b/packages/common/y-octo/core/src/doc/types/text.rs new file mode 100644 index 0000000000..07bc05d802 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/text.rs @@ -0,0 +1,293 @@ +use std::fmt::Display; + +use super::list::ListType; +use crate::{impl_type, Content, JwstCodecResult}; + +impl_type!(Text); + +impl ListType for Text {} + +impl Text { + #[inline] + pub fn len(&self) -> u64 { + self.content_len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub fn insert(&mut self, char_index: u64, str: T) -> JwstCodecResult { + self.insert_at(char_index, Content::String(str.to_string())) + } + + #[inline] + pub fn remove(&mut self, char_index: u64, len: u64) -> JwstCodecResult { + self.remove_at(char_index, len) + } +} + +impl Display for Text { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.iter_item().try_for_each(|item| { + if let Content::String(str) = &item.get().unwrap().content { + write!(f, "{}", str) + } else { + Ok(()) + } + }) + } +} + +impl serde::Serialize for Text { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +#[cfg(test)] +mod tests { + use rand::{Rng, SeedableRng}; + use rand_chacha::ChaCha20Rng; + use yrs::{Options, Text, Transact}; + + #[cfg(not(loom))] + use crate::sync::{Arc, AtomicUsize, Ordering}; + use crate::{loom_model, sync::thread, Doc}; + + #[test] + fn test_manipulate_text() { + loom_model!({ + let doc = Doc::new(); + let mut text = doc.create_text().unwrap(); + + text.insert(0, "llo").unwrap(); + text.insert(0, "he").unwrap(); + text.insert(5, " world").unwrap(); + text.insert(6, "great ").unwrap(); + text.insert(17, '!').unwrap(); + + assert_eq!(text.to_string(), "hello great world!"); + assert_eq!(text.len(), 18); + + text.remove(4, 4).unwrap(); + assert_eq!(text.to_string(), "helleat world!"); + assert_eq!(text.len(), 14); + }); + } + + #[test] + #[cfg(not(loom))] + fn test_parallel_insert_text() { + let seed = rand::rng().random(); + let rand = ChaCha20Rng::seed_from_u64(seed); + let mut handles = Vec::new(); + + let doc = Doc::with_client(1); + let mut text = doc.get_or_create_text("test").unwrap(); + text.insert(0, "This is a string with length 32.").unwrap(); + + let added_len = Arc::new(AtomicUsize::new(32)); + + // parallel editing text + { + for i in 0..2 { + let mut text = text.clone(); + let mut rand = rand.clone(); + let len = added_len.clone(); + + handles.push(thread::spawn(move || { + for j in 0..10 { + let pos = rand.random_range(0..text.len()); + let string = format!("hello {}", i * j); + + text.insert(pos, &string).unwrap(); + + len.fetch_add(string.len(), Ordering::SeqCst); + } + })); + } + } + + // parallel editing doc + { + for i in 0..2 { + let doc = doc.clone(); + let mut rand = rand.clone(); + let len = added_len.clone(); + + handles.push(thread::spawn(move || { + let mut text = doc.get_or_create_text("test").unwrap(); + for j in 0..10 { + let pos = rand.random_range(0..text.len()); + let string = format!("hello doc{}", i * j); + + text.insert(pos, &string).unwrap(); + + len.fetch_add(string.len(), Ordering::SeqCst); + } + })); + } + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(text.to_string().len(), added_len.load(Ordering::SeqCst)); + assert_eq!(text.len(), added_len.load(Ordering::SeqCst) as u64); + } + + #[cfg(not(loom))] + fn parallel_ins_del_text(seed: u64, thread: i32, iteration: i32) { + let doc = Doc::with_client(1); + let rand = ChaCha20Rng::seed_from_u64(seed); + let mut text = doc.get_or_create_text("test").unwrap(); + text.insert(0, "This is a string with length 32.").unwrap(); + + let mut handles = Vec::new(); + let len = Arc::new(AtomicUsize::new(32)); + + for i in 0..thread { + let len = len.clone(); + let mut rand = rand.clone(); + let text = text.clone(); + handles.push(thread::spawn(move || { + for j in 0..iteration { + let len = len.clone(); + let mut text = text.clone(); + let ins = i % 2 == 0; + let pos = rand.random_range(0..16); + + if ins { + let str = format!("hello {}", i * j); + text.insert(pos, &str).unwrap(); + + len.fetch_add(str.len(), Ordering::SeqCst); + } else { + text.remove(pos, 6).unwrap(); + + len.fetch_sub(6, Ordering::SeqCst); + } + } + })); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(text.to_string().len(), len.load(Ordering::SeqCst)); + assert_eq!(text.len(), len.load(Ordering::SeqCst) as u64); + } + + #[test] + #[cfg(not(loom))] + fn test_parallel_ins_del_text() { + // cases that ever broken + // wrong left/right ref + parallel_ins_del_text(973078538, 2, 2); + parallel_ins_del_text(18414938500869652479, 2, 2); + } + + #[test] + fn loom_parallel_ins_del_text() { + let seed = rand::rng().random(); + let mut rand = ChaCha20Rng::seed_from_u64(seed); + let ranges = (0..20) + .map(|_| rand.random_range(0..16)) + .collect::>(); + + loom_model!({ + let doc = Doc::new(); + let mut text = doc.get_or_create_text("test").unwrap(); + text.insert(0, "This is a string with length 32.").unwrap(); + + // enough for loom + let handles = (0..2) + .map(|i| { + let text = text.clone(); + let ranges = ranges.clone(); + thread::spawn(move || { + let mut text = text.clone(); + let ins = i % 2 == 0; + let pos = ranges[i]; + + if ins { + let str = format!("hello {}", i); + text.insert(pos, &str).unwrap(); + } else { + text.remove(pos, 6).unwrap(); + } + }) + }) + .collect::>(); + + for handle in handles { + handle.join().unwrap(); + } + }); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_recover_from_yjs_encoder() { + let yrs_options = Options { + client_id: rand::random(), + guid: nanoid::nanoid!().into(), + ..Default::default() + }; + + loom_model!({ + let binary = { + let doc = yrs::Doc::with_options(yrs_options.clone()); + let text = doc.get_or_insert_text("greating"); + let mut trx = doc.transact_mut(); + text.insert(&mut trx, 0, "hello"); + text.insert(&mut trx, 5, " world!"); + text.remove_range(&mut trx, 11, 1); + + trx.encode_update_v1() + }; + // in loom loop + #[allow(clippy::needless_borrow)] + let doc = Doc::try_from_binary_v1(&binary).unwrap(); + let mut text = doc.get_or_create_text("greating").unwrap(); + + assert_eq!(text.to_string(), "hello world"); + + text.insert(6, "great ").unwrap(); + text.insert(17, '!').unwrap(); + assert_eq!(text.to_string(), "hello great world!"); + }); + } + + #[test] + fn test_recover_from_octobase_encoder() { + loom_model!({ + let binary = { + let doc = Doc::new(); + let mut text = doc.get_or_create_text("greating").unwrap(); + text.insert(0, "hello").unwrap(); + text.insert(5, " world!").unwrap(); + text.remove(11, 1).unwrap(); + + doc.encode_update_v1().unwrap() + }; + + let doc = Doc::try_from_binary_v1(binary).unwrap(); + let mut text = doc.get_or_create_text("greating").unwrap(); + + assert_eq!(text.to_string(), "hello world"); + + text.insert(6, "great ").unwrap(); + text.insert(17, '!').unwrap(); + assert_eq!(text.to_string(), "hello great world!"); + }); + } +} diff --git a/packages/common/y-octo/core/src/doc/types/value.rs b/packages/common/y-octo/core/src/doc/types/value.rs new file mode 100644 index 0000000000..b0cd6a5883 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/value.rs @@ -0,0 +1,159 @@ +use std::fmt::Display; + +use super::*; + +#[derive(Debug, PartialEq)] +pub enum Value { + Any(Any), + Doc(Doc), + Array(Array), + Map(Map), + Text(Text), + XMLElement(XMLElement), + XMLFragment(XMLFragment), + XMLHook(XMLHook), + XMLText(XMLText), +} + +impl Value { + pub fn to_any(&self) -> Option { + match self { + Value::Any(any) => Some(any.clone()), + _ => None, + } + } + + pub fn to_array(&self) -> Option { + match self { + Value::Array(array) => Some(array.clone()), + _ => None, + } + } + + pub fn to_map(&self) -> Option { + match self { + Value::Map(map) => Some(map.clone()), + _ => None, + } + } + + pub fn to_text(&self) -> Option { + match self { + Value::Text(text) => Some(text.clone()), + _ => None, + } + } + + pub fn from_vec>(el: Vec) -> Self { + Value::Any(Any::Array( + el.into_iter().map(|item| item.into()).collect::>(), + )) + } +} + +impl From<&Content> for Value { + fn from(value: &Content) -> Value { + match value { + Content::Any(any) => Value::Any(if any.len() == 1 { + any[0].clone() + } else { + Any::Array(any.clone()) + }), + Content::String(s) => Value::Any(Any::String(s.clone())), + Content::Json(json) => Value::Any(Any::Array( + json + .iter() + .map(|item| { + if let Some(s) = item { + Any::String(s.clone()) + } else { + Any::Undefined + } + }) + .collect::>(), + )), + Content::Binary(buf) => Value::Any(Any::Binary(buf.clone())), + Content::Embed(v) => Value::Any(v.clone()), + Content::Type(ty) => match ty.ty().unwrap().kind { + YTypeKind::Array => Value::Array(Array::from_unchecked(ty.clone())), + YTypeKind::Map => Value::Map(Map::from_unchecked(ty.clone())), + YTypeKind::Text => Value::Text(Text::from_unchecked(ty.clone())), + YTypeKind::XMLElement => Value::XMLElement(XMLElement::from_unchecked(ty.clone())), + YTypeKind::XMLFragment => Value::XMLFragment(XMLFragment::from_unchecked(ty.clone())), + YTypeKind::XMLHook => Value::XMLHook(XMLHook::from_unchecked(ty.clone())), + YTypeKind::XMLText => Value::XMLText(XMLText::from_unchecked(ty.clone())), + // actually unreachable + YTypeKind::Unknown => Value::Any(Any::Undefined), + }, + Content::Doc { guid: _, opts } => Value::Doc( + DocOptions::try_from(opts.clone()) + .expect("Failed to parse doc options") + .build(), + ), + Content::Format { .. } => unimplemented!(), + // actually unreachable + Content::Deleted(_) => Value::Any(Any::Undefined), + } + } +} + +impl From for Content { + fn from(value: Value) -> Self { + match value { + Value::Any(any) => Content::from(any), + Value::Doc(doc) => Content::Doc { + guid: doc.guid().to_owned(), + opts: Any::from(doc.options().clone()), + }, + Value::Array(v) => Content::Type(v.0), + Value::Map(v) => Content::Type(v.0), + Value::Text(v) => Content::Type(v.0), + Value::XMLElement(v) => Content::Type(v.0), + Value::XMLFragment(v) => Content::Type(v.0), + Value::XMLHook(v) => Content::Type(v.0), + Value::XMLText(v) => Content::Type(v.0), + } + } +} + +impl> From for Value { + fn from(value: T) -> Self { + Value::Any(value.into()) + } +} + +impl From for Value { + fn from(value: Doc) -> Self { + Value::Doc(value) + } +} + +impl Display for Value { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Value::Any(any) => write!(f, "{}", any), + Value::Text(text) => write!(f, "{}", text), + _ => write!(f, ""), + } + } +} + +impl serde::Serialize for Value { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + Self::Any(any) => any.serialize(serializer), + Self::Array(array) => array.serialize(serializer), + Self::Map(map) => map.serialize(serializer), + Self::Text(text) => text.serialize(serializer), + // Self::XMLElement(xml_element) => xml_element.serialize(serializer), + // Self::XMLFragment(xml_fragment) => xml_fragment.serialize(serializer), + // Self::XMLHook(xml_hook) => xml_hook.serialize(serializer), + // Self::XMLText(xml_text) => xml_text.serialize(serializer), + // Self::Doc(doc) => doc.serialize(serializer), + _ => serializer.serialize_none(), + } + } +} diff --git a/packages/common/y-octo/core/src/doc/types/xml.rs b/packages/common/y-octo/core/src/doc/types/xml.rs new file mode 100644 index 0000000000..49af7a0d17 --- /dev/null +++ b/packages/common/y-octo/core/src/doc/types/xml.rs @@ -0,0 +1,14 @@ +use super::list::ListType; +use crate::impl_type; + +impl_type!(XMLElement); +impl ListType for XMLElement {} + +impl_type!(XMLFragment); +impl ListType for XMLFragment {} + +impl_type!(XMLText); +impl ListType for XMLText {} + +impl_type!(XMLHook); +impl ListType for XMLHook {} diff --git a/packages/common/y-octo/core/src/doc/utils.rs b/packages/common/y-octo/core/src/doc/utils.rs new file mode 100644 index 0000000000..45e110b6eb --- /dev/null +++ b/packages/common/y-octo/core/src/doc/utils.rs @@ -0,0 +1,28 @@ +use super::*; + +pub fn encode_awareness_as_message(awareness: AwarenessStates) -> JwstCodecResult> { + let mut buffer = Vec::new(); + write_sync_message(&mut buffer, &SyncMessage::Awareness(awareness)) + .map_err(|e| JwstCodecError::InvalidWriteBuffer(e.to_string()))?; + + Ok(buffer) +} + +pub fn encode_update_as_message(update: Vec) -> JwstCodecResult> { + let mut buffer = Vec::new(); + write_sync_message(&mut buffer, &SyncMessage::Doc(DocMessage::Update(update))) + .map_err(|e| JwstCodecError::InvalidWriteBuffer(e.to_string()))?; + + Ok(buffer) +} + +pub fn merge_updates_v1, I: IntoIterator>( + updates: I, +) -> JwstCodecResult { + let updates = updates + .into_iter() + .map(Update::decode_v1) + .collect::>>()?; + + Ok(Update::merge(updates)) +} diff --git a/packages/common/y-octo/core/src/fixtures/basic.bin b/packages/common/y-octo/core/src/fixtures/basic.bin new file mode 100644 index 0000000000000000000000000000000000000000..1507a32d76347d5b2cce0feddbfe1f3a50e7ea2b GIT binary patch literal 5726 zcma)ATWlOx8TPDq*K6;F1i>rHi6fdKt?l(aanxF|-KdocvKki)p-MeFb7pt&?40SG zGhQzcYKb}G6ci8#Z$TGGcRJNX{tnb{4!lN;e)_Joh{m**Gf7FJru$x~G}U@HRa zoLT#GsxGXqk9TPtS^ccj=M*WY8e)ds^?WBH!H5~12qPM8rTn#rFw=Kj%HLcazIeppbLQ}i zg@@1S!!PC!&rz?BzL-DSEF71h?G5@}tAFD2t!AsaJlAS2oNqm6W}K8W+SuMvh*|xy zvzON{U%I?rQy_mV&*Pl(YQ-`>*N9JgB;d@9tXqi`He|?IEq&NV#S0v=XZ1@s87lKZ zNFXd(7l*{mD`#&!+iEr8pSkhIxSYBrmz5)8bF-=%VaJP^b=8FIM%DFvAH+T|Q^v)> zI`t_?Z|tlq{}@}H@rdq_ZO`ElmgS-0WKnBY{2j7h;R8q`;05(P=kOSrH+*jv@;$G2 z-OTg-e@R0_zgfunrs7<|TIOVro9WsBBc+<+_VGNo^NQOwOUKuAm&yM_jJK{+As8UL3`TIdup*kwii!hEL{dqPbj>o;AgUDV;B98<%nLZqXMFYWY zs}O@leYe|?Qj1bz287#IAquyxqhLs^g7D<5LLk192VzkH@uC6ZZR_;IH(vkeyEo7k z673R%8G2&w3B(035c$#I2o#nK3J1j~Ea{sad^idR#V8yci9%w{pa2^c$>AL<55tmz z;Rlw%;XUig4{p8oU9>@ZQ0sXFZ361jezx>LKB9r``Gg^8CHyJwL3OgJ@8dRw`OqVv zzQ?=hniSy8QrY){oyEy6=OJ5}nd#vO1U9|nlQ8VzAee^gHWNZ-jMxm+H`vhiBF3jf z$DMphZt+^Fo-wFxZ>ep|0mOxx^;YQ=a-fPM+7Do_t`{Cb*|MSRPN`71%bKz~rMz$t z%Tsqsh3dUi%Br{2ozvl)j0?t8p#E-%nPtVy?-OQzTAEy^phN&##An(7An4^Vx&pn1 zO%4y$^Ty)`#U7v69zQsy#}A4hsS^c{QF_KEGQsYW!T@(kJ+IT{ z$O|AdVc#E!fFu#(GI2wXrsT%EL`&|*Xj$~6Wgf*uY2%xtl~u7^Y3JM8o@(biqyMnb zv(G$@MAO=vMzSVAArN~y#Kp1C(W5lvQ(o{X#0ZZ=Fky~x%xMoK52}T0`4l^14J>TO zo-bx2A$Hp*)9BL#2YqPxND~cce9reFWP$)nyJ+k%ig4i29%4YPPo`o9{zVi9c0CN% zZPI>bYaKyE%Z?Yd2j*J&yG~};_W*-Q{kk-;)2%5e&u*k zv0o{Siv3D9DzX|$EU8A`AJWJLrICk;MqaLLZc&w}4@pN7ab)8Fv5?rFJCM9ecmew_ zP@o9_#f~H5DwWZnD#J4Rb7g#0EPKqQR7vT5JtvjY!^$TGv#!@cQ)nw1AR3ZDZ7lkX zM6PQ6_3H2$uT+Q6_?DiNp7Cn63DJOfpCZR|T>@_ikP&V(1S$f3LPQ94Lc0lPvoram za3b@dnN8Z*yVZp8`~*$;-RkiuzgsP&{BAW%dF6$ea-wr#VoEu`R)JPNcTGk5^@>Rh zzp6g{8Q4Nap`eV3qU0$5<2EBN!~zo$M+~tpjeQ3NluI%s#VHXf;jfMjlW}t_n>W6v z?I{tjkG)~3oFVxTk#EBzavELo8Eqmbmd`eE!sgs23NVi&f~c$Gxhc8~pMvP%f0zU| z*Z_$035K?V21Ghw5{2wX<>iG2oAM`cnhQt0fWcEZF+xU!1fwMOqguI5eJ7I*e_X3# zzmEq@P58ND;RoO6OiJE1hh%NGz#x z4yr@SnN`a9b=4^6H?^NwPq`uoLvB`x(EyUSOt~VcfGmYCOo;^c5?~2vn*d$F40c8R ziHI8!)=e{TM|@eyBwUBI<4#AUT(lMgN9xo*2EJ5A7CjH3ESl^zM~Qp^x5y=nRHOeL zAJ*vp#3xowVPKER*C%nD_9OX3_2z_Hmy-Pb39J*9lv03f-?Mk5|Dsev3F91EISomh z!f(APpST=WMbKf1x}{69_eB9D+}PAgjoMN2Nxlp0*Nf$2z6|Wwk0+n~dZB#w>xaoF zv83|(ZEZ+C`cuGrH6x#w>(5_;7!{=$aW711gD$oWU&BxjdIc}^Nm8uY8FU$HFxkUB z7;Yk$5FdeZWNIiPTSevH>cd3;z5YlM(ak<-N8DC?cZu)o()5kSFyl8G7ge%ika={W z08tY7vNr(wAP@Cvv;*ET`TDcrxjBuV#PAw&oESIAJz2Nfj=}(3+`|*5VSes6H*ht6+n6>(3vra5=GX( z8uF1dpFDCp@BoV2@@d0q3h+!D0)K~zFL*N^V=-a+iSzHqM>8Mylu2#<)e~~-%tPm; z6BPun_OB~8~UtJm`E@^hB-^5m$sXsuYQ)|$0#y=cJ$4_*?`SVP|gRTR=8NE(p% EA1QeiMgRZ+ literal 0 HcmV?d00001 diff --git a/packages/common/y-octo/core/src/fixtures/database.bin b/packages/common/y-octo/core/src/fixtures/database.bin new file mode 100644 index 0000000000000000000000000000000000000000..ec2c51935e3aae0673a14e47b2e76cee10fcc1db GIT binary patch literal 3474 zcma)8&rcgi6y{-@-)WGVwztYv4kQwmjKMY-sYrl|P#Xso5K>W)#=8SvG2XRy))*_G z95^%_dTWr9TiZxMtrWF4`p@*JJ#UOz4J-#jh}w^DIyBXvc)%a^~@M{}-OGp7ypL zzHSfE@d?>!1jw7FgpSTQcEVk0IkqIEkx6!VXxY4GI3|blC_IhfHgnmL%i7J@8XXBZ z;CvWqBP3x-FSdIMkq%wfiZgtIanm}nOzx(f;H?HXV{tkmJMN)h!r01v!eSn4Fc~b( zz@#d4pt3kMS1c}+s>N!xQY%uD(<8DcYeB8GWz1)KGFUwn?8ND~(2XZOqb_B7D%fE( zEn&Lc9-`<(+jZKt@VuL{RF-tJf)&uI}aBP#gy@KHg#~o7_@Tq>sC6#e) zK*sZw@mgxe^C)Xg&-ktpG3hlu=_neeFTY)S3A_Tbh(YvEEoXq;|tCzt?4``O)1 zdzR_>P5LrJ;Bg!3F@p;s0W0&hUYU7SnN=87h1;Mq)U$RdW2D~k+tNiH^{m%x`{G7y zEt>0+nrL6sZ*u6-zoRBMtQHe+PHES}p%-t&g&)J7sFQ+SRB&(GvX2DMZ*##74-t1s zh}5D9A-h>61S_&TuJ-#Qk;d>3#J8V2VWQ(FmtA*kD*S#fv=wx}IAQLi$bcR@Ds-EE zOFvek_X*@60@-=G{uIR?5Xk2VkhWzXBgO$?Fv7^}ZEhh1>krZJML(vpB&u3g0(S|V zV^zn$w~S_fho4eKIaYMs^%n|WY#~upjzvXMo*=Ef3$Jp6YJ5!WBGpxbE>m-rNbm}s zS0r5}5>n7rf)vO>u974~bCpa8k-Dp7(p9I#*Q3jfyQ)XGIoMS_0y&F7Vy-$Pkb$l` zBa9b>5p&gx{t$6j$wXDrRncd~1@yC(zoJG)!dDlNsH?wOn;RGE(yTSI{E#;Qs`<;9q)_qb=dE%mzziYQBD>kKd{q zzfT4{OpBD&ReDy7nAO!sW_6XG)m2JXZ&X(F+DBV%vw9i!`OlzdwWzXs1%Q?OHhzOu ziv9-i@B-!e7RJ;Qxq0VHZ2>cV3uxu%D}TQZe%_?fEE3H-jiyXA*ZpY9h<1%MG{k-f zI{F6*=w@U6=tuFb=+lshtLv!{==*+pDtPWavXbZigv?xNem%T_zaSfY(D(0+C8D`W zchM5o_U0p9baTf=(Q)jJTZ^Fjo4CJ}ZNAf8vveJ%@~dyA4}LzYtt;_w$I?>y8H5Pq U87M;yR$vu2;K`T14Smf20Y^ynApigX literal 0 HcmV?d00001 diff --git a/packages/common/y-octo/core/src/fixtures/edge-case-left-right-same-node.bin b/packages/common/y-octo/core/src/fixtures/edge-case-left-right-same-node.bin new file mode 100644 index 0000000000000000000000000000000000000000..c598191e22f76dbbe31283bf842935d152550615 GIT binary patch literal 6673 zcma)=O>84c6~|M(i3!QfWY7@AtdKC;)n<{#gY)Sm9FP-#2?)zway zCy}Das`u}z_kQnH^{ZyS3;y%RfBpUD$P5TnMY-0}rVpB-Eq;HkATDgLL^9d!x^j@` z!P_H_E}-?aRe=L0U590peOX;(JF=kybCRvs1qrSyYy%D`I#AUFSppB388rWm{)07M z5IGe}BY1Q|QDtRS7jzLa#M9;MfzgpxHtKSzu%;;)JsFkOv8MszDOHF32n>393wf#; zYMrfNXC0C2>=<$@Ky@~7({>h$uni?$!~RT;>OBPl^9ye}oDD&TtAeCMRbs_eiPeP; zd=pJPIo4rC1A=h}%BV$~8UsO5&vjJW;lK@>P? zlB{D6j>6>tkh@cZ9Po6XwmEcrCJ)RTUk~34-&+oc@2xC;0}G~XL6xnw8meqaR#CkX z3ONY~kzUI1WmnhevPQqnw~ABHK&J zoRx{us;u)k1e7$cVdYKJaAbGKG`%N64D$6GRaxbrnqxV^(7=OnH2dol&Ujupg5bd@ z9-Ty^_7vurCW``xZ4(;Lajww`?yYJeY?v|B*YT`)#=WEsY-T^#X zSUZe&4=ZLq?I6iNRfl{E4(9*1yJ=Onp|T1;AoCSLkXi&mrwE+VAJt)3N5HYc z8eztrGQVSmN%e2kv5(zZ>z`2h{>}-;btv9~p9$=+VXd%(t%qq*pwb_uQ zaLH&i14aVx*f3`RQFhmt(n;8?u&KrM*+@aCwxX(FDW5w^`@glq9rDD6uw0UkihIRM zV>g-%M`yLAeA|7Zjh|qCY5%I@3_JuR1=K|m+lSzKIY2sJ5*`9-79N6OX>cw5JbdH{ zyKNM2Ub_rYWm>bf8`f_a^F(EbzvhumXe&-568@7O3pPK&`G@*O3Jkm zQcs-Z+o-6%lHJ5A>3mfxZ^cWCjYXjwPnTBKBJOTN6MYQ8$Y%&Ap}4XY&1Tq5K3k|{ zwC!Gutt2t0mmW@3A>#D%nw*f%$LS@V)61cpTnpj!XNQv&o;gkR745W7z|Cl8Gnj*Q zXthK&IN~|L+U&d--Kxwj*Jfu8t&y$I!}WDAGX31%c4&ovS^xn7*q%$K(xrkh-<1yw zyX(N#u%1vl%Z=Ka(LOBeyDb63IP<`u3K5L6Yr;S}AB;0P zjI*IITnnkDC!Uz8zLJ>3W;h88hiOGsFZJTX&+DCTF^j2Y>CF+Zcn z{A_5W{*F@di%41t5qfBfkWp9Dt8nY&_2o8^UVAtojEh30-+ z2(MS7?wR|sdw3Ied?F?!+AHZDK~^gIW=IxUIcC+1J@Ys@FQ_^!yYjejX0v5paL(}M z!@?Q$o)ru~H-kJv!_GlrD{tN2RiG4%30lq2`WH^^BYXT)Hcb0XMRlGqh=&q#F2C4J zX?sPz-V1M;b9>vEfv3Nw>JWpg`)k`NN9JST>#rTLmjUXp2MitxZ_Ro7XX`rDvC2aW z)=`(z3N}>8>Ot$TYFO8AtFya*|GvZNi-Z$3K*A}MQ_K51iDap6>}jx5sET}{f1cVt zl}W8obyzl7EBGEN3YtEk@F6Nw%cSf@G+s4C5n8<{Di&Of25WL#ktOTP$7OU~I=b!NAFZi}8=sen-_NSvzzb5+^FU?RCS=GOci_>z1TAe3g2;(2S zr@c8%g+n0Eub6YCBcs_~YR0>fh54LN+el-`=N`yZA%c8k}Q?F*9V-e$VN8?XHCyEa~;QW;)UmtMNHyOG<6$;yXLik#)Zwe{TE! zo;wl2oLi|j%4w~%QCKh{`?E%?W^VK^hxP-f9?Vpss~da~QNARi`RMK5DV_BxrfGjO z^)@@MrO(-U^S;N}OZ=R%8fB`lB<5CODc7~G!m6>gzn#h%jgAoMe|fPd@ZJHbLRWXa zm??Kav<~$h?B4WzF}oHL^Ak_ZR9~Hh2fmtl445-8_8aip@4?;wfxET`%*gmpn2G-b D#h@(P literal 0 HcmV?d00001 diff --git a/packages/common/y-octo/core/src/fixtures/large.bin b/packages/common/y-octo/core/src/fixtures/large.bin new file mode 100644 index 0000000000000000000000000000000000000000..06f4d1affacce4452ada14666222d5c9986bc098 GIT binary patch literal 239535 zcma&vWq94!wJrRUa#Lo;v`ryt&5$xPq|B|GIMAj^8fcnQEVIlq%Ph0ZGRrKp%reU? zv&=Hfc>b-|9J`)#tfl$kKF@u&$3F4zv~%on&w1a~JJx*d;K&W7?mJ#}>%YtFWt^H1ORpZ{Lp_Q%2V`d9tuPak@$#b2Lz_NhCb`(O0kxBf+W{j2|zc;JyoAA9hr zmXH6Re0t}TPd)MEegFN}>qSNX{a5zIiN8&K`P;me{%z`qzs*~3|J$hlR)l}_TW|jh z-TH^WCGih$Aph`p(SLY@@DG32?f>xG{uK@X=y%=zkCrdK^1uJR_}lZ(J#xo$ulZHI zd+OcOv_ZXl8a1o;8%ezGzllGzc;HVDKlQ}3k7xg4+;#80xBlh4H}~niPq%#FiN_yo z@znG8KJdh2PdxRCTmK5Y>Gpr=8r%}UGa+s$;=8E0Vfbs^b-UPHLr~v!yIMZIrGBR6 z-=2BVOK$x|?|$7kT0ZvJ6ZgFTxhI}_?7NC+qm%TWEGWtkEr4^8~E1!0xB8#sPM} z4|YF=-EUz(Q7}0MiE7yQZ-+I$1^Yn))IP_?{2L!(&Fg)~?Y?HWe0}an0GkVNA_q1Pz==-aL=K!t zfF&9z2Y|2!j#t3uw}2Bg@HPMQ>t7!~|4Vbi+XI2P)Ja^*iAxD_r6$TLAgqatGve)k zyIaO3Nh3zZ&WKT3dZ*L4lN)yu<6do)n?P6_E48uyE#r=)5!>Z##3+S*)@eM;jc1AR zqBhD|AgqmNv@vXznxqkHCI3aJo7?`~Al;q+B46L+zliot{)=qibbZn5Z{IBSWwaoz zksV&7{>>U@OUDlU zQYW&MBTET#zCp@qAgz(JG&0PY*-0b@O+^GLjb7nIR&ZnmL2fZfxecT>a=k`|X|o}T z#JH)BAf?l*oycmAtR~2l1}V3Jv_>A$$gpybCXv`Vc`xS0Bc-1AVqfv(y_gnH-ivMV zw0p5G9=Q#qHS(rLhN;s^|G&I%D;{i}!U$4Yz0ip)ojjhw8I4eH(60#lMm?49xmQd+&-i7e;Ha)MlI zka8PHYvc-z41;H75{bc66+ud?S2>YY99cz>hYV6~18I%itC3;x>`NlCcxod^Y4uts zvX&!j3G%u|;x>@h$V(a-7SH7*5{swcCA3`|NUImT#MiC`FQM&P@DkgueO{v4wSn9Q z(i++QCASw(gRppdyd*K}8es7hN08F$#ZF`~M-~(0M1z#uKw2ZmXk=JCW0OcMp0Ws1 zTD{DPEaS*Bf?Q^havMl%q_lde6IsfUr35+OAmuiY*2q~J85Ym%Bod3KB7&4wuW%wOII@Buw-}_{ z2GSb2UL(We*^oqH@l;2U((2VtWHm=t6XZ#a#BCt0kw-K#ES{rDBoAUPg;2?`5`l+PzE_PeZv4q&4!UMux@H>Sd|o!Qv^5Af?p{oybCtEF{Rm1}V3J zv_|%O+3oGxFf5+_NhB6eNdzgaUgAWSaAXNV&M-*14Wu=4vPOo*GbM?{;wg_HrPa%w z$a0P>C&;x1DYt>NMy}Avuy|G`kyt!c5u~(wl@nRTkyQkF$ROo5kk-h(8W|SPz9bTh zr#6C=R#w+y>Gbc}XL~;<=ncHp1d5csXs?MzVMcUhZqxf|t{FEqJ+Y z*FG=T?JBo{v_^J+`R&EiC@h{HFHf{Bod3KCW4e!uW=%4II@NyFBqiU2GSaNS|h{aIg>6&m5kZbJNVyH9 zHFBs%hQ%{1iNxY5jUc7fOP$D4jw~g}`35PsfwV@>(#WuQW+#zYJQWe7w0eaTS;3JN z1i8f^C?(i(Y0Bg5i3nnE_l;>mj@ zEuO}*c=BH9E1tYp(&EW`r7fOzuhhjOw}G@q-c-oOVezzjWukZ*WAPM5kkaafPGliR z782xOgOuAqS|j_t^7eLZ92QUiBod3KB!ZMyFL5GEII@HwXBed12GSZiStG;ZnUX|e z@svl9((2_-WI0Ec6XaTr#BCt0kt;MZES{A~BoEk(lvXcxB8xe)m>?$_q}&G58aYNI z!{Qm6L}KxjMUc|!Wlm%nN0t%fGJ}-cKw2XgYGhbEi;_qzp2`SPTD{VVtmMc_g4}J8 zavMl%Gbd0HWxgvE0viNxZ`e-$mB zCerHpuksa7{;O#5h|Jk5*AO#S0#$42^LRL1SzdvREQ!S8DUBed)k~eoQjRPo$oU$H+dx_)XK7?uJhPKXES`!8 zQd+&jiLBtr3WD5Xka8PHYvg*342x$&5{boA9YIQ~S38l_99d0}Ck;|=18I#sqLE?o z98Dszc=BG&i$_{L@72EI$$K>|p1fDv;%WD4RXk1PHjviHn;IDwPpem_iU*6QFoKj; zFLWXcIkJ!-2OFf^2GSbY@71bZ!{X_mL}KxjM3BGbxmO{ZhQ+fliNxZmjUc7fYn{kij;tlf>jo*efwV?m(#WuQE+>&#JO!_z?b=jY zz2G&zb}e`fZP$X=*mmvn8r`mP8%S$p_t)HBJWa#m>G7IGyEet*DUKke)r+0TVvZ~( z$cY+>+dx_)$7p0&JY$ncES|CmQd+&ti7eyDGJ;%Yka8PHYve+W42x$`5{boA89_>` zS2~fE99c<_yA4uq18I%irjcRsY)>Mwcxoa@Y4sW>vW6pT2=anK%55O6k*75>ES@t- zBoG;}I@nG>3MUc|! zMNVW9M-~y}D1(&SKw2Y*YGhbE!;(lWp3(?XTD{bXEak{jf}C%VavMl%VD>$-(Ah#H#+y>Gbxn3cgg~hWWiNxZmjv%GgtDVSdj;toglLjfb zfwV>*(a5lPjwX>mj*EuOsB+2U#UI$b<+8%S&9O^pnTr`78c z#nTLnr!az)RxflS3puioAO~wCZUbqJ?Dx9c+qGF(JpGeMES{1GQd+&li7esB5`vsz zka8PHYvg2&42x$<5{boA9zja0mphT=99d3~YYkFv18I$1p^;(ntV|-Yc&Z{uY4s{6 zvWg?C2=b6Y%55O6k$W{VES`NyBoqORaWGz8nH%Pe+q&4!AMux?6If=yL zDabwH_P_ssb~lq&FHn!T#h&3Q;7_>~yxz8JpVzB)Z6>#Yv_^J+y(*rtczV1()vj1P z#Sx^mda)B(%#pdzmQ|3gLaby`mE;C5E4Wu=4 zp+Ytfi)T?1iN#YHK}xGvI+2weSxJz)4N`6cX^q^bkzw&{Pa?5+Y9dH!^%^I#h9heT z@`6FiZ6K|Yr!_Jxo-;`#7Ek^g=&Wllt)Bk|->l1j1D$pGZ?MJF^$n_ca2rT#WQRB0 zUOdgi;_3K?MDaAo;wg$CrPYg^$Rds`BFIq&DYt>NMh?};uy}?gkyt#X5u~(wsS{bs zk);GV-yr2Skk-gq8W|SP>?9J4ry_!sR$-(Ah#H#+y>Gbxn3i~;@OZyV)0Z* zkkabaPGmJlRukk&gOuAqS|g8WWLP{$lSnL{yf^aVkyg)pqpx`K-bjll?~S&2+PzU1 zkK6{*8hKMA!{TZ6##HfO@f1do(&~jyWFbcu669cml-odBBm2EkwQE>B{WbEnw>>|I z$y1V3szi8zu*AtM;mi`koMD)99Y|~DYFLyf2xwD)&*BYlB z2-4cQ!Z^e3S($cX_*BK5((YAGXBBr=5$7S}lp8@>JNFuAm_GZ`PHdmrm{SVA*6FO} z&RXKU{>C@x#|Y*BWq`PbUNX=ye=et?*grkrgs#2)0Hox5zR6d%J>Nvjw&$B{+4g;t zfy$X6uAx1$XRy^@@YaPEyh+vV?I~=a@iC|r{dgyIJco`a(8(4mcY?Tv78_`oLF3X; z?4YGFsFeLuCv+)?E+x>F7Al8=xP~q^&@hFTq@mbCyJJwP``u3HZVuf|pj8$shl03< z?ljOahjyi**h3d%Q0e=NPUuAry-1)}EL08!aSc6hpkWeSNJFuSy1zM6MpF3Q-|Q=+ z?r)}L)cwu2jC#GcEq4k_&c1? z9UQuYK=)Xv917wZy4^s-GOA2Nv5d~epi=nfoX~R|dX7LZ87K|~aSg38(6EforlD9y zUEadWsJ;|_m$&%JsLNYu8FhJ!Eu$W9(PdO$4h3-y?erE^M)kx0aHzBXPer+H8P&%! z8Wn>|;g51cM{(#V0xh;sITXY-bcBJ1Wi&Dk#WGqDgG%8qa6%Vw=mG*=Y@u=}h->IP z0}ab)ej18pv@Hge!r$hEZsX8x1iI5gR1O7k4c%y< zVHs^oL$Qob#-LL8C!Nrf9D0&KYYY^Jg1CkrH_)(*PNbn&M(y9mml2+HZvQslGHU-e zx{TVt&6ZK8x9Kv%o6d-9Xxq1`GLlD~+r2GWMp#BeV^Aslp-$*f4joFMBP>)71#t}> zY@lHo4M{_>jAqB6Quwo-(AgY1n?UDTs2mF78amTJ!!nwchGH32#Gq366;5achgJ~i z77LYSgt&&TH_)(*Hl(3gM#o}MDg0wj=rImGMxduGR1O7k4Lxe0VHs7Yp;$(3-X19< zDf~8X_mxqbx6?9e^LATC9o}xrNDc*Y4b6MIE~BuFTE9J2M!A=s2SuQG&UTO!I*3CD z5$G@rl|w;XLkDQ6JoY>=4aG8=5rfJyn&E`b;LsTaT56$kD2Qw5Gy@IGXnGooWwb5^ zmBL@=gs$Vzbp*Q6Lgi2p*U&Wv8kW)8G!)C|NDL~4f5Ztr!l6e9^tgfIP!QM9Lk1d_ z(cv@{%c#{mcp2g8=T`6Vl~JpA&@yWE4qHZT-=WJ0Z$Bfhp*IXP{LhSTrlD9yg)yiU zexVau$f1PF{l*&YA19xhpr~j^%g422yqQ9H_)(*R;8g>Mh9b1Dg1*@=s^xWNT5e8 zR1O7k4c%{`VHq7rL$Qpm$DmU9*PYPo9D1EV^WJI7NDc*Y4ZUihVHsUZLUS)d_kAaC zT|9f+_np4h?fXvJx_#eiTX%qk%Ap{xp}pU!%19oC?xX+Hyl?xT@#S8Ho*aY9GMem! zPUg_b1Uk(^X7dxTF99m4E6D?E@1#t}>W1wLfjZH(bj26eBQuvFV z(8V0Om_V0Xs2mF78oJ0p!!jyML$Qo@#-LL8JDt#-9J-T0_Zlb;1#u0nG|;e&cBG+L zM(1NtDg5(J=y?u3PoS4AR1O7k4Lxh1VHurEL$Qpyz9&*fQutlp<13@C@1bSX^*y$X z3f`m3sIeRh;u_ldJ-Up-GV1c4WEo)@jgCR3@JBnLqd9amfsV6KITXY-bfkfXWi%=c z#WGqLgG%8qbV3($=t2TrVxe*J40}ab)K^lr>v^@ru!r$(MZs*YL1iH&Yo!~1+?)Zu-!j5@r}mQm;T=`v~}hl03E?j`EB@8@NN=XTq^-&aO$-%rb^?fY#Rb$q`mBRLerHMI5nRT;@+)NS6M zDx;=YMuTHeDg41s=wJ>VOrS*;Du;r&h7L5)u#5(!p;$&UV^AslnNH|T4xLG$b1YO2 z1#u0XZlGZq%}7JBjMm4XQuyng(DfX;oh!17c7q`~gnr01h2MphGNF4h3-y?f(JYx?vd=rlD9y(_&C5{Ao_; zG!C6cptCGg4h3-yEiurrjHafcSVn7NP$~R1PUspAT|=N7EL08!aSdH%pkW!UPD8Pb z4#l8S_=lX(LmYaDK&vfO4h3-yJz$_=868YQb1zokh(Pfy?+qvP28Z4t(AFO`Wh94! zxQ1TSPimQYiOSjsxoR8mQmji zrdqcdmeG_LR0@BJ6FP-Mrx56L3zb7bTtg=rXjn#*(@-p<@)%SKzuXBe=g@KjU2C8? z6vQ=jg@J};v@#9FGTI-5O5yK!Licm%egZvgp>imQYv?`$4a=x14aG9L8iPvVUv)yS za_Ch8y=kFxD2Qult$~JRbR`YNGU|O-q>QBSd*9_NquzJXGU|PoEu;Q->DFx~hl03< z_Pk4%QCLR3?n;&smeIr*R0@Bh6FQMYClY9hh038IuA$=%G%TYDX(*P_@)%SKf4LL7 zoI{rr=qd}9LqS|aml|kTM$6JrETg?Ks1*KQCv-1|?j_Iz7Al8=xQ6aF(6Eg5B%!&N ztuIHQc)s_t6MC6LFB9lB3zb7bTthEvs61|cDGkLkD)?@;gAEss0?ZdW=dVW}!5#EDGTtmBjSeH>)MqNLgEF&zVF)^qV{un2842O;((D4>3 zhl03IT|}TuEmRH#aSdHypkWy;Ohd7ZDq~P7{7NUZ zl0z#Abhm}dp&+iI+YB@;qwOhZgL-%X`z(iMiSPjSSts-?hn^+Sixw(}g1CmBQPAu! zcdK|L&^kuj(g{zxZuB!`Y9(6JUOhl03<4mZ%Sj7FrPSVr?>P+3OvozVFlI-fwx zG&FZ8h->Iv0}ab)UK)yJv^55m!r$tIZspLe1iHgQ?RNlZoo`zx>wfiV9BRuWf?xVgkYWGoE zM(sXo%P9Y&ri|oJ5ZBN)A5~>kKP;oRA5E1}eJrCPF{l*&5GQm9hYlgo;T9@~g1Ckb zGSIM$2B)D|Mzdm2Dg0SZ=qwJMMWAyH6o-PihR!h1u#9G=p;$&6Vo)jk4Nm9=4&6YY zn=Mq95#k!U&OpO5TAzku8CA!iQux(QXf=mc6X;0`l|w;XLys6}SVl+FP%NX?AB&Wc z6n^WE`O2vE$7mU~{+KPJ_8-$_R9_ASaSd(tFh!17lDr{DDsBKn@*9 zphGQG4h3-yEi}-uj0U8kSVq%hP$~TBPUv(Folc;$EmRH#aSfenpkWzJOGB}Y*2bVx z_-mcewH&&ZKr1X%4h3-yU2ULY8Lde|b8lcDjzICW?_nqOFozx{&|?-Vhl03<9@J2I z1N%@Kie+>&29?6U>4e_o(3=F>=HsS}omeGwg6w9dp$9d~Ekizf(abN59 z|2S>k{vWrkJIF%iP!QM9z8_bu+aN5XejiV@ZUZc%k{DD9zr+bG;m{HSonfFj6vQ=j zvVn$WG$jqiGFlabO5v|^LRWFimQYv@V?4a=xJ4aG7#5Q9qLA8g%i4hLst;!Y73P^L0m(Z8E9BW%ahRD8`%3I zP(1Cs&k5bfq5BB*poPkzAg-Z%G*sTe-kXME8P&$1Quwt_Xf21<66kdcl|w;XLoXR< zSVouAP%NXK_wX`mD23ni9$y*tyoZ)i&wFec^}WZmt{e*D8rtI?RYncNGAg(yRYnc5 zjK;^HQuyPY(D58Po%F>+ z!ZPZ1Z?cTAjK;>GQut$?(6JmkmOv+1s2mF78amoQ!!jC^hGH3&#h_C7Wlm@rhn5lO zG7FVML0m%@8faKXi;~dX8`wJ{P(1Cs!wKENp*sk4kA=#iAg-a?HB{cfu1rI*jLyZN zQuybb&~qGmjzBM2s2mF78d_tZVHurGL$Qpye3F+Dp7!nXNnaUt`6Ml)E}yhz)Z>$; zjO0)d*U(O%RAtmCEThh!OqEe1ETd5|s1*JvCv+5tjv~-v1I3{ruAw6gG%TZ$X(*P_ zf*4em(E=xQ0f#Oi(8U%ihl03<&NI-kjOM4ISVr4oP$~RvPUtoc-A150EmW2f;u^Ze zK*KWHnucN-oryuE@Xt7*XE^i>fu6TeITXY-^pt^yWpp|X#WKqORHTfg@bf?AE2I2R z(K5>alr5vKpVDR2NDc*Y4ejtLT}ELUb^KJajIfM`$DmU9!=2FK96FppM_Z^I3gQ|% z%s|63DoR7KjONCmQuuS7(77Bsmp~U{Y>8j59fG6t2xKk0;?!~to9F|eLdZ{vMjAb-529?4e>VyvE z(4ho6!a#8-h->Iz0}ab)NE(V|G&=^B!k_Jg&gRhB1Uk<`K2OB631#u1S*Fd#yldz2XH%PT^6D*^tF{l*&R3~&QhfXEX znHDOCg1ClGG0?D#O43j)qt!8}6#i-_bTx;rCeZa3D$59Q4J|j&u#8rvp;$%-V^Asl zgHGr{4n0VqM=ewi1#u1CZ=hir9Y{m5jIPI^Qux=M(CZv}oj~&%>N08~hl03-KF(TeojR+qwfRR1O7k4ei}fmr+D4zCR>4dK2(3J$b#zNZBP<8Y*vK zuSi3&jH+T#Df}uYw2DKk2=tJJ%Ap{xp?eK9ETesCD3;Nc7*q=XiW7Q;L$46%4GWb+ zL0m&G8)#TYwP`4pQLje4jG9W}_iE%Tqh5_@8TD#p%cx%?-MTmw#5J^_kt(C6VHx#o zlq#d9SVj|KP$~QgPUr*gtQE?0^gerK$lyn917wZx=2Ii4eYWs6w7F53@U}c(+SjZp&+iIotvmKY8IAJmnNw)YKCPrItG=(AMJ#W=FrguI?h7nP!QM9kp>!;(Wo>O z%V=Q?Duut$30=sc3kh_Ih038IuA%b{G%TY9X(*P__83$Of4dX9okO=1=q?MDWrVnf zZZ*)bjJBnrSVlE5s1$yU6I#QeH3WLWLgi2p*U-}j8kW(SG!)CIQ`1NpN#S>D>MNs8 zO=%f*YHG`d=gr z5uWz#(9Bmx9h%WH>d?%VQRim5jBqH3YiPS>s*IY4Wz@b|s*IXr84Zg;rSOM2p~E zG|;e&THTi@qlWdcj0VJ@QuqU$&;cAefIx>>s2mF78ruK9+gmpy+`7@iG!)BdS_~?M zKg|i9#-Y;)be4t6p&+iIB?cOn(bO~)%Vt133V)&#I*~&s5@?Bq%Ap{xq2moSETai&D3;Ok7*q;> zxf8maLzffiDhriEL0m(Z8faKX%hFIRqrEYx6#iZ(bT5bQCC~#FDu;r&hVC}du#EPk zp;$(jV^Asl%TDNJ4!umE*DO>H1#u0%XrN&kT}nf-j0!#*DI+QTg3tQOsNl1-j0!$$ z%c#$1O&Q6dAg-a^KdZ|qETbNuO_fpZ4eW6dD4zBm=Y)>q&~XGh$wK8&5ZBPL8Y*vK z7pI|EMoVH)Sw>5o&?OwYgg{qVs2mF78d_$cVHqt>L$Qo@#h_C7yPVKn9J-4@_gSbM z3gQ~N!$89_+L?x88C{4$rSLB}p%*yx0)f^VC=La24LxU|VHurIL$QpyeU6t=11bD& zpYxSbx6jcs>h?KXMm;~L%cy}I3gQ~t<#Vcx8iZxk^>fKG!ZI2YgG%9#aYDy%=okVW zZ=rH1h->I50}ab)bQ+3fv?vCZ!e8WsF5=Kd1iI8hmeImA6w9bG29?6E zbV4gRw30w~Tc{ig;u^ZmK*KWHo`zx>osB`I@XtD-XF2pNfnKywITXY-^o)UqWmJ=f zVi|S*e58z|@H>CrS4N#bPs^zD=WQ8v|GX(9ITXY-H2?FujKVVN^!Zd7<=(&^8G+(y z-;qw}NDduIpkpmm4h3-y9j>AB2KIDa-O8a`33P{r%Ap{xp_>gfETb)HD3;Ob7*q=Xv=e%oLr)XvIRnL^ zAg-Y&4KysHQ)wucQO7UvGQ!io9lzi!qmEyoWz_Ktwv4)bL6=cOITXY-wEY)U88r;c zsKXbMWrSr^6oX3P7dfFt99l%6qbyVo1#t}>YM@~m4NF6@jON6kQuuS6&^a7Bhd>ut zs2mF78amrR!!jyOL$QoD#h_C7o1D;19J+}>w^^txBg8ed!a&0^+L(r78J&nhrSMNU zp(i->1c9EhP&pLDHT0N)hGld-4aG8Q_r*vVN#VEqqOXkFeUX+?yD!=@%KxG%BRLer zHMGqabs2?a)b@+1GRnPyJtP9f)4oHT&>>Wz_mhyo~U)Z|g7l%Bc01Xc@Ks zk}ae5U(#jNNDc*Y4Q=%$RYr}%GRpf>vW&2d2F9RL_ye8LfgC!JK!;kW917wZT4DK*KVcmWE;(t&KsY@Ygz_YdLf+fmT?k zEF;7IU0}adQP#TJ5bTbB(!oTT+ z-sI4m1ls1yri|oJ5ZBP_1{#*pjU+Vp26q21^VY@FzWu-KYu)}|rmfrm%eHj~S*RQe z;u_lb%c^zd4eWkjPPJ}hETfVbR0_Yu2`%B!5(1rJp>imQYv^PH4a;as8j5AKDh8Fp zU*&|Z;?PwDy3Ru7P!QM9l?ED?QF$7QWpp40mBK&ZgdX6~0|a`+KyfIDYiN~$hGn!r z4aG9L7K2LRUvomQap*MyZS@shMvdiA5ZBNv1{#*p)ie~#sLxj-Wh8~)=PSN4>hl#^ zMt#0w%c#&o zg1Cn6G0?D#_NJj&Mzt}h6n?D}TFare1bW><KfLqS|a7aM3;MoZFAETi2qs1*Kg zCv-Q5?k3PG1I3{ruAw^(G%TZCX(*P_#TZly|DqFmkwY&M=oJfo=5$G}tl|w;XLl+uoSVoJ| zP%NVzF{l*&4kvU6hwdQIJr*j5g1ClmH_)(*D$`IbqjNE+6#h9U^c;ttBhX70Du;r& zhSnHpSVm`)(A*o?UB1rC2v7TV`MR%+x_q6MQJ1gVGV1YlQ$}(qh-+x4ud6bWH?TW@ zJyk|cv5ZE=pi=mwoX}AmI*LGxEmRH#aSa_|pkWz}Ohd7Z7Q~=Z_zRrS1suA7Ko?u6 z917wZI?q7EGMb-;Vi|3VL8b7wIicG)bQ^*0G*BE0;u^ZeK*KWHnucN-oryuE@Xt7* zXE^i>fu6TeITXY-^pt^yWpp|X#WKqOMx>0S@bkanE2I2x&@#&ZhApG6-_T{$R1O7k z4ejs^T}ELUb^J!MjIfM`$DmU9!=2FK96FppM_Z^I3gQ|%%s|63DoR7KjONCmQuuS7 z(77Bsmp~UVyvE(4ho6!b0Ux5ZBPb1{#*p zkTev_Xm$)Lg+JQ~oz0=M33Q%?%Ap{xp)(CMETdUzD3(!03@U|R;e=LjXa#|8F;E-| z;u^Z%K*KWHkcMIz9g9Jw@Q*p6$2jyDfu6EZITXY-^r(S`WmKJpVi~phR-}xi@Y{UL zS4M5VMa!tow`>`8_?9lCW^yQqYiQoLbQy(Z)cRY=GQu(%6oX3P4{|~Wap)id9cH0& zD2Qw500RxnXkZ$OWi%rOmBOFlgwEj583bBtp>imQYv?os4a;bH8j5AKE(VptU+09b z{j3AWrU}FTYcMCMyimQYiPf3tJZBEmQnw2r&_l;meJH0R0@Bp6FQYcrxNH)3zb7bTtlZAXjn!i zX(*P_>KIfCf3*|3nnPC;=z0Uip&+iI z-|zTZx9@jo>-PPQZQTJDDu;r&hW7rBE~BuF`g|u@Mp#CZV^Asl$xi5G4xLP((=1dD z1#u0XXrN&kO-e(tj8?{=Qur&K(3Kpzl0es3s2mF78oJy-!!lZthGH32#h_C7RZeIX zhgK2jAq$m5L0m)k8faKX`%=(Gxi_${aA=kYZ(v_>La%V>6#~6sp>imQYv^SK&HjS7 z-q)>7L$Qo{eV3L|qk2;Ky}s)!qh8;oWz_4twv76H*R-x23gQ}C@ZHgG%8~a6%_==mY|tVxe*IL0}ab)d>V>nv@8ad!e8cuF5}Q;1X`}4 zxkEu*Lzft6SVl|JP%NW8F{l*&9w&4UhwdTJ{T3>Rg1Cn6GSIM$cBi3OMwenxDf~-L z=p_!lM4(qKR1O7k4ZUEXVHsUaL$Qo{d@oW)QusZ-=PRQg-=k&J<9oJ@dVf!qQ6oGS zkGO_*`<^bNu#CEYFIh%dM#V9x6n?Q2TFjxv1Uk_|7Y-N~Ul33RW8%Ap{xp_K+2meGzR zH1`Jf`3MwG`<{0~&vWQ`0=;aZawv#v=vfVwH?Ys8p;$&;zt77EPy2TLzORhBexH_6 z*YDdhD)_!BBRLerHMH~hRTNe}Afs8eka> zi$SIEhdH6cICL0+jG&BvxGAfNhrSMCg&{7U9CD8c>ibFwMLuVOi zSVptcP%NX3F{l*&MkjP5hi)X$trjZF2yqSFV4z_cRivR*K_E40^MYxawv#v=vo5}%V=E^ntKELXatI< zeUCb!M>+H;fu68XITXY-^st7?8`wwEP%NXoAMrB6)4q8>@|98Ek7ya?{m7P4yC0b{ zl0!jULvI>rSVpaWlq#c!SVjY4P$~QYPUrv*9YCN%EL08!aSiSNBh|VM!!jyNL$Qpe z#h_C7)11(096F6aXBj9C1#t~6G0?D#rlz4-Mr&eFDf~4~=o$`PL!cWhRF)Cq8oJ6r z!!lZ(hGH2Vib19D4>_TSIP?&KR$HhX3gQ}iz(B(?I+%uH8Qq9MrSNY!p*J}627$Kz zu`Z*Aawv#v=rsck%jkLT#`Kenwq&_d-<5ZBN?Kh|Xw zmQmjyCtDZGXi5w!g+Ij!ox-702z0uI%Ap{xp_2?WEThS3D3(!q3@U|R?u3?eXgPtd zwNN<}#5Ht&bH*U+9nQDxL9ETdjONtIC}ETf4rs1*K0Cv+l*P9)G01I3{ruA$=%G%TYDX(*P_ z@)%SKf4LL7oI{rr=qd}9LqS|aml|kTM$6JrETg?Ks1*KQCv-1|?j_Iz7Al8=xQ6aF z(6Eg5q@h?wmt#;V{L4=0We&Yepw}!^4h3-yy=b6e8C^<4v5X3S8Yv?w{DPnQ%BbL{ zw2TUVYRjn4PjwmLv3SHawEItW8HHukh->Ir z0}aclI1R-zS`vdw;V*GQmvHD30$pLDawv#vXqkbAWwba6&AoxWD+0ySzPp^zT^zcL zK=)aw917wZx1#u0{|G6%su#7tWJXuCq zMk8ZTDg2R6=tvG7NuXmbR1O7k4IOTvVHu4`L$QqJ$DmU9^PSN796Fys%Pdq51#u0X zYoK8n%}YXaZ(whYK=HKiRwr~Thi)a%9TqBwg1Clm)=+r^drKOMWpp|QmBK&mgr4Tm z(*%0XLgi2p*U*y&8kW(iG!)CI<1cs_HIc&a_zPbdb^HY_qmI9@Wz^*tri|oJ5ZBQ5 zzffh=BrKy2zetr)6D*^m7*q{jIWuQ0|#5HuNfre!?EDgmnniGRc;m>hG z=Wys80$pICawv#v=xhTG%cwLB#WLCygG%9VazZz8=q3W)W}$K@h-+wtfre$YF%88s zIuV0P;h%6qPjKi70zG4)awv#v=rIEg%jkF-ie=R9myt4(!f*FWUm3OgB`u?NzqDnP z|4Us)P2^Az*U&b<)MXTwQQKc8%LvP8NDL~4Kg0h038I zuA%ERRNlZ|pN3)?RmY%G_|;BmHHTIc=t&EeLqS|aj~HlJMn}_7ETh)H;$_rS3cvNQ zd}Y-7SG0^;|H_t8`(K$dl0!jULtFhyl~L2MjPib!Dx;=YMgwC|Dg1#>=s*q~NT5Rv z6o-Pih87xVSVjZVP%NYAF{l*&bSHE=hfXKZ*%m5?g1ClGHPEn(rlp}+Mr&hGDg3og z=voe4OQ01NDu;r&hORcyu#DEEp;$(TV^Asl!%pa74n0hu$1GG11#t~MXrN&k9ZEy7 zjBduDQusHW(3>23lR(@2T9;8%ITXY-^tyqDWppDA#WL#u>qzTL;rIWwuXX$XnznBL zU)$ClWTA2>h-+xyU+dNl%c$S4ldX$oR1$+q;g>j}B^+8ppffB~4h3-yoot|C8BIw- zb8ld;ia_zS?H-0}adQY8r}V)aN(6jG9T| z_xX*ljQaeBmQkPI*fJ_KP#g;48rthOs*IY2Wz_pOsWNJYWi%-UmBOFogihknNd!98 zLgi2p*U$+D8kW(-G!)BdMGPv1zrqP!!J#V%bhU-bp&+iI%M3IuqvdHRmeIZ#R0@Be z6S|K>_Yvqp3zb7bTtoL5Xjn#j(@-p<+89&{zt#z@<6j$D2Qw5B?Ar1=yDp0 zWz_Sxkus9P@A+F_8TI@vEu)^lwPn=zx4Lzk$)O;wp*?=9%P1_Pg5M^~2+L@E3@U{` z-U%Jgq2md3vW3c_Ag-as1{#*pxFj_92KLej6i@pubwZbN=u!e*X`yl`h->I#4V5>r zm!zRsM!REBDg50|=xz?(O`ug4Du;r&hVC@bu#9%4p;$&2V^Asli%#f84!uaAS1eQx z1#t~MZ=hirT}VT*jJp4hmr-*m{O-T=l~MQK(K71(J6lG*ey7U_hl03xszfljbcITXY-bhLqnWi%!Y#WE_3L8b7^oX|23EhErn z7Al8=xP~q?(6EdarJ-0xJ7Q2N{2fl{4i4QxpnEJ-4h3-y-EN>^8C9mCSVre!P$~R# zPUtxfJx8FIEL08!aSg38(6EforlD9yU49=aBPskYzxS0hgPAMm>IS%190c zaSiSCdtF9h8Fl`BvW&2dM#Z2~_@kWAQ5-soK#MI@4h3-y9buqh8I4Rq8|U7@UcjMQ zBD{gUzzJQzp$iCfv4zT^Ag-bF6g2w_-g+8(ej18pv@Hge!r$hEZsX8x1iI5gtPuUk3psIhdZIeIdnLI zjsKOE`ctzP&pLDHMG<~!!nwahGH3QjzOjH zH#?!5Idn6DZnscbMu=imQYv^$U4a?|6 z8j5Aq{tuBdlEQEQ2VWVr{{t2U%Bal)w2az3V9Th(1GW5|2 z`hip#)yFa#6oX3P4{|~Wap)id9cH0&D2Qw500RxnXkZ$OWi%rOmBOFlgwEj583bBt zp>imQYv?os4a;bH8j5AKE(VptU+09bV!_^(5VDE(?aD?5ZBNt8Y*vKm!zRsMyq2`Dg4z==xPpKO`z*7R1O7k4J|j&u#8rv zp;$%-V^AslgHGr{4n0VqM=ewi1#u1CZ=hir9Y{m5jIPI^Qux=M(CZv}oj~(i>N3Ki zAg-ZT4KysHYiTH!QQwxlbsI?G_igEG-M%ep>-KGFTX%qk%Ap{xp}kwGGHMW(QJXPj*5lbLeCOoo1nOD2Qw5L<0@WXi^%AWwbH|mBL@?gs$Y!l?1xRLgi2p z*U;q#8kW(DG!)CIDh8FpuW~}GIJAmD4_T-j3gQ~N*FeKE+Lwl68C{7%rSPvfp;tKc z3W46RP&pLDHT1H9hGkTnhGH4@dMHvxQuw_d@|97qhiDn~ddQYhzlTig%Ap{xp#=}= zG78J6=R>J7%DsU-Ap*tIz7w3#2^>0sK&M!!917wZI!;674ear0D3;N(7*q;>nG?E< zLzfX~xrNH1Ag-ZH3^XjGrD-UZ(ViGo3V)9ix`#ve5a@mjl|w;XLw6ZySVp_kP%NWM zF{l*&B`5R}hh8Gks|JcgL0m&G7-(2V7t>HIqaF|QGHNJ=-{WCl8TELWmQjy~Z5j1` zSeFqVi$`2TyFIMRs9{(}-5*Yt5tdPL3@U|R?1UC`Xfc6Kv`{$|#5Ht`fre!?HVwrx zS{#E);V*VV7jx)h0$pyQawv#v=pq9R%cv|3#WLC%gG%A=bV7G>=uQINYoT%|h-+x2 zfre$YBMrqeIv;~d;h%Rx&vWQ`0=;aZawv#v=ve~|%jjGhie=RGkw_Uy;dgz+S4LeQ zp=H$d5nDzDkC-x&LqS|aJ3pe!C@iBckEF^d_XhUp2oz8Ij&?#vbLeOS9cQ6(D2Qw5 zNDY-Yut%k#SVjwDP+3L`ozR6Gx{yGZSg0Hd;u<>NK*KUxkcMIzZI3~v@V7gm+c|VQ zf$p+UITXY-bgO}eWwb2~#WJdiL8b6(oX{E$ts&4028u<tiPAXjn#P(oig;PLJ|3 z!qdK;9`%(`r$=cSb$Zm6QMX5R88wnaL0m&SKB~&7QCLR#k0#3q%Vrqq%7)meH0N zR0@BK6S{>%w-9Khg~~EQTthb*Xjn#@(@-p5oc(2*7@hl03<4$)9~1AAy1ie*$9gG%9-I-#W;T1uevEmRH#aSfei zpkW!!PD8PbHpZY*_#2(jjU2j>K(|_`917wZy1_uhGO9>Jv5bz#pi=n9ozUYPdYnK{ z8z>G1aSg3D(6Ee-rJ-0xZU4l}2v7UA{gbbZ+Wv`_QQJS+GV1szT}F-NP!QM9)_+oE z)Hp1oHh)T%5th;57*q;>uoF6%LkAORk%h{kAg-YU4KysHL1`$K(aacB3V)^(I+H_Z z66hQYl|w;XL#G>PSVl9_P%NYMF{l*&dM9)}hps2kO%^K42yqQvYoK8ntxH3(jE=^j zQus%m(4!oBlt52ds2mF78hY43!!kOOhGH4zJr*e=Dg3<0d}WmP7%ii`$7~t3d(4!P z917wZdecC|GHUf$s*G}PU=N5u@wD#%Cv*UZ4j|AW7Al8=xQ6zBOtr4OfnAt}Vi`?~ zL8b7gIib@ybQ*!qvQRk`#5J_UK*KVcnucN-t%*UU@Ygt@YdCZbfo`x+ITXY-bd`aI zWwbgC#WFe+gG%8aazYPr=ph2FHc%W2;u?CuK*KURn1*5*-H1V@@NYPwH#qbLfwum$ zE~6%LD2Qw5H3JRH=z1E8Wz_G_ymgyM;rIKquXX$VnYM1fKik$FXrXc_h-+w{KkG6I z%c$?4ldX$oG$jU=!k^-VPT|lg1UlV9YhUCyD)33Qc(%Ap{xp-T-kETd&uM@hLL-!Ks0RzRMAg-ai4KysH zJ!vSG(d8Ia3jeYbdYMBn6X-Pyl|w;XLoXU=SVou9P%NW@Cn9Aeg zge{{!Pv|njWATV{8HHukfre$YGY!Qux)6g(;a_kiMK8BRLerHMGliid4MxFm+%c%QbbQv|1LqS|a^Z%mDC@iB+e@T`RmeI%v(6EfQq@h?wr(;kl{L@b8X%0P2pyw=94h3-y zJ!zm}8J$W(b8ldGe2SM5p7!ndl&_3BK1Iu@<5RYbx;$mdNDc*Y4Q>CFDkFIVyTend zGHQ-xR1||s;TJifMI2g0prb5Q4h3-y9crLq84XKAv5e-#pi=mAoX|NOI)^|PSg0Hd z;u<>JK*KUBO+&GaHpQS)_?w*2O&q$3K(`qv4h3-ytuWBAj5emBSVkveP$~QqPUr~^ zJwc#nEL08!aSc6YpkWyuPeZYc+C3d9BPslLPy5QK-P5#;+C6Q{DF10)M$P3=5ZBN) zPwO%Y%c$+s$uhz+8WMv_;SX^_hj8c+0v&Flawv#v=pX|P%V=;Kie)q_29?5}<%G`S z&{+gJ*Fxn`5ZBNd1{#*p%rq3sXhRGtg}=cG-N2z62z0ZB%Ap{xq3aAZETi>lD3(!m z3@U|R?SxiyXf=VJv`{$|#5MGYfre#tGzD#vdjq@mGqjAdM0f+c^)tRQYW)l?qt?&Z zGHU;fDI+-)#5J_lGq;yf_7}YMkZ|5Ji85+Z56fs^3@U{`& z1{#*pfHV}#XnG7Pg+JX1oz9`t33RrF%Ap{xp;HYsETd^@D3;OM7*q;>trNPIL)Q{$ zg@)!11#t~sZJ=Qptw}?%j1I@3Quv3R(8C;hm_UzNs2mF78hX$`!!kOQhGH4rj6tRF zZ#tnjIrJuhws}^SQImRdD2Qw5bps8{=tdffWz_%KNb5@B_kY&cy8WM}t=s=u+q#1+ zR1O7k4ek4^Zr!kq`aPR$T`Z%L7*qI%0}ab)N*an~v?>Ob z!e8ZtuHw*D1iH>bl29?4;;DjFF&;tZ|#6smz5ZBNu0}ab) ze-fH|1N&M8il=?AIic4$^csP-`l~4;ITXY-^ooYc8`xLVP%NW9f8}LVUkbm^UwviN z=dZMk`ux?FQK5y(p&+iIz5c4osD4;Rz5kjjqxx7zlVVUQ{7FveBo3WKpi?bW4h3-y zonWA08BI(>v5Z#4pi=lNoX`~G1aSdH&pkWy;PeZYc_Qjx5`1_pDeH^-v zKo44|917wZy2n7nGTNJlVj0!Opi=m?PG~KM))MG-3zb7bTthDzXjn#<(@-pie8-T|5?#xQ6z4PM1+wMg`9$%LvP8d<-guKi&x)&!OW9 zbh3rYp&+iI#ReLd(YQ1e%V=o~Duut)30=yeO9^zPh038IuAz$!G%TYfX(*P_?if@G zf4395n?rXKXqAP^p&+iII}J1}qg_d8?hWjV5h$Maz37Bqv5dO^jh9gaDg5q#^OaHeztJ-4{x@4jz5Zs(NDc*Y4ek0jRYncMGV1oXR2enE zG8!9$O5u-nLdSCGSOT43p>imQYv^bL4a;au8j5997K6$%Dsw{1IJAsFml-Gy1#t~s zXrN&kElNYNjCRDJQusTZ&>bAQgFyFKs2mF78oJ#;!!oK&L$Qp`#h_C7=bX@U9D0sG zFIlJ@3gQ}CW1wLfolQfrjJiA@DI+QTF3imQYv>3A4a;a`8j5AKAO@AfU*Lo;;LrsG zy4XVHP!QM9c?KGm(fl+N%V=8+Duutz3Ejq_+X!^0h038IuAy5DG%Ta7Noej3>@yK4 zp7uTCgr4EhGX#3xLgi2p*U(cMDsNz)PD8Pb^8e1u2v7Uw|J_$c`G2Qnl>c{IMqU4I z%190caSiS8cU49W!!qjl_f#1*#4;KlgG%8KcS47A=x_oZZJ}}~h->ID0}aclC=JCj znj3>k;m>tK=W^&=0$pgJI26P+wA4VuGMbZyVi|3YL8b6FJE5C7bTff&w@_I|h->IZ z0}ab)QyPk8bTS5&!awPRp5)Mz1X^REawv#v=y3xL%jiTJie=RPACWSW!f*c%Um3Ok z2Q8!a|FC7$=^wg`8p@#{uAy!Jq01;NqjvvDmJycG&=^z-f2b2WltYIS=m-myLqS|a z2ODTuMnlq2ETh>ms1*KeCv-N4&L+@#7Al8=xQ5O&(6EeVrJ-0x6)~t3euWcS!J!od zy2V1}P!QM9^#&T2(S{^6_XhT{2oz8I9& z4&u;31Uk$@IH0}ab)dK!vl zv@Qmf!e8fvuH(>k1iH~eWf>u^p=%5@ETgq)D3;NY7*q=Xh!c8*Lyr*XaSN40L0m%* z8E9BWhtp6jqgMZll#vvEtAF{*sMWt{8MXSCEu*&o(q+_04h3-yyFqyGO&wl0>@)EHC>f2tEYl|!cz=u8WhLqS|a zrx<8hMkQ$|meJ}MR0@B!6S|s1R}<)Z3zb7bTtmwZG%TZ4Noej3?1K>~p7uTHgdXJ3 zg9LiiLgi2p*Ul}KWK=b}>%190caSgp{pkWzZOGB}Y z`u>}@ZeuC@zW?^MZr^{?*6sUm+qwfRR1O7k4ekAJRYr}&GV1g1R2enKGMXHNO5sm- zLMLimQYv=_74a?|a8j5AqqtzX}jhaZ~X9-^&WkK3U+2^*9dbhg6G?H9P?hBgR zt<@c>lbVEq)IIx?yR*OOlPw--dEb*Q9&TChwGA3JYu>zR{l<-(+}ETD7E*B(EX|)K zeBdld!P(~)Jkf%2HMuWn@E8jYLuqUsU@WD@QLs#HuRg?Tmuummo{{z*&%jv(GJfF9FLD<-VZ7l@=U^(~dg8SWf4oVEIMJ5039bhb|u6eQ2l3#`_;R9zu3eG;a;DS6;T217La$nHk z&UvP^!jS5c{Yk2{u%t#u!SV}{C4As4NWt0X7CerC<%n`$(BP3490t{>I>1;|3!`B9 zk;oE0a2BNC>~jlVLcnrFxi4t&du;faQpCU(n!9795t=<~qPwTBo95`H{#HK5!PK;Ouh? zK1;xIM7b|$@CgeJOY3AEU@WZ;ZDOS*KN4BO2hM^NoPBP=o!gkwYAQ#R`+^3yYhy|) zEUot0pQK6)OKVsZEI$%i!UxWR6r6o-!6OM+jwts94IW~_VQCGm1B|6r8U@RbM3(S@ zvmga$pIh*J0+u7neL;g~S#Vfdv(w<*o7@{?V7xV)C4As4NWt0X7QB^!<%n`$(BKUQ zEKhS+)B(oQIvxefk3^R6fwLe5XP;Z}X#$ob%6&nDt1UPztz&h7v9#K@jg(e1`H{#H zK5!PK;Ouh??%39rmK;&;3mV+ItuC!*VQIC={v=Ua&9JlvN5S$VktKZKEJ(rG=N4RK z!8oGa7c_XF1&5_Is17ie*32kaek8Jl51a)lIQ!g!=Mb34PAGYAIw2ssP#?s1b7b`9Kk;oE0a2BNC>~jll*UpqyGdZH%7c}^$1&5{8s$IIY zu(Sq5!SW-KC4As4NWt0X7CeN2<%n`$(BS?V>$%(hR|?I-(kiS2jHNX#3YH&EJu|4f(DmZa9CPX)8O12-)mxEyj`3neBdld!P(~)yn%q_h;m=h;8g}J zPkpbh1B|70C<>Mzi7ep*XF&?iKDXd%0+u7neL;f{Sa4Wc2kQW1Y2Ao|4452_HBMQgHUU1+OJwIilPbG}B(j7LoCPU3``m&L6R;dn z?h6{c&w|6!s;UEwrFAt5mLG{M;R9zu3eG;a;F|<2N0j@52G?3}SXx)=0Ap$O?hq?2 z`H{#HK5!PK;Ouh??%%<*baOeP+!r*sX9rVSVQKZs{v=geSXvXKVEK{A5Y^S_K_>X*I1UKN4BO2hM^NoPBP=eLCvW${kVe z3mV+LqbjZJuX*d=(mk?2xjXxd^8XLhn%2Y88W#o2k3^R6fwLe5XP;Z}Bm$Nr%6&nD z$69b$TE%sMv9y*%!SW-KC4As4NWt0X7QBLh<%n`$(BLu)4ohos9bhc2T~V<7NMs2g zI15s6_PGV`BVakA+!r)>hXseFwX+T|mez$RSbik3gb$nrDLDJwf@=v_jwts94L)bV zVQHPO1B|8BEk9OT@*|NYeBdld!P(~)+%w;nmK;&;3mV)d-;`EZT3xe0NtG6s)|e<* zek8Jl51a)lIQ!g!#}lv|QSJ*GJj#N@(i)ux=iVS+6a(XJunCsSHD zqTClWIKPuFt@>eUb;|xEQCjt}v_?k3@*|NYeBdld!P(~)JeGjvh;m=h;Ncb=mezg{9RY`;$~@VQCdb!SW-KC4As4 zNWt0X7Cef8<%n`$(BPpK9G2FwG&uLB`J5OSZ#8ELA2a9CRF>i}bERY$?{BatP1;4DbN+2JN-Hd_yzEa>rG=$6Fbb9*i7ep*XF&?iKDXeZ z1T06C`+^1+T5woe1JdB!8|l+yV7&dDC4As4NWt0X7Cf7P<%n`$(BP>CEKj9Rs{@Rs zwKfWtABimC17|@B&OW!`3IdiR%6&nDS6gscT5IY6V`&|Zg5^gdOZdQ9kb<+%E%+D# z%Msj+qmDE9>oUTMK$ zX_eOj#?m?v1TlrB6{U$Z0TByDiYQ`wvRFX@DGG{;fS`zo1r-%L z!lrNfrf>SDZ~CTh`lfICruSsidvBZmyg!d~XCCL?`Of??=lst7zVA7{&N?&qXLjd1 zG6c47Nh6U4Mo>d`TdY z21ZZ?qb|W0CD?8v*WiQCNU)i-sv}@KX>}+J99nhwl0YI2jGzieU4pw5ic{KtgMn-C z!G(qXp;gCBTJ7;ku0zXCTEl{1z9f)H10$${QJ3IR3Ya&MYw*ECB-l(^LnB~2Y0U|O z`I0~)4UC`)MqPp{6fkch*WiO^NwAr;W=Ftw(%KvZ^Cf{q8W=$pjJgDGQ^355T!RnZ zD8XjZ+7toXN$YeF%$EcbXr@16C#`nvLx&b$5=f+h z5mdpbOK`{b^3dW<(H{3*1#Z` zF9{^lzzC{f)FpU`0_IKR8hmhn2{x0~fC$)5TGNAIz9f)H10$${QJ3IZ3Ya&MYw*ES zCD=?_(;{FyX{`-{`I0~)4UC`)MqPq8DqxqrD_C#@D8Lx&b$5=f+h5mdpbOK_Wx^3dW<^(2iClvZ?k>RmMtqM5*iKsGgJ8ZSkVpd~sDe?K;7JOYH<4@b!DA)Z zOj_e2U^{6o4TAZSKq3u{pbAD^f>$VD-bAj!2QQXjGifb}fbFEUCkWhFmEE);DfhIu$i=WM8I~^x)=oWC4od57(o?`x&$}u zDi1B*M6SUHpO;`WX& zoYMA-7F>f5ZeJu0Ei-9#z$dv5Ejwuq4}$rUKq3u{pbAD^f=d-JZz9*=gNI76nY4yQ zz;@D_8wB$ufkYY@K^2U;1TRp)yop?c51uW-X40Ay0ozGyOAySL1QKar1XVEV61-gj z^Cof)K6sM^n@MYP1Z*d*GeIz45=f+h5mdpbOYnIG%$vwH_~26#Y$mPKxxwy@_`+_1 zL(4vIjYJw4K^2U;1b6Bt4=vtAuE7Vl?dBg^dFJKPcK9U6p_OMRtsy}$UlK^9fe}=} zs7vq&15wM-Kjs?Md zNg$C1MotuNTh)gRKciA@L&bZo5(fz;Jy-U zCar!Eu${D~2ElwuAdv<}Pz9qd!Q~2=H<4@b!ILG}Oj=VSU^{884ubiTKq3u{pbAD^ zg4YYM+eEIx2d|W1Gij}gfbFDpI0)uT0*N#*f+`qw2|liXc@wz?AAC@P&7^fG0=AP@ z^PZtYi!TWz(!dC+VALhJRZsuWYEXkWk!$e5O?!$%%S>9$@JX&i%T8LwK`>tuNTh)g zRKciA@BjtOo5(fz;NB8!CapdZu${Cf1;KntAdv<}Pz9qd!P68lZz9*=gC|O`nY79x zU^{882!i>NKq3u{pbAD^g4ZZu-bAj!2Ukk4nY5Niz;@C)5CrojfkYY@K^2U;1Rqhr zyop?c58fxiX42Z98|>bQZ_+DpXxZnjkw^n0sDe?K;DTQA(Be(x8hmi0UjCuQZ^Spo zCpiwSns(Ca6$JAofkYY@K^2U;1ou_Iyop?c5AGqsX42{z0ozGyLJ-WC1QKar1XVEV z5>M8I~^YS=q;Xz?Y1L>d@D6^yzBH|;GB zt(v@vT!Rn3EWu{d%I}@~(6WowODQ!F)*|kp@Ok1*0y( zl?s?Qk!$e53nkc0T8koJJ8A6-g87m_A`Oh73PxRm_bFiBM6SUH?~q_KY39g4-FWhSkT_$1e%Whbo>K`>tuNTh)gRKciA z@Ms0ho5(fz;9(MMCavKSu${E#1;KntAdv<}Pz9qd!3z~IZz9*=gXc)FnY89cz;@Ew z8U*ttfkYY@K^2U;1n*G5yop?c58f=nX42Y{8(iD|MtpS$Y@fGAA`Oh73PxRmFDPK% zM6SUHpB7+#BmPVTY$vVueFKM9ZN4OsNCP9Nf>D>?&VA*f#hb`A_~3SZ{X?s^nY0S= zNsdFSww<(w2ElwuAdv<}Pz9qd!6OwgZz9*=gG(gXOj<)CU^{8e4uW~oLLv=}pbAD^ zg6B)H-9)az2hWsXGil9=fbFEUDG26E0*N#*f+`qw39eGWyop?c58fcbX42Xi0ozII zR1nOU1QKar1XVEV5`0zx^Cof)KKO(Ln@Q_r1Z*d*w*5kf7GDxbq=6As!Kh1chkoMF zs?D3oHTdAx{luYVCapI3B-f#3C#}IjFkcc#q=6As!Kh2{Fa^w;$Tj%jff8&ctw9m6 zowUk>V7?@fNCP9Nf>D>?ISQCJk!$e5(`um4g9W!aQ#3wlptvYtn8W05YC4od57(o?` zx&)UfVBSQo!3Xz~U^8j;kAUr@H7y9{O9F{BFoG%=bqSs+!FCh51|K{{g3Y8gH3GJi z)|w!gF9{^lzzC{f)FpU>0_I7}HTd9F5^N@|)e*3rw2lP9d`TdY21ZZ?qb|WG6fkch z*WiN>NwAr;4oARt(kd7bI<)wbKq3u{pbAD^f?E#|hgKclM6SUHHya=hEi-8~$0xZC zEjwxT4TAZSKq3u{pbAD^f(I&K-bAj!2ltU+Gien^z;@D_90c0V7?@fNCP9Nf>D>?RSK9V zE!W_KmrAgiw3bD{cGB7x1oI_d@D6^yzBA5y@)iClvZ-XpJr>+kT|sJ@+NW(J~)4nIJC^9)exWLI<)Mh)guVzO9F{BFoG%=bqVgH zfO!+S1|Qr_g3YAWJp#6q*0>;;F9{^lzzC{f)Frq~0rMtu4L*2`1e-}~Yy@m4ttCM) zUlK^9fe}=}s7vs21)AtuNTh)gRKciA@E8Tmo5(fz;NcQ% zCan<>u${E#2f=(vAdv<}Pz9qd!HX0yZz9*=gXc=HnY8BR2D>-nt3qJ=yfqSOU<6e# z>Jq$D0rMtu4L*2_0P`F1TO(jQX`Ky%`I0~)4UC`)MqPq0Dq!A3uE7VNkzg}vRY$;f z(&{iIaA?)%O9F{BFoG%=bqVe=M4r;TiClvZE*#D>?Z3>t-k!$e58ztCGTALzZJ87K`g87m_A`Oh73PxRm z&naNuM6SUHpOj!TX`PCI?WEOiXz0-5O9F{BFoG%=bqVe`R2*9Nixymi4{kG599m}5 zYKu>D9a?tMDhY!5l0YI2jGzieU4n-zVBSQo!3PhLU^8hAj)3i?H8Tk2O9F{BFoG%= zbqSuUfO!+S1|K{_g3Y8=o*V4mh~E$b+vly3NCP9Nf>D>?Eee=7k!$e5>jaqJh+iK8 z+ezz05X_eZ5@}!rRWRxjd`1EDCUOlv_?QHnN$Yq7Y$vVO!vcqvecl?0G%$iH78RNTh)gRKciA@GJ$)o5(fz;HeU9Caq}^u${El2ElwuAdv<}Pz9qd z!5bAYPg<_Q2d|c3Gij}ffbFDpGzjKP0*N#*f+`qw2|lTSc@wz?AADGX&7^fC0=AP@ zi{YU|i!TWz(!dC+VALhJ&2VvOHQ-I;8hmi`;o{ITlU4yf$#rPiNvmHF%$EcbXF5=f+h5mdpbOK^V)wwuT`_~2d=Y$mPV5wM-K%7S3NB#=l0BdCH= zm*A-im^YDY@WB%#*i2dzBVapeEf0eEl0YI2jGzieU4mCDV4k#GgAZON!DiB`jDYQ= zwLb{vO9F{BFoG%=bqPMKfO!+S1|Pgvg3YA0F9NocR^ySOLyIp7B+|eLs$kS5xcNwN zXyx%Hat%JX;Ye|4nMtb=KFM`x*-5Ks5X_eZ5@}!rRWRxjT&#e36S)Q-++Bjrq}3w= zwv*QQAeb)+B+|eLs$kS5c#;Cd@D6^yzBHyI@lE#5?~!3STGU^8i5j)3i?)h!6-O9F{BFoG%= zbqVezz-|+{1|QsYly^$!)i9G*5k3i@(s?!Pq%|f8=1T&JG%$iH7D>?y$YB&k!$e5+a=gcT00_OJ84}Eg87m_A`Oh73PxRm8Jq$N0rMtu4L*321e-}~ za|CQBtusL|UlK^9fe}=}s7vs91U2r$oSZ(pnt^^Cf{q8W=$pjJgD`SHQf9T!Rl@DZysaS``7? zN$YSB%$EcbXre!2C#~k=1BaG<-WrKCFoG%=bqQ`Y zUL0C>6S)Q-+;qHuXw^28Rx^B(D>?X$qJ(k!$e56D8P8T4fQiowQa2!F)*|kp@Ok1*0y( zYZNd~TCTwdS4yy%w3bJ}cG5Z!1oI_d@D6^yzBA5p-(iClvZ-Y3Ck(%K&Z+exd* zgwUbImjn`NU<6e#>JnTqK^|JXiClvZZZtt0T4vH}j8AeMT6WUv6$JAofkYY@K^2U; z1ou_Iyop?c5AGqsX42}J8|>bQpAZ7u=dF=Q10$${QJ3J!3Ya&MYw*G21eo85A0GkR zNo!dU%$EcbX;ead21xnzzC{f)Frs-L~&@@P2?JU@MQ@$lUDx3oQGB& zJ85+fg87m_A`Oh73PxRmdn;hxM6SUH7lHLZ-#?sJ$4pw?B49gdjSYhNl0YI2jGzie zU4kbnVBSQo!3U3)U^8iriGb~-wKxdoO9F{BFoG%=bqTIiz&vTW1|Pgog3YA0C<3;V z)~+C!F9{^lzzC{f)FpVI0_IKR8hr2$2{x0~&Is5}T9<-gz9f)H10$${QJ3IGW%AJC zP2?JU@C6Arlh(xu*iKqq%R;9#UlK^9fe}=}s7r7U1lU5gel4G=V zZ^V~|!1j4-B+|eLs$kS5c$@;}P2?JU@JIpXH{wS{z;@DF5CrojfkYY@K^2U;1TRs* zyop?c51ucwKWLlO9F{BFoG%=bqU^~fO!+S z1|Pgxg3YA0B?7jSR&@}}mjn`NU<6e#>JofG0rMtu4LPv5lU5-<$#H18H{yqe!1j4-B+|eLs$kS5c%%a6 zP2?JUaESodGk<$HBm%aR*6bjdCoLq>zzC{f)FpVn0_IKR8hr3f2{x0~tO(dnTAPAk zz9f)H10$${QJ3H<1AKZG1e`wV+lU5silIzg2 zlh)uMm@f$=(!dC+VALgem;&Zad@D6^yzBpH{%UiClvZJ}SXx(mECa+exd{)X<^Dmjn`NU<6e#>Jr>;sywuK6S)Q- z++wOYw9KT{5})KawA>r<143Z?yfqSOU<6e#>JnU{fO!+S1|Qr{fccI1{t>X9w5A2Y zd`TdY21ZZ?qb|WS6)u>~YC#`~MfkVqaZ;eD6 z7(o?`x&*hLCJwFoyop?c4{kQiKeXzbNvkg87m_A`Oh73PxRm_bXuDM6SUH@04IOY3+)D?WA=%2D>?@d}tXk!$e5qa@f&TBQ-NowODP!F)*|kp@Ok z1*0y(OBFD0BG=%9DjzJ8A6*g87m_A`Oh73PxRm_eikaM6SUHZE^y z?WA=f2EI<)MhH8Ke1O9F{BFoG%=bqOA$fO!+S1|K|Jg3Y8g zA_BIP*8CutF9{^lzzC{f)FpV40_IKR8hr5Fnc^pGCarn+B-c;aPFht#Fkcc#q=6As z!Kh2{P6f=H$Tj%jEfQ=dt*sHTowUvd!F)*|kp@Ok1*0y(7ZosXBG=%9&q%PDw5kR8 zhX3z7#B0`YkHmMFV;;vvB#n%qN=99hyUdd3G;br<;FAky;lI@Wf6o6t6W<;m z{qKLPy_}u4hK0?1Qy`LNMo=}QF3qD9Gw&qV;G2giW;1UMjhXGhH79K5%L0)!GlHra zb!o0p%)FUggKwUtm}`~n98h`U>e<;x|L4E|_nr7J!z(o&s_{_$S~VW3U8_c&8n4p- z3mv@b3c8iH`2Nr>FAAAa1#~Z zL}fQoIJir=cIG*&nA(AE#3vr?$7y;9IEt7FvG` z9lwQI-AV&)rD?a)np^3}tyFLu^}UTI-$pBMql33m(@#+EPte3qP~|6R-zTWi?bPFT z8h1M_xt(_3PM2?|qEFK3Ptw9q(vDBkg-=rFJ80w`H2)5&x`WQ%K^;Ct!#+iGK1G{9 zMW;VS?LJKwx)bf7n ze?LvVpH|;bhwrE64^Z&~H0c3a@cRp2-)}YE7 zw66v=s!2U+(zu$mq$cgINtbIix$?R9ku8}E$UpGM%JeJwW+E$ovlqB>d>${ zG^Y-2u0yBmP`kQRQkQ1dr44oIL|tlKj|SGG>Gf!BJvv&CTGXe0^=V3dT2-G8)u(0+ zs80hbYe35z(EbM0IFEYf(fB-Cnn!!`DE|@a_6Uu6gcdzQJ0GEok5HGd(5SCa#aC$C zSLobVsN+{@_*ZG}S82;v>C9KD@M|>WYc%U?wDD_n@@v%Q>on-=G~?^E?(1~y>(ufa z)c+ea^&7PM8+7;^)cl)N{7stlOijH?e3s@vOI6R(*=MQ4 zcWKynY0h_P^LOd=cd6a?sN{Pz^Lw=6dvxM^)cQFZ_#91tj@CX$N1vk>&r`qWY0C4o z>Ulc!JT?11_4z)PeV>+ppZ0&B8vlTL{(#2+fR_G%_WXeIU!ZO;(3lr!(F?Tm1-keG zb@?HU`XN>PkhcAh&i#-&{)mSEi01x?w)}|B{D=yFOhbN5vwln)e@rKTOl@AIK`+vb z7iry#bnHcH`4j5@6Po%HTKy9`{1a;a5*5EhlU||~FVTUQsL9LJ>t&koGA(o?kpHcbGX#LOV_|K@-&uPHVY1+?e&Clt`&#B;b>iarPew|jnP6uD7 zroW)xzo3b~pvqs+zF$zIH>k%OH0}*r@&@gGgD$^8MZcuczodn~q#eJc3%{h!Z_>y& zY5tp3^(LKtlREs0hW(1>{E9aJicbHE+Py_3Z_&)RXv14{;w@_ZYZ~}#n*M8A`)fM- zYijWu>h~L(@*7(98#?qGYW7>|^IIzWEiL~o?f)$`{vGxF9gY7TE&Uzs`5ooIP2Jw6 zF>lkNw`u3wbn$KK@_QQfd#d<7ZTmf)`#p910}cNJ&HV#y`2(H#0~P*}hWwFc{gF2Q zkxu@R+Pp)9-k}-q(7Jc%*gMqnPt^ZUH1$uk`cHKDPt^R+RQzX}^k-V}XFBj_YVsHA z^%t7(7h3ig+WQx3_%3ySm&U$Ji{GVP@6x4rsq0^<^sltwueANIbpEf@>2EaRZ#3_3 zwDoUP{Wog=cN+S4n*Dd$^mjV-cWV0&8vG9`{|BxA2Oa+hwfZLw_$N*KC$0G>9r-5} z{EPbjizfezR{o0){)?Lan|lA7CjOf$|4sY;O^x289`Dh(_h`v`wEI1}{2mqkherR0 z7XF8J{D&_5hdRGcBj2a_?^D(LboPDf@Bt0`faZKan?In_A5gn|q7tl`SR1fTV6|>Y zG!Sb#)>^EiSS=b6^~0KiwF>JHRd z+JK{#bLdwqTvXD(s7&Kh`X)jaVnK+VsQEA8Q8II;>+@E&Jo=k2Mu*HP&IQ z<^%Ba$C`w-0_y-)lY#j8V@<$XhP4-~;UN6{vBqL8#@dB-39IX1{QR*NU~R`bkJYIJ zKYy%wSX;5GvDy#8&mU_x)+Ve|SZ#;m=Z{s6m3>=$_66$McZz3Uvz>kOboOP;+4mD? zU)h^|i*5GBuGx2;W?%oAeIsV}C6U?pCT3qHn0>om_JwoVcd%t&OO}08SN7#q+4nhR zUvZRu>reJYIN5i@WM7w(eS=8$r5f4yOk`iZkbN6L_CSC3W_|XId-k$<_Skp!{&n^w zboNSe_Hc0amTvaEZ1&=6_6TYAE@$>sW%hbw_Ml<*#$WdAUG@@O_BdMhURd^oR`#k? z_Rv!H_E7d5PxeAh_GnA?j!E{kNA}u9_5eioCPMa1K=zmO*&mZ*)J<(|9+qSlXmv6yxBi| zX8#tL{qtD%FGkrvqGbQBko`~X+5h^S{SUI)|3=wqCVu`{^RTvJRbyrUFLCz&B4+J5cMn@2!u~0Br1wU6hN^Ol+d6+h-iQc`bpLaWD#kMyf`$AHQwoDg zL=|<5wfOe_`QVYIznCi}P!ZUpicAm_4zOAtJdZ^%SYU_y8=J>jxOWhq(j?(wZXk)4 zY)nNnozQQ}R7X2_5iQD7=C9W6vp|VGiQ;qwbCjYKF!UV7Jts`*U!pUf|MMrotW6NH z3l>p1f=pY4rAGhTl4Gd8?B*?n;h4)jXBlnI;}m5)BIck7HMUU|ejiQi{P5ZA+b~#e zv|8+NF9?I&V{q12I>6#c1C24udI1XIdRRT{r{XBf)0gxHT-o|g3CRVOno$L|i@T>), + #[error("update not fully consumed: {0}")] + UpdateNotFullyConsumed(usize), + #[error("invalid struct clock, expect {expect}, actually {actually}")] + StructClockInvalid { expect: u64, actually: u64 }, + #[error("cannot find struct {clock} in {client_id}")] + StructSequenceInvalid { client_id: u64, clock: u64 }, + #[error("struct {0} not exists")] + StructSequenceNotExists(u64), + #[error("Invalid parent")] + InvalidParent, + #[error("Parent not found")] + ParentNotFound, + #[error("Invalid struct type, expect item, actually {0}")] + InvalidStructType(&'static str), + #[error("Can not cast known type to {0}")] + TypeCastError(&'static str), + #[error("Can not found root struct with name: {0}")] + RootStructNotFound(String), + #[error("Index {0} out of bound")] + IndexOutOfBound(u64), + #[error("Document has been released")] + DocReleased, + #[error("Unexpected type, expect {0}")] + UnexpectedType(&'static str), +} + +pub type JwstCodecResult = Result; diff --git a/packages/common/y-octo/core/src/protocol/awareness.rs b/packages/common/y-octo/core/src/protocol/awareness.rs new file mode 100644 index 0000000000..f22ed41e71 --- /dev/null +++ b/packages/common/y-octo/core/src/protocol/awareness.rs @@ -0,0 +1,151 @@ +use nom::{multi::count, Parser}; + +use super::*; + +const NULL_STR: &str = "null"; + +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +pub struct AwarenessState { + #[cfg_attr(test, proptest(strategy = "0..u32::MAX as u64"))] + pub(crate) clock: u64, + // content is usually a json + pub(crate) content: String, +} + +impl AwarenessState { + pub fn new(clock: u64, content: String) -> Self { + AwarenessState { clock, content } + } + + pub fn clock(&self) -> u64 { + self.clock + } + + pub fn content(&self) -> &str { + &self.content + } + + pub fn is_deleted(&self) -> bool { + self.content == NULL_STR + } + + pub(crate) fn add_clock(&mut self) { + self.clock += 1; + } + + pub(crate) fn set_clock(&mut self, clock: u64) { + self.clock = clock; + } + + pub fn set_content(&mut self, content: String) { + self.add_clock(); + self.content = content; + } + + pub fn delete(&mut self) { + self.set_content(NULL_STR.to_string()); + } +} + +impl Default for AwarenessState { + fn default() -> Self { + AwarenessState { + clock: 0, + content: NULL_STR.to_string(), + } + } +} + +fn read_awareness_state(input: &[u8]) -> IResult<&[u8], (u64, AwarenessState)> { + let (tail, client_id) = read_var_u64(input)?; + let (tail, clock) = read_var_u64(tail)?; + let (tail, content) = read_var_string(tail)?; + + Ok((tail, (client_id, AwarenessState { clock, content }))) +} + +fn write_awareness_state( + buffer: &mut W, + client_id: u64, + state: &AwarenessState, +) -> Result<(), IoError> { + write_var_u64(buffer, client_id)?; + write_var_u64(buffer, state.clock)?; + write_var_string(buffer, state.content.clone())?; + + Ok(()) +} + +pub type AwarenessStates = HashMap; + +pub fn read_awareness(input: &[u8]) -> IResult<&[u8], AwarenessStates> { + let (tail, len) = read_var_u64(input)?; + let (tail, messages) = count(read_awareness_state, len as usize).parse(tail)?; + + Ok((tail, messages.into_iter().collect())) +} + +pub fn write_awareness(buffer: &mut W, clients: &AwarenessStates) -> Result<(), IoError> { + write_var_u64(buffer, clients.len() as u64)?; + + for (client_id, state) in clients { + write_awareness_state(buffer, *client_id, state)?; + } + + Ok(()) +} + +// TODO(@darkskygit): impl reader/writer +// awareness state message +#[allow(dead_code)] +#[derive(Debug, PartialEq)] +pub struct AwarenessMessage { + clients: AwarenessStates, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_awareness() { + let input = [ + 3, // count of state + 1, 5, 1, 1, // first state + 2, 10, 2, 2, 3, // second state + 5, 5, 5, 1, 2, 3, 4, 5, // third state + ]; + + let expected = HashMap::from([ + ( + 1, + AwarenessState::new(5, String::from_utf8(vec![1]).unwrap()), + ), + ( + 2, + AwarenessState::new(10, String::from_utf8(vec![2, 3]).unwrap()), + ), + ( + 5, + AwarenessState::new(5, String::from_utf8(vec![1, 2, 3, 4, 5]).unwrap()), + ), + ]); + + { + let (tail, result) = read_awareness(&input).unwrap(); + assert!(tail.is_empty()); + assert_eq!(result, expected); + } + + { + let mut buffer = Vec::new(); + // hashmap has not a ordered keys, so buffer not equal each write + // we need re-parse the buffer to check result + write_awareness(&mut buffer, &expected).unwrap(); + let (tail, result) = read_awareness(&buffer).unwrap(); + assert!(tail.is_empty()); + assert_eq!(result, expected); + } + } +} diff --git a/packages/common/y-octo/core/src/protocol/doc.rs b/packages/common/y-octo/core/src/protocol/doc.rs new file mode 100644 index 0000000000..143762b6b7 --- /dev/null +++ b/packages/common/y-octo/core/src/protocol/doc.rs @@ -0,0 +1,103 @@ +use super::*; + +// doc sync message +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +pub enum DocMessage { + // state vector + // TODO: temporarily skipped in the test, because yrs decoding needs to ensure that the update + // in step1 is the correct state vector binary and any data can be included in our + // implementation (we will ensure the correctness of encoding and decoding in the subsequent + // decoding process) + #[cfg_attr(test, proptest(skip))] + Step1(Vec), + // update + Step2(Vec), + // update + Update(Vec), +} + +const DOC_MESSAGE_STEP1: u64 = 0; +const DOC_MESSAGE_STEP2: u64 = 1; +const DOC_MESSAGE_UPDATE: u64 = 2; + +pub fn read_doc_message(input: &[u8]) -> IResult<&[u8], DocMessage> { + let (tail, step) = read_var_u64(input)?; + + match step { + DOC_MESSAGE_STEP1 => { + let (tail, sv) = read_var_buffer(tail)?; + // TODO: decode state vector + Ok((tail, DocMessage::Step1(sv.into()))) + } + DOC_MESSAGE_STEP2 => { + let (tail, update) = read_var_buffer(tail)?; + // TODO: decode update + Ok((tail, DocMessage::Step2(update.into()))) + } + DOC_MESSAGE_UPDATE => { + let (tail, update) = read_var_buffer(tail)?; + // TODO: decode update + Ok((tail, DocMessage::Update(update.into()))) + } + _ => Err(nom::Err::Error(Error::new(input, ErrorKind::Tag))), + } +} + +pub fn write_doc_message(buffer: &mut W, msg: &DocMessage) -> Result<(), IoError> { + match msg { + DocMessage::Step1(sv) => { + write_var_u64(buffer, DOC_MESSAGE_STEP1)?; + write_var_buffer(buffer, sv)?; + } + DocMessage::Step2(update) => { + write_var_u64(buffer, DOC_MESSAGE_STEP2)?; + write_var_buffer(buffer, update)?; + } + DocMessage::Update(update) => { + write_var_u64(buffer, DOC_MESSAGE_UPDATE)?; + write_var_buffer(buffer, update)?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_doc_message() { + let messages = [ + DocMessage::Step1(vec![0x01, 0x02, 0x03]), + DocMessage::Step2(vec![0x04, 0x05, 0x06]), + DocMessage::Update(vec![0x07, 0x08, 0x09]), + ]; + + for msg in messages { + let mut buffer = Vec::new(); + + write_doc_message(&mut buffer, &msg).unwrap(); + let (tail, decoded) = read_doc_message(&buffer).unwrap(); + + assert_eq!(tail.len(), 0); + assert_eq!(decoded, msg); + } + + // test invalid msg + { + let mut buffer = Vec::new(); + let msg = DocMessage::Step1(vec![0x01, 0x02, 0x03]); + + write_doc_message(&mut buffer, &msg).unwrap(); + buffer[0] = 0xff; // Inject error in message tag + let res = read_doc_message(&buffer); + + match res.as_ref().unwrap_err() { + nom::Err::Error(error) => assert_eq!(error.code, ErrorKind::Tag), + _ => panic!("Expected error ErrorKind::Tag, but got {:?}", res), + } + } + } +} diff --git a/packages/common/y-octo/core/src/protocol/mod.rs b/packages/common/y-octo/core/src/protocol/mod.rs new file mode 100644 index 0000000000..1451283f73 --- /dev/null +++ b/packages/common/y-octo/core/src/protocol/mod.rs @@ -0,0 +1,23 @@ +mod awareness; +mod doc; +mod scanner; +mod sync; + +use std::{ + collections::HashMap, + io::{Error as IoError, Write}, +}; + +use awareness::{read_awareness, write_awareness}; +pub use awareness::{AwarenessState, AwarenessStates}; +pub use doc::DocMessage; +use doc::{read_doc_message, write_doc_message}; +use log::debug; +use nom::{ + error::{Error, ErrorKind}, + IResult, +}; +pub use scanner::SyncMessageScanner; +pub use sync::{read_sync_message, write_sync_message, SyncMessage}; + +use super::*; diff --git a/packages/common/y-octo/core/src/protocol/scanner.rs b/packages/common/y-octo/core/src/protocol/scanner.rs new file mode 100644 index 0000000000..e8af51f076 --- /dev/null +++ b/packages/common/y-octo/core/src/protocol/scanner.rs @@ -0,0 +1,64 @@ +use super::*; + +pub struct SyncMessageScanner<'a> { + buffer: &'a [u8], +} + +impl SyncMessageScanner<'_> { + pub fn new(buffer: &[u8]) -> SyncMessageScanner { + SyncMessageScanner { buffer } + } +} + +impl<'a> Iterator for SyncMessageScanner<'a> { + type Item = Result>>; + + fn next(&mut self) -> Option { + if self.buffer.is_empty() { + return None; + } + + match read_sync_message(self.buffer) { + Ok((tail, message)) => { + self.buffer = tail; + Some(Ok(message)) + } + Err(nom::Err::Incomplete(_)) + | Err(nom::Err::Error(nom::error::Error { + code: nom::error::ErrorKind::Eof, + .. + })) + | Err(nom::Err::Failure(nom::error::Error { + code: nom::error::ErrorKind::Eof, + .. + })) => { + debug!("incomplete sync message"); + None + } + + Err(e) => Some(Err(e)), + } + } +} + +#[cfg(test)] +mod tests { + use proptest::{collection::vec, prelude::*}; + + use super::*; + + proptest! { + #[test] + #[cfg_attr(miri, ignore)] + fn test_sync_message_scanner(messages in vec(any::(), 0..10)) { + let mut buffer = Vec::new(); + + for message in &messages { + write_sync_message(&mut buffer, message).unwrap(); + } + + let result: Result, _> = SyncMessageScanner::new(&buffer).collect(); + assert_eq!(result.unwrap(), messages); + } + } +} diff --git a/packages/common/y-octo/core/src/protocol/sync.rs b/packages/common/y-octo/core/src/protocol/sync.rs new file mode 100644 index 0000000000..3f57fbf9f6 --- /dev/null +++ b/packages/common/y-octo/core/src/protocol/sync.rs @@ -0,0 +1,165 @@ +use byteorder::WriteBytesExt; + +use super::*; + +#[derive(Debug, Clone, PartialEq)] +enum MessageType { + Auth, + Awareness, + AwarenessQuery, + Doc, +} + +fn read_sync_tag(input: &[u8]) -> IResult<&[u8], MessageType> { + let (tail, tag) = read_var_u64(input)?; + let tag = match tag { + 0 => MessageType::Doc, + 1 => MessageType::Awareness, + 2 => MessageType::Auth, + 3 => MessageType::AwarenessQuery, + _ => return Err(nom::Err::Error(Error::new(input, ErrorKind::Tag))), + }; + + Ok((tail, tag)) +} + +fn write_sync_tag(buffer: &mut W, tag: MessageType) -> Result<(), IoError> { + let tag: u64 = match tag { + MessageType::Doc => 0, + MessageType::Awareness => 1, + MessageType::Auth => 2, + MessageType::AwarenessQuery => 3, + }; + + write_var_u64(buffer, tag)?; + + Ok(()) +} + +// sync message +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +pub enum SyncMessage { + Auth(Option), + Awareness(AwarenessStates), + AwarenessQuery, + Doc(DocMessage), +} + +pub fn read_sync_message(input: &[u8]) -> IResult<&[u8], SyncMessage> { + let (tail, tag) = read_sync_tag(input)?; + + let (tail, message) = match tag { + MessageType::Doc => { + let (tail, doc) = read_doc_message(tail)?; + (tail, SyncMessage::Doc(doc)) + } + MessageType::Awareness => { + let (tail, update) = read_var_buffer(tail)?; + ( + tail, + SyncMessage::Awareness({ + let (awareness_tail, awareness) = read_awareness(update)?; + let tail_len = awareness_tail.len(); + if tail_len > 0 { + debug!("awareness update has trailing bytes: {}", tail_len); + debug_assert!(tail_len > 0, "awareness update has trailing bytes"); + } + awareness + }), + ) + } + MessageType::Auth => { + let (tail, success) = read_var_u64(tail)?; + + if success == 1 { + (tail, SyncMessage::Auth(None)) + } else { + let (tail, reason) = read_var_string(tail)?; + (tail, SyncMessage::Auth(Some(reason))) + } + } + MessageType::AwarenessQuery => (tail, SyncMessage::AwarenessQuery), + }; + + Ok((tail, message)) +} + +pub fn write_sync_message(buffer: &mut W, msg: &SyncMessage) -> Result<(), IoError> { + match msg { + SyncMessage::Auth(reason) => { + const PERMISSION_DENIED: u8 = 0; + const PERMISSION_GRANTED: u8 = 1; + + write_sync_tag(buffer, MessageType::Auth)?; + if let Some(reason) = reason { + buffer.write_u8(PERMISSION_DENIED)?; + write_var_string(buffer, reason)?; + } else { + buffer.write_u8(PERMISSION_GRANTED)?; + } + } + SyncMessage::AwarenessQuery => { + write_sync_tag(buffer, MessageType::AwarenessQuery)?; + } + SyncMessage::Awareness(awareness) => { + write_sync_tag(buffer, MessageType::Awareness)?; + write_var_buffer(buffer, &{ + let mut update = Vec::new(); + write_awareness(&mut update, awareness)?; + update + })?; + } + SyncMessage::Doc(doc) => { + write_sync_tag(buffer, MessageType::Doc)?; + write_doc_message(buffer, doc)?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{awareness::AwarenessState, *}; + + #[test] + fn test_sync_tag() { + let messages = [ + MessageType::Auth, + MessageType::Awareness, + MessageType::AwarenessQuery, + MessageType::Doc, + ]; + + for msg in messages { + let mut buffer = Vec::new(); + + write_sync_tag(&mut buffer, msg.clone()).unwrap(); + let (tail, decoded) = read_sync_tag(&buffer).unwrap(); + + assert_eq!(tail.len(), 0); + assert_eq!(decoded, msg); + } + } + + #[test] + fn test_sync_message() { + let messages = [ + SyncMessage::Auth(Some("reason".to_string())), + SyncMessage::Awareness(HashMap::from([(1, AwarenessState::new(1, "test".into()))])), + SyncMessage::AwarenessQuery, + SyncMessage::Doc(DocMessage::Step1(vec![4, 5, 6])), + SyncMessage::Doc(DocMessage::Step2(vec![7, 8, 9])), + SyncMessage::Doc(DocMessage::Update(vec![10, 11, 12])), + ]; + + for msg in messages { + let mut buffer = Vec::new(); + write_sync_message(&mut buffer, &msg).unwrap(); + let (tail, decoded) = read_sync_message(&buffer).unwrap(); + assert_eq!(tail.len(), 0); + assert_eq!(decoded, msg); + } + } +} diff --git a/packages/common/y-octo/core/src/sync.rs b/packages/common/y-octo/core/src/sync.rs new file mode 100644 index 0000000000..d670f32847 --- /dev/null +++ b/packages/common/y-octo/core/src/sync.rs @@ -0,0 +1,32 @@ +#[allow(unused)] +#[cfg(not(loom))] +pub(crate) use std::sync::{ + atomic::{AtomicBool, AtomicU16, AtomicU32, AtomicU8, Ordering}, + Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard, +}; +pub use std::sync::{Arc, Weak}; +#[cfg(all(test, not(loom)))] +pub(crate) use std::{ + sync::{atomic::AtomicUsize, MutexGuard}, + thread, +}; + +#[cfg(loom)] +pub(crate) use loom::{ + sync::{ + atomic::{AtomicBool, AtomicU16, AtomicU8, AtomicUsize, Ordering}, + Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard, + }, + thread, +}; + +#[macro_export(local_inner_macros)] +macro_rules! loom_model { + ($test:block) => { + #[cfg(loom)] + loom::model(move || $test); + + #[cfg(not(loom))] + $test + }; +} diff --git a/packages/common/y-octo/node/.gitignore b/packages/common/y-octo/node/.gitignore new file mode 100644 index 0000000000..28f2c68269 --- /dev/null +++ b/packages/common/y-octo/node/.gitignore @@ -0,0 +1,2 @@ +*.node +.coverage \ No newline at end of file diff --git a/packages/common/y-octo/node/Cargo.toml b/packages/common/y-octo/node/Cargo.toml new file mode 100644 index 0000000000..103fb2ba3f --- /dev/null +++ b/packages/common/y-octo/node/Cargo.toml @@ -0,0 +1,20 @@ +[package] +authors = ["DarkSky "] +edition = "2021" +license = "MIT" +name = "y-octo-node" +repository = "https://github.com/toeverything/y-octo" +version = "0.0.1" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +crate-type = ["cdylib"] + +[dependencies] +anyhow = { workspace = true } +napi = { workspace = true, features = ["anyhow", "napi4"] } +napi-derive = { workspace = true } +y-octo = { workspace = true, features = ["large_refs"] } + +[build-dependencies] +napi-build = { workspace = true } diff --git a/packages/common/y-octo/node/build.rs b/packages/common/y-octo/node/build.rs new file mode 100644 index 0000000000..bbfc9e4b9e --- /dev/null +++ b/packages/common/y-octo/node/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/packages/common/y-octo/node/index.d.ts b/packages/common/y-octo/node/index.d.ts new file mode 100644 index 0000000000..d499d57655 --- /dev/null +++ b/packages/common/y-octo/node/index.d.ts @@ -0,0 +1,48 @@ +/* auto-generated by NAPI-RS */ +/* eslint-disable */ +export declare class Doc { + constructor(clientId?: number | undefined | null) + get clientId(): number + get guid(): string + get keys(): Array + getOrCreateArray(key: string): YArray + getOrCreateText(key: string): YText + getOrCreateMap(key: string): YMap + createArray(): YArray + createText(): YText + createMap(): YMap + applyUpdate(update: Uint8Array): void + encodeStateAsUpdateV1(state?: Uint8Array | undefined | null): Uint8Array + gc(): void + onUpdate(callback: (result: Uint8Array) => void): void +} + +export declare class YArray { + constructor() + get length(): number + get isEmpty(): boolean + get(index: number): T + insert(index: number, value: YArray | YMap | YText | boolean | number | string | Record | null | undefined): void + remove(index: number, len: number): void + toJson(): JsArray +} + +export declare class YMap { + constructor() + get length(): number + get isEmpty(): boolean + get(key: string): T + set(key: string, value: YArray | YMap | YText | boolean | number | string | Record | null | undefined): void + remove(key: string): void + toJson(): object +} + +export declare class YText { + constructor() + get len(): number + get isEmpty(): boolean + insert(index: number, str: string): void + remove(index: number, len: number): void + get length(): number + toString(): string +} diff --git a/packages/common/y-octo/node/index.js b/packages/common/y-octo/node/index.js new file mode 100644 index 0000000000..1e41d1f1ee --- /dev/null +++ b/packages/common/y-octo/node/index.js @@ -0,0 +1,377 @@ +// prettier-ignore +/* eslint-disable */ +// @ts-nocheck +/* auto-generated by NAPI-RS */ + +const { createRequire } = require('node:module') +require = createRequire(__filename); + +const { readFileSync } = require('node:fs'); +let nativeBinding = null; +const loadErrors = []; + +const isMusl = () => { + let musl = false; + if (process.platform === 'linux') { + musl = isMuslFromFilesystem(); + if (musl === null) { + musl = isMuslFromReport(); + } + if (musl === null) { + musl = isMuslFromChildProcess(); + } + } + return musl; +}; + +const isFileMusl = f => f.includes('libc.musl-') || f.includes('ld-musl-'); + +const isMuslFromFilesystem = () => { + try { + return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl'); + } catch { + return null; + } +}; + +const isMuslFromReport = () => { + let report = null; + if (typeof process.report?.getReport === 'function') { + process.report.excludeNetwork = true; + report = process.report.getReport(); + } + if (!report) { + return null; + } + if (report.header && report.header.glibcVersionRuntime) { + return false; + } + if (Array.isArray(report.sharedObjects)) { + if (report.sharedObjects.some(isFileMusl)) { + return true; + } + } + return false; +}; + +const isMuslFromChildProcess = () => { + try { + return require('child_process') + .execSync('ldd --version', { encoding: 'utf8' }) + .includes('musl'); + } catch (e) { + // If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false + return false; + } +}; + +function requireNative() { + if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { + try { + nativeBinding = require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH); + } catch (err) { + loadErrors.push(err); + } + } else if (process.platform === 'android') { + if (process.arch === 'arm64') { + try { + return require('./y-octo.android-arm64.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-android-arm64'); + } catch (e) { + loadErrors.push(e); + } + } else if (process.arch === 'arm') { + try { + return require('./y-octo.android-arm-eabi.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-android-arm-eabi'); + } catch (e) { + loadErrors.push(e); + } + } else { + loadErrors.push( + new Error(`Unsupported architecture on Android ${process.arch}`) + ); + } + } else if (process.platform === 'win32') { + if (process.arch === 'x64') { + try { + return require('./y-octo.win32-x64-msvc.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-win32-x64-msvc'); + } catch (e) { + loadErrors.push(e); + } + } else if (process.arch === 'ia32') { + try { + return require('./y-octo.win32-ia32-msvc.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-win32-ia32-msvc'); + } catch (e) { + loadErrors.push(e); + } + } else if (process.arch === 'arm64') { + try { + return require('./y-octo.win32-arm64-msvc.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-win32-arm64-msvc'); + } catch (e) { + loadErrors.push(e); + } + } else { + loadErrors.push( + new Error(`Unsupported architecture on Windows: ${process.arch}`) + ); + } + } else if (process.platform === 'darwin') { + try { + return require('./y-octo.darwin-universal.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-darwin-universal'); + } catch (e) { + loadErrors.push(e); + } + + if (process.arch === 'x64') { + try { + return require('./y-octo.darwin-x64.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-darwin-x64'); + } catch (e) { + loadErrors.push(e); + } + } else if (process.arch === 'arm64') { + try { + return require('./y-octo.darwin-arm64.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-darwin-arm64'); + } catch (e) { + loadErrors.push(e); + } + } else { + loadErrors.push( + new Error(`Unsupported architecture on macOS: ${process.arch}`) + ); + } + } else if (process.platform === 'freebsd') { + if (process.arch === 'x64') { + try { + return require('./y-octo.freebsd-x64.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-freebsd-x64'); + } catch (e) { + loadErrors.push(e); + } + } else if (process.arch === 'arm64') { + try { + return require('./y-octo.freebsd-arm64.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-freebsd-arm64'); + } catch (e) { + loadErrors.push(e); + } + } else { + loadErrors.push( + new Error(`Unsupported architecture on FreeBSD: ${process.arch}`) + ); + } + } else if (process.platform === 'linux') { + if (process.arch === 'x64') { + if (isMusl()) { + try { + return require('./y-octo.linux-x64-musl.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-x64-musl'); + } catch (e) { + loadErrors.push(e); + } + } else { + try { + return require('./y-octo.linux-x64-gnu.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-x64-gnu'); + } catch (e) { + loadErrors.push(e); + } + } + } else if (process.arch === 'arm64') { + if (isMusl()) { + try { + return require('./y-octo.linux-arm64-musl.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-arm64-musl'); + } catch (e) { + loadErrors.push(e); + } + } else { + try { + return require('./y-octo.linux-arm64-gnu.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-arm64-gnu'); + } catch (e) { + loadErrors.push(e); + } + } + } else if (process.arch === 'arm') { + if (isMusl()) { + try { + return require('./y-octo.linux-arm-musleabihf.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-arm-musleabihf'); + } catch (e) { + loadErrors.push(e); + } + } else { + try { + return require('./y-octo.linux-arm-gnueabihf.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-arm-gnueabihf'); + } catch (e) { + loadErrors.push(e); + } + } + } else if (process.arch === 'riscv64') { + if (isMusl()) { + try { + return require('./y-octo.linux-riscv64-musl.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-riscv64-musl'); + } catch (e) { + loadErrors.push(e); + } + } else { + try { + return require('./y-octo.linux-riscv64-gnu.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-riscv64-gnu'); + } catch (e) { + loadErrors.push(e); + } + } + } else if (process.arch === 'ppc64') { + try { + return require('./y-octo.linux-ppc64-gnu.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-ppc64-gnu'); + } catch (e) { + loadErrors.push(e); + } + } else if (process.arch === 's390x') { + try { + return require('./y-octo.linux-s390x-gnu.node'); + } catch (e) { + loadErrors.push(e); + } + try { + return require('@y-octo/node-linux-s390x-gnu'); + } catch (e) { + loadErrors.push(e); + } + } else { + loadErrors.push( + new Error(`Unsupported architecture on Linux: ${process.arch}`) + ); + } + } else { + loadErrors.push( + new Error( + `Unsupported OS: ${process.platform}, architecture: ${process.arch}` + ) + ); + } +} + +nativeBinding = requireNative(); + +if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) { + try { + nativeBinding = require('./y-octo.wasi.cjs'); + } catch (err) { + if (process.env.NAPI_RS_FORCE_WASI) { + loadErrors.push(err); + } + } + if (!nativeBinding) { + try { + nativeBinding = require('@y-octo/node-wasm32-wasi'); + } catch (err) { + if (process.env.NAPI_RS_FORCE_WASI) { + loadErrors.push(err); + } + } + } +} + +if (!nativeBinding) { + if (loadErrors.length > 0) { + // TODO Link to documentation with potential fixes + // - The package owner could build/publish bindings for this arch + // - The user may need to bundle the correct files + // - The user may need to re-install node_modules to get new packages + throw new Error('Failed to load native binding', { cause: loadErrors }); + } + throw new Error(`Failed to load native binding`); +} + +module.exports.Doc = nativeBinding.Doc; +module.exports.YArray = nativeBinding.YArray; +module.exports.YMap = nativeBinding.YMap; +module.exports.YText = nativeBinding.YText; diff --git a/packages/common/y-octo/node/package.json b/packages/common/y-octo/node/package.json new file mode 100644 index 0000000000..f856acf772 --- /dev/null +++ b/packages/common/y-octo/node/package.json @@ -0,0 +1,72 @@ +{ + "name": "@y-octo/node", + "private": true, + "main": "index.js", + "types": "index.d.ts", + "napi": { + "binaryName": "y-octo", + "targets": [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl" + ], + "ts": { + "constEnum": false + } + }, + "license": "MIT", + "devDependencies": { + "@napi-rs/cli": "3.0.0-alpha.77", + "@types/node": "^22.14.1", + "@types/prompts": "^2.4.9", + "c8": "^10.1.3", + "prompts": "^2.4.2", + "ts-node": "^10.9.2", + "typescript": "^5.8.3", + "yjs": "^13.6.24" + }, + "engines": { + "node": ">= 10" + }, + "scripts": { + "artifacts": "napi artifacts", + "build": "napi build --platform --release --no-const-enum", + "build:debug": "napi build --platform --no-const-enum", + "universal": "napi universal", + "test": "NODE_NO_WARNINGS=1 node ./scripts/run-test.mts all", + "test:watch": "yarn exec ts-node-esm ./scripts/run-test.ts all --watch", + "test:coverage": "NODE_OPTIONS=\"--loader ts-node/esm\" c8 node ./scripts/run-test.mts all", + "version": "napi version" + }, + "version": "0.0.1", + "sharedConfig": { + "nodeArgs": [ + "--loader", + "ts-node/esm", + "--es-module-specifier-resolution=node" + ], + "env": { + "TS_NODE_TRANSPILE_ONLY": "1", + "TS_NODE_PROJECT": "./tsconfig.json", + "NODE_ENV": "development", + "DEBUG": "napi:*" + } + }, + "c8": { + "reporter": [ + "text", + "lcov" + ], + "report-dir": ".coverage", + "exclude": [ + "scripts", + "node_modules", + "**/*.spec.ts" + ] + } +} diff --git a/packages/common/y-octo/node/scripts/run-test.mts b/packages/common/y-octo/node/scripts/run-test.mts new file mode 100755 index 0000000000..87c326c93a --- /dev/null +++ b/packages/common/y-octo/node/scripts/run-test.mts @@ -0,0 +1,78 @@ +#!/usr/bin/env ts-node-esm +import { resolve } from 'node:path'; +import { spawn } from 'node:child_process'; +import { readdir } from 'node:fs/promises'; +import * as process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +import prompts from 'prompts'; + +import pkg from '../package.json' with { type: 'json' }; +const root = fileURLToPath(new URL('..', import.meta.url)); +const testDir = resolve(root, 'tests'); +const files = await readdir(testDir); + +const watchMode = process.argv.includes('--watch'); + +const sharedArgs = [ + ...pkg.sharedConfig.nodeArgs, + '--test', + watchMode ? '--watch' : '', +]; + +const env = { + ...pkg.sharedConfig.env, + PATH: process.env.PATH, + NODE_ENV: 'test', + NODE_NO_WARNINGS: '1', +}; + +if (process.argv[2] === 'all') { + const cp = spawn( + 'node', + [...sharedArgs, ...files.map(f => resolve(testDir, f))], + { + cwd: root, + env, + stdio: 'inherit', + shell: true, + } + ); + cp.on('exit', code => { + process.exit(code ?? 0); + }); +} else { + const result = await prompts([ + { + type: 'select', + name: 'file', + message: 'Select a file to run', + choices: files.map(file => ({ + title: file, + value: file, + })), + initial: 1, + }, + ]); + + const target = resolve(testDir, result.file); + + const cp = spawn( + 'node', + [ + ...sharedArgs, + '--test-reporter=spec', + '--test-reporter-destination=stdout', + target, + ], + { + cwd: root, + env, + stdio: 'inherit', + shell: true, + } + ); + cp.on('exit', code => { + process.exit(code ?? 0); + }); +} diff --git a/packages/common/y-octo/node/src/array.rs b/packages/common/y-octo/node/src/array.rs new file mode 100644 index 0000000000..ea3e615224 --- /dev/null +++ b/packages/common/y-octo/node/src/array.rs @@ -0,0 +1,160 @@ +use napi::{bindgen_prelude::Array as JsArray, Env, JsUnknown, ValueType}; +use y_octo::{Any, Array, Value}; + +use super::*; + +#[napi] +pub struct YArray { + pub(crate) array: Array, +} + +#[napi] +impl YArray { + #[allow(clippy::new_without_default)] + #[napi(constructor)] + pub fn new() -> Self { + unimplemented!() + } + + pub(crate) fn inner_new(array: Array) -> Self { + Self { array } + } + + #[napi(getter)] + pub fn length(&self) -> i64 { + self.array.len() as i64 + } + + #[napi(getter)] + pub fn is_empty(&self) -> bool { + self.array.is_empty() + } + + #[napi(ts_generic_types = "T = unknown", ts_return_type = "T")] + pub fn get(&self, env: Env, index: i64) -> Result { + if let Some(value) = self.array.get(index as u64) { + match value { + Value::Any(any) => get_js_unknown_from_any(env, any).map(MixedYType::D), + Value::Array(array) => Ok(MixedYType::A(YArray::inner_new(array))), + Value::Map(map) => Ok(MixedYType::B(YMap::inner_new(map))), + Value::Text(text) => Ok(MixedYType::C(YText::inner_new(text))), + _ => env.get_null().map(|v| v.into_unknown()).map(MixedYType::D), + } + .map_err(anyhow::Error::from) + } else { + Ok(MixedYType::D(env.get_null()?.into_unknown())) + } + } + + #[napi( + ts_args_type = "index: number, value: YArray | YMap | YText | boolean | number | string | \ + Record | null | undefined" + )] + pub fn insert(&mut self, index: i64, value: MixedRefYType) -> Result<()> { + match value { + MixedRefYType::A(array) => self + .array + .insert(index as u64, array.array.clone()) + .map_err(anyhow::Error::from), + MixedRefYType::B(map) => self + .array + .insert(index as u64, map.map.clone()) + .map_err(anyhow::Error::from), + MixedRefYType::C(text) => self + .array + .insert(index as u64, text.text.clone()) + .map_err(anyhow::Error::from), + MixedRefYType::D(unknown) => match unknown.get_type() { + Ok(value_type) => match value_type { + ValueType::Undefined | ValueType::Null => self + .array + .insert(index as u64, Any::Null) + .map_err(anyhow::Error::from), + ValueType::Boolean => match unknown.coerce_to_bool().and_then(|v| v.get_value()) { + Ok(boolean) => self + .array + .insert(index as u64, boolean) + .map_err(anyhow::Error::from), + Err(e) => Err(anyhow::Error::new(e).context("Failed to coerce value to boolean")), + }, + ValueType::Number => match unknown.coerce_to_number().and_then(|v| v.get_double()) { + Ok(number) => self + .array + .insert(index as u64, number) + .map_err(anyhow::Error::from), + Err(e) => Err(anyhow::Error::new(e).context("Failed to coerce value to number")), + }, + ValueType::String => { + match unknown + .coerce_to_string() + .and_then(|v| v.into_utf8()) + .and_then(|s| s.as_str().map(|s| s.to_string())) + { + Ok(string) => self + .array + .insert(index as u64, string) + .map_err(anyhow::Error::from), + Err(e) => Err(anyhow::Error::new(e).context("Failed to coerce value to string")), + } + } + ValueType::Object => match unknown + .coerce_to_object() + .and_then(|o| o.get_array_length().map(|l| (o, l))) + { + Ok((object, length)) => { + for i in 0..length { + if let Ok(any) = object + .get_element::(i) + .and_then(get_any_from_js_unknown) + { + self + .array + .insert(index as u64 + i as u64, Value::Any(any)) + .map_err(anyhow::Error::from)?; + } + } + Ok(()) + } + Err(e) => Err(anyhow::Error::new(e).context("Failed to coerce value to object")), + }, + ValueType::BigInt => Err(anyhow::Error::msg("BigInt values are not supported")), + ValueType::Symbol => Err(anyhow::Error::msg("Symbol values are not supported")), + ValueType::Function => Err(anyhow::Error::msg("Function values are not supported")), + ValueType::External => Err(anyhow::Error::msg("External values are not supported")), + ValueType::Unknown => Err(anyhow::Error::msg("Unknown values are not supported")), + }, + Err(e) => Err(anyhow::Error::from(e)), + }, + } + } + + #[napi] + pub fn remove(&mut self, index: i64, len: i64) -> Result<()> { + self + .array + .remove(index as u64, len as u64) + .map_err(anyhow::Error::from) + } + + #[napi] + pub fn to_json(&self, env: Env) -> Result { + let mut js_array = env.create_array(0)?; + for value in self.array.iter() { + js_array.insert(get_js_unknown_from_value(env, value)?)?; + } + Ok(js_array) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_array_init() { + let doc = Doc::new(None); + let array = doc.get_or_create_array("array".into()).unwrap(); + assert_eq!(array.length(), 0); + } +} diff --git a/packages/common/y-octo/node/src/doc.rs b/packages/common/y-octo/node/src/doc.rs new file mode 100644 index 0000000000..7773ab5656 --- /dev/null +++ b/packages/common/y-octo/node/src/doc.rs @@ -0,0 +1,176 @@ +use napi::{ + bindgen_prelude::{Buffer, Uint8Array}, + threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, +}; +use y_octo::{CrdtRead, Doc as YDoc, History, RawDecoder, StateVector}; + +use super::*; + +#[napi] +pub struct Doc { + doc: YDoc, +} + +#[napi] +impl Doc { + #[napi(constructor)] + pub fn new(client_id: Option) -> Self { + Self { + doc: if let Some(client_id) = client_id { + YDoc::with_client(client_id as u64) + } else { + YDoc::default() + }, + } + } + + #[napi(getter)] + pub fn client_id(&self) -> i64 { + self.doc.client() as i64 + } + + #[napi(getter)] + pub fn guid(&self) -> &str { + self.doc.guid() + } + + #[napi(getter)] + pub fn keys(&self) -> Vec { + self.doc.keys() + } + + #[napi] + pub fn get_or_create_array(&self, key: String) -> Result { + self + .doc + .get_or_create_array(key) + .map(YArray::inner_new) + .map_err(anyhow::Error::from) + } + + #[napi] + pub fn get_or_create_text(&self, key: String) -> Result { + self + .doc + .get_or_create_text(key) + .map(YText::inner_new) + .map_err(anyhow::Error::from) + } + + #[napi] + pub fn get_or_create_map(&self, key: String) -> Result { + self + .doc + .get_or_create_map(key) + .map(YMap::inner_new) + .map_err(anyhow::Error::from) + } + + #[napi] + pub fn create_array(&self) -> Result { + self + .doc + .create_array() + .map(YArray::inner_new) + .map_err(anyhow::Error::from) + } + + #[napi] + pub fn create_text(&self) -> Result { + self + .doc + .create_text() + .map(YText::inner_new) + .map_err(anyhow::Error::from) + } + + #[napi] + pub fn create_map(&self) -> Result { + self + .doc + .create_map() + .map(YMap::inner_new) + .map_err(anyhow::Error::from) + } + + #[napi] + pub fn apply_update(&mut self, update: &[u8]) -> Result<()> { + self.doc.apply_update_from_binary_v1(update)?; + + Ok(()) + } + + #[napi] + pub fn encode_state_as_update_v1(&self, state: Option<&[u8]>) -> Result { + let result = match state { + Some(state) => { + let mut decoder = RawDecoder::new(state); + let state = StateVector::read(&mut decoder)?; + self.doc.encode_state_as_update_v1(&state) + } + None => self.doc.encode_update_v1(), + }; + + result.map(|v| v.into()).map_err(anyhow::Error::from) + } + + #[napi] + pub fn gc(&self) -> Result<()> { + self.doc.gc().map_err(anyhow::Error::from) + } + + #[napi(ts_args_type = "callback: (result: Uint8Array) => void")] + pub fn on_update(&mut self, callback: ThreadsafeFunction) -> Result<()> { + let callback = move |update: &[u8], _h: &[History]| { + callback.call( + Ok(update.to_vec().into()), + ThreadsafeFunctionCallMode::Blocking, + ); + }; + self.doc.subscribe(Box::new(callback)); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_doc_client() { + let client_id = 1; + let doc = Doc::new(Some(client_id)); + assert_eq!(doc.client_id(), 1); + } + + #[test] + fn test_doc_guid() { + let doc = Doc::new(None); + assert_eq!(doc.guid().len(), 21); + } + + #[test] + fn test_create_array() { + let doc = Doc::new(None); + let array = doc.get_or_create_array("array".into()).unwrap(); + assert_eq!(array.length(), 0); + } + + #[test] + fn test_create_text() { + let doc = Doc::new(None); + let text = doc.get_or_create_text("text".into()).unwrap(); + assert_eq!(text.len(), 0); + } + + #[test] + fn test_keys() { + let doc = Doc::new(None); + doc.get_or_create_array("array".into()).unwrap(); + doc.get_or_create_text("text".into()).unwrap(); + doc.get_or_create_map("map".into()).unwrap(); + let mut keys = doc.keys(); + keys.sort(); + assert_eq!(keys, vec!["array", "map", "text"]); + } +} diff --git a/packages/common/y-octo/node/src/lib.rs b/packages/common/y-octo/node/src/lib.rs new file mode 100644 index 0000000000..a5c693c3fb --- /dev/null +++ b/packages/common/y-octo/node/src/lib.rs @@ -0,0 +1,17 @@ +use anyhow::Result; +use napi_derive::napi; + +mod array; +mod doc; +mod map; +mod text; +mod utils; + +pub use array::YArray; +pub use doc::Doc; +pub use map::YMap; +pub use text::YText; +use utils::{ + get_any_from_js_object, get_any_from_js_unknown, get_js_unknown_from_any, + get_js_unknown_from_value, MixedRefYType, MixedYType, +}; diff --git a/packages/common/y-octo/node/src/map.rs b/packages/common/y-octo/node/src/map.rs new file mode 100644 index 0000000000..17fce3f8d7 --- /dev/null +++ b/packages/common/y-octo/node/src/map.rs @@ -0,0 +1,134 @@ +use napi::{Env, JsObject, ValueType}; +use y_octo::{Any, Map, Value}; + +use super::*; + +#[napi] +pub struct YMap { + pub(crate) map: Map, +} + +#[napi] +impl YMap { + #[allow(clippy::new_without_default)] + #[napi(constructor)] + pub fn new() -> Self { + unimplemented!() + } + + pub(crate) fn inner_new(map: Map) -> Self { + Self { map } + } + + #[napi(getter)] + pub fn length(&self) -> i64 { + self.map.len() as i64 + } + + #[napi(getter)] + pub fn is_empty(&self) -> bool { + self.map.is_empty() + } + + #[napi(ts_generic_types = "T = unknown", ts_return_type = "T")] + pub fn get(&self, env: Env, key: String) -> Result { + if let Some(value) = self.map.get(&key) { + match value { + Value::Any(any) => get_js_unknown_from_any(env, any).map(MixedYType::D), + Value::Array(array) => Ok(MixedYType::A(YArray::inner_new(array))), + Value::Map(map) => Ok(MixedYType::B(YMap::inner_new(map))), + Value::Text(text) => Ok(MixedYType::C(YText::inner_new(text))), + _ => env.get_null().map(|v| v.into_unknown()).map(MixedYType::D), + } + .map_err(anyhow::Error::from) + } else { + Ok(MixedYType::D(env.get_null()?.into_unknown())) + } + } + + #[napi( + ts_args_type = "key: string, value: YArray | YMap | YText | boolean | number | string | \ + Record | null | undefined" + )] + pub fn set(&mut self, key: String, value: MixedRefYType) -> Result<()> { + match value { + MixedRefYType::A(array) => self + .map + .insert(key, array.array.clone()) + .map_err(anyhow::Error::from), + MixedRefYType::B(map) => self + .map + .insert(key, map.map.clone()) + .map_err(anyhow::Error::from), + MixedRefYType::C(text) => self + .map + .insert(key, text.text.clone()) + .map_err(anyhow::Error::from), + MixedRefYType::D(unknown) => match unknown.get_type() { + Ok(value_type) => match value_type { + ValueType::Undefined | ValueType::Null => { + self.map.insert(key, Any::Null).map_err(anyhow::Error::from) + } + ValueType::Boolean => match unknown.coerce_to_bool().and_then(|v| v.get_value()) { + Ok(boolean) => self.map.insert(key, boolean).map_err(anyhow::Error::from), + Err(e) => Err(anyhow::Error::from(e).context("Failed to coerce value to boolean")), + }, + ValueType::Number => match unknown.coerce_to_number().and_then(|v| v.get_double()) { + Ok(number) => self.map.insert(key, number).map_err(anyhow::Error::from), + Err(e) => Err(anyhow::Error::from(e).context("Failed to coerce value to number")), + }, + ValueType::String => { + match unknown + .coerce_to_string() + .and_then(|v| v.into_utf8()) + .and_then(|s| s.as_str().map(|s| s.to_string())) + { + Ok(string) => self.map.insert(key, string).map_err(anyhow::Error::from), + Err(e) => Err(anyhow::Error::from(e).context("Failed to coerce value to string")), + } + } + ValueType::Object => match unknown.coerce_to_object().and_then(get_any_from_js_object) { + Ok(any) => self + .map + .insert(key, Value::Any(any)) + .map_err(anyhow::Error::from), + Err(e) => Err(anyhow::Error::from(e).context("Failed to coerce value to object")), + }, + ValueType::BigInt => Err(anyhow::Error::msg("BigInt values are not supported")), + ValueType::Symbol => Err(anyhow::Error::msg("Symbol values are not supported")), + ValueType::Function => Err(anyhow::Error::msg("Function values are not supported")), + ValueType::External => Err(anyhow::Error::msg("External values are not supported")), + ValueType::Unknown => Err(anyhow::Error::msg("Unknown values are not supported")), + }, + Err(e) => Err(anyhow::Error::from(e)), + }, + } + } + + #[napi] + pub fn remove(&mut self, key: String) { + self.map.remove(&key); + } + + #[napi] + pub fn to_json(&self, env: Env) -> Result { + let mut js_object = env.create_object()?; + for (key, value) in self.map.iter() { + js_object.set(key, get_js_unknown_from_value(env, value))?; + } + Ok(js_object) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_map_init() { + let doc = Doc::new(None); + let text = doc.get_or_create_map("map".into()).unwrap(); + assert_eq!(text.length(), 0); + } +} diff --git a/packages/common/y-octo/node/src/text.rs b/packages/common/y-octo/node/src/text.rs new file mode 100644 index 0000000000..ca6478698a --- /dev/null +++ b/packages/common/y-octo/node/src/text.rs @@ -0,0 +1,82 @@ +use y_octo::Text; + +use super::*; + +#[napi] +pub struct YText { + pub(crate) text: Text, +} + +#[napi] +impl YText { + #[allow(clippy::new_without_default)] + #[napi(constructor)] + pub fn new() -> Self { + unimplemented!() + } + + pub(crate) fn inner_new(text: Text) -> Self { + Self { text } + } + + #[napi(getter)] + pub fn len(&self) -> i64 { + self.text.len() as i64 + } + + #[napi(getter)] + pub fn is_empty(&self) -> bool { + self.text.is_empty() + } + + #[napi] + pub fn insert(&mut self, index: i64, str: String) -> Result<()> { + self + .text + .insert(index as u64, str) + .map_err(anyhow::Error::from) + } + + #[napi] + pub fn remove(&mut self, index: i64, len: i64) -> Result<()> { + self + .text + .remove(index as u64, len as u64) + .map_err(anyhow::Error::from) + } + + #[napi(getter)] + pub fn length(&self) -> i64 { + self.text.len() as i64 + } + + #[allow(clippy::inherent_to_string)] + #[napi] + pub fn to_string(&self) -> String { + self.text.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_text_init() { + let doc = Doc::new(None); + let text = doc.get_or_create_text("text".into()).unwrap(); + assert_eq!(text.len(), 0); + } + + #[test] + fn test_text_edit() { + let doc = Doc::new(None); + let mut text = doc.get_or_create_text("text".into()).unwrap(); + text.insert(0, "hello".into()).unwrap(); + assert_eq!(text.to_string(), "hello"); + text.insert(5, " world".into()).unwrap(); + assert_eq!(text.to_string(), "hello world"); + text.remove(5, 6).unwrap(); + assert_eq!(text.to_string(), "hello"); + } +} diff --git a/packages/common/y-octo/node/src/utils.rs b/packages/common/y-octo/node/src/utils.rs new file mode 100644 index 0000000000..7103041401 --- /dev/null +++ b/packages/common/y-octo/node/src/utils.rs @@ -0,0 +1,117 @@ +use napi::{bindgen_prelude::Either4, Env, Error, JsObject, JsUnknown, Result, Status, ValueType}; +use y_octo::{AHashMap, Any, HashMapExt, Value}; + +use super::*; + +pub type MixedYType = Either4; +pub type MixedRefYType<'a> = Either4<&'a YArray, &'a YMap, &'a YText, JsUnknown>; + +pub fn get_js_unknown_from_any(env: Env, any: Any) -> Result { + match any { + Any::Null | Any::Undefined => env.get_null().map(|v| v.into_unknown()), + Any::True => env.get_boolean(true).map(|v| v.into_unknown()), + Any::False => env.get_boolean(false).map(|v| v.into_unknown()), + Any::Integer(number) => env.create_int32(number).map(|v| v.into_unknown()), + Any::BigInt64(number) => env.create_int64(number).map(|v| v.into_unknown()), + Any::Float32(number) => env.create_double(number.0 as f64).map(|v| v.into_unknown()), + Any::Float64(number) => env.create_double(number.0).map(|v| v.into_unknown()), + Any::String(string) => env.create_string(string.as_str()).map(|v| v.into_unknown()), + Any::Array(array) => { + let mut js_array = env.create_array_with_length(array.len())?; + for (i, value) in array.into_iter().enumerate() { + js_array.set_element(i as u32, get_js_unknown_from_any(env, value)?)?; + } + Ok(js_array.into_unknown()) + } + _ => env.get_null().map(|v| v.into_unknown()), + } +} + +#[allow(deprecated)] +// Wait for NAPI-RS External::into_unknown to be stabilized +pub fn get_js_unknown_from_value(env: Env, value: Value) -> Result { + match value { + Value::Any(any) => get_js_unknown_from_any(env, any), + Value::Array(array) => env + .create_external(YArray::inner_new(array), None) + .map(|o| o.into_unknown()), + Value::Map(map) => env + .create_external(YMap::inner_new(map), None) + .map(|o| o.into_unknown()), + Value::Text(text) => env + .create_external(YText::inner_new(text), None) + .map(|o| o.into_unknown()), + _ => env.get_null().map(|v| v.into_unknown()), + } +} + +pub fn get_any_from_js_object(object: JsObject) -> Result { + if let Ok(length) = object.get_array_length() { + let mut array = Vec::with_capacity(length as usize); + for i in 0..length { + if let Ok(value) = object.get_element::(i) { + array.push(get_any_from_js_unknown(value)?); + } + } + Ok(Any::Array(array)) + } else { + let mut map = AHashMap::new(); + let keys = object.get_property_names()?; + if let Ok(length) = keys.get_array_length() { + for i in 0..length { + if let Ok((obj, key)) = keys.get_element::(i).and_then(|o| { + o.coerce_to_string().and_then(|obj| { + obj + .into_utf8() + .and_then(|s| s.as_str().map(|s| (obj, s.to_string()))) + }) + }) { + if let Ok(value) = object.get_property::<_, JsUnknown>(obj) { + println!("key: {}", key); + map.insert(key, get_any_from_js_unknown(value)?); + } + } + } + } + Ok(Any::Object(map)) + } +} + +pub fn get_any_from_js_unknown(js_unknown: JsUnknown) -> Result { + match js_unknown.get_type()? { + ValueType::Undefined | ValueType::Null => Ok(Any::Null), + ValueType::Boolean => Ok( + js_unknown + .coerce_to_bool() + .and_then(|v| v.get_value())? + .into(), + ), + ValueType::Number => Ok( + js_unknown + .coerce_to_number() + .and_then(|v| v.get_double()) + .map(|v| v.into())?, + ), + ValueType::String => Ok( + js_unknown + .coerce_to_string() + .and_then(|v| v.into_utf8()) + .and_then(|s| s.as_str().map(|s| s.to_string()))? + .into(), + ), + ValueType::Object => { + if let Ok(object) = js_unknown.coerce_to_object() { + get_any_from_js_object(object) + } else { + Err(Error::new( + Status::InvalidArg, + "Failed to coerce value to object", + )) + } + } + _ => Err(Error::new( + Status::InvalidArg, + "Failed to coerce value to any", + )), + } +} diff --git a/packages/common/y-octo/node/tests/array.spec.mts b/packages/common/y-octo/node/tests/array.spec.mts new file mode 100644 index 0000000000..0b2f57d078 --- /dev/null +++ b/packages/common/y-octo/node/tests/array.spec.mts @@ -0,0 +1,62 @@ +import assert, { equal, deepEqual } from 'node:assert'; +import { test } from 'node:test'; + +import { Doc, type YArray } from '../index'; + +test('array test', { concurrency: false }, async t => { + let client_id: number; + let doc: Doc; + t.beforeEach(async () => { + client_id = (Math.random() * 100000) | 0; + doc = new Doc(client_id); + }); + + t.afterEach(async () => { + client_id = -1; + // @ts-expect-error - doc must not null in next range + doc = null; + }); + + await t.test('array should be created', () => { + let arr = doc.getOrCreateArray('arr'); + deepEqual(doc.keys, ['arr']); + equal(arr.length, 0); + }); + + await t.test('array editing', () => { + let arr = doc.getOrCreateArray('arr'); + arr.insert(0, true); + arr.insert(1, false); + arr.insert(2, 1); + arr.insert(3, 'hello world'); + equal(arr.length, 4); + equal(arr.get(0), true); + equal(arr.get(1), false); + equal(arr.get(2), 1); + equal(arr.get(3), 'hello world'); + equal(arr.length, 4); + arr.remove(1, 1); + equal(arr.length, 3); + equal(arr.get(2), 'hello world'); + }); + + await t.test('sub array should can edit', () => { + let map = doc.getOrCreateMap('map'); + let sub = doc.createArray(); + map.set('sub', sub); + + sub.insert(0, true); + sub.insert(1, false); + sub.insert(2, 1); + sub.insert(3, 'hello world'); + equal(sub.length, 4); + + let sub2 = map.get('sub'); + assert(sub2); + equal(sub2.get(0), true); + equal(sub2.get(1), false); + equal(sub2.get(2), 1); + equal(sub2.get(3), 'hello world'); + equal(sub2.length, 4); + }); +}); diff --git a/packages/common/y-octo/node/tests/doc.spec.mts b/packages/common/y-octo/node/tests/doc.spec.mts new file mode 100644 index 0000000000..65f36d5a7c --- /dev/null +++ b/packages/common/y-octo/node/tests/doc.spec.mts @@ -0,0 +1,99 @@ +import { equal } from 'node:assert'; +import { test } from 'node:test'; + +import { Doc } from '../index'; +import * as Y from 'yjs'; + +test('doc test', { concurrency: false }, async t => { + let client_id: number; + let doc: Doc; + t.beforeEach(async () => { + client_id = (Math.random() * 100000) | 0; + doc = new Doc(client_id); + }); + + t.afterEach(async () => { + client_id = -1; + // @ts-expect-error - doc must not null in next range + doc = null; + }); + + await t.test('doc id should be set', () => { + equal(doc.clientId, client_id); + }); + + await t.test('y-octo doc update should be apply', () => { + let array = doc.getOrCreateArray('array'); + let map = doc.getOrCreateMap('map'); + let text = doc.getOrCreateText('text'); + + array.insert(0, true); + array.insert(1, false); + array.insert(2, 1); + array.insert(3, 'hello world'); + map.set('a', true); + map.set('b', false); + map.set('c', 1); + map.set('d', 'hello world'); + text.insert(0, 'a'); + text.insert(1, 'b'); + text.insert(2, 'c'); + + let doc2 = new Doc(client_id); + doc2.applyUpdate(doc.encodeStateAsUpdateV1()); + + let array2 = doc2.getOrCreateArray('array'); + let map2 = doc2.getOrCreateMap('map'); + let text2 = doc2.getOrCreateText('text'); + + equal(doc2.clientId, client_id); + equal(array2.length, 4); + equal(array2.get(0), true); + equal(array2.get(1), false); + equal(array2.get(2), 1); + equal(array2.get(3), 'hello world'); + equal(map2.length, 4); + equal(map2.get('a'), true); + equal(map2.get('b'), false); + equal(map2.get('c'), 1); + equal(map2.get('d'), 'hello world'); + equal(text2.toString(), 'abc'); + }); + + await t.test('yjs doc update should be apply', () => { + let doc2 = new Y.Doc(); + let array2 = doc2.getArray('array'); + let map2 = doc2.getMap('map'); + let text2 = doc2.getText('text'); + + array2.insert(0, [true]); + array2.insert(1, [false]); + array2.insert(2, [1]); + array2.insert(3, ['hello world']); + map2.set('a', true); + map2.set('b', false); + map2.set('c', 1); + map2.set('d', 'hello world'); + text2.insert(0, 'a'); + text2.insert(1, 'b'); + text2.insert(2, 'c'); + + doc.applyUpdate(Buffer.from(Y.encodeStateAsUpdate(doc2))); + + let array = doc.getOrCreateArray('array'); + let map = doc.getOrCreateMap('map'); + let text = doc.getOrCreateText('text'); + + equal(array.length, 4); + equal(array.get(0), true); + equal(array.get(1), false); + equal(array.get(2), 1); + equal(array.get(3), 'hello world'); + equal(map.length, 4); + equal(map.get('a'), true); + equal(map.get('b'), false); + equal(map.get('c'), 1); + equal(map.get('d'), 'hello world'); + equal(text.toString(), 'abc'); + }); +}); diff --git a/packages/common/y-octo/node/tests/map.spec.mts b/packages/common/y-octo/node/tests/map.spec.mts new file mode 100644 index 0000000000..c31cf6846c --- /dev/null +++ b/packages/common/y-octo/node/tests/map.spec.mts @@ -0,0 +1,152 @@ +import assert, { equal, deepEqual } from 'node:assert'; +import { test } from 'node:test'; + +import * as Y from 'yjs'; +import { Doc, type YArray, type YMap, type YText } from '../index'; + +test('map test', { concurrency: false }, async t => { + let client_id: number; + let doc: Doc; + t.beforeEach(async () => { + client_id = (Math.random() * 100000) | 0; + doc = new Doc(client_id); + }); + + t.afterEach(async () => { + client_id = -1; + // @ts-expect-error - doc must not null in next range + doc = null; + }); + + await t.test('map should be created', () => { + let map = doc.getOrCreateMap('map'); + deepEqual(doc.keys, ['map']); + equal(map.length, 0); + }); + + await t.test('map editing', () => { + let map = doc.getOrCreateMap('map'); + map.set('a', true); + map.set('b', false); + map.set('c', 1); + map.set('d', 'hello world'); + equal(map.length, 4); + equal(map.get('a'), true); + equal(map.get('b'), false); + equal(map.get('c'), 1); + equal(map.get('d'), 'hello world'); + equal(map.length, 4); + map.remove('b'); + equal(map.length, 3); + equal(map.get('d'), 'hello world'); + }); + + await t.test('map should can be nested', () => { + let map = doc.getOrCreateMap('map'); + let sub = doc.createMap(); + map.set('sub', sub); + + sub.set('a', true); + sub.set('b', false); + sub.set('c', 1); + sub.set('d', 'hello world'); + equal(sub.length, 4); + + let sub2 = map.get('sub'); + assert(sub2); + equal(sub2.get('a'), true); + equal(sub2.get('b'), false); + equal(sub2.get('c'), 1); + equal(sub2.get('d'), 'hello world'); + equal(sub2.length, 4); + }); + + await t.test('y-octo to yjs compatibility test with nested type', () => { + let map = doc.getOrCreateMap('map'); + let sub_array = doc.createArray(); + let sub_map = doc.createMap(); + let sub_text = doc.createText(); + + map.set('array', sub_array); + map.set('map', sub_map); + map.set('text', sub_text); + + sub_array.insert(0, true); + sub_array.insert(1, false); + sub_array.insert(2, 1); + sub_array.insert(3, 'hello world'); + sub_map.set('a', true); + sub_map.set('b', false); + sub_map.set('c', 1); + sub_map.set('d', 'hello world'); + sub_text.insert(0, 'a'); + sub_text.insert(1, 'b'); + sub_text.insert(2, 'c'); + + let doc2 = new Y.Doc(); + Y.applyUpdate(doc2, doc.encodeStateAsUpdateV1()); + + let map2 = doc2.getMap('map'); + let sub_array2 = map2.get('array') as Y.Array; + let sub_map2 = map2.get('map') as Y.Map; + let sub_text2 = map2.get('text') as Y.Text; + + assert(sub_array2); + equal(sub_array2.length, 4); + equal(sub_array2.get(0), true); + equal(sub_array2.get(1), false); + equal(sub_array2.get(2), 1); + equal(sub_array2.get(3), 'hello world'); + assert(sub_map2); + equal(sub_map2.get('a'), true); + equal(sub_map2.get('b'), false); + equal(sub_map2.get('c'), 1); + equal(sub_map2.get('d'), 'hello world'); + assert(sub_text2); + equal(sub_text2.toString(), 'abc'); + }); + + await t.test('yjs to y-octo compatibility test with nested type', () => { + let doc2 = new Y.Doc(); + let map2 = doc2.getMap('map'); + let sub_array2 = new Y.Array(); + let sub_map2 = new Y.Map(); + let sub_text2 = new Y.Text(); + map2.set('array', sub_array2); + map2.set('map', sub_map2); + map2.set('text', sub_text2); + + sub_array2.insert(0, [true]); + sub_array2.insert(1, [false]); + sub_array2.insert(2, [1]); + sub_array2.insert(3, ['hello world']); + sub_map2.set('a', true); + sub_map2.set('b', false); + sub_map2.set('c', 1); + sub_map2.set('d', 'hello world'); + sub_text2.insert(0, 'a'); + sub_text2.insert(1, 'b'); + sub_text2.insert(2, 'c'); + + doc.applyUpdate(Buffer.from(Y.encodeStateAsUpdate(doc2))); + + let map = doc.getOrCreateMap('map'); + let sub_array = map.get('array'); + let sub_map = map.get('map'); + let sub_text = map.get('text'); + + assert(sub_array); + equal(sub_array.length, 4); + equal(sub_array.get(0), true); + equal(sub_array.get(1), false); + equal(sub_array.get(2), 1); + equal(sub_array.get(3), 'hello world'); + assert(sub_map); + equal(sub_map.get('a'), true); + equal(sub_map.get('b'), false); + equal(sub_map.get('c'), 1); + equal(sub_map.get('d'), 'hello world'); + assert(sub_text); + equal(sub_text.toString(), 'abc'); + }); +}); diff --git a/packages/common/y-octo/node/tests/text.spec.mts b/packages/common/y-octo/node/tests/text.spec.mts new file mode 100644 index 0000000000..123440d4f5 --- /dev/null +++ b/packages/common/y-octo/node/tests/text.spec.mts @@ -0,0 +1,54 @@ +import assert, { equal, deepEqual } from 'node:assert'; +import { test } from 'node:test'; + +import { Doc, type YText } from '../index'; + +test('text test', { concurrency: false }, async t => { + let client_id: number; + let doc: Doc; + t.beforeEach(async () => { + client_id = (Math.random() * 100000) | 0; + doc = new Doc(client_id); + }); + + t.afterEach(async () => { + client_id = -1; + // @ts-expect-error - doc must not null in next range + doc = null; + }); + + await t.test('text should be created', () => { + let text = doc.getOrCreateText('text'); + deepEqual(doc.keys, ['text']); + equal(text.len, 0); + }); + + await t.test('text editing', () => { + let text = doc.getOrCreateText('text'); + text.insert(0, 'a'); + text.insert(1, 'b'); + text.insert(2, 'c'); + equal(text.toString(), 'abc'); + text.remove(0, 1); + equal(text.toString(), 'bc'); + text.remove(1, 1); + equal(text.toString(), 'b'); + text.remove(0, 1); + equal(text.toString(), ''); + }); + + await t.test('sub text should can edit', () => { + let map = doc.getOrCreateMap('map'); + let sub = doc.createText(); + map.set('sub', sub); + + sub.insert(0, 'a'); + sub.insert(1, 'b'); + sub.insert(2, 'c'); + equal(sub.toString(), 'abc'); + + let sub2 = map.get('sub'); + assert(sub2); + equal(sub2.toString(), 'abc'); + }); +}); diff --git a/packages/common/y-octo/node/tsconfig.json b/packages/common/y-octo/node/tsconfig.json new file mode 100644 index 0000000000..fdb88feac5 --- /dev/null +++ b/packages/common/y-octo/node/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../../tsconfig.node.json", + "compilerOptions": { + "noEmit": false, + "outDir": "lib", + "composite": true + }, + "include": ["index.d.ts", "tests/**/*.mts"], + "references": [] +} diff --git a/packages/common/y-octo/utils/Cargo.toml b/packages/common/y-octo/utils/Cargo.toml new file mode 100644 index 0000000000..55292d0899 --- /dev/null +++ b/packages/common/y-octo/utils/Cargo.toml @@ -0,0 +1,71 @@ +[package] +authors = ["x1a0t <405028157@qq.com>", "DarkSky "] +edition = "2021" +license = "MIT" +name = "y-octo-utils" +version = "0.0.1" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +bench = ["regex"] +default = ["merger"] +fuzz = ["arbitrary", "phf"] +merger = ["clap", "y-octo/large_refs"] + +[dependencies] +arbitrary = { workspace = true, features = ["derive"], optional = true } +clap = { workspace = true, features = ["derive"], optional = true } +lib0 = { workspace = true, features = ["lib0-serde"] } +phf = { workspace = true, features = ["macros"], optional = true } +rand = { workspace = true } +rand_chacha = { workspace = true } +regex = { workspace = true, optional = true } +y-octo = { workspace = true } +y-sync = { workspace = true } +yrs = { workspace = true } + +[dev-dependencies] +criterion = { workspace = true } +path-ext = { workspace = true } +proptest = { workspace = true } +proptest-derive = { workspace = true } + +[[bin]] +name = "bench_result_render" +path = "bin/bench_result_render.rs" + +[[bin]] +name = "doc_merger" +path = "bin/doc_merger.rs" + +[[bin]] +name = "memory_leak_test" +path = "bin/memory_leak_test.rs" + +[[bench]] +harness = false +name = "array_ops_benchmarks" + +[[bench]] +harness = false +name = "codec_benchmarks" + +[[bench]] +harness = false +name = "map_ops_benchmarks" + +[[bench]] +harness = false +name = "text_ops_benchmarks" + +[[bench]] +harness = false +name = "apply_benchmarks" + +[[bench]] +harness = false +name = "update_benchmarks" + +[lib] +bench = true diff --git a/packages/common/y-octo/utils/benches/apply_benchmarks.rs b/packages/common/y-octo/utils/benches/apply_benchmarks.rs new file mode 100644 index 0000000000..050f0f970e --- /dev/null +++ b/packages/common/y-octo/utils/benches/apply_benchmarks.rs @@ -0,0 +1,35 @@ +mod utils; + +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use path_ext::PathExt; +use utils::Files; + +fn apply(c: &mut Criterion) { + let files = Files::load(); + + let mut group = c.benchmark_group("apply"); + group.measurement_time(Duration::from_secs(15)); + + for file in &files.files { + group.throughput(Throughput::Bytes(file.content.len() as u64)); + group.bench_with_input( + BenchmarkId::new("apply with yrs", file.path.name_str()), + &file.content, + |b, content| { + b.iter(|| { + use yrs::{updates::decoder::Decode, Doc, Transact, Update}; + let update = Update::decode_v1(content).unwrap(); + let doc = Doc::new(); + doc.transact_mut().apply_update(update).unwrap(); + }); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, apply); +criterion_main!(benches); diff --git a/packages/common/y-octo/utils/benches/array_ops_benchmarks.rs b/packages/common/y-octo/utils/benches/array_ops_benchmarks.rs new file mode 100644 index 0000000000..acaf54e133 --- /dev/null +++ b/packages/common/y-octo/utils/benches/array_ops_benchmarks.rs @@ -0,0 +1,79 @@ +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Criterion}; +use rand::{Rng, SeedableRng}; + +fn operations(c: &mut Criterion) { + let mut group = c.benchmark_group("ops/array"); + group.measurement_time(Duration::from_secs(15)); + + group.bench_function("yrs/insert", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); + + let idxs = (0..99) + .map(|_| rng.random_range(0..base_text.len() as u32)) + .collect::>(); + b.iter(|| { + use yrs::{Array, Doc, Transact}; + let doc = Doc::new(); + let array = doc.get_or_insert_array("test"); + + let mut trx = doc.transact_mut(); + for c in base_text.chars() { + array.push_back(&mut trx, c.to_string()); + } + for idx in &idxs { + array.insert(&mut trx, *idx, "test"); + } + drop(trx); + }); + }); + + group.bench_function("yrs/insert range", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); + + let idxs = (0..99) + .map(|_| rng.random_range(0..base_text.len() as u32)) + .collect::>(); + b.iter(|| { + use yrs::{Array, Doc, Transact}; + let doc = Doc::new(); + let array = doc.get_or_insert_array("test"); + + let mut trx = doc.transact_mut(); + for c in base_text.chars() { + array.push_back(&mut trx, c.to_string()); + } + for idx in &idxs { + array.insert_range(&mut trx, *idx, vec!["test1", "test2"]); + } + drop(trx); + }); + }); + + group.bench_function("yrs/remove", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + + b.iter(|| { + use yrs::{Array, Doc, Transact}; + let doc = Doc::new(); + let array = doc.get_or_insert_array("test"); + + let mut trx = doc.transact_mut(); + for c in base_text.chars() { + array.push_back(&mut trx, c.to_string()); + } + for idx in (base_text.len() as u32)..0 { + array.remove(&mut trx, idx); + } + drop(trx); + }); + }); + + group.finish(); +} + +criterion_group!(benches, operations); +criterion_main!(benches); diff --git a/packages/common/y-octo/utils/benches/codec_benchmarks.rs b/packages/common/y-octo/utils/benches/codec_benchmarks.rs new file mode 100644 index 0000000000..efd1b4597a --- /dev/null +++ b/packages/common/y-octo/utils/benches/codec_benchmarks.rs @@ -0,0 +1,89 @@ +use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; +use lib0::{ + decoding::{Cursor, Read}, + encoding::Write, +}; + +const BENCHMARK_SIZE: u32 = 100000; + +fn codec(c: &mut Criterion) { + let mut codec_group = c.benchmark_group("codec"); + codec_group.sampling_mode(SamplingMode::Flat); + { + codec_group.bench_function("lib0 encode var_int (64 bit)", |b| { + b.iter(|| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..(BENCHMARK_SIZE as i64) { + encoder.write_var(i); + } + }) + }); + codec_group.bench_function("lib0 decode var_int (64 bit)", |b| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..(BENCHMARK_SIZE as i64) { + encoder.write_var(i); + } + + b.iter(|| { + let mut decoder = Cursor::from(&encoder); + for i in 0..(BENCHMARK_SIZE as i64) { + let num: i64 = decoder.read_var().unwrap(); + assert_eq!(num, i); + } + }) + }); + } + + { + codec_group.bench_function("lib0 encode var_uint (32 bit)", |b| { + b.iter(|| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..BENCHMARK_SIZE { + encoder.write_var(i); + } + }) + }); + codec_group.bench_function("lib0 decode var_uint (32 bit)", |b| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..BENCHMARK_SIZE { + encoder.write_var(i); + } + + b.iter(|| { + let mut decoder = Cursor::from(&encoder); + for i in 0..BENCHMARK_SIZE { + let num: u32 = decoder.read_var().unwrap(); + assert_eq!(num, i); + } + }) + }); + } + + { + codec_group.bench_function("lib0 encode var_uint (64 bit)", |b| { + b.iter(|| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..(BENCHMARK_SIZE as u64) { + encoder.write_var(i); + } + }) + }); + codec_group.bench_function("lib0 decode var_uint (64 bit)", |b| { + let mut encoder = Vec::with_capacity(BENCHMARK_SIZE as usize * 8); + for i in 0..(BENCHMARK_SIZE as u64) { + encoder.write_var(i); + } + + b.iter(|| { + let mut decoder = Cursor::from(&encoder); + for i in 0..(BENCHMARK_SIZE as u64) { + let num: u64 = decoder.read_var().unwrap(); + assert_eq!(num, i); + } + }) + }); + } +} + +criterion_group!(benches, codec); +criterion_main!(benches); diff --git a/packages/common/y-octo/utils/benches/map_ops_benchmarks.rs b/packages/common/y-octo/utils/benches/map_ops_benchmarks.rs new file mode 100644 index 0000000000..7092e29a1f --- /dev/null +++ b/packages/common/y-octo/utils/benches/map_ops_benchmarks.rs @@ -0,0 +1,79 @@ +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Criterion}; + +fn operations(c: &mut Criterion) { + let mut group = c.benchmark_group("ops/map"); + group.measurement_time(Duration::from_secs(15)); + + group.bench_function("yrs/insert", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" + .split(' ') + .collect::>(); + + b.iter(|| { + use yrs::{Doc, Map, Transact}; + let doc = Doc::new(); + let map = doc.get_or_insert_map("test"); + + let mut trx = doc.transact_mut(); + for (idx, key) in base_text.iter().enumerate() { + map.insert(&mut trx, key.to_string(), idx as f64); + } + + drop(trx); + }); + }); + + group.bench_function("yrs/get", |b| { + use yrs::{Doc, Map, Transact}; + + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" + .split(' ') + .collect::>(); + + let doc = Doc::new(); + let map = doc.get_or_insert_map("test"); + + let mut trx = doc.transact_mut(); + for (idx, key) in base_text.iter().enumerate() { + map.insert(&mut trx, key.to_string(), idx as f64); + } + drop(trx); + + b.iter(|| { + let trx = doc.transact(); + for key in &base_text { + map.get(&trx, key).unwrap(); + } + }); + }); + + group.bench_function("yrs/remove", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" + .split(' ') + .collect::>(); + + b.iter(|| { + use yrs::{Doc, Map, Transact}; + let doc = Doc::new(); + let map = doc.get_or_insert_map("test"); + + let mut trx = doc.transact_mut(); + for (idx, key) in base_text.iter().enumerate() { + map.insert(&mut trx, key.to_string(), idx as f64); + } + + for key in &base_text { + map.remove(&mut trx, key).unwrap(); + } + + drop(trx); + }); + }); + + group.finish(); +} + +criterion_group!(benches, operations); +criterion_main!(benches); diff --git a/packages/common/y-octo/utils/benches/text_ops_benchmarks.rs b/packages/common/y-octo/utils/benches/text_ops_benchmarks.rs new file mode 100644 index 0000000000..7ed21df1ad --- /dev/null +++ b/packages/common/y-octo/utils/benches/text_ops_benchmarks.rs @@ -0,0 +1,54 @@ +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Criterion}; +use rand::{Rng, SeedableRng}; + +fn operations(c: &mut Criterion) { + let mut group = c.benchmark_group("ops/text"); + group.measurement_time(Duration::from_secs(15)); + + group.bench_function("yrs/insert", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(1234); + + let idxs = (0..99) + .map(|_| rng.random_range(0..base_text.len() as u32)) + .collect::>(); + b.iter(|| { + use yrs::{Doc, Text, Transact}; + let doc = Doc::new(); + let text = doc.get_or_insert_text("test"); + let mut trx = doc.transact_mut(); + + text.push(&mut trx, base_text); + for idx in &idxs { + text.insert(&mut trx, *idx, "test"); + } + drop(trx); + }); + }); + + group.bench_function("yrs/remove", |b| { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9"; + + b.iter(|| { + use yrs::{Doc, Text, Transact}; + let doc = Doc::new(); + let text = doc.get_or_insert_text("test"); + let mut trx = doc.transact_mut(); + + text.push(&mut trx, base_text); + text.push(&mut trx, base_text); + text.push(&mut trx, base_text); + for idx in (base_text.len() as u32)..0 { + text.remove_range(&mut trx, idx, 1); + } + drop(trx); + }); + }); + + group.finish(); +} + +criterion_group!(benches, operations); +criterion_main!(benches); diff --git a/packages/common/y-octo/utils/benches/update_benchmarks.rs b/packages/common/y-octo/utils/benches/update_benchmarks.rs new file mode 100644 index 0000000000..5f553845d2 --- /dev/null +++ b/packages/common/y-octo/utils/benches/update_benchmarks.rs @@ -0,0 +1,33 @@ +mod utils; + +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use path_ext::PathExt; +use utils::Files; + +fn update(c: &mut Criterion) { + let files = Files::load(); + + let mut group = c.benchmark_group("update"); + group.measurement_time(Duration::from_secs(15)); + + for file in &files.files { + group.throughput(Throughput::Bytes(file.content.len() as u64)); + group.bench_with_input( + BenchmarkId::new("parse with yrs", file.path.name_str()), + &file.content, + |b, content| { + b.iter(|| { + use yrs::{updates::decoder::Decode, Update}; + Update::decode_v1(content).unwrap() + }); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, update); +criterion_main!(benches); diff --git a/packages/common/y-octo/utils/benches/utils/files.rs b/packages/common/y-octo/utils/benches/utils/files.rs new file mode 100644 index 0000000000..b390d94678 --- /dev/null +++ b/packages/common/y-octo/utils/benches/utils/files.rs @@ -0,0 +1,42 @@ +use std::{ + fs::{read, read_dir}, + path::{Path, PathBuf}, +}; + +use path_ext::PathExt; + +pub struct File { + pub path: PathBuf, + pub content: Vec, +} + +const BASE: &str = "../y-octo/src/fixtures/"; + +impl File { + fn new(path: &Path) -> Self { + let content = read(path).unwrap(); + Self { + path: path.into(), + content, + } + } +} + +pub struct Files { + pub files: Vec, +} + +impl Files { + pub fn load() -> Self { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(BASE); + + let files = read_dir(path).unwrap(); + let files = files + .flatten() + .filter(|f| f.path().is_file() && f.path().ext_str() == "bin") + .map(|f| File::new(&f.path())) + .collect::>(); + + Self { files } + } +} diff --git a/packages/common/y-octo/utils/benches/utils/mod.rs b/packages/common/y-octo/utils/benches/utils/mod.rs new file mode 100644 index 0000000000..412eb3d0a6 --- /dev/null +++ b/packages/common/y-octo/utils/benches/utils/mod.rs @@ -0,0 +1,3 @@ +mod files; + +pub use files::Files; diff --git a/packages/common/y-octo/utils/bin/bench_result_render.rs b/packages/common/y-octo/utils/bin/bench_result_render.rs new file mode 100644 index 0000000000..ecb54a743b --- /dev/null +++ b/packages/common/y-octo/utils/bin/bench_result_render.rs @@ -0,0 +1,134 @@ +use std::{ + collections::HashMap, + io::{self, BufRead}, +}; + +fn process_duration(duration: &str) -> Option<(f64, f64)> { + let dur_split: Vec = duration.split('±').map(String::from).collect(); + if dur_split.len() != 2 { + return None; + } + let units = dur_split[1] + .chars() + .skip_while(|c| c.is_ascii_digit()) + .collect::(); + let dur_secs = dur_split[0].parse::().ok()?; + let error_secs = dur_split[1] + .chars() + .take_while(|c| c.is_ascii_digit()) + .collect::() + .parse::() + .ok()?; + Some(( + convert_dur_to_seconds(dur_secs, &units), + convert_dur_to_seconds(error_secs, &units), + )) +} + +fn convert_dur_to_seconds(dur: f64, units: &str) -> f64 { + let factors: HashMap<_, _> = [ + ("s", 1.0), + ("ms", 1.0 / 1000.0), + ("µs", 1.0 / 1_000_000.0), + ("ns", 1.0 / 1_000_000_000.0), + ] + .iter() + .cloned() + .collect(); + dur * factors.get(units).unwrap_or(&1.0) +} + +fn is_significant(changes_dur: f64, changes_err: f64, base_dur: f64, base_err: f64) -> bool { + if changes_dur < base_dur { + changes_dur + changes_err < base_dur || base_dur - base_err > changes_dur + } else { + changes_dur - changes_err > base_dur || base_dur + base_err < changes_dur + } +} + +fn convert_to_markdown() -> impl Iterator { + #[cfg(feature = "bench")] + let re = regex::Regex::new(r"\s{2,}").unwrap(); + io::stdin() + .lock() + .lines() + .skip(2) + .flat_map(move |row| { + if let Ok(_row) = row { + let columns = { + #[cfg(feature = "bench")] + { + re.split(&_row).collect::>() + } + #[cfg(not(feature = "bench"))] + Vec::<&str>::new() + }; + let name = columns.first()?; + let base_duration = columns.get(2)?; + let changes_duration = columns.get(5)?; + Some(( + name.to_string(), + base_duration.to_string(), + changes_duration.to_string(), + )) + } else { + None + } + }) + .flat_map(|(name, base_duration, changes_duration)| { + let mut difference = "N/A".to_string(); + let base_undefined = base_duration == "?"; + let changes_undefined = changes_duration == "?"; + + if !base_undefined && !changes_undefined { + let (base_dur_secs, base_err_secs) = process_duration(&base_duration)?; + let (changes_dur_secs, changes_err_secs) = process_duration(&changes_duration)?; + + let diff = -(1.0 - changes_dur_secs / base_dur_secs) * 100.0; + difference = format!("{:+.2}%", diff); + + if is_significant( + changes_dur_secs, + changes_err_secs, + base_dur_secs, + base_err_secs, + ) { + difference = format!("**{}**", difference); + } + } + + Some(format!( + "| {} | {} | {} | {} |", + name.replace('|', "\\|"), + if base_undefined { + "N/A" + } else { + &base_duration + }, + if changes_undefined { + "N/A" + } else { + &changes_duration + }, + difference + )) + }) +} + +fn main() { + let platform = std::env::args().nth(1).expect("Missing platform argument"); + + let headers = vec![ + format!("## Benchmark for {}", platform), + "
".to_string(), + " Click to view benchmark".to_string(), + "".to_string(), + "| Test | Base | PR | % |".to_string(), + "| --- | --- | --- | --- |".to_string(), + ]; + + for line in headers.into_iter().chain(convert_to_markdown()) { + println!("{}", line); + } + println!("
"); +} diff --git a/packages/common/y-octo/utils/bin/doc_merger.rs b/packages/common/y-octo/utils/bin/doc_merger.rs new file mode 100644 index 0000000000..c43c320a6a --- /dev/null +++ b/packages/common/y-octo/utils/bin/doc_merger.rs @@ -0,0 +1,100 @@ +use std::{ + fs::read, + io::{Error, ErrorKind}, + path::PathBuf, + time::Instant, +}; + +use clap::Parser; +use y_octo::Doc; + +/// ybinary merger +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Path of the ybinary to read + #[arg(short, long)] + path: String, +} + +fn load_path(path: &str) -> Result>, Error> { + let path = PathBuf::from(path); + if path.is_dir() { + let mut updates = Vec::new(); + let mut paths = path + .read_dir()? + .filter_map(|entry| { + let entry = entry.ok()?; + if entry.path().is_file() { + Some(entry.path()) + } else { + None + } + }) + .collect::>(); + paths.sort(); + + for path in paths { + println!("read {:?}", path); + updates.push(read(path)?); + } + Ok(updates) + } else if path.is_file() { + Ok(vec![read(path)?]) + } else { + Err(Error::new(ErrorKind::NotFound, "not a file or directory")) + } +} + +fn main() { + let args = Args::parse(); + jwst_merge(&args.path); +} + +fn jwst_merge(path: &str) { + let updates = load_path(path).unwrap(); + + let mut doc = Doc::default(); + for (i, update) in updates.iter().enumerate() { + println!("apply update{i} {} bytes", update.len()); + doc.apply_update_from_binary_v1(update.clone()).unwrap(); + } + + println!("press enter to continue"); + std::io::stdin().read_line(&mut String::new()).unwrap(); + let ts = Instant::now(); + let history = doc.history().parse_store(Default::default()); + println!("history: {:?}", ts.elapsed()); + for history in history.iter().take(100) { + println!("history: {:?}", history); + } + + doc.gc().unwrap(); + + let binary = { + let binary = doc.encode_update_v1().unwrap(); + + println!("merged {} bytes", binary.len()); + + binary + }; + + { + let mut doc = Doc::default(); + doc.apply_update_from_binary_v1(binary.clone()).unwrap(); + let new_binary = doc.encode_update_v1().unwrap(); + + println!("re-encoded {} bytes", new_binary.len(),); + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[ignore = "only for debug"] + fn test_gc() { + jwst_merge("/Users/ds/Downloads/out"); + } +} diff --git a/packages/common/y-octo/utils/bin/memory_leak_test.rs b/packages/common/y-octo/utils/bin/memory_leak_test.rs new file mode 100644 index 0000000000..cfa4b4eed2 --- /dev/null +++ b/packages/common/y-octo/utils/bin/memory_leak_test.rs @@ -0,0 +1,79 @@ +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha20Rng; +use y_octo::*; + +fn run_text_test(seed: u64) { + let doc = Doc::with_client(1); + let mut rand = ChaCha20Rng::seed_from_u64(seed); + let mut text = doc.get_or_create_text("test").unwrap(); + text.insert(0, "This is a string with length 32.").unwrap(); + + let iteration = 20; + let mut len = 32; + + for i in 0..iteration { + let mut text = text.clone(); + let ins = i % 2 == 0; + let pos = rand.random_range(0..if ins { text.len() } else { len / 2 }); + if ins { + let str = format!("hello {i}"); + text.insert(pos, &str).unwrap(); + len += str.len() as u64; + } else { + text.remove(pos, 6).unwrap(); + len -= 6; + } + } + + assert_eq!(text.to_string().len(), len as usize); + assert_eq!(text.len(), len); +} + +fn run_array_test(seed: u64) { + let doc = Doc::with_client(1); + let mut rand = ChaCha20Rng::seed_from_u64(seed); + let mut array = doc.get_or_create_array("test").unwrap(); + array.push(1).unwrap(); + + let iteration = 20; + let mut len = 1; + + for i in 0..iteration { + let mut array = array.clone(); + let ins = i % 2 == 0; + let pos = rand.random_range(0..if ins { array.len() } else { len / 2 }); + if ins { + array.insert(pos, 1).unwrap(); + len += 1; + } else { + array.remove(pos, 1).unwrap(); + len -= 1; + } + } + + assert_eq!(array.len(), len); +} + +fn run_map_test() { + let base_text = "test1 test2 test3 test4 test5 test6 test7 test8 test9" + .split(' ') + .collect::>(); + + for _ in 0..10000 { + let doc = Doc::default(); + let mut map = doc.get_or_create_map("test").unwrap(); + for (idx, key) in base_text.iter().enumerate() { + map.insert(key.to_string(), idx).unwrap(); + } + } +} + +fn main() { + let mut rand = ChaCha20Rng::seed_from_u64(rand::rng().random()); + for _ in 0..10000 { + let seed = rand.random(); + run_array_test(seed); + run_text_test(seed); + run_map_test(); + } +} diff --git a/packages/common/y-octo/utils/fuzz/.gitignore b/packages/common/y-octo/utils/fuzz/.gitignore new file mode 100644 index 0000000000..1a45eee776 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/packages/common/y-octo/utils/fuzz/Cargo.lock b/packages/common/y-octo/utils/fuzz/Cargo.lock new file mode 100644 index 0000000000..d4e834f81c --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/Cargo.lock @@ -0,0 +1,1483 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom 0.2.15", + "once_cell", + "version_check", + "zerocopy 0.7.35", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys", +] + +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener", + "event-listener-strategy", + "loom", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "borsh" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" +dependencies = [ + "cfg_aliases", +] + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "clap" +version = "4.5.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", + "loom", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "loom", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +dependencies = [ + "getrandom 0.2.15", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "generator" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" +dependencies = [ + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.2", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lasso" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" +dependencies = [ + "dashmap", + "hashbrown", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lib0" +version = "0.16.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29dc19a026a0d45fc391898c6d4a6d0a5aab5ae6a826ebddc0f33572ffdae8dc" +dependencies = [ + "serde", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "nanoid" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8" +dependencies = [ + "rand 0.8.5", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "ordered-float" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2c1f9f56e534ac6a9b8a4600bdf0f530fb393b5f393e7b4d03489c3cf0c3f01" +dependencies = [ + "arbitrary", + "num-traits", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +dependencies = [ + "loom", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy 0.8.24", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.24", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.0", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "redox_syscall" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + +[[package]] +name = "smallstr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b1aefdf380735ff8ded0b15f31aab05daf1f70216c01c02a12926badd1df9d" +dependencies = [ + "smallvec", +] + +[[package]] +name = "smallvec" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" + +[[package]] +name = "smol_str" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9676b89cd56310a87b93dec47b11af744f34d5fc9f367b829474eec0a891350d" +dependencies = [ + "borsh", + "serde", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "y-octo" +version = "0.0.1" +dependencies = [ + "ahash", + "arbitrary", + "async-lock", + "bitvec", + "byteorder", + "lasso", + "log", + "loom", + "nanoid", + "nom", + "ordered-float", + "rand 0.9.0", + "rand_chacha 0.9.0", + "rand_distr", + "serde", + "serde_json", + "smol_str", + "thiserror 2.0.12", +] + +[[package]] +name = "y-octo-fuzz" +version = "0.0.0" +dependencies = [ + "lib0", + "libfuzzer-sys", + "rand 0.9.0", + "rand_chacha 0.9.0", + "y-octo", + "y-octo-utils", + "yrs 0.23.0", +] + +[[package]] +name = "y-octo-utils" +version = "0.0.1" +dependencies = [ + "arbitrary", + "clap", + "lib0", + "phf", + "rand 0.9.0", + "rand_chacha 0.9.0", + "y-octo", + "y-sync", + "yrs 0.23.0", +] + +[[package]] +name = "y-sync" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e3675a497cde881a71e7e5c2ae1d087dfc7733ddece9b24a9a61408e969d3b" +dependencies = [ + "thiserror 1.0.69", + "yrs 0.17.4", +] + +[[package]] +name = "yrs" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4830316bfee4bec0044fe34a001cda783506d5c4c0852f8433c6041dfbfce51" +dependencies = [ + "atomic_refcell", + "rand 0.7.3", + "serde", + "serde_json", + "smallstr", + "smallvec", + "thiserror 1.0.69", +] + +[[package]] +name = "yrs" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0189b51d8ab1283e7c1f1f515c610875262e629cf258bec530da5cd4aa115d59" +dependencies = [ + "arc-swap", + "async-lock", + "async-trait", + "dashmap", + "fastrand", + "serde", + "serde_json", + "smallstr", + "smallvec", + "thiserror 1.0.69", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive 0.8.24", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/packages/common/y-octo/utils/fuzz/Cargo.toml b/packages/common/y-octo/utils/fuzz/Cargo.toml new file mode 100644 index 0000000000..999782018c --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/Cargo.toml @@ -0,0 +1,88 @@ +[package] +edition = "2021" +name = "y-octo-fuzz" +publish = false +version = "0.0.0" + + [package.metadata] + cargo-fuzz = true + +[dependencies] +lib0 = "=0.16.10" +libfuzzer-sys = "0.4" +rand = "0.9" +rand_chacha = "0.9" +yrs = "=0.23.0" + +y-octo-utils = { path = "..", features = ["fuzz"] } + + [dependencies.y-octo] + path = "../../core" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[profile.release] +debug = 1 + +[[bin]] +doc = false +name = "codec_doc_any_struct" +path = "fuzz_targets/codec_doc_any_struct.rs" +test = false + +[[bin]] +doc = false +name = "codec_doc_any" +path = "fuzz_targets/codec_doc_any.rs" +test = false + +[[bin]] +doc = false +name = "decode_bytes" +path = "fuzz_targets/decode_bytes.rs" +test = false + +[[bin]] +doc = false +name = "ins_del_text" +path = "fuzz_targets/ins_del_text.rs" +test = false + +[[bin]] +doc = false +name = "sync_message" +path = "fuzz_targets/sync_message.rs" +test = false + +[[bin]] +doc = false +name = "i32_decode" +path = "fuzz_targets/i32_decode.rs" +test = false + +[[bin]] +doc = false +name = "i32_encode" +path = "fuzz_targets/i32_encode.rs" +test = false + +[[bin]] +doc = false +name = "u64_decode" +path = "fuzz_targets/u64_decode.rs" +test = false + + +[[bin]] +doc = false +name = "u64_encode" +path = "fuzz_targets/u64_encode.rs" +test = false + +[[bin]] +doc = false +name = "apply_update" +path = "fuzz_targets/apply_update.rs" +test = false diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/apply_update.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/apply_update.rs new file mode 100644 index 0000000000..9a4a365cc0 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/apply_update.rs @@ -0,0 +1,51 @@ +#![no_main] + +use std::collections::HashSet; + +use libfuzzer_sys::fuzz_target; +use y_octo_utils::{ + gen_nest_type_from_nest_type, gen_nest_type_from_root, CRDTParam, ManipulateSource, OpType, + OpsRegistry, YrsNestType, +}; +use yrs::Transact; + +fuzz_target!(|crdt_params: Vec| { + let mut doc = yrs::Doc::new(); + let mut cur_crdt_nest_type: Option = None; + let ops_registry = OpsRegistry::new(); + let mut key_set = HashSet::::new(); + for crdt_param in crdt_params { + if key_set.contains(&crdt_param.key) { + continue; + } + + key_set.insert(crdt_param.key.clone()); + match crdt_param.op_type { + OpType::HandleCurrent => { + if cur_crdt_nest_type.is_some() { + ops_registry.operate_yrs_nest_type(&doc, cur_crdt_nest_type.clone().unwrap(), crdt_param); + } + } + OpType::CreateCRDTNestType => { + cur_crdt_nest_type = match cur_crdt_nest_type { + None => gen_nest_type_from_root(&mut doc, &crdt_param), + Some(mut nest_type) => match crdt_param.manipulate_source { + ManipulateSource::CurrentNestType => Some(nest_type), + ManipulateSource::NewNestTypeFromYDocRoot => { + gen_nest_type_from_root(&mut doc, &crdt_param) + } + ManipulateSource::NewNestTypeFromCurrent => { + gen_nest_type_from_nest_type(&mut doc, crdt_param.clone(), &mut nest_type) + } + }, + }; + } + }; + } + + let trx = doc.transact_mut(); + let binary_from_yrs = trx.encode_update_v1(); + let doc = y_octo::Doc::try_from_binary_v1(&binary_from_yrs).unwrap(); + let binary = doc.encode_update_v1().unwrap(); + assert_eq!(binary, binary_from_yrs); +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/codec_doc_any.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/codec_doc_any.rs new file mode 100644 index 0000000000..eafe19ad74 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/codec_doc_any.rs @@ -0,0 +1,17 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use y_octo::{Any, CrdtRead, CrdtWrite, RawDecoder, RawEncoder}; + +fuzz_target!(|data: &[u8]| { + if let Ok(any) = Any::read(&mut RawDecoder::new(data)) { + // ensure decoding and re-encoding results has same result + let mut buffer = RawEncoder::default(); + if let Err(e) = any.write(&mut buffer) { + panic!("Failed to write message: {:?}, {:?}", any, e); + } + if let Ok(any2) = Any::read(&mut RawDecoder::new(&buffer.into_inner())) { + assert_eq!(any, any2); + } + } +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/codec_doc_any_struct.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/codec_doc_any_struct.rs new file mode 100644 index 0000000000..4001499848 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/codec_doc_any_struct.rs @@ -0,0 +1,43 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rand::{distr::Alphanumeric, Rng}; +use y_octo::{Any, CrdtRead, CrdtWrite, RawDecoder, RawEncoder}; + +fn get_random_string() -> String { + rand::rng() + .sample_iter(&Alphanumeric) + .take(7) + .map(char::from) + .collect() +} + +fuzz_target!(|data: Vec| { + { + let any = Any::Object( + data + .iter() + .map(|a| (get_random_string(), a.clone())) + .collect(), + ); + + let mut buffer = RawEncoder::default(); + if let Err(e) = any.write(&mut buffer) { + panic!("Failed to write message: {:?}, {:?}", any, e); + } + if let Ok(any2) = Any::read(&mut RawDecoder::new(&buffer.into_inner())) { + assert_eq!(any, any2); + } + } + + { + let any = Any::Array(data); + let mut buffer = RawEncoder::default(); + if let Err(e) = any.write(&mut buffer) { + panic!("Failed to write message: {:?}, {:?}", any, e); + } + if let Ok(any2) = Any::read(&mut RawDecoder::new(&buffer.into_inner())) { + assert_eq!(any, any2); + } + } +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/decode_bytes.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/decode_bytes.rs new file mode 100644 index 0000000000..33f0f785c1 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/decode_bytes.rs @@ -0,0 +1,11 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use y_octo::{read_var_buffer, read_var_i32, read_var_string, read_var_u64}; + +fuzz_target!(|data: Vec| { + let _ = read_var_i32(&data); + let _ = read_var_u64(&data); + let _ = read_var_buffer(&data); + let _ = read_var_string(&data); +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/i32_decode.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/i32_decode.rs new file mode 100644 index 0000000000..ef18f9f9c2 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/i32_decode.rs @@ -0,0 +1,18 @@ +#![no_main] + +use lib0::encoding::Write; +use libfuzzer_sys::fuzz_target; +use y_octo::{read_var_i32, write_var_i32}; + +fuzz_target!(|data: Vec| { + for i in data { + let mut buf1 = Vec::new(); + write_var_i32(&mut buf1, i).unwrap(); + + let mut buf2 = Vec::new(); + buf2.write_var(i); + + assert_eq!(read_var_i32(&buf1).unwrap().1, i); + assert_eq!(read_var_i32(&buf2).unwrap().1, i); + } +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/i32_encode.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/i32_encode.rs new file mode 100644 index 0000000000..3ae9d1c52d --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/i32_encode.rs @@ -0,0 +1,17 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use y_octo::write_var_i32; + +fuzz_target!(|data: Vec| { + use lib0::encoding::Write; + + for i in data { + let mut buf1 = Vec::new(); + write_var_i32(&mut buf1, i).unwrap(); + let mut buf2 = Vec::new(); + buf2.write_var(i); + + assert_eq!(buf1, buf2); + } +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/ins_del_text.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/ins_del_text.rs new file mode 100644 index 0000000000..2e43151bb0 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/ins_del_text.rs @@ -0,0 +1,34 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha20Rng; +use y_octo::*; + +fuzz_target!(|seed: u64| { + // println!("seed: {}", seed); + let doc = Doc::with_client(1); + let mut rand = ChaCha20Rng::seed_from_u64(seed); + let mut text = doc.get_or_create_text("test").unwrap(); + text.insert(0, "This is a string with length 32.").unwrap(); + + let iteration = 20; + let mut len = 32; + + for i in 0..iteration { + let mut text = text.clone(); + let ins = i % 2 == 0; + let pos = rand.random_range(0..if ins { text.len() } else { len / 2 }); + if ins { + let str = format!("hello {i}"); + text.insert(pos, &str).unwrap(); + len += str.len() as u64; + } else { + text.remove(pos, 6).unwrap(); + len -= 6; + } + } + + assert_eq!(text.to_string().len(), len as usize); + assert_eq!(text.len(), len); +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/sync_message.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/sync_message.rs new file mode 100644 index 0000000000..d3b44b5a68 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/sync_message.rs @@ -0,0 +1,20 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use y_octo::{read_sync_message, write_sync_message}; + +fuzz_target!(|data: &[u8]| { + let result = read_sync_message(data); + + if let Ok((_, msg)) = result { + // ensure decoding and re-encoding results has same result + let mut buffer = Vec::new(); + if let Err(e) = write_sync_message(&mut buffer, &msg) { + panic!("Failed to write message: {:?}, {:?}", msg, e); + } + let result = read_sync_message(&buffer); + if let Ok((_, msg2)) = result { + assert_eq!(msg, msg2); + } + } +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/u64_decode.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/u64_decode.rs new file mode 100644 index 0000000000..0f1fe53513 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/u64_decode.rs @@ -0,0 +1,18 @@ +#![no_main] + +use lib0::encoding::Write; +use libfuzzer_sys::fuzz_target; +use y_octo::{read_var_u64, write_var_u64}; + +fuzz_target!(|data: Vec| { + for i in data { + let mut buf1 = Vec::new(); + write_var_u64(&mut buf1, i).unwrap(); + + let mut buf2 = Vec::new(); + buf2.write_var(i); + + assert_eq!(read_var_u64(&buf1).unwrap().1, i); + assert_eq!(read_var_u64(&buf2).unwrap().1, i); + } +}); diff --git a/packages/common/y-octo/utils/fuzz/fuzz_targets/u64_encode.rs b/packages/common/y-octo/utils/fuzz/fuzz_targets/u64_encode.rs new file mode 100644 index 0000000000..e24c90b755 --- /dev/null +++ b/packages/common/y-octo/utils/fuzz/fuzz_targets/u64_encode.rs @@ -0,0 +1,17 @@ +#![no_main] + +use lib0::encoding::Write; +use libfuzzer_sys::fuzz_target; +use y_octo::write_var_u64; + +fuzz_target!(|data: Vec| { + for i in data { + let mut buf1 = Vec::new(); + buf1.write_var(i); + + let mut buf2 = Vec::new(); + write_var_u64(&mut buf2, i).unwrap(); + + assert_eq!(buf1, buf2); + } +}); diff --git a/packages/common/y-octo/utils/src/codec.rs b/packages/common/y-octo/utils/src/codec.rs new file mode 100644 index 0000000000..5a6702bedb --- /dev/null +++ b/packages/common/y-octo/utils/src/codec.rs @@ -0,0 +1,78 @@ +use super::*; + +#[cfg(test)] +mod tests { + + use super::*; + + use lib0::encoding::Write; + + fn test_var_uint_enc_dec(num: u64) { + let mut buf1 = Vec::new(); + write_var_u64(&mut buf1, num).unwrap(); + + let mut buf2 = Vec::new(); + buf2.write_var(num); + + { + let (rest, decoded_num) = read_var_u64(&buf1).unwrap(); + assert_eq!(num, decoded_num); + assert_eq!(rest.len(), 0); + } + + { + let (rest, decoded_num) = read_var_u64(&buf2).unwrap(); + assert_eq!(num, decoded_num); + assert_eq!(rest.len(), 0); + } + } + + fn test_var_int_enc_dec(num: i32) { + { + let mut buf1: Vec = Vec::new(); + write_var_i32(&mut buf1, num).unwrap(); + + let (rest, decoded_num) = read_var_i32(&buf1).unwrap(); + assert_eq!(num, decoded_num); + assert_eq!(rest.len(), 0); + } + + { + let mut buf2 = Vec::new(); + buf2.write_var(num); + + let (rest, decoded_num) = read_var_i32(&buf2).unwrap(); + assert_eq!(num, decoded_num); + assert_eq!(rest.len(), 0); + } + } + + #[test] + fn test_var_uint_codec() { + test_var_uint_enc_dec(0); + test_var_uint_enc_dec(1); + test_var_uint_enc_dec(127); + test_var_uint_enc_dec(0b1000_0000); + test_var_uint_enc_dec(0b1_0000_0000); + test_var_uint_enc_dec(0b1_1111_1111); + test_var_uint_enc_dec(0b10_0000_0000); + test_var_uint_enc_dec(0b11_1111_1111); + test_var_uint_enc_dec(0x7fff_ffff_ffff_ffff); + test_var_uint_enc_dec(u64::max_value()); + } + + #[test] + fn test_var_int() { + test_var_int_enc_dec(0); + test_var_int_enc_dec(1); + test_var_int_enc_dec(-1); + test_var_int_enc_dec(63); + test_var_int_enc_dec(-63); + test_var_int_enc_dec(64); + test_var_int_enc_dec(-64); + test_var_int_enc_dec(i32::MAX); + test_var_int_enc_dec(i32::MIN); + test_var_int_enc_dec(((1 << 20) - 1) * 8); + test_var_int_enc_dec(-((1 << 20) - 1) * 8); + } +} diff --git a/packages/common/y-octo/utils/src/doc.rs b/packages/common/y-octo/utils/src/doc.rs new file mode 100644 index 0000000000..6051ca8ff5 --- /dev/null +++ b/packages/common/y-octo/utils/src/doc.rs @@ -0,0 +1,21 @@ +#[cfg(test)] +mod tests { + use y_octo::Doc; + use yrs::{Map, Transact}; + + #[test] + fn test_basic_yrs_binary_compatibility() { + let yrs_doc = yrs::Doc::new(); + + let map = yrs_doc.get_or_insert_map("abc"); + let mut trx = yrs_doc.transact_mut(); + map.insert(&mut trx, "a", 1); + + let binary_from_yrs = trx.encode_update_v1(); + + let doc = Doc::try_from_binary_v1(&binary_from_yrs).unwrap(); + let binary = doc.encode_update_v1().unwrap(); + + assert_eq!(binary_from_yrs, binary); + } +} diff --git a/packages/common/y-octo/utils/src/doc_operation/mod.rs b/packages/common/y-octo/utils/src/doc_operation/mod.rs new file mode 100644 index 0000000000..cca3e84e7c --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/mod.rs @@ -0,0 +1,5 @@ +pub mod types; +pub mod yrs_op; + +pub use types::*; +pub use yrs_op::*; diff --git a/packages/common/y-octo/utils/src/doc_operation/types.rs b/packages/common/y-octo/utils/src/doc_operation/types.rs new file mode 100644 index 0000000000..b2812e8b53 --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/types.rs @@ -0,0 +1,63 @@ +use yrs::{ArrayRef, MapRef, TextRef, XmlFragmentRef, XmlTextRef}; + +pub const NEST_DATA_INSERT: &str = "insert"; +pub const NEST_DATA_DELETE: &str = "delete"; +pub const NEST_DATA_CLEAR: &str = "clear"; + +#[derive(Hash, PartialEq, Eq, Clone, Debug, arbitrary::Arbitrary)] +pub enum OpType { + HandleCurrent, + CreateCRDTNestType, +} + +#[derive(Hash, PartialEq, Eq, Clone, Debug, arbitrary::Arbitrary)] +pub enum NestDataOpType { + Insert, + Delete, + Clear, +} + +#[derive(PartialEq, Clone, Debug, arbitrary::Arbitrary)] +pub struct CRDTParam { + pub op_type: OpType, + pub new_nest_type: CRDTNestType, + pub manipulate_source: ManipulateSource, + pub insert_pos: InsertPos, + pub key: String, + pub value: String, + pub nest_data_op_type: NestDataOpType, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, arbitrary::Arbitrary)] +pub enum CRDTNestType { + Array, + Map, + Text, + XMLElement, + XMLFragment, + XMLText, +} + +#[derive(Debug, Clone, PartialEq, arbitrary::Arbitrary)] +pub enum ManipulateSource { + NewNestTypeFromYDocRoot, + CurrentNestType, + NewNestTypeFromCurrent, +} + +#[derive(Debug, Clone, PartialEq, arbitrary::Arbitrary)] +pub enum InsertPos { + BEGIN, + MID, + END, +} + +#[derive(Clone)] +pub enum YrsNestType { + ArrayType(ArrayRef), + MapType(MapRef), + TextType(TextRef), + XMLElementType(XmlFragmentRef), + XMLFragmentType(XmlFragmentRef), + XMLTextType(XmlTextRef), +} diff --git a/packages/common/y-octo/utils/src/doc_operation/yrs_op/array.rs b/packages/common/y-octo/utils/src/doc_operation/yrs_op/array.rs new file mode 100644 index 0000000000..afe59aeb86 --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/yrs_op/array.rs @@ -0,0 +1,172 @@ +use phf::phf_map; + +use super::*; + +fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let array = match nest_input { + YrsNestType::ArrayType(array) => array, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = array.len(&trx); + let index = random_pick_num(len, ¶ms.insert_pos); + array.insert(&mut trx, index, params.value); +} + +fn delete_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let array = match nest_input { + YrsNestType::ArrayType(array) => array, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = array.len(&trx); + if len >= 1 { + let index = random_pick_num(len - 1, ¶ms.insert_pos); + array.remove(&mut trx, index); + } +} + +fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { + let array = match nest_input { + YrsNestType::ArrayType(array) => array, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = array.len(&trx); + for _ in 0..len { + array.remove(&mut trx, 0); + } +} + +pub static ARRAY_OPS: TestOps = phf_map! { + "insert" => insert_op, + "delete" => delete_op, + "clear" => clear_op, +}; + +pub fn yrs_create_array_from_nest_type( + doc: &yrs::Doc, + current: &mut YrsNestType, + insert_pos: &InsertPos, + key: String, +) -> Option { + let cal_index = |len: u32| -> u32 { + match insert_pos { + InsertPos::BEGIN => 0, + InsertPos::MID => len / 2, + InsertPos::END => len, + } + }; + let mut trx = doc.transact_mut(); + let array_prelim = ArrayPrelim::default(); + match current { + YrsNestType::ArrayType(array) => { + let index = cal_index(array.len(&trx)); + Some(array.insert(&mut trx, index, array_prelim)) + } + YrsNestType::MapType(map) => Some(map.insert(&mut trx, key, array_prelim)), + YrsNestType::TextType(text) => { + let str = text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + let index = random_pick_num(len, insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + Some(text.insert_embed(&mut trx, byte_start_offset as u32, array_prelim)) + } + YrsNestType::XMLTextType(xml_text) => { + let str = xml_text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + let index = random_pick_num(len, insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + Some(xml_text.insert_embed(&mut trx, byte_start_offset as u32, array_prelim)) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use yrs::Doc; + + use super::*; + + #[test] + fn test_gen_array_ref_ops() { + let doc = Doc::new(); + let array_ref = doc.get_or_insert_array("test_array"); + + let ops_registry = OpsRegistry::new(); + + let mut params = CRDTParam { + op_type: OpType::CreateCRDTNestType, + new_nest_type: CRDTNestType::Array, + manipulate_source: ManipulateSource::NewNestTypeFromYDocRoot, + insert_pos: InsertPos::BEGIN, + key: String::from("test_key"), + value: String::from("test_value"), + nest_data_op_type: NestDataOpType::Insert, + }; + + ops_registry.operate_yrs_nest_type( + &doc, + YrsNestType::ArrayType(array_ref.clone()), + params.clone(), + ); + assert_eq!(array_ref.len(&doc.transact()), 1); + params.nest_data_op_type = NestDataOpType::Delete; + ops_registry.operate_yrs_nest_type( + &doc, + YrsNestType::ArrayType(array_ref.clone()), + params.clone(), + ); + assert_eq!(array_ref.len(&doc.transact()), 0); + + params.nest_data_op_type = NestDataOpType::Clear; + ops_registry.operate_yrs_nest_type( + &doc, + YrsNestType::ArrayType(array_ref.clone()), + params.clone(), + ); + assert_eq!(array_ref.len(&doc.transact()), 0); + } + + #[test] + fn test_yrs_create_array_from_nest_type() { + let doc = Doc::new(); + let array_ref = doc.get_or_insert_array("test_array"); + let key = String::from("test_key"); + + let new_array_ref = yrs_create_array_from_nest_type( + &doc, + &mut YrsNestType::ArrayType(array_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(new_array_ref.is_some()); + + let map_ref = doc.get_or_insert_map("test_map"); + let new_array_ref = yrs_create_array_from_nest_type( + &doc, + &mut YrsNestType::MapType(map_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(new_array_ref.is_some()); + + let text_ref = doc.get_or_insert_text("test_text"); + let new_array_ref = yrs_create_array_from_nest_type( + &doc, + &mut YrsNestType::TextType(text_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(new_array_ref.is_some()); + } +} diff --git a/packages/common/y-octo/utils/src/doc_operation/yrs_op/map.rs b/packages/common/y-octo/utils/src/doc_operation/yrs_op/map.rs new file mode 100644 index 0000000000..754a5e5514 --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/yrs_op/map.rs @@ -0,0 +1,168 @@ +use phf::phf_map; + +use super::*; + +fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let map = match nest_input { + YrsNestType::MapType(map) => map, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + map.insert(&mut trx, params.key, params.value); +} + +fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let map = match nest_input { + YrsNestType::MapType(map) => map, + _ => unreachable!(), + }; + let rand_key = { + let trx = doc.transact_mut(); + let mut iter = map.iter(&trx); + let len = map.len(&trx) as usize; + let skip_step = if len <= 1 { + 0 + } else { + random_pick_num((len - 1) as u32, ¶ms.insert_pos) + }; + + iter + .nth(skip_step as usize) + .map(|(key, _value)| key.to_string()) + }; + + if let Some(key) = rand_key { + let mut trx = doc.transact_mut(); + map.remove(&mut trx, &key).unwrap(); + } +} + +fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { + let map = match nest_input { + YrsNestType::MapType(map) => map, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + map.clear(&mut trx); +} + +pub static MAP_OPS: TestOps = phf_map! { + "insert" => insert_op, + "delete" => remove_op, + "clear" => clear_op, +}; + +pub fn yrs_create_map_from_nest_type( + doc: &yrs::Doc, + current: &mut YrsNestType, + insert_pos: &InsertPos, + key: String, +) -> Option { + let cal_index = |len: u32| -> u32 { + match insert_pos { + InsertPos::BEGIN => 0, + InsertPos::MID => len / 2, + InsertPos::END => len, + } + }; + let mut trx = doc.transact_mut(); + let map_prelim = MapPrelim::from([("deepkey".to_owned(), "deepvalue")]); + match current { + YrsNestType::ArrayType(array) => { + let index = cal_index(array.len(&trx)); + Some(array.insert(&mut trx, index, map_prelim)) + } + YrsNestType::MapType(map) => Some(map.insert(&mut trx, key, map_prelim)), + YrsNestType::TextType(text) => { + let str = text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + let index = random_pick_num(len, insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + Some(text.insert_embed(&mut trx, byte_start_offset as u32, map_prelim)) + } + YrsNestType::XMLTextType(xml_text) => { + let str = xml_text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + let index = random_pick_num(len, insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + Some(xml_text.insert_embed(&mut trx, byte_start_offset as u32, map_prelim)) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use yrs::Doc; + + use super::*; + + #[test] + fn test_gen_map_ref_ops() { + let doc = Doc::new(); + let map_ref = doc.get_or_insert_map("test_map"); + + let ops_registry = OpsRegistry::new(); + + let mut params = CRDTParam { + op_type: OpType::CreateCRDTNestType, + new_nest_type: CRDTNestType::Map, + manipulate_source: ManipulateSource::NewNestTypeFromYDocRoot, + insert_pos: InsertPos::BEGIN, + key: String::from("test_key"), + value: String::from("test_value"), + nest_data_op_type: NestDataOpType::Insert, + }; + + ops_registry.operate_yrs_nest_type(&doc, YrsNestType::MapType(map_ref.clone()), params.clone()); + assert_eq!(map_ref.len(&doc.transact()), 1); + params.nest_data_op_type = NestDataOpType::Delete; + ops_registry.operate_yrs_nest_type(&doc, YrsNestType::MapType(map_ref.clone()), params.clone()); + assert_eq!(map_ref.len(&doc.transact()), 0); + + params.nest_data_op_type = NestDataOpType::Clear; + ops_registry.operate_yrs_nest_type(&doc, YrsNestType::MapType(map_ref.clone()), params.clone()); + assert_eq!(map_ref.len(&doc.transact()), 0); + } + + #[test] + fn test_yrs_create_map_from_nest_type() { + let doc = Doc::new(); + let map_ref = doc.get_or_insert_map("test_map"); + let key = String::from("test_key"); + + let new_map_ref = yrs_create_map_from_nest_type( + &doc, + &mut YrsNestType::MapType(map_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(new_map_ref.is_some()); + + let map_ref = doc.get_or_insert_map("test_map"); + let new_map_ref = yrs_create_map_from_nest_type( + &doc, + &mut YrsNestType::MapType(map_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(new_map_ref.is_some()); + + let text_ref = doc.get_or_insert_text("test_text"); + let new_map_ref = yrs_create_map_from_nest_type( + &doc, + &mut YrsNestType::TextType(text_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(new_map_ref.is_some()); + } +} diff --git a/packages/common/y-octo/utils/src/doc_operation/yrs_op/mod.rs b/packages/common/y-octo/utils/src/doc_operation/yrs_op/mod.rs new file mode 100644 index 0000000000..f23d770841 --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/yrs_op/mod.rs @@ -0,0 +1,193 @@ +pub mod array; +pub mod map; +pub mod text; +pub mod xml_element; +pub mod xml_fragment; +pub mod xml_text; + +use std::collections::HashMap; + +use array::*; +use map::*; +use text::*; +use xml_element::*; +use xml_fragment::*; +use xml_text::*; +use yrs::{ + Array, ArrayPrelim, ArrayRef, Doc, GetString, Map, MapPrelim, MapRef, Text, TextPrelim, TextRef, + Transact, XmlFragment, XmlTextPrelim, XmlTextRef, +}; + +use super::*; + +type TestOp = fn(doc: &Doc, nest_input: &YrsNestType, params: CRDTParam) -> (); +type TestOps = phf::Map<&'static str, TestOp>; + +pub struct OpsRegistry<'a>(HashMap); + +impl Default for OpsRegistry<'_> { + fn default() -> Self { + OpsRegistry::new() + } +} + +impl OpsRegistry<'_> { + pub fn new() -> Self { + let mut map = HashMap::new(); + map.insert(CRDTNestType::Map, &MAP_OPS); + map.insert(CRDTNestType::Array, &ARRAY_OPS); + map.insert(CRDTNestType::Text, &TEXT_OPS); + map.insert(CRDTNestType::XMLElement, &XML_ELEMENT_OPS); + map.insert(CRDTNestType::XMLText, &XML_TEXT_OPS); + map.insert(CRDTNestType::XMLFragment, &XML_FRAGMENT_OPS); + + OpsRegistry(map) + } + + pub fn get_ops(&self, crdt_nest_type: &CRDTNestType) -> &TestOps { + match crdt_nest_type { + CRDTNestType::Map => self.0.get(&CRDTNestType::Map).unwrap(), + CRDTNestType::Array => self.0.get(&CRDTNestType::Array).unwrap(), + CRDTNestType::Text => self.0.get(&CRDTNestType::Text).unwrap(), + CRDTNestType::XMLElement => self.0.get(&CRDTNestType::XMLElement).unwrap(), + CRDTNestType::XMLFragment => self.0.get(&CRDTNestType::XMLFragment).unwrap(), + CRDTNestType::XMLText => self.0.get(&CRDTNestType::XMLText).unwrap(), + } + } + + pub fn get_ops_from_yrs_nest_type(&self, yrs_nest_type: &YrsNestType) -> &TestOps { + match yrs_nest_type { + YrsNestType::MapType(_) => self.get_ops(&CRDTNestType::Map), + YrsNestType::ArrayType(_) => self.get_ops(&CRDTNestType::Array), + YrsNestType::TextType(_) => self.get_ops(&CRDTNestType::Text), + YrsNestType::XMLElementType(_) => self.get_ops(&CRDTNestType::XMLElement), + YrsNestType::XMLTextType(_) => self.get_ops(&CRDTNestType::XMLText), + YrsNestType::XMLFragmentType(_) => self.get_ops(&CRDTNestType::XMLFragment), + } + } + + pub fn operate_yrs_nest_type( + &self, + doc: &yrs::Doc, + cur_crdt_nest_type: YrsNestType, + crdt_param: CRDTParam, + ) { + let ops = self.get_ops_from_yrs_nest_type(&cur_crdt_nest_type); + ops + .get(match &crdt_param.nest_data_op_type { + NestDataOpType::Insert => NEST_DATA_INSERT, + NestDataOpType::Delete => NEST_DATA_DELETE, + NestDataOpType::Clear => NEST_DATA_CLEAR, + }) + .unwrap()(doc, &cur_crdt_nest_type, crdt_param) + } +} + +pub fn yrs_create_nest_type_from_root( + doc: &yrs::Doc, + target_type: CRDTNestType, + key: &str, +) -> YrsNestType { + match target_type { + CRDTNestType::Array => YrsNestType::ArrayType(doc.get_or_insert_array(key)), + CRDTNestType::Map => YrsNestType::MapType(doc.get_or_insert_map(key)), + CRDTNestType::Text => YrsNestType::TextType(doc.get_or_insert_text(key)), + CRDTNestType::XMLElement => YrsNestType::XMLElementType(doc.get_or_insert_xml_fragment(key)), + CRDTNestType::XMLFragment => YrsNestType::XMLFragmentType(doc.get_or_insert_xml_fragment(key)), + CRDTNestType::XMLText => { + YrsNestType::XMLTextType((AsRef::::as_ref(&doc.get_or_insert_text(key))).clone()) + } + } +} + +pub fn gen_nest_type_from_root(doc: &mut Doc, crdt_param: &CRDTParam) -> Option { + match crdt_param.new_nest_type { + CRDTNestType::Array => Some(yrs_create_nest_type_from_root( + doc, + CRDTNestType::Array, + crdt_param.key.as_str(), + )), + CRDTNestType::Map => Some(yrs_create_nest_type_from_root( + doc, + CRDTNestType::Map, + crdt_param.key.as_str(), + )), + CRDTNestType::Text => Some(yrs_create_nest_type_from_root( + doc, + CRDTNestType::Text, + crdt_param.key.as_str(), + )), + CRDTNestType::XMLText => Some(yrs_create_nest_type_from_root( + doc, + CRDTNestType::XMLText, + crdt_param.key.as_str(), + )), + CRDTNestType::XMLElement => Some(yrs_create_nest_type_from_root( + doc, + CRDTNestType::XMLElement, + crdt_param.key.as_str(), + )), + CRDTNestType::XMLFragment => Some(yrs_create_nest_type_from_root( + doc, + CRDTNestType::XMLFragment, + crdt_param.key.as_str(), + )), + } +} + +pub fn gen_nest_type_from_nest_type( + doc: &mut Doc, + crdt_param: CRDTParam, + nest_type: &mut YrsNestType, +) -> Option { + match crdt_param.new_nest_type { + CRDTNestType::Array => { + yrs_create_array_from_nest_type(doc, nest_type, &crdt_param.insert_pos, crdt_param.key) + .map(YrsNestType::ArrayType) + } + CRDTNestType::Map => { + yrs_create_map_from_nest_type(doc, nest_type, &crdt_param.insert_pos, crdt_param.key) + .map(YrsNestType::MapType) + } + CRDTNestType::Text => { + yrs_create_text_from_nest_type(doc, nest_type, &crdt_param.insert_pos, crdt_param.key) + .map(YrsNestType::TextType) + } + _ => None, + } +} + +pub fn random_pick_num(len: u32, insert_pos: &InsertPos) -> u32 { + match insert_pos { + InsertPos::BEGIN => 0, + InsertPos::MID => len / 2, + InsertPos::END => len, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ops_registry_new() { + let ops_registry = OpsRegistry::new(); + assert_eq!(ops_registry.0.len(), 6); + } + + #[test] + fn test_ops_registry_get_ops() { + let ops_registry = OpsRegistry::new(); + let ops = ops_registry.get_ops(&CRDTNestType::Array); + assert!(!ops.is_empty()); + } + + #[test] + fn test_ops_registry_get_ops_from_yrs_nest_type() { + let doc = yrs::Doc::new(); + let array = doc.get_or_insert_array("array"); + let ops_registry = OpsRegistry::new(); + let ops = ops_registry.get_ops_from_yrs_nest_type(&YrsNestType::ArrayType(array)); + assert!(!ops.is_empty()); + } +} diff --git a/packages/common/y-octo/utils/src/doc_operation/yrs_op/text.rs b/packages/common/y-octo/utils/src/doc_operation/yrs_op/text.rs new file mode 100644 index 0000000000..47dddfecd5 --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/yrs_op/text.rs @@ -0,0 +1,180 @@ +use phf::phf_map; + +use super::*; + +fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let text = match nest_input { + YrsNestType::TextType(text) => text, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + + let str = text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + let index = random_pick_num(len, ¶ms.insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + text.insert(&mut trx, byte_start_offset as u32, ¶ms.value); +} + +fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let text = match nest_input { + YrsNestType::TextType(text) => text, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + + let str = text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + if len < 1 { + return; + } + let index = random_pick_num(len - 1, ¶ms.insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + let char_byte_len = str.chars().nth(index).unwrap().len_utf8(); + text.remove_range(&mut trx, byte_start_offset as u32, char_byte_len as u32); +} + +fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { + let text = match nest_input { + YrsNestType::TextType(text) => text, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + + let str = text.get_string(&trx); + let byte_len = str.chars().fold(0, |acc, ch| acc + ch.len_utf8()); + + text.remove_range(&mut trx, 0, byte_len as u32); +} + +pub static TEXT_OPS: TestOps = phf_map! { + "insert" => insert_op, + "delete" => remove_op, + "clear" => clear_op, +}; + +pub fn yrs_create_text_from_nest_type( + doc: &yrs::Doc, + current: &mut YrsNestType, + insert_pos: &InsertPos, + key: String, +) -> Option { + let cal_index_closure = |len: u32| -> u32 { random_pick_num(len, insert_pos) }; + let mut trx = doc.transact_mut(); + let text_prelim = TextPrelim::new(""); + match current { + YrsNestType::ArrayType(array) => { + let index = cal_index_closure(array.len(&trx)); + Some(array.insert(&mut trx, index, text_prelim)) + } + YrsNestType::MapType(map) => Some(map.insert(&mut trx, key, text_prelim)), + YrsNestType::TextType(text) => { + let str = text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + let index = random_pick_num(len, insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + Some(text.insert_embed(&mut trx, byte_start_offset as u32, text_prelim)) + } + YrsNestType::XMLTextType(xml_text) => { + let str = xml_text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + let index = random_pick_num(len, insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + Some(xml_text.insert_embed(&mut trx, byte_start_offset as u32, text_prelim)) + } + _ => None, + } +} +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gen_array_ref_ops() { + let doc = Doc::new(); + let text_ref = doc.get_or_insert_text("test_text"); + + let ops_registry = OpsRegistry::new(); + + let mut params = CRDTParam { + op_type: OpType::CreateCRDTNestType, + new_nest_type: CRDTNestType::Text, + manipulate_source: ManipulateSource::NewNestTypeFromYDocRoot, + insert_pos: InsertPos::BEGIN, + key: String::from("test_key"), + value: String::from("test_value"), + nest_data_op_type: NestDataOpType::Insert, + }; + + ops_registry.operate_yrs_nest_type( + &doc, + YrsNestType::TextType(text_ref.clone()), + params.clone(), + ); + assert_eq!(text_ref.len(&doc.transact()), 10); + params.nest_data_op_type = NestDataOpType::Delete; + ops_registry.operate_yrs_nest_type( + &doc, + YrsNestType::TextType(text_ref.clone()), + params.clone(), + ); + assert_eq!(text_ref.len(&doc.transact()), 9); + + params.nest_data_op_type = NestDataOpType::Clear; + ops_registry.operate_yrs_nest_type( + &doc, + YrsNestType::TextType(text_ref.clone()), + params.clone(), + ); + assert_eq!(text_ref.len(&doc.transact()), 0); + } + + #[test] + fn test_yrs_create_array_from_nest_type() { + let doc = Doc::new(); + let array_ref = doc.get_or_insert_array("test_array"); + let key = String::from("test_key"); + + let next_text_ref = yrs_create_text_from_nest_type( + &doc, + &mut YrsNestType::ArrayType(array_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(next_text_ref.is_some()); + + let map_ref = doc.get_or_insert_map("test_map"); + let next_text_ref = yrs_create_text_from_nest_type( + &doc, + &mut YrsNestType::MapType(map_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(next_text_ref.is_some()); + + let text_ref = doc.get_or_insert_text("test_text"); + let next_text_ref = yrs_create_text_from_nest_type( + &doc, + &mut YrsNestType::TextType(text_ref.clone()), + &InsertPos::BEGIN, + key.clone(), + ); + assert!(next_text_ref.is_some()); + } +} diff --git a/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_element.rs b/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_element.rs new file mode 100644 index 0000000000..5cd4b95fa9 --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_element.rs @@ -0,0 +1,45 @@ +use phf::phf_map; + +use super::*; + +fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let xml_element = match nest_input { + YrsNestType::XMLElementType(xml_element) => xml_element, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = xml_element.len(&trx); + let index = random_pick_num(len, ¶ms.insert_pos); + xml_element.insert(&mut trx, index, XmlTextPrelim::new(params.value)); +} + +fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let xml_element = match nest_input { + YrsNestType::XMLElementType(xml_element) => xml_element, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = xml_element.len(&trx); + if len >= 1 { + let index = random_pick_num(len - 1, ¶ms.insert_pos); + xml_element.remove_range(&mut trx, index, 1); + } +} + +fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { + let xml_element = match nest_input { + YrsNestType::XMLElementType(xml_element) => xml_element, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = xml_element.len(&trx); + for _ in 0..len { + xml_element.remove_range(&mut trx, 0, 1); + } +} + +pub static XML_ELEMENT_OPS: TestOps = phf_map! { + "insert" => insert_op, + "delete" => remove_op, + "clear" => clear_op, +}; diff --git a/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_fragment.rs b/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_fragment.rs new file mode 100644 index 0000000000..0c10bbf81d --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_fragment.rs @@ -0,0 +1,45 @@ +use phf::phf_map; + +use super::*; + +fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let xml_fragment = match nest_input { + YrsNestType::XMLFragmentType(xml_fragment) => xml_fragment, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = xml_fragment.len(&trx); + let index = random_pick_num(len, ¶ms.insert_pos); + xml_fragment.insert(&mut trx, index, XmlTextPrelim::new(params.value)); +} + +fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let xml_fragment = match nest_input { + YrsNestType::XMLFragmentType(xml_fragment) => xml_fragment, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = xml_fragment.len(&trx); + if len >= 1 { + let index = random_pick_num(len - 1, ¶ms.insert_pos); + xml_fragment.remove_range(&mut trx, index, 1); + } +} + +fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { + let xml_fragment = match nest_input { + YrsNestType::XMLFragmentType(xml_fragment) => xml_fragment, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + let len = xml_fragment.len(&trx); + for _ in 0..len { + xml_fragment.remove_range(&mut trx, 0, 1); + } +} + +pub static XML_FRAGMENT_OPS: TestOps = phf_map! { + "insert" => insert_op, + "delete" => remove_op, + "clear" => clear_op, +}; diff --git a/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_text.rs b/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_text.rs new file mode 100644 index 0000000000..4f9b65b1c4 --- /dev/null +++ b/packages/common/y-octo/utils/src/doc_operation/yrs_op/xml_text.rs @@ -0,0 +1,62 @@ +use phf::phf_map; + +use super::*; + +fn insert_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let xml_text = match nest_input { + YrsNestType::XMLTextType(xml_text) => xml_text, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + + let str = xml_text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + let index = random_pick_num(len, ¶ms.insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + xml_text.insert(&mut trx, byte_start_offset as u32, ¶ms.value); +} + +fn remove_op(doc: &yrs::Doc, nest_input: &YrsNestType, params: CRDTParam) { + let xml_text = match nest_input { + YrsNestType::XMLTextType(xml_text) => xml_text, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + + let str = xml_text.get_string(&trx); + let len = str.chars().fold(0, |acc, _| acc + 1); + if len < 1 { + return; + } + let index = random_pick_num(len - 1, ¶ms.insert_pos) as usize; + let byte_start_offset = str + .chars() + .take(index) + .fold(0, |acc, ch| acc + ch.len_utf8()); + + let char_byte_len = str.chars().nth(index).unwrap().len_utf8(); + xml_text.remove_range(&mut trx, byte_start_offset as u32, char_byte_len as u32); +} + +fn clear_op(doc: &yrs::Doc, nest_input: &YrsNestType, _params: CRDTParam) { + let xml_text = match nest_input { + YrsNestType::XMLTextType(xml_text) => xml_text, + _ => unreachable!(), + }; + let mut trx = doc.transact_mut(); + + let str = xml_text.get_string(&trx); + let byte_len = str.chars().fold(0, |acc, ch| acc + ch.len_utf8()); + + xml_text.remove_range(&mut trx, 0, byte_len as u32); +} + +pub static XML_TEXT_OPS: TestOps = phf_map! { + "insert" => insert_op, + "delete" => remove_op, + "clear" => clear_op, +}; diff --git a/packages/common/y-octo/utils/src/lib.rs b/packages/common/y-octo/utils/src/lib.rs new file mode 100644 index 0000000000..ad9f827328 --- /dev/null +++ b/packages/common/y-octo/utils/src/lib.rs @@ -0,0 +1,7 @@ +mod doc; + +#[cfg(feature = "fuzz")] +pub mod doc_operation; + +#[cfg(feature = "fuzz")] +pub use doc_operation::*; diff --git a/tools/commitlint/.commitlintrc.json b/tools/commitlint/.commitlintrc.json index ac82f3ece6..245a409423 100644 --- a/tools/commitlint/.commitlintrc.json +++ b/tools/commitlint/.commitlintrc.json @@ -26,7 +26,8 @@ "nbstore", "infra", "editor", - "tools" + "tools", + "y-octo" ] ] } diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 97d30ed426..5962ce466d 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -979,6 +979,11 @@ export const PackageList = [ 'blocksuite/affine/all', ], }, + { + location: 'packages/common/y-octo/node', + name: '@y-octo/node', + workspaceDependencies: [], + }, { location: 'packages/frontend/admin', name: '@affine/admin', @@ -1291,6 +1296,7 @@ export type PackageName = | '@affine/graphql' | '@toeverything/infra' | '@affine/nbstore' + | '@y-octo/node' | '@affine/admin' | '@affine/android' | '@affine/electron' diff --git a/tsconfig.json b/tsconfig.json index ebc110e570..303b13c148 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -118,6 +118,7 @@ { "path": "./packages/common/graphql" }, { "path": "./packages/common/infra" }, { "path": "./packages/common/nbstore" }, + { "path": "./packages/common/y-octo/node" }, { "path": "./packages/frontend/admin" }, { "path": "./packages/frontend/apps/android" }, { "path": "./packages/frontend/apps/electron" }, diff --git a/yarn.lock b/yarn.lock index 0199cfb740..06f0cd2080 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15103,7 +15103,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*, @types/node@npm:>=10.0.0, @types/node@npm:>=13.7.0, @types/node@npm:>=18.0.0, @types/node@npm:>=8.1.0, @types/node@npm:^22.0.0, @types/node@npm:^22.7.7": +"@types/node@npm:*, @types/node@npm:>=10.0.0, @types/node@npm:>=13.7.0, @types/node@npm:>=18.0.0, @types/node@npm:>=8.1.0, @types/node@npm:^22.0.0, @types/node@npm:^22.14.1, @types/node@npm:^22.7.7": version: 22.14.1 resolution: "@types/node@npm:22.14.1" dependencies: @@ -15189,6 +15189,16 @@ __metadata: languageName: node linkType: hard +"@types/prompts@npm:^2.4.9": + version: 2.4.9 + resolution: "@types/prompts@npm:2.4.9" + dependencies: + "@types/node": "npm:*" + kleur: "npm:^3.0.3" + checksum: 10/69b8372f4c790b45fea16a46ff8d1bcc71b14579481776b67bd6263637118a7ecb1f12e1311506c29fadc81bf618dc64f1a91f903cfd5be67a0455a227b3e462 + languageName: node + linkType: hard + "@types/qs@npm:*": version: 6.9.18 resolution: "@types/qs@npm:6.9.18" @@ -16264,6 +16274,21 @@ __metadata: languageName: node linkType: hard +"@y-octo/node@workspace:packages/common/y-octo/node": + version: 0.0.0-use.local + resolution: "@y-octo/node@workspace:packages/common/y-octo/node" + dependencies: + "@napi-rs/cli": "npm:3.0.0-alpha.77" + "@types/node": "npm:^22.14.1" + "@types/prompts": "npm:^2.4.9" + c8: "npm:^10.1.3" + prompts: "npm:^2.4.2" + ts-node: "npm:^10.9.2" + typescript: "npm:^5.8.3" + yjs: "npm:^13.6.24" + languageName: unknown + linkType: soft + "@zeit/schemas@npm:2.36.0": version: 2.36.0 resolution: "@zeit/schemas@npm:2.36.0" @@ -32909,7 +32934,7 @@ __metadata: languageName: node linkType: hard -"typescript@npm:^5, typescript@npm:^5.3.3, typescript@npm:^5.4.3, typescript@npm:^5.5.4, typescript@npm:^5.7.2": +"typescript@npm:^5, typescript@npm:^5.3.3, typescript@npm:^5.4.3, typescript@npm:^5.5.4, typescript@npm:^5.7.2, typescript@npm:^5.8.3": version: 5.8.3 resolution: "typescript@npm:5.8.3" bin: @@ -32919,7 +32944,7 @@ __metadata: languageName: node linkType: hard -"typescript@patch:typescript@npm%3A^5#optional!builtin, typescript@patch:typescript@npm%3A^5.3.3#optional!builtin, typescript@patch:typescript@npm%3A^5.4.3#optional!builtin, typescript@patch:typescript@npm%3A^5.5.4#optional!builtin, typescript@patch:typescript@npm%3A^5.7.2#optional!builtin": +"typescript@patch:typescript@npm%3A^5#optional!builtin, typescript@patch:typescript@npm%3A^5.3.3#optional!builtin, typescript@patch:typescript@npm%3A^5.4.3#optional!builtin, typescript@patch:typescript@npm%3A^5.5.4#optional!builtin, typescript@patch:typescript@npm%3A^5.7.2#optional!builtin, typescript@patch:typescript@npm%3A^5.8.3#optional!builtin": version: 5.8.3 resolution: "typescript@patch:typescript@npm%3A5.8.3#optional!builtin::version=5.8.3&hash=5786d5" bin: