refactor(infra): directory structure (#4615)

packages/backend/storage/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "affine_storage"
+version = "1.0.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+chrono = "0.4"
+jwst-codec = { git = "https://github.com/toeverything/OctoBase.git", rev = "ad51b2c" }
+jwst-core = { git = "https://github.com/toeverything/OctoBase.git", rev = "ad51b2c" }
+jwst-storage = { git = "https://github.com/toeverything/OctoBase.git", rev = "ad51b2c" }
+napi = { version = "2", default-features = false, features = [
+  "napi5",
+  "async",
+] }
+napi-derive = { version = "2", features = ["type-def"] }
+rand = "0.8"
+sha3 = "0.10"
+
+[dev-dependencies]
+tokio = "1"
+
+[build-dependencies]
+napi-build = "2"

packages/backend/storage/__tests__/storage.spec.js

@@ -0,0 +1,165 @@
+import assert from 'node:assert';
+import { beforeEach, describe, test } from 'node:test';
+
+import { encoding } from 'lib0';
+import { applyUpdate, Doc } from 'yjs';
+
+import { Storage } from '../index.js';
+
+// update binary by y.doc.text('content').insert('hello world')
+// prettier-ignore
+let init = Buffer.from([
+  1,
+  1,
+  160,
+  238,
+  169,
+  240,
+  10,
+  0,
+  4,
+  1,
+  7,
+  99,
+  111,
+  110,
+  116,
+  101,
+  110,
+  116,
+  11,
+  104,
+  101,
+  108,
+  108,
+  111,
+  32,
+  119,
+  111,
+  114,
+  108,
+  100,
+  0])
+describe('Test jwst storage binding', () => {
+  /** @type { Storage } */
+  let storage;
+  beforeEach(async () => {
+    storage = await Storage.connect('sqlite::memory:', true);
+  });
+
+  test('should be able to create workspace', async () => {
+    const workspace = await storage.createWorkspace('test-workspace', init);
+
+    assert(workspace.id === 'test-workspace');
+    assert.deepEqual(init, await storage.load(workspace.doc.guid));
+  });
+
+  test('should not create workspace with same id', async () => {
+    await storage.createWorkspace('test-workspace', init);
+    await assert.rejects(
+      storage.createWorkspace('test-workspace', init),
+      /Workspace [\w-]+ already exists/
+    );
+  });
+
+  test('should be able to delete workspace', async () => {
+    const workspace = await storage.createWorkspace('test-workspace', init);
+
+    await storage.deleteWorkspace(workspace.id);
+
+    await assert.rejects(
+      storage.load(workspace.doc.guid),
+      /Doc [\w-]+ not exists/
+    );
+  });
+
+  test('should be able to sync update', async () => {
+    const workspace = await storage.createWorkspace('test-workspace', init);
+
+    const update = await storage.load(workspace.doc.guid);
+    assert(update !== null);
+
+    const doc = new Doc();
+    applyUpdate(doc, update);
+
+    let text = doc.getText('content');
+    assert.equal(text.toJSON(), 'hello world');
+
+    const updates = [];
+    doc.on('update', async (/** @type { UInt8Array } */ update) => {
+      updates.push(Buffer.from(update));
+    });
+
+    text.insert(5, ' my');
+    text.insert(14, '!');
+
+    for (const update of updates) {
+      await storage.sync(workspace.id, workspace.doc.guid, update);
+    }
+
+    const update2 = await storage.load(workspace.doc.guid);
+    const doc2 = new Doc();
+    applyUpdate(doc2, update2);
+
+    text = doc2.getText('content');
+    assert.equal(text.toJSON(), 'hello my world!');
+  });
+
+  test('should be able to sync update with guid encoded', async () => {
+    const workspace = await storage.createWorkspace('test-workspace', init);
+
+    const update = await storage.load(workspace.doc.guid);
+    assert(update !== null);
+
+    const doc = new Doc();
+    applyUpdate(doc, update);
+
+    let text = doc.getText('content');
+    assert.equal(text.toJSON(), 'hello world');
+
+    const updates = [];
+    doc.on('update', async (/** @type { UInt8Array } */ update) => {
+      const prefix = encoding.encode(encoder => {
+        encoding.writeVarString(encoder, workspace.doc.guid);
+      });
+
+      updates.push(Buffer.concat([prefix, update]));
+    });
+
+    text.insert(5, ' my');
+    text.insert(14, '!');
+
+    for (const update of updates) {
+      await storage.syncWithGuid(workspace.id, update);
+    }
+
+    const update2 = await storage.load(workspace.doc.guid);
+    const doc2 = new Doc();
+    applyUpdate(doc2, update2);
+
+    text = doc2.getText('content');
+    assert.equal(text.toJSON(), 'hello my world!');
+  });
+
+  test('should be able to store blob', async () => {
+    let workspace = await storage.createWorkspace('test-workspace');
+    await storage.sync(workspace.id, workspace.doc.guid, init);
+    const blobId = await storage.uploadBlob(workspace.id, Buffer.from([1]));
+
+    assert(blobId !== null);
+
+    let list = await storage.listBlobs(workspace.id);
+    assert.deepEqual(list, [blobId]);
+
+    let blob = await storage.getBlob(workspace.id, blobId);
+    assert.deepEqual(blob.data, Buffer.from([1]));
+    assert.strictEqual(blob.size, 1);
+    assert.equal(blob.contentType, 'application/octet-stream');
+
+    await storage.uploadBlob(workspace.id, Buffer.from([1, 2, 3, 4, 5]));
+
+    const spaceTaken = await storage.blobsSize(workspace.id);
+
+    assert.equal(spaceTaken, 6);
+  });
+});

packages/backend/storage/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+  napi_build::setup();
+}

packages/backend/storage/index.d.ts

@@ -0,0 +1,42 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+export function verifyChallengeResponse(
+  response: string,
+  bits: number,
+  resource: string
+): Promise<boolean>;
+export function mintChallengeResponse(
+  resource: string,
+  bits?: number | undefined | null
+): Promise<string>;
+export interface Blob {
+  contentType: string;
+  lastModified: string;
+  size: number;
+  data: Buffer;
+}
+/**
+ * Merge updates in form like `Y.applyUpdate(doc, update)` way and return the
+ * result binary.
+ */
+export function mergeUpdatesInApplyWay(updates: Array<Buffer>): Buffer;
+export class Storage {
+  /** Create a storage instance and establish connection to persist store. */
+  static connect(
+    database: string,
+    debugOnlyAutoMigrate?: boolean | undefined | null
+  ): Promise<Storage>;
+  /** List all blobs in a workspace. */
+  listBlobs(workspaceId?: string | undefined | null): Promise<Array<string>>;
+  /** Fetch a workspace blob. */
+  getBlob(workspaceId: string, name: string): Promise<Blob | null>;
+  /** Upload a blob into workspace storage. */
+  uploadBlob(workspaceId: string, blob: Buffer): Promise<string>;
+  /** Delete a blob from workspace storage. */
+  deleteBlob(workspaceId: string, hash: string): Promise<boolean>;
+  /** Workspace size taken by blobs. */
+  blobsSize(workspaces: Array<string>): Promise<number>;
+}

packages/backend/storage/index.js

@@ -0,0 +1,11 @@
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+/** @type {import('.')} */
+const binding = require('./storage.node');
+
+export const Storage = binding.Storage;
+export const mergeUpdatesInApplyWay = binding.mergeUpdatesInApplyWay;
+export const verifyChallengeResponse = binding.verifyChallengeResponse;
+export const mintChallengeResponse = binding.mintChallengeResponse;

packages/backend/storage/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@affine/storage",
+  "version": "0.10.0-canary.1",
+  "engines": {
+    "node": ">= 10.16.0 < 11 || >= 11.8.0"
+  },
+  "type": "module",
+  "main": "./index.js",
+  "module": "./index.js",
+  "types": "index.d.ts",
+  "exports": {
+    ".": {
+      "require": "./storage.node",
+      "import": "./index.js",
+      "types": "./index.d.ts"
+    }
+  },
+  "napi": {
+    "name": "storage",
+    "targets": [
+      "aarch64-apple-darwin",
+      "aarch64-unknown-linux-gnu",
+      "aarch64-pc-windows-msvc",
+      "x86_64-apple-darwin",
+      "x86_64-pc-windows-msvc",
+      "x86_64-unknown-linux-gnu",
+      "universal-apple-darwin"
+    ]
+  },
+  "scripts": {
+    "test": "node --test ./__tests__/**/*.spec.js",
+    "build": "napi build --release --strip",
+    "build:debug": "napi build",
+    "prepublishOnly": "napi prepublish -t npm",
+    "artifacts": "napi artifacts",
+    "version": "napi version"
+  },
+  "devDependencies": {
+    "@napi-rs/cli": "^2.16.3",
+    "lib0": "^0.2.87",
+    "nx": "^16.10.0",
+    "nx-cloud": "^16.5.2",
+    "yjs": "^13.6.8"
+  }
+}

packages/backend/storage/project.json

@@ -0,0 +1,23 @@
+{
+  "name": "@affine/storage",
+  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+  "projectType": "application",
+  "root": "packages/backend/storage",
+  "sourceRoot": "packages/backend/storage/src",
+  "targets": {
+    "build": {
+      "executor": "nx:run-script",
+      "dependsOn": ["^build"],
+      "options": {
+        "script": "build"
+      },
+      "inputs": [
+        { "runtime": "rustc --version" },
+        { "runtime": "node -v" },
+        { "runtime": "clang --version" },
+        { "runtime": "cargo tree" }
+      ],
+      "outputs": ["{projectRoot}/*.node", "{workspaceRoot}/*.node"]
+    }
+  }
+}

packages/backend/storage/src/hashcash.rs

@@ -0,0 +1 @@
+../../../frontend/native/src/hashcash.rs

packages/backend/storage/src/lib.rs

@@ -0,0 +1,169 @@
+#![deny(clippy::all)]
+
+pub mod hashcash;
+
+use std::{
+  collections::HashMap,
+  fmt::{Debug, Display},
+  path::PathBuf,
+};
+
+use jwst_codec::Doc;
+use jwst_core::BlobStorage;
+use jwst_storage::{BlobStorageType, JwstStorage, JwstStorageError};
+use napi::{bindgen_prelude::*, Error, Result, Status};
+
+#[macro_use]
+extern crate napi_derive;
+
+fn map_err_inner<T, E: Display + Debug>(v: std::result::Result<T, E>, status: Status) -> Result<T> {
+  match v {
+    Ok(val) => Ok(val),
+    Err(e) => {
+      dbg!(&e);
+      Err(Error::new(status, e.to_string()))
+    }
+  }
+}
+
+macro_rules! map_err {
+  ($val: expr) => {
+    map_err_inner($val, Status::GenericFailure)
+  };
+  ($val: expr, $stauts: ident) => {
+    map_err_inner($val, $stauts)
+  };
+}
+
+macro_rules! napi_wrap {
+    ($( ($name: ident, $target: ident) ),*) => {
+        $(
+            #[napi]
+            pub struct $name($target);
+
+            impl std::ops::Deref for $name {
+                type Target = $target;
+
+                fn deref(&self) -> &Self::Target {
+                    &self.0
+                }
+            }
+
+            impl From<$target> for $name {
+                fn from(t: $target) -> Self {
+                    Self(t)
+                }
+            }
+        )*
+    };
+}
+
+napi_wrap!((Storage, JwstStorage));
+
+#[napi(object)]
+pub struct Blob {
+  pub content_type: String,
+  pub last_modified: String,
+  pub size: i64,
+  pub data: Buffer,
+}
+
+#[napi]
+impl Storage {
+  /// Create a storage instance and establish connection to persist store.
+  #[napi]
+  pub async fn connect(database: String, debug_only_auto_migrate: Option<bool>) -> Result<Storage> {
+    let inner = match if cfg!(debug_assertions) && debug_only_auto_migrate.unwrap_or(false) {
+      JwstStorage::new_with_migration(&database, BlobStorageType::DB).await
+    } else {
+      JwstStorage::new(&database, BlobStorageType::DB).await
+    } {
+      Ok(storage) => storage,
+      Err(JwstStorageError::Db(e)) => {
+        return Err(Error::new(
+          Status::GenericFailure,
+          format!("failed to connect to database: {}", e),
+        ));
+      }
+      Err(e) => return Err(Error::new(Status::GenericFailure, e.to_string())),
+    };
+
+    Ok(inner.into())
+  }
+
+  /// List all blobs in a workspace.
+  #[napi]
+  pub async fn list_blobs(&self, workspace_id: Option<String>) -> Result<Vec<String>> {
+    map_err!(self.blobs().list_blobs(workspace_id).await)
+  }
+
+  /// Fetch a workspace blob.
+  #[napi]
+  pub async fn get_blob(&self, workspace_id: String, name: String) -> Result<Option<Blob>> {
+    let (id, params) = {
+      let path = PathBuf::from(name.clone());
+      let ext = path
+        .extension()
+        .and_then(|s| s.to_str().map(|s| s.to_string()));
+      let id = path
+        .file_stem()
+        .and_then(|s| s.to_str().map(|s| s.to_string()))
+        .unwrap_or(name);
+
+      (id, ext.map(|ext| HashMap::from([("format".into(), ext)])))
+    };
+
+    let Ok(meta) = self
+      .blobs()
+      .get_metadata(Some(workspace_id.clone()), id.clone(), params.clone())
+      .await
+    else {
+      return Ok(None);
+    };
+
+    let Ok(file) = self.blobs().get_blob(Some(workspace_id), id, params).await else {
+      return Ok(None);
+    };
+
+    Ok(Some(Blob {
+      content_type: meta.content_type,
+      last_modified: format!("{:?}", meta.last_modified),
+      size: meta.size,
+      data: file.into(),
+    }))
+  }
+
+  /// Upload a blob into workspace storage.
+  #[napi]
+  pub async fn upload_blob(&self, workspace_id: String, blob: Buffer) -> Result<String> {
+    // TODO: can optimize, avoid copy
+    let blob = blob.as_ref().to_vec();
+    map_err!(self.blobs().put_blob(Some(workspace_id), blob).await)
+  }
+
+  /// Delete a blob from workspace storage.
+  #[napi]
+  pub async fn delete_blob(&self, workspace_id: String, hash: String) -> Result<bool> {
+    map_err!(self.blobs().delete_blob(Some(workspace_id), hash).await)
+  }
+
+  /// Workspace size taken by blobs.
+  #[napi]
+  pub async fn blobs_size(&self, workspaces: Vec<String>) -> Result<i64> {
+    map_err!(self.blobs().get_blobs_size(workspaces).await)
+  }
+}
+
+/// Merge updates in form like `Y.applyUpdate(doc, update)` way and return the
+/// result binary.
+#[napi(catch_unwind)]
+pub fn merge_updates_in_apply_way(updates: Vec<Buffer>) -> Result<Buffer> {
+  let mut doc = Doc::default();
+  for update in updates {
+    map_err!(doc.apply_update_from_binary(update.as_ref().to_vec()))?;
+  }
+
+  let buf = map_err!(doc.encode_update_v1())?;
+
+  Ok(buf.into())
+}

packages/backend/storage/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": false,
+    "outDir": "lib",
+    "composite": true
+  },
+  "include": ["index.d.ts"]
+}