milestone: publish alpha version (#637)

- document folder
- full-text search
- blob storage
- basic edgeless support

Co-authored-by: tzhangchi <terry.zhangchi@outlook.com>
Co-authored-by: QiShaoXuan <qishaoxuan777@gmail.com>
Co-authored-by: DiamondThree <diamond.shx@gmail.com>
Co-authored-by: MingLiang Wang <mingliangwang0o0@gmail.com>
Co-authored-by: JimmFly <yangjinfei001@gmail.com>
Co-authored-by: Yifeng Wang <doodlewind@toeverything.info>
Co-authored-by: Himself65 <himself65@outlook.com>
Co-authored-by: lawvs <18554747+lawvs@users.noreply.github.com>
Co-authored-by: Qi <474021214@qq.com>

packages/data-services/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+/test-results/
+/playwright-report/
+/playwright/.cache/

packages/data-services/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@affine/data-services",
+  "version": "0.3.0",
+  "description": "",
+  "type": "module",
+  "main": "dist/src/index.js",
+  "types": "dist/src/index.d.ts",
+  "exports": {
+    "./src/*": "./dist/src/*.js",
+    ".": "./dist/src/index.js"
+  },
+  "scripts": {
+    "build": "tsc --project ./tsconfig.json",
+    "test": "playwright test"
+  },
+  "keywords": [],
+  "author": "",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/toeverything/AFFiNE.git"
+  },
+  "devDependencies": {
+    "@playwright/test": "^1.29.1",
+    "typescript": "^4.8.4"
+  },
+  "dependencies": {
+    "encoding": "^0.1.13",
+    "firebase": "^9.13.0",
+    "ky": "^0.33.0",
+    "lib0": "^0.2.58",
+    "swr": "^2.0.0",
+    "y-protocols": "^1.0.5"
+  }
+}

packages/data-services/playwright.config.ts

@@ -0,0 +1,25 @@
+import type { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  testDir: './tests',
+  timeout: 30 * 1000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 5000,
+  },
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  use: {
+    actionTimeout: 0,
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+};
+
+export default config;

packages/data-services/src/auth.ts

@@ -0,0 +1,44 @@
+import { initializeApp } from 'firebase/app';
+import {
+  getAuth,
+  createUserWithEmailAndPassword,
+  signInWithEmailAndPassword,
+  GoogleAuthProvider,
+  signInWithPopup,
+} from 'firebase/auth';
+import type { User } from 'firebase/auth';
+import { token } from './request';
+
+/**
+ * firebaseConfig reference: https://firebase.google.com/docs/web/setup#add_firebase_to_your_app
+ */
+const app = initializeApp({
+  apiKey: process.env.NEXT_PUBLIC_FIREBASE_API_KEY,
+  authDomain: process.env.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN,
+  projectId: process.env.NEXT_PUBLIC_FIREBASE_PROJECT_ID,
+  storageBucket: process.env.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET,
+  messagingSenderId: process.env.NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID,
+  appId: process.env.NEXT_PUBLIC_FIREBASE_APP_ID,
+  measurementId: process.env.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID,
+});
+
+export const firebaseAuth = getAuth(app);
+
+const signUp = (email: string, password: string) => {
+  return createUserWithEmailAndPassword(firebaseAuth, email, password);
+};
+
+const signIn = (email: string, password: string) => {
+  return signInWithEmailAndPassword(firebaseAuth, email, password);
+};
+
+const googleAuthProvider = new GoogleAuthProvider();
+export const signInWithGoogle = async () => {
+  const user = await signInWithPopup(firebaseAuth, googleAuthProvider);
+  const idToken = await user.user.getIdToken();
+  await token.initToken(idToken);
+};
+
+export const onAuthStateChanged = (callback: (user: User | null) => void) => {
+  firebaseAuth.onAuthStateChanged(callback);
+};

packages/data-services/src/data-center.ts

@@ -0,0 +1,19 @@
+class DataCenter {
+  static async init() {
+    return new DataCenter();
+  }
+
+  private constructor() {
+    // TODO
+  }
+}
+
+let _dataCenterInstance: Promise<DataCenter>;
+
+export const getDataCenter = () => {
+  if (!_dataCenterInstance) {
+    _dataCenterInstance = DataCenter.init();
+  }
+
+  return _dataCenterInstance;
+};

packages/data-services/src/index.ts

@@ -0,0 +1,6 @@
+export { signInWithGoogle, onAuthStateChanged } from './auth';
+export * from './request';
+export * from './sdks';
+export * from './websocket';
+
+export { getDataCenter } from './data-center';

packages/data-services/src/request/events.ts

@@ -0,0 +1,28 @@
+import { AccessTokenMessage } from './token';
+
+export type Callback = (user: AccessTokenMessage | null) => void;
+
+export class AuthorizationEvent {
+  private callbacks: Callback[] = [];
+  private lastState: AccessTokenMessage | null = null;
+
+  /**
+   * Callback will execute when call this function.
+   */
+  onChange(callback: Callback) {
+    this.callbacks.push(callback);
+    callback(this.lastState);
+  }
+
+  triggerChange(user: AccessTokenMessage | null) {
+    this.lastState = user;
+    this.callbacks.forEach(callback => callback(user));
+  }
+
+  removeCallback(callback: Callback) {
+    const index = this.callbacks.indexOf(callback);
+    if (index > -1) {
+      this.callbacks.splice(index, 1);
+    }
+  }
+}

packages/data-services/src/request/index.ts

@@ -0,0 +1,45 @@
+import ky from 'ky';
+import { token } from './token';
+
+export const bareClient = ky.extend({
+  retry: 1,
+  hooks: {
+    // afterResponse: [
+    //   async (_request, _options, response) => {
+    //     if (response.status === 200) {
+    //       const data = await response.json();
+    //       if (data.error) {
+    //         return new Response(data.error.message, {
+    //           status: data.error.code,
+    //         });
+    //       }
+    //     }
+    //     return response;
+    //   },
+    // ],
+  },
+});
+export const client = bareClient.extend({
+  hooks: {
+    beforeRequest: [
+      async request => {
+        if (token.isLogin) {
+          if (token.isExpired) await token.refreshToken();
+          request.headers.set('Authorization', token.token);
+        } else {
+          return new Response('Unauthorized', { status: 401 });
+        }
+      },
+    ],
+
+    beforeRetry: [
+      async ({ request }) => {
+        await token.refreshToken();
+        request.headers.set('Authorization', token.token);
+      },
+    ],
+  },
+});
+
+export type { AccessTokenMessage } from './token';
+export { token };

packages/data-services/src/request/token.ts

@@ -0,0 +1,138 @@
+import { bareClient } from '.';
+import { AuthorizationEvent, Callback } from './events';
+
+export interface AccessTokenMessage {
+  create_at: number;
+  exp: number;
+  email: string;
+  id: string;
+  name: string;
+  avatar_url: string;
+}
+
+const TOKEN_KEY = 'affine_token';
+
+type LoginParams = {
+  type: 'Google' | 'Refresh';
+  token: string;
+};
+
+type LoginResponse = {
+  // JWT, expires in a very short time
+  token: string;
+  // Refresh token
+  refresh: string;
+};
+
+const login = (params: LoginParams): Promise<LoginResponse> =>
+  bareClient.post('/api/user/token', { json: params }).json();
+
+function b64DecodeUnicode(str: string) {
+  // Going backwards: from byte stream, to percent-encoding, to original string.
+  return decodeURIComponent(
+    window
+      .atob(str)
+      .split('')
+      .map(function (c) {
+        return '%' + ('00' + c.charCodeAt(0).toString(16)).slice(-2);
+      })
+      .join('')
+  );
+}
+
+function getRefreshToken() {
+  try {
+    return localStorage.getItem(TOKEN_KEY) || '';
+  } catch (_) {
+    return '';
+  }
+}
+function setRefreshToken(token: string) {
+  try {
+    localStorage.setItem(TOKEN_KEY, token);
+  } catch (_) {}
+}
+
+class Token {
+  private readonly _event: AuthorizationEvent;
+  private _accessToken: string;
+  private _refreshToken: string;
+
+  private _user: AccessTokenMessage | null;
+  private _padding?: Promise<LoginResponse>;
+
+  constructor(refreshToken?: string) {
+    this._accessToken = '';
+    this._refreshToken = refreshToken || getRefreshToken();
+    this._event = new AuthorizationEvent();
+
+    this._user = Token.parse(this._accessToken);
+    this._event.triggerChange(this._user);
+  }
+
+  private _setToken(login: LoginResponse) {
+    this._accessToken = login.token;
+    this._refreshToken = login.refresh;
+    this._user = Token.parse(login.token);
+
+    this._event.triggerChange(this._user);
+
+    setRefreshToken(login.refresh);
+  }
+
+  async initToken(token: string) {
+    this._setToken(await login({ token, type: 'Google' }));
+  }
+
+  async refreshToken() {
+    if (!this._refreshToken) {
+      throw new Error('No authorization token.');
+    }
+    if (!this._padding) {
+      this._padding = login({
+        type: 'Refresh',
+        token: this._refreshToken,
+      });
+    }
+    this._setToken(await this._padding);
+    this._padding = undefined;
+  }
+
+  get token() {
+    return this._accessToken;
+  }
+
+  get refresh() {
+    return this._refreshToken;
+  }
+
+  get isLogin() {
+    return !!this._refreshToken;
+  }
+
+  get isExpired() {
+    if (!this._user) return true;
+    return Date.now() - this._user.create_at > this._user.exp;
+  }
+
+  static parse(token: string): AccessTokenMessage | null {
+    try {
+      const message: AccessTokenMessage = JSON.parse(
+        b64DecodeUnicode(token.split('.')[1])
+      );
+      return message;
+    } catch (error) {
+      return null;
+    }
+  }
+
+  onChange(callback: Callback) {
+    this._event.onChange(callback);
+  }
+
+  offChange(callback: Callback) {
+    this._event.removeCallback(callback);
+  }
+}
+
+export const token = new Token();

packages/data-services/src/sdks/index.ts

@@ -0,0 +1,4 @@
+export * from './workspace';
+export * from './workspace.hook';
+export * from './user';
+export * from './user.hook';

packages/data-services/src/sdks/types/common.ts

@@ -0,0 +1,2 @@
+export type CommonError = { error: { code: string; message: string } };
+export type MayError = Partial<CommonError>;

packages/data-services/src/sdks/types/index.ts

@@ -0,0 +1 @@
+export * from './common';

packages/data-services/src/sdks/user.hook.ts

@@ -0,0 +1,23 @@
+import useSWR from 'swr';
+import type { SWRConfiguration } from 'swr';
+import { getUserByEmail } from './user';
+import type { GetUserByEmailParams, User } from './user';
+
+export const GET_USER_BY_EMAIL_SWR_TOKEN = 'user.getUserByEmail';
+export function useGetUserByEmail(
+  params: GetUserByEmailParams,
+  config?: SWRConfiguration
+) {
+  const { data, error, isLoading, mutate } = useSWR<User | null>(
+    [GET_USER_BY_EMAIL_SWR_TOKEN, params],
+    ([_, params]) => getUserByEmail(params),
+    config
+  );
+
+  return {
+    loading: isLoading,
+    data,
+    error,
+    mutate,
+  };
+}

packages/data-services/src/sdks/user.ts

@@ -0,0 +1,21 @@
+import { client } from '../request';
+
+export interface GetUserByEmailParams {
+  email: string;
+  workspace_id: string;
+}
+
+export interface User {
+  id: string;
+  name: string;
+  email: string;
+  avatar_url: string;
+  create_at: string;
+}
+
+export async function getUserByEmail(
+  params: GetUserByEmailParams
+): Promise<User | null> {
+  const searchParams = new URLSearchParams({ ...params });
+  return client.get('/api/user', { searchParams }).json<User | null>();
+}

packages/data-services/src/sdks/workspace.hook.ts

@@ -0,0 +1,72 @@
+import useSWR from 'swr';
+import type { SWRConfiguration } from 'swr';
+import {
+  getWorkspaceDetail,
+  updateWorkspace,
+  deleteWorkspace,
+  inviteMember,
+  Workspace,
+} from './workspace';
+import {
+  GetWorkspaceDetailParams,
+  WorkspaceDetail,
+  UpdateWorkspaceParams,
+  DeleteWorkspaceParams,
+  InviteMemberParams,
+  getWorkspaces,
+} from './workspace';
+
+export const GET_WORKSPACE_DETAIL_SWR_TOKEN = 'workspace.getWorkspaceDetail';
+export function useGetWorkspaceDetail(
+  params: GetWorkspaceDetailParams,
+  config?: SWRConfiguration
+) {
+  const { data, error, isLoading, mutate } = useSWR<WorkspaceDetail | null>(
+    [GET_WORKSPACE_DETAIL_SWR_TOKEN, params],
+    ([_, params]) => getWorkspaceDetail(params),
+    config
+  );
+
+  return {
+    data,
+    error,
+    loading: isLoading,
+    mutate,
+  };
+}
+
+export const GET_WORKSPACES_SWR_TOKEN = 'workspace.getWorkspaces';
+export function useGetWorkspaces(config?: SWRConfiguration) {
+  const { data, error, isLoading } = useSWR<Workspace[]>(
+    [GET_WORKSPACES_SWR_TOKEN],
+    () => getWorkspaces(),
+    config
+  );
+
+  return {
+    data,
+    error,
+    loading: isLoading,
+  };
+}
+
+export const UPDATE_WORKSPACE_SWR_TOKEN = 'workspace.updateWorkspace';
+/**
+ * I don't think a hook needed for update workspace.
+ * If you figure out the scene, please implement this function.
+ */
+export function useUpdateWorkspace() {}
+
+export const DELETE_WORKSPACE_SWR_TOKEN = 'workspace.deleteWorkspace';
+/**
+ * I don't think a hook needed for delete workspace.
+ * If you figure out the scene, please implement this function.
+ */
+export function useDeleteWorkspace() {}
+
+export const INVITE_MEMBER_SWR_TOKEN = 'workspace.inviteMember';
+/**
+ * I don't think a hook needed for invite member.
+ * If you figure out the scene, please implement this function.
+ */
+export function useInviteMember() {}

packages/data-services/src/sdks/workspace.ts

@@ -0,0 +1,178 @@
+import { client, bareClient } from '../request';
+import { User } from './user';
+
+export interface GetWorkspaceDetailParams {
+  id: string;
+}
+
+export enum WorkspaceType {
+  Private = 0,
+  Normal = 1,
+}
+
+export enum PermissionType {
+  Read = 0,
+  Write = 1,
+  Admin = 2,
+  Owner = 99,
+}
+
+export interface Workspace {
+  id: string;
+  type: WorkspaceType;
+  public: boolean;
+  permission_type: PermissionType;
+  create_at: number;
+}
+
+export async function getWorkspaces(): Promise<Workspace[]> {
+  return client
+    .get('/api/workspace', {
+      headers: {
+        'Cache-Control': 'no-cache',
+      },
+    })
+    .json();
+}
+
+export interface WorkspaceDetail extends Workspace {
+  owner: User;
+  member_count: number;
+}
+
+export async function getWorkspaceDetail(
+  params: GetWorkspaceDetailParams
+): Promise<WorkspaceDetail | null> {
+  return client.get(`/api/workspace/${params.id}`).json();
+}
+
+export interface Permission {
+  id: number;
+  type: PermissionType;
+  workspace_id: string;
+  accepted: boolean;
+  create_at: number;
+}
+
+export interface RegisteredUser extends User {
+  type: 'Registered';
+}
+
+export interface UnregisteredUser {
+  type: 'Unregistered';
+  email: string;
+}
+
+export interface Member extends Permission {
+  user: RegisteredUser | UnregisteredUser;
+}
+
+export interface GetWorkspaceMembersParams {
+  id: string;
+}
+
+export async function getWorkspaceMembers(
+  params: GetWorkspaceDetailParams
+): Promise<Member[]> {
+  return client.get(`/api/workspace/${params.id}/permission`).json();
+}
+
+export interface CreateWorkspaceParams {
+  name: string;
+  avatar: string;
+}
+
+export async function createWorkspace(
+  params: CreateWorkspaceParams
+): Promise<void> {
+  return client.post('/api/workspace', { json: params }).json();
+}
+
+export interface UpdateWorkspaceParams {
+  id: string;
+  public: boolean;
+}
+
+export async function updateWorkspace(
+  params: UpdateWorkspaceParams
+): Promise<{ public: boolean | null }> {
+  return client
+    .post(`/api/workspace/${params.id}`, {
+      json: {
+        public: params.public,
+      },
+    })
+    .json();
+}
+
+export interface DeleteWorkspaceParams {
+  id: string;
+}
+
+export async function deleteWorkspace(
+  params: DeleteWorkspaceParams
+): Promise<void> {
+  await client.delete(`/api/workspace/${params.id}`);
+}
+
+export interface InviteMemberParams {
+  id: string;
+  email: string;
+}
+
+/**
+ * Notice: Only support normal(contrast to private) workspace.
+ */
+export async function inviteMember(params: InviteMemberParams): Promise<void> {
+  return client
+    .post(`/api/workspace/${params.id}/permission`, {
+      json: {
+        email: params.email,
+      },
+    })
+    .json();
+}
+
+export interface RemoveMemberParams {
+  permissionId: number;
+}
+
+export async function removeMember(params: RemoveMemberParams): Promise<void> {
+  await client.delete(`/api/permission/${params.permissionId}`);
+}
+
+export interface AcceptInvitingParams {
+  invitingCode: string;
+}
+
+export async function acceptInviting(
+  params: AcceptInvitingParams
+): Promise<void> {
+  await bareClient.post(`/api/invitation/${params.invitingCode}`);
+}
+
+export interface DownloadWOrkspaceParams {
+  workspaceId: string;
+}
+export async function downloadWorkspace(
+  params: DownloadWOrkspaceParams
+): Promise<ArrayBuffer> {
+  return client.get(`/api/workspace/${params.workspaceId}/doc`).arrayBuffer();
+}
+
+export async function uploadBlob(params: { blob: Blob }): Promise<string> {
+  return client.put('/api/blob', { body: params.blob }).text();
+}
+
+export async function getBlob(params: {
+  blobId: string;
+}): Promise<ArrayBuffer> {
+  return client.get(`/api/blob/${params.blobId}`).arrayBuffer();
+}
+
+export interface LeaveWorkspaceParams {
+  id: number | string;
+}
+export async function leaveWorkspace({ id }: LeaveWorkspaceParams) {
+  await client.delete(`/api/workspace/${id}/permission`).json();
+}

packages/data-services/src/websocket/index.ts

@@ -0,0 +1 @@
+export { WebsocketProvider } from './y-websocket';

packages/data-services/src/websocket/y-websocket.js

@@ -0,0 +1,508 @@
+/* eslint-disable no-undef */
+/**
+ * @module provider/websocket
+ */
+
+/* eslint-env browser */
+
+// import * as Y from 'yjs'; // eslint-disable-line
+import * as bc from 'lib0/broadcastchannel';
+import * as time from 'lib0/time';
+import * as encoding from 'lib0/encoding';
+import * as decoding from 'lib0/decoding';
+import * as syncProtocol from 'y-protocols/sync';
+import * as authProtocol from 'y-protocols/auth';
+import * as awarenessProtocol from 'y-protocols/awareness';
+import { Observable } from 'lib0/observable';
+import * as math from 'lib0/math';
+import * as url from 'lib0/url';
+
+export const messageSync = 0;
+export const messageQueryAwareness = 3;
+export const messageAwareness = 1;
+export const messageAuth = 2;
+
+/**
+ *                       encoder,          decoder,          provider,          emitSynced, messageType
+ * @type {Array<function(encoding.Encoder, decoding.Decoder, WebsocketProvider, boolean,    number):void>}
+ */
+const messageHandlers = [];
+
+messageHandlers[messageSync] = (
+  encoder,
+  decoder,
+  provider,
+  emitSynced,
+  _messageType
+) => {
+  encoding.writeVarUint(encoder, messageSync);
+  const syncMessageType = syncProtocol.readSyncMessage(
+    decoder,
+    encoder,
+    provider.doc,
+    provider
+  );
+  if (
+    emitSynced &&
+    syncMessageType === syncProtocol.messageYjsSyncStep2 &&
+    !provider.synced
+  ) {
+    provider.synced = true;
+  }
+};
+
+messageHandlers[messageQueryAwareness] = (
+  encoder,
+  _decoder,
+  provider,
+  _emitSynced,
+  _messageType
+) => {
+  encoding.writeVarUint(encoder, messageAwareness);
+  encoding.writeVarUint8Array(
+    encoder,
+    awarenessProtocol.encodeAwarenessUpdate(
+      provider.awareness,
+      Array.from(provider.awareness.getStates().keys())
+    )
+  );
+};
+
+messageHandlers[messageAwareness] = (
+  _encoder,
+  decoder,
+  provider,
+  _emitSynced,
+  _messageType
+) => {
+  awarenessProtocol.applyAwarenessUpdate(
+    provider.awareness,
+    decoding.readVarUint8Array(decoder),
+    provider
+  );
+};
+
+messageHandlers[messageAuth] = (
+  _encoder,
+  decoder,
+  provider,
+  _emitSynced,
+  _messageType
+) => {
+  authProtocol.readAuthMessage(decoder, provider.doc, (_ydoc, reason) =>
+    permissionDeniedHandler(provider, reason)
+  );
+};
+
+// @todo - this should depend on awareness.outdatedTime
+const messageReconnectTimeout = 30000;
+
+/**
+ * @param {WebsocketProvider} provider
+ * @param {string} reason
+ */
+const permissionDeniedHandler = (provider, reason) =>
+  console.warn(`Permission denied to access ${provider.url}.\n${reason}`);
+
+/**
+ * @param {WebsocketProvider} provider
+ * @param {Uint8Array} buf
+ * @param {boolean} emitSynced
+ * @return {encoding.Encoder}
+ */
+const readMessage = (provider, buf, emitSynced) => {
+  const decoder = decoding.createDecoder(buf);
+  const encoder = encoding.createEncoder();
+  const messageType = decoding.readVarUint(decoder);
+  const messageHandler = provider.messageHandlers[messageType];
+  if (/** @type {any} */ (messageHandler)) {
+    messageHandler(encoder, decoder, provider, emitSynced, messageType);
+  } else {
+    console.error('Unable to compute message');
+  }
+  return encoder;
+};
+
+/**
+ * @param {WebsocketProvider} provider
+ */
+const setupWS = provider => {
+  if (provider.shouldConnect && provider.ws === null) {
+    const websocket = new provider._WS(provider.url);
+    websocket.binaryType = 'arraybuffer';
+    provider.ws = websocket;
+    provider.wsconnecting = true;
+    provider.wsconnected = false;
+    provider.synced = false;
+
+    websocket.onmessage = event => {
+      provider.wsLastMessageReceived = time.getUnixTime();
+      const encoder = readMessage(provider, new Uint8Array(event.data), true);
+      if (encoding.length(encoder) > 1) {
+        websocket.send(encoding.toUint8Array(encoder));
+      }
+    };
+    websocket.onerror = event => {
+      provider.emit('connection-error', [event, provider]);
+    };
+    websocket.onclose = event => {
+      provider.emit('connection-close', [event, provider]);
+      provider.ws = null;
+      provider.wsconnecting = false;
+      if (provider.wsconnected) {
+        provider.wsconnected = false;
+        provider.synced = false;
+        // update awareness (all users except local left)
+        awarenessProtocol.removeAwarenessStates(
+          provider.awareness,
+          Array.from(provider.awareness.getStates().keys()).filter(
+            client => client !== provider.doc.clientID
+          ),
+          provider
+        );
+        provider.emit('status', [
+          {
+            status: 'disconnected',
+          },
+        ]);
+      } else {
+        provider.wsUnsuccessfulReconnects++;
+      }
+      // Start with no reconnect timeout and increase timeout by
+      // using exponential backoff starting with 100ms
+      setTimeout(
+        setupWS,
+        math.min(
+          math.pow(2, provider.wsUnsuccessfulReconnects) * 100,
+          provider.maxBackoffTime
+        ),
+        provider
+      );
+    };
+    websocket.onopen = () => {
+      provider.wsLastMessageReceived = time.getUnixTime();
+      provider.wsconnecting = false;
+      provider.wsconnected = true;
+      provider.wsUnsuccessfulReconnects = 0;
+      provider.emit('status', [
+        {
+          status: 'connected',
+        },
+      ]);
+      // always send sync step 1 when connected
+      const encoder = encoding.createEncoder();
+      encoding.writeVarUint(encoder, messageSync);
+      syncProtocol.writeSyncStep1(encoder, provider.doc);
+      websocket.send(encoding.toUint8Array(encoder));
+      // broadcast local awareness state
+      if (provider.awareness.getLocalState() !== null) {
+        const encoderAwarenessState = encoding.createEncoder();
+        encoding.writeVarUint(encoderAwarenessState, messageAwareness);
+        encoding.writeVarUint8Array(
+          encoderAwarenessState,
+          awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [
+            provider.doc.clientID,
+          ])
+        );
+        websocket.send(encoding.toUint8Array(encoderAwarenessState));
+      }
+    };
+
+    provider.emit('status', [
+      {
+        status: 'connecting',
+      },
+    ]);
+  }
+};
+
+/**
+ * @param {WebsocketProvider} provider
+ * @param {ArrayBuffer} buf
+ */
+const broadcastMessage = (provider, buf) => {
+  if (provider.wsconnected) {
+    /** @type {WebSocket} */ (provider.ws).send(buf);
+  }
+  if (provider.bcconnected) {
+    bc.publish(provider.bcChannel, buf, provider);
+  }
+};
+
+/**
+ * Websocket Provider for Yjs. Creates a websocket connection to sync the shared document.
+ * The document name is attached to the provided url. I.e. the following example
+ * creates a websocket connection to http://localhost:1234/my-document-name
+ *
+ * @example
+ *   import * as Y from 'yjs'
+ *   import { WebsocketProvider } from 'y-websocket'
+ *   const doc = new Y.Doc()
+ *   const provider = new WebsocketProvider('http://localhost:1234', 'my-document-name', doc)
+ *
+ * @extends {Observable<string>}
+ */
+export class WebsocketProvider extends Observable {
+  /**
+   * @param {string} serverUrl
+   * @param {string} roomname
+   * @param {Y.Doc} doc
+   * @param {object} [opts]
+   * @param {boolean} [opts.connect]
+   * @param {awarenessProtocol.Awareness} [opts.awareness]
+   * @param {Object<string,string>} [opts.params]
+   * @param {typeof WebSocket} [opts.WebSocketPolyfill] Optionall provide a WebSocket polyfill
+   * @param {number} [opts.resyncInterval] Request server state every `resyncInterval` milliseconds
+   * @param {number} [opts.maxBackoffTime] Maximum amount of time to wait before trying to reconnect (we try to reconnect using exponential backoff)
+   * @param {boolean} [opts.disableBc] Disable cross-tab BroadcastChannel communication
+   */
+  constructor(
+    serverUrl,
+    roomname,
+    doc,
+    {
+      connect = true,
+      awareness = new awarenessProtocol.Awareness(doc),
+      params = {},
+      WebSocketPolyfill = WebSocket,
+      resyncInterval = -1,
+      maxBackoffTime = 2500,
+      disableBc = false,
+    } = {}
+  ) {
+    super();
+    // ensure that url is always ends with /
+    while (serverUrl[serverUrl.length - 1] === '/') {
+      serverUrl = serverUrl.slice(0, serverUrl.length - 1);
+    }
+    const encodedParams = url.encodeQueryParams(params);
+    this.maxBackoffTime = maxBackoffTime;
+    this.bcChannel = serverUrl + '/' + roomname;
+    this.url =
+      serverUrl +
+      '/' +
+      roomname +
+      (encodedParams.length === 0 ? '' : '?' + encodedParams);
+    this.roomname = roomname;
+    this.doc = doc;
+    this._WS = WebSocketPolyfill;
+    this.awareness = awareness;
+    this.wsconnected = false;
+    this.wsconnecting = false;
+    this.bcconnected = false;
+    this.disableBc = disableBc;
+    this.wsUnsuccessfulReconnects = 0;
+    this.messageHandlers = messageHandlers.slice();
+    /**
+     * @type {boolean}
+     */
+    this._synced = false;
+    /**
+     * @type {WebSocket?}
+     */
+    this.ws = null;
+    this.wsLastMessageReceived = 0;
+    /**
+     * Whether to connect to other peers or not
+     * @type {boolean}
+     */
+    this.shouldConnect = connect;
+
+    /**
+     * @type {number}
+     */
+    this._resyncInterval = 0;
+    if (resyncInterval > 0) {
+      this._resyncInterval = /** @type {any} */ (
+        setInterval(() => {
+          if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            // resend sync step 1
+            const encoder = encoding.createEncoder();
+            encoding.writeVarUint(encoder, messageSync);
+            syncProtocol.writeSyncStep1(encoder, doc);
+            this.ws.send(encoding.toUint8Array(encoder));
+          }
+        }, resyncInterval)
+      );
+    }
+
+    /**
+     * @param {ArrayBuffer} data
+     * @param {any} origin
+     */
+    this._bcSubscriber = (data, origin) => {
+      if (origin !== this) {
+        const encoder = readMessage(this, new Uint8Array(data), false);
+        if (encoding.length(encoder) > 1) {
+          bc.publish(this.bcChannel, encoding.toUint8Array(encoder), this);
+        }
+      }
+    };
+    /**
+     * Listens to Yjs updates and sends them to remote peers (ws and broadcastchannel)
+     * @param {Uint8Array} update
+     * @param {any} origin
+     */
+    this._updateHandler = (update, origin) => {
+      if (origin !== this) {
+        const encoder = encoding.createEncoder();
+        encoding.writeVarUint(encoder, messageSync);
+        syncProtocol.writeUpdate(encoder, update);
+        broadcastMessage(this, encoding.toUint8Array(encoder));
+      }
+    };
+    this.doc.on('update', this._updateHandler);
+    /**
+     * @param {any} changed
+     * @param {any} _origin
+     */
+    this._awarenessUpdateHandler = ({ added, updated, removed }, _origin) => {
+      const changedClients = added.concat(updated).concat(removed);
+      const encoder = encoding.createEncoder();
+      encoding.writeVarUint(encoder, messageAwareness);
+      encoding.writeVarUint8Array(
+        encoder,
+        awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients)
+      );
+      broadcastMessage(this, encoding.toUint8Array(encoder));
+    };
+    this._unloadHandler = () => {
+      awarenessProtocol.removeAwarenessStates(
+        this.awareness,
+        [doc.clientID],
+        'window unload'
+      );
+    };
+    if (typeof window !== 'undefined') {
+      window.addEventListener('unload', this._unloadHandler);
+    } else if (typeof process !== 'undefined') {
+      process.on('exit', this._unloadHandler);
+    }
+    awareness.on('update', this._awarenessUpdateHandler);
+    this._checkInterval = /** @type {any} */ (
+      setInterval(() => {
+        if (
+          this.wsconnected &&
+          messageReconnectTimeout <
+            time.getUnixTime() - this.wsLastMessageReceived
+        ) {
+          // no message received in a long time - not even your own awareness
+          // updates (which are updated every 15 seconds)
+          /** @type {WebSocket} */ (this.ws).close();
+        }
+      }, messageReconnectTimeout / 10)
+    );
+    if (connect) {
+      this.connect();
+    }
+  }
+
+  /**
+   * @type {boolean}
+   */
+  get synced() {
+    return this._synced;
+  }
+
+  set synced(state) {
+    if (this._synced !== state) {
+      this._synced = state;
+      this.emit('synced', [state]);
+      this.emit('sync', [state]);
+    }
+  }
+
+  destroy() {
+    if (this._resyncInterval !== 0) {
+      clearInterval(this._resyncInterval);
+    }
+    clearInterval(this._checkInterval);
+    this.disconnect();
+    if (typeof window !== 'undefined') {
+      window.removeEventListener('unload', this._unloadHandler);
+    } else if (typeof process !== 'undefined') {
+      process.off('exit', this._unloadHandler);
+    }
+    this.awareness.off('update', this._awarenessUpdateHandler);
+    this.doc.off('update', this._updateHandler);
+    super.destroy();
+  }
+
+  connectBc() {
+    if (this.disableBc) {
+      return;
+    }
+    if (!this.bcconnected) {
+      bc.subscribe(this.bcChannel, this._bcSubscriber);
+      this.bcconnected = true;
+    }
+    // send sync step1 to bc
+    // write sync step 1
+    const encoderSync = encoding.createEncoder();
+    encoding.writeVarUint(encoderSync, messageSync);
+    syncProtocol.writeSyncStep1(encoderSync, this.doc);
+    bc.publish(this.bcChannel, encoding.toUint8Array(encoderSync), this);
+    // broadcast local state
+    const encoderState = encoding.createEncoder();
+    encoding.writeVarUint(encoderState, messageSync);
+    syncProtocol.writeSyncStep2(encoderState, this.doc);
+    bc.publish(this.bcChannel, encoding.toUint8Array(encoderState), this);
+    // write queryAwareness
+    const encoderAwarenessQuery = encoding.createEncoder();
+    encoding.writeVarUint(encoderAwarenessQuery, messageQueryAwareness);
+    bc.publish(
+      this.bcChannel,
+      encoding.toUint8Array(encoderAwarenessQuery),
+      this
+    );
+    // broadcast local awareness state
+    const encoderAwarenessState = encoding.createEncoder();
+    encoding.writeVarUint(encoderAwarenessState, messageAwareness);
+    encoding.writeVarUint8Array(
+      encoderAwarenessState,
+      awarenessProtocol.encodeAwarenessUpdate(this.awareness, [
+        this.doc.clientID,
+      ])
+    );
+    bc.publish(
+      this.bcChannel,
+      encoding.toUint8Array(encoderAwarenessState),
+      this
+    );
+  }
+
+  disconnectBc() {
+    // broadcast message with local awareness state set to null (indicating disconnect)
+    const encoder = encoding.createEncoder();
+    encoding.writeVarUint(encoder, messageAwareness);
+    encoding.writeVarUint8Array(
+      encoder,
+      awarenessProtocol.encodeAwarenessUpdate(
+        this.awareness,
+        [this.doc.clientID],
+        new Map()
+      )
+    );
+    broadcastMessage(this, encoding.toUint8Array(encoder));
+    if (this.bcconnected) {
+      bc.unsubscribe(this.bcChannel, this._bcSubscriber);
+      this.bcconnected = false;
+    }
+  }
+
+  disconnect() {
+    this.shouldConnect = false;
+    this.disconnectBc();
+    if (this.ws !== null) {
+      this.ws.close();
+    }
+  }
+
+  connect() {
+    this.shouldConnect = true;
+    if (!this.wsconnected && this.ws === null) {
+      setupWS(this);
+      this.connectBc();
+    }
+  }
+}

packages/data-services/tests/datacenter.spec.ts

@@ -0,0 +1,8 @@
+import { test, expect } from '@playwright/test';
+import { getDataCenter } from './utils.js';
+
+test('can init data center', async () => {
+  const dataCenter = await getDataCenter();
+
+  expect(dataCenter).toBeTruthy();
+});

packages/data-services/tests/utils.ts

@@ -0,0 +1,4 @@
+export const getDataCenter = () =>
+  import('../src/data-center.js').then(async dataCenter =>
+    dataCenter.getDataCenter()
+  );

packages/data-services/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": false,
+    "esModuleInterop": true,
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "experimentalDecorators": true,
+    "declaration": true,
+    "baseUrl": ".",
+    "rootDir": ".",
+    "outDir": "./dist"
+  },
+  "include": ["next-env.d.ts", "src/**/*.ts", "pages/**/*.tsx"],
+  "exclude": ["node_modules", "dist"]
+}