refactor(infra): directory structure (#4615)

tools/@types/env/__all.d.ts

@@ -0,0 +1,71 @@
+import type { Environment, Platform, RuntimeConfig } from '@affine/env/global';
+import type {
+  DBHandlerManager,
+  DebugHandlerManager,
+  DialogHandlerManager,
+  EventMap,
+  ExportHandlerManager,
+  UIHandlerManager,
+  UnwrapManagerHandlerToClientSide,
+  UpdaterHandlerManager,
+  WorkspaceHandlerManager,
+} from '@toeverything/infra/index';
+
+declare global {
+  interface Window {
+    appInfo: {
+      electron: boolean;
+    };
+    apis: {
+      db: UnwrapManagerHandlerToClientSide<DBHandlerManager>;
+      debug: UnwrapManagerHandlerToClientSide<DebugHandlerManager>;
+      dialog: UnwrapManagerHandlerToClientSide<DialogHandlerManager>;
+      export: UnwrapManagerHandlerToClientSide<ExportHandlerManager>;
+      ui: UnwrapManagerHandlerToClientSide<UIHandlerManager>;
+      updater: UnwrapManagerHandlerToClientSide<UpdaterHandlerManager>;
+      workspace: UnwrapManagerHandlerToClientSide<WorkspaceHandlerManager>;
+    };
+    events: EventMap;
+  }
+
+  interface WindowEventMap {
+    'migration-done': CustomEvent;
+  }
+
+  // eslint-disable-next-line no-var
+  var process: {
+    env: Record<string, string>;
+  };
+  // eslint-disable-next-line no-var
+  var $migrationDone: boolean;
+  // eslint-disable-next-line no-var
+  var platform: Platform | undefined;
+  // eslint-disable-next-line no-var
+  var environment: Environment;
+  // eslint-disable-next-line no-var
+  var runtimeConfig: RuntimeConfig;
+  // eslint-disable-next-line no-var
+  var $AFFINE_SETUP: boolean | undefined;
+  // eslint-disable-next-line no-var
+  var editorVersion: string | undefined;
+  // eslint-disable-next-line no-var
+  var prefixUrl: string;
+  // eslint-disable-next-line no-var
+  var websocketPrefixUrl: string;
+}
+
+declare module '@blocksuite/store' {
+  interface PageMeta {
+    favorite?: boolean;
+    subpageIds: string[];
+    // If a page remove to trash, and it is a subpage, it will remove from its parent `subpageIds`, 'trashRelate' is use for save it parent
+    trashRelate?: string;
+    trash?: boolean;
+    trashDate?: number;
+    updatedDate?: number;
+    mode?: 'page' | 'edgeless';
+    jumpOnce?: boolean;
+    // todo: support `number` in the future
+    isPublic?: boolean;
+  }
+}

tools/@types/env/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@types/affine__env",
+  "private": true,
+  "types": "./__all.d.ts",
+  "type": "module",
+  "dependencies": {
+    "@affine/env": "workspace:*",
+    "@toeverything/infra": "workspace:*"
+  },
+  "version": "0.10.0-canary.1"
+}

tools/cli/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@affine/cli",
+  "type": "module",
+  "private": true,
+  "bin": {
+    "build-core": "./src/bin/build-core.mjs",
+    "dev-core": "./src/bin/dev-core.mjs"
+  },
+  "exports": {
+    "./config": "./src/config/index.ts"
+  },
+  "devDependencies": {
+    "@clack/core": "^0.3.3",
+    "@clack/prompts": "^0.7.0",
+    "@magic-works/i18n-codegen": "^0.5.0",
+    "ts-node": "^10.9.1"
+  },
+  "dependencies": {
+    "dotenv": "^16.3.1",
+    "vite": "^4.4.11"
+  },
+  "peerDependencies": {
+    "ts-node": "*"
+  },
+  "version": "0.10.0-canary.1"
+}

tools/cli/src/bin/build-core.mjs

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const child = spawnSync(
+  process.execPath,
+  [
+    '--loader',
+    'ts-node/esm/transpile-only',
+    fileURLToPath(new URL('./build-core.ts', import.meta.url)),
+    ...process.argv.slice(2),
+  ],
+  { stdio: 'inherit' }
+);
+
+if (child.status) process.exit(child.status);

tools/cli/src/bin/build-core.ts

@@ -0,0 +1,74 @@
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+
+import type { BuildFlags } from '../config/index.js';
+import { projectRoot } from '../config/index.js';
+import { buildI18N } from '../util/i18n.js';
+
+const cwd = path.resolve(projectRoot, 'packages/frontend/core');
+
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+const buildType = process.env.BUILD_TYPE_OVERRIDE || process.env.BUILD_TYPE;
+
+if (process.env.BUILD_TYPE_OVERRIDE) {
+  process.env.BUILD_TYPE = process.env.BUILD_TYPE_OVERRIDE;
+}
+
+const getChannel = () => {
+  switch (buildType) {
+    case 'canary':
+    case 'beta':
+    case 'stable':
+    case 'internal':
+      return buildType;
+    case '':
+      throw new Error('BUILD_TYPE is not set');
+    default: {
+      throw new Error(
+        `BUILD_TYPE must be one of canary, beta, stable, internal, received [${buildType}]`
+      );
+    }
+  }
+};
+
+const getDistribution = () => {
+  switch (process.env.DISTRIBUTION) {
+    case 'browser':
+    case 'desktop':
+      return process.env.DISTRIBUTION;
+    case undefined: {
+      console.log('DISTRIBUTION is not set, defaulting to browser');
+      return 'browser';
+    }
+    default: {
+      throw new Error('DISTRIBUTION must be one of browser, desktop');
+    }
+  }
+};
+
+const flags = {
+  distribution: getDistribution(),
+  mode: 'production',
+  channel: getChannel(),
+  coverage: process.env.COVERAGE === 'true',
+} satisfies BuildFlags;
+
+buildI18N();
+spawn(
+  'node',
+  [
+    '--loader',
+    'ts-node/esm/transpile-only',
+    '../../../node_modules/webpack/bin/webpack.js',
+    '--mode',
+    'production',
+    '--env',
+    'flags=' + Buffer.from(JSON.stringify(flags), 'utf-8').toString('hex'),
+  ].filter((v): v is string => !!v),
+  {
+    cwd,
+    stdio: 'inherit',
+    shell: true,
+    env: process.env,
+  }
+);

tools/cli/src/bin/dev-core.mjs

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const child = spawnSync(
+  process.execPath,
+  [
+    '--loader',
+    'ts-node/esm/transpile-only',
+    fileURLToPath(new URL('./dev-core.ts', import.meta.url)),
+    ...process.argv.slice(2),
+  ],
+  { stdio: 'inherit' }
+);
+
+if (child.status) process.exit(child.status);

tools/cli/src/bin/dev-core.ts

@@ -0,0 +1,219 @@
+import type { ChildProcess } from 'node:child_process';
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+
+import * as p from '@clack/prompts';
+import { config } from 'dotenv';
+
+import { type BuildFlags, projectRoot } from '../config/index.js';
+import { watchI18N } from '../util/i18n.js';
+
+const cwd = path.resolve(projectRoot, 'packages/frontend/core');
+
+const flags: BuildFlags = {
+  distribution: 'browser',
+  mode: 'development',
+  channel: 'canary',
+  coverage: process.env.COVERAGE === 'true',
+  localBlockSuite: undefined,
+};
+
+if (process.argv.includes('--static')) {
+  await awaitChildProcess(
+    spawn(
+      'node',
+      [
+        '--loader',
+        'ts-node/esm/transpile-only',
+        '../../../node_modules/webpack/bin/webpack.js',
+        'serve',
+        '--mode',
+        'development',
+        '--no-client-overlay',
+        '--no-live-reload',
+        '--env',
+        'flags=' + Buffer.from(JSON.stringify(flags), 'utf-8').toString('hex'),
+      ].filter((v): v is string => !!v),
+      {
+        cwd,
+        stdio: 'inherit',
+        shell: true,
+        env: process.env,
+      }
+    )
+  );
+  process.exit(0);
+}
+
+const files = ['.env', '.env.local'];
+
+for (const file of files) {
+  if (existsSync(path.resolve(projectRoot, file))) {
+    config({
+      path: path.resolve(projectRoot, file),
+    });
+    console.log(`${file} loaded`);
+    break;
+  }
+}
+
+const buildFlags = await p.group(
+  {
+    distribution: () =>
+      p.select({
+        message: 'Distribution',
+        options: [
+          {
+            value: 'browser',
+          },
+          {
+            value: 'desktop',
+          },
+        ],
+        initialValue: 'browser',
+      }),
+    mode: () =>
+      p.select({
+        message: 'Mode',
+        options: [
+          {
+            value: 'development',
+          },
+          {
+            value: 'production',
+          },
+        ],
+        initialValue: 'development',
+      }),
+    channel: () =>
+      p.select({
+        message: 'Channel',
+        options: [
+          {
+            value: 'canary',
+          },
+          {
+            value: 'beta',
+          },
+          {
+            value: 'stable',
+          },
+        ],
+        initialValue: 'canary',
+      }),
+    coverage: () =>
+      p.confirm({
+        message: 'Enable coverage',
+        initialValue: process.env.COVERAGE === 'true',
+      }),
+    debugBlockSuite: () =>
+      p.confirm({
+        message: 'Debug blocksuite locally?',
+        initialValue: false,
+      }),
+  },
+  {
+    onCancel: () => {
+      p.cancel('Operation cancelled.');
+      process.exit(0);
+    },
+  }
+);
+
+if (buildFlags.debugBlockSuite) {
+  const { config } = await import('dotenv');
+  const envLocal = config({
+    path: path.resolve(cwd, '.env.local'),
+  });
+
+  const localBlockSuite = await p.text({
+    message: 'local blocksuite PATH',
+    initialValue: envLocal.error
+      ? undefined
+      : envLocal.parsed?.LOCAL_BLOCK_SUITE,
+  });
+  if (typeof localBlockSuite !== 'string') {
+    throw new Error('local blocksuite PATH is required');
+  }
+  if (!existsSync(localBlockSuite)) {
+    throw new Error(`local blocksuite not found: ${localBlockSuite}`);
+  }
+  flags.localBlockSuite = localBlockSuite;
+}
+
+flags.distribution = buildFlags.distribution as any;
+flags.mode = buildFlags.mode as any;
+flags.channel = buildFlags.channel as any;
+flags.coverage = buildFlags.coverage;
+
+watchI18N();
+
+function awaitChildProcess(child: ChildProcess): Promise<number> {
+  return new Promise<number>((resolve, reject) => {
+    const handleExitCode = (code: number | null) => {
+      if (code) {
+        reject(
+          new Error(
+            `Child process at ${
+              (child as any).cwd
+            } fails: ${child.spawnargs.join(' ')}`
+          )
+        );
+      } else {
+        resolve(0);
+      }
+    };
+
+    child.on('error', () => handleExitCode(child.exitCode));
+    child.on('exit', code => handleExitCode(code));
+  });
+}
+
+try {
+  // Build:infra
+  await awaitChildProcess(
+    spawn('yarn', ['build:infra'], {
+      cwd,
+      stdio: 'inherit',
+      shell: true,
+      env: process.env,
+    })
+  );
+
+  // Build:plugins
+  await awaitChildProcess(
+    spawn('yarn', ['build:plugins'], {
+      cwd,
+      stdio: 'inherit',
+      shell: true,
+      env: process.env,
+    })
+  );
+
+  // Start webpack
+  await awaitChildProcess(
+    spawn(
+      'node',
+      [
+        '--loader',
+        'ts-node/esm/transpile-only',
+        '../../../node_modules/webpack/bin/webpack.js',
+        flags.mode === 'development' ? 'serve' : undefined,
+        '--mode',
+        flags.mode === 'development' ? 'development' : 'production',
+        '--env',
+        'flags=' + Buffer.from(JSON.stringify(flags), 'utf-8').toString('hex'),
+      ].filter((v): v is string => !!v),
+      {
+        cwd,
+        stdio: 'inherit',
+        shell: true,
+        env: process.env,
+      }
+    )
+  );
+} catch (error) {
+  console.error('Error during build:', error);
+  process.exit(1);
+}

tools/cli/src/config/index.ts

@@ -0,0 +1,13 @@
+import { fileURLToPath } from 'node:url';
+
+export type BuildFlags = {
+  distribution: 'browser' | 'desktop';
+  mode: 'development' | 'production';
+  channel: 'stable' | 'beta' | 'canary' | 'internal';
+  coverage?: boolean;
+  localBlockSuite?: string;
+};
+
+export const projectRoot = fileURLToPath(
+  new URL('../../../../', import.meta.url)
+);

tools/cli/src/util/i18n.ts

@@ -0,0 +1,32 @@
+import { resolve } from 'node:path';
+
+import { runCli } from '@magic-works/i18n-codegen';
+
+import { projectRoot } from '../config/index.js';
+
+const configPath = resolve(projectRoot, '.i18n-codegen.json');
+
+export const watchI18N = () => {
+  runCli(
+    {
+      config: configPath,
+      watch: true,
+    },
+    error => {
+      console.error(error);
+    }
+  );
+};
+
+export const buildI18N = () => {
+  runCli(
+    {
+      config: configPath,
+      watch: false,
+    },
+    error => {
+      console.error(error);
+      process.exit(1);
+    }
+  );
+};

tools/cli/src/util/infra.ts

@@ -0,0 +1,41 @@
+import { spawn } from 'node:child_process';
+import { resolve } from 'node:path';
+
+import { build } from 'vite';
+
+import { projectRoot } from '../config/index.js';
+
+const infraFilePath = resolve(
+  projectRoot,
+  'packages',
+  'infra',
+  'vite.config.ts'
+);
+const pluginInfraFilePath = resolve(
+  projectRoot,
+  'packages',
+  'plugin-infra',
+  'vite.config.ts'
+);
+
+export const buildInfra = async () => {
+  await build({
+    configFile: infraFilePath,
+  });
+  await build({
+    configFile: pluginInfraFilePath,
+  });
+};
+
+export const watchInfra = async () => {
+  spawn('vite', ['build', '--watch'], {
+    cwd: resolve(projectRoot, 'packages/common/infra'),
+    shell: true,
+    stdio: 'inherit',
+  });
+  spawn('vite', ['build', '--watch'], {
+    cwd: resolve(projectRoot, 'packages/plugin-infra'),
+    shell: true,
+    stdio: 'inherit',
+  });
+};

tools/cli/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "composite": true,
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "outDir": "lib"
+  },
+  "include": ["src"]
+}

tools/plugin-cli/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@affine/plugin-cli",
+  "type": "module",
+  "version": "0.10.0-canary.1",
+  "bin": {
+    "af": "./src/af.mjs"
+  },
+  "files": [
+    "src",
+    "tsconfig.json"
+  ],
+  "dependencies": {
+    "@endo/static-module-record": "^0.8.2",
+    "@plugxjs/vite-plugin": "0.1.0",
+    "@swc/core": "^1.3.93",
+    "@toeverything/infra": "workspace:^",
+    "@vanilla-extract/rollup-plugin": "^1.3.0",
+    "@vitejs/plugin-vue": "^4.4.0",
+    "rollup": "^3.29.4",
+    "rollup-plugin-swc3": "^0.10.2",
+    "ts-node": "^10.9.1",
+    "vue": "^3.3.4"
+  }
+}

tools/plugin-cli/src/af.mjs

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const child = spawnSync(
+  process.execPath,
+  [
+    '--loader',
+    'ts-node/esm/transpile-only',
+    fileURLToPath(new URL('./af.ts', import.meta.url)),
+    ...process.argv.slice(2),
+  ],
+  { stdio: 'inherit' }
+);
+
+if (child.status) process.exit(child.status);

tools/plugin-cli/src/af.ts

@@ -0,0 +1,183 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+
+import { plugx } from '@plugxjs/vite-plugin';
+import {
+  packageJsonInputSchema,
+  packageJsonOutputSchema,
+} from '@toeverything/infra/type';
+import { vanillaExtractPlugin } from '@vanilla-extract/vite-plugin';
+import react from '@vitejs/plugin-react-swc';
+import vue from '@vitejs/plugin-vue';
+import { build, type PluginOption } from 'vite';
+import type { z } from 'zod';
+
+const projectRoot = fileURLToPath(new URL('../../..', import.meta.url));
+
+const args = process.argv.splice(2);
+
+const result = parseArgs({
+  args,
+  allowPositionals: true,
+});
+
+const plugin = process.cwd().split(path.sep).pop();
+if (!plugin) {
+  throw new Error('plugin name not found');
+}
+
+const command = result.positionals[0];
+
+const isWatch = (() => {
+  switch (command) {
+    case 'dev': {
+      return true;
+    }
+    case 'build': {
+      return false;
+    }
+    default: {
+      throw new Error('invalid command');
+    }
+  }
+})();
+
+const external = [
+  // built-in packages
+  /^@affine/,
+  /^@blocksuite/,
+  /^@toeverything/,
+
+  // react
+  'react',
+  /^react\//,
+  /^react-dom/,
+
+  // store
+  /^jotai/,
+
+  // utils
+  'swr',
+
+  // css
+  /^@vanilla-extract/,
+];
+
+const allPluginDir = path.resolve(projectRoot, 'packages/plugins');
+
+const getPluginDir = (plugin: string) => path.resolve(allPluginDir, plugin);
+const pluginDir = getPluginDir(plugin);
+const packageJsonFile = path.resolve(pluginDir, 'package.json');
+
+const json: z.infer<typeof packageJsonInputSchema> = await readFile(
+  packageJsonFile,
+  {
+    encoding: 'utf-8',
+  }
+)
+  .then(text => JSON.parse(text))
+  .then(async json => {
+    const result = await packageJsonInputSchema.safeParseAsync(json);
+    if (result.success) {
+      return json;
+    } else {
+      throw new Error('invalid package.json', result.error);
+    }
+  });
+
+type Metadata = {
+  assets: Set<string>;
+};
+
+const metadata: Metadata = {
+  assets: new Set(),
+};
+
+const outDir = path.resolve(
+  projectRoot,
+  'packages/frontend/core/public/plugins'
+);
+
+const coreOutDir = path.resolve(outDir, plugin);
+
+const coreEntry = path.resolve(pluginDir, json.affinePlugin.entry.core);
+
+const generatePackageJson: PluginOption = {
+  name: 'generate-package.json',
+  async generateBundle() {
+    const packageJson = {
+      name: json.name,
+      version: json.version,
+      description: json.description,
+      affinePlugin: {
+        release: json.affinePlugin.release,
+        entry: {
+          core: 'index.js',
+        },
+        assets: [...metadata.assets],
+      },
+    } satisfies z.infer<typeof packageJsonOutputSchema>;
+    packageJsonOutputSchema.parse(packageJson);
+    this.emitFile({
+      type: 'asset',
+      fileName: 'package.json',
+      source: JSON.stringify(packageJson, null, 2),
+    });
+  },
+};
+
+// step 1: generate core bundle
+await build({
+  build: {
+    watch: isWatch ? {} : undefined,
+    minify: false,
+    target: 'esnext',
+    outDir: coreOutDir,
+    emptyOutDir: true,
+    lib: {
+      entry: coreEntry,
+      fileName: 'index',
+      formats: ['es'],
+    },
+    rollupOptions: {
+      output: {
+        assetFileNames: chunkInfo => {
+          if (chunkInfo.name) {
+            metadata.assets.add(chunkInfo.name);
+            return chunkInfo.name;
+          } else {
+            throw new Error('no name');
+          }
+        },
+        chunkFileNames: chunkInfo => {
+          if (chunkInfo.name) {
+            const hash = createHash('md5')
+              .update(
+                Object.values(chunkInfo.moduleIds)
+                  .map(m => m)
+                  .join()
+              )
+              .digest('hex')
+              .substring(0, 6);
+            return `${chunkInfo.name}-${hash}.mjs`;
+          } else {
+            throw new Error('no name');
+          }
+        },
+      },
+      external,
+    },
+  },
+  plugins: [
+    vanillaExtractPlugin(),
+    vue(),
+    react(),
+    plugx({
+      staticJsonSuffix: '.json',
+    }),
+    generatePackageJson,
+  ],
+});

tools/plugin-cli/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "composite": true,
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "outDir": "lib"
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../../packages/common/infra"
+    }
+  ]
+}

tools/workers/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@affine/workers",
+  "version": "0.10.0-canary.1",
+  "private": true,
+  "scripts": {
+    "dev": "wrangler dev"
+  },
+  "devDependencies": {
+    "wrangler": "^3.13.1"
+  }
+}

tools/workers/src/index.ts

@@ -0,0 +1,73 @@
+const ALLOW_ORIGIN = [
+  'https://affine.pro',
+  'https://app.affine.pro',
+  'https://insider.affine.pro',
+  'https://affine.fail',
+];
+
+function isString(s: any): boolean {
+  return typeof s === 'string' || s instanceof String;
+}
+
+function isOriginAllowed(
+  origin: string,
+  allowedOrigin: string | RegExp | Array<string | RegExp>
+): boolean {
+  if (Array.isArray(allowedOrigin)) {
+    for (let i = 0; i < allowedOrigin.length; ++i) {
+      if (isOriginAllowed(origin, allowedOrigin[i])) {
+        return true;
+      }
+    }
+    return false;
+  } else if (isString(allowedOrigin)) {
+    return origin === allowedOrigin;
+  } else if (allowedOrigin instanceof RegExp) {
+    return allowedOrigin.test(origin);
+  } else {
+    return !!allowedOrigin;
+  }
+}
+
+async function proxyImage(request: Request): Promise<Response> {
+  const url = new URL(request.url);
+  const imageURL = url.searchParams.get('url');
+
+  if (!imageURL) {
+    return new Response('Missing "url" parameter', { status: 400 });
+  }
+
+  const imageRequest = new Request(imageURL, {
+    method: 'GET',
+    headers: request.headers,
+  });
+
+  const response = await fetch(imageRequest);
+  const modifiedResponse = new Response(response.body);
+
+  modifiedResponse.headers.set(
+    'Access-Control-Allow-Origin',
+    request.headers.get('Origin') ?? 'null'
+  );
+  modifiedResponse.headers.set('Vary', 'Origin');
+  modifiedResponse.headers.set('Access-Control-Allow-Methods', 'GET');
+
+  return modifiedResponse;
+}
+
+const handler = {
+  async fetch(request: Request) {
+    if (!isOriginAllowed(request.headers.get('Origin') ?? '', ALLOW_ORIGIN)) {
+      return new Response('unauthorized', { status: 401 });
+    }
+
+    const url = new URL(request.url);
+    if (url.pathname.startsWith('/proxy/image')) {
+      return await proxyImage(request);
+    }
+
+    return new Response('not found', { status: 404 });
+  },
+};
+
+export default handler;

tools/workers/wrangler.toml

@@ -0,0 +1,3 @@
+name = "workers"
+main = "./src/index.ts"
+compatibility_date = "2023-07-11"