refactor(infra): directory structure (#4615)

packages/frontend/electron/scripts/build-layers.ts

@@ -0,0 +1,26 @@
+import * as esbuild from 'esbuild';
+
+import { config, mode } from './common';
+
+async function buildLayers() {
+  const common = config();
+
+  const define: Record<string, string> = {
+    'process.env.NODE_ENV': `"${mode}"`,
+    'process.env.BUILD_TYPE': `"${process.env.BUILD_TYPE || 'stable'}"`,
+  };
+
+  if (process.env.BUILD_TYPE_OVERRIDE) {
+    define[
+      'process.env.BUILD_TYPE_OVERRIDE'
+    ] = `"${process.env.BUILD_TYPE_OVERRIDE}"`;
+  }
+
+  await esbuild.build({
+    ...common,
+    define: define,
+  });
+}
+
+await buildLayers();
+console.log('Build layers done');

packages/frontend/electron/scripts/common.ts

@@ -0,0 +1,57 @@
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import type { BuildOptions } from 'esbuild';
+
+export const electronDir = fileURLToPath(new URL('..', import.meta.url));
+
+export const rootDir = resolve(electronDir, '..', '..', '..');
+
+export const NODE_MAJOR_VERSION = 18;
+
+// hard-coded for now:
+// fixme(xp): report error if app is not running on DEV_SERVER_URL
+const DEV_SERVER_URL = process.env.DEV_SERVER_URL;
+
+export const mode = (process.env.NODE_ENV =
+  process.env.NODE_ENV || 'development');
+
+export const config = (): BuildOptions => {
+  const define = Object.fromEntries([
+    ['process.env.NODE_ENV', `"${mode}"`],
+    ['process.env.USE_WORKER', '"true"'],
+  ]);
+
+  if (DEV_SERVER_URL) {
+    define['process.env.DEV_SERVER_URL'] = `"${DEV_SERVER_URL}"`;
+  }
+
+  return {
+    entryPoints: [
+      resolve(electronDir, './src/main/index.ts'),
+      resolve(electronDir, './src/preload/index.ts'),
+      resolve(electronDir, './src/helper/index.ts'),
+    ],
+    entryNames: '[dir]',
+    outdir: resolve(electronDir, './dist'),
+    bundle: true,
+    target: `node${NODE_MAJOR_VERSION}`,
+    platform: 'node',
+    external: [
+      'electron',
+      'electron-updater',
+      '@toeverything/plugin-infra',
+      'yjs',
+      'semver',
+      'tinykeys',
+    ],
+    define: define,
+    format: 'cjs',
+    loader: {
+      '.node': 'copy',
+    },
+    assetNames: '[name]',
+    treeShaking: true,
+    sourcemap: 'linked',
+  };
+};

packages/frontend/electron/scripts/dev.ts

@@ -0,0 +1,94 @@
+/* eslint-disable no-async-promise-executor */
+import { spawn } from 'node:child_process';
+
+import type { ChildProcessWithoutNullStreams } from 'child_process';
+import electronPath from 'electron';
+import * as esbuild from 'esbuild';
+
+import { config } from './common';
+
+// this means we don't spawn electron windows, mainly for testing
+const watchMode = process.argv.includes('--watch');
+
+/** Messages on stderr that match any of the contained patterns will be stripped from output */
+const stderrFilterPatterns = [
+  // warning about devtools extension
+  // https://github.com/cawa-93/vite-electron-builder/issues/492
+  // https://github.com/MarshallOfSound/electron-devtools-installer/issues/143
+  /ExtensionLoadWarning/,
+];
+
+let spawnProcess: ChildProcessWithoutNullStreams | null = null;
+
+function spawnOrReloadElectron() {
+  if (watchMode) {
+    return;
+  }
+  if (spawnProcess !== null) {
+    spawnProcess.off('exit', process.exit);
+    spawnProcess.kill('SIGINT');
+    spawnProcess = null;
+  }
+
+  spawnProcess = spawn(String(electronPath), ['.']);
+
+  spawnProcess.stdout.on('data', d => {
+    const str = d.toString().trim();
+    if (str) {
+      console.log(str);
+    }
+  });
+  spawnProcess.stderr.on('data', d => {
+    const data = d.toString().trim();
+    if (!data) return;
+    const mayIgnore = stderrFilterPatterns.some(r => r.test(data));
+    if (mayIgnore) return;
+    console.error(data);
+  });
+
+  // Stops the watch script when the application has quit
+  spawnProcess.on('exit', process.exit);
+}
+
+const common = config();
+
+async function watchLayers() {
+  return new Promise<void>(async resolve => {
+    let initialBuild = false;
+
+    const buildContext = await esbuild.context({
+      ...common,
+      plugins: [
+        ...(common.plugins ?? []),
+        {
+          name: 'electron-dev:reload-app-on-layers-change',
+          setup(build) {
+            build.onEnd(() => {
+              if (initialBuild) {
+                console.log(`[layers] has changed, [re]launching electron...`);
+                spawnOrReloadElectron();
+              } else {
+                resolve();
+                initialBuild = true;
+              }
+            });
+          },
+        },
+      ],
+    });
+    await buildContext.watch();
+  });
+}
+
+async function main() {
+  await watchLayers();
+
+  if (watchMode) {
+    console.log(`Watching for changes...`);
+  } else {
+    spawnOrReloadElectron();
+    console.log(`Electron is started, watching for changes...`);
+  }
+}
+
+main();

packages/frontend/electron/scripts/generate-assets.ts

@@ -0,0 +1,91 @@
+import { spawnSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import fs from 'fs-extra';
+import { glob } from 'glob';
+
+const require = createRequire(import.meta.url);
+const __dirname = fileURLToPath(new URL('.', import.meta.url));
+
+const repoRootDir = path.join(__dirname, '..', '..', '..');
+const electronRootDir = path.join(__dirname, '..');
+const publicDistDir = path.join(electronRootDir, 'resources');
+const affineCoreDir = path.join(repoRootDir, 'apps', 'core');
+const affineCoreOutDir = path.join(affineCoreDir, 'dist');
+const publicAffineOutDir = path.join(publicDistDir, `web-static`);
+const releaseVersionEnv = process.env.RELEASE_VERSION || '';
+
+console.log('build with following dir', {
+  repoRootDir,
+  electronRootDir,
+  publicDistDir,
+  affineSrcDir: affineCoreDir,
+  affineSrcOutDir: affineCoreOutDir,
+  publicAffineOutDir,
+});
+
+// step 0: check version match
+const electronPackageJson = require(`${electronRootDir}/package.json`);
+if (releaseVersionEnv && electronPackageJson.version !== releaseVersionEnv) {
+  throw new Error(
+    `Version mismatch, expected ${releaseVersionEnv} but got ${electronPackageJson.version}`
+  );
+}
+// copy web dist files to electron dist
+
+process.env.DISTRIBUTION = 'desktop';
+
+const cwd = repoRootDir;
+
+if (!process.env.SKIP_PLUGIN_BUILD) {
+  spawnSync('yarn', ['build:plugins'], {
+    stdio: 'inherit',
+    env: process.env,
+    cwd,
+  });
+}
+
+// step 1: build web dist
+if (!process.env.SKIP_WEB_BUILD) {
+  spawnSync('yarn', ['nx', 'build', '@affine/core'], {
+    stdio: 'inherit',
+    env: process.env,
+    cwd,
+  });
+
+  spawnSync('yarn', ['workspace', '@affine/electron', 'build'], {
+    stdio: 'inherit',
+    env: process.env,
+    cwd,
+  });
+
+  // step 1.5: amend sourceMappingURL to allow debugging in devtools
+  await glob('**/*.{js,css}', { cwd: affineCoreOutDir }).then(files => {
+    return files.map(async file => {
+      const dir = path.dirname(file);
+      const fullpath = path.join(affineCoreOutDir, file);
+      let content = await fs.readFile(fullpath, 'utf-8');
+      // replace # sourceMappingURL=76-6370cd185962bc89.js.map
+      // to      # sourceMappingURL=assets://./{dir}/76-6370cd185962bc89.js.map
+      content = content.replace(/# sourceMappingURL=(.*)\.map/g, (_, p1) => {
+        return `# sourceMappingURL=assets://./${dir}/${p1}.map`;
+      });
+      await fs.writeFile(fullpath, content);
+    });
+  });
+
+  await fs.move(affineCoreOutDir, publicAffineOutDir, { overwrite: true });
+}
+
+// step 2: update app-updater.yml content with build type in resources folder
+if (process.env.BUILD_TYPE === 'internal') {
+  const appUpdaterYml = path.join(publicDistDir, 'app-update.yml');
+  const appUpdaterYmlContent = await fs.readFile(appUpdaterYml, 'utf-8');
+  const newAppUpdaterYmlContent = appUpdaterYmlContent.replace(
+    'AFFiNE',
+    'AFFiNE-Releases'
+  );
+  await fs.writeFile(appUpdaterYml, newAppUpdaterYmlContent);
+}

packages/frontend/electron/scripts/generate-yml.js

@@ -0,0 +1,56 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const yml = {
+  version: process.env.RELEASE_VERSION ?? '0.0.0',
+  files: [],
+};
+
+const generateYml = platform => {
+  const regex = new RegExp(`^affine-.*-${platform}-.*.(exe|zip|dmg|AppImage)$`);
+  const files = fs.readdirSync(process.cwd()).filter(file => regex.test(file));
+  files.forEach(fileName => {
+    const filePath = path.join(process.cwd(), './', fileName);
+    try {
+      const fileData = fs.readFileSync(filePath);
+      const hash = crypto
+        .createHash('sha512')
+        .update(fileData)
+        .digest('base64');
+      const size = fs.statSync(filePath).size;
+
+      yml.files.push({
+        url: fileName,
+        sha512: hash,
+        size: size,
+      });
+    } catch (e) {}
+  });
+  yml.path = yml.files[0].url;
+  yml.sha512 = yml.files[0].sha512;
+  yml.releaseDate = new Date().toISOString();
+
+  const ymlStr =
+    `version: ${yml.version}\n` +
+    `files:\n` +
+    yml.files
+      .map(file => {
+        return (
+          `  - url: ${file.url}\n` +
+          `    sha512: ${file.sha512}\n` +
+          `    size: ${file.size}\n`
+        );
+      })
+      .join('') +
+    `path: ${yml.path}\n` +
+    `sha512: ${yml.sha512}\n` +
+    `releaseDate: ${yml.releaseDate}\n`;
+
+  const fileName = platform === 'windows' ? 'latest.yml' : 'latest-mac.yml';
+
+  fs.writeFileSync(fileName, ymlStr);
+};
+
+generateYml('windows');
+generateYml('macos');

packages/frontend/electron/scripts/macos-arm64-output-check.mts

@@ -0,0 +1,26 @@
+import { readdir } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+
+const outputRoot = fileURLToPath(
+  new URL(
+    '../out/canary/AFFiNE-canary-darwin-arm64/AFFiNE-canary.app/Contents/Resources/app',
+    import.meta.url
+  )
+);
+
+const outputList = [
+  ['dist', ['main.js', 'helper.js', 'preload.js', 'affine.darwin-arm64.node']],
+] as [entry: string, expected: string[]][];
+
+await Promise.all(
+  outputList.map(async ([entry, output]) => {
+    const files = await readdir(`${outputRoot}/${entry}`);
+    output.forEach(file => {
+      if (!files.includes(file)) {
+        throw new Error(`File ${entry}/${file} not found`);
+      }
+    });
+  })
+);
+
+console.log('Output check passed');

packages/frontend/electron/scripts/make-env.ts

@@ -0,0 +1,54 @@
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { z } from 'zod';
+
+const ReleaseTypeSchema = z.enum(['stable', 'beta', 'canary', 'internal']);
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url));
+
+const ROOT = path.resolve(__dirname, '..');
+
+const envBuildType = (process.env.BUILD_TYPE || 'canary').trim().toLowerCase();
+const buildType = ReleaseTypeSchema.parse(envBuildType);
+const stableBuild = buildType === 'stable';
+const productName = !stableBuild ? `AFFiNE-${buildType}` : 'AFFiNE';
+const icoPath = path.join(
+  ROOT,
+  !stableBuild
+    ? `./resources/icons/icon_${buildType}.ico`
+    : './resources/icons/icon.ico'
+);
+
+const icnsPath = path.join(
+  ROOT,
+  !stableBuild
+    ? `./resources/icons/icon_${buildType}.icns`
+    : './resources/icons/icon.icns'
+);
+
+const iconPngPath = path.join(ROOT, './resources/icons/icon.png');
+
+const iconUrl = `https://cdn.affine.pro/app-icons/icon_${buildType}.ico`;
+const arch =
+  process.argv.indexOf('--arch') > 0
+    ? process.argv[process.argv.indexOf('--arch') + 1]
+    : process.arch;
+
+const platform =
+  process.argv.indexOf('--platform') > 0
+    ? process.argv[process.argv.indexOf('--platform') + 1]
+    : process.platform;
+
+export {
+  arch,
+  buildType,
+  icnsPath,
+  iconPngPath,
+  iconUrl,
+  icoPath,
+  platform,
+  productName,
+  ROOT,
+  stableBuild,
+};

packages/frontend/electron/scripts/make-squirrel.ts

@@ -0,0 +1,84 @@
+import type { Options as ElectronWinstallerOptions } from 'electron-winstaller';
+import { convertVersion, createWindowsInstaller } from 'electron-winstaller';
+import fs from 'fs-extra';
+import path from 'path';
+
+import {
+  arch,
+  buildType,
+  iconUrl,
+  icoPath,
+  platform,
+  productName,
+  ROOT,
+} from './make-env.js';
+
+async function ensureDirectory(dir: string) {
+  if (await fs.pathExists(dir)) {
+    await fs.remove(dir);
+  }
+  return fs.mkdirs(dir);
+}
+
+// taking from https://github.com/electron/forge/blob/main/packages/maker/squirrel/src/MakerSquirrel.ts
+// it was for forge's maker, but can be used standalone as well
+async function make() {
+  const appName = productName;
+  const makeDir = path.resolve(ROOT, 'out', buildType, 'make');
+  const outPath = path.resolve(makeDir, `squirrel.windows/${arch}`);
+  const appDirectory = path.resolve(
+    ROOT,
+    'out',
+    buildType,
+    `${appName}-${platform}-${arch}`
+  );
+  await ensureDirectory(outPath);
+
+  const packageJSON = await fs.readJson(path.resolve(ROOT, 'package.json'));
+
+  const winstallerConfig: ElectronWinstallerOptions = {
+    name: appName,
+    title: appName,
+    noMsi: true,
+    exe: `${appName}.exe`,
+    setupExe: `${appName}-${packageJSON.version} Setup.exe`,
+    version: packageJSON.version,
+    appDirectory: appDirectory,
+    outputDirectory: outPath,
+    iconUrl: iconUrl,
+    setupIcon: icoPath,
+    loadingGif: path.resolve(ROOT, './resources/icons/affine_installing.gif'),
+  };
+
+  await createWindowsInstaller(winstallerConfig);
+  const nupkgVersion = convertVersion(packageJSON.version);
+  const artifacts = [
+    path.resolve(outPath, 'RELEASES'),
+    path.resolve(outPath, winstallerConfig.setupExe || `${appName}Setup.exe`),
+    path.resolve(
+      outPath,
+      `${winstallerConfig.name}-${nupkgVersion}-full.nupkg`
+    ),
+  ];
+  const deltaPath = path.resolve(
+    outPath,
+    `${winstallerConfig.name}-${nupkgVersion}-delta.nupkg`
+  );
+  if (
+    (winstallerConfig.remoteReleases && !winstallerConfig.noDelta) ||
+    (await fs.pathExists(deltaPath))
+  ) {
+    artifacts.push(deltaPath);
+  }
+  const msiPath = path.resolve(
+    outPath,
+    winstallerConfig.setupMsi || `${appName}Setup.msi`
+  );
+  if (!winstallerConfig.noMsi && (await fs.pathExists(msiPath))) {
+    artifacts.push(msiPath);
+  }
+  console.log('making squirrel.windows done:', artifacts);
+  return artifacts;
+}
+
+make();

packages/frontend/electron/scripts/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}