chore: cli to create self signed ca to dev with domain (#12466)

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Introduced a CLI command to manage local Certificate Authority (CA) and generate SSL certificates for development domains.
  - Added example and template files for Nginx and OpenSSL configurations to support local development with SSL.
  - Provided new DNS and Nginx configuration files for enhanced local development setup.

- **Documentation**
  - Added a README with step-by-step instructions for setting up development containers and managing certificates on MacOS with OrbStack.

- **Chores**
  - Updated ignore patterns to exclude additional development files and directories.
  - Enhanced example Docker Compose files with commented service configurations and new volume definitions.
  - Removed the Elasticsearch example Docker Compose file.

- **Refactor**
  - Extended utility and command classes with new methods to support file operations and command execution.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.docker/dev/.gitignore

@@ -1,3 +1,6 @@
 postgres
 .env
-compose.yml
+compose.yml
+certs/*
+!certs/.gitkeep
+nginx/conf.d/*

.docker/dev/README.md

@@ -0,0 +1,27 @@
+# Dev containers
+
+## Develop with domain
+
+> MacOs only, OrbStack only
+
+### 1. Generate and install Root CA
+
+```bash
+# the root ca file will be located at `./.docker/dev/certs/ca`
+yarn affine cert --install
+```
+
+### 2. Generate domain certs
+
+```bash
+# certificates will be located at `./.docker/dev/certs/${domain}`
+yarn affine cert --domain dev.affine.fail
+```
+
+### 3. Enable dns and nginx service in compose.yml
+
+### 4. Add custom dns server
+
+```bash
+echo "nameserver 127.0.0.1" | sudo tee /etc/resolver/dev.affine.fail
+```

.docker/dev/compose.yml.elasticsearch.example

@@ -1,65 +0,0 @@
-name: affine_dev_services
-services:
-  postgres:
-    env_file:
-      - .env
-    image: pgvector/pgvector:pg${DB_VERSION:-16}
-    ports:
-      - 5432:5432
-    environment:
-      POSTGRES_PASSWORD: ${DB_PASSWORD}
-      POSTGRES_USER: ${DB_USERNAME}
-      POSTGRES_DB: ${DB_DATABASE_NAME}
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-
-  redis:
-    image: redis:latest
-    ports:
-      - 6379:6379
-
-  mailhog:
-    image: mailhog/mailhog:latest
-    ports:
-      - 1025:1025
-      - 8025:8025
-
-  elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.0.1}${ELASTIC_VERSION_ARM64}
-    platform: ${ELASTIC_PLATFORM}
-    labels:
-      co.elastic.logs/module: elasticsearch
-    volumes:
-      - elasticsearch_data:/usr/share/elasticsearch/data
-    ports:
-      - ${ES_PORT:-9200}:9200
-    environment:
-      - node.name=es01
-      - cluster.name=affine-dev
-      - discovery.type=single-node
-      - bootstrap.memory_lock=true
-      - xpack.security.enabled=false
-      - xpack.security.http.ssl.enabled=false
-      - xpack.security.transport.ssl.enabled=false
-      - xpack.license.self_generated.type=basic
-    mem_limit: ${ES_MEM_LIMIT:-1073741824}
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    healthcheck:
-      test:
-        [
-          "CMD-SHELL",
-          "curl -s http://localhost:9200 | grep -q 'affine-dev'",
-        ]
-      interval: 10s
-      timeout: 10s
-      retries: 120
-
-networks:
-  dev:
-
-volumes:
-  postgres_data:
-  elasticsearch_data:

.docker/dev/compose.yml.example

@@ -27,7 +27,6 @@ services:
   # https://manual.manticoresearch.com/Starting_the_server/Docker
   manticoresearch:
     image: manticoresearch/manticore:${MANTICORE_VERSION:-9.2.14}
-    restart: always
     ports:
       - 9308:9308
     ulimits:
@@ -40,6 +39,58 @@ services:
         hard: -1
     volumes:
       - manticoresearch_data:/var/lib/manticore
+  
+  # elasticsearch:
+  #   image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.0.1}${ELASTIC_VERSION_ARM64}
+  #   platform: ${ELASTIC_PLATFORM}
+  #   labels:
+  #     co.elastic.logs/module: elasticsearch
+  #   volumes:
+  #     - elasticsearch_data:/usr/share/elasticsearch/data
+  #   ports:
+  #     - ${ES_PORT:-9200}:9200
+  #   environment:
+  #     - node.name=es01
+  #     - cluster.name=affine-dev
+  #     - discovery.type=single-node
+  #     - bootstrap.memory_lock=true
+  #     - xpack.security.enabled=false
+  #     - xpack.security.http.ssl.enabled=false
+  #     - xpack.security.transport.ssl.enabled=false
+  #     - xpack.license.self_generated.type=basic
+  #   mem_limit: ${ES_MEM_LIMIT:-1073741824}
+  #   ulimits:
+  #     memlock:
+  #       soft: -1
+  #       hard: -1
+  #   healthcheck:
+  #     test:
+  #       [
+  #         "CMD-SHELL",
+  #         "curl -s http://localhost:9200 | grep -q 'affine-dev'",
+  #       ]
+  #     interval: 10s
+  #     timeout: 10s
+  #     retries: 120
+
+  # dns:
+  #   image: strm/dnsmasq
+  #   volumes:
+  #     - ./dnsmasq.conf:/etc/dnsmasq.d/local.conf
+  #   ports:
+  #     - "53:53/udp"
+  #   cap_add:
+  #     - NET_ADMIN
+  #   depends_on:
+  #     - nginx
+
+  # nginx:
+  #   image: nginx:alpine
+  #   volumes:
+  #     - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+  #     - ./nginx/conf.d:/etc/nginx/conf.d:ro
+  #     - ./certs:/etc/nginx/certs:ro
+  #   network_mode: host
 
 networks:
   dev:
@@ -47,3 +98,4 @@ networks:
 volumes:
   postgres_data:
   manticoresearch_data:
+  elasticsearch_data:

.docker/dev/dnsmasq.conf

@@ -0,0 +1,2 @@
+log-queries
+address=/dev.affine.fail/127.0.0.1

.docker/dev/nginx/nginx.conf

@@ -0,0 +1,28 @@
+user nginx;
+worker_processes auto;
+
+error_log   /var/log/nginx/error.log;
+pid         /var/run/nginx.pid;
+
+events {
+  worker_connections  1024;
+}
+
+http {
+  include mime.types;
+  default_type  application/octet-stream;
+
+  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+  access_log  /var/log/nginx/access.log  main;
+
+  sendfile on;
+  keepalive_timeout  65;
+  types_hash_max_size 2048;
+  client_max_body_size 512M;
+  server_names_hash_bucket_size 128;
+  ssi on;
+  gzip  on;
+  include "/etc/nginx/conf.d/*";
+}

.docker/dev/templates/nginx.conf

@@ -0,0 +1,27 @@
+server {
+  listen 80;
+  server_name DEV_DOMAIN;
+  
+  location / {
+    return 301 https://$host$request_uri;
+  }
+}
+
+server {
+  listen 443 ssl;
+  http2 on;
+  ssl_certificate /etc/nginx/certs/$host/crt;
+  ssl_certificate_key /etc/nginx/certs/$host/key;
+  server_name DEV_DOMAIN;
+
+  location / {
+    proxy_pass http://localhost:8080;
+    proxy_set_header   Host              $host;
+    proxy_set_header   X-Real-IP         $remote_addr;
+    proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+    proxy_set_header   X-Forwarded-Proto $scheme;
+    proxy_set_header   Upgrade           $http_upgrade;
+    proxy_set_header   Connection        "upgrade";
+    resolver 127.0.0.1;
+  }
+}

.docker/dev/templates/openssl.conf

@@ -0,0 +1,25 @@
+[req]
+distinguished_name = req_distinguished_name
+req_extensions = v3_req
+
+[req_distinguished_name]
+countryName = Country Name (2 letter code)
+countryName_default = US
+stateOrProvinceName = State or Province Name (full name)
+stateOrProvinceName_default = MN
+localityName = Locality Name (eg, city)
+localityName_default = Minneapolis
+organizationalUnitName	= Organizational Unit Name (eg, section)
+organizationalUnitName_default	= Domain Control Validated
+commonName = Internet Widgits Ltd
+commonName_max	= 64
+
+[ v3_req ]
+# Extensions to add to a certificate request
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = DEV_DOMAIN
+DNS.2 = *.DEV_DOMAIN

tools/cli/src/affine.ts

@@ -3,6 +3,7 @@ import { Cli } from 'clipanion';
 
 import { BuildCommand } from './build';
 import { BundleCommand } from './bundle';
+import { CertCommand } from './cert';
 import { CleanCommand } from './clean';
 import type { CliContext } from './context';
 import { DevCommand } from './dev';
@@ -23,6 +24,7 @@ cli.register(CleanCommand);
 cli.register(BuildCommand);
 cli.register(DevCommand);
 cli.register(BundleCommand);
+cli.register(CertCommand);
 
 await cli.runExit(process.argv.slice(2), {
   workspace: new Workspace(),

tools/cli/src/cert.ts

@@ -0,0 +1,124 @@
+import { type Path, ProjectRoot } from '@affine-tools/utils/path';
+
+import { Command, Option } from './command';
+
+const CERT_DIR = ProjectRoot.join('.docker/dev/certs');
+const CA_DIR = CERT_DIR.join('ca');
+const TEMPLATES_DIR = ProjectRoot.join('.docker/dev/templates');
+const NGINX_CONF_DIR = ProjectRoot.join('.docker/dev/nginx/conf.d');
+const CA_PEM_PATH = CA_DIR.join('affine-self-signed.pem');
+const CA_KEY_PATH = CA_DIR.join('affine-self-signed.key');
+
+const CA_ORG = 'AFFiNE Dev CA Self Signed Org';
+const CA_NAME = 'AFFiNE Dev CA Self Signed CN';
+
+export class CertCommand extends Command {
+  static override paths = [['cert']];
+
+  install = Option.Boolean('--install', {
+    description: 'Install the CA',
+  });
+
+  domain = Option.String('--domain', {
+    description:
+      'Generate certificates for given domain. e.g. "dev.affine.fail"',
+  });
+
+  uninstall = Option.Boolean('--uninstall', {
+    description: 'Uninstall the CA',
+  });
+
+  async execute() {
+    if (this.install) {
+      this.installCa();
+    } else if (this.uninstall) {
+      this.uninstallCa(CA_PEM_PATH);
+    } else if (this.domain) {
+      this.createCert(this.domain);
+    }
+  }
+
+  private createCert(domain: string) {
+    if (!this.checkInstalled(CA_PEM_PATH)) {
+      this.logger.error(
+        'CA not installed. Please run `yarn affine cert --install` first.'
+      );
+      process.exit(1);
+    }
+
+    const domainDir = CERT_DIR.join(domain);
+    domainDir.rm({ recursive: true });
+    domainDir.mkdir();
+    NGINX_CONF_DIR.mkdir();
+
+    const keyPath = domainDir.join('key');
+    const crtPath = domainDir.join('crt');
+    const csrPath = domainDir.join('csr');
+    const confPath = domainDir.join('conf');
+    const nginxConfPath = NGINX_CONF_DIR.join(`${domain}.conf`);
+
+    const configTemp = TEMPLATES_DIR.join('openssl.conf')
+      .readAsFile()
+      .toString('utf-8');
+    const config = configTemp.replaceAll('DEV_DOMAIN', domain);
+
+    confPath.writeFile(config);
+    this.exec(`openssl genrsa -out ${keyPath} 2048`);
+    this.exec(
+      `openssl req -new -key ${keyPath} -out ${csrPath} -config ${confPath} -subj "/C=/ST=/O=/localityName=/commonName=${domain}/organizationalUnitName=/emailAddress=${domain}@affine.pro/"`
+    );
+    this.exec(
+      `openssl x509 -req -days 1024 -in ${csrPath} -CA ${CA_PEM_PATH} -CAkey ${CA_KEY_PATH} -CAcreateserial -out ${crtPath} -extensions v3_req -extfile ${confPath}`
+    );
+
+    const nginxConfTemp = TEMPLATES_DIR.join('nginx.conf')
+      .readAsFile()
+      .toString('utf-8');
+    const nginxConf = nginxConfTemp.replaceAll('DEV_DOMAIN', domain);
+    nginxConfPath.writeFile(nginxConf);
+  }
+
+  private installCa() {
+    if (this.checkInstalled(CA_PEM_PATH)) {
+      return;
+    }
+
+    // remove old ca
+    CA_PEM_PATH.rm();
+    CA_KEY_PATH.rm();
+    CA_DIR.mkdir();
+
+    this.exec(
+      `openssl req -new -newkey rsa:2048 -days 1024 -nodes -x509 -subj "/C=/ST=/O=${CA_ORG}/localityName=/commonName=${CA_NAME}/organizationalUnitName=Developers/emailAddress=dev@affine.pro/" -keyout ${CA_KEY_PATH} -out ${CA_PEM_PATH}`
+    );
+    this.trustCa(CA_PEM_PATH);
+  }
+
+  private trustCa(pem: Path) {
+    this.logger.info(`Trusting AFFiNE Dev Self Signed CA`);
+    this.exec(
+      `sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ${pem}`
+    );
+  }
+
+  private uninstallCa(pem: Path) {
+    if (!this.checkInstalled(pem)) {
+      this.logger.error('CA not installed');
+      return;
+    }
+
+    this.exec(
+      `sudo security delete-certificate -c ${CA_NAME} /Library/Keychains/System.keychain`
+    );
+  }
+
+  private checkInstalled(pem: Path) {
+    if (!pem.exists()) {
+      return false;
+    }
+
+    const ret = this.exec(`security verify-cert -c ${pem}`);
+
+    return ret.includes('...certificate verification successful.');
+  }
+}

tools/cli/src/command.ts

@@ -1,5 +1,6 @@
 import { AliasToPackage } from '@affine-tools/utils/distribution';
 import { Logger } from '@affine-tools/utils/logger';
+import { exec, execAsync, spawn } from '@affine-tools/utils/process';
 import { type PackageName, Workspace } from '@affine-tools/utils/workspace';
 import { Command as BaseCommand, Option } from 'clipanion';
 import inquirer from 'inquirer';
@@ -8,9 +9,11 @@ import * as t from 'typanion';
 import type { CliContext } from './context';
 
 export abstract class Command extends BaseCommand<CliContext> {
+  // @ts-expect-error hack: Get the command name
+  cmd = this.constructor.paths[0][0];
+
   get logger() {
-    // @ts-expect-error hack: Get the command name
-    return new Logger(this.constructor.paths[0][0]);
+    return new Logger(this.cmd);
   }
 
   get workspace() {
@@ -20,6 +23,10 @@ export abstract class Command extends BaseCommand<CliContext> {
   set workspace(workspace: Workspace) {
     this.context.workspace = workspace;
   }
+
+  exec = exec.bind(null, this.cmd);
+  execAsync = execAsync.bind(null, this.cmd);
+  spawn = spawn.bind(null, this.cmd);
 }
 
 export abstract class PackageCommand extends Command {

tools/utils/src/path.ts

@@ -1,4 +1,11 @@
-import { existsSync, statSync } from 'node:fs';
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
 import { join, relative, sep } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 
@@ -41,6 +48,22 @@ export class Path {
     return existsSync(this.path);
   }
 
+  rm(opts: { recursive?: boolean } = {}) {
+    rmSync(this.path, { ...opts, force: true });
+  }
+
+  mkdir() {
+    mkdirSync(this.path, { recursive: true });
+  }
+
+  readAsFile() {
+    return readFileSync(this.path);
+  }
+
+  writeFile(content: Buffer | string) {
+    writeFileSync(this.path, content);
+  }
+
   stats() {
     return statSync(this.path);
   }