refactor(server): better selfhost deployment (#9036)

This commit is contained in:
forehalo
2024-12-06 01:37:26 +00:00
parent f96f08dcec
commit 0a1fa8911f
24 changed files with 226 additions and 167 deletions
+23
View File
@@ -0,0 +1,23 @@
# select a revision to deploy, available values: stable, beta, canary
AFFINE_REVISION=stable
# set the port for the server container it will expose the server on
PORT=3010
# set the host for the server for outgoing links
# AFFINE_SERVER_HTTPS=true
# AFFINE_SERVER_HOST=affine.yourdomain.com
# or
# AFFINE_SERVER_EXTERNAL_URL=https://affine.yourdomain.com
# position of the database data to persist
DB_DATA_LOCATION=~/.affine/self-host/postgres
# position of the upload data(images, files, etc.) to persist
UPLOAD_LOCATION=~/.affine/self-host/storage
# position of the configuration files to persist
CONFIG_LOCATION=~/.affine/self-host/config
# database credentials
DB_USERNAME=affine
DB_PASSWORD=
DB_DATABASE=affine
+47 -41
View File
@@ -1,60 +1,66 @@
name: affine
services:
affine:
image: ghcr.io/toeverything/affine-graphql:stable
container_name: affine_selfhosted
command:
['sh', '-c', 'node ./scripts/self-host-predeploy && node ./dist/index.js']
image: ghcr.io/toeverything/affine-graphql:${AFFINE_REVISION:-stable}
container_name: affine_server
ports:
- '3010:3010'
- '5555:5555'
- '${PORT:-3010}:3010'
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
affine_migration:
condition: service_completed_successfully
volumes:
# custom configurations
- ~/.affine/self-host/config:/root/.affine/config
# blob storage
- ~/.affine/self-host/storage:/root/.affine/storage
logging:
driver: 'json-file'
options:
max-size: '1000m'
restart: unless-stopped
- ${UPLOAD_LOCATION}:/root/.affine/storage
- ${CONFIG_LOCATION}:/root/.affine/config
env_file:
- .env
environment:
- NODE_OPTIONS="--import=./scripts/register.js"
- AFFINE_CONFIG_PATH=/root/.affine/config
- REDIS_SERVER_HOST=redis
- DATABASE_URL=postgres://affine:affine@postgres:5432/affine
- NODE_ENV=production
# Telemetry allows us to collect data on how you use the affine. This data will helps us improve the app and provide better features.
# Uncomment next line if you wish to quit telemetry.
# - TELEMETRY_ENABLE=false
- DATABASE_URL=postgresql://${DB_USERNAME}:${DB_PASSWORD}@postgres:5432/${DB_DATABASE:-affine}
restart: unless-stopped
affine_migration:
image: ghcr.io/toeverything/affine-graphql:${AFFINE_REVISION:-stable}
container_name: affine_migration_job
volumes:
# custom configurations
- ${UPLOAD_LOCATION}:/root/.affine/storage
- ${CONFIG_LOCATION}:/root/.affine/config
command: ['sh', '-c', 'node ./scripts/self-host-predeploy.js']
env_file:
- .env
environment:
- DATABASE_URL=postgresql://${DB_USERNAME}:${DB_PASSWORD}@postgres:5432/${DB_DATABASE:-affine}
depends_on:
postgres:
condition: service_healthy
redis:
image: redis
container_name: affine_redis
restart: unless-stopped
volumes:
- ~/.affine/self-host/redis:/data
container_name: redis
healthcheck:
test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping']
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped
postgres:
image: postgres:16
container_name: affine_postgres
restart: unless-stopped
container_name: postgres
volumes:
- ~/.affine/self-host/postgres:/var/lib/postgresql/data
healthcheck:
test: ['CMD-SHELL', 'pg_isready -U affine']
interval: 10s
timeout: 5s
retries: 5
- ${DB_DATA_LOCATION}:/var/lib/postgresql/data
environment:
POSTGRES_USER: affine
POSTGRES_PASSWORD: affine
POSTGRES_DB: affine
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_USER: ${DB_USERNAME}
POSTGRES_PASSWORD: ${DB_PASSWORD}
POSTGRES_DB: ${DB_DATABASE:-affine}
POSTGRES_INITDB_ARGS: '--data-checksums'
# you better set a password for you database
# or you may add 'POSTGRES_HOST_AUTH_METHOD=trust' to ignore postgres security policy
POSTGRES_HOST_AUTH_METHOD: trust
healthcheck:
test: ['CMD-SHELL', 'pg_isready']
interval: 1m
start_interval: 10s
start_period: 1m
restart: unless-stopped
+12 -26
View File
@@ -415,35 +415,39 @@ jobs:
uses: actions/download-artifact@v4
with:
name: affine-darwin-x64-builds
path: ./
path: ./release
- name: Download Artifacts (macos-arm64)
uses: actions/download-artifact@v4
with:
name: affine-darwin-arm64-builds
path: ./
path: ./release
- name: Download Artifacts (windows-x64)
uses: actions/download-artifact@v4
with:
name: affine-win32-x64-builds
path: ./
path: ./release
- name: Download Artifacts (windows-arm64)
uses: actions/download-artifact@v4
with:
name: affine-win32-arm64-builds
path: ./
path: ./release
- name: Download Artifacts (linux-x64)
uses: actions/download-artifact@v4
with:
name: affine-linux-x64-builds
path: ./
path: ./release
- uses: actions/setup-node@v4
with:
node-version: 20
- name: Generate Release yml
run: |
node ./packages/frontend/apps/electron/scripts/generate-yml.js
node ./scripts/generate-release-yml.js
env:
RELEASE_VERSION: ${{ needs.before-make.outputs.RELEASE_VERSION }}
- name: Copy Selfhost Release Files
run: |
cp ./.github/deployment/self-host/compose.yaml ./release/docker-compose.yaml
cp ./.github/deployment/self-host/.env.example ./release/.env.example
- name: Create Release Draft
if: ${{ github.ref_type == 'tag' }}
uses: softprops/action-gh-release@v2
@@ -452,16 +456,7 @@ jobs:
body: ''
draft: ${{ github.event.inputs.is-draft }}
prerelease: ${{ github.event.inputs.is-pre-release }}
files: |
./VERSION
./*.zip
./*.dmg
./*.exe
./*.appimage
./*.deb
./*.flatpak
./*.apk
./*.yml
files: ./release/*
- name: Create Nightly Release Draft
if: ${{ github.ref_type == 'branch' }}
uses: softprops/action-gh-release@v2
@@ -476,13 +471,4 @@ jobs:
body: ''
draft: false
prerelease: true
files: |
./VERSION
./*.zip
./*.dmg
./*.exe
./*.appimage
./*.deb
./*.apk
./*.flatpak
./*.yml
files: ./release/*
+2
View File
@@ -0,0 +1,2 @@
# required DATABASE_URL affects app start
schema.prisma
+1
View File
@@ -165,6 +165,7 @@
"*.gen.*"
],
"env": {
"NODE_ENV": "development",
"AFFINE_SERVER_EXTERNAL_URL": "http://localhost:8080",
"TS_NODE_TRANSPILE_ONLY": true,
"TS_NODE_PROJECT": "./tsconfig.json",
@@ -3,61 +3,47 @@ import { generateKeyPairSync } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { parse } from 'dotenv';
const SELF_HOST_CONFIG_DIR = '/root/.affine/config';
/**
* @type {Array<{ from: string; to?: string, modifier?: (content: string): string }>}
*/
const configFiles = [
{ from: './.env.example', to: '.env' },
{ from: './dist/config/affine.js', modifier: configCleaner },
{ from: './dist/config/affine.env.js', modifier: configCleaner },
];
function configCleaner(content) {
function generateConfigFile() {
const content = fs.readFileSync('./dist/config/affine.js', 'utf-8');
return content.replace(
/(^\/\/#.*$)|(^\/\/\s+TODO.*$)|("use\sstrict";?)|(^.*eslint-disable.*$)/gm,
''
);
}
function generatePrivateKey() {
const key = generateKeyPairSync('ec', {
namedCurve: 'prime256v1',
}).privateKey.export({
type: 'sec1',
format: 'pem',
});
if (key instanceof Buffer) {
return key.toString('utf-8');
}
return key;
}
/**
* @type {Array<{ to: string; generator: () => string }>}
*/
const configFiles = [
{ to: 'affine.js', generator: generateConfigFile },
{ to: 'private.key', generator: generatePrivateKey },
];
function prepare() {
fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
for (const { from, to, modifier } of configFiles) {
const targetFileName = to ?? path.parse(from).base;
const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, targetFileName);
for (const { to, generator } of configFiles) {
const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, to);
if (!fs.existsSync(targetFilePath)) {
console.log(`creating config file [${targetFilePath}].`);
if (modifier) {
const content = fs.readFileSync(from, 'utf-8');
fs.writeFileSync(targetFilePath, modifier(content), 'utf-8');
} else {
fs.cpSync(from, targetFilePath, {
force: false,
});
}
}
// make the default .env
if (to === '.env') {
const dotenvFile = fs.readFileSync(targetFilePath, 'utf-8');
const envs = parse(dotenvFile);
// generate a new private key
if (!envs.AFFINE_PRIVATE_KEY) {
const privateKey = generateKeyPairSync('ec', {
namedCurve: 'prime256v1',
}).privateKey.export({
type: 'sec1',
format: 'pem',
});
fs.writeFileSync(
targetFilePath,
`AFFINE_PRIVATE_KEY=${privateKey}\n` + dotenvFile
);
}
fs.writeFileSync(targetFilePath, generator(), 'utf-8');
}
}
}
@@ -12,7 +12,7 @@ AFFiNE.ENV_MAP = {
MAILER_PASSWORD: 'mailer.auth.pass',
MAILER_SENDER: 'mailer.from.address',
MAILER_SECURE: ['mailer.secure', 'boolean'],
DATABASE_URL: 'database.datasourceUrl',
DATABASE_URL: 'prisma.datasourceUrl',
OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId',
OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret',
OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId',
@@ -17,6 +17,11 @@
// ====================================================================================
const env = process.env;
AFFiNE.serverName = AFFiNE.affine.canary
? 'AFFiNE Canary Cloud'
: AFFiNE.affine.beta
? 'AFFiNE Beta Cloud'
: 'AFFiNE Cloud';
AFFiNE.metrics.enabled = !AFFiNE.node.test;
if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
+1 -11
View File
@@ -8,22 +8,12 @@
// Any changes in this file won't take effect before server restarted.
//
//
// > Configurations merge order
// 1. load environment variables (`.env` if provided, and from system)
// 2. load `src/fundamentals/config/default.ts` for all default settings
// 3. apply `./affine.ts` patches (this file)
// 4. apply `./affine.env.ts` patches
//
//
// ###############################################################
// ## General settings ##
// ###############################################################
//
// /* The unique identity of the server */
// AFFiNE.serverId = 'some-randome-uuid';
//
// /* The name of AFFiNE Server, may show on the UI */
// AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud';
AFFiNE.serverName = 'My Selfhosted AFFiNE Cloud';
//
// /* Whether the server is deployed behind a HTTPS proxied environment */
AFFiNE.server.https = false;
@@ -261,7 +261,7 @@ export class ServerServiceConfigResolver {
}
database(): ServerDatabaseConfig {
const url = new URL(this.config.database.datasourceUrl);
const url = new URL(this.config.prisma.datasourceUrl);
return {
host: url.hostname,
@@ -14,7 +14,7 @@ import { readEnv } from './env';
import { defaultStartupConfig } from './register';
function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'development', [
const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'production', [
'development',
'test',
'production',
@@ -29,7 +29,7 @@ export {
mapSseError,
OptionalModule,
} from './nestjs';
export type { PrismaTransaction } from './prisma';
export { type PrismaTransaction } from './prisma';
export * from './storage';
export { type StorageProvider, StorageProviderFactory } from './storage';
export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler';
@@ -8,10 +8,10 @@ interface PrismaStartupConfiguration extends Prisma.PrismaClientOptions {
declare module '../config' {
interface AppConfig {
database: ModuleConfig<PrismaStartupConfiguration>;
prisma: ModuleConfig<PrismaStartupConfiguration>;
}
}
defineStartupConfig('database', {
defineStartupConfig('prisma', {
datasourceUrl: '',
});
@@ -14,7 +14,7 @@ const clientProvider: Provider = {
return PrismaService.INSTANCE;
}
return new PrismaService(config.database);
return new PrismaService(config.prisma);
},
inject: [Config],
};
+38 -23
View File
@@ -1,6 +1,7 @@
import 'reflect-metadata';
import { cpSync } from 'node:fs';
import { cpSync, existsSync, readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { join, parse } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -10,42 +11,57 @@ import {
applyEnvToConfig,
getAFFiNEConfigModifier,
} from './fundamentals/config';
import { enablePlugin } from './plugins';
const PROJECT_CONFIG_PATH = join(fileURLToPath(import.meta.url), '../config');
async function loadRemote(remoteDir: string, file: string) {
let fileToLoad = join(PROJECT_CONFIG_PATH, file);
const CUSTOM_CONFIG_PATH = `${homedir()}/.affine/config`;
if (PROJECT_CONFIG_PATH !== remoteDir) {
const remoteFile = join(remoteDir, file);
async function loadConfig(configDir: string, file: string) {
let fileToLoad: string | undefined;
if (PROJECT_CONFIG_PATH !== configDir) {
const remoteFile = join(configDir, file);
const remoteFileAtLocal = join(
PROJECT_CONFIG_PATH,
parse(file).name + '.remote.js'
);
cpSync(remoteFile, remoteFileAtLocal, {
force: true,
});
fileToLoad = remoteFileAtLocal;
if (existsSync(remoteFile)) {
cpSync(remoteFile, remoteFileAtLocal, {
force: true,
});
fileToLoad = remoteFileAtLocal;
}
} else {
fileToLoad = join(PROJECT_CONFIG_PATH, file);
}
await import(pathToFileURL(fileToLoad).href);
if (fileToLoad) {
await import(pathToFileURL(fileToLoad).href);
}
}
function loadPrivateKey() {
const file = join(CUSTOM_CONFIG_PATH, 'private.key');
if (!process.env.AFFINE_PRIVATE_KEY && existsSync(file)) {
const privateKey = readFileSync(file, 'utf-8');
process.env.AFFINE_PRIVATE_KEY = privateKey;
}
}
async function load() {
const AFFiNE_CONFIG_PATH =
process.env.AFFINE_CONFIG_PATH ?? PROJECT_CONFIG_PATH;
// Initializing AFFiNE config
//
// 1. load dotenv file to `process.env`
// load `.env` under pwd
config();
// @deprecated removed
// load `.env` under user config folder
config({
path: join(AFFiNE_CONFIG_PATH, '.env'),
path: join(CUSTOM_CONFIG_PATH, '.env'),
});
// 2. generate AFFiNE default config and assign to `globalThis.AFFiNE`
globalThis.AFFiNE = getAFFiNEConfigModifier();
const { enablePlugin } = await import('./plugins/registry');
globalThis.AFFiNE.use = enablePlugin;
globalThis.AFFiNE.plugins.use = enablePlugin;
@@ -53,24 +69,23 @@ async function load() {
// Modules may contribute to ENV_MAP, figure out a good way to involve them instead of hardcoding in `./config/affine.env`
// 3. load env => config map to `globalThis.AFFiNE.ENV_MAP
// load local env map as well in case there are new env added
await loadRemote(PROJECT_CONFIG_PATH, 'affine.env.js');
const projectEnvMap = AFFiNE.ENV_MAP;
await loadRemote(AFFiNE_CONFIG_PATH, 'affine.env.js');
const customEnvMap = AFFiNE.ENV_MAP;
AFFiNE.ENV_MAP = { ...projectEnvMap, ...customEnvMap };
await loadConfig(PROJECT_CONFIG_PATH, 'affine.env');
// 4. load `config/affine` to patch custom configs
// load local config as well in case there are new default configurations added
await loadRemote(PROJECT_CONFIG_PATH, 'affine.js');
await loadRemote(AFFiNE_CONFIG_PATH, 'affine.js');
await loadConfig(PROJECT_CONFIG_PATH, 'affine');
await loadConfig(CUSTOM_CONFIG_PATH, 'affine');
// 5. load `config/affine.self` to patch custom configs
// This is the file only take effect in [AFFiNE Cloud]
if (!AFFiNE.isSelfhosted) {
await loadRemote(PROJECT_CONFIG_PATH, 'affine.self.js');
await loadConfig(PROJECT_CONFIG_PATH, 'affine.self');
}
// 6. apply `process.env` map overriding to `globalThis.AFFiNE`
// 6. load `config/private.key` to patch app configs
loadPrivateKey();
// 7. apply `process.env` map overriding to `globalThis.AFFiNE`
applyEnvToConfig(globalThis.AFFiNE);
}
+1
View File
@@ -17,6 +17,7 @@ export type BUILD_CONFIG_TYPE = {
isMobileWeb: boolean;
isIOS: boolean;
isAndroid: boolean;
isAdmin: boolean;
// this is for the electron app
/**
+1
View File
@@ -34,6 +34,7 @@
"@radix-ui/react-tooltip": "^1.1.1",
"@sentry/react": "^8.9.0",
"@tanstack/react-table": "^8.19.3",
"@toeverything/infra": "workspace:*",
"cmdk": "^1.0.0",
"embla-carousel-react": "^8.1.5",
"input-otp": "^1.2.4",
+46 -11
View File
@@ -1,5 +1,20 @@
import { Toaster } from '@affine/admin/components/ui/sonner';
import {
configureCloudModule,
DefaultServerService,
} from '@affine/core/modules/cloud';
import { configureLocalStorageStateStorageImpls } from '@affine/core/modules/storage';
import { configureUrlModule } from '@affine/core/modules/url';
import { wrapCreateBrowserRouter } from '@sentry/react';
import {
configureGlobalContextModule,
configureGlobalStorageModule,
configureLifecycleModule,
Framework,
FrameworkRoot,
FrameworkScope,
LifecycleService,
} from '@toeverything/infra';
import { useEffect } from 'react';
import {
createBrowserRouter as reactRouterCreateBrowserRouter,
@@ -109,18 +124,38 @@ export const router = _createBrowserRouter(
}
);
const framework = new Framework();
configureLifecycleModule(framework);
configureLocalStorageStateStorageImpls(framework);
configureGlobalStorageModule(framework);
configureGlobalContextModule(framework);
configureUrlModule(framework);
configureCloudModule(framework);
const frameworkProvider = framework.provider();
// setup application lifecycle events, and emit application start event
window.addEventListener('focus', () => {
frameworkProvider.get(LifecycleService).applicationFocus();
});
frameworkProvider.get(LifecycleService).applicationStart();
const serverService = frameworkProvider.get(DefaultServerService);
export const App = () => {
return (
<TooltipProvider>
<SWRConfig
value={{
revalidateOnFocus: false,
revalidateOnMount: false,
}}
>
<RouterProvider router={router} />
</SWRConfig>
<Toaster />
</TooltipProvider>
<FrameworkRoot framework={frameworkProvider}>
<FrameworkScope scope={serverService.server.scope}>
<TooltipProvider>
<SWRConfig
value={{
revalidateOnFocus: false,
revalidateOnMount: false,
}}
>
<RouterProvider router={router} />
</SWRConfig>
<Toaster />
</TooltipProvider>
</FrameworkScope>
</FrameworkRoot>
);
};
+5 -1
View File
@@ -8,6 +8,10 @@
"verbatimModuleSyntax": false,
"jsx": "react-jsx"
},
"references": [{ "path": "../core" }, { "path": "../graphql" }],
"references": [
{ "path": "../core" },
{ "path": "../graphql" },
{ "path": "../../common/infra" }
],
"exclude": ["dist"]
}
@@ -9,7 +9,7 @@ import type { GraphQLError } from 'graphql';
import { useCallback, useMemo } from 'react';
import type { SWRConfiguration, SWRResponse } from 'swr';
import useSWR from 'swr';
import useSWRImutable from 'swr/immutable';
import useSWRImmutable from 'swr/immutable';
import useSWRInfinite from 'swr/infinite';
/**
@@ -60,7 +60,7 @@ const createUseQuery =
);
const graphqlService = useService(GraphQLService);
const useSWRFn = immutable ? useSWRImutable : useSWR;
const useSWRFn = immutable ? useSWRImmutable : useSWR;
return useSWRFn(
options ? () => ['cloud', options.query.id, options.variables] : null,
options ? () => graphqlService.gql(options) : null,
@@ -2,6 +2,8 @@ import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
const releaseDir = path.join(process.cwd(), './release');
const filenamesMapping = {
all: 'latest.yml',
macos: 'latest-mac.yml',
@@ -22,11 +24,11 @@ const generateYml = platform => {
? new RegExp(`.(${releaseFiles.join('|')})$`)
: new RegExp(`.+-${platform}-.+.(${releaseFiles.join('|')})$`);
const files = fs.readdirSync(process.cwd()).filter(file => regex.test(file));
const files = fs.readdirSync(releaseDir).filter(file => regex.test(file));
const outputFileName = filenamesMapping[platform];
files.forEach(fileName => {
const filePath = path.join(process.cwd(), './', fileName);
const filePath = path.join(releaseDir, fileName);
try {
const fileData = fs.readFileSync(filePath);
const hash = crypto
+1 -1
View File
@@ -341,7 +341,7 @@ export const createConfiguration: (
],
}),
buildFlags.mode === 'production' &&
(buildConfig.isWeb || buildConfig.isMobileWeb) &&
(buildConfig.isWeb || buildConfig.isMobileWeb || buildConfig.isAdmin) &&
process.env.R2_SECRET_ACCESS_KEY
? new WebpackS3Plugin()
: null,
+1
View File
@@ -20,6 +20,7 @@ export function getBuildConfig(buildFlags: BuildFlags): BUILD_CONFIG_TYPE {
isMobileWeb: buildFlags.distribution === 'mobile',
isIOS: buildFlags.distribution === 'ios',
isAndroid: buildFlags.distribution === 'android',
isAdmin: buildFlags.distribution === 'admin',
isSelfHosted: process.env.SELF_HOSTED === 'true',
appBuildType: 'stable' as const,
+1
View File
@@ -181,6 +181,7 @@ __metadata:
"@radix-ui/react-tooltip": "npm:^1.1.1"
"@sentry/react": "npm:^8.9.0"
"@tanstack/react-table": "npm:^8.19.3"
"@toeverything/infra": "workspace:*"
class-variance-authority: "npm:^0.7.0"
clsx: "npm:^2.1.1"
cmdk: "npm:^1.0.0"