Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-13 21:05:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(server): better selfhost deployment (#9036)

Browse Source
This commit is contained in:
forehalo
2024-12-06 01:37:26 +00:00
parent f96f08dcec
commit 0a1fa8911f
24 changed files with 226 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend/server/.dockerignore Normal file
View File

@@ -0,0 +1,2 @@
# required DATABASE_URL affects app start
schema.prisma

1
packages/backend/server/package.json
View File

@@ -165,6 +165,7 @@
"*.gen.*"
],
"env": {
"NODE_ENV": "development",
"AFFINE_SERVER_EXTERNAL_URL": "http://localhost:8080",
"TS_NODE_TRANSPILE_ONLY": true,
"TS_NODE_PROJECT": "./tsconfig.json",

70
packages/backend/server/scripts/self-host-predeploy.js
View File

@@ -3,61 +3,47 @@ import { generateKeyPairSync } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { parse } from 'dotenv';
const SELF_HOST_CONFIG_DIR = '/root/.affine/config';
/**
* @type {Array<{ from: string; to?: string, modifier?: (content: string): string }>}
*/
const configFiles = [
{ from: './.env.example', to: '.env' },
{ from: './dist/config/affine.js', modifier: configCleaner },
{ from: './dist/config/affine.env.js', modifier: configCleaner },
];
function configCleaner(content) {
function generateConfigFile() {
const content = fs.readFileSync('./dist/config/affine.js', 'utf-8');
return content.replace(
/(^\/\/#.*$)|(^\/\/\s+TODO.*$)|("use\sstrict";?)|(^.*eslint-disable.*$)/gm,
''
);
}
function generatePrivateKey() {
const key = generateKeyPairSync('ec', {
namedCurve: 'prime256v1',
}).privateKey.export({
type: 'sec1',
format: 'pem',
});
if (key instanceof Buffer) {
return key.toString('utf-8');
}
return key;
}
/**
* @type {Array<{ to: string; generator: () => string }>}
*/
const configFiles = [
{ to: 'affine.js', generator: generateConfigFile },
{ to: 'private.key', generator: generatePrivateKey },
];
function prepare() {
fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true });
for (const { from, to, modifier } of configFiles) {
const targetFileName = to ?? path.parse(from).base;
const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, targetFileName);
for (const { to, generator } of configFiles) {
const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, to);
if (!fs.existsSync(targetFilePath)) {
console.log(`creating config file [${targetFilePath}].`);
if (modifier) {
const content = fs.readFileSync(from, 'utf-8');
fs.writeFileSync(targetFilePath, modifier(content), 'utf-8');
} else {
fs.cpSync(from, targetFilePath, {
force: false,
});
}
}
// make the default .env
if (to === '.env') {
const dotenvFile = fs.readFileSync(targetFilePath, 'utf-8');
const envs = parse(dotenvFile);
// generate a new private key
if (!envs.AFFINE_PRIVATE_KEY) {
const privateKey = generateKeyPairSync('ec', {
namedCurve: 'prime256v1',
}).privateKey.export({
type: 'sec1',
format: 'pem',
});
fs.writeFileSync(
targetFilePath,
`AFFINE_PRIVATE_KEY=${privateKey}\n` + dotenvFile
);
}
fs.writeFileSync(targetFilePath, generator(), 'utf-8');
}
}
}

2
packages/backend/server/src/config/affine.env.ts
View File

@@ -12,7 +12,7 @@ AFFiNE.ENV_MAP = {
MAILER_PASSWORD: 'mailer.auth.pass',
MAILER_SENDER: 'mailer.from.address',
MAILER_SECURE: ['mailer.secure', 'boolean'],
DATABASE_URL: 'database.datasourceUrl',
DATABASE_URL: 'prisma.datasourceUrl',
OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId',
OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret',
OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId',

5
packages/backend/server/src/config/affine.self.ts
View File

@@ -17,6 +17,11 @@
// ====================================================================================
const env = process.env;
AFFiNE.serverName = AFFiNE.affine.canary
? 'AFFiNE Canary Cloud'
: AFFiNE.affine.beta
? 'AFFiNE Beta Cloud'
: 'AFFiNE Cloud';
AFFiNE.metrics.enabled = !AFFiNE.node.test;
if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) {

12
packages/backend/server/src/config/affine.ts
View File

@@ -8,22 +8,12 @@
// Any changes in this file won't take effect before server restarted.
//
//
// > Configurations merge order
// 1. load environment variables (`.env` if provided, and from system)
// 2. load `src/fundamentals/config/default.ts` for all default settings
// 3. apply `./affine.ts` patches (this file)
// 4. apply `./affine.env.ts` patches
//
//
// ###############################################################
// ## General settings ##
// ###############################################################
//
// /* The unique identity of the server */
// AFFiNE.serverId = 'some-randome-uuid';
//
// /* The name of AFFiNE Server, may show on the UI */
// AFFiNE.serverName = 'Your Cool AFFiNE Selfhosted Cloud';
AFFiNE.serverName = 'My Selfhosted AFFiNE Cloud';
//
// /* Whether the server is deployed behind a HTTPS proxied environment */
AFFiNE.server.https = false;

2
packages/backend/server/src/core/config/resolver.ts
View File

@@ -261,7 +261,7 @@ export class ServerServiceConfigResolver {
}
database(): ServerDatabaseConfig {
const url = new URL(this.config.database.datasourceUrl);
const url = new URL(this.config.prisma.datasourceUrl);
return {
host: url.hostname,

2
packages/backend/server/src/fundamentals/config/default.ts
View File

@@ -14,7 +14,7 @@ import { readEnv } from './env';
import { defaultStartupConfig } from './register';
function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'development', [
const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'production', [
'development',
'test',
'production',

2
packages/backend/server/src/fundamentals/index.ts
View File

@@ -29,7 +29,7 @@ export {
mapSseError,
OptionalModule,
} from './nestjs';
export type { PrismaTransaction } from './prisma';
export { type PrismaTransaction } from './prisma';
export * from './storage';
export { type StorageProvider, StorageProviderFactory } from './storage';
export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler';

4
packages/backend/server/src/fundamentals/prisma/config.ts
View File

@@ -8,10 +8,10 @@ interface PrismaStartupConfiguration extends Prisma.PrismaClientOptions {
declare module '../config' {
interface AppConfig {
database: ModuleConfig<PrismaStartupConfiguration>;
prisma: ModuleConfig<PrismaStartupConfiguration>;
}
}
defineStartupConfig('database', {
defineStartupConfig('prisma', {
datasourceUrl: '',
});

2
packages/backend/server/src/fundamentals/prisma/index.ts
View File

@@ -14,7 +14,7 @@ const clientProvider: Provider = {
return PrismaService.INSTANCE;
}
return new PrismaService(config.database);
return new PrismaService(config.prisma);
},
inject: [Config],
};

61
packages/backend/server/src/prelude.ts
View File

@@ -1,6 +1,7 @@
import 'reflect-metadata';
import { cpSync } from 'node:fs';
import { cpSync, existsSync, readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { join, parse } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -10,42 +11,57 @@ import {
applyEnvToConfig,
getAFFiNEConfigModifier,
} from './fundamentals/config';
import { enablePlugin } from './plugins';
const PROJECT_CONFIG_PATH = join(fileURLToPath(import.meta.url), '../config');
async function loadRemote(remoteDir: string, file: string) {
let fileToLoad = join(PROJECT_CONFIG_PATH, file);
const CUSTOM_CONFIG_PATH = `${homedir()}/.affine/config`;
if (PROJECT_CONFIG_PATH !== remoteDir) {
const remoteFile = join(remoteDir, file);
async function loadConfig(configDir: string, file: string) {
let fileToLoad: string | undefined;
if (PROJECT_CONFIG_PATH !== configDir) {
const remoteFile = join(configDir, file);
const remoteFileAtLocal = join(
PROJECT_CONFIG_PATH,
parse(file).name + '.remote.js'
);
cpSync(remoteFile, remoteFileAtLocal, {
force: true,
});
fileToLoad = remoteFileAtLocal;
if (existsSync(remoteFile)) {
cpSync(remoteFile, remoteFileAtLocal, {
force: true,
});
fileToLoad = remoteFileAtLocal;
}
} else {
fileToLoad = join(PROJECT_CONFIG_PATH, file);
}
await import(pathToFileURL(fileToLoad).href);
if (fileToLoad) {
await import(pathToFileURL(fileToLoad).href);
}
}
function loadPrivateKey() {
const file = join(CUSTOM_CONFIG_PATH, 'private.key');
if (!process.env.AFFINE_PRIVATE_KEY && existsSync(file)) {
const privateKey = readFileSync(file, 'utf-8');
process.env.AFFINE_PRIVATE_KEY = privateKey;
}
}
async function load() {
const AFFiNE_CONFIG_PATH =
process.env.AFFINE_CONFIG_PATH ?? PROJECT_CONFIG_PATH;
// Initializing AFFiNE config
//
// 1. load dotenv file to `process.env`
// load `.env` under pwd
config();
// @deprecated removed
// load `.env` under user config folder
config({
path: join(AFFiNE_CONFIG_PATH, '.env'),
path: join(CUSTOM_CONFIG_PATH, '.env'),
});
// 2. generate AFFiNE default config and assign to `globalThis.AFFiNE`
globalThis.AFFiNE = getAFFiNEConfigModifier();
const { enablePlugin } = await import('./plugins/registry');
globalThis.AFFiNE.use = enablePlugin;
globalThis.AFFiNE.plugins.use = enablePlugin;
@@ -53,24 +69,23 @@ async function load() {
// Modules may contribute to ENV_MAP, figure out a good way to involve them instead of hardcoding in `./config/affine.env`
// 3. load env => config map to `globalThis.AFFiNE.ENV_MAP
// load local env map as well in case there are new env added
await loadRemote(PROJECT_CONFIG_PATH, 'affine.env.js');
const projectEnvMap = AFFiNE.ENV_MAP;
await loadRemote(AFFiNE_CONFIG_PATH, 'affine.env.js');
const customEnvMap = AFFiNE.ENV_MAP;
AFFiNE.ENV_MAP = { ...projectEnvMap, ...customEnvMap };
await loadConfig(PROJECT_CONFIG_PATH, 'affine.env');
// 4. load `config/affine` to patch custom configs
// load local config as well in case there are new default configurations added
await loadRemote(PROJECT_CONFIG_PATH, 'affine.js');
await loadRemote(AFFiNE_CONFIG_PATH, 'affine.js');
await loadConfig(PROJECT_CONFIG_PATH, 'affine');
await loadConfig(CUSTOM_CONFIG_PATH, 'affine');
// 5. load `config/affine.self` to patch custom configs
// This is the file only take effect in [AFFiNE Cloud]
if (!AFFiNE.isSelfhosted) {
await loadRemote(PROJECT_CONFIG_PATH, 'affine.self.js');
await loadConfig(PROJECT_CONFIG_PATH, 'affine.self');
}
// 6. apply `process.env` map overriding to `globalThis.AFFiNE`
// 6. load `config/private.key` to patch app configs
loadPrivateKey();
// 7. apply `process.env` map overriding to `globalThis.AFFiNE`
applyEnvToConfig(globalThis.AFFiNE);
}

1
packages/common/env/src/global.ts vendored
View File

@@ -17,6 +17,7 @@ export type BUILD_CONFIG_TYPE = {
isMobileWeb: boolean;
isIOS: boolean;
isAndroid: boolean;
isAdmin: boolean;
// this is for the electron app
/**

1
packages/frontend/admin/package.json
View File

@@ -34,6 +34,7 @@
"@radix-ui/react-tooltip": "^1.1.1",
"@sentry/react": "^8.9.0",
"@tanstack/react-table": "^8.19.3",
"@toeverything/infra": "workspace:*",
"cmdk": "^1.0.0",
"embla-carousel-react": "^8.1.5",
"input-otp": "^1.2.4",

57
packages/frontend/admin/src/app.tsx
View File

@@ -1,5 +1,20 @@
import { Toaster } from '@affine/admin/components/ui/sonner';
import {
configureCloudModule,
DefaultServerService,
} from '@affine/core/modules/cloud';
import { configureLocalStorageStateStorageImpls } from '@affine/core/modules/storage';
import { configureUrlModule } from '@affine/core/modules/url';
import { wrapCreateBrowserRouter } from '@sentry/react';
import {
configureGlobalContextModule,
configureGlobalStorageModule,
configureLifecycleModule,
Framework,
FrameworkRoot,
FrameworkScope,
LifecycleService,
} from '@toeverything/infra';
import { useEffect } from 'react';
import {
createBrowserRouter as reactRouterCreateBrowserRouter,
@@ -109,18 +124,38 @@ export const router = _createBrowserRouter(
}
);
const framework = new Framework();
configureLifecycleModule(framework);
configureLocalStorageStateStorageImpls(framework);
configureGlobalStorageModule(framework);
configureGlobalContextModule(framework);
configureUrlModule(framework);
configureCloudModule(framework);
const frameworkProvider = framework.provider();
// setup application lifecycle events, and emit application start event
window.addEventListener('focus', () => {
frameworkProvider.get(LifecycleService).applicationFocus();
});
frameworkProvider.get(LifecycleService).applicationStart();
const serverService = frameworkProvider.get(DefaultServerService);
export const App = () => {
return (
<TooltipProvider>
<SWRConfig
value={{
revalidateOnFocus: false,
revalidateOnMount: false,
}}
>
<RouterProvider router={router} />
</SWRConfig>
<Toaster />
</TooltipProvider>
<FrameworkRoot framework={frameworkProvider}>
<FrameworkScope scope={serverService.server.scope}>
<TooltipProvider>
<SWRConfig
value={{
revalidateOnFocus: false,
revalidateOnMount: false,
}}
>
<RouterProvider router={router} />
</SWRConfig>
<Toaster />
</TooltipProvider>
</FrameworkScope>
</FrameworkRoot>
);
};

6
packages/frontend/admin/tsconfig.json
View File

@@ -8,6 +8,10 @@
"verbatimModuleSyntax": false,
"jsx": "react-jsx"
},
"references": [{ "path": "../core" }, { "path": "../graphql" }],
"references": [
{ "path": "../core" },
{ "path": "../graphql" },
{ "path": "../../common/infra" }
],
"exclude": ["dist"]
}

72
packages/frontend/apps/electron/scripts/generate-yml.js
View File

@@ -1,72 +0,0 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
const filenamesMapping = {
all: 'latest.yml',
macos: 'latest-mac.yml',
linux: 'latest-linux.yml',
};
const releaseFiles = ['zip', 'exe', 'dmg', 'appimage', 'deb', 'flatpak'];
const generateYml = platform => {
const yml = {
version: process.env.RELEASE_VERSION ?? '0.0.0',
files: [],
};
const regex =
// we involves all distribution files in one release file to enforce we handle auto updater correctly
platform === 'all'
? new RegExp(`.(${releaseFiles.join('|')})$`)
: new RegExp(`.+-${platform}-.+.(${releaseFiles.join('|')})$`);
const files = fs.readdirSync(process.cwd()).filter(file => regex.test(file));
const outputFileName = filenamesMapping[platform];
files.forEach(fileName => {
const filePath = path.join(process.cwd(), './', fileName);
try {
const fileData = fs.readFileSync(filePath);
const hash = crypto
.createHash('sha512')
.update(fileData)
.digest('base64');
const size = fs.statSync(filePath).size;
yml.files.push({
url: fileName,
sha512: hash,
size: size,
});
} catch {}
});
yml.releaseDate = new Date().toISOString();
// NOTE(@forehalo): make sure old windows x64 won't fetch windows arm64 by default
// maybe we need to separate arm64 builds to separated yml file `latest-arm64.yml`, `latest-linux-arm64.yml`
// check https://github.com/electron-userland/electron-builder/blob/master/packages/electron-updater/src/providers/Provider.ts#L30
// and packages/frontend/apps/electron/src/main/updater/affine-update-provider.ts#L100
yml.files.sort(a => (a.url.includes('windows-arm64') ? 1 : -1));
const ymlStr =
`version: ${yml.version}\n` +
`files:\n` +
yml.files
.map(file => {
return (
` - url: ${file.url}\n` +
` sha512: ${file.sha512}\n` +
` size: ${file.size}\n`
);
})
.join('') +
`releaseDate: ${yml.releaseDate}\n`;
fs.writeFileSync(outputFileName, ymlStr);
};
generateYml('macos');
generateYml('linux');
generateYml('all');

4
packages/frontend/core/src/components/hooks/use-query.ts
View File

@@ -9,7 +9,7 @@ import type { GraphQLError } from 'graphql';
import { useCallback, useMemo } from 'react';
import type { SWRConfiguration, SWRResponse } from 'swr';
import useSWR from 'swr';
import useSWRImutable from 'swr/immutable';
import useSWRImmutable from 'swr/immutable';
import useSWRInfinite from 'swr/infinite';
/**
@@ -60,7 +60,7 @@ const createUseQuery =
);
const graphqlService = useService(GraphQLService);
const useSWRFn = immutable ? useSWRImutable : useSWR;
const useSWRFn = immutable ? useSWRImmutable : useSWR;
return useSWRFn(
options ? () => ['cloud', options.query.id, options.variables] : null,
options ? () => graphqlService.gql(options) : null,