Compare commits

...

23 Commits

Author SHA1 Message Date
LongYinan
39674e2686 Merge branch 'beta' into stable 2024-03-18 12:03:59 +08:00
LongYinan
c00cddfad7 build(core): add source-map-loader for blocksuite codes (#6137) 2024-03-18 12:02:50 +08:00
liuyi
92d7f318ba fix(server): ensure selfhost admin created after all data migrated (#6163)
fix #6154 

cp to canary
2024-03-18 11:40:14 +08:00
LongYinan
3bdfcfaf40 Merge remote-tracking branch 'origin/beta' into stable 2024-03-18 10:38:22 +08:00
liuyi
d82ee7d43d fix(server): hotfix (#6161) 2024-03-18 10:29:51 +08:00
EYHN
27989c6401 fix(core): fix error when switch to local workspace (#6144) 2024-03-15 18:06:34 +08:00
LongYinan
d8b57d00c0 Merge remote-tracking branch 'origin/canary' into beta 2024-03-15 17:01:17 +08:00
DarkSky
530959b868 fix(server): wrap read-modify-write apis with distributed lock (#5979) 2024-03-15 16:46:41 +08:00
Chen
dd2c6cf544 fix: note added with template should be edgeless only (#6122) 2024-03-15 16:46:41 +08:00
EYHN
f5b1d041c5 fix(core): fix active view undefined (#6131)
close https://github.com/toeverything/AFFiNE/issues/6127, #6132
2024-03-15 16:46:40 +08:00
EYHN
81f3e65bde feat(core): allow switch workspace in loading fallback (#6129) 2024-03-15 16:46:40 +08:00
EYHN
4389378689 fix(core): catch auth error (#6128) 2024-03-15 16:46:39 +08:00
Cats Juice
364bb6ccb0 fix(core): shared page's present button not working (#6117) 2024-03-15 16:46:39 +08:00
Peng Xiao
e47b271e9d fix: update docs (#6094) 2024-03-15 16:46:38 +08:00
LongYinan
72a4bf5294 ci: fix canary backend auto release job (#6121) 2024-03-15 16:46:38 +08:00
liuyi
f5108c6788 feat(server): cleanup gateway code (#6118) 2024-03-15 16:46:37 +08:00
liuyi
f9945a073c feat(server): allow prefetch doc stats before sync (#6115) 2024-03-15 16:46:37 +08:00
LongYinan
1b1e40133a Merge remote-tracking branch 'origin/canary' into beta 2024-03-14 15:31:51 +08:00
DarkSky
003986d657 feat: add cloud logger sa integrate (#6089) 2024-03-12 23:25:00 +08:00
forehalo
0d66519523 ci: only enable jwst codec in canary 2024-03-12 17:47:33 +08:00
LongYinan
aaffc80f82 ci: add write packages permission to release workflow 2024-03-06 18:18:27 +08:00
LongYinan
2c8861ae49 ci: add write permission to release workflow 2024-03-06 18:14:22 +08:00
LongYinan
ec1edfd70b ci: add write permission to release workflow 2024-03-06 18:12:20 +08:00
29 changed files with 434 additions and 85 deletions

View File

@@ -21,6 +21,7 @@ const {
AFFINE_GOOGLE_CLIENT_ID, AFFINE_GOOGLE_CLIENT_ID,
AFFINE_GOOGLE_CLIENT_SECRET, AFFINE_GOOGLE_CLIENT_SECRET,
CLOUD_SQL_IAM_ACCOUNT, CLOUD_SQL_IAM_ACCOUNT,
CLOUD_LOGGER_IAM_ACCOUNT,
GCLOUD_CONNECTION_NAME, GCLOUD_CONNECTION_NAME,
GCLOUD_CLOUD_SQL_INTERNAL_ENDPOINT, GCLOUD_CLOUD_SQL_INTERNAL_ENDPOINT,
REDIS_HOST, REDIS_HOST,
@@ -59,7 +60,9 @@ const createHelmCommand = ({ isDryRun }) => {
? [ ? [
`--set-json web.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`, `--set-json web.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`,
`--set-json graphql.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`, `--set-json graphql.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`,
`--set-json graphql.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_LOGGER_IAM_ACCOUNT}\\"}\"`,
`--set-json sync.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`, `--set-json sync.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`,
`--set-json sync.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_LOGGER_IAM_ACCOUNT}\\"}\"`,
`--set-json cloud-sql-proxy.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_SQL_IAM_ACCOUNT}\\" }\"`, `--set-json cloud-sql-proxy.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_SQL_IAM_ACCOUNT}\\" }\"`,
`--set-json cloud-sql-proxy.nodeSelector=\"{ \\"iam.gke.io/gke-metadata-server-enabled\\": \\"true\\" }\"`, `--set-json cloud-sql-proxy.nodeSelector=\"{ \\"iam.gke.io/gke-metadata-server-enabled\\": \\"true\\" }\"`,
] ]

View File

@@ -295,6 +295,7 @@ jobs:
REDIS_HOST: ${{ secrets.REDIS_HOST }} REDIS_HOST: ${{ secrets.REDIS_HOST }}
REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
CLOUD_SQL_IAM_ACCOUNT: ${{ secrets.CLOUD_SQL_IAM_ACCOUNT }} CLOUD_SQL_IAM_ACCOUNT: ${{ secrets.CLOUD_SQL_IAM_ACCOUNT }}
CLOUD_LOGGER_IAM_ACCOUNT: ${{ secrets.CLOUD_LOGGER_IAM_ACCOUNT }}
STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }} STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }}
STRIPE_WEBHOOK_KEY: ${{ secrets.STRIPE_WEBHOOK_KEY }} STRIPE_WEBHOOK_KEY: ${{ secrets.STRIPE_WEBHOOK_KEY }}
STATIC_IP_NAME: ${{ secrets.STATIC_IP_NAME }} STATIC_IP_NAME: ${{ secrets.STATIC_IP_NAME }}

View File

@@ -28,6 +28,7 @@ import { GqlModule } from './fundamentals/graphql';
import { HelpersModule } from './fundamentals/helpers'; import { HelpersModule } from './fundamentals/helpers';
import { MailModule } from './fundamentals/mailer'; import { MailModule } from './fundamentals/mailer';
import { MetricsModule } from './fundamentals/metrics'; import { MetricsModule } from './fundamentals/metrics';
import { MutexModule } from './fundamentals/mutex';
import { PrismaModule } from './fundamentals/prisma'; import { PrismaModule } from './fundamentals/prisma';
import { StorageProviderModule } from './fundamentals/storage'; import { StorageProviderModule } from './fundamentals/storage';
import { RateLimiterModule } from './fundamentals/throttler'; import { RateLimiterModule } from './fundamentals/throttler';
@@ -39,6 +40,7 @@ export const FunctionalityModules = [
ScheduleModule.forRoot(), ScheduleModule.forRoot(),
EventModule, EventModule,
CacheModule, CacheModule,
MutexModule,
PrismaModule, PrismaModule,
MetricsModule, MetricsModule,
RateLimiterModule, RateLimiterModule,

View File

@@ -226,6 +226,10 @@ export class AuthService implements OnApplicationBootstrap {
} }
async getSession(token: string) { async getSession(token: string) {
if (!token) {
return null;
}
return this.db.$transaction(async tx => { return this.db.$transaction(async tx => {
const session = await tx.session.findUnique({ const session = await tx.session.findUnique({
where: { where: {

View File

@@ -1,5 +1,4 @@
import { PrismaClient } from '@prisma/client'; import { PrismaTransaction } from '../../fundamentals';
import { Feature, FeatureSchema, FeatureType } from './types'; import { Feature, FeatureSchema, FeatureType } from './types';
class FeatureConfig { class FeatureConfig {
@@ -67,7 +66,7 @@ export type FeatureConfigType<F extends FeatureType> = InstanceType<
const FeatureCache = new Map<number, FeatureConfigType<FeatureType>>(); const FeatureCache = new Map<number, FeatureConfigType<FeatureType>>();
export async function getFeature(prisma: PrismaClient, featureId: number) { export async function getFeature(prisma: PrismaTransaction, featureId: number) {
const cachedQuota = FeatureCache.get(featureId); const cachedQuota = FeatureCache.get(featureId);
if (cachedQuota) { if (cachedQuota) {

View File

@@ -1,5 +1,4 @@
import { PrismaClient } from '@prisma/client'; import { PrismaTransaction } from '../../fundamentals';
import { formatDate, formatSize, Quota, QuotaSchema } from './types'; import { formatDate, formatSize, Quota, QuotaSchema } from './types';
const QuotaCache = new Map<number, QuotaConfig>(); const QuotaCache = new Map<number, QuotaConfig>();
@@ -7,14 +6,14 @@ const QuotaCache = new Map<number, QuotaConfig>();
export class QuotaConfig { export class QuotaConfig {
readonly config: Quota; readonly config: Quota;
static async get(prisma: PrismaClient, featureId: number) { static async get(tx: PrismaTransaction, featureId: number) {
const cachedQuota = QuotaCache.get(featureId); const cachedQuota = QuotaCache.get(featureId);
if (cachedQuota) { if (cachedQuota) {
return cachedQuota; return cachedQuota;
} }
const quota = await prisma.features.findFirst({ const quota = await tx.features.findFirst({
where: { where: {
id: featureId, id: featureId,
}, },

View File

@@ -1,13 +1,15 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { PrismaClient } from '@prisma/client'; import { PrismaClient } from '@prisma/client';
import { type EventPayload, OnEvent } from '../../fundamentals'; import {
type EventPayload,
OnEvent,
PrismaTransaction,
} from '../../fundamentals';
import { FeatureKind } from '../features'; import { FeatureKind } from '../features';
import { QuotaConfig } from './quota'; import { QuotaConfig } from './quota';
import { QuotaType } from './types'; import { QuotaType } from './types';
type Transaction = Parameters<Parameters<PrismaClient['$transaction']>[0]>[0];
@Injectable() @Injectable()
export class QuotaService { export class QuotaService {
constructor(private readonly prisma: PrismaClient) {} constructor(private readonly prisma: PrismaClient) {}
@@ -140,8 +142,8 @@ export class QuotaService {
}); });
} }
async hasQuota(userId: string, quota: QuotaType, transaction?: Transaction) { async hasQuota(userId: string, quota: QuotaType, tx?: PrismaTransaction) {
const executor = transaction ?? this.prisma; const executor = tx ?? this.prisma;
return executor.userFeatures return executor.userFeatures
.count({ .count({

View File

@@ -54,7 +54,7 @@ export class UserService {
return this.createUser({ return this.createUser({
email, email,
name: 'Unnamed', name: email.split('@')[0],
...data, ...data,
}); });
} }

View File

@@ -25,7 +25,9 @@ import {
EventEmitter, EventEmitter,
type FileUpload, type FileUpload,
MailService, MailService,
MutexService,
Throttle, Throttle,
TooManyRequestsException,
} from '../../../fundamentals'; } from '../../../fundamentals';
import { CurrentUser, Public } from '../../auth'; import { CurrentUser, Public } from '../../auth';
import { QuotaManagementService, QuotaQueryType } from '../../quota'; import { QuotaManagementService, QuotaQueryType } from '../../quota';
@@ -58,7 +60,8 @@ export class WorkspaceResolver {
private readonly quota: QuotaManagementService, private readonly quota: QuotaManagementService,
private readonly users: UserService, private readonly users: UserService,
private readonly event: EventEmitter, private readonly event: EventEmitter,
private readonly blobStorage: WorkspaceBlobStorage private readonly blobStorage: WorkspaceBlobStorage,
private readonly mutex: MutexService
) {} ) {}
@ResolveField(() => Permission, { @ResolveField(() => Permission, {
@@ -336,74 +339,87 @@ export class WorkspaceResolver {
throw new ForbiddenException('Cannot change owner'); throw new ForbiddenException('Cannot change owner');
} }
// member limit check try {
const [memberCount, quota] = await Promise.all([ // lock to prevent concurrent invite
this.prisma.workspaceUserPermission.count({ const lockFlag = `invite:${workspaceId}`;
where: { workspaceId }, await using lock = await this.mutex.lock(lockFlag);
}), if (!lock) {
this.quota.getWorkspaceUsage(workspaceId), return new TooManyRequestsException('Server is busy');
]); }
if (memberCount >= quota.memberLimit) {
throw new PayloadTooLargeException('Workspace member limit reached.');
}
let target = await this.users.findUserByEmail(email); // member limit check
if (target) { const [memberCount, quota] = await Promise.all([
const originRecord = await this.prisma.workspaceUserPermission.findFirst({ this.prisma.workspaceUserPermission.count({
where: { where: { workspaceId },
workspaceId, }),
userId: target.id, this.quota.getWorkspaceUsage(workspaceId),
}, ]);
}); if (memberCount >= quota.memberLimit) {
// only invite if the user is not already in the workspace return new PayloadTooLargeException('Workspace member limit reached.');
if (originRecord) return originRecord.id; }
} else {
target = await this.users.createAnonymousUser(email, {
registered: false,
});
}
const inviteId = await this.permissions.grant( let target = await this.users.findUserByEmail(email);
workspaceId, if (target) {
target.id, const originRecord =
permission await this.prisma.workspaceUserPermission.findFirst({
); where: {
if (sendInviteMail) { workspaceId,
const inviteInfo = await this.getInviteInfo(inviteId); userId: target.id,
},
try { });
await this.mailer.sendInviteEmail(email, inviteId, { // only invite if the user is not already in the workspace
workspace: { if (originRecord) return originRecord.id;
id: inviteInfo.workspace.id, } else {
name: inviteInfo.workspace.name, target = await this.users.createAnonymousUser(email, {
avatar: inviteInfo.workspace.avatar, registered: false,
},
user: {
avatar: inviteInfo.user?.avatarUrl || '',
name: inviteInfo.user?.name || '',
},
}); });
} catch (e) { }
const ret = await this.permissions.revokeWorkspace(
workspaceId,
target.id
);
if (!ret) { const inviteId = await this.permissions.grant(
this.logger.fatal( workspaceId,
`failed to send ${workspaceId} invite email to ${email} and failed to revoke permission: ${inviteId}, ${e}` target.id,
permission
);
if (sendInviteMail) {
const inviteInfo = await this.getInviteInfo(inviteId);
try {
await this.mailer.sendInviteEmail(email, inviteId, {
workspace: {
id: inviteInfo.workspace.id,
name: inviteInfo.workspace.name,
avatar: inviteInfo.workspace.avatar,
},
user: {
avatar: inviteInfo.user?.avatarUrl || '',
name: inviteInfo.user?.name || '',
},
});
} catch (e) {
const ret = await this.permissions.revokeWorkspace(
workspaceId,
target.id
); );
} else {
this.logger.warn( if (!ret) {
`failed to send ${workspaceId} invite email to ${email}, but successfully revoked permission: ${e}` this.logger.fatal(
`failed to send ${workspaceId} invite email to ${email} and failed to revoke permission: ${inviteId}, ${e}`
);
} else {
this.logger.warn(
`failed to send ${workspaceId} invite email to ${email}, but successfully revoked permission: ${e}`
);
}
return new InternalServerErrorException(
'Failed to send invite email. Please try again.'
); );
} }
return new InternalServerErrorException(
'Failed to send invite email. Please try again.'
);
} }
return inviteId;
} catch (e) {
this.logger.error('failed to invite user', e);
return new TooManyRequestsException('Server is busy');
} }
return inviteId;
} }
@Throttle({ @Throttle({

View File

@@ -23,6 +23,8 @@ export async function collectMigrations(): Promise<Migration[]> {
) )
.map(desc => join(folder, desc)); .map(desc => join(folder, desc));
migrationFiles.sort((a, b) => a.localeCompare(b));
const migrations: Migration[] = await Promise.all( const migrations: Migration[] = await Promise.all(
migrationFiles.map(async file => { migrationFiles.map(async file => {
return import(pathToFileURL(file).href).then(mod => { return import(pathToFileURL(file).href).then(mod => {

View File

@@ -4,7 +4,7 @@ import { PrismaClient } from '@prisma/client';
import { UserService } from '../../core/user'; import { UserService } from '../../core/user';
import { Config, CryptoHelper } from '../../fundamentals'; import { Config, CryptoHelper } from '../../fundamentals';
export class SelfHostAdmin1605053000403 { export class SelfHostAdmin99999999 {
// do the migration // do the migration
static async up(_db: PrismaClient, ref: ModuleRef) { static async up(_db: PrismaClient, ref: ModuleRef) {
const config = ref.get(Config, { strict: false }); const config = ref.get(Config, { strict: false });

View File

@@ -1 +1,2 @@
export * from './payment-required'; export * from './payment-required';
export * from './too-many-requests';

View File

@@ -0,0 +1,14 @@
import { HttpException, HttpStatus } from '@nestjs/common';
export class TooManyRequestsException extends HttpException {
constructor(desc?: string, code: string = 'Too Many Requests') {
super(
HttpException.createBody(
desc ?? code,
code,
HttpStatus.TOO_MANY_REQUESTS
),
HttpStatus.TOO_MANY_REQUESTS
);
}
}

View File

@@ -11,6 +11,12 @@ import { GraphQLError } from 'graphql';
import { Config } from '../config'; import { Config } from '../config';
import { GQLLoggerPlugin } from './logger-plugin'; import { GQLLoggerPlugin } from './logger-plugin';
export type GraphqlContext = {
req: Request;
res: Response;
isAdminQuery: boolean;
};
@Global() @Global()
@Module({ @Module({
imports: [ imports: [
@@ -30,7 +36,13 @@ import { GQLLoggerPlugin } from './logger-plugin';
: '../../../schema.gql' : '../../../schema.gql'
), ),
sortSchema: true, sortSchema: true,
context: ({ req, res }: { req: Request; res: Response }) => ({ context: ({
req,
res,
}: {
req: Request;
res: Response;
}): GraphqlContext => ({
req, req,
res, res,
isAdminQuery: false, isAdminQuery: false,

View File

@@ -14,14 +14,23 @@ export {
} from './config'; } from './config';
export * from './error'; export * from './error';
export { EventEmitter, type EventPayload, OnEvent } from './event'; export { EventEmitter, type EventPayload, OnEvent } from './event';
export type { GraphqlContext } from './graphql';
export { CryptoHelper, URLHelper } from './helpers'; export { CryptoHelper, URLHelper } from './helpers';
export { MailService } from './mailer'; export { MailService } from './mailer';
export { CallCounter, CallTimer, metrics } from './metrics'; export { CallCounter, CallTimer, metrics } from './metrics';
export {
BucketService,
LockGuard,
MUTEX_RETRY,
MUTEX_WAIT,
MutexService,
} from './mutex';
export { export {
getOptionalModuleMetadata, getOptionalModuleMetadata,
GlobalExceptionFilter, GlobalExceptionFilter,
OptionalModule, OptionalModule,
} from './nestjs'; } from './nestjs';
export type { PrismaTransaction } from './prisma';
export * from './storage'; export * from './storage';
export { type StorageProvider, StorageProviderFactory } from './storage'; export { type StorageProvider, StorageProviderFactory } from './storage';
export { AuthThrottlerGuard, CloudThrottlerGuard, Throttle } from './throttler'; export { AuthThrottlerGuard, CloudThrottlerGuard, Throttle } from './throttler';

View File

@@ -0,0 +1,15 @@
export class BucketService {
private readonly bucket = new Map<string, string>();
get(key: string) {
return this.bucket.get(key);
}
set(key: string, value: string) {
this.bucket.set(key, value);
}
delete(key: string) {
this.bucket.delete(key);
}
}

View File

@@ -0,0 +1,14 @@
import { Global, Module } from '@nestjs/common';
import { BucketService } from './bucket';
import { MutexService } from './mutex';
@Global()
@Module({
providers: [BucketService, MutexService],
exports: [BucketService, MutexService],
})
export class MutexModule {}
export { BucketService, MutexService };
export { LockGuard, MUTEX_RETRY, MUTEX_WAIT } from './mutex';

View File

@@ -0,0 +1,96 @@
import { randomUUID } from 'node:crypto';
import { setTimeout } from 'node:timers/promises';
import { Inject, Injectable, Logger, Scope } from '@nestjs/common';
import { CONTEXT } from '@nestjs/graphql';
import type { GraphqlContext } from '../graphql';
import { BucketService } from './bucket';
export class LockGuard<M extends MutexService = MutexService>
implements AsyncDisposable
{
constructor(
private readonly mutex: M,
private readonly key: string
) {}
async [Symbol.asyncDispose]() {
return this.mutex.unlock(this.key);
}
}
export const MUTEX_RETRY = 5;
export const MUTEX_WAIT = 100;
@Injectable({ scope: Scope.REQUEST })
export class MutexService {
protected logger = new Logger(MutexService.name);
constructor(
@Inject(CONTEXT) private readonly context: GraphqlContext,
private readonly bucket: BucketService
) {}
protected getId() {
let id = this.context.req.headers['x-transaction-id'] as string;
if (!id) {
id = randomUUID();
this.context.req.headers['x-transaction-id'] = id;
}
return id;
}
/**
* lock an resource and return a lock guard, which will release the lock when disposed
*
* if the lock is not available, it will retry for [MUTEX_RETRY] times
*
* usage:
* ```typescript
* {
* // lock is acquired here
* await using lock = await mutex.lock('resource-key');
* if (lock) {
* // do something
* } else {
* // failed to lock
* }
* }
* // lock is released here
* ```
* @param key resource key
* @returns LockGuard
*/
async lock(key: string): Promise<LockGuard | undefined> {
const id = this.getId();
const fetchLock = async (retry: number): Promise<LockGuard | undefined> => {
if (retry === 0) {
this.logger.error(
`Failed to fetch lock ${key} after ${MUTEX_RETRY} retry`
);
return undefined;
}
const current = this.bucket.get(key);
if (current && current !== id) {
this.logger.warn(
`Failed to fetch lock ${key}, retrying in ${MUTEX_WAIT} ms`
);
await setTimeout(MUTEX_WAIT * (MUTEX_RETRY - retry + 1));
return fetchLock(retry - 1);
}
this.bucket.set(key, id);
return new LockGuard(this, key);
};
return fetchLock(MUTEX_RETRY);
}
async unlock(key: string): Promise<void> {
if (this.bucket.get(key) === this.getId()) {
this.bucket.delete(key);
}
}
}

View File

@@ -16,3 +16,7 @@ const clientProvider: Provider = {
}) })
export class PrismaModule {} export class PrismaModule {}
export { PrismaService } from './service'; export { PrismaService } from './service';
export type PrismaTransaction = Parameters<
Parameters<PrismaClient['$transaction']>[0]
>[0];

View File

@@ -12,6 +12,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
import type { Request } from 'express'; import type { Request } from 'express';
import Stripe from 'stripe'; import Stripe from 'stripe';
import { Public } from '../../core/auth';
import { Config } from '../../fundamentals'; import { Config } from '../../fundamentals';
@Controller('/api/stripe') @Controller('/api/stripe')
@@ -28,6 +29,7 @@ export class StripeWebhook {
this.webhookKey = config.plugins.payment.stripe.keys.webhookKey; this.webhookKey = config.plugins.payment.stripe.keys.webhookKey;
} }
@Public()
@Post('/webhook') @Post('/webhook')
async handleWebhook(@Req() req: RawBodyRequest<Request>) { async handleWebhook(@Req() req: RawBodyRequest<Request>) {
// Check if webhook signing is configured. // Check if webhook signing is configured.

View File

@@ -1,18 +1,27 @@
import { Global, Provider, Type } from '@nestjs/common'; import { Global, Provider, Type } from '@nestjs/common';
import { CONTEXT } from '@nestjs/graphql';
import { Redis, type RedisOptions } from 'ioredis'; import { Redis, type RedisOptions } from 'ioredis';
import { ThrottlerStorageRedisService } from 'nestjs-throttler-storage-redis'; import { ThrottlerStorageRedisService } from 'nestjs-throttler-storage-redis';
import { Cache, SessionCache } from '../../fundamentals'; import {
BucketService,
Cache,
type GraphqlContext,
MutexService,
SessionCache,
} from '../../fundamentals';
import { ThrottlerStorage } from '../../fundamentals/throttler'; import { ThrottlerStorage } from '../../fundamentals/throttler';
import { SocketIoAdapterImpl } from '../../fundamentals/websocket'; import { SocketIoAdapterImpl } from '../../fundamentals/websocket';
import { Plugin } from '../registry'; import { Plugin } from '../registry';
import { RedisCache } from './cache'; import { RedisCache } from './cache';
import { import {
CacheRedis, CacheRedis,
MutexRedis,
SessionRedis, SessionRedis,
SocketIoRedis, SocketIoRedis,
ThrottlerRedis, ThrottlerRedis,
} from './instances'; } from './instances';
import { MutexRedisService } from './mutex';
import { createSockerIoAdapterImpl } from './ws-adapter'; import { createSockerIoAdapterImpl } from './ws-adapter';
function makeProvider(token: Type, impl: Type<Redis>): Provider { function makeProvider(token: Type, impl: Type<Redis>): Provider {
@@ -47,15 +56,31 @@ const socketIoRedisAdapterProvider: Provider = {
inject: [SocketIoRedis], inject: [SocketIoRedis],
}; };
// mutex
const mutexRedisAdapterProvider: Provider = {
provide: MutexService,
useFactory: (redis: Redis, ctx: GraphqlContext, bucket: BucketService) => {
return new MutexRedisService(redis, ctx, bucket);
},
inject: [MutexRedis, CONTEXT, BucketService],
};
@Global() @Global()
@Plugin({ @Plugin({
name: 'redis', name: 'redis',
providers: [CacheRedis, SessionRedis, ThrottlerRedis, SocketIoRedis], providers: [
CacheRedis,
SessionRedis,
ThrottlerRedis,
SocketIoRedis,
MutexRedis,
],
overrides: [ overrides: [
cacheProvider, cacheProvider,
sessionCacheProvider, sessionCacheProvider,
socketIoRedisAdapterProvider, socketIoRedisAdapterProvider,
throttlerStorageProvider, throttlerStorageProvider,
mutexRedisAdapterProvider,
], ],
requires: ['plugins.redis.host'], requires: ['plugins.redis.host'],
}) })

View File

@@ -54,3 +54,10 @@ export class SocketIoRedis extends Redis {
super({ ...config.plugins.redis, db: (config.plugins.redis?.db ?? 0) + 3 }); super({ ...config.plugins.redis, db: (config.plugins.redis?.db ?? 0) + 3 });
} }
} }
@Injectable()
export class MutexRedis extends Redis {
constructor(config: Config) {
super({ ...config.plugins.redis, db: (config.plugins.redis?.db ?? 0) + 4 });
}
}

View File

@@ -0,0 +1,96 @@
import { setTimeout } from 'node:timers/promises';
import { Injectable, Logger } from '@nestjs/common';
import Redis, { Command } from 'ioredis';
import {
BucketService,
type GraphqlContext,
LockGuard,
MUTEX_RETRY,
MUTEX_WAIT,
MutexService,
} from '../../fundamentals';
const lockScript = `local key = KEYS[1]
local clientId = ARGV[1]
local releaseTime = ARGV[2]
if redis.call("get", key) == clientId or redis.call("set", key, clientId, "NX", "PX", releaseTime) then
return 1
else
return 0
end`;
const unlockScript = `local key = KEYS[1]
local clientId = ARGV[1]
if redis.call("get", key) == clientId then
return redis.call("del", key)
else
return 0
end`;
@Injectable()
export class MutexRedisService extends MutexService {
constructor(
private readonly redis: Redis,
context: GraphqlContext,
bucket: BucketService
) {
super(context, bucket);
this.logger = new Logger(MutexRedisService.name);
}
override async lock(
key: string,
releaseTimeInMS: number = 200
): Promise<LockGuard | undefined> {
const clientId = this.getId();
this.logger.debug(`Client ${clientId} lock try to lock ${key}`);
const releaseTime = releaseTimeInMS.toString();
const fetchLock = async (retry: number): Promise<LockGuard | undefined> => {
if (retry === 0) {
this.logger.error(
`Failed to fetch lock ${key} after ${MUTEX_RETRY} retry`
);
return undefined;
}
try {
const success = await this.redis.sendCommand(
new Command('EVAL', [lockScript, '1', key, clientId, releaseTime])
);
if (success === 1) {
return new LockGuard(this, key);
} else {
this.logger.warn(
`Failed to fetch lock ${key}, retrying in ${MUTEX_WAIT} ms`
);
await setTimeout(MUTEX_WAIT * (MUTEX_RETRY - retry + 1));
return fetchLock(retry - 1);
}
} catch (error: any) {
this.logger.error(
`Unexpected error when fetch lock ${key}: ${error.message}`
);
return undefined;
}
};
return fetchLock(MUTEX_RETRY);
}
override async unlock(key: string, ignoreUnlockFail = false): Promise<void> {
const clientId = this.getId();
const result = await this.redis.sendCommand(
new Command('EVAL', [unlockScript, '1', key, clientId])
);
if (result === 0) {
if (!ignoreUnlockFail) {
throw new Error(`Failed to release lock ${key}`);
} else {
this.logger.warn(`Failed to release lock ${key}`);
}
}
}
}

View File

@@ -2,7 +2,7 @@
"extends": "../../../../tsconfig.json", "extends": "../../../../tsconfig.json",
"compilerOptions": { "compilerOptions": {
"composite": true, "composite": true,
"target": "ESNext", "target": "ES2022",
"emitDecoratorMetadata": true, "emitDecoratorMetadata": true,
"experimentalDecorators": true, "experimentalDecorators": true,
"rootDir": ".", "rootDir": ".",

View File

@@ -104,7 +104,7 @@ test('should create user if not exist', async t => {
const user = await auth.getUserByEmail('u2@affine.pro'); const user = await auth.getUserByEmail('u2@affine.pro');
t.not(user, undefined, 'failed to create user'); t.not(user, undefined, 'failed to create user');
t.is(user?.name, 'Unnamed', 'failed to create user'); t.is(user?.name, 'u2', 'failed to create user');
}); });
test('should invite a user by link', async t => { test('should invite a user by link', async t => {
@@ -255,3 +255,25 @@ test('should support pagination for member', async t => {
); );
t.is(secondPageWorkspace.members.length, 1, 'failed to check invite id'); t.is(secondPageWorkspace.members.length, 1, 'failed to check invite id');
}); });
test('should limit member count correctly', async t => {
const { app } = t.context;
const u1 = await signUp(app, 'u1', 'u1@affine.pro', '1');
for (let i = 0; i < 10; i++) {
const workspace = await createWorkspace(app, u1.token.token);
await Promise.allSettled(
Array.from({ length: 10 }).map(async (_, i) =>
inviteUser(
app,
u1.token.token,
workspace.id,
`u${i}@affine.pro`,
'Admin'
)
)
);
const ws = await getWorkspace(app, u1.token.token, workspace.id);
t.assert(ws.members.length <= 3, 'failed to check member list');
}
});

View File

@@ -2,7 +2,7 @@
"extends": "../../../tsconfig.json", "extends": "../../../tsconfig.json",
"compilerOptions": { "compilerOptions": {
"composite": true, "composite": true,
"target": "ESNext", "target": "ES2022",
"module": "ESNext", "module": "ESNext",
"emitDecoratorMetadata": true, "emitDecoratorMetadata": true,
"experimentalDecorators": true, "experimentalDecorators": true,

View File

@@ -242,6 +242,12 @@ export const createConfiguration: (
fullySpecified: false, fullySpecified: false,
}, },
}, },
{
test: /\.js$/,
enforce: 'pre',
include: /@blocksuite/,
use: ['source-map-loader'],
},
{ {
oneOf: [ oneOf: [
{ {

View File

@@ -1,6 +1,6 @@
import { style } from '@vanilla-extract/css'; import { style } from '@vanilla-extract/css';
export const fallbackStyle = style({ export const fallbackStyle = style({
margin: '12px 16px', margin: '5px 16px',
height: '100%', height: '100%',
}); });
export const fallbackHeaderStyle = style({ export const fallbackHeaderStyle = style({

View File

@@ -1,5 +1,5 @@
import { WorkspaceFlavour } from '@affine/env/workspace'; import { WorkspaceFlavour } from '@affine/env/workspace';
import { WorkspaceManager } from '@toeverything/infra'; import { Workspace, WorkspaceManager } from '@toeverything/infra';
import { useService } from '@toeverything/infra/di'; import { useService } from '@toeverything/infra/di';
import { useLiveData } from '@toeverything/infra/livedata'; import { useLiveData } from '@toeverything/infra/livedata';
import { useAtom } from 'jotai'; import { useAtom } from 'jotai';
@@ -172,9 +172,7 @@ export const AuthModal = (): ReactElement => {
}; };
export function CurrentWorkspaceModals() { export function CurrentWorkspaceModals() {
const currentWorkspace = useLiveData( const currentWorkspace = useService(Workspace);
useService(CurrentWorkspaceService).currentWorkspace
);
const [openDisableCloudAlertModal, setOpenDisableCloudAlertModal] = useAtom( const [openDisableCloudAlertModal, setOpenDisableCloudAlertModal] = useAtom(
openDisableCloudAlertModalAtom openDisableCloudAlertModalAtom
); );