refactor(server): folder structure (#5573)

packages/backend/server/src/fundamentals/cache/index.ts

@@ -0,0 +1,35 @@
+import { Global, Module, Provider, Type } from '@nestjs/common';
+import { Redis } from 'ioredis';
+
+import { SessionCache, ThrottlerCache } from './instances';
+import { LocalCache } from './providers/cache';
+import { RedisCache } from './providers/redis';
+import { CacheRedis, RedisModule, SessionRedis, ThrottlerRedis } from './redis';
+
+function makeCacheProvider(CacheToken: Type, RedisToken: Type): Provider {
+  return {
+    provide: CacheToken,
+    useFactory: (redis?: Redis) => {
+      return redis ? new RedisCache(redis) : new LocalCache();
+    },
+    inject: [{ token: RedisToken, optional: true }],
+  };
+}
+
+const CacheProvider = makeCacheProvider(LocalCache, CacheRedis);
+const SessionCacheProvider = makeCacheProvider(SessionCache, SessionRedis);
+const ThrottlerCacheProvider = makeCacheProvider(
+  ThrottlerCache,
+  ThrottlerRedis
+);
+
+@Global()
+@Module({
+  imports: AFFiNE.redis.enabled ? [RedisModule] : [],
+  providers: [CacheProvider, SessionCacheProvider, ThrottlerCacheProvider],
+  exports: [CacheProvider, SessionCacheProvider, ThrottlerCacheProvider],
+})
+export class CacheModule {}
+export { LocalCache as Cache, SessionCache, ThrottlerCache };
+
+export { CacheInterceptor, MakeCache, PreventCache } from './interceptor';

packages/backend/server/src/fundamentals/cache/instances.ts

@@ -0,0 +1,4 @@
+import { LocalCache } from './providers/cache';
+
+export class SessionCache extends LocalCache {}
+export class ThrottlerCache extends LocalCache {}

packages/backend/server/src/fundamentals/cache/interceptor.ts

@@ -0,0 +1,99 @@
+import {
+  CallHandler,
+  ExecutionContext,
+  Injectable,
+  Logger,
+  NestInterceptor,
+  SetMetadata,
+} from '@nestjs/common';
+import { Reflector } from '@nestjs/core';
+import { GqlContextType, GqlExecutionContext } from '@nestjs/graphql';
+import { mergeMap, Observable, of } from 'rxjs';
+
+import { LocalCache } from './providers/cache';
+
+export const MakeCache = (key: string[], args?: string[]) =>
+  SetMetadata('cacheKey', [key, args]);
+export const PreventCache = (key: string[], args?: string[]) =>
+  SetMetadata('preventCache', [key, args]);
+
+type CacheConfig = [string[], string[]?];
+
+@Injectable()
+export class CacheInterceptor implements NestInterceptor {
+  private readonly logger = new Logger(CacheInterceptor.name);
+  constructor(
+    private readonly reflector: Reflector,
+    private readonly cache: LocalCache
+  ) {}
+  async intercept(
+    ctx: ExecutionContext,
+    next: CallHandler<any>
+  ): Promise<Observable<any>> {
+    const key = this.reflector.get<CacheConfig | undefined>(
+      'cacheKey',
+      ctx.getHandler()
+    );
+    const preventKey = this.reflector.get<CacheConfig | undefined>(
+      'preventCache',
+      ctx.getHandler()
+    );
+
+    if (preventKey) {
+      this.logger.debug(`prevent cache: ${JSON.stringify(preventKey)}`);
+      const key = await this.getCacheKey(ctx, preventKey);
+      if (key) {
+        await this.cache.delete(key);
+      }
+
+      return next.handle();
+    } else if (!key) {
+      return next.handle();
+    }
+
+    const cacheKey = await this.getCacheKey(ctx, key);
+
+    if (!cacheKey) {
+      return next.handle();
+    }
+
+    const cachedData = await this.cache.get(cacheKey);
+
+    if (cachedData) {
+      this.logger.debug('cache hit', cacheKey, cachedData);
+      return of(cachedData);
+    } else {
+      return next.handle().pipe(
+        mergeMap(async result => {
+          this.logger.debug('cache miss', cacheKey, result);
+          await this.cache.set(cacheKey, result);
+
+          return result;
+        })
+      );
+    }
+  }
+
+  private async getCacheKey(
+    ctx: ExecutionContext,
+    config: CacheConfig
+  ): Promise<string | null> {
+    const [key, params] = config;
+
+    if (!params) {
+      return key.join(':');
+    } else if (ctx.getType<GqlContextType>() === 'graphql') {
+      const args = GqlExecutionContext.create(ctx).getArgs();
+      const cacheKey = params
+        .map(name => args[name])
+        .filter(v => v)
+        .join(':');
+      if (cacheKey) {
+        return [...key, cacheKey].join(':');
+      } else {
+        return key.join(':');
+      }
+    }
+    return null;
+  }
+}

packages/backend/server/src/fundamentals/cache/providers/cache.ts

@@ -0,0 +1,332 @@
+import { Injectable } from '@nestjs/common';
+import Keyv from 'keyv';
+
+export interface CacheSetOptions {
+  // in milliseconds
+  ttl?: number;
+}
+
+// extends if needed
+export interface Cache {
+  // standard operation
+  get<T = unknown>(key: string): Promise<T | undefined>;
+  set<T = unknown>(
+    key: string,
+    value: T,
+    opts?: CacheSetOptions
+  ): Promise<boolean>;
+  setnx<T = unknown>(
+    key: string,
+    value: T,
+    opts?: CacheSetOptions
+  ): Promise<boolean>;
+  increase(key: string, count?: number): Promise<number>;
+  decrease(key: string, count?: number): Promise<number>;
+  delete(key: string): Promise<boolean>;
+  has(key: string): Promise<boolean>;
+  ttl(key: string): Promise<number>;
+  expire(key: string, ttl: number): Promise<boolean>;
+
+  // list operations
+  pushBack<T = unknown>(key: string, ...values: T[]): Promise<number>;
+  pushFront<T = unknown>(key: string, ...values: T[]): Promise<number>;
+  len(key: string): Promise<number>;
+  list<T = unknown>(key: string, start: number, end: number): Promise<T[]>;
+  popFront<T = unknown>(key: string, count?: number): Promise<T[]>;
+  popBack<T = unknown>(key: string, count?: number): Promise<T[]>;
+
+  // map operations
+  mapSet<T = unknown>(
+    map: string,
+    key: string,
+    value: T,
+    opts: CacheSetOptions
+  ): Promise<boolean>;
+  mapIncrease(map: string, key: string, count?: number): Promise<number>;
+  mapDecrease(map: string, key: string, count?: number): Promise<number>;
+  mapGet<T = unknown>(map: string, key: string): Promise<T | undefined>;
+  mapDelete(map: string, key: string): Promise<boolean>;
+  mapKeys(map: string): Promise<string[]>;
+  mapRandomKey(map: string): Promise<string | undefined>;
+  mapLen(map: string): Promise<number>;
+}
+
+@Injectable()
+export class LocalCache implements Cache {
+  private readonly kv: Keyv;
+
+  constructor(opts: Keyv.Options<any> = {}) {
+    this.kv = new Keyv(opts);
+  }
+
+  // standard operation
+  async get<T = unknown>(key: string): Promise<T | undefined> {
+    return this.kv.get(key).catch(() => undefined);
+  }
+
+  async set<T = unknown>(
+    key: string,
+    value: T,
+    opts: CacheSetOptions = {}
+  ): Promise<boolean> {
+    return this.kv
+      .set(key, value, opts.ttl)
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  async setnx<T = unknown>(
+    key: string,
+    value: T,
+    opts?: CacheSetOptions | undefined
+  ): Promise<boolean> {
+    if (!(await this.has(key))) {
+      return this.set(key, value, opts);
+    }
+    return false;
+  }
+
+  async increase(key: string, count: number = 1): Promise<number> {
+    const prev = (await this.get(key)) ?? 0;
+    if (typeof prev !== 'number') {
+      throw new Error(
+        `Expect a Number keyed by ${key}, but found ${typeof prev}`
+      );
+    }
+
+    const curr = prev + count;
+    return (await this.set(key, curr)) ? curr : prev;
+  }
+
+  async decrease(key: string, count: number = 1): Promise<number> {
+    return this.increase(key, -count);
+  }
+
+  async delete(key: string): Promise<boolean> {
+    return this.kv.delete(key).catch(() => false);
+  }
+
+  async has(key: string): Promise<boolean> {
+    return this.kv.has(key).catch(() => false);
+  }
+
+  async ttl(key: string): Promise<number> {
+    return this.kv
+      .get(key, { raw: true })
+      .then(raw => (raw?.expires ? raw.expires - Date.now() : Infinity))
+      .catch(() => 0);
+  }
+
+  async expire(key: string, ttl: number): Promise<boolean> {
+    const value = await this.kv.get(key);
+    return this.set(key, value, { ttl });
+  }
+
+  // list operations
+  private async getArray<T = unknown>(key: string) {
+    const raw = await this.kv.get(key, { raw: true });
+    if (raw && !Array.isArray(raw.value)) {
+      throw new Error(
+        `Expect an Array keyed by ${key}, but found ${raw.value}`
+      );
+    }
+
+    return raw as Keyv.DeserializedData<T[]>;
+  }
+
+  private async setArray<T = unknown>(
+    key: string,
+    value: T[],
+    opts: CacheSetOptions = {}
+  ) {
+    return this.set(key, value, opts).then(() => value.length);
+  }
+
+  async pushBack<T = unknown>(key: string, ...values: T[]): Promise<number> {
+    let list: any[] = [];
+    let ttl: number | undefined = undefined;
+    const raw = await this.getArray(key);
+    if (raw) {
+      list = raw.value;
+      if (raw.expires) {
+        ttl = raw.expires - Date.now();
+      }
+    }
+
+    list = list.concat(values);
+    return this.setArray(key, list, { ttl });
+  }
+
+  async pushFront<T = unknown>(key: string, ...values: T[]): Promise<number> {
+    let list: any[] = [];
+    let ttl: number | undefined = undefined;
+    const raw = await this.getArray(key);
+    if (raw) {
+      list = raw.value;
+      if (raw.expires) {
+        ttl = raw.expires - Date.now();
+      }
+    }
+
+    list = values.concat(list);
+    return this.setArray(key, list, { ttl });
+  }
+
+  async len(key: string): Promise<number> {
+    return this.getArray(key).then(v => v?.value.length ?? 0);
+  }
+
+  /**
+   * list array elements with `[start, end]`
+   * the end indice is inclusive
+   */
+  async list<T = unknown>(
+    key: string,
+    start: number,
+    end: number
+  ): Promise<T[]> {
+    const raw = await this.getArray<T>(key);
+    if (raw?.value) {
+      start = (raw.value.length + start) % raw.value.length;
+      end = ((raw.value.length + end) % raw.value.length) + 1;
+      return raw.value.slice(start, end);
+    } else {
+      return [];
+    }
+  }
+
+  private async trim<T = unknown>(key: string, start: number, end: number) {
+    const raw = await this.getArray<T>(key);
+    if (raw) {
+      start = (raw.value.length + start) % raw.value.length;
+      // make negative end index work, and end indice is inclusive
+      end = ((raw.value.length + end) % raw.value.length) + 1;
+      const result = raw.value.splice(start, end);
+
+      await this.set(key, raw.value, {
+        ttl: raw.expires ? raw.expires - Date.now() : undefined,
+      });
+
+      return result;
+    }
+
+    return [];
+  }
+
+  async popFront<T = unknown>(key: string, count: number = 1) {
+    return this.trim<T>(key, 0, count - 1);
+  }
+
+  async popBack<T = unknown>(key: string, count: number = 1) {
+    return this.trim<T>(key, -count, count - 1);
+  }
+
+  // map operations
+  private async getMap<T = unknown>(map: string) {
+    const raw = await this.kv.get(map, { raw: true });
+
+    if (raw) {
+      if (typeof raw.value !== 'object') {
+        throw new Error(
+          `Expect an Object keyed by ${map}, but found ${typeof raw}`
+        );
+      }
+
+      if (Array.isArray(raw.value)) {
+        throw new Error(`Expect an Object keyed by ${map}, but found an Array`);
+      }
+    }
+
+    return raw as Keyv.DeserializedData<Record<string, T>>;
+  }
+
+  private async setMap<T = unknown>(
+    map: string,
+    value: Record<string, T>,
+    opts: CacheSetOptions = {}
+  ) {
+    return this.kv.set(map, value, opts.ttl).then(() => true);
+  }
+
+  async mapGet<T = unknown>(map: string, key: string): Promise<T | undefined> {
+    const raw = await this.getMap<T>(map);
+    if (raw?.value) {
+      return raw.value[key];
+    }
+
+    return undefined;
+  }
+
+  async mapSet<T = unknown>(
+    map: string,
+    key: string,
+    value: T
+  ): Promise<boolean> {
+    const raw = await this.getMap(map);
+    const data = raw?.value ?? {};
+
+    data[key] = value;
+
+    return this.setMap(map, data, {
+      ttl: raw?.expires ? raw.expires - Date.now() : undefined,
+    });
+  }
+
+  async mapDelete(map: string, key: string): Promise<boolean> {
+    const raw = await this.getMap(map);
+
+    if (raw?.value) {
+      delete raw.value[key];
+      return this.setMap(map, raw.value, {
+        ttl: raw.expires ? raw.expires - Date.now() : undefined,
+      });
+    }
+
+    return false;
+  }
+
+  async mapIncrease(
+    map: string,
+    key: string,
+    count: number = 1
+  ): Promise<number> {
+    const prev = (await this.mapGet(map, key)) ?? 0;
+
+    if (typeof prev !== 'number') {
+      throw new Error(
+        `Expect a Number keyed by ${key}, but found ${typeof prev}`
+      );
+    }
+
+    const curr = prev + count;
+
+    return (await this.mapSet(map, key, curr)) ? curr : prev;
+  }
+
+  async mapDecrease(
+    map: string,
+    key: string,
+    count: number = 1
+  ): Promise<number> {
+    return this.mapIncrease(map, key, -count);
+  }
+
+  async mapKeys(map: string): Promise<string[]> {
+    const raw = await this.getMap(map);
+    if (raw) {
+      return Object.keys(raw.value);
+    }
+
+    return [];
+  }
+
+  async mapRandomKey(map: string): Promise<string | undefined> {
+    const keys = await this.mapKeys(map);
+    return keys[Math.floor(Math.random() * keys.length)];
+  }
+
+  async mapLen(map: string): Promise<number> {
+    const raw = await this.getMap(map);
+    return raw ? Object.keys(raw.value).length : 0;
+  }
+}

packages/backend/server/src/fundamentals/cache/providers/redis.ts

@@ -0,0 +1,194 @@
+import { Redis } from 'ioredis';
+
+import { Cache, CacheSetOptions } from './cache';
+
+export class RedisCache implements Cache {
+  constructor(private readonly redis: Redis) {}
+
+  // standard operation
+  async get<T = unknown>(key: string): Promise<T> {
+    return this.redis
+      .get(key)
+      .then(v => {
+        if (v) {
+          return JSON.parse(v);
+        }
+        return undefined;
+      })
+      .catch(() => undefined);
+  }
+
+  async set<T = unknown>(
+    key: string,
+    value: T,
+    opts: CacheSetOptions = {}
+  ): Promise<boolean> {
+    if (opts.ttl) {
+      return this.redis
+        .set(key, JSON.stringify(value), 'PX', opts.ttl)
+        .then(() => true)
+        .catch(() => false);
+    }
+
+    return this.redis
+      .set(key, JSON.stringify(value))
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  async increase(key: string, count: number = 1): Promise<number> {
+    return this.redis.incrby(key, count).catch(() => 0);
+  }
+
+  async decrease(key: string, count: number = 1): Promise<number> {
+    return this.redis.decrby(key, count).catch(() => 0);
+  }
+
+  async setnx<T = unknown>(
+    key: string,
+    value: T,
+    opts: CacheSetOptions = {}
+  ): Promise<boolean> {
+    if (opts.ttl) {
+      return this.redis
+        .set(key, JSON.stringify(value), 'PX', opts.ttl, 'NX')
+        .then(v => !!v)
+        .catch(() => false);
+    }
+
+    return this.redis
+      .set(key, JSON.stringify(value), 'NX')
+      .then(v => !!v)
+      .catch(() => false);
+  }
+
+  async delete(key: string): Promise<boolean> {
+    return this.redis
+      .del(key)
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  async has(key: string): Promise<boolean> {
+    return this.redis
+      .exists(key)
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  async ttl(key: string): Promise<number> {
+    return this.redis.ttl(key).catch(() => 0);
+  }
+
+  async expire(key: string, ttl: number): Promise<boolean> {
+    return this.redis
+      .pexpire(key, ttl)
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  // list operations
+  async pushBack<T = unknown>(key: string, ...values: T[]): Promise<number> {
+    return this.redis
+      .rpush(key, ...values.map(v => JSON.stringify(v)))
+      .catch(() => 0);
+  }
+
+  async pushFront<T = unknown>(key: string, ...values: T[]): Promise<number> {
+    return this.redis
+      .lpush(key, ...values.map(v => JSON.stringify(v)))
+      .catch(() => 0);
+  }
+
+  async len(key: string): Promise<number> {
+    return this.redis.llen(key).catch(() => 0);
+  }
+
+  async list<T = unknown>(
+    key: string,
+    start: number,
+    end: number
+  ): Promise<T[]> {
+    return this.redis
+      .lrange(key, start, end)
+      .then(data => data.map(v => JSON.parse(v)))
+      .catch(() => []);
+  }
+
+  async popFront<T = unknown>(key: string, count: number = 1): Promise<T[]> {
+    return this.redis
+      .lpop(key, count)
+      .then(data => (data ?? []).map(v => JSON.parse(v)))
+      .catch(() => []);
+  }
+
+  async popBack<T = unknown>(key: string, count: number = 1): Promise<T[]> {
+    return this.redis
+      .rpop(key, count)
+      .then(data => (data ?? []).map(v => JSON.parse(v)))
+      .catch(() => []);
+  }
+
+  // map operations
+  async mapSet<T = unknown>(
+    map: string,
+    key: string,
+    value: T
+  ): Promise<boolean> {
+    return this.redis
+      .hset(map, key, JSON.stringify(value))
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  async mapIncrease(
+    map: string,
+    key: string,
+    count: number = 1
+  ): Promise<number> {
+    return this.redis.hincrby(map, key, count);
+  }
+
+  async mapDecrease(
+    map: string,
+    key: string,
+    count: number = 1
+  ): Promise<number> {
+    return this.redis.hincrby(map, key, -count);
+  }
+
+  async mapGet<T = unknown>(map: string, key: string): Promise<T | undefined> {
+    return this.redis
+      .hget(map, key)
+      .then(v => (v ? JSON.parse(v) : undefined))
+      .catch(() => undefined);
+  }
+
+  async mapDelete(map: string, key: string): Promise<boolean> {
+    return this.redis
+      .hdel(map, key)
+      .then(v => v > 0)
+      .catch(() => false);
+  }
+
+  async mapKeys(map: string): Promise<string[]> {
+    return this.redis.hkeys(map).catch(() => []);
+  }
+
+  async mapRandomKey(map: string): Promise<string | undefined> {
+    return this.redis
+      .hrandfield(map, 1)
+      .then(v =>
+        typeof v === 'string'
+          ? v
+          : Array.isArray(v)
+            ? (v[0] as string)
+            : undefined
+      )
+      .catch(() => undefined);
+  }
+
+  async mapLen(map: string): Promise<number> {
+    return this.redis.hlen(map).catch(() => 0);
+  }
+}

packages/backend/server/src/fundamentals/cache/redis/index.ts

@@ -0,0 +1,38 @@
+import { Global, Injectable, Module, OnModuleDestroy } from '@nestjs/common';
+import { Redis as IORedis } from 'ioredis';
+
+import { Config } from '../../config';
+
+class Redis extends IORedis implements OnModuleDestroy {
+  onModuleDestroy() {
+    this.disconnect();
+  }
+}
+
+@Injectable()
+export class CacheRedis extends Redis {
+  constructor(config: Config) {
+    super({ ...config.redis, db: config.redis.database });
+  }
+}
+
+@Injectable()
+export class ThrottlerRedis extends Redis {
+  constructor(config: Config) {
+    super({ ...config.redis, db: config.redis.database + 1 });
+  }
+}
+
+@Injectable()
+export class SessionRedis extends Redis {
+  constructor(config: Config) {
+    super({ ...config.redis, db: config.redis.database + 2 });
+  }
+}
+
+@Global()
+@Module({
+  providers: [CacheRedis, ThrottlerRedis, SessionRedis],
+  exports: [CacheRedis, ThrottlerRedis, SessionRedis],
+})
+export class RedisModule {}