feat(server): enable share og information for docs (#7794)

This commit is contained in:
forehalo
2024-09-10 04:03:52 +00:00
parent 34eac4c24e
commit 0add8917f9
24 changed files with 449 additions and 40 deletions

View File

@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "workspaces" ADD COLUMN "enable_url_preview" BOOLEAN NOT NULL DEFAULT false;

View File

@@ -94,6 +94,7 @@
"ts-node": "^10.9.2",
"typescript": "^5.4.5",
"ws": "^8.16.0",
"xss": "^1.0.15",
"yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch",
"zod": "^3.22.4"
},

View File

@@ -97,9 +97,10 @@ model VerificationToken {
}
model Workspace {
id String @id @default(uuid()) @db.VarChar
public Boolean
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
id String @id @default(uuid()) @db.VarChar
public Boolean
enableUrlPreview Boolean @default(false) @map("enable_url_preview")
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
pages WorkspacePage[]
permissions WorkspaceUserPermission[]

View File

@@ -11,6 +11,7 @@ import { AppController } from './app.controller';
import { AuthModule } from './core/auth';
import { ADD_ENABLED_FEATURES, ServerConfigModule } from './core/config';
import { DocStorageModule } from './core/doc';
import { DocRendererModule } from './core/doc-renderer';
import { FeatureModule } from './core/features';
import { PermissionModule } from './core/permission';
import { QuotaModule } from './core/quota';
@@ -42,7 +43,6 @@ import { ENABLED_PLUGINS } from './plugins/registry';
export const FunctionalityModules = [
ConfigModule.forRoot(),
ScheduleModule.forRoot(),
EventModule,
CacheModule,
MutexModule,
@@ -156,7 +156,7 @@ export function buildAppModule() {
.use(UserModule, AuthModule, PermissionModule)
// business modules
.use(DocStorageModule)
.use(FeatureModule, QuotaModule, DocStorageModule)
// sync server only
.useIf(config => config.flavor.sync, SyncModule)
@@ -164,16 +164,16 @@ export function buildAppModule() {
// graphql server only
.useIf(
config => config.flavor.graphql,
ScheduleModule.forRoot(),
GqlModule,
StorageModule,
ServerConfigModule,
WorkspaceModule,
FeatureModule,
QuotaModule
WorkspaceModule
)
// self hosted server only
.useIf(config => config.isSelfhosted, SelfhostModule);
.useIf(config => config.isSelfhosted, SelfhostModule)
.useIf(config => config.flavor.renderer, DocRendererModule);
// plugin modules
ENABLED_PLUGINS.forEach(name => {

View File

@@ -0,0 +1,95 @@
import { Controller, Get, Param, Res } from '@nestjs/common';
import type { Response } from 'express';
import xss from 'xss';
import { DocNotFound } from '../../fundamentals';
import { PermissionService } from '../permission';
import { PageDocContent } from '../utils/blocksuite';
import { DocContentService } from './service';
interface RenderOptions {
og: boolean;
content: boolean;
}
@Controller('/workspace/:workspaceId/:docId')
export class DocRendererController {
constructor(
private readonly doc: DocContentService,
private readonly permission: PermissionService
) {}
@Get()
async render(
@Res() res: Response,
@Param('workspaceId') workspaceId: string,
@Param('docId') docId: string
) {
if (workspaceId === docId) {
throw new DocNotFound({ spaceId: workspaceId, docId });
}
// if page is public, show all
// if page is private, but workspace public og is on, show og but not content
const opts: RenderOptions = {
og: false,
content: false,
};
const isPagePublic = await this.permission.isPublicPage(workspaceId, docId);
if (isPagePublic) {
opts.og = true;
opts.content = true;
} else {
const allowPreview = await this.permission.allowUrlPreview(workspaceId);
if (allowPreview) {
opts.og = true;
}
}
let docContent = opts.og
? await this.doc.getPageContent(workspaceId, docId)
: null;
if (!docContent) {
docContent = { title: 'untitled', summary: '' };
}
res.setHeader('Content-Type', 'text/html');
if (!opts.og) {
res.setHeader('X-Robots-Tag', 'noindex');
}
res.send(this._render(docContent, opts));
}
_render(doc: PageDocContent, { og }: RenderOptions): string {
const title = xss(doc.title);
const summary = xss(doc.summary);
return `
<!DOCTYPE html>
<html>
<head>
<title>${title} | AFFiNE</title>
<meta name="theme-color" content="#fafafa" />
<link rel="manifest" href="/manifest.json" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
<link rel="icon" sizes="192x192" href="/favicon-192.png" />
${!og ? '<meta name="robots" content="noindex, nofollow" />' : ''}
<meta
name="twitter:title"
content="AFFiNE: There can be more than Notion and Miro."
/>
<meta name="twitter:description" content="${title}" />
<meta name="twitter:site" content="@AffineOfficial" />
<meta name="twitter:image" content="https://affine.pro/og.jpeg" />
<meta property="og:title" content="${title}" />
<meta property="og:description" content="${summary}" />
<meta property="og:image" content="https://affine.pro/og.jpeg" />
</head>
<body>
</body>
</html>
`;
}
}

View File

@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { DocStorageModule } from '../doc';
import { PermissionModule } from '../permission';
import { DocRendererController } from './controller';
import { DocContentService } from './service';
@Module({
imports: [DocStorageModule, PermissionModule],
providers: [DocContentService],
controllers: [DocRendererController],
exports: [DocContentService],
})
export class DocRendererModule {}
export { DocContentService };

View File

@@ -0,0 +1,88 @@
import { Injectable } from '@nestjs/common';
import { applyUpdate, Doc } from 'yjs';
import { Cache } from '../../fundamentals';
import { PgWorkspaceDocStorageAdapter } from '../doc';
import {
type PageDocContent,
parsePageDoc,
parseWorkspaceDoc,
type WorkspaceDocContent,
} from '../utils/blocksuite';
@Injectable()
export class DocContentService {
constructor(
private readonly cache: Cache,
private readonly workspace: PgWorkspaceDocStorageAdapter
) {}
async getPageContent(
workspaceId: string,
guid: string
): Promise<PageDocContent | null> {
const cacheKey = `workspace:${workspaceId}:doc:${guid}:content`;
const cachedResult = await this.cache.get<PageDocContent>(cacheKey);
if (cachedResult) {
return cachedResult;
}
const docRecord = await this.workspace.getDoc(workspaceId, guid);
if (!docRecord) {
return null;
}
const doc = new Doc();
applyUpdate(doc, docRecord.bin);
const content = parsePageDoc(doc);
if (content) {
await this.cache.set(cacheKey, content, {
ttl:
7 *
24 *
60 *
60 *
1000 /* TODO(@forehalo): we need time constants helper */,
});
}
return content;
}
async getWorkspaceContent(
workspaceId: string
): Promise<WorkspaceDocContent | null> {
const cacheKey = `workspace:${workspaceId}:content`;
const cachedResult = await this.cache.get<WorkspaceDocContent>(cacheKey);
if (cachedResult) {
return cachedResult;
}
const docRecord = await this.workspace.getDoc(workspaceId, workspaceId);
if (!docRecord) {
return null;
}
const doc = new Doc();
applyUpdate(doc, docRecord.bin);
const content = parseWorkspaceDoc(doc);
if (content) {
await this.cache.set(cacheKey, content);
}
return content;
}
async markDocContentCacheStale(workspaceId: string, guid: string) {
const key =
workspaceId === guid
? `workspace:${workspaceId}:content`
: `workspace:${workspaceId}:doc:${guid}:content`;
await this.cache.delete(key);
}
}

View File

@@ -1,4 +1,4 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { Injectable, Logger, OnModuleInit, Optional } from '@nestjs/common';
import { Cron, CronExpression, SchedulerRegistry } from '@nestjs/schedule';
import { PrismaClient } from '@prisma/client';
@@ -11,14 +11,14 @@ export class DocStorageCronJob implements OnModuleInit {
private busy = false;
constructor(
private readonly registry: SchedulerRegistry,
private readonly config: Config,
private readonly db: PrismaClient,
private readonly workspace: PgWorkspaceDocStorageAdapter
private readonly workspace: PgWorkspaceDocStorageAdapter,
@Optional() private readonly registry?: SchedulerRegistry
) {}
onModuleInit() {
if (this.config.doc.manager.enableUpdateAutoMerging) {
if (this.registry && this.config.doc.manager.enableUpdateAutoMerging) {
this.registry.addInterval(
this.autoMergePendingDocUpdates.name,
// scheduler registry will clean up the interval when the app is stopped

View File

@@ -212,7 +212,7 @@ export class PermissionService {
const count = await this.prisma.workspace.count({
where: {
id: ws,
public: true,
enableUrlPreview: true,
},
});

View File

@@ -0,0 +1,129 @@
// TODO(@forehalo):
// Because of the `@affine/server` package can't import directly from workspace packages,
// this is a temprory solution to get the block suite data(title, description) from given yjs binary or yjs doc.
// The logic is mainly copied from
// - packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
// - packages/frontend/core/src/components/page-list/use-block-suite-page-preview.ts
// and it's better to be provided by blocksuite
import { Array, Doc, Map } from 'yjs';
export interface PageDocContent {
title: string;
summary: string;
}
export interface WorkspaceDocContent {
name: string;
avatarKey: string;
}
type KnownFlavour =
| 'affine:page'
| 'affine:note'
| 'affine:surface'
| 'affine:paragraph'
| 'affine:list'
| 'affine:code'
| 'affine:image';
export function parseWorkspaceDoc(doc: Doc): WorkspaceDocContent | null {
// not a workspace doc
if (!doc.share.has('meta')) {
return null;
}
const meta = doc.getMap('meta');
return {
name: meta.get('name') as string,
avatarKey: meta.get('avatar') as string,
};
}
export interface ParsePageOptions {
maxSummaryLength: number;
}
export function parsePageDoc(
doc: Doc,
opts: ParsePageOptions = { maxSummaryLength: 150 }
): PageDocContent | null {
// not a page doc
if (!doc.share.has('blocks')) {
return null;
}
const blocks = doc.getMap<Map<any>>('blocks');
if (!blocks.size) {
return null;
}
const content: PageDocContent = {
title: '',
summary: '',
};
let summaryLenNeeded = opts.maxSummaryLength;
let root: Map<any> | null = null;
for (const block of blocks.values()) {
const flavour = block.get('sys:flavour') as KnownFlavour;
if (flavour === 'affine:page') {
content.title = block.get('prop:title') as string;
root = block;
}
}
if (!root) {
return null;
}
const queue: string[] = [root.get('sys:id')];
function pushChildren(block: Map<any>) {
const children = block.get('sys:children') as Array<string> | undefined;
if (children?.length) {
for (let i = children.length - 1; i >= 0; i--) {
queue.push(children.get(i));
}
}
}
while (queue.length) {
const blockId = queue.pop();
const block = blockId ? blocks.get(blockId) : null;
if (!block) {
break;
}
const flavour = block.get('sys:flavour') as KnownFlavour;
switch (flavour) {
case 'affine:page':
case 'affine:note': {
pushChildren(block);
break;
}
case 'affine:paragraph':
case 'affine:list':
case 'affine:code': {
pushChildren(block);
const text = block.get('prop:text');
if (!text) {
continue;
}
if (summaryLenNeeded > 0) {
content.summary += text.toString();
summaryLenNeeded -= text.length;
} else {
break;
}
}
}
}
return content;
}

View File

@@ -1,6 +1,7 @@
import { Module } from '@nestjs/common';
import { DocStorageModule } from '../doc';
import { DocRendererModule } from '../doc-renderer';
import { FeatureModule } from '../features';
import { PermissionModule } from '../permission';
import { QuotaModule } from '../quota';
@@ -18,6 +19,7 @@ import {
@Module({
imports: [
DocStorageModule,
DocRendererModule,
FeatureModule,
QuotaModule,
StorageModule,

View File

@@ -13,7 +13,6 @@ import {
import { PrismaClient } from '@prisma/client';
import { getStreamAsBuffer } from 'get-stream';
import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
import { applyUpdate, Doc } from 'yjs';
import type { FileUpload } from '../../../fundamentals';
import {
@@ -32,6 +31,7 @@ import {
} from '../../../fundamentals';
import { CurrentUser, Public } from '../../auth';
import type { Editor } from '../../doc';
import { DocContentService } from '../../doc-renderer';
import { Permission, PermissionService } from '../../permission';
import { QuotaManagementService, QuotaQueryType } from '../../quota';
import { WorkspaceBlobStorage } from '../../storage';
@@ -85,7 +85,8 @@ export class WorkspaceResolver {
private readonly users: UserService,
private readonly event: EventEmitter,
private readonly blobStorage: WorkspaceBlobStorage,
private readonly mutex: RequestMutex
private readonly mutex: RequestMutex,
private readonly doc: DocContentService
) {}
@ResolveField(() => Permission, {
@@ -471,17 +472,7 @@ export class WorkspaceResolver {
})
.then(({ workspaceId }) => workspaceId);
const snapshot = await this.prisma.snapshot.findFirstOrThrow({
where: {
id: workspaceId,
workspaceId,
},
});
const doc = new Doc();
applyUpdate(doc, new Uint8Array(snapshot.blob));
const metaJSON = doc.getMap('meta').toJSON();
const workspaceContent = await this.doc.getWorkspaceContent(workspaceId);
const owner = await this.permissions.getWorkspaceOwner(workspaceId);
const invitee = await this.permissions.getWorkspaceInvitation(
@@ -490,11 +481,10 @@ export class WorkspaceResolver {
);
let avatar = '';
if (metaJSON.avatar) {
if (workspaceContent?.avatarKey) {
const avatarBlob = await this.blobStorage.get(
workspaceId,
metaJSON.avatar
workspaceContent.avatarKey
);
if (avatarBlob.body) {
@@ -504,7 +494,7 @@ export class WorkspaceResolver {
return {
workspace: {
name: metaJSON.name || '',
name: workspaceContent?.name ?? '',
avatar: avatar || defaultWorkspaceAvatar,
id: workspaceId,
},

View File

@@ -46,6 +46,9 @@ export class WorkspaceType implements Partial<Workspace> {
@Field({ description: 'is Public workspace' })
public!: boolean;
@Field({ description: 'Enable url previous when sharing' })
enableUrlPreview!: boolean;
@Field({ description: 'Workspace created date' })
createdAt!: Date;
@@ -89,7 +92,7 @@ export class InvitationType {
@InputType()
export class UpdateWorkspaceInput extends PickType(
PartialType(WorkspaceType),
['public'],
['public', 'enableUrlPreview'],
InputType
) {
@Field(() => ID)

View File

@@ -2,7 +2,7 @@ import type { LeafPaths } from '../utils/types';
import { AppStartupConfig } from './types';
export type EnvConfigType = 'string' | 'int' | 'float' | 'boolean';
export type ServerFlavor = 'allinone' | 'graphql' | 'sync';
export type ServerFlavor = 'allinone' | 'graphql' | 'sync' | 'renderer';
export type AFFINE_ENV = 'dev' | 'beta' | 'production';
export type NODE_ENV = 'development' | 'test' | 'production';
@@ -23,7 +23,7 @@ export interface PreDefinedAFFiNEConfig {
readonly version: string;
readonly type: DeploymentType;
readonly isSelfhosted: boolean;
readonly flavor: { type: string; graphql: boolean; sync: boolean };
readonly flavor: { type: string } & { [key in ServerFlavor]: boolean };
readonly affine: { canary: boolean; beta: boolean; stable: boolean };
readonly node: { prod: boolean; dev: boolean; test: boolean };
readonly deploy: boolean;

View File

@@ -28,6 +28,7 @@ function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
'allinone',
'graphql',
'sync',
'renderer',
]);
const deploymentType = readEnv<DeploymentType>(
'DEPLOYMENT_TYPE',
@@ -59,8 +60,10 @@ function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
isSelfhosted,
flavor: {
type: flavor,
allinone: flavor === 'allinone',
graphql: flavor === 'graphql' || flavor === 'allinone',
sync: flavor === 'sync' || flavor === 'allinone',
renderer: flavor === 'renderer' || flavor === 'allinone',
},
affine,
node,

View File

@@ -87,6 +87,17 @@ export class UserFriendlyError extends Error {
};
}
toText() {
const json = this.toJSON();
return [
`Status: ${json.status}`,
`Type: ${json.type}`,
`Name: ${json.name}`,
`Message: ${json.message}`,
`Data: ${JSON.stringify(json.data)}`,
].join('\n');
}
log(context: string) {
// ignore all user behavior error log
if (this.type !== 'internal_server_error') {

View File

@@ -48,7 +48,16 @@ export class GlobalExceptionFilter extends BaseExceptionFilter {
error.log('HTTP');
metrics.controllers.counter('error').add(1, { status: error.status });
const res = host.switchToHttp().getResponse<Response>();
res.status(error.status).send(error.toJSON());
const respondText = res.getHeader('content-type') === 'text/plain';
if (respondText) {
res
.setHeader('content-type', 'text/plain')
.status(error.status)
.send(error.toText());
} else {
res.status(error.status).send(error.toJSON());
}
return;
}
}

View File

@@ -773,6 +773,8 @@ input UpdateUserInput {
}
input UpdateWorkspaceInput {
"""Enable url previous when sharing"""
enableUrlPreview: Boolean
id: ID!
"""is Public workspace"""
@@ -904,6 +906,9 @@ type WorkspaceType {
"""Workspace created date"""
createdAt: DateTime!
"""Enable url previous when sharing"""
enableUrlPreview: Boolean!
"""Enabled features of workspace"""
features: [FeatureType!]!
histories(before: DateTime, guid: String!, take: Int): [DocHistoryType!]!

View File

@@ -14,7 +14,13 @@ const test = ava as TestFn<{
test.before('start app', async t => {
// @ts-expect-error override
AFFiNE.flavor = { type: 'graphql', graphql: true, sync: false };
AFFiNE.flavor = {
type: 'graphql',
allinone: false,
graphql: true,
sync: false,
renderer: false,
} satisfies typeof AFFiNE.flavor;
const { app } = await createTestingApp({
imports: [buildAppModule()],
});

View File

@@ -0,0 +1,39 @@
import type { INestApplication } from '@nestjs/common';
import type { TestFn } from 'ava';
import ava from 'ava';
import request from 'supertest';
import { buildAppModule } from '../../src/app.module';
import { createTestingApp } from '../utils';
const test = ava as TestFn<{
app: INestApplication;
}>;
test.before('start app', async t => {
// @ts-expect-error override
AFFiNE.flavor = {
type: 'renderer',
allinone: false,
graphql: false,
sync: false,
renderer: true,
} satisfies typeof AFFiNE.flavor;
const { app } = await createTestingApp({
imports: [buildAppModule()],
});
t.context.app = app;
});
test.after.always(async t => {
await t.context.app.close();
});
test('should init app', async t => {
const res = await request(t.context.app.getHttpServer())
.get('/info')
.expect(200);
t.is(res.body.flavor, 'renderer');
});

View File

@@ -12,7 +12,13 @@ const test = ava as TestFn<{
test.before('start app', async t => {
// @ts-expect-error override
AFFiNE.flavor = { type: 'sync', graphql: false, sync: true };
AFFiNE.flavor = {
type: 'sync',
allinone: false,
graphql: false,
sync: true,
renderer: false,
} satisfies typeof AFFiNE.flavor;
const { app } = await createTestingApp({
imports: [buildAppModule()],
});

View File

@@ -1,5 +1,6 @@
import { mock } from 'node:test';
import { ScheduleModule } from '@nestjs/schedule';
import { TestingModule } from '@nestjs/testing';
import { PrismaClient } from '@prisma/client';
import test from 'ava';
@@ -20,7 +21,7 @@ test.before(async () => {
toFake: ['setInterval'],
});
m = await createTestingModule({
imports: [DocStorageModule],
imports: [ScheduleModule.forRoot(), DocStorageModule],
});
db = m.get(PrismaClient);

View File

@@ -52,7 +52,7 @@ class TestResolver {
}
@Public()
@Controller()
@Controller('/')
class TestController {
@Get('/ok')
ok() {
@@ -154,6 +154,7 @@ test('should be able to handle known user error in http request', async t => {
const res = await request(t.context.app.getHttpServer())
.get('/throw-known-error')
.expect(HttpStatus.FORBIDDEN);
t.is(res.body.message, 'You do not have permission to access this resource.');
t.is(res.body.name, 'ACCESS_DENIED');
t.true(t.context.logger.error.notCalled);

View File

@@ -896,6 +896,7 @@ __metadata:
ts-node: "npm:^10.9.2"
typescript: "npm:^5.4.5"
ws: "npm:^8.16.0"
xss: "npm:^1.0.15"
yjs: "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch"
zod: "npm:^3.22.4"
bin:
@@ -36500,7 +36501,7 @@ __metadata:
languageName: node
linkType: hard
"xss@npm:^1.0.8":
"xss@npm:^1.0.15, xss@npm:^1.0.8":
version: 1.0.15
resolution: "xss@npm:1.0.15"
dependencies: