feat(infra): framework

This commit is contained in:
EYHN
2024-04-17 14:12:29 +08:00
parent ab17a05df3
commit 06fda3b62c
467 changed files with 9996 additions and 8697 deletions

View File

@@ -1,15 +1,8 @@
import { nanoid } from 'nanoid';
import type { Mock } from 'vitest';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { gqlFetcherFactory } from '../fetcher';
import type { GraphQLQuery } from '../graphql';
import {
generateRandUTF16Chars,
SPAN_ID_BYTES,
TRACE_ID_BYTES,
TraceReporter,
} from '../utils';
const query: GraphQLQuery = {
id: 'query',
@@ -19,6 +12,7 @@ const query: GraphQLQuery = {
};
let fetch: Mock;
let gql: ReturnType<typeof gqlFetcherFactory>;
describe('GraphQL fetcher', () => {
beforeEach(() => {
fetch = vi.fn(() =>
@@ -30,15 +24,13 @@ describe('GraphQL fetcher', () => {
})
)
);
vi.stubGlobal('fetch', fetch);
gql = gqlFetcherFactory('https://example.com/graphql', fetch);
});
afterEach(() => {
fetch.mockReset();
});
const gql = gqlFetcherFactory('https://example.com/graphql');
it('should send POST request to given endpoint', async () => {
await gql(
// @ts-expect-error variables is actually optional
@@ -65,7 +57,6 @@ describe('GraphQL fetcher', () => {
'content-type': 'application/json',
'x-definition-name': 'query',
'x-operation-name': 'query',
'x-request-id': expect.any(String),
}),
method: 'POST',
})
@@ -119,41 +110,3 @@ describe('GraphQL fetcher', () => {
`);
});
});
describe('Trace Reporter', () => {
const startTime = new Date().toISOString();
const traceId = generateRandUTF16Chars(TRACE_ID_BYTES);
const spanId = generateRandUTF16Chars(SPAN_ID_BYTES);
const requestId = nanoid();
it('spanId, traceId should be right format', () => {
expect(
new RegExp(`^[0-9a-f]{${SPAN_ID_BYTES * 2}}$`).test(
generateRandUTF16Chars(SPAN_ID_BYTES)
)
).toBe(true);
expect(
new RegExp(`^[0-9a-f]{${TRACE_ID_BYTES * 2}}$`).test(
generateRandUTF16Chars(TRACE_ID_BYTES)
)
).toBe(true);
});
it('test createTraceSpan', () => {
const traceSpan = TraceReporter.createTraceSpan(
traceId,
spanId,
startTime,
{ requestId }
);
expect(traceSpan.startTime).toBe(startTime);
expect(
traceSpan.name ===
`projects/{GCP_PROJECT_ID}/traces/${traceId}/spans/${spanId}`
).toBe(true);
expect(traceSpan.spanId).toBe(spanId);
expect(traceSpan.attributes.attributeMap.requestId?.stringValue.value).toBe(
requestId
);
});
});

View File

@@ -1,18 +1,9 @@
import type { ExecutionResult } from 'graphql';
import { GraphQLError } from 'graphql';
import { isNil, isObject, merge } from 'lodash-es';
import { nanoid } from 'nanoid';
import type { GraphQLQuery } from './graphql';
import type { Mutations, Queries } from './schema';
import {
generateRandUTF16Chars,
SPAN_ID_BYTES,
TRACE_FLAG,
TRACE_ID_BYTES,
TRACE_VERSION,
traceReporter,
} from './utils';
export type NotArray<T> = T extends Array<unknown> ? never : T;
@@ -166,7 +157,10 @@ function formatRequestBody<Q extends GraphQLQuery>({
return body;
}
export const gqlFetcherFactory = (endpoint: string) => {
export const gqlFetcherFactory = (
endpoint: string,
fetcher: (input: string, init?: RequestInit) => Promise<Response> = fetch
) => {
const gqlFetch = async <Query extends GraphQLQuery>(
options: QueryOptions<Query>
): Promise<QueryResponse<Query>> => {
@@ -180,14 +174,13 @@ export const gqlFetcherFactory = (endpoint: string) => {
if (!isFormData) {
headers['content-type'] = 'application/json';
}
const ret = fetchWithTraceReport(
const ret = fetcher(
endpoint,
merge(options.context, {
method: 'POST',
headers,
body: isFormData ? body : JSON.stringify(body),
}),
{ event: 'GraphQLRequest' }
})
).then(async res => {
if (res.headers.get('content-type')?.startsWith('application/json')) {
const result = (await res.json()) as ExecutionResult;
@@ -205,7 +198,10 @@ export const gqlFetcherFactory = (endpoint: string) => {
}
}
throw new GraphQLError('GraphQL query responds unexpected result');
throw new GraphQLError(
'GraphQL query responds unexpected result, query ' +
options.query.operationName
);
});
return ret;
@@ -213,47 +209,3 @@ export const gqlFetcherFactory = (endpoint: string) => {
return gqlFetch;
};
export const fetchWithTraceReport = async (
input: RequestInfo | URL,
init?: RequestInit & { priority?: 'auto' | 'low' | 'high' }, // https://github.com/microsoft/TypeScript/issues/54472
traceOptions?: { event: string }
): Promise<Response> => {
const startTime = new Date().toISOString();
const spanId = generateRandUTF16Chars(SPAN_ID_BYTES);
const traceId = generateRandUTF16Chars(TRACE_ID_BYTES);
const traceparent = `${TRACE_VERSION}-${traceId}-${spanId}-${TRACE_FLAG}`;
init = init || {};
init.headers = init.headers || new Headers();
const requestId = nanoid();
const event = traceOptions?.event;
if (init.headers instanceof Headers) {
init.headers.append('x-request-id', requestId);
init.headers.append('traceparent', traceparent);
} else {
const headers = init.headers as Record<string, string>;
headers['x-request-id'] = requestId;
headers['traceparent'] = traceparent;
}
if (!traceReporter) {
return fetch(input, init);
}
try {
const response = await fetch(input, init);
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
traceReporter!.cacheTrace(traceId, spanId, startTime, {
requestId,
...(event ? { event } : {}),
});
return response;
} catch (err) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
traceReporter!.uploadTrace(traceId, spanId, startTime, {
requestId,
...(event ? { event } : {}),
});
throw err;
}
};

View File

@@ -1,5 +0,0 @@
query checkBlobSizes($workspaceId: String!, $size: SafeInt!) {
checkBlobSize(workspaceId: $workspaceId, size: $size) {
size
}
}

View File

@@ -1,5 +0,0 @@
query blobSizes($workspaceId: String!) {
workspace(id: $workspaceId) {
blobsSize
}
}

View File

@@ -1,5 +0,0 @@
query allBlobSizes {
collectAllBlobSizes {
size
}
}

View File

@@ -1,3 +0,0 @@
mutation addToEarlyAccess($email: String!) {
addToEarlyAccess(email: $email)
}

View File

@@ -1,5 +1,6 @@
query getMembersByWorkspaceId($workspaceId: String!, $skip: Int!, $take: Int!) {
workspace(id: $workspaceId) {
memberCount
members(skip: $skip, take: $take) {
id
name

View File

@@ -1,5 +1,6 @@
query getUserFeatures {
currentUser {
id
features
}
}

View File

@@ -0,0 +1,8 @@
query getWorkspacePublicPageById($workspaceId: String!, $pageId: String!) {
workspace(id: $workspaceId) {
publicPage(pageId: $pageId) {
id
mode
}
}
}

View File

@@ -1,5 +1,8 @@
query getWorkspaces {
workspaces {
id
owner {
id
}
}
}

View File

@@ -18,19 +18,6 @@ fragment CredentialsRequirement on CredentialsRequirementType {
...PasswordLimits
}
}`
export const checkBlobSizesQuery = {
id: 'checkBlobSizesQuery' as const,
operationName: 'checkBlobSizes',
definitionName: 'checkBlobSize',
containsFile: false,
query: `
query checkBlobSizes($workspaceId: String!, $size: SafeInt!) {
checkBlobSize(workspaceId: $workspaceId, size: $size) {
size
}
}`,
};
export const deleteBlobMutation = {
id: 'deleteBlobMutation' as const,
operationName: 'deleteBlob',
@@ -64,32 +51,6 @@ mutation setBlob($workspaceId: String!, $blob: Upload!) {
}`,
};
export const blobSizesQuery = {
id: 'blobSizesQuery' as const,
operationName: 'blobSizes',
definitionName: 'workspace',
containsFile: false,
query: `
query blobSizes($workspaceId: String!) {
workspace(id: $workspaceId) {
blobsSize
}
}`,
};
export const allBlobSizesQuery = {
id: 'allBlobSizesQuery' as const,
operationName: 'allBlobSizes',
definitionName: 'collectAllBlobSizes',
containsFile: false,
query: `
query allBlobSizes {
collectAllBlobSizes {
size
}
}`,
};
export const cancelSubscriptionMutation = {
id: 'cancelSubscriptionMutation' as const,
operationName: 'cancelSubscription',
@@ -216,17 +177,6 @@ mutation deleteWorkspace($id: String!) {
}`,
};
export const addToEarlyAccessMutation = {
id: 'addToEarlyAccessMutation' as const,
operationName: 'addToEarlyAccess',
definitionName: 'addToEarlyAccess',
containsFile: false,
query: `
mutation addToEarlyAccess($email: String!) {
addToEarlyAccess(email: $email)
}`,
};
export const earlyAccessUsersQuery = {
id: 'earlyAccessUsersQuery' as const,
operationName: 'earlyAccessUsers',
@@ -395,6 +345,7 @@ export const getMembersByWorkspaceIdQuery = {
query: `
query getMembersByWorkspaceId($workspaceId: String!, $skip: Int!, $take: Int!) {
workspace(id: $workspaceId) {
memberCount
members(skip: $skip, take: $take) {
id
name
@@ -443,6 +394,7 @@ export const getUserFeaturesQuery = {
query: `
query getUserFeatures {
currentUser {
id
features
}
}`,
@@ -498,6 +450,22 @@ query getWorkspacePublicById($id: String!) {
}`,
};
export const getWorkspacePublicPageByIdQuery = {
id: 'getWorkspacePublicPageByIdQuery' as const,
operationName: 'getWorkspacePublicPageById',
definitionName: 'workspace',
containsFile: false,
query: `
query getWorkspacePublicPageById($workspaceId: String!, $pageId: String!) {
workspace(id: $workspaceId) {
publicPage(pageId: $pageId) {
id
mode
}
}
}`,
};
export const getWorkspacePublicPagesQuery = {
id: 'getWorkspacePublicPagesQuery' as const,
operationName: 'getWorkspacePublicPages',
@@ -536,6 +504,9 @@ export const getWorkspacesQuery = {
query getWorkspaces {
workspaces {
id
owner {
id
}
}
}`,
};
@@ -642,11 +613,18 @@ mutation publishPage($workspaceId: String!, $pageId: String!, $mode: PublicPageM
export const quotaQuery = {
id: 'quotaQuery' as const,
operationName: 'quota',
definitionName: 'currentUser',
definitionName: 'currentUser,collectAllBlobSizes',
containsFile: false,
query: `
query quota {
currentUser {
id
copilot {
quota {
limit
used
}
}
quota {
name
blobLimit
@@ -662,6 +640,9 @@ query quota {
}
}
}
collectAllBlobSizes {
size
}
}`,
};
@@ -829,6 +810,7 @@ export const subscriptionQuery = {
query: `
query subscription {
currentUser {
id
subscriptions {
id
status

View File

@@ -1,5 +1,12 @@
query quota {
currentUser {
id
copilot {
quota {
limit
used
}
}
quota {
name
blobLimit
@@ -15,4 +22,7 @@ query quota {
}
}
}
collectAllBlobSizes {
size
}
}

View File

@@ -1,5 +1,6 @@
query subscription {
currentUser {
id
subscriptions {
id
status

View File

@@ -2,7 +2,6 @@ export * from './error';
export * from './fetcher';
export * from './graphql';
export * from './schema';
export * from './utils';
import { setupGlobal } from '@affine/env/global';
@@ -14,6 +13,10 @@ export function getBaseUrl(): string {
if (environment.isDesktop) {
return runtimeConfig.serverUrlPrefix;
}
if (typeof window === 'undefined') {
// is nodejs
return '';
}
const { protocol, hostname, port } = window.location;
return `${protocol}//${hostname}${port ? `:${port}` : ''}`;
}

View File

@@ -58,8 +58,14 @@ export interface CreateCheckoutSessionInput {
successCallbackLink: InputMaybe<Scalars['String']['input']>;
}
export enum EarlyAccessType {
AI = 'AI',
App = 'App',
}
/** The type of workspace feature */
export enum FeatureType {
AIEarlyAccess = 'AIEarlyAccess',
Copilot = 'Copilot',
EarlyAccess = 'EarlyAccess',
UnlimitedCopilot = 'UnlimitedCopilot',
@@ -147,16 +153,6 @@ export interface UpdateWorkspaceInput {
public: InputMaybe<Scalars['Boolean']['input']>;
}
export type CheckBlobSizesQueryVariables = Exact<{
workspaceId: Scalars['String']['input'];
size: Scalars['SafeInt']['input'];
}>;
export type CheckBlobSizesQuery = {
__typename?: 'Query';
checkBlobSize: { __typename?: 'WorkspaceBlobSizes'; size: number };
};
export type DeleteBlobMutationVariables = Exact<{
workspaceId: Scalars['String']['input'];
hash: Scalars['String']['input'];
@@ -180,22 +176,6 @@ export type SetBlobMutationVariables = Exact<{
export type SetBlobMutation = { __typename?: 'Mutation'; setBlob: string };
export type BlobSizesQueryVariables = Exact<{
workspaceId: Scalars['String']['input'];
}>;
export type BlobSizesQuery = {
__typename?: 'Query';
workspace: { __typename?: 'WorkspaceType'; blobsSize: number };
};
export type AllBlobSizesQueryVariables = Exact<{ [key: string]: never }>;
export type AllBlobSizesQuery = {
__typename?: 'Query';
collectAllBlobSizes: { __typename?: 'WorkspaceBlobSizes'; size: number };
};
export type CancelSubscriptionMutationVariables = Exact<{
idempotencyKey: Scalars['String']['input'];
plan?: InputMaybe<SubscriptionPlan>;
@@ -296,15 +276,6 @@ export type DeleteWorkspaceMutation = {
deleteWorkspace: boolean;
};
export type AddToEarlyAccessMutationVariables = Exact<{
email: Scalars['String']['input'];
}>;
export type AddToEarlyAccessMutation = {
__typename?: 'Mutation';
addToEarlyAccess: number;
};
export type EarlyAccessUsersQueryVariables = Exact<{ [key: string]: never }>;
export type EarlyAccessUsersQuery = {
@@ -476,6 +447,7 @@ export type GetMembersByWorkspaceIdQuery = {
__typename?: 'Query';
workspace: {
__typename?: 'WorkspaceType';
memberCount: number;
members: Array<{
__typename?: 'InviteUserType';
id: string;
@@ -513,7 +485,11 @@ export type GetUserFeaturesQueryVariables = Exact<{ [key: string]: never }>;
export type GetUserFeaturesQuery = {
__typename?: 'Query';
currentUser: { __typename?: 'UserType'; features: Array<FeatureType> } | null;
currentUser: {
__typename?: 'UserType';
id: string;
features: Array<FeatureType>;
} | null;
};
export type GetUserQueryVariables = Exact<{
@@ -557,6 +533,23 @@ export type GetWorkspacePublicByIdQuery = {
workspace: { __typename?: 'WorkspaceType'; public: boolean };
};
export type GetWorkspacePublicPageByIdQueryVariables = Exact<{
workspaceId: Scalars['String']['input'];
pageId: Scalars['String']['input'];
}>;
export type GetWorkspacePublicPageByIdQuery = {
__typename?: 'Query';
workspace: {
__typename?: 'WorkspaceType';
publicPage: {
__typename?: 'WorkspacePage';
id: string;
mode: PublicPageMode;
} | null;
};
};
export type GetWorkspacePublicPagesQueryVariables = Exact<{
workspaceId: Scalars['String']['input'];
}>;
@@ -586,7 +579,11 @@ export type GetWorkspacesQueryVariables = Exact<{ [key: string]: never }>;
export type GetWorkspacesQuery = {
__typename?: 'Query';
workspaces: Array<{ __typename?: 'WorkspaceType'; id: string }>;
workspaces: Array<{
__typename?: 'WorkspaceType';
id: string;
owner: { __typename?: 'UserType'; id: string };
}>;
};
export type ListHistoryQueryVariables = Exact<{
@@ -686,6 +683,15 @@ export type QuotaQuery = {
__typename?: 'Query';
currentUser: {
__typename?: 'UserType';
id: string;
copilot: {
__typename?: 'Copilot';
quota: {
__typename?: 'CopilotQuota';
limit: number | null;
used: number;
};
};
quota: {
__typename?: 'UserQuota';
name: string;
@@ -703,6 +709,7 @@ export type QuotaQuery = {
};
} | null;
} | null;
collectAllBlobSizes: { __typename?: 'WorkspaceBlobSizes'; size: number };
};
export type RecoverDocMutationVariables = Exact<{
@@ -850,6 +857,7 @@ export type SubscriptionQuery = {
__typename?: 'Query';
currentUser: {
__typename?: 'UserType';
id: string;
subscriptions: Array<{
__typename?: 'UserSubscription';
id: string;
@@ -1032,26 +1040,11 @@ export type WorkspaceQuotaQuery = {
};
export type Queries =
| {
name: 'checkBlobSizesQuery';
variables: CheckBlobSizesQueryVariables;
response: CheckBlobSizesQuery;
}
| {
name: 'listBlobsQuery';
variables: ListBlobsQueryVariables;
response: ListBlobsQuery;
}
| {
name: 'blobSizesQuery';
variables: BlobSizesQueryVariables;
response: BlobSizesQuery;
}
| {
name: 'allBlobSizesQuery';
variables: AllBlobSizesQueryVariables;
response: AllBlobSizesQuery;
}
| {
name: 'earlyAccessUsersQuery';
variables: EarlyAccessUsersQueryVariables;
@@ -1127,6 +1120,11 @@ export type Queries =
variables: GetWorkspacePublicByIdQueryVariables;
response: GetWorkspacePublicByIdQuery;
}
| {
name: 'getWorkspacePublicPageByIdQuery';
variables: GetWorkspacePublicPageByIdQueryVariables;
response: GetWorkspacePublicPageByIdQuery;
}
| {
name: 'getWorkspacePublicPagesQuery';
variables: GetWorkspacePublicPagesQueryVariables;
@@ -1259,11 +1257,6 @@ export type Mutations =
variables: DeleteWorkspaceMutationVariables;
response: DeleteWorkspaceMutation;
}
| {
name: 'addToEarlyAccessMutation';
variables: AddToEarlyAccessMutationVariables;
response: AddToEarlyAccessMutation;
}
| {
name: 'removeEarlyAccessMutation';
variables: RemoveEarlyAccessMutationVariables;

View File

@@ -1,209 +0,0 @@
export const SPAN_ID_BYTES = 8;
export const TRACE_ID_BYTES = 16;
export const TRACE_VERSION = '00';
export const TRACE_FLAG = '01';
const BytesBuffer = Array.from<number>({ length: 32 });
type TraceSpan = {
name: string;
spanId: string;
displayName: {
value: string;
truncatedByteCount: number;
};
startTime: string;
endTime: string;
attributes: {
attributeMap: {
requestId?: {
stringValue: {
value: string;
truncatedByteCount: number;
};
};
event?: {
stringValue: {
value: string;
truncatedByteCount: 0;
};
};
};
droppedAttributesCount: number;
};
};
/**
* inspired by open-telemetry/opentelemetry-js
*/
export function generateRandUTF16Chars(bytes: number) {
for (let i = 0; i < bytes * 2; i++) {
BytesBuffer[i] = Math.floor(Math.random() * 16) + 48;
// valid hex characters in the range 48-57 and 97-102
if (BytesBuffer[i] >= 58) {
BytesBuffer[i] += 39;
}
}
return String.fromCharCode(...BytesBuffer.slice(0, bytes * 2));
}
export class TraceReporter {
static traceReportEndpoint = process.env.TRACE_REPORT_ENDPOINT;
static shouldReportTrace = process.env.SHOULD_REPORT_TRACE;
private spansCache = new Array<TraceSpan>();
private reportIntervalId: number | undefined | NodeJS.Timeout;
private readonly reportInterval = 60_000;
private static instance: TraceReporter;
public static getInstance(): TraceReporter {
if (!TraceReporter.instance) {
const instance = (TraceReporter.instance = new TraceReporter());
instance.initTraceReport();
}
return TraceReporter.instance;
}
public cacheTrace(
traceId: string,
spanId: string,
startTime: string,
attributes: {
requestId?: string;
event?: string;
}
) {
const span = TraceReporter.createTraceSpan(
traceId,
spanId,
startTime,
attributes
);
this.spansCache.push(span);
if (this.spansCache.length <= 1) {
this.initTraceReport();
}
}
public uploadTrace(
traceId: string,
spanId: string,
startTime: string,
attributes: {
requestId?: string;
event?: string;
}
) {
const span = TraceReporter.createTraceSpan(
traceId,
spanId,
startTime,
attributes
);
TraceReporter.reportToTraceEndpoint(JSON.stringify({ spans: [span] }));
}
public static reportToTraceEndpoint(payload: string): void {
if (!TraceReporter.traceReportEndpoint) {
console.warn('No trace report endpoint found!');
return;
}
if (typeof navigator !== 'undefined') {
navigator.sendBeacon(TraceReporter.traceReportEndpoint, payload);
} else {
fetch(TraceReporter.traceReportEndpoint, {
method: 'POST',
mode: 'cors',
cache: 'no-cache',
headers: {
'Content-Type': 'application/json',
},
body: payload,
}).catch(console.warn);
}
}
public static createTraceSpan(
traceId: string,
spanId: string,
startTime: string,
attributes: {
requestId?: string;
event?: string;
}
): TraceSpan {
const requestId = attributes.requestId;
const event = attributes.event;
return {
name: `projects/{GCP_PROJECT_ID}/traces/${traceId}/spans/${spanId}`,
spanId,
displayName: {
value: 'AFFiNE_REQUEST',
truncatedByteCount: 0,
},
startTime,
endTime: new Date().toISOString(),
attributes: {
attributeMap: {
...(!requestId
? {}
: {
requestId: {
stringValue: {
value: requestId,
truncatedByteCount: 0,
},
},
}),
...(!event
? {}
: {
event: {
stringValue: {
value: event,
truncatedByteCount: 0,
},
},
}),
},
droppedAttributesCount: 0,
},
};
}
private readonly initTraceReport = () => {
if (!this.reportIntervalId && TraceReporter.shouldReportTrace) {
if (typeof window !== 'undefined') {
this.reportIntervalId = window.setInterval(
this.reportHandler,
this.reportInterval
);
} else {
this.reportIntervalId = setInterval(
this.reportHandler,
this.reportInterval
);
}
}
};
private readonly reportHandler = () => {
if (this.spansCache.length <= 0) {
clearInterval(this.reportIntervalId);
this.reportIntervalId = undefined;
return;
}
TraceReporter.reportToTraceEndpoint(
JSON.stringify({ spans: [...this.spansCache] })
);
this.spansCache = [];
};
}
export const traceReporter = process.env.SHOULD_REPORT_TRACE
? TraceReporter.getInstance()
: null;