Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-04 08:38:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: pre-aggregation workspace stats

Browse Source
This commit is contained in:
DarkSky
2026-01-01 05:01:52 +08:00
parent 97507e7043
commit f745f7b669
16 changed files with 722 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
packages/backend/server/migrations/20251231203615_workspace_admin_stats/migration.sql Normal file
View File

@@ -0,0 +1,85 @@
-- Pre-aggregated admin workspace stats tables
CREATE TABLE IF NOT EXISTS "workspace_admin_stats" (
"workspace_id" VARCHAR NOT NULL,
"snapshot_count" BIGINT NOT NULL DEFAULT 0,
"snapshot_size" BIGINT NOT NULL DEFAULT 0,
"blob_count" BIGINT NOT NULL DEFAULT 0,
"blob_size" BIGINT NOT NULL DEFAULT 0,
"member_count" BIGINT NOT NULL DEFAULT 0,
"public_page_count" BIGINT NOT NULL DEFAULT 0,
"features" TEXT[] NOT NULL DEFAULT '{}'::text[],
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT NOW(),
CONSTRAINT "workspace_admin_stats_pkey" PRIMARY KEY ("workspace_id"),
CONSTRAINT "workspace_admin_stats_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE
);
CREATE TABLE IF NOT EXISTS "workspace_admin_stats_dirty" (
"workspace_id" VARCHAR NOT NULL,
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT NOW(),
CONSTRAINT "workspace_admin_stats_dirty_pkey" PRIMARY KEY ("workspace_id"),
CONSTRAINT "workspace_admin_stats_dirty_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- Sorting indexes for admin queries
CREATE INDEX IF NOT EXISTS "workspace_admin_stats_snapshot_size_idx" ON "workspace_admin_stats" ("snapshot_size" DESC);
CREATE INDEX IF NOT EXISTS "workspace_admin_stats_blob_count_idx" ON "workspace_admin_stats" ("blob_count" DESC);
CREATE INDEX IF NOT EXISTS "workspace_admin_stats_blob_size_idx" ON "workspace_admin_stats" ("blob_size" DESC);
CREATE INDEX IF NOT EXISTS "workspace_admin_stats_snapshot_count_idx" ON "workspace_admin_stats" ("snapshot_count" DESC);
CREATE INDEX IF NOT EXISTS "workspace_admin_stats_member_count_idx" ON "workspace_admin_stats" ("member_count" DESC);
CREATE INDEX IF NOT EXISTS "workspace_admin_stats_public_page_count_idx" ON "workspace_admin_stats" ("public_page_count" DESC);
CREATE INDEX IF NOT EXISTS "workspace_admin_stats_dirty_updated_at_idx" ON "workspace_admin_stats_dirty" ("updated_at");
-- Feature filtering index
CREATE INDEX IF NOT EXISTS "workspace_features_workspace_id_name_activated_idx" ON "workspace_features" ("workspace_id", "name", "activated");
-- Dirty marker trigger
CREATE OR REPLACE FUNCTION workspace_admin_stats_mark_dirty() RETURNS TRIGGER AS $$
DECLARE
wid VARCHAR;
BEGIN
wid := COALESCE(NEW."workspace_id", OLD."workspace_id");
IF wid IS NULL THEN
RETURN NULL;
END IF;
INSERT INTO "workspace_admin_stats_dirty" ("workspace_id", "updated_at")
VALUES (wid, NOW())
ON CONFLICT ("workspace_id")
DO UPDATE SET "updated_at" = EXCLUDED."updated_at";
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS "workspace_admin_stats_dirty_snapshots" ON "snapshots";
CREATE TRIGGER "workspace_admin_stats_dirty_snapshots"
AFTER INSERT OR UPDATE OR DELETE ON "snapshots"
FOR EACH ROW EXECUTE FUNCTION workspace_admin_stats_mark_dirty();
DROP TRIGGER IF EXISTS "workspace_admin_stats_dirty_blobs" ON "blobs";
CREATE TRIGGER "workspace_admin_stats_dirty_blobs"
AFTER INSERT OR UPDATE OR DELETE ON "blobs"
FOR EACH ROW EXECUTE FUNCTION workspace_admin_stats_mark_dirty();
DROP TRIGGER IF EXISTS "workspace_admin_stats_dirty_wup" ON "workspace_user_permissions";
CREATE TRIGGER "workspace_admin_stats_dirty_wup"
AFTER INSERT OR UPDATE OR DELETE ON "workspace_user_permissions"
FOR EACH ROW EXECUTE FUNCTION workspace_admin_stats_mark_dirty();
DROP TRIGGER IF EXISTS "workspace_admin_stats_dirty_pages" ON "workspace_pages";
CREATE TRIGGER "workspace_admin_stats_dirty_pages"
AFTER INSERT OR UPDATE OR DELETE ON "workspace_pages"
FOR EACH ROW EXECUTE FUNCTION workspace_admin_stats_mark_dirty();
DROP TRIGGER IF EXISTS "workspace_admin_stats_dirty_features" ON "workspace_features";
CREATE TRIGGER "workspace_admin_stats_dirty_features"
AFTER INSERT OR UPDATE OR DELETE ON "workspace_features"
FOR EACH ROW EXECUTE FUNCTION workspace_admin_stats_mark_dirty();
-- Mark existing workspaces dirty for initial backfill
INSERT INTO "workspace_admin_stats_dirty" ("workspace_id", "updated_at")
SELECT id, NOW() FROM "workspaces"
ON CONFLICT ("workspace_id") DO NOTHING;
-- DropIndex
DROP INDEX "idx_wur_workspace";

56
packages/backend/server/schema.prisma
View File

@@ -124,15 +124,17 @@ model Workspace {
indexed Boolean @default(false)
lastCheckEmbeddings DateTime @default("1970-01-01T00:00:00-00:00") @map("last_check_embeddings") @db.Timestamptz(3)
features WorkspaceFeature[]
docs WorkspaceDoc[]
permissions WorkspaceUserRole[]
docPermissions WorkspaceDocUserRole[]
blobs Blob[]
ignoredDocs AiWorkspaceIgnoredDocs[]
embedFiles AiWorkspaceFiles[]
comments Comment[]
commentAttachments CommentAttachment[]
features WorkspaceFeature[]
docs WorkspaceDoc[]
permissions WorkspaceUserRole[]
docPermissions WorkspaceDocUserRole[]
blobs Blob[]
ignoredDocs AiWorkspaceIgnoredDocs[]
embedFiles AiWorkspaceFiles[]
comments Comment[]
commentAttachments CommentAttachment[]
workspaceAdminStats WorkspaceAdminStats[]
workspaceAdminStatsDirties WorkspaceAdminStatsDirty[]
@@index([lastCheckEmbeddings])
@@index([createdAt])
@@ -241,7 +243,7 @@ model UserFeature {
expiredAt DateTime? @map("expired_at") @db.Timestamptz(3)
activated Boolean @default(false)
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@index([userId])
@@index([name])
@@ -267,9 +269,43 @@ model WorkspaceFeature {
@@index([workspaceId])
@@index([name])
// Partial index on (workspace_id, name) where activated = true is created via SQL migration
@@index([workspaceId, name, activated])
@@map("workspace_features")
}
model WorkspaceAdminStats {
workspaceId String @id @map("workspace_id") @db.VarChar
snapshotCount BigInt @default(0) @map("snapshot_count") @db.BigInt
snapshotSize BigInt @default(0) @map("snapshot_size") @db.BigInt
blobCount BigInt @default(0) @map("blob_count") @db.BigInt
blobSize BigInt @default(0) @map("blob_size") @db.BigInt
memberCount BigInt @default(0) @map("member_count") @db.BigInt
publicPageCount BigInt @default(0) @map("public_page_count") @db.BigInt
features String[] @default([]) @db.Text
updatedAt DateTime @default(now()) @map("updated_at") @db.Timestamptz(3)
workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
@@index([snapshotSize(sort: Desc)])
@@index([blobCount(sort: Desc)])
@@index([blobSize(sort: Desc)])
@@index([snapshotCount(sort: Desc)])
@@index([memberCount(sort: Desc)])
@@index([publicPageCount(sort: Desc)])
@@map("workspace_admin_stats")
}
model WorkspaceAdminStatsDirty {
workspaceId String @id @map("workspace_id") @db.VarChar
updatedAt DateTime @default(now()) @map("updated_at") @db.Timestamptz(3)
workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
@@index([updatedAt])
@@map("workspace_admin_stats_dirty")
}
// the latest snapshot of each doc that we've seen
// Snapshot + Updates are the latest state of the doc
model Snapshot {

3
packages/backend/server/src/base/metrics/metrics.ts
View File

@@ -61,7 +61,8 @@ export type KnownMetricScopes =
| 'event'
| 'queue'
| 'storage'
| 'process';
| 'process'
| 'workspace';
const metricCreators: MetricCreators = {
counter(meter: Meter, name: string, opts?: MetricOptions) {

2
packages/backend/server/src/core/workspaces/index.ts
View File

@@ -21,6 +21,7 @@ import {
} from './resolvers';
import { AdminWorkspaceResolver } from './resolvers/admin';
import { WorkspaceService } from './service';
import { WorkspaceStatsJob } from './stats.job';
@Module({
imports: [
@@ -45,6 +46,7 @@ import { WorkspaceService } from './service';
WorkspaceService,
WorkspaceEvents,
AdminWorkspaceResolver,
WorkspaceStatsJob,
],
exports: [WorkspaceService],
})

9
packages/backend/server/src/core/workspaces/resolvers/admin.ts
View File

@@ -31,6 +31,9 @@ enum AdminWorkspaceSort {
SnapshotSize = 'SnapshotSize',
BlobCount = 'BlobCount',
BlobSize = 'BlobSize',
SnapshotCount = 'SnapshotCount',
MemberCount = 'MemberCount',
PublicPageCount = 'PublicPageCount',
}
registerEnumType(AdminWorkspaceSort, {
@@ -298,6 +301,12 @@ export class AdminWorkspaceResolver {
return 'blobCount';
case AdminWorkspaceSort.BlobSize:
return 'blobSize';
case AdminWorkspaceSort.SnapshotCount:
return 'snapshotCount';
case AdminWorkspaceSort.MemberCount:
return 'memberCount';
case AdminWorkspaceSort.PublicPageCount:
return 'publicPageCount';
case AdminWorkspaceSort.CreatedAt:
default:
return 'createdAt';

305
packages/backend/server/src/core/workspaces/stats.job.ts Normal file
View File

@@ -0,0 +1,305 @@
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { Prisma, PrismaClient } from '@prisma/client';
import { metrics } from '../../base';
const LOCK_NAMESPACE = 97_301;
const LOCK_KEY = 1;
const DIRTY_BATCH_SIZE = 500;
const FULL_REFRESH_BATCH_SIZE = 2000;
const TRANSACTION_TIMEOUT_MS = 120_000;
@Injectable()
export class WorkspaceStatsJob {
private readonly logger = new Logger(WorkspaceStatsJob.name);
constructor(private readonly prisma: PrismaClient) {}
@Cron(CronExpression.EVERY_MINUTE)
async refreshDirty() {
const started = Date.now();
try {
const result = await this.withAdvisoryLock(async tx => {
const backlog = await this.countDirty(tx);
metrics.workspace
.gauge('admin_stats_dirty_backlog')
.record(Number(backlog));
const dirty = await this.loadDirty(tx, DIRTY_BATCH_SIZE);
if (!dirty.length) {
return { processed: 0, backlog };
}
await this.upsertStats(tx, dirty);
await this.clearDirty(tx, dirty);
return { processed: dirty.length, backlog };
});
if (!result) {
this.logger.debug('skip admin stats refresh, lock not acquired');
return;
}
metrics.workspace
.histogram('admin_stats_refresh_duration_ms')
.record(Date.now() - started, { mode: 'incremental' });
if (result.processed > 0) {
this.logger.log(
`Refreshed admin stats for ${result.processed} workspace(s); backlog ${result.backlog}`
);
}
} catch (error) {
metrics.workspace.counter('admin_stats_refresh_failed').add(1, {
mode: 'incremental',
});
this.logger.error('Failed to refresh admin stats', error as Error);
}
}
@Cron(CronExpression.EVERY_DAY_AT_1AM)
async recalibrate() {
let lastSid = 0;
let processed = 0;
while (true) {
const started = Date.now();
try {
const result = await this.withAdvisoryLock(async tx => {
const workspaces = await this.fetchWorkspaceBatch(
tx,
lastSid,
FULL_REFRESH_BATCH_SIZE
);
if (!workspaces.length) {
return { processed: 0, lastSid };
}
const ids = workspaces.map(({ id }) => id);
await this.upsertStats(tx, ids);
return {
processed: ids.length,
lastSid: workspaces[workspaces.length - 1].sid,
};
});
if (!result) {
this.logger.debug(
'skip admin stats recalibration, lock not acquired'
);
break;
}
if (result.processed === 0) {
break;
}
processed += result.processed;
lastSid = result.lastSid;
metrics.workspace
.histogram('admin_stats_refresh_duration_ms')
.record(Date.now() - started, { mode: 'full' });
if (result.processed < FULL_REFRESH_BATCH_SIZE) {
break;
}
} catch (error) {
metrics.workspace.counter('admin_stats_refresh_failed').add(1, {
mode: 'full',
});
this.logger.error(
`Failed to recalibrate admin stats after sid ${lastSid}`,
error as Error
);
break;
}
}
if (processed > 0) {
this.logger.verbose(
`Recalibrate admin stats for ${processed} workspace(s) (last sid ${lastSid})`
);
}
}
private async withAdvisoryLock<T>(
callback: (tx: Prisma.TransactionClient) => Promise<T>
): Promise<T | null> {
return await this.prisma.$transaction(
async tx => {
const [lock] = await tx.$queryRaw<{ locked: boolean }[]>`
SELECT pg_try_advisory_lock(${LOCK_NAMESPACE}, ${LOCK_KEY}) AS locked
`;
if (!lock?.locked) {
return null;
}
try {
return await callback(tx);
} finally {
await tx.$executeRaw`SELECT pg_advisory_unlock(${LOCK_NAMESPACE}, ${LOCK_KEY})`;
}
},
{
maxWait: 5_000,
timeout: TRANSACTION_TIMEOUT_MS,
}
);
}
private async loadDirty(
tx: Prisma.TransactionClient,
limit: number
): Promise<string[]> {
const rows = await tx.$queryRaw<{ workspace_id: string }[]>`
SELECT workspace_id
FROM workspace_admin_stats_dirty
ORDER BY updated_at ASC
LIMIT ${limit}
FOR UPDATE SKIP LOCKED
`;
return rows.map(row => row.workspace_id);
}
private async countDirty(tx: Prisma.TransactionClient) {
const [row] = await tx.$queryRaw<{ total: bigint | number }[]>`
SELECT COUNT(*) AS total FROM workspace_admin_stats_dirty
`;
return row?.total ? Number(row.total) : 0;
}
private async clearDirty(
tx: Prisma.TransactionClient,
workspaceIds: string[]
) {
if (!workspaceIds.length) {
return;
}
await tx.$executeRaw`
DELETE FROM workspace_admin_stats_dirty
WHERE workspace_id IN (${Prisma.join(
workspaceIds.map(id => Prisma.sql`${id}`)
)})
`;
}
private async upsertStats(
tx: Prisma.TransactionClient,
workspaceIds: string[]
) {
if (!workspaceIds.length) {
return;
}
const targetIds = Prisma.join(workspaceIds.map(id => Prisma.sql`${id}`));
await tx.$executeRaw`
WITH targets AS (
SELECT UNNEST(ARRAY[${targetIds}]::varchar[]) AS workspace_id
),
snapshot_stats AS (
SELECT workspace_id,
COUNT(*) AS snapshot_count,
COALESCE(SUM(COALESCE(size, octet_length(blob))), 0) AS snapshot_size
FROM snapshots
WHERE workspace_id IN (SELECT workspace_id FROM targets)
GROUP BY workspace_id
),
blob_stats AS (
SELECT workspace_id,
COUNT(*) FILTER (WHERE deleted_at IS NULL AND status = 'completed') AS blob_count,
COALESCE(SUM(size) FILTER (WHERE deleted_at IS NULL AND status = 'completed'), 0) AS blob_size
FROM blobs
WHERE workspace_id IN (SELECT workspace_id FROM targets)
GROUP BY workspace_id
),
member_stats AS (
SELECT workspace_id, COUNT(*) AS member_count
FROM workspace_user_permissions
WHERE workspace_id IN (SELECT workspace_id FROM targets)
GROUP BY workspace_id
),
public_page_stats AS (
SELECT workspace_id, COUNT(*) AS public_page_count
FROM workspace_pages
WHERE public = TRUE AND workspace_id IN (SELECT workspace_id FROM targets)
GROUP BY workspace_id
),
feature_stats AS (
SELECT workspace_id,
ARRAY_AGG(DISTINCT name ORDER BY name) FILTER (WHERE activated) AS features
FROM workspace_features
WHERE workspace_id IN (SELECT workspace_id FROM targets)
GROUP BY workspace_id
),
aggregated AS (
SELECT t.workspace_id,
COALESCE(ss.snapshot_count, 0) AS snapshot_count,
COALESCE(ss.snapshot_size, 0) AS snapshot_size,
COALESCE(bs.blob_count, 0) AS blob_count,
COALESCE(bs.blob_size, 0) AS blob_size,
COALESCE(ms.member_count, 0) AS member_count,
COALESCE(pp.public_page_count, 0) AS public_page_count,
COALESCE(fs.features, ARRAY[]::text[]) AS features
FROM targets t
LEFT JOIN snapshot_stats ss ON ss.workspace_id = t.workspace_id
LEFT JOIN blob_stats bs ON bs.workspace_id = t.workspace_id
LEFT JOIN member_stats ms ON ms.workspace_id = t.workspace_id
LEFT JOIN public_page_stats pp ON pp.workspace_id = t.workspace_id
LEFT JOIN feature_stats fs ON fs.workspace_id = t.workspace_id
)
INSERT INTO workspace_admin_stats (
workspace_id,
snapshot_count,
snapshot_size,
blob_count,
blob_size,
member_count,
public_page_count,
features,
updated_at
)
SELECT
workspace_id,
snapshot_count,
snapshot_size,
blob_count,
blob_size,
member_count,
public_page_count,
features,
NOW()
FROM aggregated
ON CONFLICT (workspace_id) DO UPDATE SET
snapshot_count = EXCLUDED.snapshot_count,
snapshot_size = EXCLUDED.snapshot_size,
blob_count = EXCLUDED.blob_count,
blob_size = EXCLUDED.blob_size,
member_count = EXCLUDED.member_count,
public_page_count = EXCLUDED.public_page_count,
features = EXCLUDED.features,
updated_at = EXCLUDED.updated_at
`;
}
private async fetchWorkspaceBatch(
tx: Prisma.TransactionClient,
lastSid: number,
limit: number
) {
return tx.$queryRaw<{ id: string; sid: number }[]>`
SELECT id, sid
FROM workspaces
WHERE sid > ${lastSid}
ORDER BY sid
LIMIT ${limit}
`;
}
}

392
packages/backend/server/src/models/workspace.ts
View File

@@ -185,7 +185,14 @@ export class WorkspaceModel extends BaseModel {
first: number;
keyword?: string | null;
features?: WorkspaceFeatureName[] | null;
order?: 'createdAt' | 'snapshotSize' | 'blobCount' | 'blobSize';
order?:
| 'createdAt'
| 'snapshotSize'
| 'blobCount'
| 'blobSize'
| 'snapshotCount'
| 'memberCount'
| 'publicPageCount';
includeTotal?: boolean;
}): Promise<{ rows: AdminWorkspaceSummary[]; total: number }> {
const keyword = options.keyword?.trim();
@@ -198,189 +205,100 @@ export class WorkspaceModel extends BaseModel {
return { rows: [], total: 0 };
}
const rows =
options.order === 'createdAt' || !options.order
? await this.db.$queryRaw<RawWorkspaceSummary[]>`
WITH feature_set AS (
SELECT workspace_id, array_agg(DISTINCT name) FILTER (WHERE activated) AS features
FROM workspace_features
GROUP BY workspace_id
),
owner AS (
SELECT wur.workspace_id,
u.id AS owner_id,
u.name AS owner_name,
u.email AS owner_email,
u.avatar_url AS owner_avatar_url
FROM workspace_user_permissions AS wur
JOIN users u ON wur.user_id = u.id
WHERE wur.type = ${WorkspaceRole.Owner}
AND wur.status = ${Prisma.sql`${WorkspaceMemberStatus.Accepted}::"WorkspaceMemberStatus"`}
),
filtered AS (
SELECT w.id,
w.public,
w.created_at AS "createdAt",
w.name,
w.avatar_key AS "avatarKey",
w.enable_ai AS "enableAi",
w.enable_url_preview AS "enableUrlPreview",
w.enable_doc_embedding AS "enableDocEmbedding",
COALESCE(fs.features, ARRAY[]::text[]) AS features,
o.owner_id AS "ownerId",
o.owner_name AS "ownerName",
o.owner_email AS "ownerEmail",
o.owner_avatar_url AS "ownerAvatarUrl"
FROM workspaces w
LEFT JOIN feature_set fs ON fs.workspace_id = w.id
LEFT JOIN owner o ON o.workspace_id = w.id
WHERE ${
keyword
? Prisma.sql`
(
w.id ILIKE ${'%' + keyword + '%'}
OR o.owner_id ILIKE ${'%' + keyword + '%'}
OR o.owner_email ILIKE ${'%' + keyword + '%'}
)
`
: Prisma.sql`TRUE`
}
AND ${
features.length
? Prisma.sql`CAST(COALESCE(fs.features, ARRAY[]::text[]) AS text[]) @> ${Prisma.sql`${features}::text[]`}`
: Prisma.sql`TRUE`
}
ORDER BY w.created_at DESC
LIMIT ${options.first}
OFFSET ${options.skip}
)
SELECT f.*,
COALESCE(ms.member_count, 0) AS "memberCount",
COALESCE(pp.public_page_count, 0) AS "publicPageCount",
COALESCE(ss.snapshot_count, 0) AS "snapshotCount",
COALESCE(ss.snapshot_size, 0) AS "snapshotSize",
COALESCE(bs.blob_count, 0) AS "blobCount",
COALESCE(bs.blob_size, 0) AS "blobSize"
FROM filtered f
LEFT JOIN LATERAL (
SELECT COUNT(*) AS member_count
FROM workspace_user_permissions
WHERE workspace_id = f.id
) ms ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*) AS public_page_count
FROM workspace_pages
WHERE workspace_id = f.id AND public = true
) pp ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*) AS snapshot_count,
SUM(size) AS snapshot_size
FROM snapshots
WHERE workspace_id = f.id
) ss ON TRUE
LEFT JOIN LATERAL (
SELECT COUNT(*) FILTER (WHERE deleted_at IS NULL AND status = 'completed') AS blob_count,
SUM(size) FILTER (WHERE deleted_at IS NULL AND status = 'completed') AS blob_size
FROM blobs
WHERE workspace_id = f.id
) bs ON TRUE
ORDER BY f."createdAt" DESC
const featuresHaving =
features.length > 0
? Prisma.sql`
HAVING COUNT(
DISTINCT CASE
WHEN wf.name = ANY(${Prisma.sql`${features}::text[]`}) THEN wf.name
END
) = ${features.length}
`
: await this.db.$queryRaw<RawWorkspaceSummary[]>`
WITH feature_set AS (
SELECT workspace_id, array_agg(DISTINCT name) FILTER (WHERE activated) AS features
FROM workspace_features
GROUP BY workspace_id
),
owner AS (
SELECT wur.workspace_id,
u.id AS owner_id,
u.name AS owner_name,
u.email AS owner_email,
u.avatar_url AS owner_avatar_url
FROM workspace_user_permissions AS wur
JOIN users u ON wur.user_id = u.id
WHERE wur.type = ${WorkspaceRole.Owner}
AND wur.status = ${Prisma.sql`${WorkspaceMemberStatus.Accepted}::"WorkspaceMemberStatus"`}
),
filtered AS (
SELECT w.id,
: Prisma.empty;
const featureJoin =
features.length > 0
? Prisma.sql`
LEFT JOIN workspace_features wf
ON wf.workspace_id = w.id AND wf.activated = TRUE
`
: Prisma.empty;
const groupAndHaving =
features.length > 0
? Prisma.sql`
GROUP BY w.id,
w.public,
w.created_at AS "createdAt",
w.created_at,
w.name,
w.avatar_key AS "avatarKey",
w.enable_ai AS "enableAi",
w.enable_url_preview AS "enableUrlPreview",
w.enable_doc_embedding AS "enableDocEmbedding",
COALESCE(fs.features, ARRAY[]::text[]) AS features,
o.owner_id AS "ownerId",
o.owner_name AS "ownerName",
o.owner_email AS "ownerEmail",
o.owner_avatar_url AS "ownerAvatarUrl"
FROM workspaces w
LEFT JOIN feature_set fs ON fs.workspace_id = w.id
LEFT JOIN owner o ON o.workspace_id = w.id
WHERE ${
keyword
? Prisma.sql`
(
w.id ILIKE ${'%' + keyword + '%'}
OR o.owner_id ILIKE ${'%' + keyword + '%'}
OR o.owner_email ILIKE ${'%' + keyword + '%'}
)
`
: Prisma.sql`TRUE`
}
AND ${
features.length
? Prisma.sql`CAST(COALESCE(fs.features, ARRAY[]::text[]) AS text[]) @> ${Prisma.sql`${features}::text[]`}`
: Prisma.sql`TRUE`
}
),
snapshot_stats AS (
SELECT workspace_id,
SUM(size) AS snapshot_size,
COUNT(*) AS snapshot_count
FROM snapshots
WHERE workspace_id IN (SELECT id FROM filtered)
GROUP BY workspace_id
),
blob_stats AS (
SELECT workspace_id,
SUM(size) FILTER (WHERE deleted_at IS NULL AND status = 'completed') AS blob_size,
COUNT(*) FILTER (WHERE deleted_at IS NULL AND status = 'completed') AS blob_count
FROM blobs
WHERE workspace_id IN (SELECT id FROM filtered)
GROUP BY workspace_id
),
member_stats AS (
SELECT workspace_id, COUNT(*) AS member_count
FROM workspace_user_permissions
WHERE workspace_id IN (SELECT id FROM filtered)
GROUP BY workspace_id
),
public_pages AS (
SELECT workspace_id, COUNT(*) AS public_page_count
FROM workspace_pages
WHERE public = true AND workspace_id IN (SELECT id FROM filtered)
GROUP BY workspace_id
)
SELECT f.*,
COALESCE(ms.member_count, 0) AS "memberCount",
COALESCE(pp.public_page_count, 0) AS "publicPageCount",
COALESCE(ss.snapshot_count, 0) AS "snapshotCount",
COALESCE(ss.snapshot_size, 0) AS "snapshotSize",
COALESCE(bs.blob_count, 0) AS "blobCount",
COALESCE(bs.blob_size, 0) AS "blobSize"
FROM filtered f
LEFT JOIN snapshot_stats ss ON ss.workspace_id = f.id
LEFT JOIN blob_stats bs ON bs.workspace_id = f.id
LEFT JOIN member_stats ms ON ms.workspace_id = f.id
LEFT JOIN public_pages pp ON pp.workspace_id = f.id
ORDER BY ${Prisma.raw(this.buildAdminOrder(options.order))}
LIMIT ${options.first}
OFFSET ${options.skip}
`;
w.avatar_key,
w.enable_ai,
w.enable_url_preview,
w.enable_doc_embedding,
o.owner_id,
o.owner_name,
o.owner_email,
o.owner_avatar_url
${featuresHaving}
`
: Prisma.empty;
const rows = await this.db.$queryRaw<RawWorkspaceSummary[]>`
WITH filtered AS (
SELECT w.id,
w.public,
w.created_at AS "createdAt",
w.name,
w.avatar_key AS "avatarKey",
w.enable_ai AS "enableAi",
w.enable_url_preview AS "enableUrlPreview",
w.enable_doc_embedding AS "enableDocEmbedding",
o.owner_id AS "ownerId",
o.owner_name AS "ownerName",
o.owner_email AS "ownerEmail",
o.owner_avatar_url AS "ownerAvatarUrl"
FROM workspaces w
LEFT JOIN LATERAL (
SELECT u.id AS owner_id,
u.name AS owner_name,
u.email AS owner_email,
u.avatar_url AS owner_avatar_url
FROM workspace_user_permissions AS wur
JOIN users u ON wur.user_id = u.id
WHERE wur.workspace_id = w.id
AND wur.type = ${WorkspaceRole.Owner}
AND wur.status = ${Prisma.sql`${WorkspaceMemberStatus.Accepted}::"WorkspaceMemberStatus"`}
ORDER BY u.created_at ASC
LIMIT 1
) o ON TRUE
${featureJoin}
WHERE ${
keyword
? Prisma.sql`
(
w.id ILIKE ${'%' + keyword + '%'}
OR o.owner_id ILIKE ${'%' + keyword + '%'}
OR o.owner_email ILIKE ${'%' + keyword + '%'}
)
`
: Prisma.sql`TRUE`
}
${groupAndHaving}
)
SELECT f.*,
COALESCE(s.snapshot_count, 0) AS "snapshotCount",
COALESCE(s.snapshot_size, 0) AS "snapshotSize",
COALESCE(s.blob_count, 0) AS "blobCount",
COALESCE(s.blob_size, 0) AS "blobSize",
COALESCE(s.member_count, 0) AS "memberCount",
COALESCE(s.public_page_count, 0) AS "publicPageCount",
COALESCE(s.features, ARRAY[]::text[]) AS features
FROM filtered f
LEFT JOIN workspace_admin_stats s ON s.workspace_id = f.id
ORDER BY ${Prisma.raw(this.buildAdminOrder(options.order))}
LIMIT ${options.first}
OFFSET ${options.skip}
`;
const mapped = rows.map(row => ({
id: row.id,
@@ -418,56 +336,94 @@ export class WorkspaceModel extends BaseModel {
const keyword = options.keyword?.trim();
const features = options.features ?? [];
const featuresHaving =
features.length > 0
? Prisma.sql`
HAVING COUNT(
DISTINCT CASE
WHEN wf.name = ANY(${Prisma.sql`${features}::text[]`}) THEN wf.name
END
) = ${features.length}
`
: Prisma.empty;
const featureJoin =
features.length > 0
? Prisma.sql`
LEFT JOIN workspace_features wf
ON wf.workspace_id = w.id AND wf.activated = TRUE
`
: Prisma.empty;
const groupAndHaving =
features.length > 0
? Prisma.sql`
GROUP BY w.id, o.owner_id, o.owner_email
${featuresHaving}
`
: Prisma.empty;
const [row] = await this.db.$queryRaw<{ total: bigint | number }[]>`
WITH feature_set AS (
SELECT workspace_id, array_agg(DISTINCT name) FILTER (WHERE activated) AS features
FROM workspace_features
GROUP BY workspace_id
),
owner AS (
SELECT wur.workspace_id,
u.id AS owner_id,
u.email AS owner_email
FROM workspace_user_permissions AS wur
JOIN users u ON wur.user_id = u.id
WHERE wur.type = ${WorkspaceRole.Owner}
AND wur.status = ${Prisma.sql`${WorkspaceMemberStatus.Accepted}::"WorkspaceMemberStatus"`}
WITH filtered AS (
SELECT w.id,
o.owner_id AS "ownerId",
o.owner_email AS "ownerEmail"
FROM workspaces w
LEFT JOIN LATERAL (
SELECT wur.workspace_id,
u.id AS owner_id,
u.email AS owner_email
FROM workspace_user_permissions AS wur
JOIN users u ON wur.user_id = u.id
WHERE wur.workspace_id = w.id
AND wur.type = ${WorkspaceRole.Owner}
AND wur.status = ${Prisma.sql`${WorkspaceMemberStatus.Accepted}::"WorkspaceMemberStatus"`}
ORDER BY u.created_at ASC
LIMIT 1
) o ON TRUE
${featureJoin}
WHERE ${
keyword
? Prisma.sql`
(
w.id ILIKE ${'%' + keyword + '%'}
OR o.owner_id ILIKE ${'%' + keyword + '%'}
OR o.owner_email ILIKE ${'%' + keyword + '%'}
)
`
: Prisma.sql`TRUE`
}
${groupAndHaving}
)
SELECT COUNT(*) AS total
FROM workspaces w
LEFT JOIN feature_set fs ON fs.workspace_id = w.id
LEFT JOIN owner o ON o.workspace_id = w.id
WHERE ${
keyword
? Prisma.sql`
(
w.id ILIKE ${'%' + keyword + '%'}
OR o.owner_id ILIKE ${'%' + keyword + '%'}
OR o.owner_email ILIKE ${'%' + keyword + '%'}
)
`
: Prisma.sql`TRUE`
}
AND ${
features.length
? Prisma.sql`CAST(COALESCE(fs.features, ARRAY[]::text[]) AS text[]) @> ${Prisma.sql`${features}::text[]`}`
: Prisma.sql`TRUE`
}
SELECT COUNT(*) AS total FROM filtered
`;
return row?.total ? Number(row.total) : 0;
}
private buildAdminOrder(
order?: 'createdAt' | 'snapshotSize' | 'blobCount' | 'blobSize'
order?:
| 'createdAt'
| 'snapshotSize'
| 'blobCount'
| 'blobSize'
| 'snapshotCount'
| 'memberCount'
| 'publicPageCount'
) {
switch (order) {
case 'snapshotSize':
return `"snapshotSize" DESC NULLS LAST`;
return `"snapshotSize" DESC NULLS LAST, "createdAt" DESC`;
case 'blobCount':
return `"blobCount" DESC NULLS LAST`;
return `"blobCount" DESC NULLS LAST, "createdAt" DESC`;
case 'blobSize':
return `"blobSize" DESC NULLS LAST`;
return `"blobSize" DESC NULLS LAST, "createdAt" DESC`;
case 'snapshotCount':
return `"snapshotCount" DESC NULLS LAST, "createdAt" DESC`;
case 'memberCount':
return `"memberCount" DESC NULLS LAST, "createdAt" DESC`;
case 'publicPageCount':
return `"publicPageCount" DESC NULLS LAST, "createdAt" DESC`;
case 'createdAt':
default:
return `"createdAt" DESC`;

3
packages/backend/server/src/schema.gql
View File

@@ -77,6 +77,9 @@ enum AdminWorkspaceSort {
BlobCount
BlobSize
CreatedAt
MemberCount
PublicPageCount
SnapshotCount
SnapshotSize
}

3
packages/common/graphql/src/schema.ts
View File

@@ -120,6 +120,9 @@ export enum AdminWorkspaceSort {
BlobCount = 'BlobCount',
BlobSize = 'BlobSize',
CreatedAt = 'CreatedAt',
MemberCount = 'MemberCount',
PublicPageCount = 'PublicPageCount',
SnapshotCount = 'SnapshotCount',
SnapshotSize = 'SnapshotSize',
}

11
packages/frontend/admin/src/components/shared/data-table-pagination.tsx
View File

@@ -17,10 +17,12 @@ import { useCallback, useTransition } from 'react';
interface DataTablePaginationProps<TData> {
table: Table<TData>;
disabled?: boolean;
}
export function DataTablePagination<TData>({
table,
disabled = false,
}: DataTablePaginationProps<TData>) {
const [, startTransition] = useTransition();
@@ -63,6 +65,7 @@ export function DataTablePagination<TData>({
<Select
value={`${table.getState().pagination.pageSize}`}
onValueChange={onPageSizeChange}
disabled={disabled}
>
<SelectTrigger className="h-8 w-[70px]">
<SelectValue placeholder={table.getState().pagination.pageSize} />
@@ -86,7 +89,7 @@ export function DataTablePagination<TData>({
variant="outline"
className="hidden h-8 w-8 p-0 lg:flex"
onClick={handleFirstPage}
disabled={!table.getCanPreviousPage()}
disabled={disabled || !table.getCanPreviousPage()}
>
<span className="sr-only">Go to first page</span>
<ChevronsLeftIcon className="h-4 w-4" />
@@ -95,7 +98,7 @@ export function DataTablePagination<TData>({
variant="outline"
className="h-8 w-8 p-0"
onClick={handlePreviousPage}
disabled={!table.getCanPreviousPage()}
disabled={disabled || !table.getCanPreviousPage()}
>
<span className="sr-only">Go to previous page</span>
<ChevronLeftIcon className="h-4 w-4" />
@@ -104,7 +107,7 @@ export function DataTablePagination<TData>({
variant="outline"
className="h-8 w-8 p-0"
onClick={handleNextPage}
disabled={!table.getCanNextPage()}
disabled={disabled || !table.getCanNextPage()}
>
<span className="sr-only">Go to next page</span>
<ChevronRightIcon className="h-4 w-4" />
@@ -113,7 +116,7 @@ export function DataTablePagination<TData>({
variant="outline"
className="h-8 w-8 p-0"
onClick={handleLastPage}
disabled={!table.getCanNextPage()}
disabled={disabled || !table.getCanNextPage()}
>
<span className="sr-only">Go to last page</span>
<ChevronsRightIcon className="h-4 w-4" />

38
packages/frontend/admin/src/components/shared/data-table.tsx
View File

@@ -27,6 +27,8 @@ interface DataTableProps<TData, TValue> {
totalCount: number;
pagination: PaginationState;
onPaginationChange: OnChangeFn<PaginationState>;
loading?: boolean;
disablePagination?: boolean;
// Row Selection
rowSelection?: RowSelectionState;
@@ -51,6 +53,8 @@ export function SharedDataTable<TData extends { id: string }, TValue>({
totalCount,
pagination,
onPaginationChange,
loading = false,
disablePagination = false,
rowSelection,
onRowSelectionChange,
renderToolbar,
@@ -83,9 +87,34 @@ export function SharedDataTable<TData extends { id: string }, TValue>({
});
return (
<div className="flex flex-col gap-4 py-5 px-6 h-full overflow-auto">
<div className="flex flex-col gap-4 py-5 px-6 h-full overflow-auto relative">
{renderToolbar?.(table)}
<div className="rounded-md border h-full flex flex-col overflow-auto">
<div className="rounded-md border h-full flex flex-col overflow-auto relative">
{loading ? (
<div className="absolute inset-0 z-10 bg-gray-50/70 backdrop-blur-[1px] flex flex-col items-center justify-center gap-2 text-sm text-gray-600">
<svg
className="h-5 w-5 animate-spin text-gray-500"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<circle
className="opacity-25"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="4"
/>
<path
className="opacity-75"
fill="currentColor"
d="M4 12a8 8 0 018-8v4a4 4 0 00-4 4H4z"
/>
</svg>
<span>Loading...</span>
</div>
) : null}
<Table>
<TableHeader>
{table.getHeaderGroups().map(headerGroup => (
@@ -160,7 +189,10 @@ export function SharedDataTable<TData extends { id: string }, TValue>({
</Table>
</div>
</div>
<DataTablePagination table={table} />
<DataTablePagination
table={table}
disabled={disablePagination || loading}
/>
</div>
);
}

18
packages/frontend/admin/src/components/shared/feature-filter-popover.tsx
View File

@@ -14,6 +14,7 @@ type FeatureFilterPopoverProps = {
onChange: (features: FeatureType[]) => void;
align?: 'start' | 'center' | 'end';
buttonLabel?: string;
disabled?: boolean;
};
export const FeatureFilterPopover = ({
@@ -22,29 +23,37 @@ export const FeatureFilterPopover = ({
onChange,
align = 'start',
buttonLabel = 'Features',
disabled = false,
}: FeatureFilterPopoverProps) => {
const handleFeatureToggle = useCallback(
(feature: FeatureType, checked: boolean) => {
if (disabled) {
return;
}
if (checked) {
onChange([...new Set([...selectedFeatures, feature])]);
} else {
onChange(selectedFeatures.filter(enabled => enabled !== feature));
}
},
[onChange, selectedFeatures]
[disabled, onChange, selectedFeatures]
);
const handleClearFeatures = useCallback(() => {
if (disabled) {
return;
}
onChange([]);
}, [onChange]);
}, [disabled, onChange]);
return (
<Popover>
<Popover open={disabled ? false : undefined}>
<PopoverTrigger asChild>
<Button
variant="outline"
size="sm"
className="h-8 px-2 lg:px-3 space-x-1"
disabled={disabled}
>
<span>{buttonLabel}</span>
{selectedFeatures.length > 0 ? (
@@ -70,6 +79,7 @@ export const FeatureFilterPopover = ({
onCheckedChange={checked =>
handleFeatureToggle(feature, !!checked)
}
disabled={disabled}
/>
<span className="text-sm truncate">{feature}</span>
</label>
@@ -80,7 +90,7 @@ export const FeatureFilterPopover = ({
variant="ghost"
size="sm"
onClick={handleClearFeatures}
disabled={selectedFeatures.length === 0}
disabled={disabled || selectedFeatures.length === 0}
>
Clear
</Button>

17
packages/frontend/admin/src/modules/workspaces/components/data-table-toolbar.tsx
View File

@@ -27,12 +27,16 @@ interface DataTableToolbarProps<TData> {
onFeaturesChange: (features: FeatureType[]) => void;
sort: AdminWorkspaceSort | undefined;
onSortChange: (sort: AdminWorkspaceSort | undefined) => void;
disabled?: boolean;
}
const sortOptions: { value: AdminWorkspaceSort; label: string }[] = [
{ value: AdminWorkspaceSort.SnapshotSize, label: 'Snapshot size' },
{ value: AdminWorkspaceSort.BlobCount, label: 'Blob count' },
{ value: AdminWorkspaceSort.BlobSize, label: 'Blob size' },
{ value: AdminWorkspaceSort.SnapshotCount, label: 'Snapshot count' },
{ value: AdminWorkspaceSort.MemberCount, label: 'Member count' },
{ value: AdminWorkspaceSort.PublicPageCount, label: 'Public pages' },
{ value: AdminWorkspaceSort.CreatedAt, label: 'Created time' },
];
@@ -43,6 +47,7 @@ export function DataTableToolbar<TData>({
onFeaturesChange,
sort,
onSortChange,
disabled = false,
}: DataTableToolbarProps<TData>) {
const [value, setValue] = useState(keyword);
const debouncedValue = useDebouncedValue(value, 400);
@@ -82,12 +87,18 @@ export function DataTableToolbar<TData>({
availableFeatures={availableFeatures}
onChange={onFeaturesChange}
align="start"
disabled={disabled}
/>
<div className="flex items-center gap-y-2 flex-wrap justify-end gap-2">
<Popover>
<Popover open={disabled ? false : undefined}>
<PopoverTrigger asChild>
<Button variant="outline" size="sm" className="h-8 px-2 lg:px-3">
<Button
variant="outline"
size="sm"
className="h-8 px-2 lg:px-3"
disabled={disabled}
>
Sort: {selectedSortLabel}
</Button>
</PopoverTrigger>
@@ -99,6 +110,7 @@ export function DataTableToolbar<TData>({
variant="ghost"
className="justify-start"
size="sm"
disabled={disabled}
onClick={() => handleSortChange(option.value)}
>
{option.label}
@@ -113,6 +125,7 @@ export function DataTableToolbar<TData>({
value={value}
onChange={onValueChange}
className="h-8 w-[150px] lg:w-[250px]"
disabled={disabled}
/>
</div>
</div>

5
packages/frontend/admin/src/modules/workspaces/components/data-table.tsx
View File

@@ -16,6 +16,7 @@ interface DataTableProps<TData, TValue> {
onFeaturesChange: (features: FeatureType[]) => void;
sort: AdminWorkspaceSort | undefined;
onSortChange: (sort: AdminWorkspaceSort | undefined) => void;
loading?: boolean;
onPaginationChange: Dispatch<
SetStateAction<{
pageIndex: number;
@@ -36,6 +37,7 @@ export function DataTable<TData extends { id: string }, TValue>({
sort,
onSortChange,
onPaginationChange,
loading = false,
}: DataTableProps<TData, TValue>) {
return (
<SharedDataTable
@@ -54,8 +56,11 @@ export function DataTable<TData extends { id: string }, TValue>({
onFeaturesChange={onFeaturesChange}
sort={sort}
onSortChange={onSortChange}
disabled={loading}
/>
)}
loading={loading}
disablePagination={loading}
/>
);
}

3
packages/frontend/admin/src/modules/workspaces/index.tsx
View File

@@ -13,7 +13,7 @@ export function WorkspacePage() {
AdminWorkspaceSort.CreatedAt
);
const { workspaces, pagination, setPagination, workspacesCount } =
const { workspaces, pagination, setPagination, workspacesCount, loading } =
useWorkspaceList({
keyword,
features: featureFilters,
@@ -38,6 +38,7 @@ export function WorkspacePage() {
onFeaturesChange={setFeatureFilters}
sort={sort}
onSortChange={setSort}
loading={loading}
/>
</div>
);

17
packages/frontend/admin/src/modules/workspaces/use-workspace-list.ts
View File

@@ -51,30 +51,43 @@ export const useWorkspaceList = (filter?: {
]
);
const { data: listData } = useQuery(
const { data: listData, isValidating: isListValidating } = useQuery(
{
query: adminWorkspacesQuery,
variables,
},
{
keepPreviousData: true,
revalidateOnFocus: false,
revalidateIfStale: true,
revalidateOnReconnect: true,
}
);
const { data: countData } = useQuery(
const { data: countData, isValidating: isCountValidating } = useQuery(
{
query: adminWorkspacesCountQuery,
variables,
},
{
keepPreviousData: true,
revalidateOnFocus: false,
revalidateIfStale: true,
revalidateOnReconnect: true,
}
);
const loading =
isListValidating ||
isCountValidating ||
listData === undefined ||
countData === undefined;
return {
workspaces: listData?.adminWorkspaces ?? [],
workspacesCount: countData?.adminWorkspacesCount ?? 0,
pagination,
setPagination,
loading,
};
};