feat(core): unused blob management in settings (#9795)

fix AF-2144, PD-2064, PD-2065, PD-2066
This commit is contained in:
pengx17
2025-01-23 07:12:16 +00:00
parent 8021b89944
commit 6ac6a8d6d6
26 changed files with 846 additions and 30 deletions
@@ -1,5 +1,5 @@
import { ShadowlessElement } from '@blocksuite/affine/block-std';
import { getAttachmentFileIcons } from '@blocksuite/affine/blocks';
import { getAttachmentFileIcon } from '@blocksuite/affine/blocks';
import { SignalWatcher, WithDisposable } from '@blocksuite/affine/global/utils';
import { html } from 'lit';
import { property } from 'lit/decorators.js';
@@ -17,7 +17,7 @@ export class ChatPanelFileChip extends SignalWatcher(
const { state, fileName, fileType } = this.chip;
const isLoading = state === 'embedding' || state === 'uploading';
const tooltip = getChipTooltip(state, fileName, this.chip.tooltip);
const fileIcon = getAttachmentFileIcons(fileType);
const fileIcon = getAttachmentFileIcon(fileType);
const icon = getChipIcon(state, fileIcon);
return html`<chat-panel-chip
@@ -74,9 +74,11 @@ export const sidebarSelectSubItem = style({
export const sidebarSelectItemIcon = style({
width: '16px',
height: '16px',
fontSize: '16px',
marginRight: '10px',
flexShrink: 0,
color: cssVarV2('icon/primary'),
display: 'inline-flex',
});
export const sidebarSelectItemName = style({
@@ -43,7 +43,7 @@ export const WorkspaceSetting = ({
case 'workspace:billing':
return <WorkspaceSettingBilling />;
case 'workspace:storage':
return <WorkspaceSettingStorage />;
return <WorkspaceSettingStorage onCloseSetting={onCloseSetting} />;
default:
return null;
}
@@ -0,0 +1,276 @@
import {
Button,
Checkbox,
Loading,
templateToString,
useConfirmModal,
useDisposable,
} from '@affine/component';
import { Pagination } from '@affine/component/member-components';
import { BlobManagementService } from '@affine/core/modules/blob-management/services';
import { useI18n } from '@affine/i18n';
import type { ListedBlobRecord } from '@affine/nbstore';
import { getAttachmentFileIcon } from '@blocksuite/affine/blocks';
import { DeleteIcon } from '@blocksuite/icons/rc';
import { useLiveData, useService } from '@toeverything/infra';
import bytes from 'bytes';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import * as styles from './style.css';
const Empty = () => {
const t = useI18n();
return (
<div className={styles.empty}>
{t['com.affine.settings.workspace.storage.unused-blobs.empty']()}
</div>
);
};
const useBlob = (blobRecord: ListedBlobRecord) => {
const unusedBlobsEntity = useService(BlobManagementService).unusedBlobs;
return useDisposable(
(abortSignal?: AbortSignal) =>
unusedBlobsEntity.hydrateBlob(blobRecord, abortSignal),
[blobRecord]
);
};
const BlobPreview = ({ blobRecord }: { blobRecord: ListedBlobRecord }) => {
const { data, loading, error } = useBlob(blobRecord);
const element = useMemo(() => {
if (loading) return <Loading size={24} />;
if (!data?.url || !data.type) return null;
const { url, type, mime } = data;
const icon = templateToString(getAttachmentFileIcon(type));
if (error) {
return (
<div
className={styles.unknownBlobIcon}
dangerouslySetInnerHTML={{ __html: icon }}
/>
);
}
if (mime?.startsWith('image/')) {
return (
<img
className={styles.blobImagePreview}
src={url}
alt={blobRecord.key}
/>
);
} else {
return (
<div
className={styles.unknownBlobIcon}
dangerouslySetInnerHTML={{ __html: icon }}
/>
);
}
}, [loading, data, error, blobRecord.key]);
return (
<div className={styles.blobPreviewContainer}>
<div className={styles.blobPreview}>{element}</div>
<div className={styles.blobPreviewFooter}>
<div className={styles.blobPreviewName}>{blobRecord.key}</div>
<div className={styles.blobPreviewInfo}>
{data?.type} · {bytes(blobRecord.size)}
</div>
</div>
</div>
);
};
const BlobCard = ({
blobRecord,
onClick,
selected,
}: {
blobRecord: ListedBlobRecord;
onClick: () => void;
selected: boolean;
}) => {
return (
<div
data-testid="blob-preview-card"
className={styles.blobCard}
data-selected={selected}
onClick={onClick}
>
<Checkbox className={styles.blobGridItemCheckbox} checked={selected} />
<BlobPreview blobRecord={blobRecord} />
</div>
);
};
const PAGE_SIZE = 9;
export const BlobManagementPanel = () => {
const t = useI18n();
const unusedBlobsEntity = useService(BlobManagementService).unusedBlobs;
const originalUnusedBlobs = useLiveData(unusedBlobsEntity.unusedBlobs$);
const isLoading = useLiveData(unusedBlobsEntity.isLoading$);
const [pageNum, setPageNum] = useState(0);
const [skip, setSkip] = useState(0);
const [unusedBlobs, setUnusedBlobs] = useState<ListedBlobRecord[]>([]);
const unusedBlobsPage = useMemo(() => {
return unusedBlobs.slice(skip, skip + PAGE_SIZE);
}, [unusedBlobs, skip]);
useEffect(() => {
setUnusedBlobs(originalUnusedBlobs);
}, [originalUnusedBlobs]);
useEffect(() => {
unusedBlobsEntity.revalidate();
}, [unusedBlobsEntity]);
const [selectedBlobs, setSelectedBlobs] = useState<ListedBlobRecord[]>([]);
const [deleting, setDeleting] = useState(false);
const handleSelectBlob = useCallback((blob: ListedBlobRecord) => {
setSelectedBlobs(prev => {
if (prev.includes(blob)) {
return prev;
}
return [...prev, blob];
});
}, []);
const handleUnselectBlob = useCallback((blob: ListedBlobRecord) => {
setSelectedBlobs(prev => prev.filter(b => b.key !== blob.key));
}, []);
const handleSelectAll = useCallback(() => {
unusedBlobsPage.forEach(blob => handleSelectBlob(blob));
}, [unusedBlobsPage, handleSelectBlob]);
const { openConfirmModal } = useConfirmModal();
const handleDeleteSelectedBlobs = useCallback(
(e: React.MouseEvent) => {
e.stopPropagation();
const currentSelectedBlobs = selectedBlobs;
openConfirmModal({
title:
t[
'com.affine.settings.workspace.storage.unused-blobs.delete.title'
](),
children:
t[
'com.affine.settings.workspace.storage.unused-blobs.delete.warning'
](),
onConfirm: async () => {
setDeleting(true);
for (const blob of currentSelectedBlobs) {
await unusedBlobsEntity.deleteBlob(blob.key, true);
handleUnselectBlob(blob);
setUnusedBlobs(prev => prev.filter(b => b.key !== blob.key));
}
setDeleting(false);
},
confirmText: t['Delete'](),
cancelText: t['Cancel'](),
confirmButtonOptions: {
variant: 'error',
},
});
},
[selectedBlobs, openConfirmModal, t, unusedBlobsEntity, handleUnselectBlob]
);
const blobPreviewGridRef = useRef<HTMLDivElement>(null);
useEffect(() => {
if (blobPreviewGridRef.current) {
const unselectBlobs = (e: MouseEvent) => {
const target = e.target as HTMLElement;
if (
!blobPreviewGridRef.current?.contains(target) &&
!target.closest('modal-transition-container')
) {
setSelectedBlobs([]);
}
};
document.addEventListener('click', unselectBlobs);
return () => {
document.removeEventListener('click', unselectBlobs);
};
}
return;
}, [unusedBlobs]);
return (
<>
{selectedBlobs.length > 0 ? (
<div className={styles.blobManagementControls}>
<div className={styles.blobManagementName}>
{`${selectedBlobs.length} ${t['com.affine.settings.workspace.storage.unused-blobs.selected']()}`}
</div>
<div className={styles.spacer} />
<Button onClick={handleSelectAll} variant="primary">
{t['com.affine.keyboardShortcuts.selectAll']()}
</Button>
<Button
loading={deleting}
onClick={handleDeleteSelectedBlobs}
prefix={<DeleteIcon />}
disabled={deleting}
>
{t['Delete']()}
</Button>
</div>
) : (
<div className={styles.blobManagementNameInactive}>
{`${t['com.affine.settings.workspace.storage.unused-blobs']()} (${unusedBlobs.length})`}
</div>
)}
<div className={styles.blobManagementContainer}>
{isLoading ? (
<div className={styles.loadingContainer}>
<Loading size={32} />
</div>
) : unusedBlobs.length === 0 ? (
<Empty />
) : (
<>
<div className={styles.blobPreviewGrid} ref={blobPreviewGridRef}>
{unusedBlobs.slice(skip, skip + PAGE_SIZE).map(blob => {
const selected = selectedBlobs.includes(blob);
return (
<BlobCard
key={blob.key}
blobRecord={blob}
onClick={() =>
selected
? handleUnselectBlob(blob)
: handleSelectBlob(blob)
}
selected={selected}
/>
);
})}
</div>
<Pagination
pageNum={pageNum}
totalCount={unusedBlobs.length}
countPerPage={PAGE_SIZE}
onPageChange={(_, pageNum) => {
setPageNum(pageNum);
setSkip(pageNum * PAGE_SIZE);
}}
/>
</>
)}
</div>
</>
);
};
@@ -6,10 +6,16 @@ import { WorkspaceService } from '@affine/core/modules/workspace';
import { useI18n } from '@affine/i18n';
import { useService } from '@toeverything/infra';
import { EnableCloudPanel } from '../preference/enable-cloud';
import { BlobManagementPanel } from './blob-management';
import { DesktopExportPanel } from './export';
import { WorkspaceQuotaPanel } from './workspace-quota';
export const WorkspaceSettingStorage = () => {
export const WorkspaceSettingStorage = ({
onCloseSetting,
}: {
onCloseSetting: () => void;
}) => {
const t = useI18n();
const workspace = useService(WorkspaceService).workspace;
return (
@@ -18,10 +24,18 @@ export const WorkspaceSettingStorage = () => {
title={t['Storage']()}
subtitle={t['com.affine.settings.workspace.storage.subtitle']()}
/>
{workspace.flavour !== 'local' && (
<SettingWrapper>
<WorkspaceQuotaPanel />
</SettingWrapper>
{workspace.flavour === 'local' ? (
<EnableCloudPanel onCloseSetting={onCloseSetting} />
) : (
<>
<SettingWrapper>
<WorkspaceQuotaPanel />
</SettingWrapper>
<SettingWrapper>
<BlobManagementPanel />
</SettingWrapper>
</>
)}
{BUILD_CONFIG.isElectron && (
<SettingWrapper>
@@ -28,3 +28,136 @@ globalStyle(`${storageProgressWrapper} .storage-progress-bar-wrapper`, {
export const storageProgressBar = style({
height: '100%',
});
// blob management
// when no blob is selected
export const blobManagementControls = style({
display: 'flex',
alignItems: 'center',
justifyContent: 'space-between',
gap: 16,
});
export const spacer = style({
flexGrow: 1,
});
export const blobManagementName = style({
fontSize: cssVar('fontSm'),
fontWeight: 600,
height: '28px',
});
export const blobManagementNameInactive = style([
blobManagementName,
{
color: cssVarV2('text/secondary'),
},
]);
export const blobManagementContainer = style({
marginTop: '24px',
display: 'flex',
flexDirection: 'column',
gap: '12px',
padding: '12px',
borderRadius: '8px',
background: cssVarV2('layer/background/primary'),
border: `1px solid ${cssVarV2('layer/insideBorder/border')}`,
});
export const blobPreviewGrid = style({
display: 'grid',
gridTemplateColumns: 'repeat(3, minmax(30%, 1fr))',
gap: '12px',
});
export const blobCard = style({
borderRadius: '4px',
overflow: 'hidden',
position: 'relative',
});
export const loadingContainer = style({
display: 'flex',
justifyContent: 'center',
alignItems: 'center',
height: '320px',
});
export const empty = style({
padding: '8px 16px',
});
export const blobPreviewContainer = style({
display: 'flex',
flexDirection: 'column',
justifyContent: 'center',
alignItems: 'center',
gap: 8,
});
export const blobPreview = style({
width: '100%',
overflow: 'hidden',
aspectRatio: '1',
borderRadius: '4px',
padding: 6,
backgroundColor: cssVarV2('layer/background/secondary'),
display: 'flex',
justifyContent: 'center',
alignItems: 'center',
border: `2px solid transparent`,
selectors: {
[`${blobCard}[data-selected="true"] &`]: {
borderColor: cssVarV2('button/primary'),
},
[`${blobCard}:hover &`]: {
backgroundColor: cssVarV2('layer/background/hoverOverlay'),
},
},
});
export const blobGridItemCheckbox = style({
position: 'absolute',
top: 8,
right: 8,
fontSize: 16,
opacity: 0,
selectors: {
[`${blobCard}:hover &`]: {
opacity: 1,
},
[`${blobCard}[data-selected="true"] &`]: {
opacity: 1,
},
},
});
export const blobImagePreview = style({
width: '100%',
height: '100%',
objectFit: 'contain',
});
export const unknownBlobIcon = style({});
export const blobPreviewFooter = style({
fontSize: cssVar('fontXs'),
width: '100%',
});
export const blobPreviewName = style({
fontSize: cssVar('fontSm'),
fontWeight: 600,
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
maxWidth: '100%',
});
export const blobPreviewInfo = style({
fontSize: cssVar('fontXs'),
color: cssVarV2('text/secondary'),
});
@@ -46,7 +46,7 @@ export const StorageProgress = () => {
if (loadError) {
return <ErrorMessage>Load error</ErrorMessage>;
}
return <Skeleton height={42} />;
return <Skeleton height={26} />;
}
if (!isTeam) {
@@ -0,0 +1,164 @@
import type { ListedBlobRecord } from '@affine/nbstore';
import {
effect,
Entity,
fromPromise,
LiveData,
onComplete,
onStart,
} from '@toeverything/infra';
import { fileTypeFromBuffer } from 'file-type';
import { EMPTY, mergeMap, switchMap } from 'rxjs';
import type { DocsSearchService } from '../../docs-search';
import type { WorkspaceService } from '../../workspace';
import type { WorkspaceFlavoursService } from '../../workspace/services/flavours';
interface HydratedBlobRecord extends ListedBlobRecord, Disposable {
url: string;
extension?: string;
type?: string;
}
export class UnusedBlobs extends Entity {
constructor(
private readonly flavoursService: WorkspaceFlavoursService,
private readonly workspaceService: WorkspaceService,
private readonly docsSearchService: DocsSearchService
) {
super();
}
isLoading$ = new LiveData(false);
unusedBlobs$ = new LiveData<ListedBlobRecord[]>([]);
readonly revalidate = effect(
switchMap(() =>
fromPromise(async () => {
return await this.getUnusedBlobs();
}).pipe(
mergeMap(data => {
this.unusedBlobs$.setValue(data);
return EMPTY;
}),
onStart(() => this.isLoading$.setValue(true)),
onComplete(() => this.isLoading$.setValue(false))
)
)
);
private get flavourProvider() {
return this.flavoursService.flavours$.value.find(
f => f.flavour === this.workspaceService.workspace.flavour
);
}
private get localFlavourProvider() {
return this.flavoursService.flavours$.value.find(
f => f.flavour === 'local'
);
}
async listBlobs() {
const blobs = await this.flavourProvider?.listBlobs(
this.workspaceService.workspace.id
);
return blobs;
}
async getBlob(blobKey: string) {
const blob = await this.flavourProvider?.getWorkspaceBlob(
this.workspaceService.workspace.id,
blobKey
);
return blob;
}
async deleteBlob(blob: string, permanent: boolean) {
await this.flavourProvider?.deleteBlob(
this.workspaceService.workspace.id,
blob,
permanent
);
if (this.localFlavourProvider !== this.flavourProvider) {
await this.localFlavourProvider?.deleteBlob(
this.workspaceService.workspace.id,
blob,
permanent
);
}
}
async getUnusedBlobs(abortSignal?: AbortSignal) {
// wait for the indexer to finish
await this.docsSearchService.indexer.status$.waitFor(
status => status.remaining === undefined || status.remaining === 0,
abortSignal
);
const [blobs, usedBlobs] = await Promise.all([
this.listBlobs(),
this.getUsedBlobs(),
]);
// ignore the workspace avatar
const workspaceAvatar = this.workspaceService.workspace.avatar$.value;
return (
blobs?.filter(
blob => !usedBlobs.includes(blob.key) && blob.key !== workspaceAvatar
) ?? []
);
}
private async getUsedBlobs(): Promise<string[]> {
const result = await this.docsSearchService.indexer.blockIndex.aggregate(
{
type: 'boolean',
occur: 'must',
queries: [
{
type: 'exists',
field: 'blob',
},
],
},
'blob'
);
return result.buckets.map(bucket => bucket.key);
}
async hydrateBlob(
record: ListedBlobRecord,
abortSignal?: AbortSignal
): Promise<HydratedBlobRecord | null> {
const blob = await this.getBlob(record.key);
if (!blob || abortSignal?.aborted) {
return null;
}
const fileType = await fileTypeFromBuffer(await blob.arrayBuffer());
if (abortSignal?.aborted) {
return null;
}
const url = URL.createObjectURL(new Blob([blob]));
const mime = record.mime || fileType?.mime || 'unknown';
// todo(@pengx17): the following may not be sufficient
const extension = fileType?.ext;
const type = extension ?? (mime?.startsWith('text/') ? 'txt' : 'unknown');
return {
...record,
url,
extension,
type,
mime,
[Symbol.dispose]: () => {
URL.revokeObjectURL(url);
},
};
}
}
@@ -0,0 +1,18 @@
import { type Framework } from '@toeverything/infra';
import { DocsSearchService } from '../docs-search';
import { WorkspaceScope, WorkspaceService } from '../workspace';
import { WorkspaceFlavoursService } from '../workspace/services/flavours';
import { UnusedBlobs } from './entity/unused-blobs';
import { BlobManagementService } from './services';
export function configureBlobManagementModule(framework: Framework) {
framework
.scope(WorkspaceScope)
.entity(UnusedBlobs, [
WorkspaceFlavoursService,
WorkspaceService,
DocsSearchService,
])
.service(BlobManagementService);
}
@@ -0,0 +1,11 @@
import { Service } from '@toeverything/infra';
import { UnusedBlobs } from '../entity/unused-blobs';
export class BlobManagementService extends Service {
constructor() {
super();
}
unusedBlobs = this.framework.createEntity(UnusedBlobs);
}
@@ -7,6 +7,7 @@ import {
} from './ai-button';
import { configureAppSidebarModule } from './app-sidebar';
import { configAtMenuConfigModule } from './at-menu-config';
import { configureBlobManagementModule } from './blob-management';
import { configureCloudModule } from './cloud';
import { configureCollectionModule } from './collection';
import { configureWorkspaceDBModule } from './db';
@@ -98,4 +99,5 @@ export function configureCommonModules(framework: Framework) {
configureAINetworkSearchModule(framework);
configureAIButtonModule(framework);
configureTemplateDocModule(framework);
configureBlobManagementModule(framework);
}
@@ -5,7 +5,11 @@ import {
getWorkspaceInfoQuery,
getWorkspacesQuery,
} from '@affine/graphql';
import type { BlobStorage, DocStorage } from '@affine/nbstore';
import type {
BlobStorage,
DocStorage,
ListedBlobRecord,
} from '@affine/nbstore';
import { CloudBlobStorage, StaticCloudDocStorage } from '@affine/nbstore/cloud';
import {
IndexedDBBlobStorage,
@@ -364,6 +368,26 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
return new Blob([cloudBlob.data], { type: cloudBlob.mime });
}
async listBlobs(id: string): Promise<ListedBlobRecord[]> {
const cloudStorage = new CloudBlobStorage({
id,
serverBaseUrl: this.server.serverMetadata.baseUrl,
});
return cloudStorage.list();
}
async deleteBlob(
id: string,
blob: string,
permanent: boolean
): Promise<void> {
const cloudStorage = new CloudBlobStorage({
id,
serverBaseUrl: this.server.serverMetadata.baseUrl,
});
await cloudStorage.delete(blob, permanent);
}
onWorkspaceInitialized(workspace: Workspace): void {
// bind the workspace to the affine cloud server
workspace.scope.get(WorkspaceServerService).bindServer(this.server);
@@ -2,6 +2,7 @@ import { DebugLogger } from '@affine/debug';
import {
type BlobStorage,
type DocStorage,
type ListedBlobRecord,
universalId,
} from '@affine/nbstore';
import {
@@ -276,6 +277,33 @@ class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
return blob ? new Blob([blob.data], { type: blob.mime }) : null;
}
async listBlobs(id: string): Promise<ListedBlobRecord[]> {
const storage = new this.BlobStorageType({
id: id,
flavour: this.flavour,
type: 'workspace',
});
storage.connection.connect();
await storage.connection.waitForConnected();
return storage.list();
}
async deleteBlob(
id: string,
blob: string,
permanent: boolean
): Promise<void> {
const storage = new this.BlobStorageType({
id: id,
flavour: this.flavour,
type: 'workspace',
});
storage.connection.connect();
await storage.connection.waitForConnected();
await storage.delete(blob, permanent);
}
getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions {
return {
local: {
@@ -1,4 +1,8 @@
import type { BlobStorage, DocStorage } from '@affine/nbstore';
import type {
BlobStorage,
DocStorage,
ListedBlobRecord,
} from '@affine/nbstore';
import type { WorkerInitOptions } from '@affine/nbstore/worker/client';
import type { Workspace as BSWorkspace } from '@blocksuite/affine/store';
import { createIdentifier, type LiveData } from '@toeverything/infra';
@@ -41,6 +45,14 @@ export interface WorkspaceFlavourProvider {
getWorkspaceBlob(id: string, blob: string): Promise<Blob | null>;
listBlobs(workspaceId: string): Promise<ListedBlobRecord[]>;
deleteBlob(
workspaceId: string,
blob: string,
permanent: boolean
): Promise<void>;
getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions;
onWorkspaceInitialized?(workspace: Workspace): void;