mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-20 11:36:25 +08:00
chore: rename fundamentals to base (#9119)
This commit is contained in:
forehalo
@@ -0,0 +1,113 @@
|
||||
import { promises as fs } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import test from 'ava';
|
||||
import { getStreamAsBuffer } from 'get-stream';
|
||||
|
||||
import { ListObjectsMetadata } from '../providers';
|
||||
import { FsStorageProvider } from '../providers/fs';
|
||||
|
||||
const config = {
|
||||
path: join(process.cwd(), 'node_modules', '.cache/affine-test-storage'),
|
||||
};
|
||||
|
||||
function createProvider() {
|
||||
return new FsStorageProvider(
|
||||
config,
|
||||
'test' + Math.random().toString(16).substring(2, 8)
|
||||
);
|
||||
}
|
||||
|
||||
function keys(list: ListObjectsMetadata[]) {
|
||||
return list.map(i => i.key);
|
||||
}
|
||||
|
||||
async function randomPut(
|
||||
provider: FsStorageProvider,
|
||||
prefix = ''
|
||||
): Promise<string> {
|
||||
const key = prefix + 'test-key-' + Math.random().toString(16).substring(2, 8);
|
||||
const body = Buffer.from(key);
|
||||
provider.put(key, body);
|
||||
return key;
|
||||
}
|
||||
|
||||
test.after.always(() => {
|
||||
fs.rm(config.path, { recursive: true });
|
||||
});
|
||||
|
||||
test('put & get', async t => {
|
||||
const provider = createProvider();
|
||||
const key = 'testKey';
|
||||
const body = Buffer.from('testBody');
|
||||
await provider.put(key, body);
|
||||
|
||||
const result = await provider.get(key);
|
||||
|
||||
t.deepEqual(await getStreamAsBuffer(result.body!), body);
|
||||
t.is(result.metadata?.contentLength, body.length);
|
||||
});
|
||||
|
||||
test('list - one level', async t => {
|
||||
const provider = createProvider();
|
||||
const list = await Promise.all(
|
||||
Array.from({ length: 100 }).map(() => randomPut(provider))
|
||||
);
|
||||
list.sort();
|
||||
// random order, use set
|
||||
const result = await provider.list();
|
||||
t.deepEqual(keys(result), list);
|
||||
|
||||
const result2 = await provider.list('test-key');
|
||||
t.deepEqual(keys(result2), list);
|
||||
|
||||
const result3 = await provider.list('testKey');
|
||||
t.is(result3.length, 0);
|
||||
});
|
||||
|
||||
test('list recursively', async t => {
|
||||
const provider = createProvider();
|
||||
|
||||
await Promise.all([
|
||||
Promise.all(Array.from({ length: 10 }).map(() => randomPut(provider))),
|
||||
Promise.all(
|
||||
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/'))
|
||||
),
|
||||
Promise.all(
|
||||
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/'))
|
||||
),
|
||||
Promise.all(
|
||||
Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/t/'))
|
||||
),
|
||||
]);
|
||||
|
||||
const r1 = await provider.list();
|
||||
t.is(r1.length, 40);
|
||||
|
||||
// contains all `a/xxx` and `a/b/xxx` and `a/b/c/xxx`
|
||||
const r2 = await provider.list('a');
|
||||
t.is(r2.length, 30);
|
||||
|
||||
// contains only `a/b/xxx`
|
||||
const r3 = await provider.list('a/b');
|
||||
const r4 = await provider.list('a/b/');
|
||||
t.is(r3.length, 20);
|
||||
t.deepEqual(r3, r4);
|
||||
|
||||
// prefix is not ended with '/', it's open to all files and sub dirs
|
||||
// contains all `a/b/t/xxx` and `a/b/t{xxxx}`
|
||||
const r5 = await provider.list('a/b/t');
|
||||
|
||||
t.is(r5.length, 20);
|
||||
});
|
||||
|
||||
test('delete', async t => {
|
||||
const provider = createProvider();
|
||||
const key = 'testKey';
|
||||
const body = Buffer.from('testBody');
|
||||
await provider.put(key, body);
|
||||
|
||||
await provider.delete(key);
|
||||
|
||||
await t.throwsAsync(() => fs.access(join(config.path, provider.bucket, key)));
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { defineStartupConfig, ModuleConfig } from '../config';
|
||||
|
||||
export interface FsStorageConfig {
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface StorageProvidersConfig {
|
||||
fs?: FsStorageConfig;
|
||||
}
|
||||
|
||||
declare module '../config' {
|
||||
interface AppConfig {
|
||||
storageProviders: ModuleConfig<StorageProvidersConfig>;
|
||||
}
|
||||
}
|
||||
|
||||
defineStartupConfig('storageProviders', {
|
||||
fs: {
|
||||
path: join(homedir(), '.affine/storage'),
|
||||
},
|
||||
});
|
||||
|
||||
export type StorageProviderType = keyof StorageProvidersConfig;
|
||||
|
||||
export type StorageConfig<Ext = unknown> = {
|
||||
provider: StorageProviderType;
|
||||
bucket: string;
|
||||
} & Ext;
|
||||
|
||||
export interface StoragesConfig {
|
||||
avatar: StorageConfig<{ publicLinkFactory: (key: string) => string }>;
|
||||
blob: StorageConfig;
|
||||
copilot: StorageConfig;
|
||||
}
|
||||
|
||||
export interface AFFiNEStorageConfig {
|
||||
/**
|
||||
* All providers for object storage
|
||||
*
|
||||
* Support different providers for different usage at the same time.
|
||||
*/
|
||||
providers: StorageProvidersConfig;
|
||||
storages: StoragesConfig;
|
||||
}
|
||||
|
||||
export type StorageProviders = AFFiNEStorageConfig['providers'];
|
||||
export type Storages = keyof AFFiNEStorageConfig['storages'];
|
||||
|
||||
export function getDefaultAFFiNEStorageConfig(): AFFiNEStorageConfig {
|
||||
return {
|
||||
providers: {
|
||||
fs: {
|
||||
path: join(homedir(), '.affine/storage'),
|
||||
},
|
||||
},
|
||||
storages: {
|
||||
avatar: {
|
||||
provider: 'fs',
|
||||
bucket: 'avatars',
|
||||
publicLinkFactory: key => `/api/avatars/${key}`,
|
||||
},
|
||||
blob: {
|
||||
provider: 'fs',
|
||||
bucket: 'blobs',
|
||||
},
|
||||
copilot: {
|
||||
provider: 'fs',
|
||||
bucket: 'copilot',
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import './config';
|
||||
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
|
||||
import { registerStorageProvider, StorageProviderFactory } from './providers';
|
||||
import { FsStorageProvider } from './providers/fs';
|
||||
|
||||
registerStorageProvider('fs', (config, bucket) => {
|
||||
if (!config.storageProviders.fs) {
|
||||
throw new Error('Missing fs storage provider configuration');
|
||||
}
|
||||
|
||||
return new FsStorageProvider(config.storageProviders.fs, bucket);
|
||||
});
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [StorageProviderFactory],
|
||||
exports: [StorageProviderFactory],
|
||||
})
|
||||
export class StorageProviderModule {}
|
||||
|
||||
export * from '../../native';
|
||||
export type { StorageProviderType } from './config';
|
||||
export type {
|
||||
BlobInputType,
|
||||
BlobOutputType,
|
||||
GetObjectMetadata,
|
||||
ListObjectsMetadata,
|
||||
PutObjectMetadata,
|
||||
StorageProvider,
|
||||
} from './providers';
|
||||
export { registerStorageProvider, StorageProviderFactory } from './providers';
|
||||
export { autoMetadata, toBuffer } from './providers/utils';
|
||||
@@ -0,0 +1,264 @@
|
||||
import {
|
||||
accessSync,
|
||||
constants,
|
||||
createReadStream,
|
||||
Dirent,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { join, parse, resolve } from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import { FsStorageConfig } from '../config';
|
||||
import {
|
||||
BlobInputType,
|
||||
GetObjectMetadata,
|
||||
ListObjectsMetadata,
|
||||
PutObjectMetadata,
|
||||
StorageProvider,
|
||||
} from './provider';
|
||||
import { autoMetadata, toBuffer } from './utils';
|
||||
|
||||
function escapeKey(key: string): string {
|
||||
// avoid '../' and './' in key
|
||||
return key.replace(/\.?\.[/\\]/g, '%');
|
||||
}
|
||||
|
||||
export class FsStorageProvider implements StorageProvider {
|
||||
private readonly path: string;
|
||||
private readonly logger: Logger;
|
||||
|
||||
readonly type = 'fs';
|
||||
|
||||
constructor(
|
||||
config: FsStorageConfig,
|
||||
public readonly bucket: string
|
||||
) {
|
||||
this.path = resolve(config.path, bucket);
|
||||
this.ensureAvailability();
|
||||
|
||||
this.logger = new Logger(`${FsStorageProvider.name}:${bucket}`);
|
||||
}
|
||||
|
||||
async put(
|
||||
key: string,
|
||||
body: BlobInputType,
|
||||
metadata: PutObjectMetadata = {}
|
||||
): Promise<void> {
|
||||
key = escapeKey(key);
|
||||
const blob = await toBuffer(body);
|
||||
|
||||
// write object
|
||||
this.writeObject(key, blob);
|
||||
// write metadata
|
||||
await this.writeMetadata(key, blob, metadata);
|
||||
this.logger.verbose(`Object \`${key}\` put`);
|
||||
}
|
||||
|
||||
async get(key: string): Promise<{
|
||||
body?: Readable;
|
||||
metadata?: GetObjectMetadata;
|
||||
}> {
|
||||
key = escapeKey(key);
|
||||
|
||||
try {
|
||||
const metadata = this.readMetadata(key);
|
||||
const stream = this.readObject(this.join(key));
|
||||
this.logger.verbose(`Read object \`${key}\``);
|
||||
return {
|
||||
body: stream,
|
||||
metadata,
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to read object \`${key}\``, e);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async list(prefix?: string): Promise<ListObjectsMetadata[]> {
|
||||
// prefix cases:
|
||||
// - `undefined`: list all objects
|
||||
// - `a/b`: list objects under dir `a` with prefix `b`, `b` might be a dir under `a` as well.
|
||||
// - `a/b/` list objects under dir `a/b`
|
||||
|
||||
// read dir recursively and filter out '.metadata.json' files
|
||||
let dir = this.path;
|
||||
if (prefix) {
|
||||
prefix = escapeKey(prefix);
|
||||
const parts = prefix.split(/[/\\]/);
|
||||
// for prefix `a/b/c`, move `a/b` to dir and `c` to key prefix
|
||||
if (parts.length > 1) {
|
||||
dir = join(dir, ...parts.slice(0, -1));
|
||||
prefix = parts[parts.length - 1];
|
||||
}
|
||||
}
|
||||
|
||||
const results: ListObjectsMetadata[] = [];
|
||||
async function getFiles(dir: string, prefix?: string): Promise<void> {
|
||||
try {
|
||||
const entries: Dirent[] = readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const res = join(dir, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
if (!prefix || entry.name.startsWith(prefix)) {
|
||||
await getFiles(res);
|
||||
}
|
||||
} else if (
|
||||
(!prefix || entry.name.startsWith(prefix)) &&
|
||||
!entry.name.endsWith('.metadata.json')
|
||||
) {
|
||||
const stat = statSync(res);
|
||||
results.push({
|
||||
key: res,
|
||||
lastModified: stat.mtime,
|
||||
contentLength: stat.size,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// failed to read dir, stop recursion
|
||||
}
|
||||
}
|
||||
|
||||
await getFiles(dir, prefix);
|
||||
|
||||
// trim path with `this.path` prefix
|
||||
results.forEach(r => (r.key = r.key.slice(this.path.length + 1)));
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
delete(key: string): Promise<void> {
|
||||
key = escapeKey(key);
|
||||
|
||||
try {
|
||||
rmSync(this.join(key), { force: true });
|
||||
rmSync(this.join(`${key}.metadata.json`), { force: true });
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to delete object \`${key}\``, {
|
||||
cause: e,
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.verbose(`Object \`${key}\` deleted`);
|
||||
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
ensureAvailability() {
|
||||
// check stats
|
||||
const stats = statSync(this.path, {
|
||||
throwIfNoEntry: false,
|
||||
});
|
||||
|
||||
// not existing, create it
|
||||
if (!stats) {
|
||||
try {
|
||||
mkdirSync(this.path, { recursive: true });
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Failed to create target directory for fs storage provider: ${this.path}`,
|
||||
{
|
||||
cause: e,
|
||||
}
|
||||
);
|
||||
}
|
||||
} else if (stats.isDirectory()) {
|
||||
// the target directory has already existed, check if it is readable & writable
|
||||
try {
|
||||
accessSync(this.path, constants.W_OK | constants.R_OK);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`The target directory for fs storage provider has already existed, but it is not readable & writable: ${this.path}`,
|
||||
{
|
||||
cause: e,
|
||||
}
|
||||
);
|
||||
}
|
||||
} else if (stats.isFile()) {
|
||||
throw new Error(
|
||||
`The target directory for fs storage provider is a file: ${this.path}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private join(...paths: string[]) {
|
||||
return join(this.path, ...paths);
|
||||
}
|
||||
|
||||
private readObject(file: string): Readable | undefined {
|
||||
const state = statSync(file, { throwIfNoEntry: false });
|
||||
|
||||
if (state?.isFile()) {
|
||||
return createReadStream(file);
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private writeObject(key: string, blob: Buffer) {
|
||||
const path = this.join(key);
|
||||
mkdirSync(parse(path).dir, { recursive: true });
|
||||
writeFileSync(path, blob);
|
||||
}
|
||||
|
||||
private async writeMetadata(
|
||||
key: string,
|
||||
blob: Buffer,
|
||||
raw: PutObjectMetadata
|
||||
) {
|
||||
try {
|
||||
const metadata = autoMetadata(blob, raw);
|
||||
|
||||
if (raw.checksumCRC32 && metadata.checksumCRC32 !== raw.checksumCRC32) {
|
||||
throw new Error(
|
||||
'The checksum of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'
|
||||
);
|
||||
}
|
||||
|
||||
if (raw.contentLength && metadata.contentLength !== raw.contentLength) {
|
||||
throw new Error(
|
||||
'The content length of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'
|
||||
);
|
||||
}
|
||||
|
||||
writeFileSync(
|
||||
this.join(`${key}.metadata.json`),
|
||||
JSON.stringify({
|
||||
...metadata,
|
||||
lastModified: Date.now(),
|
||||
})
|
||||
);
|
||||
} catch (e) {
|
||||
this.logger.warn(`Failed to write metadata of object \`${key}\``, e);
|
||||
}
|
||||
}
|
||||
|
||||
private readMetadata(key: string): GetObjectMetadata | undefined {
|
||||
try {
|
||||
const raw = JSON.parse(
|
||||
readFileSync(this.join(`${key}.metadata.json`), {
|
||||
encoding: 'utf-8',
|
||||
})
|
||||
);
|
||||
|
||||
return {
|
||||
...raw,
|
||||
lastModified: new Date(raw.lastModified),
|
||||
expires: raw.expires ? new Date(raw.expires) : undefined,
|
||||
};
|
||||
} catch (e) {
|
||||
this.logger.warn(`Failed to read metadata of object \`${key}\``, e);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Config } from '../../config';
|
||||
import { StorageConfig, StorageProviderType } from '../config';
|
||||
import type { StorageProvider } from './provider';
|
||||
|
||||
const availableProviders = new Map<
|
||||
StorageProviderType,
|
||||
(config: Config, bucket: string) => StorageProvider
|
||||
>();
|
||||
|
||||
export function registerStorageProvider(
|
||||
type: StorageProviderType,
|
||||
providerFactory: (config: Config, bucket: string) => StorageProvider
|
||||
) {
|
||||
availableProviders.set(type, providerFactory);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class StorageProviderFactory {
|
||||
constructor(private readonly config: Config) {}
|
||||
|
||||
create(storage: StorageConfig): StorageProvider {
|
||||
const providerFactory = availableProviders.get(storage.provider);
|
||||
|
||||
if (!providerFactory) {
|
||||
throw new Error(`Unknown storage provider type: ${storage.provider}`);
|
||||
}
|
||||
|
||||
return providerFactory(this.config, storage.bucket);
|
||||
}
|
||||
}
|
||||
|
||||
export type * from './provider';
|
||||
@@ -0,0 +1,42 @@
|
||||
import type { Readable } from 'node:stream';
|
||||
|
||||
import { StorageProviderType } from '../config';
|
||||
|
||||
export interface GetObjectMetadata {
|
||||
/**
|
||||
* @default 'application/octet-stream'
|
||||
*/
|
||||
contentType: string;
|
||||
contentLength: number;
|
||||
lastModified: Date;
|
||||
checksumCRC32?: string;
|
||||
}
|
||||
|
||||
export interface PutObjectMetadata {
|
||||
contentType?: string;
|
||||
contentLength?: number;
|
||||
checksumCRC32?: string;
|
||||
}
|
||||
|
||||
export interface ListObjectsMetadata {
|
||||
key: string;
|
||||
lastModified: Date;
|
||||
contentLength: number;
|
||||
}
|
||||
|
||||
export type BlobInputType = Buffer | Readable | string;
|
||||
export type BlobOutputType = Readable;
|
||||
|
||||
export interface StorageProvider {
|
||||
readonly type: StorageProviderType;
|
||||
put(
|
||||
key: string,
|
||||
body: BlobInputType,
|
||||
metadata?: PutObjectMetadata
|
||||
): Promise<void>;
|
||||
get(
|
||||
key: string
|
||||
): Promise<{ body?: BlobOutputType; metadata?: GetObjectMetadata }>;
|
||||
list(prefix?: string): Promise<ListObjectsMetadata[]>;
|
||||
delete(key: string): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { crc32 } from '@node-rs/crc32';
|
||||
import { getStreamAsBuffer } from 'get-stream';
|
||||
|
||||
import { getMime } from '../../../native';
|
||||
import { BlobInputType, PutObjectMetadata } from './provider';
|
||||
|
||||
export async function toBuffer(input: BlobInputType): Promise<Buffer> {
|
||||
return input instanceof Readable
|
||||
? await getStreamAsBuffer(input)
|
||||
: input instanceof Buffer
|
||||
? input
|
||||
: Buffer.from(input);
|
||||
}
|
||||
|
||||
export function autoMetadata(
|
||||
blob: Buffer,
|
||||
raw: PutObjectMetadata = {}
|
||||
): PutObjectMetadata {
|
||||
const metadata = {
|
||||
...raw,
|
||||
};
|
||||
|
||||
if (!metadata.contentLength) {
|
||||
metadata.contentLength = blob.byteLength;
|
||||
}
|
||||
|
||||
try {
|
||||
// checksum
|
||||
if (!metadata.checksumCRC32) {
|
||||
metadata.checksumCRC32 = crc32(blob).toString(16);
|
||||
}
|
||||
|
||||
// mime type
|
||||
if (!metadata.contentType) {
|
||||
metadata.contentType = getMime(blob);
|
||||
}
|
||||
} catch {
|
||||
// noop
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
Reference in New Issue
Block a user