mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-08-10 13:38:49 +08:00
feat(server): converge legacy compatibility (#15426)
#### PR Dependency Tree * **PR #15426** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added workspace BYOK profiles with provider/model catalogs, capability validation, connection probing, credential rotation, reordering, and secure local leases. * Added Copilot route options, selectable targets, managed tiers, explicit profile/model overrides, and improved streaming with tool callbacks and abort support. * Added Copilot availability controls to prevent access when the feature is disabled. * **Changes** * Simplified Copilot configuration and removed legacy provider-specific settings. * Removed obsolete model, token-cost, transcript strategy, and provider metadata fields from public responses. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -1,2 +1 @@
|
||||
export * from './feature-coverage';
|
||||
export * from './policy';
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { CopilotQuotaExceeded } from '../../../base';
|
||||
import { ByokService } from '../byok/service';
|
||||
import type { ByokFeatureKind } from '../byok/types';
|
||||
import type { CopilotProviderProfile } from '../config';
|
||||
import { ConversationPolicy } from '../conversation/policy';
|
||||
import {
|
||||
getByokSourceCoverage,
|
||||
getCopilotFeatureAccess,
|
||||
} from './feature-coverage';
|
||||
|
||||
export type CopilotAccessContext = {
|
||||
userId?: string;
|
||||
workspaceId?: string;
|
||||
byokLeaseId?: string;
|
||||
featureKind?: ByokFeatureKind;
|
||||
quotaBackedRoutesAllowed?: boolean;
|
||||
};
|
||||
|
||||
export type CopilotRouteAccess = {
|
||||
byokProfiles: CopilotProviderProfile[];
|
||||
quotaBackedRoutesAvailable: boolean;
|
||||
};
|
||||
|
||||
export type CopilotTurnRouteAccess = {
|
||||
byokProfiles: CopilotProviderProfile[];
|
||||
quotaBackedRoutesAllowed?: boolean;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CopilotAccessPolicy {
|
||||
constructor(
|
||||
private readonly conversationPolicy: ConversationPolicy,
|
||||
private readonly byok: ByokService
|
||||
) {}
|
||||
|
||||
async getByokProfiles(context: CopilotAccessContext = {}) {
|
||||
const coverage = getByokSourceCoverage(context.featureKind);
|
||||
return await this.byok.getProfiles(context, coverage);
|
||||
}
|
||||
|
||||
async canUseQuotaBackedRoutes(context: CopilotAccessContext = {}) {
|
||||
if (context.quotaBackedRoutesAllowed !== undefined) {
|
||||
return context.quotaBackedRoutesAllowed;
|
||||
}
|
||||
if (!getCopilotFeatureAccess(context.featureKind).quotaMetered) {
|
||||
return true;
|
||||
}
|
||||
if (!context.userId) {
|
||||
return true;
|
||||
}
|
||||
return await this.conversationPolicy.hasQuota(context.userId);
|
||||
}
|
||||
|
||||
async getQuota(userId: string) {
|
||||
return await this.conversationPolicy.getQuota(userId);
|
||||
}
|
||||
|
||||
async checkQuota(userId: string) {
|
||||
await this.conversationPolicy.checkQuota(userId);
|
||||
}
|
||||
|
||||
async resolveRouteAccess(
|
||||
context: CopilotAccessContext = {}
|
||||
): Promise<CopilotRouteAccess> {
|
||||
const [byokProfiles, quotaBackedRoutesAvailable] = await Promise.all([
|
||||
this.getByokProfiles(context),
|
||||
this.canUseQuotaBackedRoutes(context),
|
||||
]);
|
||||
|
||||
return { byokProfiles, quotaBackedRoutesAvailable };
|
||||
}
|
||||
|
||||
async resolveTurnRouteAccess(
|
||||
context: CopilotAccessContext
|
||||
): Promise<CopilotTurnRouteAccess> {
|
||||
const byokProfiles = await this.getByokProfiles(context);
|
||||
if (context.quotaBackedRoutesAllowed === false) {
|
||||
return { byokProfiles, quotaBackedRoutesAllowed: false };
|
||||
}
|
||||
const featureAccess = getCopilotFeatureAccess(context.featureKind);
|
||||
if (!byokProfiles.length && context.userId && featureAccess.quotaMetered) {
|
||||
await this.conversationPolicy.checkQuota(context.userId);
|
||||
}
|
||||
|
||||
const quotaBackedRoutesAllowed = byokProfiles.length
|
||||
? context.quotaBackedRoutesAllowed
|
||||
: true;
|
||||
return { byokProfiles, quotaBackedRoutesAllowed };
|
||||
}
|
||||
|
||||
async assertQuotaOrByok(context: CopilotAccessContext) {
|
||||
const byokProfiles = await this.getByokProfiles(context);
|
||||
if (context.quotaBackedRoutesAllowed === false) {
|
||||
if (!byokProfiles.length) {
|
||||
throw new CopilotQuotaExceeded();
|
||||
}
|
||||
return;
|
||||
}
|
||||
const featureAccess = getCopilotFeatureAccess(context.featureKind);
|
||||
if (!byokProfiles.length && context.userId && featureAccess.quotaMetered) {
|
||||
await this.conversationPolicy.checkQuota(context.userId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import type { Config } from '../../base/config';
|
||||
import { ActionForbidden } from '../../base/error/errors.gen';
|
||||
|
||||
export function assertCopilotEnabled(config: Config) {
|
||||
if (!config.copilot.enabled) {
|
||||
throw new ActionForbidden('Copilot is disabled.');
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
export { ByokEntitlementPolicy } from './policy';
|
||||
export { WorkspaceByokResolver } from './resolver';
|
||||
export { type ByokProviderRequestContext, ByokService } from './service';
|
||||
export * from './types';
|
||||
|
||||
@@ -103,6 +103,16 @@ export class ByokEntitlementPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
async assertEntitled(workspaceId: string, userId?: string) {
|
||||
const [serverEntitled, localEntitled] = await this.hasEntitlement(
|
||||
workspaceId,
|
||||
userId
|
||||
);
|
||||
if (!serverEntitled && !localEntitled) {
|
||||
throw new ActionForbidden('BYOK requires Pro, Team, or Believer.');
|
||||
}
|
||||
}
|
||||
|
||||
private async hasWorkspaceTeamPlan(workspaceId: string) {
|
||||
try {
|
||||
const state =
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
|
||||
import type { safeFetch } from '../../../base';
|
||||
import { ByokProvider } from './types';
|
||||
|
||||
const TEST_TIMEOUT_MS = 10_000;
|
||||
export const PROVIDER_PROBE_MAX_BYTES = 1024 * 1024;
|
||||
|
||||
type ProbeFetch = typeof safeFetch;
|
||||
|
||||
export async function runProviderProbe(
|
||||
probeFetch: ProbeFetch,
|
||||
provider: ByokProvider,
|
||||
apiKey: string,
|
||||
endpoint: string | null,
|
||||
allowPrivateEndpoint: boolean
|
||||
) {
|
||||
const request = buildProbeRequest(provider, apiKey, endpoint);
|
||||
const response = await probeFetch(
|
||||
request.url,
|
||||
{
|
||||
method: request.method,
|
||||
headers: request.headers,
|
||||
},
|
||||
{
|
||||
timeoutMs: TEST_TIMEOUT_MS,
|
||||
maxRedirects: 3,
|
||||
maxBytes: PROVIDER_PROBE_MAX_BYTES,
|
||||
allowedHeaders: Object.keys(request.headers),
|
||||
allowHttp: endpoint?.startsWith('http:') ?? false,
|
||||
allowPrivateTargetOrigin: allowPrivateEndpoint,
|
||||
}
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new BadRequestException(providerProbeFailureMessage(response.status));
|
||||
}
|
||||
}
|
||||
|
||||
function buildProbeRequest(
|
||||
provider: ByokProvider,
|
||||
apiKey: string,
|
||||
endpoint: string | null
|
||||
): {
|
||||
method: 'GET';
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
} {
|
||||
switch (provider) {
|
||||
case ByokProvider.openai:
|
||||
return {
|
||||
method: 'GET',
|
||||
url: `${endpoint ?? 'https://api.openai.com/v1'}/models`,
|
||||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
};
|
||||
case ByokProvider.anthropic:
|
||||
return {
|
||||
method: 'GET',
|
||||
url: `${endpoint ?? 'https://api.anthropic.com/v1'}/models`,
|
||||
headers: {
|
||||
'x-api-key': apiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
};
|
||||
case ByokProvider.gemini:
|
||||
return {
|
||||
method: 'GET',
|
||||
url: `${endpoint ?? 'https://generativelanguage.googleapis.com/v1beta'}/models`,
|
||||
headers: { 'x-goog-api-key': apiKey },
|
||||
};
|
||||
case ByokProvider.fal:
|
||||
return {
|
||||
method: 'GET',
|
||||
url: 'https://api.fal.ai/v1/models?limit=10',
|
||||
headers: { Authorization: `Key ${apiKey}` },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function providerProbeFailureMessage(status: number) {
|
||||
switch (status) {
|
||||
case 401:
|
||||
return 'Provider rejected the BYOK key.';
|
||||
case 403:
|
||||
return 'Provider rejected the BYOK key permissions.';
|
||||
case 404:
|
||||
return 'Provider probe endpoint was not found.';
|
||||
case 429:
|
||||
return 'Provider rate limit exceeded while testing the key.';
|
||||
default:
|
||||
return status >= 500
|
||||
? 'Provider service is unavailable.'
|
||||
: `Provider key test failed with HTTP ${status}.`;
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import {
|
||||
Args,
|
||||
Field,
|
||||
@@ -11,18 +12,139 @@ import {
|
||||
} from '@nestjs/graphql';
|
||||
import { SafeIntResolver } from 'graphql-scalars';
|
||||
|
||||
import { Throttle } from '../../../base';
|
||||
import { Config, Throttle } from '../../../base';
|
||||
import { CurrentUser } from '../../../core/auth';
|
||||
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
|
||||
import { PermissionAccess } from '../../../core/permission';
|
||||
import { WorkspaceType } from '../../../core/workspaces';
|
||||
import { Models } from '../../../models';
|
||||
import { llmGetByokCatalog } from '../../../native';
|
||||
import { CopilotEnabled } from '../feature';
|
||||
import { ByokEntitlementPolicy } from './policy';
|
||||
import { ByokKeyConfig, ByokLocalLeaseProvider, ByokService } from './service';
|
||||
import { ByokKeyStorage, ByokKeyTestStatus, ByokProvider } from './types';
|
||||
import {
|
||||
BYOK_ALLOWED_PROVIDERS,
|
||||
ByokProvider,
|
||||
ByokProviderSource,
|
||||
} from './types';
|
||||
|
||||
@ObjectType()
|
||||
export class WorkspaceByokKeyConfigType implements ByokKeyConfig {
|
||||
class WorkspaceByokCapabilityType {
|
||||
@Field(() => [String])
|
||||
input!: string[];
|
||||
|
||||
@Field(() => [String])
|
||||
output!: string[];
|
||||
|
||||
@Field(() => [String])
|
||||
features!: string[];
|
||||
|
||||
@Field(() => [String])
|
||||
attachmentKinds!: string[];
|
||||
|
||||
@Field(() => [String])
|
||||
attachmentSources!: string[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokModelDeclarationType {
|
||||
@Field(() => String)
|
||||
modelId!: string;
|
||||
|
||||
@Field(() => Boolean)
|
||||
enabled!: boolean;
|
||||
|
||||
@Field(() => [WorkspaceByokCapabilityType])
|
||||
capabilities!: WorkspaceByokCapabilityType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokEndpointType {
|
||||
@Field(() => String)
|
||||
kind!: string;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
url!: string | null;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokProfileDefinitionType {
|
||||
@Field(() => SafeIntResolver)
|
||||
version!: number;
|
||||
|
||||
@Field(() => WorkspaceByokEndpointType)
|
||||
endpoint!: WorkspaceByokEndpointType;
|
||||
|
||||
@Field(() => [WorkspaceByokModelDeclarationType])
|
||||
models!: WorkspaceByokModelDeclarationType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokProbeStatusType {
|
||||
@Field(() => String)
|
||||
kind!: string;
|
||||
|
||||
@Field(() => Date, { nullable: true })
|
||||
testedAt!: Date | null;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
errorKind!: string | null;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokModelProbeCheckType {
|
||||
@Field(() => String)
|
||||
operation!: string;
|
||||
|
||||
@Field(() => WorkspaceByokProbeStatusType)
|
||||
status!: WorkspaceByokProbeStatusType;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokModelProbeType {
|
||||
@Field(() => String)
|
||||
modelId!: string;
|
||||
|
||||
@Field(() => [WorkspaceByokModelProbeCheckType])
|
||||
checks!: WorkspaceByokModelProbeCheckType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokValidationType {
|
||||
@Field(() => String)
|
||||
definitionFingerprint!: string;
|
||||
|
||||
@Field(() => SafeIntResolver)
|
||||
credentialGeneration!: number;
|
||||
|
||||
@Field(() => WorkspaceByokProbeStatusType)
|
||||
connection!: WorkspaceByokProbeStatusType;
|
||||
|
||||
@Field(() => [WorkspaceByokModelProbeType])
|
||||
models!: WorkspaceByokModelProbeType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokProbeResultType {
|
||||
@Field(() => String)
|
||||
definitionFingerprint!: string;
|
||||
|
||||
@Field(() => Boolean)
|
||||
stale!: boolean;
|
||||
|
||||
@Field(() => WorkspaceByokProbeStatusType)
|
||||
connection!: WorkspaceByokProbeStatusType;
|
||||
|
||||
@Field(() => [WorkspaceByokModelProbeType])
|
||||
models!: WorkspaceByokModelProbeType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
export class WorkspaceByokProfileType {
|
||||
@Field(() => ID)
|
||||
id!: string;
|
||||
profileId!: string;
|
||||
|
||||
@Field(() => String)
|
||||
workspaceId!: string;
|
||||
|
||||
@Field(() => ByokProvider)
|
||||
provider!: ByokProvider;
|
||||
@@ -33,59 +155,53 @@ export class WorkspaceByokKeyConfigType implements ByokKeyConfig {
|
||||
@Field(() => String, { nullable: true })
|
||||
description!: string | null;
|
||||
|
||||
@Field(() => ByokKeyStorage)
|
||||
storage!: ByokKeyStorage;
|
||||
|
||||
@Field(() => Boolean)
|
||||
configured!: boolean;
|
||||
@Field(() => WorkspaceByokProfileDefinitionType)
|
||||
definition!: WorkspaceByokProfileDefinitionType;
|
||||
|
||||
@Field(() => Boolean)
|
||||
enabled!: boolean;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
endpoint!: string | null;
|
||||
|
||||
@Field(() => Boolean)
|
||||
endpointEditable!: boolean;
|
||||
|
||||
@Field(() => SafeIntResolver)
|
||||
sortOrder!: number;
|
||||
|
||||
@Field(() => [String])
|
||||
capabilities!: string[];
|
||||
@Field(() => SafeIntResolver)
|
||||
revision!: number;
|
||||
|
||||
@Field(() => ByokKeyTestStatus)
|
||||
testStatus!: ByokKeyTestStatus;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
disabledReason!: string | null;
|
||||
|
||||
@Field(() => Date, { nullable: true })
|
||||
lastTestedAt!: Date | null;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
lastTestError!: string | null;
|
||||
|
||||
@Field(() => Date, { nullable: true })
|
||||
lastUsedAt!: Date | null;
|
||||
|
||||
@Field(() => Date, { nullable: true })
|
||||
lastErrorAt!: Date | null;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
lastError!: string | null;
|
||||
@Field(() => WorkspaceByokValidationType, { nullable: true })
|
||||
validation!: WorkspaceByokValidationType | null;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokCapabilityWarningType {
|
||||
class WorkspaceByokCatalogModelType {
|
||||
@Field(() => String)
|
||||
featureKind!: string;
|
||||
modelId!: string;
|
||||
|
||||
@Field(() => String)
|
||||
reason!: string;
|
||||
displayName!: string;
|
||||
|
||||
@Field(() => [ByokProvider])
|
||||
requiredProviders!: ByokProvider[];
|
||||
@Field(() => Boolean)
|
||||
recommended!: boolean;
|
||||
|
||||
@Field(() => [WorkspaceByokCapabilityType])
|
||||
capabilities!: WorkspaceByokCapabilityType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokCatalogProviderType {
|
||||
@Field(() => ByokProvider)
|
||||
provider!: ByokProvider;
|
||||
|
||||
@Field(() => [WorkspaceByokCatalogModelType])
|
||||
models!: WorkspaceByokCatalogModelType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class WorkspaceByokCatalogType {
|
||||
@Field(() => String)
|
||||
version!: string;
|
||||
|
||||
@Field(() => [WorkspaceByokCatalogProviderType])
|
||||
providers!: WorkspaceByokCatalogProviderType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
@@ -102,29 +218,20 @@ class WorkspaceByokSettingsType {
|
||||
@Field(() => Boolean)
|
||||
localEntitled!: boolean;
|
||||
|
||||
@Field(() => [String])
|
||||
entitlementRequired!: string[];
|
||||
|
||||
@Field(() => [WorkspaceByokKeyConfigType])
|
||||
keys!: WorkspaceByokKeyConfigType[];
|
||||
@Field(() => [WorkspaceByokProfileType])
|
||||
profiles!: WorkspaceByokProfileType[];
|
||||
|
||||
@Field(() => [ByokProvider])
|
||||
allowedProviders!: ByokProvider[];
|
||||
|
||||
@Field(() => Boolean)
|
||||
localStorageSupported!: boolean;
|
||||
|
||||
@Field(() => Boolean)
|
||||
customEndpointSupported!: boolean;
|
||||
|
||||
@Field(() => Boolean)
|
||||
privateEndpointSupported!: boolean;
|
||||
|
||||
@Field(() => Boolean)
|
||||
hasAiPlan!: boolean;
|
||||
|
||||
@Field(() => [WorkspaceByokCapabilityWarningType])
|
||||
warnings!: WorkspaceByokCapabilityWarningType[];
|
||||
@Field(() => WorkspaceByokCatalogType)
|
||||
catalog!: WorkspaceByokCatalogType;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
@@ -139,18 +246,6 @@ class WorkspaceByokUsagePointType {
|
||||
totalTokens!: number;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class TestWorkspaceByokConfigResultType {
|
||||
@Field(() => Boolean)
|
||||
ok!: boolean;
|
||||
|
||||
@Field(() => ByokKeyTestStatus)
|
||||
status!: ByokKeyTestStatus;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
message!: string | null;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class CreateWorkspaceByokLocalLeaseResultType {
|
||||
@Field(() => String)
|
||||
@@ -161,10 +256,58 @@ class CreateWorkspaceByokLocalLeaseResultType {
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class UpsertWorkspaceByokConfigInput {
|
||||
@Field(() => ID, { nullable: true })
|
||||
id?: string;
|
||||
class WorkspaceByokCapabilityInput {
|
||||
@Field(() => [String])
|
||||
input!: string[];
|
||||
|
||||
@Field(() => [String])
|
||||
output!: string[];
|
||||
|
||||
@Field(() => [String])
|
||||
features!: string[];
|
||||
|
||||
@Field(() => [String])
|
||||
attachmentKinds!: string[];
|
||||
|
||||
@Field(() => [String])
|
||||
attachmentSources!: string[];
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class WorkspaceByokModelDeclarationInput {
|
||||
@Field(() => String)
|
||||
modelId!: string;
|
||||
|
||||
@Field(() => Boolean)
|
||||
enabled!: boolean;
|
||||
|
||||
@Field(() => [WorkspaceByokCapabilityInput])
|
||||
capabilities!: WorkspaceByokCapabilityInput[];
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class WorkspaceByokEndpointInput {
|
||||
@Field(() => String)
|
||||
kind!: string;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
url!: string | null;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class WorkspaceByokProfileDefinitionInput {
|
||||
@Field(() => SafeIntResolver)
|
||||
version!: number;
|
||||
|
||||
@Field(() => WorkspaceByokEndpointInput)
|
||||
endpoint!: WorkspaceByokEndpointInput;
|
||||
|
||||
@Field(() => [WorkspaceByokModelDeclarationInput])
|
||||
models!: WorkspaceByokModelDeclarationInput[];
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class CreateWorkspaceByokProfileInput {
|
||||
@Field(() => String)
|
||||
workspaceId!: string;
|
||||
|
||||
@@ -175,59 +318,125 @@ class UpsertWorkspaceByokConfigInput {
|
||||
name!: string;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
description?: string | null;
|
||||
description!: string | null;
|
||||
|
||||
@Field(() => ByokKeyStorage)
|
||||
storage!: ByokKeyStorage;
|
||||
@Field(() => String)
|
||||
credential!: string;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
apiKey?: string | null;
|
||||
@Field(() => WorkspaceByokProfileDefinitionInput)
|
||||
definition!: WorkspaceByokProfileDefinitionInput;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
endpoint?: string | null;
|
||||
|
||||
@Field(() => SafeIntResolver, { nullable: true })
|
||||
sortOrder?: number | null;
|
||||
|
||||
@Field(() => Boolean, { nullable: true })
|
||||
enabled?: boolean | null;
|
||||
@Field(() => Boolean)
|
||||
enabled!: boolean;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class TestWorkspaceByokConfigInput {
|
||||
class ReplaceWorkspaceByokProfileInput {
|
||||
@Field(() => String)
|
||||
workspaceId!: string;
|
||||
|
||||
@Field(() => ID)
|
||||
profileId!: string;
|
||||
|
||||
@Field(() => SafeIntResolver)
|
||||
expectedRevision!: number;
|
||||
|
||||
@Field(() => String)
|
||||
name!: string;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
description!: string | null;
|
||||
|
||||
@Field(() => WorkspaceByokProfileDefinitionInput)
|
||||
definition!: WorkspaceByokProfileDefinitionInput;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
credential!: string | null;
|
||||
|
||||
@Field(() => Boolean)
|
||||
enabled!: boolean;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class RotateWorkspaceByokCredentialInput {
|
||||
@Field(() => String)
|
||||
workspaceId!: string;
|
||||
|
||||
@Field(() => ID)
|
||||
profileId!: string;
|
||||
|
||||
@Field(() => SafeIntResolver)
|
||||
expectedRevision!: number;
|
||||
|
||||
@Field(() => String)
|
||||
credential!: string;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class WorkspaceByokProbeCheckInput {
|
||||
@Field(() => String)
|
||||
modelId!: string;
|
||||
|
||||
@Field(() => String)
|
||||
operation!: string;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class ProbeWorkspaceByokProfileInput {
|
||||
@Field(() => String)
|
||||
workspaceId!: string;
|
||||
|
||||
@Field(() => ID)
|
||||
profileId!: string;
|
||||
|
||||
@Field(() => [WorkspaceByokProbeCheckInput])
|
||||
checks!: WorkspaceByokProbeCheckInput[];
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class ProbeWorkspaceByokDraftInput {
|
||||
@Field(() => String)
|
||||
workspaceId!: string;
|
||||
|
||||
@Field(() => ByokProvider)
|
||||
provider!: ByokProvider;
|
||||
|
||||
@Field(() => ByokKeyStorage)
|
||||
storage!: ByokKeyStorage;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
apiKey?: string | null;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
endpoint?: string | null;
|
||||
credential!: string | null;
|
||||
|
||||
@Field(() => ID, { nullable: true })
|
||||
configId?: string | null;
|
||||
profileId!: string | null;
|
||||
|
||||
@Field(() => SafeIntResolver, { nullable: true })
|
||||
expectedRevision!: number | null;
|
||||
|
||||
@Field(() => WorkspaceByokProfileDefinitionInput)
|
||||
definition!: WorkspaceByokProfileDefinitionInput;
|
||||
|
||||
@Field(() => [WorkspaceByokProbeCheckInput])
|
||||
checks!: WorkspaceByokProbeCheckInput[];
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class ReorderWorkspaceByokConfigsInput {
|
||||
class WorkspaceByokProfileOrderInput {
|
||||
@Field(() => ID)
|
||||
profileId!: string;
|
||||
|
||||
@Field(() => SafeIntResolver)
|
||||
expectedRevision!: number;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class ReorderWorkspaceByokProfilesInput {
|
||||
@Field(() => String)
|
||||
workspaceId!: string;
|
||||
|
||||
@Field(() => ByokKeyStorage)
|
||||
storage!: ByokKeyStorage;
|
||||
|
||||
@Field(() => [ID])
|
||||
ids!: string[];
|
||||
@Field(() => [WorkspaceByokProfileOrderInput])
|
||||
profiles!: WorkspaceByokProfileOrderInput[];
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class CreateWorkspaceByokLocalLeaseProviderInput implements ByokLocalLeaseProvider {
|
||||
class CreateWorkspaceByokLocalLeaseProviderInput {
|
||||
@Field(() => ByokProvider)
|
||||
provider!: ByokProvider;
|
||||
|
||||
@@ -235,19 +444,16 @@ class CreateWorkspaceByokLocalLeaseProviderInput implements ByokLocalLeaseProvid
|
||||
name!: string;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
description?: string | null;
|
||||
description!: string | null;
|
||||
|
||||
@Field(() => String)
|
||||
apiKey!: string;
|
||||
credential!: string;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
endpoint?: string | null;
|
||||
@Field(() => WorkspaceByokProfileDefinitionInput)
|
||||
definition!: WorkspaceByokProfileDefinitionInput;
|
||||
|
||||
@Field(() => SafeIntResolver, { nullable: true })
|
||||
sortOrder?: number | null;
|
||||
|
||||
@Field(() => Boolean, { nullable: true })
|
||||
enabled?: boolean | null;
|
||||
@Field(() => Boolean)
|
||||
enabled!: boolean;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
@@ -259,12 +465,15 @@ class CreateWorkspaceByokLocalLeaseInput {
|
||||
providers!: CreateWorkspaceByokLocalLeaseProviderInput[];
|
||||
}
|
||||
|
||||
@CopilotEnabled()
|
||||
@Resolver(() => WorkspaceType)
|
||||
export class WorkspaceByokResolver {
|
||||
constructor(
|
||||
private readonly ac: PermissionAccess,
|
||||
private readonly entitlement: ByokEntitlementPolicy,
|
||||
private readonly byok: ByokService
|
||||
private readonly runtime: BackendRuntimeProvider,
|
||||
private readonly models: Models,
|
||||
private readonly config: Config
|
||||
) {}
|
||||
|
||||
@ResolveField(() => WorkspaceByokSettingsType, {
|
||||
@@ -275,13 +484,35 @@ export class WorkspaceByokResolver {
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Parent() workspace: WorkspaceType
|
||||
) {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspace.id)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Read');
|
||||
await this.assertRead(user.id, workspace.id);
|
||||
await this.entitlement.assertManagementAccess(workspace.id, user.id);
|
||||
return await this.byok.getSettings(workspace.id, user.id);
|
||||
const [serverEntitled, localEntitled] =
|
||||
await this.entitlement.hasEntitlement(workspace.id, user.id);
|
||||
const profiles = serverEntitled
|
||||
? await this.runtime.listByokProfiles(workspace.id)
|
||||
: [];
|
||||
const customEndpointSupported =
|
||||
this.config.copilot.byok.allowCustomEndpoint;
|
||||
const catalog = llmGetByokCatalog();
|
||||
return {
|
||||
workspaceId: workspace.id,
|
||||
entitled: serverEntitled || localEntitled,
|
||||
serverEntitled,
|
||||
localEntitled,
|
||||
profiles: profiles.map(profile => projectProfile(profile)),
|
||||
allowedProviders: [...BYOK_ALLOWED_PROVIDERS],
|
||||
customEndpointSupported,
|
||||
privateEndpointSupported:
|
||||
customEndpointSupported &&
|
||||
this.config.copilot.byok.allowPrivateEndpoint,
|
||||
catalog: {
|
||||
...catalog,
|
||||
providers: catalog.providers.map(provider => ({
|
||||
...provider,
|
||||
provider: provider.provider as ByokProvider,
|
||||
})),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@ResolveField(() => [WorkspaceByokUsagePointType], {
|
||||
@@ -294,100 +525,131 @@ export class WorkspaceByokResolver {
|
||||
@Args('from', { type: () => Date }) from: Date,
|
||||
@Args('to', { type: () => Date }) to: Date
|
||||
) {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspace.id)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Read');
|
||||
await this.assertRead(user.id, workspace.id);
|
||||
await this.entitlement.assertManagementAccess(workspace.id, user.id);
|
||||
return await this.byok.getUsage(workspace.id, from, to);
|
||||
return await this.models.copilotUsage.aggregateByDay({
|
||||
workspaceId: workspace.id,
|
||||
from,
|
||||
to,
|
||||
providerSources: [ByokProviderSource.Server, ByokProviderSource.Local],
|
||||
});
|
||||
}
|
||||
|
||||
@Mutation(() => WorkspaceByokProfileType)
|
||||
@Throttle('strict')
|
||||
@Mutation(() => TestWorkspaceByokConfigResultType)
|
||||
async testWorkspaceByokConfig(
|
||||
async createWorkspaceByokProfile(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('input') input: TestWorkspaceByokConfigInput
|
||||
@Args('input') input: CreateWorkspaceByokProfileInput
|
||||
) {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(input.workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Update');
|
||||
await this.entitlement.assertManagementAccess(input.workspaceId, user.id);
|
||||
if (input.storage === ByokKeyStorage.server) {
|
||||
await this.assertUpdate(user.id, input.workspaceId);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
requireExplicitDescription(input);
|
||||
return projectProfile(
|
||||
await this.runtime.createByokProfile({
|
||||
...input,
|
||||
description: input.description ?? undefined,
|
||||
definition: nativeDefinition(input.definition),
|
||||
actorUserId: user.id,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@Mutation(() => WorkspaceByokProfileType)
|
||||
@Throttle('strict')
|
||||
async replaceWorkspaceByokProfile(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('input') input: ReplaceWorkspaceByokProfileInput
|
||||
) {
|
||||
await this.assertUpdate(user.id, input.workspaceId);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
requireExplicitDescription(input);
|
||||
return projectProfile(
|
||||
await this.runtime.replaceByokProfile({
|
||||
...input,
|
||||
description: input.description ?? undefined,
|
||||
credential: input.credential ?? undefined,
|
||||
definition: nativeDefinition(input.definition),
|
||||
actorUserId: user.id,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@Mutation(() => WorkspaceByokProfileType)
|
||||
@Throttle('strict')
|
||||
async rotateWorkspaceByokCredential(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('input') input: RotateWorkspaceByokCredentialInput
|
||||
) {
|
||||
await this.assertUpdate(user.id, input.workspaceId);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
return projectProfile(
|
||||
await this.runtime.rotateByokCredential({
|
||||
...input,
|
||||
actorUserId: user.id,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@Mutation(() => WorkspaceByokProbeResultType)
|
||||
@Throttle('strict')
|
||||
async probeWorkspaceByokProfile(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('input') input: ProbeWorkspaceByokProfileInput
|
||||
) {
|
||||
await this.assertUpdate(user.id, input.workspaceId);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
return projectProbeResult(await this.runtime.probeByokProfile(input));
|
||||
}
|
||||
|
||||
@Mutation(() => WorkspaceByokProbeResultType)
|
||||
@Throttle('strict')
|
||||
async probeWorkspaceByokDraft(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('input') input: ProbeWorkspaceByokDraftInput
|
||||
) {
|
||||
await this.assertUpdate(user.id, input.workspaceId);
|
||||
if (input.profileId) {
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
} else {
|
||||
await this.entitlement.assertLocalEntitled(input.workspaceId, user.id);
|
||||
await this.entitlement.assertEntitled(input.workspaceId, user.id);
|
||||
}
|
||||
return await this.byok.testConfig({ ...input, userId: user.id });
|
||||
}
|
||||
|
||||
@Mutation(() => WorkspaceByokKeyConfigType)
|
||||
@Throttle('strict')
|
||||
async upsertWorkspaceByokConfig(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('input') input: UpsertWorkspaceByokConfigInput
|
||||
) {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(input.workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Update');
|
||||
await this.entitlement.assertManagementAccess(input.workspaceId, user.id);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
return await this.byok.upsertConfig({ ...input, userId: user.id });
|
||||
}
|
||||
|
||||
@Mutation(() => [WorkspaceByokKeyConfigType])
|
||||
@Throttle('strict')
|
||||
async reorderWorkspaceByokConfigs(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('input') input: ReorderWorkspaceByokConfigsInput
|
||||
) {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(input.workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Update');
|
||||
await this.entitlement.assertManagementAccess(input.workspaceId, user.id);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
return await this.byok.reorderConfigs({ ...input, userId: user.id });
|
||||
return projectProbeResult(
|
||||
await this.runtime.probeByokDraft({
|
||||
...input,
|
||||
credential: input.credential ?? undefined,
|
||||
profileId: input.profileId ?? undefined,
|
||||
expectedRevision: input.expectedRevision ?? undefined,
|
||||
definition: nativeDefinition(input.definition),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@Mutation(() => Boolean)
|
||||
@Throttle('strict')
|
||||
async deleteWorkspaceByokConfig(
|
||||
async deleteWorkspaceByokProfile(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('id', { type: () => ID }) id: string,
|
||||
@Args('profileId', { type: () => ID }) profileId: string,
|
||||
@Args('workspaceId', { type: () => String }) workspaceId: string
|
||||
) {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Update');
|
||||
await this.entitlement.assertManagementAccess(workspaceId, user.id);
|
||||
await this.assertUpdate(user.id, workspaceId);
|
||||
await this.entitlement.assertServerEntitled(workspaceId);
|
||||
return await this.byok.deleteConfig(workspaceId, id, user.id);
|
||||
return await this.runtime.deleteByokProfile(workspaceId, profileId);
|
||||
}
|
||||
|
||||
@Mutation(() => Boolean)
|
||||
@Mutation(() => [WorkspaceByokProfileType])
|
||||
@Throttle('strict')
|
||||
async clearWorkspaceByokConfigs(
|
||||
async reorderWorkspaceByokProfiles(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('workspaceId', { type: () => String }) workspaceId: string,
|
||||
@Args('provider', { type: () => ByokProvider, nullable: true })
|
||||
provider?: ByokProvider | null
|
||||
@Args('input') input: ReorderWorkspaceByokProfilesInput
|
||||
) {
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Update');
|
||||
await this.entitlement.assertManagementAccess(workspaceId, user.id);
|
||||
await this.entitlement.assertServerEntitled(workspaceId);
|
||||
return await this.byok.clearConfigs(workspaceId, provider, user.id);
|
||||
await this.assertUpdate(user.id, input.workspaceId);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
return (
|
||||
await this.runtime.reorderByokProfiles({
|
||||
...input,
|
||||
actorUserId: user.id,
|
||||
})
|
||||
).map(profile => projectProfile(profile));
|
||||
}
|
||||
|
||||
@Mutation(() => CreateWorkspaceByokLocalLeaseResultType)
|
||||
@@ -403,6 +665,119 @@ export class WorkspaceByokResolver {
|
||||
.assert('Workspace.Copilot');
|
||||
await this.entitlement.assertManagementAccess(input.workspaceId, user.id);
|
||||
await this.entitlement.assertLocalEntitled(input.workspaceId, user.id);
|
||||
return await this.byok.createLocalLease({ ...input, userId: user.id });
|
||||
input.providers.forEach(requireExplicitDescription);
|
||||
const result = await this.runtime.createByokLocalLease({
|
||||
...input,
|
||||
providers: input.providers.map(provider => ({
|
||||
...provider,
|
||||
description: provider.description ?? undefined,
|
||||
definition: nativeDefinition(provider.definition),
|
||||
})),
|
||||
userId: user.id,
|
||||
});
|
||||
return {
|
||||
leaseId: result.leaseId,
|
||||
expiresAt: new Date(result.expiresAtMs),
|
||||
};
|
||||
}
|
||||
|
||||
private async assertRead(userId: string, workspaceId: string) {
|
||||
await this.ac
|
||||
.user(userId)
|
||||
.workspace(workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Read');
|
||||
}
|
||||
|
||||
private async assertUpdate(userId: string, workspaceId: string) {
|
||||
await this.ac
|
||||
.user(userId)
|
||||
.workspace(workspaceId)
|
||||
.allowLocal()
|
||||
.assert('Workspace.Settings.Update');
|
||||
await this.entitlement.assertManagementAccess(workspaceId, userId);
|
||||
}
|
||||
}
|
||||
|
||||
function requireExplicitDescription(input: { description: string | null }) {
|
||||
if (!Object.hasOwn(input, 'description')) {
|
||||
throw new BadRequestException('description must be provided explicitly.');
|
||||
}
|
||||
}
|
||||
|
||||
function nativeDefinition(input: WorkspaceByokProfileDefinitionInput) {
|
||||
return {
|
||||
...input,
|
||||
endpoint: {
|
||||
...input.endpoint,
|
||||
url: input.endpoint.url ?? undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function projectProbe(probe: {
|
||||
kind: string;
|
||||
testedAtMs?: number;
|
||||
errorKind?: string;
|
||||
}) {
|
||||
return {
|
||||
kind: probe.kind,
|
||||
testedAt: probe.testedAtMs ? new Date(probe.testedAtMs) : null,
|
||||
errorKind: probe.errorKind ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function projectProbeResult(result: {
|
||||
definitionFingerprint: string;
|
||||
stale: boolean;
|
||||
connection: {
|
||||
kind: string;
|
||||
testedAtMs?: number;
|
||||
errorKind?: string;
|
||||
};
|
||||
models: Array<{
|
||||
modelId: string;
|
||||
checks: Array<{
|
||||
operation: string;
|
||||
status: {
|
||||
kind: string;
|
||||
testedAtMs?: number;
|
||||
errorKind?: string;
|
||||
};
|
||||
}>;
|
||||
}>;
|
||||
}) {
|
||||
return {
|
||||
...result,
|
||||
connection: projectProbe(result.connection),
|
||||
models: result.models.map(model => ({
|
||||
...model,
|
||||
checks: model.checks.map(check => ({
|
||||
...check,
|
||||
status: projectProbe(check.status),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function projectProfile(
|
||||
profile: Awaited<ReturnType<BackendRuntimeProvider['createByokProfile']>>
|
||||
) {
|
||||
return {
|
||||
...profile,
|
||||
provider: profile.provider as ByokProvider,
|
||||
validation: profile.validation
|
||||
? {
|
||||
...profile.validation,
|
||||
connection: projectProbe(profile.validation.connection),
|
||||
models: profile.validation.models.map(model => ({
|
||||
...model,
|
||||
checks: model.checks.map(check => ({
|
||||
...check,
|
||||
status: projectProbe(check.status),
|
||||
})),
|
||||
})),
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,833 +0,0 @@
|
||||
import { createHash, createHmac, randomUUID } from 'node:crypto';
|
||||
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
BadRequest,
|
||||
Cache,
|
||||
Config,
|
||||
CryptoHelper,
|
||||
metrics,
|
||||
safeFetch,
|
||||
} from '../../../base';
|
||||
import { Models } from '../../../models';
|
||||
import type { CopilotProviderProfile } from '../config';
|
||||
import { ByokEntitlementPolicy } from './policy';
|
||||
import { runProviderProbe } from './probe';
|
||||
import {
|
||||
BYOK_ALLOWED_PROVIDERS,
|
||||
type ByokFeatureKind,
|
||||
ByokKeyStorage,
|
||||
ByokKeyTestStatus,
|
||||
ByokProvider,
|
||||
ByokProviderSource,
|
||||
byokProviderToCopilotType,
|
||||
isByokProvider,
|
||||
} from './types';
|
||||
|
||||
const LOCAL_LEASE_TTL_MS = 10 * 60 * 1000;
|
||||
const BYOK_PROFILE_PRIORITY_BASE = 10_000;
|
||||
const SERVER_PROFILE_PRIORITY_OFFSET = 2_000;
|
||||
|
||||
export type ByokProviderRequestContext = {
|
||||
userId?: string;
|
||||
workspaceId?: string;
|
||||
byokLeaseId?: string;
|
||||
};
|
||||
|
||||
export type ByokProfileSourceFilter = {
|
||||
local?: boolean;
|
||||
server?: boolean;
|
||||
};
|
||||
|
||||
export type ByokKeyConfig = {
|
||||
id: string;
|
||||
provider: ByokProvider;
|
||||
name: string;
|
||||
description: string | null;
|
||||
storage: ByokKeyStorage;
|
||||
configured: boolean;
|
||||
enabled: boolean;
|
||||
endpoint: string | null;
|
||||
endpointEditable: boolean;
|
||||
sortOrder: number;
|
||||
capabilities: string[];
|
||||
testStatus: ByokKeyTestStatus;
|
||||
disabledReason: string | null;
|
||||
lastTestedAt: Date | null;
|
||||
lastTestError: string | null;
|
||||
lastUsedAt: Date | null;
|
||||
lastErrorAt: Date | null;
|
||||
lastError: string | null;
|
||||
};
|
||||
|
||||
export type ByokSettings = {
|
||||
workspaceId: string;
|
||||
entitled: boolean;
|
||||
serverEntitled: boolean;
|
||||
localEntitled: boolean;
|
||||
entitlementRequired: string[];
|
||||
keys: ByokKeyConfig[];
|
||||
allowedProviders: ByokProvider[];
|
||||
localStorageSupported: boolean;
|
||||
customEndpointSupported: boolean;
|
||||
privateEndpointSupported: boolean;
|
||||
hasAiPlan: boolean;
|
||||
warnings: Array<{
|
||||
featureKind: string;
|
||||
reason: string;
|
||||
requiredProviders: ByokProvider[];
|
||||
}>;
|
||||
};
|
||||
|
||||
export type ByokLocalLeaseProvider = {
|
||||
provider: ByokProvider;
|
||||
name: string;
|
||||
description?: string | null;
|
||||
apiKey: string;
|
||||
endpoint?: string | null;
|
||||
sortOrder?: number | null;
|
||||
enabled?: boolean | null;
|
||||
};
|
||||
|
||||
type LocalLeasePayload = {
|
||||
workspaceId: string;
|
||||
userId: string;
|
||||
providers: Array<
|
||||
Omit<ByokLocalLeaseProvider, 'apiKey'> & { encryptedApiKey: string }
|
||||
>;
|
||||
};
|
||||
|
||||
type LocalLeaseActive = {
|
||||
leaseId: string;
|
||||
expiresAt: string;
|
||||
};
|
||||
|
||||
type ByokProfileMeta = {
|
||||
source: ByokProviderSource.Server | ByokProviderSource.Local;
|
||||
keyId?: string;
|
||||
provider: ByokProvider;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class ByokService {
|
||||
private readonly probeFetch = safeFetch;
|
||||
|
||||
constructor(
|
||||
private readonly models: Models,
|
||||
private readonly crypto: CryptoHelper,
|
||||
private readonly cache: Cache,
|
||||
private readonly entitlement: ByokEntitlementPolicy,
|
||||
private readonly config: Config
|
||||
) {}
|
||||
|
||||
get customEndpointSupported() {
|
||||
return env.selfhosted && this.config.copilot.byok.allowCustomEndpoint;
|
||||
}
|
||||
|
||||
get privateEndpointSupported() {
|
||||
return (
|
||||
this.customEndpointSupported &&
|
||||
this.config.copilot.byok.allowPrivateEndpoint
|
||||
);
|
||||
}
|
||||
|
||||
async getSettings(
|
||||
workspaceId: string,
|
||||
userId?: string
|
||||
): Promise<ByokSettings> {
|
||||
if (!(await this.entitlement.hasManagementAccess(workspaceId, userId))) {
|
||||
return {
|
||||
workspaceId,
|
||||
entitled: false,
|
||||
serverEntitled: false,
|
||||
localEntitled: false,
|
||||
entitlementRequired: ['Workspace owner or admin'],
|
||||
keys: [],
|
||||
allowedProviders: [...BYOK_ALLOWED_PROVIDERS],
|
||||
localStorageSupported: false,
|
||||
customEndpointSupported: this.customEndpointSupported,
|
||||
privateEndpointSupported: this.privateEndpointSupported,
|
||||
hasAiPlan: await this.entitlement.hasAiPlan(userId),
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
const [serverEntitled, localEntitled] =
|
||||
await this.entitlement.hasEntitlement(workspaceId, userId);
|
||||
const entitled = serverEntitled || localEntitled;
|
||||
if (!entitled) {
|
||||
return {
|
||||
workspaceId,
|
||||
entitled: false,
|
||||
serverEntitled: false,
|
||||
localEntitled: false,
|
||||
entitlementRequired: ['Pro', 'Team', 'Believer'],
|
||||
keys: [],
|
||||
allowedProviders: [...BYOK_ALLOWED_PROVIDERS],
|
||||
localStorageSupported: false,
|
||||
customEndpointSupported: this.customEndpointSupported,
|
||||
privateEndpointSupported: this.privateEndpointSupported,
|
||||
hasAiPlan: await this.entitlement.hasAiPlan(userId),
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
const rows = serverEntitled
|
||||
? await this.models.copilotWorkspaceByokConfig.list(workspaceId)
|
||||
: [];
|
||||
const keys = rows.map(row => this.toKeyConfig(row));
|
||||
|
||||
return {
|
||||
workspaceId,
|
||||
entitled: true,
|
||||
serverEntitled,
|
||||
localEntitled,
|
||||
entitlementRequired: ['Pro', 'Team', 'Believer'],
|
||||
keys,
|
||||
allowedProviders: [...BYOK_ALLOWED_PROVIDERS],
|
||||
localStorageSupported: false,
|
||||
customEndpointSupported: this.customEndpointSupported,
|
||||
privateEndpointSupported: this.privateEndpointSupported,
|
||||
hasAiPlan: await this.entitlement.hasAiPlan(userId),
|
||||
warnings: this.buildWarnings(keys),
|
||||
};
|
||||
}
|
||||
|
||||
async upsertConfig(input: {
|
||||
id?: string | null;
|
||||
workspaceId: string;
|
||||
provider: ByokProvider;
|
||||
name: string;
|
||||
description?: string | null;
|
||||
storage: ByokKeyStorage;
|
||||
apiKey?: string | null;
|
||||
endpoint?: string | null;
|
||||
sortOrder?: number | null;
|
||||
enabled?: boolean | null;
|
||||
userId?: string;
|
||||
}): Promise<ByokKeyConfig> {
|
||||
await this.entitlement.assertManagementAccess(
|
||||
input.workspaceId,
|
||||
input.userId
|
||||
);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
this.assertProvider(input.provider);
|
||||
if (input.storage !== ByokKeyStorage.server) {
|
||||
throw new BadRequestException('Only server BYOK keys are persisted.');
|
||||
}
|
||||
const existing = input.id
|
||||
? await this.models.copilotWorkspaceByokConfig.get(input.id)
|
||||
: null;
|
||||
if (input.id && (!existing || existing.workspaceId !== input.workspaceId)) {
|
||||
throw new BadRequest('BYOK config not found.');
|
||||
}
|
||||
const encryptedApiKey = input.apiKey
|
||||
? this.crypto.encrypt(input.apiKey)
|
||||
: undefined;
|
||||
|
||||
if (!input.id && !encryptedApiKey) {
|
||||
throw new BadRequestException('apiKey is required.');
|
||||
}
|
||||
|
||||
const description =
|
||||
input.description !== undefined
|
||||
? input.description?.trim() || null
|
||||
: (existing?.description ?? null);
|
||||
const endpoint =
|
||||
input.endpoint !== undefined
|
||||
? this.normalizeEndpoint(input.endpoint)
|
||||
: (existing?.endpoint ?? null);
|
||||
const sortOrder = input.sortOrder ?? existing?.sortOrder ?? 0;
|
||||
const enabled = input.enabled ?? existing?.enabled ?? true;
|
||||
|
||||
const row = await this.models.copilotWorkspaceByokConfig.upsert({
|
||||
id: input.id,
|
||||
workspaceId: input.workspaceId,
|
||||
provider: input.provider,
|
||||
name: input.name.trim(),
|
||||
description,
|
||||
encryptedApiKey,
|
||||
endpoint,
|
||||
sortOrder,
|
||||
enabled,
|
||||
userId: input.userId,
|
||||
});
|
||||
|
||||
return this.toKeyConfig(row);
|
||||
}
|
||||
|
||||
async reorderConfigs(input: {
|
||||
workspaceId: string;
|
||||
storage: ByokKeyStorage;
|
||||
ids: string[];
|
||||
userId?: string;
|
||||
}) {
|
||||
await this.entitlement.assertManagementAccess(
|
||||
input.workspaceId,
|
||||
input.userId
|
||||
);
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
if (input.storage !== ByokKeyStorage.server) {
|
||||
throw new BadRequestException('Only server BYOK keys are persisted.');
|
||||
}
|
||||
await this.models.copilotWorkspaceByokConfig.reorder(
|
||||
input.workspaceId,
|
||||
input.ids,
|
||||
input.userId
|
||||
);
|
||||
return (await this.getSettings(input.workspaceId, input.userId)).keys;
|
||||
}
|
||||
|
||||
async deleteConfig(workspaceId: string, id: string, _userId?: string) {
|
||||
await this.entitlement.assertManagementAccess(workspaceId, _userId);
|
||||
await this.entitlement.assertServerEntitled(workspaceId);
|
||||
await this.models.copilotWorkspaceByokConfig.delete(workspaceId, id);
|
||||
return true;
|
||||
}
|
||||
|
||||
async clearConfigs(
|
||||
workspaceId: string,
|
||||
provider: ByokProvider | null | undefined,
|
||||
_userId?: string
|
||||
) {
|
||||
await this.entitlement.assertManagementAccess(workspaceId, _userId);
|
||||
await this.entitlement.assertServerEntitled(workspaceId);
|
||||
await this.models.copilotWorkspaceByokConfig.clear(workspaceId, provider);
|
||||
return true;
|
||||
}
|
||||
|
||||
async testConfig(input: {
|
||||
workspaceId: string;
|
||||
provider: ByokProvider;
|
||||
storage: ByokKeyStorage;
|
||||
apiKey?: string | null;
|
||||
endpoint?: string | null;
|
||||
configId?: string | null;
|
||||
userId?: string;
|
||||
}) {
|
||||
await this.entitlement.assertManagementAccess(
|
||||
input.workspaceId,
|
||||
input.userId
|
||||
);
|
||||
if (input.storage === ByokKeyStorage.server) {
|
||||
await this.entitlement.assertServerEntitled(input.workspaceId);
|
||||
} else {
|
||||
await this.entitlement.assertLocalEntitled(
|
||||
input.workspaceId,
|
||||
input.userId
|
||||
);
|
||||
}
|
||||
this.assertProvider(input.provider);
|
||||
let apiKey = input.apiKey;
|
||||
let endpoint = this.normalizeEndpoint(input.endpoint);
|
||||
if (!apiKey && input.configId && input.storage === ByokKeyStorage.server) {
|
||||
const config = await this.models.copilotWorkspaceByokConfig.get(
|
||||
input.configId
|
||||
);
|
||||
if (
|
||||
!config ||
|
||||
config.workspaceId !== input.workspaceId ||
|
||||
config.provider !== input.provider
|
||||
) {
|
||||
throw new BadRequestException('BYOK config not found.');
|
||||
}
|
||||
apiKey = this.crypto.decrypt(config.encryptedApiKey);
|
||||
endpoint =
|
||||
input.endpoint !== undefined
|
||||
? endpoint
|
||||
: this.normalizeEndpoint(config.endpoint);
|
||||
}
|
||||
if (!apiKey) {
|
||||
throw new BadRequestException('apiKey is required.');
|
||||
}
|
||||
|
||||
try {
|
||||
await runProviderProbe(
|
||||
this.probeFetch,
|
||||
input.provider,
|
||||
apiKey,
|
||||
endpoint,
|
||||
this.privateEndpointSupported
|
||||
);
|
||||
if (input.configId && input.storage === ByokKeyStorage.server) {
|
||||
await this.models.copilotWorkspaceByokConfig.markValidated(
|
||||
input.workspaceId,
|
||||
input.configId,
|
||||
input.userId
|
||||
);
|
||||
}
|
||||
metrics.ai.counter('byok_test_key').add(1, {
|
||||
workspace: input.workspaceId,
|
||||
provider: input.provider,
|
||||
storage: input.storage,
|
||||
result: 'passed',
|
||||
});
|
||||
return { ok: true, status: ByokKeyTestStatus.passed, message: null };
|
||||
} catch (error) {
|
||||
const message = this.sanitizeError(error);
|
||||
if (input.configId && input.storage === ByokKeyStorage.server) {
|
||||
await this.models.copilotWorkspaceByokConfig.markFailure(
|
||||
input.workspaceId,
|
||||
input.configId,
|
||||
message
|
||||
);
|
||||
}
|
||||
metrics.ai.counter('byok_test_key').add(1, {
|
||||
workspace: input.workspaceId,
|
||||
provider: input.provider,
|
||||
storage: input.storage,
|
||||
result: 'failed',
|
||||
});
|
||||
return { ok: false, status: ByokKeyTestStatus.failed, message };
|
||||
}
|
||||
}
|
||||
|
||||
async createLocalLease(input: {
|
||||
workspaceId: string;
|
||||
providers: ByokLocalLeaseProvider[];
|
||||
userId: string;
|
||||
}) {
|
||||
await this.entitlement.assertManagementAccess(
|
||||
input.workspaceId,
|
||||
input.userId
|
||||
);
|
||||
await this.entitlement.assertLocalEntitled(input.workspaceId, input.userId);
|
||||
const providers = input.providers.map(provider => {
|
||||
this.assertProvider(provider.provider);
|
||||
const endpoint = this.normalizeEndpoint(provider.endpoint);
|
||||
return { ...provider, endpoint };
|
||||
});
|
||||
const activeCacheKey = this.localLeaseActiveCacheKey({
|
||||
...input,
|
||||
providers,
|
||||
});
|
||||
const activeLease = await this.getActiveLocalLease(activeCacheKey);
|
||||
if (activeLease) return activeLease;
|
||||
|
||||
const leaseId = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + LOCAL_LEASE_TTL_MS);
|
||||
const payload: LocalLeasePayload = {
|
||||
workspaceId: input.workspaceId,
|
||||
userId: input.userId,
|
||||
providers: providers.map(provider => ({
|
||||
provider: provider.provider,
|
||||
name: provider.name,
|
||||
description: provider.description,
|
||||
encryptedApiKey: this.crypto.encrypt(provider.apiKey),
|
||||
endpoint: provider.endpoint,
|
||||
sortOrder: provider.sortOrder,
|
||||
enabled: provider.enabled,
|
||||
})),
|
||||
};
|
||||
await this.cache.set(this.leaseCacheKey(leaseId), payload, {
|
||||
ttl: LOCAL_LEASE_TTL_MS,
|
||||
});
|
||||
const registered = await this.cache.setnx<LocalLeaseActive>(
|
||||
activeCacheKey,
|
||||
{ leaseId, expiresAt: expiresAt.toISOString() },
|
||||
{ ttl: LOCAL_LEASE_TTL_MS }
|
||||
);
|
||||
if (!registered) {
|
||||
const current = await this.getActiveLocalLease(activeCacheKey);
|
||||
if (current) {
|
||||
await this.cache.delete(this.leaseCacheKey(leaseId));
|
||||
return current;
|
||||
}
|
||||
}
|
||||
return { leaseId, expiresAt };
|
||||
}
|
||||
|
||||
async getProfiles(
|
||||
context: ByokProviderRequestContext = {},
|
||||
sources: ByokProfileSourceFilter = { local: true, server: true }
|
||||
): Promise<CopilotProviderProfile[]> {
|
||||
if (!context.workspaceId) {
|
||||
return [];
|
||||
}
|
||||
const [localEntitled, serverEntitled] = await Promise.all([
|
||||
this.entitlement.hasLocalEntitlement(context.workspaceId, context.userId),
|
||||
this.entitlement.hasServerEntitlement(context.workspaceId),
|
||||
]);
|
||||
const [localProfiles, serverProfiles] = await Promise.all([
|
||||
sources.local && localEntitled
|
||||
? this.getLocalProfiles(context)
|
||||
: Promise.resolve([]),
|
||||
sources.server && serverEntitled
|
||||
? this.getServerProfiles(context.workspaceId)
|
||||
: Promise.resolve([]),
|
||||
]);
|
||||
|
||||
return [...localProfiles, ...serverProfiles];
|
||||
}
|
||||
|
||||
async recordUsage(input: {
|
||||
workspaceId?: string;
|
||||
userId?: string;
|
||||
providerId?: string;
|
||||
model?: string | null;
|
||||
featureKind: ByokFeatureKind;
|
||||
sessionId?: string;
|
||||
taskId?: string;
|
||||
actionId?: string;
|
||||
billingUnitId?: string;
|
||||
usage?: {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
total_tokens?: number;
|
||||
cached_tokens?: number;
|
||||
};
|
||||
}) {
|
||||
if (!input.workspaceId || !input.providerId) return;
|
||||
const meta = this.parseProfileMeta(input.providerId, input.workspaceId);
|
||||
if (!meta) return;
|
||||
|
||||
metrics.ai.counter('byok_usage').add(1, {
|
||||
workspace: input.workspaceId,
|
||||
provider: meta.provider,
|
||||
source: meta.source,
|
||||
feature: input.featureKind,
|
||||
});
|
||||
await this.models.copilotUsage.create({
|
||||
workspaceId: input.workspaceId,
|
||||
userId: input.userId,
|
||||
provider: meta.provider,
|
||||
providerSource: meta.source,
|
||||
featureKind: input.featureKind,
|
||||
model: input.model ?? null,
|
||||
sessionId: input.sessionId,
|
||||
taskId: input.taskId,
|
||||
actionId: input.actionId,
|
||||
billingUnitId: input.billingUnitId,
|
||||
promptTokens: input.usage?.prompt_tokens ?? 0,
|
||||
completionTokens: input.usage?.completion_tokens ?? 0,
|
||||
totalTokens: input.usage?.total_tokens ?? 0,
|
||||
cachedTokens: input.usage?.cached_tokens ?? 0,
|
||||
});
|
||||
if (meta.source === ByokProviderSource.Server && meta.keyId) {
|
||||
await this.models.copilotWorkspaceByokConfig.touchUsed(
|
||||
input.workspaceId,
|
||||
meta.keyId
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async recordProviderFailure(input: {
|
||||
workspaceId?: string;
|
||||
providerId?: string;
|
||||
featureKind: ByokFeatureKind;
|
||||
error: unknown;
|
||||
}) {
|
||||
if (!input.workspaceId || !input.providerId) return;
|
||||
const meta = this.parseProfileMeta(input.providerId, input.workspaceId);
|
||||
if (!meta) return;
|
||||
|
||||
const message = this.sanitizeError(input.error);
|
||||
metrics.ai.counter('byok_route_failure').add(1, {
|
||||
workspace: input.workspaceId,
|
||||
provider: meta.provider,
|
||||
source: meta.source,
|
||||
feature: input.featureKind,
|
||||
});
|
||||
if (meta.source === ByokProviderSource.Server && meta.keyId) {
|
||||
await this.models.copilotWorkspaceByokConfig.markFailure(
|
||||
input.workspaceId,
|
||||
meta.keyId,
|
||||
message
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async getUsage(workspaceId: string, from: Date, to: Date) {
|
||||
return await this.models.copilotUsage.aggregateByDay({
|
||||
workspaceId,
|
||||
from,
|
||||
to,
|
||||
providerSources: [ByokProviderSource.Server, ByokProviderSource.Local],
|
||||
});
|
||||
}
|
||||
|
||||
private async getServerProfiles(workspaceId: string) {
|
||||
const rows =
|
||||
await this.models.copilotWorkspaceByokConfig.listEnabled(workspaceId);
|
||||
|
||||
return rows
|
||||
.filter(row => isByokProvider(row.provider))
|
||||
.map((row, index): CopilotProviderProfile => {
|
||||
const provider = row.provider as ByokProvider;
|
||||
return {
|
||||
id: this.profileId(workspaceId, provider, row.id, 'server'),
|
||||
type: byokProviderToCopilotType(provider),
|
||||
priority:
|
||||
BYOK_PROFILE_PRIORITY_BASE - SERVER_PROFILE_PRIORITY_OFFSET - index,
|
||||
config: this.providerConfig(
|
||||
provider,
|
||||
row.encryptedApiKey,
|
||||
row.endpoint
|
||||
),
|
||||
} as CopilotProviderProfile;
|
||||
});
|
||||
}
|
||||
|
||||
private async getLocalProfiles(context: ByokProviderRequestContext) {
|
||||
if (!context.byokLeaseId || !context.workspaceId || !context.userId) {
|
||||
return [];
|
||||
}
|
||||
if (
|
||||
!(await this.entitlement.hasManagementAccess(
|
||||
context.workspaceId,
|
||||
context.userId
|
||||
))
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
const lease = await this.cache.get<LocalLeasePayload>(
|
||||
this.leaseCacheKey(context.byokLeaseId)
|
||||
);
|
||||
if (
|
||||
!lease ||
|
||||
lease.workspaceId !== context.workspaceId ||
|
||||
lease.userId !== context.userId
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
return lease.providers
|
||||
.filter(provider => provider.enabled !== false)
|
||||
.map((provider, index): CopilotProviderProfile => {
|
||||
return {
|
||||
id: this.profileId(
|
||||
context.workspaceId ?? lease.workspaceId,
|
||||
provider.provider,
|
||||
`${index}`,
|
||||
'local'
|
||||
),
|
||||
type: byokProviderToCopilotType(provider.provider),
|
||||
priority: BYOK_PROFILE_PRIORITY_BASE - index,
|
||||
config: this.providerConfig(
|
||||
provider.provider,
|
||||
provider.encryptedApiKey,
|
||||
provider.endpoint ?? null
|
||||
),
|
||||
} as CopilotProviderProfile;
|
||||
});
|
||||
}
|
||||
|
||||
private providerConfig(
|
||||
provider: ByokProvider,
|
||||
encryptedApiKey: string,
|
||||
endpoint: string | null
|
||||
) {
|
||||
const apiKey = this.crypto.decrypt(encryptedApiKey);
|
||||
switch (provider) {
|
||||
case ByokProvider.openai:
|
||||
case ByokProvider.gemini:
|
||||
case ByokProvider.anthropic:
|
||||
return { apiKey, ...(endpoint ? { baseURL: endpoint } : {}) };
|
||||
case ByokProvider.fal:
|
||||
return { apiKey };
|
||||
}
|
||||
}
|
||||
|
||||
private profileId(
|
||||
workspaceId: string,
|
||||
provider: ByokProvider,
|
||||
keyId: string,
|
||||
storage: 'server' | 'local'
|
||||
) {
|
||||
const hash = this.workspaceHash(workspaceId);
|
||||
const sanitizedKeyId = keyId.replaceAll(/[^a-zA-Z0-9-_]/g, '');
|
||||
return storage === 'local'
|
||||
? `byok-${hash}-${provider}-local-${sanitizedKeyId}`
|
||||
: `byok-${hash}-${provider}-${sanitizedKeyId}`;
|
||||
}
|
||||
|
||||
parseProfileMeta(
|
||||
providerId: string,
|
||||
workspaceId?: string
|
||||
): ByokProfileMeta | null {
|
||||
const match =
|
||||
/^byok-([a-f0-9]{12})-(openai|anthropic|gemini|fal)-(.+)$/.exec(
|
||||
providerId
|
||||
);
|
||||
if (!match) return null;
|
||||
if (workspaceId && match[1] !== this.workspaceHash(workspaceId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const keyId = match[3];
|
||||
return {
|
||||
provider: match[2] as ByokProvider,
|
||||
source: keyId.startsWith('local-')
|
||||
? ByokProviderSource.Local
|
||||
: ByokProviderSource.Server,
|
||||
keyId: keyId.startsWith('local-') ? undefined : keyId,
|
||||
};
|
||||
}
|
||||
|
||||
private toKeyConfig(row: {
|
||||
id: string;
|
||||
provider: string;
|
||||
name: string;
|
||||
description: string | null;
|
||||
endpoint: string | null;
|
||||
sortOrder: number;
|
||||
enabled: boolean;
|
||||
disabledReason: string | null;
|
||||
lastValidatedAt: Date | null;
|
||||
lastValidationError: string | null;
|
||||
lastUsedAt: Date | null;
|
||||
lastErrorAt: Date | null;
|
||||
lastError: string | null;
|
||||
}): ByokKeyConfig {
|
||||
const provider = row.provider as ByokProvider;
|
||||
return {
|
||||
id: row.id,
|
||||
provider,
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
storage: ByokKeyStorage.server,
|
||||
configured: true,
|
||||
enabled: row.enabled,
|
||||
endpoint: row.endpoint,
|
||||
endpointEditable: this.customEndpointSupported,
|
||||
sortOrder: row.sortOrder,
|
||||
capabilities: this.capabilities(provider, 'server'),
|
||||
testStatus: row.lastValidationError
|
||||
? ByokKeyTestStatus.failed
|
||||
: row.lastValidatedAt
|
||||
? ByokKeyTestStatus.passed
|
||||
: ByokKeyTestStatus.untested,
|
||||
disabledReason: row.disabledReason,
|
||||
lastTestedAt: row.lastValidatedAt,
|
||||
lastTestError: row.lastValidationError,
|
||||
lastUsedAt: row.lastUsedAt,
|
||||
lastErrorAt: row.lastErrorAt,
|
||||
lastError: row.lastError,
|
||||
};
|
||||
}
|
||||
|
||||
private capabilities(provider: ByokProvider, storage: 'server' | 'local') {
|
||||
switch (provider) {
|
||||
case ByokProvider.openai:
|
||||
return ['Text', 'Image input', 'Actions', 'Image generate'];
|
||||
case ByokProvider.anthropic:
|
||||
return ['Text', 'Image input'];
|
||||
case ByokProvider.gemini:
|
||||
return storage === 'server'
|
||||
? [
|
||||
'Text',
|
||||
'Image input',
|
||||
'Actions',
|
||||
'Image generate',
|
||||
'Transcript',
|
||||
'Indexing',
|
||||
]
|
||||
: ['Text', 'Image input', 'Actions', 'Image generate'];
|
||||
case ByokProvider.fal:
|
||||
return ['Image generate'];
|
||||
}
|
||||
}
|
||||
|
||||
private buildWarnings(keys: ByokKeyConfig[]) {
|
||||
const activeServerGemini = keys.some(
|
||||
key =>
|
||||
key.provider === ByokProvider.gemini &&
|
||||
key.storage === ByokKeyStorage.server &&
|
||||
key.enabled
|
||||
);
|
||||
if (activeServerGemini) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
featureKind: 'transcript',
|
||||
reason:
|
||||
'Transcript and workspace indexing require a server Gemini BYOK key or AFFiNE AI plan fallback.',
|
||||
requiredProviders: [ByokProvider.gemini],
|
||||
},
|
||||
{
|
||||
featureKind: 'workspace_indexing',
|
||||
reason:
|
||||
'Workspace indexing requires a server Gemini BYOK key or AFFiNE AI plan fallback.',
|
||||
requiredProviders: [ByokProvider.gemini],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
private normalizeEndpoint(endpoint?: string | null) {
|
||||
if (!endpoint) return null;
|
||||
if (!this.customEndpointSupported) {
|
||||
throw new BadRequestException('Custom BYOK endpoint is not supported.');
|
||||
}
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(endpoint);
|
||||
} catch {
|
||||
throw new BadRequestException('Invalid BYOK endpoint.');
|
||||
}
|
||||
if (!['https:', 'http:'].includes(parsed.protocol)) {
|
||||
throw new BadRequestException('BYOK endpoint must use HTTP or HTTPS.');
|
||||
}
|
||||
return parsed.toString().replace(/\/$/, '');
|
||||
}
|
||||
|
||||
private assertProvider(provider: ByokProvider) {
|
||||
if (!BYOK_ALLOWED_PROVIDERS.includes(provider)) {
|
||||
throw new BadRequestException('Unsupported BYOK provider.');
|
||||
}
|
||||
}
|
||||
|
||||
private sanitizeError(error: unknown) {
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
return 'Provider key test timed out.';
|
||||
}
|
||||
if (error instanceof BadRequestException && error.message) {
|
||||
return error.message.slice(0, 300);
|
||||
}
|
||||
return 'Provider request failed.';
|
||||
}
|
||||
|
||||
private workspaceHash(workspaceId: string) {
|
||||
return createHash('sha256').update(workspaceId).digest('hex').slice(0, 12);
|
||||
}
|
||||
|
||||
private leaseCacheKey(leaseId: string) {
|
||||
return `copilot:byok:lease:${leaseId}`;
|
||||
}
|
||||
|
||||
private async getActiveLocalLease(activeCacheKey: string) {
|
||||
const active = await this.cache.get<LocalLeaseActive>(activeCacheKey);
|
||||
if (!active) return null;
|
||||
if (await this.cache.has(this.leaseCacheKey(active.leaseId))) {
|
||||
return { leaseId: active.leaseId, expiresAt: new Date(active.expiresAt) };
|
||||
}
|
||||
await this.cache.delete(activeCacheKey);
|
||||
return null;
|
||||
}
|
||||
|
||||
private localLeaseActiveCacheKey(input: {
|
||||
workspaceId: string;
|
||||
userId: string;
|
||||
providers: ByokLocalLeaseProvider[];
|
||||
}) {
|
||||
const fingerprint = createHmac(
|
||||
'sha256',
|
||||
this.crypto.keyPair.sha256.privateKey
|
||||
)
|
||||
.update(
|
||||
JSON.stringify(
|
||||
input.providers.map(provider => ({
|
||||
provider: provider.provider,
|
||||
name: provider.name,
|
||||
description: provider.description ?? null,
|
||||
apiKey: provider.apiKey,
|
||||
endpoint: provider.endpoint ?? null,
|
||||
sortOrder: provider.sortOrder ?? 0,
|
||||
enabled: provider.enabled ?? true,
|
||||
}))
|
||||
)
|
||||
)
|
||||
.digest('hex');
|
||||
return `copilot:byok:lease:active:${input.workspaceId}:${input.userId}:${fingerprint}`;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { AiPromptRole } from '@prisma/client';
|
||||
import { AiSessionMessageRole } from '@prisma/client';
|
||||
|
||||
import type { Conversation, Turn } from '../core';
|
||||
import { chatMessageFromTurn } from '../core';
|
||||
@@ -16,7 +16,6 @@ export type CanonicalConversationHistory = {
|
||||
conversation: Conversation;
|
||||
turns: Turn[];
|
||||
prompt: ResolvedPrompt;
|
||||
tokenCost: number;
|
||||
};
|
||||
|
||||
export type CanonicalConversationMeta = Omit<
|
||||
@@ -35,7 +34,7 @@ export class CompatHistoryProjector {
|
||||
private projectSessionBase(
|
||||
history: CanonicalConversationMeta
|
||||
): Omit<ChatHistory, 'messages'> {
|
||||
const { conversation, prompt, tokenCost } = history;
|
||||
const { conversation, prompt } = history;
|
||||
return {
|
||||
userId: conversation.userId,
|
||||
sessionId: conversation.id,
|
||||
@@ -45,10 +44,7 @@ export class CompatHistoryProjector {
|
||||
pinned: conversation.pinned,
|
||||
title: conversation.title,
|
||||
action: prompt.action || null,
|
||||
model: prompt.model,
|
||||
optionalModels: prompt.optionalModels || [],
|
||||
promptName: prompt.name,
|
||||
tokens: tokenCost,
|
||||
createdAt: conversation.createdAt,
|
||||
updatedAt: conversation.updatedAt,
|
||||
};
|
||||
@@ -84,7 +80,7 @@ export class CompatHistoryProjector {
|
||||
.concat(messages)
|
||||
.filter(
|
||||
message =>
|
||||
message.role !== AiPromptRole.user ||
|
||||
message.role !== AiSessionMessageRole.user ||
|
||||
!!message.content.trim() ||
|
||||
!!message.attachments?.length
|
||||
)
|
||||
|
||||
+4
-2
@@ -1,5 +1,5 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { AiPromptRole } from '@prisma/client';
|
||||
import { AiSessionMessageRole } from '@prisma/client';
|
||||
|
||||
import { PromptService } from '../prompt/service';
|
||||
import type { ChatMessage } from '../types';
|
||||
@@ -24,7 +24,9 @@ export class HistoryPromptPreloadProjector {
|
||||
history.turns[0] ? history.turns[0].metadata : {},
|
||||
history.conversation.id
|
||||
)
|
||||
.filter(({ role }) => role !== AiPromptRole.system) as ChatMessage[];
|
||||
.filter(
|
||||
({ role }) => role !== AiSessionMessageRole.system
|
||||
) as ChatMessage[];
|
||||
|
||||
preload.forEach((message, index) => {
|
||||
message.createdAt = new Date(
|
||||
|
||||
@@ -5,32 +5,9 @@ import {
|
||||
StorageJSONSchema,
|
||||
StorageProviderConfig,
|
||||
} from '../../base';
|
||||
import {
|
||||
AnthropicOfficialConfig,
|
||||
AnthropicVertexConfig,
|
||||
} from './providers/anthropic';
|
||||
import { CloudflareWorkersAIConfig } from './providers/cloudflare';
|
||||
import type { FalConfig } from './providers/fal';
|
||||
import { GeminiGenerativeConfig, GeminiVertexConfig } from './providers/gemini';
|
||||
import { OpenAIConfig } from './providers/openai';
|
||||
import {
|
||||
CopilotProviderType,
|
||||
ModelOutputType,
|
||||
VertexSchema,
|
||||
} from './providers/types';
|
||||
import { CopilotProviderType } from './providers/types';
|
||||
|
||||
export type CopilotProviderConfigMap = {
|
||||
[CopilotProviderType.OpenAI]: OpenAIConfig;
|
||||
[CopilotProviderType.CloudflareWorkersAi]: CloudflareWorkersAIConfig;
|
||||
[CopilotProviderType.FAL]: FalConfig;
|
||||
[CopilotProviderType.Gemini]: GeminiGenerativeConfig;
|
||||
[CopilotProviderType.GeminiVertex]: GeminiVertexConfig;
|
||||
[CopilotProviderType.Anthropic]: AnthropicOfficialConfig;
|
||||
[CopilotProviderType.AnthropicVertex]: AnthropicVertexConfig;
|
||||
};
|
||||
|
||||
export type ProviderSpecificConfig =
|
||||
CopilotProviderConfigMap[keyof CopilotProviderConfigMap];
|
||||
export type ProviderSpecificConfig = Record<string, unknown>;
|
||||
|
||||
export const RustRequestMiddlewareValues = [
|
||||
'normalize_messages',
|
||||
@@ -65,24 +42,13 @@ type CopilotProviderProfileCommon = {
|
||||
displayName?: string;
|
||||
priority?: number;
|
||||
enabled?: boolean;
|
||||
models?: string[];
|
||||
models: string[];
|
||||
middleware?: ProviderMiddlewareConfig;
|
||||
};
|
||||
|
||||
type CopilotProviderProfileVariant<T extends CopilotProviderType> = {
|
||||
type: T;
|
||||
config: CopilotProviderConfigMap[T];
|
||||
};
|
||||
|
||||
export type CopilotProviderProfile = CopilotProviderProfileCommon &
|
||||
{
|
||||
[Type in CopilotProviderType]: CopilotProviderProfileVariant<Type>;
|
||||
}[CopilotProviderType];
|
||||
|
||||
export type CopilotProviderDefaults = Partial<
|
||||
Record<Exclude<ModelOutputType, typeof ModelOutputType.Rerank>, string>
|
||||
> & {
|
||||
fallback?: string;
|
||||
export type CopilotProviderProfile = CopilotProviderProfileCommon & {
|
||||
type: CopilotProviderType;
|
||||
config: ProviderSpecificConfig;
|
||||
};
|
||||
|
||||
const CopilotProviderProfileBaseShape = z.object({
|
||||
@@ -90,7 +56,7 @@ const CopilotProviderProfileBaseShape = z.object({
|
||||
displayName: z.string().optional(),
|
||||
priority: z.number().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
models: z.array(z.string()).optional(),
|
||||
models: z.array(z.string().min(1)).min(1),
|
||||
middleware: z
|
||||
.object({
|
||||
rust: z
|
||||
@@ -106,79 +72,9 @@ const CopilotProviderProfileBaseShape = z.object({
|
||||
.optional(),
|
||||
});
|
||||
|
||||
const OpenAIConfigShape = z.object({
|
||||
apiKey: z.string(),
|
||||
baseURL: z.string().optional(),
|
||||
oldApiStyle: z.boolean().optional(),
|
||||
});
|
||||
|
||||
const FalConfigShape = z.object({
|
||||
apiKey: z.string(),
|
||||
});
|
||||
|
||||
const CloudflareWorkersAIConfigShape = z.object({
|
||||
apiToken: z.string(),
|
||||
accountId: z.string().optional(),
|
||||
baseURL: z.string().optional(),
|
||||
});
|
||||
|
||||
const GeminiGenerativeConfigShape = z.object({
|
||||
apiKey: z.string(),
|
||||
baseURL: z.string().optional(),
|
||||
});
|
||||
|
||||
const VertexProviderConfigShape = z.object({
|
||||
location: z.string().optional(),
|
||||
project: z.string().optional(),
|
||||
baseURL: z.string().optional(),
|
||||
googleAuthOptions: z.any().optional(),
|
||||
fetch: z.any().optional(),
|
||||
});
|
||||
|
||||
const AnthropicOfficialConfigShape = z.object({
|
||||
apiKey: z.string(),
|
||||
baseURL: z.string().optional(),
|
||||
});
|
||||
|
||||
const CopilotProviderProfileShape = z.discriminatedUnion('type', [
|
||||
CopilotProviderProfileBaseShape.extend({
|
||||
type: z.literal(CopilotProviderType.OpenAI),
|
||||
config: OpenAIConfigShape,
|
||||
}),
|
||||
CopilotProviderProfileBaseShape.extend({
|
||||
type: z.literal(CopilotProviderType.FAL),
|
||||
config: FalConfigShape,
|
||||
}),
|
||||
CopilotProviderProfileBaseShape.extend({
|
||||
type: z.literal(CopilotProviderType.CloudflareWorkersAi),
|
||||
config: CloudflareWorkersAIConfigShape,
|
||||
}),
|
||||
CopilotProviderProfileBaseShape.extend({
|
||||
type: z.literal(CopilotProviderType.Gemini),
|
||||
config: GeminiGenerativeConfigShape,
|
||||
}),
|
||||
CopilotProviderProfileBaseShape.extend({
|
||||
type: z.literal(CopilotProviderType.GeminiVertex),
|
||||
config: VertexProviderConfigShape,
|
||||
}),
|
||||
CopilotProviderProfileBaseShape.extend({
|
||||
type: z.literal(CopilotProviderType.Anthropic),
|
||||
config: AnthropicOfficialConfigShape,
|
||||
}),
|
||||
CopilotProviderProfileBaseShape.extend({
|
||||
type: z.literal(CopilotProviderType.AnthropicVertex),
|
||||
config: VertexProviderConfigShape,
|
||||
}),
|
||||
]);
|
||||
|
||||
const CopilotProviderDefaultsShape = z.object({
|
||||
[ModelOutputType.Text]: z.string().optional(),
|
||||
[ModelOutputType.Object]: z.string().optional(),
|
||||
[ModelOutputType.Embedding]: z.string().optional(),
|
||||
[ModelOutputType.Image]: z.string().optional(),
|
||||
[ModelOutputType.Rerank]: z.string().optional(),
|
||||
[ModelOutputType.Structured]: z.string().optional(),
|
||||
fallback: z.string().optional(),
|
||||
const CopilotProviderProfileShape = CopilotProviderProfileBaseShape.extend({
|
||||
type: z.nativeEnum(CopilotProviderType),
|
||||
config: z.record(z.string(), z.unknown()),
|
||||
});
|
||||
|
||||
declare global {
|
||||
@@ -202,14 +98,6 @@ declare global {
|
||||
storage: ConfigItem<StorageProviderConfig>;
|
||||
providers: {
|
||||
profiles: ConfigItem<CopilotProviderProfile[]>;
|
||||
defaults: ConfigItem<CopilotProviderDefaults>;
|
||||
openai: ConfigItem<OpenAIConfig>;
|
||||
cloudflareWorkersAi: ConfigItem<CloudflareWorkersAIConfig>;
|
||||
fal: ConfigItem<FalConfig>;
|
||||
gemini: ConfigItem<GeminiGenerativeConfig>;
|
||||
geminiVertex: ConfigItem<GeminiVertexConfig>;
|
||||
anthropic: ConfigItem<AnthropicOfficialConfig>;
|
||||
anthropicVertex: ConfigItem<AnthropicVertexConfig>;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -245,56 +133,6 @@ defineModuleConfig('copilot', {
|
||||
default: [],
|
||||
shape: z.array(CopilotProviderProfileShape),
|
||||
},
|
||||
'providers.defaults': {
|
||||
desc: 'The default provider ids for model output types and global fallback.',
|
||||
default: {},
|
||||
shape: CopilotProviderDefaultsShape,
|
||||
},
|
||||
'providers.openai': {
|
||||
desc: 'The config for the openai provider.',
|
||||
default: {
|
||||
apiKey: '',
|
||||
baseURL: 'https://api.openai.com/v1',
|
||||
},
|
||||
link: 'https://github.com/openai/openai-node',
|
||||
},
|
||||
'providers.cloudflareWorkersAi': {
|
||||
desc: 'The config for the Cloudflare Workers AI provider.',
|
||||
default: {
|
||||
apiToken: '',
|
||||
accountId: '',
|
||||
},
|
||||
},
|
||||
'providers.fal': {
|
||||
desc: 'The config for the fal provider.',
|
||||
default: {
|
||||
apiKey: '',
|
||||
},
|
||||
},
|
||||
'providers.gemini': {
|
||||
desc: 'The config for the gemini provider.',
|
||||
default: {
|
||||
apiKey: '',
|
||||
baseURL: 'https://generativelanguage.googleapis.com/v1beta',
|
||||
},
|
||||
},
|
||||
'providers.geminiVertex': {
|
||||
desc: 'The config for the gemini provider in Google Vertex AI.',
|
||||
default: {},
|
||||
schema: VertexSchema,
|
||||
},
|
||||
'providers.anthropic': {
|
||||
desc: 'The config for the anthropic provider.',
|
||||
default: {
|
||||
apiKey: '',
|
||||
baseURL: 'https://api.anthropic.com/v1',
|
||||
},
|
||||
},
|
||||
'providers.anthropicVertex': {
|
||||
desc: 'The config for the anthropic provider in Google Vertex AI.',
|
||||
default: {},
|
||||
schema: VertexSchema,
|
||||
},
|
||||
unsplash: {
|
||||
desc: 'The config for the unsplash key.',
|
||||
default: {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { OnEvent } from '../../../base';
|
||||
import { Config } from '../../../base/config';
|
||||
import { OnEvent } from '../../../base/event';
|
||||
import { PermissionAccess } from '../../../core/permission';
|
||||
import {
|
||||
RealtimePublisher,
|
||||
@@ -10,6 +11,7 @@ import {
|
||||
registerRealtimeLiveQuery,
|
||||
} from '../../../core/realtime';
|
||||
import { Models } from '../../../models';
|
||||
import { assertCopilotEnabled } from '../availability';
|
||||
|
||||
export function workspaceEmbeddingRoom(workspaceId: string) {
|
||||
return realtimeWorkspaceEmbeddingProgressRoom(workspaceId);
|
||||
@@ -21,7 +23,8 @@ export class CopilotEmbeddingRealtimeProvider implements OnModuleInit {
|
||||
private readonly ac: PermissionAccess,
|
||||
private readonly models: Models,
|
||||
private readonly registry: RealtimeRegistry,
|
||||
private readonly publisher: RealtimePublisher
|
||||
private readonly publisher: RealtimePublisher,
|
||||
private readonly config: Config
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
@@ -118,6 +121,7 @@ export class CopilotEmbeddingRealtimeProvider implements OnModuleInit {
|
||||
}
|
||||
|
||||
private async assertCopilot(userId: string, workspaceId: string) {
|
||||
assertCopilotEnabled(this.config);
|
||||
await this.ac
|
||||
.user(userId)
|
||||
.workspace(workspaceId)
|
||||
|
||||
@@ -50,6 +50,7 @@ import {
|
||||
Models,
|
||||
} from '../../../models';
|
||||
import { CopilotEmbeddingJob } from '../embedding/job';
|
||||
import { CopilotEnabled } from '../feature';
|
||||
import { COPILOT_LOCKER, CopilotType } from '../resolver';
|
||||
import { ChatSessionService } from '../session';
|
||||
import { CopilotStorage } from '../storage';
|
||||
@@ -286,6 +287,7 @@ class ContextMatchedDocChunk implements DocChunkSimilarity {
|
||||
}
|
||||
|
||||
@Throttle()
|
||||
@CopilotEnabled()
|
||||
@Resolver(() => CopilotType)
|
||||
export class CopilotContextRootResolver {
|
||||
constructor(
|
||||
@@ -435,6 +437,7 @@ export class CopilotContextRootResolver {
|
||||
}
|
||||
|
||||
@Throttle()
|
||||
@CopilotEnabled()
|
||||
@Resolver(() => CopilotContextType)
|
||||
export class CopilotContextResolver {
|
||||
constructor(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
/* oxlint-disable import/no-cycle -- Context embedding reuses the shared capability runtime. */
|
||||
import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
UnsplashIsNotConfigured,
|
||||
} from '../../base';
|
||||
import { CurrentUser, Public } from '../../core/auth';
|
||||
import { CopilotEnabled } from './feature';
|
||||
import {
|
||||
ActionStreamHost,
|
||||
projectActionEventToChatEvent,
|
||||
@@ -52,6 +53,7 @@ export interface ChatEvent {
|
||||
|
||||
const PING_INTERVAL = 5000;
|
||||
|
||||
@CopilotEnabled()
|
||||
@Controller('/api/copilot')
|
||||
export class CopilotController implements BeforeApplicationShutdown {
|
||||
private readonly logger = new Logger(CopilotController.name);
|
||||
|
||||
@@ -83,7 +83,6 @@ export class ConversationStore {
|
||||
conversation: Conversation;
|
||||
turns: Turn[];
|
||||
promptName: string;
|
||||
tokenCost: number;
|
||||
}
|
||||
| undefined
|
||||
> {
|
||||
@@ -96,7 +95,6 @@ export class ConversationStore {
|
||||
conversation: this.toConversation(session),
|
||||
turns: this.toTurns(session),
|
||||
promptName: session.promptName,
|
||||
tokenCost: session.tokenCost,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -104,7 +102,6 @@ export class ConversationStore {
|
||||
| {
|
||||
conversation: Conversation;
|
||||
promptName: string;
|
||||
tokenCost: number;
|
||||
}
|
||||
| undefined
|
||||
> {
|
||||
@@ -124,7 +121,6 @@ export class ConversationStore {
|
||||
updatedAt: session.updatedAt,
|
||||
},
|
||||
promptName: session.promptName,
|
||||
tokenCost: session.tokenCost,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -146,7 +142,6 @@ export class ConversationStore {
|
||||
turnFromChatMessage(message, session.id)
|
||||
),
|
||||
promptName: session.promptName,
|
||||
tokenCost: session.tokenCost,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -168,14 +163,12 @@ export class ConversationStore {
|
||||
updatedAt: session.updatedAt,
|
||||
} satisfies Conversation,
|
||||
promptName: session.promptName,
|
||||
tokenCost: session.tokenCost,
|
||||
}));
|
||||
}
|
||||
|
||||
async appendTurns(input: {
|
||||
sessionId: string;
|
||||
userId: string;
|
||||
prompt: { model: string };
|
||||
turns: Turn[];
|
||||
}) {
|
||||
return await this.models.copilotSession.updateMessages({
|
||||
@@ -190,14 +183,12 @@ export class ConversationStore {
|
||||
async appendTurn(input: {
|
||||
sessionId: string;
|
||||
userId: string;
|
||||
prompt: { model: string };
|
||||
turn: Turn;
|
||||
compatSubmissionId?: string;
|
||||
}) {
|
||||
const message = await this.models.copilotSession.appendMessage({
|
||||
sessionId: input.sessionId,
|
||||
userId: input.userId,
|
||||
prompt: input.prompt,
|
||||
message: (() => {
|
||||
const { id: _id, ...message } = chatMessageFromTurn(input.turn);
|
||||
return { ...message, compatSubmissionId: input.compatSubmissionId };
|
||||
|
||||
@@ -5,6 +5,7 @@ import { JOB_SIGNAL, JobQueue, OneDay, OnJob } from '../../base';
|
||||
import { Models } from '../../models';
|
||||
|
||||
const CLEANUP_EMBEDDING_JOB_BATCH_SIZE = 100;
|
||||
const BACKGROUND_COPILOT_JOB_PRIORITY = 100;
|
||||
|
||||
declare global {
|
||||
interface Jobs {
|
||||
@@ -71,9 +72,11 @@ export class CopilotCronJobs {
|
||||
const sessions = await this.models.copilotSession.toBeGenerateTitle();
|
||||
|
||||
for (const session of sessions) {
|
||||
await this.jobs.add('copilot.session.generateTitle', {
|
||||
sessionId: session.id,
|
||||
});
|
||||
await this.jobs.add(
|
||||
'copilot.session.generateTitle',
|
||||
{ sessionId: session.id },
|
||||
{ priority: BACKGROUND_COPILOT_JOB_PRIORITY }
|
||||
);
|
||||
}
|
||||
this.logger.log(
|
||||
`Scheduled title generation for ${sessions.length} sessions`
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/* oxlint-disable import/no-cycle -- Embedding delegates to the shared capability runtime. */
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { CopilotFailedToGenerateEmbedding } from '../../../base/error/errors.gen';
|
||||
import {
|
||||
@@ -10,7 +11,6 @@ import {
|
||||
} from '../../../models';
|
||||
import { type CopilotRerankRequest } from '../providers/types';
|
||||
import { CapabilityRuntime } from '../runtime/capability-runtime';
|
||||
import { TaskPolicy } from '../runtime/task-policy';
|
||||
import {
|
||||
type EmbeddingCallOptionsInput,
|
||||
EmbeddingClient,
|
||||
@@ -18,20 +18,20 @@ import {
|
||||
type ReRankResult,
|
||||
} from './types';
|
||||
|
||||
type EmbeddingRuntime = Pick<
|
||||
CapabilityRuntime,
|
||||
'embeddingConfigured' | 'embed' | 'rerank'
|
||||
>;
|
||||
|
||||
class ProductionEmbeddingClient extends EmbeddingClient {
|
||||
private readonly logger = new Logger(ProductionEmbeddingClient.name);
|
||||
|
||||
constructor(
|
||||
private readonly taskPolicy: TaskPolicy,
|
||||
private readonly runtime: CapabilityRuntime
|
||||
) {
|
||||
constructor(private readonly runtime: EmbeddingRuntime) {
|
||||
super();
|
||||
}
|
||||
|
||||
override async configured(): Promise<boolean> {
|
||||
const result = await this.runtime.embeddingConfigured(
|
||||
this.taskPolicy.resolveEmbeddingModelId()
|
||||
);
|
||||
const result = await this.runtime.embeddingConfigured('route-selected');
|
||||
if (!result) {
|
||||
this.logger.warn(
|
||||
'Copilot embedding client is not configured properly, please check your configuration.'
|
||||
@@ -45,7 +45,7 @@ class ProductionEmbeddingClient extends EmbeddingClient {
|
||||
options?: EmbeddingCallOptionsInput
|
||||
): Promise<Embedding[]> {
|
||||
const normalizedOptions = normalizeEmbeddingCallOptions(options);
|
||||
const modelId = this.taskPolicy.resolveEmbeddingModelId();
|
||||
const modelId = 'route-selected';
|
||||
const embeddings = await this.runtime.embed(modelId, input, {
|
||||
dimensions: EMBEDDING_DIMENSIONS,
|
||||
signal: normalizedOptions.signal,
|
||||
@@ -94,17 +94,13 @@ class ProductionEmbeddingClient extends EmbeddingClient {
|
||||
})),
|
||||
};
|
||||
|
||||
const ranks = await this.runtime.rerank(
|
||||
this.taskPolicy.resolveRerankModelId(),
|
||||
rerankRequest,
|
||||
{
|
||||
signal: normalizedOptions.signal,
|
||||
user: normalizedOptions.userId,
|
||||
workspace: normalizedOptions.workspaceId,
|
||||
byokLeaseId: normalizedOptions.byokLeaseId,
|
||||
featureKind: 'rerank',
|
||||
}
|
||||
);
|
||||
const ranks = await this.runtime.rerank('route-selected', rerankRequest, {
|
||||
signal: normalizedOptions.signal,
|
||||
user: normalizedOptions.userId,
|
||||
workspace: normalizedOptions.workspaceId,
|
||||
byokLeaseId: normalizedOptions.byokLeaseId,
|
||||
featureKind: 'rerank',
|
||||
});
|
||||
|
||||
try {
|
||||
return ranks.map((score, i) => {
|
||||
@@ -206,12 +202,12 @@ export class CopilotEmbeddingClientService {
|
||||
private client: EmbeddingClient | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly taskPolicy: TaskPolicy,
|
||||
private readonly runtime: CapabilityRuntime
|
||||
@Inject(forwardRef(() => CapabilityRuntime))
|
||||
private readonly runtime: EmbeddingRuntime
|
||||
) {}
|
||||
|
||||
async refresh() {
|
||||
const client = new ProductionEmbeddingClient(this.taskPolicy, this.runtime);
|
||||
const client = new ProductionEmbeddingClient(this.runtime);
|
||||
await client.configured();
|
||||
this.client = client;
|
||||
return this.client;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { CanActivate, Injectable, UseGuards } from '@nestjs/common';
|
||||
|
||||
import { Config } from '../../base/config';
|
||||
import { OnEvent } from '../../base/event';
|
||||
import { ServerFeature, ServerService } from '../../core/config';
|
||||
import { assertCopilotEnabled } from './availability';
|
||||
|
||||
@Injectable()
|
||||
export class CopilotFeatureService {
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly server: ServerService
|
||||
) {}
|
||||
|
||||
get enabled() {
|
||||
return this.config.copilot.enabled;
|
||||
}
|
||||
|
||||
@OnEvent('config.init')
|
||||
onConfigInit() {
|
||||
this.syncServerFeature();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
onConfigChanged(event: Events['config.changed']) {
|
||||
if ('copilot' in event.updates) {
|
||||
this.syncServerFeature();
|
||||
}
|
||||
}
|
||||
|
||||
assertEnabled() {
|
||||
assertCopilotEnabled(this.config);
|
||||
}
|
||||
|
||||
private syncServerFeature() {
|
||||
if (this.enabled) {
|
||||
this.server.enableFeature(ServerFeature.Copilot);
|
||||
} else {
|
||||
this.server.disableFeature(ServerFeature.Copilot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class CopilotFeatureGuard implements CanActivate {
|
||||
constructor(private readonly feature: CopilotFeatureService) {}
|
||||
|
||||
canActivate() {
|
||||
this.feature.assertEnabled();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export const CopilotEnabled = () => UseGuards(CopilotFeatureGuard);
|
||||
@@ -11,6 +11,7 @@ import { StorageModule } from '../../core/storage';
|
||||
import { WorkspaceModule } from '../../core/workspaces';
|
||||
import { IndexerModule } from '../indexer';
|
||||
import { CopilotController } from './controller';
|
||||
import { CopilotFeatureGuard, CopilotFeatureService } from './feature';
|
||||
import { WorkspaceMcpController } from './mcp/controller';
|
||||
import { McpCredentialService } from './mcp/credential';
|
||||
import { McpCredentialResolver } from './mcp/resolver';
|
||||
@@ -34,20 +35,27 @@ const COPILOT_SHARED_IMPORTS = [
|
||||
];
|
||||
|
||||
@Module({
|
||||
imports: [...COPILOT_SHARED_IMPORTS],
|
||||
imports: [ServerConfigModule],
|
||||
providers: [CopilotFeatureService, CopilotFeatureGuard],
|
||||
exports: [CopilotFeatureService, CopilotFeatureGuard],
|
||||
})
|
||||
export class CopilotAvailabilityModule {}
|
||||
|
||||
@Module({
|
||||
imports: [...COPILOT_SHARED_IMPORTS, CopilotAvailabilityModule],
|
||||
providers: [...COPILOT_KERNEL_PROVIDERS],
|
||||
exports: [...COPILOT_KERNEL_PROVIDERS],
|
||||
exports: [CopilotAvailabilityModule, ...COPILOT_KERNEL_PROVIDERS],
|
||||
})
|
||||
export class CopilotKernelModule {}
|
||||
|
||||
@Module({
|
||||
imports: [PermissionModule],
|
||||
imports: [PermissionModule, CopilotAvailabilityModule],
|
||||
providers: [...COPILOT_TRANSCRIPT_REALTIME_PROVIDERS],
|
||||
})
|
||||
export class CopilotRealtimeModule {}
|
||||
|
||||
@Module({
|
||||
imports: [PermissionModule],
|
||||
imports: [PermissionModule, CopilotAvailabilityModule],
|
||||
providers: [...COPILOT_CONTEXT_REALTIME_PROVIDERS],
|
||||
})
|
||||
export class CopilotEmbeddingRealtimeModule {}
|
||||
|
||||
@@ -16,6 +16,7 @@ import type { Request, Response } from 'express';
|
||||
import { ActionForbidden, Throttle } from '../../../base';
|
||||
import { Public } from '../../../core/auth';
|
||||
import { extractTokenFromHeader } from '../../../core/auth/input';
|
||||
import { CopilotEnabled } from '../feature';
|
||||
import { McpCredentialService } from './credential';
|
||||
import { WorkspaceMcpProvider, type WorkspaceMcpServer } from './provider';
|
||||
|
||||
@@ -46,6 +47,7 @@ const SUPPORTED_PROTOCOL_VERSIONS = new Set([
|
||||
'2024-10-07',
|
||||
]);
|
||||
|
||||
@CopilotEnabled()
|
||||
@Controller('/api/workspaces/:workspaceId/mcp')
|
||||
export class WorkspaceMcpController {
|
||||
private readonly logger = new Logger(WorkspaceMcpController.name);
|
||||
|
||||
@@ -15,6 +15,7 @@ import { McpAccessMode } from '@prisma/client';
|
||||
|
||||
import { CurrentUser } from '../../../core/auth';
|
||||
import { PermissionAccess } from '../../../core/permission';
|
||||
import { CopilotEnabled } from '../feature';
|
||||
import { McpCredentialService } from './credential';
|
||||
|
||||
registerEnumType(McpAccessMode, { name: 'McpAccessMode' });
|
||||
@@ -89,6 +90,7 @@ class CreateMcpCredentialInput {
|
||||
expirationDays!: number;
|
||||
}
|
||||
|
||||
@CopilotEnabled()
|
||||
@Resolver()
|
||||
export class McpCredentialResolver {
|
||||
constructor(
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
import { CopilotAccessPolicy } from './access';
|
||||
import {
|
||||
ByokEntitlementPolicy,
|
||||
ByokService,
|
||||
WorkspaceByokResolver,
|
||||
} from './byok';
|
||||
import { ByokEntitlementPolicy, WorkspaceByokResolver } from './byok';
|
||||
import { HistoryAttachmentUrlProjector } from './compat/history-attachment-url-projector';
|
||||
import { CompatHistoryProjector } from './compat/history-projector';
|
||||
import { HistoryPromptPreloadProjector } from './compat/history-prompt-preload-projector';
|
||||
@@ -25,30 +20,18 @@ import {
|
||||
} from './embedding';
|
||||
import { WorkspaceMcpProvider } from './mcp/provider';
|
||||
import { PromptService } from './prompt';
|
||||
import {
|
||||
CopilotProviderFactory,
|
||||
CopilotProviderLifecycleService,
|
||||
CopilotProviderRegistryService,
|
||||
CopilotProviders,
|
||||
} from './providers';
|
||||
import { CopilotResolver, UserCopilotResolver } from './resolver';
|
||||
import { ActionRuntimeBridge } from './runtime/action-runtime-bridge';
|
||||
import { CapabilityRuntime } from './runtime/capability-runtime';
|
||||
import { CopilotExecutionMetrics } from './runtime/execution-metrics';
|
||||
import { ExecutionPlanBuilder } from './runtime/execution-plan';
|
||||
import { CopilotRuntimeEventConsumer } from './runtime/copilot-runtime-event-consumer';
|
||||
import { ActionStreamHost } from './runtime/hosts/action-stream-host';
|
||||
import { AttachmentAdmissionHost } from './runtime/hosts/attachment-admission';
|
||||
import { AttachmentMaterializer } from './runtime/hosts/attachment-materializer';
|
||||
import { CapabilityPolicyHost } from './runtime/hosts/capability-policy-host';
|
||||
import { ConversationHost } from './runtime/hosts/conversation-host';
|
||||
import { ImageResultHost } from './runtime/hosts/image-result-host';
|
||||
import { ResponsePostprocessor } from './runtime/hosts/response-postprocessor';
|
||||
import { ToolExecutorHost } from './runtime/hosts/tool-executor-host';
|
||||
import { TurnPersistence } from './runtime/hosts/turn-persistence';
|
||||
import { ModelSelectionPolicy } from './runtime/model-selection-policy';
|
||||
import { NativeExecutionEngine } from './runtime/native-execution-engine';
|
||||
import { PromptRuntime } from './runtime/prompt-runtime';
|
||||
import { TaskPolicy } from './runtime/task-policy';
|
||||
import { ToolRuntime } from './runtime/tool-runtime';
|
||||
import { TurnOrchestrator } from './runtime/turn-orchestrator';
|
||||
import { ChatSessionService } from './session';
|
||||
@@ -65,21 +48,14 @@ import {
|
||||
CopilotWorkspaceService,
|
||||
} from './workspace';
|
||||
|
||||
export const COPILOT_PROVIDER_PROVIDERS = [
|
||||
...CopilotProviders,
|
||||
CopilotProviderRegistryService,
|
||||
CopilotProviderFactory,
|
||||
CopilotProviderLifecycleService,
|
||||
];
|
||||
export const COPILOT_PROVIDER_PROVIDERS: [] = [];
|
||||
|
||||
export const COPILOT_RUNTIME_PROVIDERS = [
|
||||
ByokEntitlementPolicy,
|
||||
ByokService,
|
||||
ChatSessionService,
|
||||
ConversationStore,
|
||||
ConversationInboxService,
|
||||
ConversationPolicy,
|
||||
CopilotAccessPolicy,
|
||||
HistoryAttachmentUrlProjector,
|
||||
CompatHistoryProjector,
|
||||
HistoryPromptPreloadProjector,
|
||||
@@ -88,18 +64,12 @@ export const COPILOT_RUNTIME_PROVIDERS = [
|
||||
CopilotContextService,
|
||||
CopilotEmbeddingClientService,
|
||||
PromptService,
|
||||
ModelSelectionPolicy,
|
||||
ActionRuntimeBridge,
|
||||
CopilotExecutionMetrics,
|
||||
ExecutionPlanBuilder,
|
||||
CopilotRuntimeEventConsumer,
|
||||
PromptRuntime,
|
||||
CapabilityPolicyHost,
|
||||
ConversationHost,
|
||||
CapabilityRuntime,
|
||||
NativeExecutionEngine,
|
||||
TaskPolicy,
|
||||
ToolRuntime,
|
||||
ToolExecutorHost,
|
||||
AttachmentMaterializer,
|
||||
AttachmentAdmissionHost,
|
||||
ActionStreamHost,
|
||||
|
||||
@@ -1,36 +1,17 @@
|
||||
import {
|
||||
llmCollectPromptMetadata,
|
||||
llmCountPromptTokens,
|
||||
llmGetBuiltInPromptSpec,
|
||||
llmListBuiltInPromptSpecs,
|
||||
llmRenderBuiltInPrompt,
|
||||
llmRenderBuiltInSessionPrompt,
|
||||
llmRenderPrompt,
|
||||
llmRenderSessionPrompt,
|
||||
type NativeBuiltInPromptRenderRequest as NativeBuiltInPromptRenderContract,
|
||||
type NativeBuiltInPromptSessionRenderRequest as NativeBuiltInPromptSessionContract,
|
||||
type NativePromptCountTokensRequest as NativePromptTokenCountContract,
|
||||
type NativePromptCountTokensResponse as NativePromptTokenCountResult,
|
||||
type NativePromptMetadataRequest as NativePromptMetadataContract,
|
||||
type NativePromptMetadataResponse as NativePromptMetadataResult,
|
||||
type NativePromptRenderRequest as NativePromptRenderContract,
|
||||
type NativePromptRenderResponse as NativePromptRenderResult,
|
||||
type NativePromptSessionRenderRequest as NativePromptSessionContract,
|
||||
type NativePromptSessionRenderResponse as NativePromptSessionResult,
|
||||
} from '../../../native';
|
||||
import type { PromptMessage, PromptParams } from '../providers/types';
|
||||
import { projectPromptMessageForNative } from '../runtime/contracts';
|
||||
import type { PromptSpec } from './spec';
|
||||
|
||||
export type NativePromptRenderRequest = Omit<
|
||||
NativePromptRenderContract,
|
||||
'messages' | 'templateParams' | 'renderParams'
|
||||
> & {
|
||||
messages: PromptMessage[];
|
||||
templateParams: PromptParams;
|
||||
renderParams: PromptParams;
|
||||
};
|
||||
|
||||
export type NativePromptRenderResponse = Omit<
|
||||
NativePromptRenderResult,
|
||||
'messages'
|
||||
@@ -45,46 +26,6 @@ export type NativeBuiltInPromptRenderRequest = Omit<
|
||||
renderParams: PromptParams;
|
||||
};
|
||||
|
||||
export type NativePromptCountTokensRequest = Omit<
|
||||
NativePromptTokenCountContract,
|
||||
'messages' | 'model'
|
||||
> & {
|
||||
model?: string | null;
|
||||
messages: Pick<PromptMessage, 'content'>[];
|
||||
};
|
||||
|
||||
export type NativePromptCountTokensResponse = NativePromptTokenCountResult;
|
||||
|
||||
export type NativePromptMetadataRequest = Omit<
|
||||
NativePromptMetadataContract,
|
||||
'messages'
|
||||
> & {
|
||||
messages: PromptMessage[];
|
||||
};
|
||||
|
||||
export type NativePromptMetadataResponse = Omit<
|
||||
NativePromptMetadataResult,
|
||||
'templateParams'
|
||||
> & {
|
||||
templateParams: PromptParams;
|
||||
};
|
||||
|
||||
export type NativePromptSessionRenderRequest = Omit<
|
||||
NativePromptSessionContract,
|
||||
'prompt' | 'turns' | 'renderParams'
|
||||
> & {
|
||||
prompt: Omit<
|
||||
NativePromptSessionContract['prompt'],
|
||||
'templateParams' | 'messages' | 'model'
|
||||
> & {
|
||||
model?: string | null;
|
||||
templateParams: PromptParams;
|
||||
messages: PromptMessage[];
|
||||
};
|
||||
turns: PromptMessage[];
|
||||
renderParams: PromptParams;
|
||||
};
|
||||
|
||||
export type NativePromptSessionRenderResponse = Omit<
|
||||
NativePromptSessionResult,
|
||||
'messages'
|
||||
@@ -100,8 +41,7 @@ export type NativeBuiltInPromptSessionRenderRequest = Omit<
|
||||
renderParams: PromptParams;
|
||||
};
|
||||
|
||||
type NativePromptContractMessage =
|
||||
NativePromptRenderContract['messages'][number];
|
||||
type NativePromptContractMessage = NativePromptRenderResult['messages'][number];
|
||||
|
||||
function toNativePromptMessage(
|
||||
message: PromptMessage
|
||||
@@ -123,22 +63,6 @@ function fromNativePromptMessage(
|
||||
};
|
||||
}
|
||||
|
||||
export function renderPromptNative(
|
||||
request: NativePromptRenderRequest
|
||||
): NativePromptRenderResponse {
|
||||
const normalizedMessages = request.messages.map(toNativePromptMessage);
|
||||
const rendered = llmRenderPrompt({
|
||||
messages: normalizedMessages,
|
||||
templateParams: request.templateParams,
|
||||
renderParams: request.renderParams,
|
||||
});
|
||||
|
||||
return {
|
||||
...rendered,
|
||||
messages: rendered.messages.map(fromNativePromptMessage),
|
||||
};
|
||||
}
|
||||
|
||||
export function renderBuiltInPromptNative(
|
||||
request: NativeBuiltInPromptRenderRequest
|
||||
): NativePromptRenderResponse {
|
||||
@@ -153,25 +77,6 @@ export function renderBuiltInPromptNative(
|
||||
};
|
||||
}
|
||||
|
||||
export function renderPromptSessionNative(
|
||||
request: NativePromptSessionRenderRequest
|
||||
): NativePromptSessionRenderResponse {
|
||||
const rendered = llmRenderSessionPrompt({
|
||||
...request,
|
||||
prompt: {
|
||||
...request.prompt,
|
||||
messages: request.prompt.messages.map(toNativePromptMessage),
|
||||
model: request.prompt.model ?? undefined,
|
||||
},
|
||||
turns: request.turns.map(toNativePromptMessage),
|
||||
renderParams: request.renderParams,
|
||||
});
|
||||
return {
|
||||
...rendered,
|
||||
messages: rendered.messages.map(fromNativePromptMessage),
|
||||
};
|
||||
}
|
||||
|
||||
export function renderBuiltInPromptSessionNative(
|
||||
request: NativeBuiltInPromptSessionRenderRequest
|
||||
): NativePromptSessionRenderResponse {
|
||||
@@ -187,29 +92,10 @@ export function renderBuiltInPromptSessionNative(
|
||||
};
|
||||
}
|
||||
|
||||
export function countPromptTokensNative(
|
||||
request: NativePromptCountTokensRequest
|
||||
): NativePromptCountTokensResponse {
|
||||
return llmCountPromptTokens({
|
||||
...request,
|
||||
model: request.model ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export function collectPromptMetadataNative(
|
||||
request: NativePromptMetadataRequest
|
||||
): NativePromptMetadataResponse {
|
||||
return llmCollectPromptMetadata({
|
||||
messages: request.messages.map(toNativePromptMessage),
|
||||
});
|
||||
}
|
||||
|
||||
export function listBuiltInPromptSpecsNative(): PromptSpec[] {
|
||||
return llmListBuiltInPromptSpecs().map(spec => ({
|
||||
name: spec.name,
|
||||
action: spec.action,
|
||||
model: spec.model,
|
||||
optionalModels: spec.optionalModels,
|
||||
config: spec.config,
|
||||
params: spec.params
|
||||
? Object.fromEntries(
|
||||
@@ -238,8 +124,6 @@ export function getBuiltInPromptSpecNative(name: string): PromptSpec | null {
|
||||
return {
|
||||
name: spec.name,
|
||||
action: spec.action,
|
||||
model: spec.model,
|
||||
optionalModels: spec.optionalModels,
|
||||
config: spec.config,
|
||||
params: spec.params
|
||||
? Object.fromEntries(
|
||||
|
||||
@@ -2,15 +2,11 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import type { PromptMessage, PromptParams } from '../providers/types';
|
||||
import {
|
||||
collectPromptMetadataNative,
|
||||
countPromptTokensNative,
|
||||
getBuiltInPromptSpecNative,
|
||||
renderBuiltInPromptNative,
|
||||
renderBuiltInPromptSessionNative,
|
||||
renderPromptNative,
|
||||
renderPromptSessionNative,
|
||||
} from './native-contract';
|
||||
import type { Prompt, PromptSpec, ResolvedPrompt } from './spec';
|
||||
import type { PromptSpec, ResolvedPrompt } from './spec';
|
||||
|
||||
@Injectable()
|
||||
export class PromptService {
|
||||
@@ -20,11 +16,6 @@ export class PromptService {
|
||||
}
|
||||
|
||||
async get(name: string): Promise<ResolvedPrompt | null> {
|
||||
const compatPrompt = this.lookupCompatPrompt(name);
|
||||
if (compatPrompt) {
|
||||
return this.describeCompatPrompt(this.clonePrompt(compatPrompt));
|
||||
}
|
||||
|
||||
const builtInPromptSpec = this.lookupBuiltInPromptSpec(name);
|
||||
if (!builtInPromptSpec) return null;
|
||||
|
||||
@@ -36,17 +27,10 @@ export class PromptService {
|
||||
params: PromptParams,
|
||||
sessionId?: string
|
||||
): PromptMessage[] {
|
||||
const rendered =
|
||||
prompt.source === 'built_in'
|
||||
? renderBuiltInPromptNative({
|
||||
name: prompt.name,
|
||||
renderParams: params,
|
||||
})
|
||||
: renderPromptNative({
|
||||
messages: this.requireCompatMessages(prompt),
|
||||
templateParams: prompt.params,
|
||||
renderParams: params,
|
||||
});
|
||||
const rendered = renderBuiltInPromptNative({
|
||||
name: prompt.name,
|
||||
renderParams: params,
|
||||
});
|
||||
|
||||
this.logWarnings(rendered.warnings, sessionId);
|
||||
return rendered.messages;
|
||||
@@ -56,38 +40,18 @@ export class PromptService {
|
||||
prompt: ResolvedPrompt,
|
||||
turns: PromptMessage[],
|
||||
params: PromptParams,
|
||||
maxTokenSize = prompt.config?.maxTokens || 128 * 1024,
|
||||
sessionId?: string
|
||||
): PromptMessage[] {
|
||||
const rendered =
|
||||
prompt.source === 'built_in'
|
||||
? renderBuiltInPromptSessionNative({
|
||||
name: prompt.name,
|
||||
turns,
|
||||
renderParams: params,
|
||||
maxTokenSize,
|
||||
})
|
||||
: renderPromptSessionNative({
|
||||
prompt: {
|
||||
action: prompt.action,
|
||||
model: prompt.model,
|
||||
promptTokens: this.countCompatPromptTokens(prompt),
|
||||
templateParams: prompt.params,
|
||||
messages: this.requireCompatMessages(prompt),
|
||||
},
|
||||
turns,
|
||||
renderParams: params,
|
||||
maxTokenSize,
|
||||
});
|
||||
const rendered = renderBuiltInPromptSessionNative({
|
||||
name: prompt.name,
|
||||
turns,
|
||||
renderParams: params,
|
||||
});
|
||||
|
||||
this.logWarnings(rendered.warnings, sessionId);
|
||||
return rendered.messages;
|
||||
}
|
||||
|
||||
protected lookupCompatPrompt(_name: string): Prompt | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected lookupBuiltInPromptSpec(name: string): PromptSpec | null {
|
||||
const spec = getBuiltInPromptSpecNative(name);
|
||||
return spec ? this.clonePromptSpec(spec) : null;
|
||||
@@ -104,23 +68,9 @@ export class PromptService {
|
||||
}));
|
||||
}
|
||||
|
||||
protected clonePrompt(prompt: Prompt): Prompt {
|
||||
return {
|
||||
...prompt,
|
||||
optionalModels: prompt.optionalModels
|
||||
? [...prompt.optionalModels]
|
||||
: undefined,
|
||||
config: prompt.config ? structuredClone(prompt.config) : undefined,
|
||||
messages: this.cloneMessages(prompt.messages),
|
||||
};
|
||||
}
|
||||
|
||||
protected clonePromptSpec(spec: PromptSpec): PromptSpec {
|
||||
return {
|
||||
...spec,
|
||||
optionalModels: spec.optionalModels
|
||||
? [...spec.optionalModels]
|
||||
: undefined,
|
||||
config: spec.config ? structuredClone(spec.config) : undefined,
|
||||
params: spec.params ? structuredClone(spec.params) : undefined,
|
||||
messages: spec.messages.map(message => ({ ...message })),
|
||||
@@ -132,27 +82,9 @@ export class PromptService {
|
||||
return {
|
||||
name: spec.name,
|
||||
action: spec.action,
|
||||
model: spec.model,
|
||||
optionalModels: spec.optionalModels ?? [],
|
||||
config: spec.config ? structuredClone(spec.config) : undefined,
|
||||
paramKeys: Object.keys(params),
|
||||
params,
|
||||
source: 'built_in',
|
||||
};
|
||||
}
|
||||
|
||||
private describeCompatPrompt(prompt: Prompt): ResolvedPrompt {
|
||||
const metadata = collectPromptMetadataNative({ messages: prompt.messages });
|
||||
return {
|
||||
name: prompt.name,
|
||||
action: prompt.action,
|
||||
model: prompt.model,
|
||||
optionalModels: prompt.optionalModels ?? [],
|
||||
config: prompt.config ? structuredClone(prompt.config) : undefined,
|
||||
paramKeys: metadata.paramKeys,
|
||||
params: metadata.templateParams,
|
||||
source: 'compat',
|
||||
messages: prompt.messages,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -178,23 +110,6 @@ export class PromptService {
|
||||
);
|
||||
}
|
||||
|
||||
private countCompatPromptTokens(prompt: ResolvedPrompt): number {
|
||||
return countPromptTokensNative({
|
||||
model: prompt.model,
|
||||
messages: this.requireCompatMessages(prompt).map(message => ({
|
||||
content: message.content,
|
||||
})),
|
||||
}).tokens;
|
||||
}
|
||||
|
||||
private requireCompatMessages(prompt: ResolvedPrompt): PromptMessage[] {
|
||||
if (prompt.source === 'compat' && prompt.messages) {
|
||||
return this.cloneMessages(prompt.messages);
|
||||
}
|
||||
|
||||
throw new Error(`Prompt ${prompt.name} does not expose compat messages`);
|
||||
}
|
||||
|
||||
private logWarnings(warnings: string[], sessionId?: string) {
|
||||
if (!sessionId) {
|
||||
return;
|
||||
|
||||
@@ -1,28 +1,11 @@
|
||||
import type {
|
||||
PromptConfig,
|
||||
PromptMessage,
|
||||
PromptParams,
|
||||
} from '../providers/types';
|
||||
|
||||
export type Prompt = {
|
||||
name: string;
|
||||
model: string;
|
||||
optionalModels?: string[];
|
||||
action?: string;
|
||||
messages: PromptMessage[];
|
||||
config?: PromptConfig;
|
||||
};
|
||||
import type { PromptConfig, PromptParams } from '../providers/types';
|
||||
|
||||
export type ResolvedPrompt = {
|
||||
name: string;
|
||||
model: string;
|
||||
optionalModels: string[];
|
||||
action?: string;
|
||||
config?: PromptConfig;
|
||||
paramKeys: string[];
|
||||
params: PromptParams;
|
||||
source: 'built_in' | 'compat';
|
||||
messages?: PromptMessage[];
|
||||
};
|
||||
|
||||
type PromptParamSpec = {
|
||||
@@ -38,8 +21,6 @@ type PromptSpecMessage = {
|
||||
export type PromptSpec = {
|
||||
name: string;
|
||||
action?: string;
|
||||
model: string;
|
||||
optionalModels?: string[];
|
||||
config?: PromptConfig;
|
||||
params?: Record<string, PromptParamSpec>;
|
||||
messages: PromptSpecMessage[];
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
import { CopilotProviderSideError, UserFriendlyError } from '../../../../base';
|
||||
import {
|
||||
type LlmBackendConfig,
|
||||
llmResolveRequestIntentOptions,
|
||||
} from '../../../../native';
|
||||
import { CopilotProvider } from '../provider';
|
||||
import { hasProviderModelBehaviorFlag } from '../provider-model-runtime';
|
||||
import {
|
||||
type CopilotProviderExecution,
|
||||
type ProviderDriverSpec,
|
||||
} from '../provider-runtime-contract';
|
||||
import { CopilotProviderType } from '../types';
|
||||
import {
|
||||
getGoogleAuth,
|
||||
getVertexAnthropicBaseUrl,
|
||||
type VertexAnthropicProviderConfig,
|
||||
} from '../utils';
|
||||
|
||||
export abstract class AnthropicProvider<T> extends CopilotProvider<T> {
|
||||
protected resolveModelBackendKind() {
|
||||
return this.type === CopilotProviderType.AnthropicVertex
|
||||
? ('anthropic_vertex' as const)
|
||||
: ('anthropic' as const);
|
||||
}
|
||||
|
||||
override getDriverSpec(): ProviderDriverSpec {
|
||||
return {
|
||||
createBackendConfig: execution => this.createNativeConfig(execution),
|
||||
mapError: error => this.handleError(error),
|
||||
chat: {
|
||||
resolveRequestOptions: async context => {
|
||||
const requestIntent = await llmResolveRequestIntentOptions({
|
||||
protocol: context.protocol,
|
||||
backendConfig: context.backendConfig,
|
||||
reasoning: {
|
||||
enabled: context.options.reasoning,
|
||||
supported: hasProviderModelBehaviorFlag(
|
||||
context.model,
|
||||
'reasoning_budget_12000'
|
||||
),
|
||||
budgetTokens: hasProviderModelBehaviorFlag(
|
||||
context.model,
|
||||
'reasoning_budget_12000'
|
||||
)
|
||||
? 12000
|
||||
: undefined,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
attachmentCapability: this.getAttachCapability(
|
||||
context.model,
|
||||
context.outputType
|
||||
),
|
||||
reasoning: requestIntent.reasoning,
|
||||
};
|
||||
},
|
||||
},
|
||||
structured: false,
|
||||
embedding: false,
|
||||
rerank: false,
|
||||
};
|
||||
}
|
||||
|
||||
private handleError(e: any) {
|
||||
if (e instanceof UserFriendlyError) {
|
||||
return e;
|
||||
}
|
||||
return new CopilotProviderSideError({
|
||||
provider: this.type,
|
||||
kind: 'unexpected_response',
|
||||
message: e?.message || 'Unexpected anthropic response',
|
||||
});
|
||||
}
|
||||
|
||||
private async createNativeConfig(
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<LlmBackendConfig> {
|
||||
const config = this.getConfig(execution);
|
||||
if (this.type === CopilotProviderType.AnthropicVertex) {
|
||||
const vertexConfig = config as VertexAnthropicProviderConfig;
|
||||
const auth = await getGoogleAuth(vertexConfig, 'anthropic');
|
||||
const { Authorization: authHeader } = auth.headers();
|
||||
const token = authHeader.replace(/^Bearer\s+/i, '');
|
||||
const baseUrl = getVertexAnthropicBaseUrl(vertexConfig) || auth.baseUrl;
|
||||
return {
|
||||
base_url: baseUrl || '',
|
||||
auth_token: token,
|
||||
headers: { Authorization: authHeader },
|
||||
};
|
||||
}
|
||||
|
||||
const officialConfig = config as { apiKey: string; baseURL?: string };
|
||||
const baseUrl = officialConfig.baseURL || 'https://api.anthropic.com/v1';
|
||||
return {
|
||||
base_url: baseUrl.replace(/\/v1\/?$/, ''),
|
||||
auth_token: officialConfig.apiKey,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
export * from './official';
|
||||
export * from './vertex';
|
||||
@@ -1,16 +0,0 @@
|
||||
import type { CopilotProviderExecution } from '../provider-runtime-contract';
|
||||
import { CopilotProviderType } from '../types';
|
||||
import { AnthropicProvider } from './anthropic';
|
||||
|
||||
export type AnthropicOfficialConfig = {
|
||||
apiKey: string;
|
||||
baseURL?: string;
|
||||
};
|
||||
|
||||
export class AnthropicOfficialProvider extends AnthropicProvider<AnthropicOfficialConfig> {
|
||||
override readonly type = CopilotProviderType.Anthropic;
|
||||
|
||||
override configured(execution?: CopilotProviderExecution): boolean {
|
||||
return !!this.getConfig(execution).apiKey;
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
import type { CopilotProviderExecution } from '../provider-runtime-contract';
|
||||
import { CopilotProviderType } from '../types';
|
||||
import { getVertexAnthropicBaseUrl, type VertexProviderConfig } from '../utils';
|
||||
import { AnthropicProvider } from './anthropic';
|
||||
|
||||
export type AnthropicVertexConfig = VertexProviderConfig;
|
||||
|
||||
export class AnthropicVertexProvider extends AnthropicProvider<AnthropicVertexConfig> {
|
||||
override readonly type = CopilotProviderType.AnthropicVertex;
|
||||
|
||||
override configured(execution?: CopilotProviderExecution): boolean {
|
||||
const config = this.getConfig(execution);
|
||||
if (!config.location || !config.googleAuthOptions) return false;
|
||||
return !!config.project || !!getVertexAnthropicBaseUrl(config);
|
||||
}
|
||||
}
|
||||
@@ -1,42 +1,4 @@
|
||||
import type {
|
||||
ModelAttachmentCapability,
|
||||
PromptAttachment,
|
||||
PromptMessage,
|
||||
} from './types';
|
||||
|
||||
export const IMAGE_ATTACHMENT_CAPABILITY: ModelAttachmentCapability = {
|
||||
kinds: ['image'],
|
||||
sourceKinds: ['url', 'data'],
|
||||
allowRemoteUrls: true,
|
||||
};
|
||||
|
||||
export const GEMINI_ATTACHMENT_CAPABILITY: ModelAttachmentCapability = {
|
||||
kinds: ['image', 'audio', 'file'],
|
||||
sourceKinds: ['url', 'data', 'bytes', 'file_handle'],
|
||||
allowRemoteUrls: true,
|
||||
};
|
||||
|
||||
export function promptAttachmentHasSource(
|
||||
attachment: PromptAttachment
|
||||
): boolean {
|
||||
if (typeof attachment === 'string') {
|
||||
return !!attachment.trim();
|
||||
}
|
||||
|
||||
if ('attachment' in attachment) {
|
||||
return !!attachment.attachment;
|
||||
}
|
||||
|
||||
switch (attachment.kind) {
|
||||
case 'url':
|
||||
return !!attachment.url;
|
||||
case 'data':
|
||||
case 'bytes':
|
||||
return !!attachment.data;
|
||||
case 'file_handle':
|
||||
return !!attachment.fileHandle;
|
||||
}
|
||||
}
|
||||
import type { PromptAttachment, PromptMessage } from './types';
|
||||
|
||||
export function applyPromptAttachmentMimeTypeHintForNative(
|
||||
attachment: PromptAttachment,
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
import { CopilotProviderSideError, UserFriendlyError } from '../../../base';
|
||||
import { type LlmBackendConfig } from '../../../native';
|
||||
import { CopilotProvider } from './provider';
|
||||
import {
|
||||
type CopilotProviderExecution,
|
||||
type ProviderDriverSpec,
|
||||
} from './provider-runtime-contract';
|
||||
import { CopilotProviderType } from './types';
|
||||
|
||||
export type CloudflareWorkersAIConfig = {
|
||||
apiToken: string;
|
||||
accountId?: string;
|
||||
baseURL?: string;
|
||||
};
|
||||
|
||||
export class CloudflareWorkersAIProvider extends CopilotProvider<CloudflareWorkersAIConfig> {
|
||||
override readonly type = CopilotProviderType.CloudflareWorkersAi;
|
||||
|
||||
protected resolveModelBackendKind() {
|
||||
return 'cloudflare_workers_ai' as const;
|
||||
}
|
||||
|
||||
override configured(execution?: CopilotProviderExecution): boolean {
|
||||
const config = this.getConfig(execution);
|
||||
return !!config.apiToken && (!!config.accountId || !!config.baseURL);
|
||||
}
|
||||
private handleError(e: any) {
|
||||
if (e instanceof UserFriendlyError) {
|
||||
return e;
|
||||
}
|
||||
return new CopilotProviderSideError({
|
||||
provider: this.type,
|
||||
kind: 'unexpected_response',
|
||||
message: e?.message || 'Unexpected cloudflare workers ai response',
|
||||
});
|
||||
}
|
||||
|
||||
private createNativeConfig(
|
||||
execution?: CopilotProviderExecution
|
||||
): LlmBackendConfig {
|
||||
const config = this.getConfig(execution);
|
||||
return {
|
||||
base_url: this.resolveBaseUrl(execution),
|
||||
auth_token: config.apiToken,
|
||||
};
|
||||
}
|
||||
|
||||
private resolveBaseUrl(execution?: CopilotProviderExecution) {
|
||||
const config = this.getConfig(execution);
|
||||
if (config.baseURL) {
|
||||
return config.baseURL.replace(/\/v1\/?$/, '').replace(/\/$/, '');
|
||||
}
|
||||
const accountId = config.accountId ?? '';
|
||||
return `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai`;
|
||||
}
|
||||
|
||||
override getDriverSpec(): ProviderDriverSpec {
|
||||
return {
|
||||
createBackendConfig: execution => this.createNativeConfig(execution),
|
||||
mapError: error => this.handleError(error),
|
||||
structured: false,
|
||||
embedding: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,527 +0,0 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { CopilotQuotaExceeded } from '../../../base';
|
||||
import { ServerFeature, ServerService } from '../../../core';
|
||||
import { type CopilotAccessContext, CopilotAccessPolicy } from '../access';
|
||||
import type { RequiredStructuredOutputContract } from '../runtime/contracts';
|
||||
import { getProviderRuntimeHost } from '../runtime/provider-runtime-context';
|
||||
import type { CopilotProvider } from './provider';
|
||||
import {
|
||||
buildProviderRegistry,
|
||||
type CopilotProviderRegistry,
|
||||
type NormalizedCopilotProviderProfile,
|
||||
resolveModel,
|
||||
stripProviderPrefix,
|
||||
} from './provider-registry';
|
||||
import type {
|
||||
CopilotProviderExecution,
|
||||
PreparedNativeEmbeddingExecution,
|
||||
PreparedNativeExecution,
|
||||
PreparedNativeImageExecution,
|
||||
PreparedNativeRerankExecution,
|
||||
PreparedNativeStructuredExecution,
|
||||
} from './provider-runtime-contract';
|
||||
import { CopilotProviderRegistryService } from './registry-service';
|
||||
import {
|
||||
type CopilotChatOptions,
|
||||
type CopilotEmbeddingOptions,
|
||||
type CopilotImageOptions,
|
||||
CopilotProviderType,
|
||||
type CopilotRerankRequest,
|
||||
type CopilotStructuredOptions,
|
||||
ModelFullConditions,
|
||||
ModelOutputType,
|
||||
type PromptMessage,
|
||||
} from './types';
|
||||
|
||||
export type ResolvedCopilotProvider = {
|
||||
providerId: string;
|
||||
provider: CopilotProvider;
|
||||
execution: CopilotProviderExecution;
|
||||
profile: NormalizedCopilotProviderProfile;
|
||||
rawModelId?: string;
|
||||
modelId?: string;
|
||||
explicitProviderId?: string;
|
||||
prepared?: PreparedNativeExecution;
|
||||
preparedStructured?: PreparedNativeStructuredExecution;
|
||||
preparedEmbedding?: PreparedNativeEmbeddingExecution;
|
||||
preparedRerank?: PreparedNativeRerankExecution;
|
||||
preparedImage?: PreparedNativeImageExecution;
|
||||
};
|
||||
|
||||
type RoutePreparationResult = Partial<
|
||||
Pick<
|
||||
ResolvedCopilotProvider,
|
||||
| 'prepared'
|
||||
| 'preparedStructured'
|
||||
| 'preparedEmbedding'
|
||||
| 'preparedRerank'
|
||||
| 'preparedImage'
|
||||
| 'modelId'
|
||||
>
|
||||
>;
|
||||
|
||||
type EffectiveProviderRegistry = {
|
||||
byokRegistry: CopilotProviderRegistry;
|
||||
quotaBackedRegistry: CopilotProviderRegistry;
|
||||
quotaBackedRoutesAvailable: boolean;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CopilotProviderFactory {
|
||||
constructor(
|
||||
private readonly server: ServerService,
|
||||
private readonly registries: CopilotProviderRegistryService,
|
||||
private readonly access: CopilotAccessPolicy
|
||||
) {}
|
||||
|
||||
private readonly logger = new Logger(CopilotProviderFactory.name);
|
||||
|
||||
readonly #providers = new Map<string, CopilotProvider>();
|
||||
readonly #providerIdsByType = new Map<CopilotProviderType, Set<string>>();
|
||||
|
||||
private getRegistry() {
|
||||
return this.registries.getRegistry();
|
||||
}
|
||||
|
||||
private getProviderByProfile(
|
||||
providerId: string,
|
||||
profile: NormalizedCopilotProviderProfile
|
||||
) {
|
||||
return (
|
||||
this.#providers.get(providerId) ??
|
||||
Array.from(this.#providerIdsByType.get(profile.type) ?? [])
|
||||
.map(id => this.#providers.get(id))
|
||||
.find((provider): provider is CopilotProvider => !!provider)
|
||||
);
|
||||
}
|
||||
|
||||
private providerAvailable(
|
||||
providerId: string,
|
||||
profile: NormalizedCopilotProviderProfile
|
||||
) {
|
||||
return !!this.getProviderByProfile(providerId, profile);
|
||||
}
|
||||
|
||||
private getAvailableProviderIds(registry: CopilotProviderRegistry) {
|
||||
return Array.from(registry.profiles.entries())
|
||||
.filter(([providerId, profile]) =>
|
||||
this.providerAvailable(providerId, profile)
|
||||
)
|
||||
.map(([providerId]) => providerId);
|
||||
}
|
||||
|
||||
private getPreferredProviderIds(
|
||||
registry: CopilotProviderRegistry,
|
||||
type?: CopilotProviderType
|
||||
) {
|
||||
if (!type) return undefined;
|
||||
return registry.byType.get(type)?.filter(providerId => {
|
||||
const profile = registry.profiles.get(providerId);
|
||||
return profile ? this.providerAvailable(providerId, profile) : false;
|
||||
});
|
||||
}
|
||||
|
||||
private normalizeCond(
|
||||
registry: CopilotProviderRegistry,
|
||||
providerId: string,
|
||||
cond: ModelFullConditions
|
||||
): ModelFullConditions {
|
||||
const modelId = stripProviderPrefix(registry, providerId, cond.modelId);
|
||||
return { ...cond, modelId };
|
||||
}
|
||||
|
||||
private async getEffectiveRegistry(
|
||||
context: CopilotAccessContext = {}
|
||||
): Promise<EffectiveProviderRegistry> {
|
||||
const quotaBackedRegistry = this.getRegistry();
|
||||
const routeAccess = await this.access.resolveRouteAccess(context);
|
||||
|
||||
return {
|
||||
byokRegistry: buildProviderRegistry({
|
||||
profiles: routeAccess.byokProfiles,
|
||||
defaults: {},
|
||||
}),
|
||||
quotaBackedRegistry,
|
||||
quotaBackedRoutesAvailable: routeAccess.quotaBackedRoutesAvailable,
|
||||
};
|
||||
}
|
||||
|
||||
private getRequestContext(
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions
|
||||
): CopilotAccessContext {
|
||||
return {
|
||||
userId: options?.user,
|
||||
workspaceId: options?.workspace,
|
||||
byokLeaseId: options?.byokLeaseId,
|
||||
featureKind: options?.featureKind,
|
||||
quotaBackedRoutesAllowed: options?.quotaBackedRoutesAllowed,
|
||||
};
|
||||
}
|
||||
|
||||
private filterPreparedRoutes(routes: Array<ResolvedCopilotProvider | null>) {
|
||||
return routes.filter(
|
||||
(route): route is ResolvedCopilotProvider => route !== null
|
||||
);
|
||||
}
|
||||
|
||||
private async prepareResolvedRoutes(
|
||||
routes: ResolvedCopilotProvider[],
|
||||
prepare: (
|
||||
route: ResolvedCopilotProvider
|
||||
) => Promise<RoutePreparationResult | null | undefined>
|
||||
) {
|
||||
const preparedRoutes = await Promise.all(
|
||||
routes.map(async route => {
|
||||
const prepared = await prepare(route);
|
||||
return prepared ? { ...route, ...prepared } : null;
|
||||
})
|
||||
);
|
||||
|
||||
return this.filterPreparedRoutes(preparedRoutes);
|
||||
}
|
||||
|
||||
async resolveProvider(
|
||||
cond: ModelFullConditions,
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {},
|
||||
context: CopilotAccessContext = {}
|
||||
): Promise<ResolvedCopilotProvider | null> {
|
||||
return (await this.resolveRoutes(cond, filter, context))[0] ?? null;
|
||||
}
|
||||
|
||||
async resolveRoutes(
|
||||
cond: ModelFullConditions,
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {},
|
||||
context: CopilotAccessContext = {}
|
||||
): Promise<ResolvedCopilotProvider[]> {
|
||||
this.logger.debug(
|
||||
`Resolving copilot provider for output type: ${cond.outputType}`
|
||||
);
|
||||
const { byokRegistry, quotaBackedRegistry, quotaBackedRoutesAvailable } =
|
||||
await this.getEffectiveRegistry(context);
|
||||
const byokRoutes = await this.resolveRoutesFromRegistry(
|
||||
byokRegistry,
|
||||
cond,
|
||||
filter
|
||||
);
|
||||
const resolved = byokRoutes.length
|
||||
? byokRoutes
|
||||
: quotaBackedRoutesAvailable
|
||||
? await this.resolveRoutesFromRegistry(
|
||||
quotaBackedRegistry,
|
||||
cond,
|
||||
filter
|
||||
)
|
||||
: [];
|
||||
for (const route of resolved) {
|
||||
this.logger.debug(
|
||||
`Copilot provider candidate found: ${route.provider.type} (${route.providerId})`
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
!resolved.length &&
|
||||
!quotaBackedRoutesAvailable &&
|
||||
context.quotaBackedRoutesAllowed !== false
|
||||
) {
|
||||
const quotaBackedRoutes = await this.resolveRoutesFromRegistry(
|
||||
quotaBackedRegistry,
|
||||
cond,
|
||||
filter
|
||||
);
|
||||
if (quotaBackedRoutes.length) {
|
||||
throw new CopilotQuotaExceeded();
|
||||
}
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
private async resolveRoutesFromRegistry(
|
||||
registry: CopilotProviderRegistry,
|
||||
cond: ModelFullConditions,
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<ResolvedCopilotProvider[]> {
|
||||
const route = resolveModel({
|
||||
registry,
|
||||
modelId: cond.modelId,
|
||||
outputType: cond.outputType,
|
||||
availableProviderIds: this.getAvailableProviderIds(registry),
|
||||
preferredProviderIds: this.getPreferredProviderIds(
|
||||
registry,
|
||||
filter.prefer
|
||||
),
|
||||
});
|
||||
|
||||
const resolved: ResolvedCopilotProvider[] = [];
|
||||
for (const providerId of route.candidateProviderIds) {
|
||||
const profile = registry.profiles.get(providerId);
|
||||
const provider = profile
|
||||
? this.getProviderByProfile(providerId, profile)
|
||||
: undefined;
|
||||
if (!provider || !profile) continue;
|
||||
|
||||
const normalizedCond = this.normalizeCond(registry, providerId, cond);
|
||||
if (
|
||||
normalizedCond.modelId &&
|
||||
profile.models?.length &&
|
||||
!profile.models.includes(normalizedCond.modelId)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const execution = { providerId, profile };
|
||||
const matched = await provider.match(normalizedCond, execution);
|
||||
if (!matched) continue;
|
||||
|
||||
resolved.push({
|
||||
providerId,
|
||||
provider,
|
||||
execution,
|
||||
profile,
|
||||
rawModelId: route.rawModelId,
|
||||
modelId: normalizedCond.modelId,
|
||||
explicitProviderId: route.explicitProviderId,
|
||||
});
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async prepareRoutes(
|
||||
kind: 'text' | 'streamText' | 'streamObject',
|
||||
cond: ModelFullConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotChatOptions = {},
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<ResolvedCopilotProvider[]> {
|
||||
const routes = await this.resolveRoutes(
|
||||
cond,
|
||||
filter,
|
||||
this.getRequestContext(options)
|
||||
);
|
||||
return await this.prepareResolvedRoutes(routes, async route => {
|
||||
const prepared = await getProviderRuntimeHost(
|
||||
route.provider
|
||||
).prepare.chat(
|
||||
kind,
|
||||
{ ...cond, modelId: route.modelId },
|
||||
messages,
|
||||
options,
|
||||
route.execution
|
||||
);
|
||||
const normalizedPrepared = prepared?.route ? prepared : undefined;
|
||||
if (!normalizedPrepared) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
modelId: normalizedPrepared.route.model,
|
||||
prepared: normalizedPrepared,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async prepareStructuredRoutes(
|
||||
cond: ModelFullConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotStructuredOptions = {},
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {},
|
||||
responseContract?: RequiredStructuredOutputContract
|
||||
): Promise<ResolvedCopilotProvider[]> {
|
||||
const routes = await this.resolveRoutes(
|
||||
cond,
|
||||
filter,
|
||||
this.getRequestContext(options)
|
||||
);
|
||||
return await this.prepareResolvedRoutes(routes, async route => {
|
||||
const preparedStructured =
|
||||
(await getProviderRuntimeHost(route.provider).prepare.structured(
|
||||
{ ...cond, modelId: route.modelId },
|
||||
messages,
|
||||
options,
|
||||
responseContract,
|
||||
route.execution
|
||||
)) ?? undefined;
|
||||
if (!preparedStructured) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
modelId: preparedStructured.route.model,
|
||||
preparedStructured,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async prepareEmbeddingRoutes(
|
||||
modelId: string,
|
||||
input: string | string[],
|
||||
options: CopilotEmbeddingOptions = {}
|
||||
): Promise<ResolvedCopilotProvider[]> {
|
||||
const routes = await this.resolveRoutes(
|
||||
{ modelId, outputType: ModelOutputType.Embedding },
|
||||
{},
|
||||
{
|
||||
...this.getRequestContext(options),
|
||||
featureKind: options?.featureKind ?? 'embedding',
|
||||
}
|
||||
);
|
||||
return await this.prepareResolvedRoutes(routes, async route => {
|
||||
const preparedEmbedding =
|
||||
(await getProviderRuntimeHost(route.provider).prepare.embedding(
|
||||
{ modelId: route.modelId },
|
||||
input,
|
||||
options,
|
||||
route.execution
|
||||
)) ?? undefined;
|
||||
if (!preparedEmbedding) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
modelId: preparedEmbedding.route.model,
|
||||
preparedEmbedding,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async prepareRerankRoutes(
|
||||
modelId: string,
|
||||
request: CopilotRerankRequest,
|
||||
options: CopilotChatOptions = {}
|
||||
): Promise<ResolvedCopilotProvider[]> {
|
||||
const routes = await this.resolveRoutes(
|
||||
{
|
||||
modelId,
|
||||
outputType: ModelOutputType.Rerank,
|
||||
},
|
||||
{},
|
||||
{ ...this.getRequestContext(options), featureKind: 'rerank' }
|
||||
);
|
||||
return await this.prepareResolvedRoutes(routes, async route => {
|
||||
const preparedRerank =
|
||||
(await getProviderRuntimeHost(route.provider).prepare.rerank(
|
||||
{ modelId: route.modelId },
|
||||
request,
|
||||
options,
|
||||
route.execution
|
||||
)) ?? undefined;
|
||||
if (!preparedRerank) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
modelId: preparedRerank.route.model,
|
||||
preparedRerank,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async prepareImageRoutes(
|
||||
cond: ModelFullConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotImageOptions = {},
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<ResolvedCopilotProvider[]> {
|
||||
const routes = await this.resolveRoutes(cond, filter, {
|
||||
...this.getRequestContext(options),
|
||||
featureKind: options?.featureKind ?? 'image',
|
||||
});
|
||||
return await this.prepareResolvedRoutes(routes, async route => {
|
||||
const preparedImage =
|
||||
(await getProviderRuntimeHost(route.provider).prepare.image(
|
||||
{ ...cond, modelId: route.modelId },
|
||||
messages,
|
||||
options,
|
||||
route.execution
|
||||
)) ?? undefined;
|
||||
if (!preparedImage) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
modelId: preparedImage.route.model,
|
||||
preparedImage,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async getProvider(
|
||||
cond: ModelFullConditions,
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<CopilotProvider | null> {
|
||||
return (await this.resolveProvider(cond, filter))?.provider ?? null;
|
||||
}
|
||||
|
||||
async getProviderByModel(
|
||||
modelId: string,
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<CopilotProvider | null> {
|
||||
this.logger.debug(`Resolving copilot provider for model: ${modelId}`);
|
||||
return this.getProvider({ modelId }, filter);
|
||||
}
|
||||
|
||||
register(providerId: string, provider: CopilotProvider) {
|
||||
const existed = this.#providers.get(providerId);
|
||||
if (existed?.type && existed.type !== provider.type) {
|
||||
const ids = this.#providerIdsByType.get(existed.type);
|
||||
ids?.delete(providerId);
|
||||
if (!ids?.size) {
|
||||
this.#providerIdsByType.delete(existed.type);
|
||||
}
|
||||
}
|
||||
|
||||
this.#providers.set(providerId, provider);
|
||||
|
||||
const ids = this.#providerIdsByType.get(provider.type) ?? new Set<string>();
|
||||
ids.add(providerId);
|
||||
this.#providerIdsByType.set(provider.type, ids);
|
||||
|
||||
this.logger.log(
|
||||
`Copilot provider [${provider.type}] registered as [${providerId}].`
|
||||
);
|
||||
this.server.enableFeature(ServerFeature.Copilot);
|
||||
}
|
||||
|
||||
unregister(providerId: string, provider: CopilotProvider) {
|
||||
const existed = this.#providers.get(providerId);
|
||||
if (!existed || existed !== provider) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.#providers.delete(providerId);
|
||||
|
||||
const ids = this.#providerIdsByType.get(provider.type);
|
||||
ids?.delete(providerId);
|
||||
if (!ids?.size) {
|
||||
this.#providerIdsByType.delete(provider.type);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Copilot provider [${provider.type}] unregistered from [${providerId}].`
|
||||
);
|
||||
if (this.#providers.size === 0) {
|
||||
this.server.disableFeature(ServerFeature.Copilot);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { CopilotProviderSideError, UserFriendlyError } from '../../../base';
|
||||
import { CopilotProvider } from './provider';
|
||||
import type {
|
||||
CopilotProviderExecution,
|
||||
ProviderDriverSpec,
|
||||
} from './provider-runtime-contract';
|
||||
import { CopilotProviderType } from './types';
|
||||
|
||||
export type FalConfig = {
|
||||
apiKey: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class FalProvider extends CopilotProvider<FalConfig> {
|
||||
override type = CopilotProviderType.FAL;
|
||||
|
||||
protected resolveModelBackendKind() {
|
||||
return 'fal' as const;
|
||||
}
|
||||
|
||||
override configured(execution?: CopilotProviderExecution): boolean {
|
||||
return !!this.getConfig(execution).apiKey;
|
||||
}
|
||||
|
||||
private createNativeConfig(execution?: CopilotProviderExecution) {
|
||||
return {
|
||||
base_url: 'https://fal.run',
|
||||
auth_token: this.getConfig(execution).apiKey,
|
||||
};
|
||||
}
|
||||
|
||||
override getDriverSpec(): ProviderDriverSpec {
|
||||
return {
|
||||
createBackendConfig: execution => this.createNativeConfig(execution),
|
||||
mapError: error => this.handleError(error),
|
||||
chat: false,
|
||||
structured: false,
|
||||
embedding: false,
|
||||
rerank: false,
|
||||
image: {},
|
||||
};
|
||||
}
|
||||
|
||||
private handleError(e: any) {
|
||||
if (e instanceof UserFriendlyError) {
|
||||
// pass through user friendly errors
|
||||
return e;
|
||||
} else {
|
||||
const error = new CopilotProviderSideError({
|
||||
provider: this.type,
|
||||
kind: 'unexpected_response',
|
||||
message: e?.message || 'Unexpected fal response',
|
||||
});
|
||||
return error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
|
||||
import { Inject } from '@nestjs/common';
|
||||
import { ZodError } from 'zod';
|
||||
|
||||
import {
|
||||
CopilotProviderSideError,
|
||||
OneMB,
|
||||
UserFriendlyError,
|
||||
} from '../../../../base';
|
||||
import {
|
||||
isInvalidStructuredOutputError,
|
||||
type LlmBackendConfig,
|
||||
llmResolveRequestIntentOptions,
|
||||
} from '../../../../native';
|
||||
import {
|
||||
admittedAttachmentToPromptAttachment,
|
||||
AttachmentAdmissionHost,
|
||||
} from '../../runtime/hosts/attachment-admission';
|
||||
import {
|
||||
planAdmittedAttachmentMaterialization,
|
||||
planHostUrlAttachmentMaterialization,
|
||||
} from '../../runtime/hosts/attachment-materialization-planner';
|
||||
import { AttachmentMaterializer } from '../../runtime/hosts/attachment-materializer';
|
||||
import { CopilotProvider } from '../provider';
|
||||
import { hasProviderModelBehaviorFlag } from '../provider-model-runtime';
|
||||
import {
|
||||
type CopilotProviderExecution,
|
||||
type ProviderDriverSpec,
|
||||
} from '../provider-runtime-contract';
|
||||
import type { PromptAttachment, PromptMessage } from '../types';
|
||||
import { promptAttachmentMimeType, promptAttachmentToUrl } from '../utils';
|
||||
|
||||
export const DEFAULT_DIMENSIONS = 256;
|
||||
const GEMINI_REMOTE_ATTACHMENT_MAX_BYTES = 64 * OneMB;
|
||||
const TRUSTED_ATTACHMENT_HOST_SUFFIXES = ['cdn.affine.pro'];
|
||||
const GEMINI_RETRY_INITIAL_DELAY_MS = 2_000;
|
||||
|
||||
function normalizeMimeType(mediaType?: string) {
|
||||
return mediaType?.split(';', 1)[0]?.trim() || 'application/octet-stream';
|
||||
}
|
||||
|
||||
export abstract class GeminiProvider<T> extends CopilotProvider<T> {
|
||||
@Inject() protected readonly attachmentMaterializer!: AttachmentMaterializer;
|
||||
@Inject()
|
||||
protected readonly attachmentAdmissionHost?: AttachmentAdmissionHost;
|
||||
|
||||
protected resolveModelBackendKind() {
|
||||
return this.type === 'geminiVertex'
|
||||
? ('gemini_vertex' as const)
|
||||
: ('gemini_api' as const);
|
||||
}
|
||||
|
||||
protected abstract createNativeConfig(
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<LlmBackendConfig>;
|
||||
|
||||
private handleError(e: any) {
|
||||
if (e instanceof UserFriendlyError) {
|
||||
return e;
|
||||
} else {
|
||||
return new CopilotProviderSideError({
|
||||
provider: this.type,
|
||||
kind: 'unexpected_response',
|
||||
message: e?.message || 'Unexpected google response',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private getAttachmentAdmissionHost() {
|
||||
return (
|
||||
this.attachmentAdmissionHost ??
|
||||
new AttachmentAdmissionHost(this.attachmentMaterializer)
|
||||
);
|
||||
}
|
||||
|
||||
protected async prepareMessages(
|
||||
messages: PromptMessage[],
|
||||
backendConfig: LlmBackendConfig,
|
||||
options?: {
|
||||
signal?: AbortSignal;
|
||||
user?: string;
|
||||
workspace?: string;
|
||||
session?: string;
|
||||
}
|
||||
): Promise<PromptMessage[]> {
|
||||
const prepared: PromptMessage[] = [];
|
||||
|
||||
for (const message of messages) {
|
||||
options?.signal?.throwIfAborted();
|
||||
if (!Array.isArray(message.attachments) || !message.attachments.length) {
|
||||
prepared.push(message);
|
||||
continue;
|
||||
}
|
||||
|
||||
const attachments: PromptAttachment[] = [];
|
||||
let changed = false;
|
||||
for (const attachment of message.attachments) {
|
||||
options?.signal?.throwIfAborted();
|
||||
const rawUrl = promptAttachmentToUrl(attachment);
|
||||
if (!rawUrl || rawUrl.startsWith('data:')) {
|
||||
attachments.push(attachment);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
new URL(rawUrl);
|
||||
} catch {
|
||||
attachments.push(attachment);
|
||||
continue;
|
||||
}
|
||||
|
||||
const declaredMimeType = promptAttachmentMimeType(
|
||||
attachment,
|
||||
typeof message.params?.mimetype === 'string'
|
||||
? message.params.mimetype
|
||||
: undefined
|
||||
);
|
||||
const referencePlan = await planHostUrlAttachmentMaterialization(
|
||||
'gemini',
|
||||
backendConfig,
|
||||
{
|
||||
attachmentId: rawUrl,
|
||||
url: rawUrl,
|
||||
expectedMime: declaredMimeType
|
||||
? normalizeMimeType(declaredMimeType)
|
||||
: undefined,
|
||||
maxSize: GEMINI_REMOTE_ATTACHMENT_MAX_BYTES,
|
||||
}
|
||||
);
|
||||
if (referencePlan.mode === 'remote_reference') {
|
||||
attachments.push(attachment);
|
||||
continue;
|
||||
}
|
||||
|
||||
const admitted =
|
||||
await this.getAttachmentAdmissionHost().admitPromptAttachment(
|
||||
attachment,
|
||||
{
|
||||
userId: options?.user ?? 'provider-runtime',
|
||||
workspaceId: options?.workspace ?? 'provider-runtime',
|
||||
sessionId: options?.session,
|
||||
signal: options?.signal,
|
||||
maxBytes: referencePlan.request.maxSize,
|
||||
trustedHostSuffixes: TRUSTED_ATTACHMENT_HOST_SUFFIXES,
|
||||
}
|
||||
);
|
||||
const materialization = planAdmittedAttachmentMaterialization(admitted);
|
||||
attachments.push(
|
||||
materialization.mode === 'inline'
|
||||
? materialization.attachment
|
||||
: admittedAttachmentToPromptAttachment(admitted)
|
||||
);
|
||||
changed = true;
|
||||
}
|
||||
|
||||
prepared.push(changed ? { ...message, attachments } : message);
|
||||
}
|
||||
|
||||
return prepared;
|
||||
}
|
||||
|
||||
protected async waitForStructuredRetry(
|
||||
delayMs: number,
|
||||
signal?: AbortSignal
|
||||
) {
|
||||
await delay(delayMs, undefined, signal ? { signal } : undefined);
|
||||
}
|
||||
|
||||
override getDriverSpec(): ProviderDriverSpec {
|
||||
return {
|
||||
createBackendConfig: execution => this.createNativeConfig(execution),
|
||||
mapError: error => this.handleError(error),
|
||||
chat: {
|
||||
prepareMessages: async context =>
|
||||
await this.prepareMessages(
|
||||
context.input.messages,
|
||||
context.backendConfig,
|
||||
context.options
|
||||
),
|
||||
resolveRequestOptions: async context => {
|
||||
const requestIntent = await llmResolveRequestIntentOptions({
|
||||
protocol: context.protocol,
|
||||
backendConfig: context.backendConfig,
|
||||
reasoning: {
|
||||
enabled: context.options.reasoning,
|
||||
supported:
|
||||
hasProviderModelBehaviorFlag(
|
||||
context.model,
|
||||
'reasoning_medium'
|
||||
) ||
|
||||
hasProviderModelBehaviorFlag(context.model, 'reasoning_high'),
|
||||
effort: hasProviderModelBehaviorFlag(
|
||||
context.model,
|
||||
'reasoning_high'
|
||||
)
|
||||
? 'high'
|
||||
: 'medium',
|
||||
includeReasoning:
|
||||
hasProviderModelBehaviorFlag(
|
||||
context.model,
|
||||
'reasoning_medium'
|
||||
) ||
|
||||
hasProviderModelBehaviorFlag(context.model, 'reasoning_high'),
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
attachmentCapability: this.getAttachCapability(
|
||||
context.model,
|
||||
context.outputType
|
||||
),
|
||||
include: requestIntent.include,
|
||||
reasoning: requestIntent.reasoning,
|
||||
};
|
||||
},
|
||||
},
|
||||
structured: {
|
||||
prepareMessages: (inputMessages, backendConfig, structuredOptions) =>
|
||||
this.prepareMessages(inputMessages, backendConfig, structuredOptions),
|
||||
shouldRetry: async ({ error, attempt, options: structuredOptions }) => {
|
||||
const isParsingError =
|
||||
isInvalidStructuredOutputError(error) || error instanceof ZodError;
|
||||
const retryableError =
|
||||
isParsingError || !(error instanceof UserFriendlyError);
|
||||
const maxRetries = Math.max(structuredOptions.maxRetries ?? 3, 0);
|
||||
if (!retryableError || attempt >= maxRetries) {
|
||||
return false;
|
||||
}
|
||||
if (!isParsingError) {
|
||||
await this.waitForStructuredRetry(
|
||||
GEMINI_RETRY_INITIAL_DELAY_MS * 2 ** attempt,
|
||||
structuredOptions.signal
|
||||
);
|
||||
}
|
||||
return true;
|
||||
},
|
||||
},
|
||||
embedding: {
|
||||
defaultDimensions: DEFAULT_DIMENSIONS,
|
||||
taskType: 'RETRIEVAL_DOCUMENT',
|
||||
},
|
||||
rerank: false,
|
||||
image: {
|
||||
prepareMessages: (inputMessages, backendConfig, imageOptions) =>
|
||||
this.prepareMessages(inputMessages, backendConfig, imageOptions),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import type { LlmBackendConfig } from '../../../../native';
|
||||
import type { CopilotProviderExecution } from '../provider-runtime-contract';
|
||||
import { CopilotProviderType } from '../types';
|
||||
import { GeminiProvider } from './gemini';
|
||||
|
||||
export type GeminiGenerativeConfig = {
|
||||
apiKey: string;
|
||||
baseURL?: string;
|
||||
};
|
||||
|
||||
export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeConfig> {
|
||||
override readonly type = CopilotProviderType.Gemini;
|
||||
override configured(execution?: CopilotProviderExecution): boolean {
|
||||
return !!this.getConfig(execution).apiKey;
|
||||
}
|
||||
|
||||
protected override async createNativeConfig(
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<LlmBackendConfig> {
|
||||
const config = this.getConfig(execution);
|
||||
return {
|
||||
base_url: (
|
||||
config.baseURL || 'https://generativelanguage.googleapis.com/v1beta'
|
||||
).replace(/\/$/, ''),
|
||||
auth_token: config.apiKey,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
export * from './generative';
|
||||
export * from './vertex';
|
||||
@@ -1,34 +0,0 @@
|
||||
import type { LlmBackendConfig } from '../../../../native';
|
||||
import type { CopilotProviderExecution } from '../provider-runtime-contract';
|
||||
import { CopilotProviderType } from '../types';
|
||||
import {
|
||||
getGoogleAuth,
|
||||
getVertexGoogleBaseUrl,
|
||||
type VertexProviderConfig,
|
||||
} from '../utils';
|
||||
import { GeminiProvider } from './gemini';
|
||||
|
||||
export type GeminiVertexConfig = VertexProviderConfig;
|
||||
|
||||
export class GeminiVertexProvider extends GeminiProvider<GeminiVertexConfig> {
|
||||
override readonly type = CopilotProviderType.GeminiVertex;
|
||||
override configured(execution?: CopilotProviderExecution): boolean {
|
||||
const config = this.getConfig(execution);
|
||||
return !!getVertexGoogleBaseUrl(config) && !!config.googleAuthOptions;
|
||||
}
|
||||
protected async resolveVertexAuth(execution?: CopilotProviderExecution) {
|
||||
return await getGoogleAuth(this.getConfig(execution), 'google');
|
||||
}
|
||||
|
||||
protected override async createNativeConfig(
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<LlmBackendConfig> {
|
||||
const auth = await this.resolveVertexAuth(execution);
|
||||
const { Authorization: authHeader } = auth.headers();
|
||||
|
||||
return {
|
||||
base_url: auth.baseUrl || '',
|
||||
auth_token: authHeader.replace(/^Bearer\s+/i, ''),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
export {
|
||||
AnthropicOfficialProvider,
|
||||
AnthropicVertexProvider,
|
||||
} from './anthropic';
|
||||
export { CloudflareWorkersAIProvider } from './cloudflare';
|
||||
export { CopilotProviderFactory } from './factory';
|
||||
export { FalProvider } from './fal';
|
||||
export { GeminiGenerativeProvider, GeminiVertexProvider } from './gemini';
|
||||
export { CopilotProviderLifecycleService } from './lifecycle-service';
|
||||
export { OpenAIProvider } from './openai';
|
||||
export type { CopilotProvider } from './provider';
|
||||
export { CopilotProviders } from './provider-tokens';
|
||||
export { CopilotProviderRegistryService } from './registry-service';
|
||||
export * from './types';
|
||||
@@ -1,90 +0,0 @@
|
||||
import { Injectable, Type } from '@nestjs/common';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
|
||||
import { OnEvent } from '../../../base';
|
||||
import { CopilotProviderFactory } from './factory';
|
||||
import type { CopilotProvider } from './provider';
|
||||
import type { CopilotProviderExecution } from './provider-runtime-contract';
|
||||
import { CopilotProviders } from './provider-tokens';
|
||||
import { CopilotProviderRegistryService } from './registry-service';
|
||||
|
||||
@Injectable()
|
||||
export class CopilotProviderLifecycleService {
|
||||
private readonly registeredByProvider = new WeakMap<
|
||||
CopilotProvider,
|
||||
Set<string>
|
||||
>();
|
||||
|
||||
constructor(
|
||||
private readonly moduleRef: ModuleRef,
|
||||
private readonly factory: CopilotProviderFactory,
|
||||
private readonly registries: CopilotProviderRegistryService
|
||||
) {}
|
||||
|
||||
private getProviders(): CopilotProvider[] {
|
||||
return CopilotProviders.flatMap(token => {
|
||||
const provider = this.moduleRef.get(token as Type<CopilotProvider>, {
|
||||
strict: false,
|
||||
});
|
||||
return provider ? [provider] : [];
|
||||
});
|
||||
}
|
||||
|
||||
private getRegisteredProviderIds(provider: CopilotProvider) {
|
||||
const current = this.registeredByProvider.get(provider);
|
||||
if (current) {
|
||||
return current;
|
||||
}
|
||||
|
||||
const next = new Set<string>();
|
||||
this.registeredByProvider.set(provider, next);
|
||||
return next;
|
||||
}
|
||||
|
||||
private async syncProvider(provider: CopilotProvider) {
|
||||
const registry = this.registries.getRegistry();
|
||||
const configuredIds = new Set<string>();
|
||||
|
||||
for (const providerId of registry.byType.get(provider.type) ?? []) {
|
||||
const profile = registry.profiles.get(providerId);
|
||||
if (!profile) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const execution: CopilotProviderExecution = { providerId, profile };
|
||||
if (!provider.configured(execution)) {
|
||||
this.factory.unregister(providerId, provider);
|
||||
continue;
|
||||
}
|
||||
|
||||
configuredIds.add(providerId);
|
||||
this.factory.register(providerId, provider);
|
||||
}
|
||||
|
||||
const previous = this.getRegisteredProviderIds(provider);
|
||||
for (const providerId of previous) {
|
||||
if (!configuredIds.has(providerId)) {
|
||||
this.factory.unregister(providerId, provider);
|
||||
}
|
||||
}
|
||||
this.registeredByProvider.set(provider, configuredIds);
|
||||
}
|
||||
|
||||
async syncProviders() {
|
||||
for (const provider of this.getProviders()) {
|
||||
await this.syncProvider(provider);
|
||||
}
|
||||
}
|
||||
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
await this.syncProviders();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
async onConfigChanged(event: Events['config.changed']) {
|
||||
if ('copilot' in event.updates) {
|
||||
await this.syncProviders();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
import { Inject } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
CopilotProviderSideError,
|
||||
OneMB,
|
||||
UserFriendlyError,
|
||||
} from '../../../base';
|
||||
import {
|
||||
type LlmBackendConfig,
|
||||
llmResolveRequestIntentOptions,
|
||||
} from '../../../native';
|
||||
import {
|
||||
admittedAttachmentToPromptAttachment,
|
||||
AttachmentAdmissionHost,
|
||||
} from '../runtime/hosts/attachment-admission';
|
||||
import { AttachmentMaterializer } from '../runtime/hosts/attachment-materializer';
|
||||
import { CopilotProvider } from './provider';
|
||||
import { hasProviderModelBehaviorFlag } from './provider-model-runtime';
|
||||
import type {
|
||||
CopilotProviderExecution,
|
||||
ProviderDriverSpec,
|
||||
} from './provider-runtime-contract';
|
||||
import {
|
||||
CopilotProviderType,
|
||||
type PromptAttachment,
|
||||
type PromptMessage,
|
||||
} from './types';
|
||||
import { promptAttachmentToUrl } from './utils';
|
||||
|
||||
export const DEFAULT_DIMENSIONS = 256;
|
||||
|
||||
export type OpenAIConfig = {
|
||||
apiKey: string;
|
||||
baseURL?: string;
|
||||
oldApiStyle?: boolean;
|
||||
};
|
||||
|
||||
export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
|
||||
readonly type = CopilotProviderType.OpenAI;
|
||||
@Inject() protected readonly attachmentMaterializer!: AttachmentMaterializer;
|
||||
@Inject()
|
||||
protected readonly attachmentAdmissionHost?: AttachmentAdmissionHost;
|
||||
|
||||
protected resolveModelBackendKind(execution?: CopilotProviderExecution) {
|
||||
return this.getConfig(execution).oldApiStyle
|
||||
? ('openai_chat' as const)
|
||||
: ('openai_responses' as const);
|
||||
}
|
||||
|
||||
override configured(execution?: CopilotProviderExecution): boolean {
|
||||
return !!this.getConfig(execution).apiKey;
|
||||
}
|
||||
|
||||
private handleError(e: any) {
|
||||
if (e instanceof UserFriendlyError) {
|
||||
return e;
|
||||
}
|
||||
return new CopilotProviderSideError({
|
||||
provider: this.type,
|
||||
kind: 'unexpected_response',
|
||||
message: e?.message || 'Unexpected openai response',
|
||||
});
|
||||
}
|
||||
|
||||
protected createNativeConfig(
|
||||
execution?: CopilotProviderExecution
|
||||
): LlmBackendConfig {
|
||||
const config = this.getConfig(execution);
|
||||
const baseUrl = config.baseURL || 'https://api.openai.com/v1';
|
||||
return {
|
||||
base_url: baseUrl.replace(/\/v1\/?$/, ''),
|
||||
auth_token: config.apiKey,
|
||||
};
|
||||
}
|
||||
|
||||
private getAttachmentAdmissionHost() {
|
||||
return (
|
||||
this.attachmentAdmissionHost ??
|
||||
new AttachmentAdmissionHost(this.attachmentMaterializer)
|
||||
);
|
||||
}
|
||||
|
||||
private async prepareImageMessages(
|
||||
messages: PromptMessage[],
|
||||
options: {
|
||||
signal?: AbortSignal;
|
||||
user?: string;
|
||||
workspace?: string;
|
||||
session?: string;
|
||||
}
|
||||
) {
|
||||
const prepared: PromptMessage[] = [];
|
||||
|
||||
for (const message of messages) {
|
||||
options.signal?.throwIfAborted();
|
||||
if (!Array.isArray(message.attachments) || !message.attachments.length) {
|
||||
prepared.push(message);
|
||||
continue;
|
||||
}
|
||||
|
||||
let changed = false;
|
||||
const attachments: PromptAttachment[] = [];
|
||||
for (const attachment of message.attachments) {
|
||||
options.signal?.throwIfAborted();
|
||||
const url = promptAttachmentToUrl(attachment);
|
||||
if (!url || url.startsWith('data:')) {
|
||||
attachments.push(attachment);
|
||||
continue;
|
||||
}
|
||||
|
||||
const admitted =
|
||||
await this.getAttachmentAdmissionHost().admitPromptAttachment(
|
||||
attachment,
|
||||
{
|
||||
userId: options.user ?? 'provider-runtime',
|
||||
workspaceId: options.workspace ?? 'provider-runtime',
|
||||
sessionId: options.session,
|
||||
signal: options.signal,
|
||||
maxBytes: 50 * OneMB,
|
||||
}
|
||||
);
|
||||
attachments.push(admittedAttachmentToPromptAttachment(admitted));
|
||||
changed = true;
|
||||
}
|
||||
|
||||
prepared.push(changed ? { ...message, attachments } : message);
|
||||
}
|
||||
|
||||
return prepared;
|
||||
}
|
||||
|
||||
override getDriverSpec(): ProviderDriverSpec {
|
||||
return {
|
||||
createBackendConfig: execution => this.createNativeConfig(execution),
|
||||
mapError: error => this.handleError(error),
|
||||
chat: {
|
||||
resolveRequestOptions: async context => {
|
||||
const requestIntent = await llmResolveRequestIntentOptions({
|
||||
protocol: context.protocol,
|
||||
backendConfig: context.backendConfig,
|
||||
include: context.options.webSearch ? ['citations'] : undefined,
|
||||
reasoning: {
|
||||
enabled: context.options.reasoning,
|
||||
supported: hasProviderModelBehaviorFlag(
|
||||
context.model,
|
||||
'reasoning_supported'
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
attachmentCapability: this.getAttachCapability(
|
||||
context.model,
|
||||
context.outputType
|
||||
),
|
||||
include: requestIntent.include,
|
||||
reasoning: requestIntent.reasoning,
|
||||
};
|
||||
},
|
||||
},
|
||||
structured: {},
|
||||
embedding: {
|
||||
defaultDimensions: DEFAULT_DIMENSIONS,
|
||||
taskType: 'RETRIEVAL_DOCUMENT',
|
||||
},
|
||||
image: {
|
||||
prepareMessages: async (messages, _backendConfig, options) =>
|
||||
await this.prepareImageMessages(messages, options),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import type { ProviderMiddlewareConfig } from '../config';
|
||||
import { CopilotProviderType } from './types';
|
||||
|
||||
const DEFAULT_NODE_TEXT_MIDDLEWARE: NonNullable<
|
||||
NonNullable<ProviderMiddlewareConfig['node']>['text']
|
||||
> = ['citation_footnote', 'callout'];
|
||||
|
||||
const DEFAULT_MIDDLEWARE_BY_TYPE: Record<
|
||||
CopilotProviderType,
|
||||
ProviderMiddlewareConfig
|
||||
> = {
|
||||
[CopilotProviderType.OpenAI]: {
|
||||
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
|
||||
},
|
||||
[CopilotProviderType.CloudflareWorkersAi]: {
|
||||
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
|
||||
},
|
||||
[CopilotProviderType.Anthropic]: {
|
||||
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
|
||||
},
|
||||
[CopilotProviderType.AnthropicVertex]: {
|
||||
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
|
||||
},
|
||||
[CopilotProviderType.Gemini]: {
|
||||
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
|
||||
},
|
||||
[CopilotProviderType.GeminiVertex]: {
|
||||
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
|
||||
},
|
||||
[CopilotProviderType.FAL]: {},
|
||||
};
|
||||
|
||||
function unique<T>(items: T[]) {
|
||||
return [...new Set(items)];
|
||||
}
|
||||
|
||||
function mergeArray<T>(base: T[] | undefined, override: T[] | undefined) {
|
||||
if (!base?.length && !override?.length) {
|
||||
return undefined;
|
||||
}
|
||||
return unique([...(base ?? []), ...(override ?? [])]);
|
||||
}
|
||||
|
||||
function compactMiddlewareSection<T extends Record<string, unknown>>(
|
||||
section: T
|
||||
): T | undefined {
|
||||
return Object.values(section).some(value => value !== undefined)
|
||||
? section
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function mergeProviderMiddleware(
|
||||
defaults: ProviderMiddlewareConfig,
|
||||
override?: ProviderMiddlewareConfig
|
||||
): ProviderMiddlewareConfig {
|
||||
return {
|
||||
rust: compactMiddlewareSection({
|
||||
request: mergeArray(defaults.rust?.request, override?.rust?.request),
|
||||
stream: mergeArray(defaults.rust?.stream, override?.rust?.stream),
|
||||
}),
|
||||
node: compactMiddlewareSection({
|
||||
text: mergeArray(defaults.node?.text, override?.node?.text),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveProviderMiddleware(
|
||||
type: CopilotProviderType,
|
||||
override?: ProviderMiddlewareConfig
|
||||
): ProviderMiddlewareConfig {
|
||||
const defaults = DEFAULT_MIDDLEWARE_BY_TYPE[type] ?? {};
|
||||
return mergeProviderMiddleware(defaults, override);
|
||||
}
|
||||
@@ -1,385 +0,0 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
import { CopilotPromptInvalid } from '../../../base';
|
||||
import {
|
||||
type LlmBackendConfig,
|
||||
llmInferPromptModelConditions,
|
||||
llmMatchModelCapabilities,
|
||||
llmMatchModelRegistry,
|
||||
type LlmProtocol,
|
||||
llmResolveModelRegistryVariant,
|
||||
} from '../../../native';
|
||||
import { applyPromptAttachmentMimeTypeHintForNative } from './attachments';
|
||||
import {
|
||||
type CopilotChatOptions,
|
||||
type CopilotImageOptions,
|
||||
type CopilotModelBackendKind,
|
||||
type CopilotProviderModel,
|
||||
type CopilotProviderType,
|
||||
type CopilotStructuredOptions,
|
||||
EmbeddingMessage,
|
||||
type ModelAttachmentCapability,
|
||||
type ModelCapability,
|
||||
type ModelFullConditions,
|
||||
ModelInputType,
|
||||
ModelOutputType,
|
||||
type PromptAttachmentKind,
|
||||
type PromptAttachmentSourceKind,
|
||||
type PromptMessage,
|
||||
PromptMessageSchema,
|
||||
} from './types';
|
||||
|
||||
// Owner: backend host model-selection glue.
|
||||
// Capability matching and catalog lookup are delegated to native/adapter; this
|
||||
// file keeps provider prefix/default/prefer behavior and Node prompt checks.
|
||||
export type ProviderModelRuntimeContext = {
|
||||
type: CopilotProviderType;
|
||||
backendKind: CopilotModelBackendKind;
|
||||
};
|
||||
|
||||
export type ResolvedProviderModel = CopilotProviderModel & {
|
||||
backendKind: CopilotModelBackendKind;
|
||||
canonicalKey: string;
|
||||
protocol?: LlmProtocol;
|
||||
requestLayer?: LlmBackendConfig['request_layer'];
|
||||
routeOverrides?: Partial<
|
||||
Record<
|
||||
ModelOutputType,
|
||||
{
|
||||
protocol?: LlmProtocol;
|
||||
requestLayer?: LlmBackendConfig['request_layer'];
|
||||
}
|
||||
>
|
||||
>;
|
||||
behaviorFlags?: string[];
|
||||
};
|
||||
|
||||
function unique<T>(values: Iterable<T>) {
|
||||
return Array.from(new Set(values));
|
||||
}
|
||||
|
||||
function resolveAttachmentCapability(
|
||||
cap: ModelCapability,
|
||||
outputType?: ModelOutputType
|
||||
): ModelAttachmentCapability | undefined {
|
||||
if (outputType === ModelOutputType.Structured) {
|
||||
return cap.structuredAttachments ?? cap.attachments;
|
||||
}
|
||||
return cap.attachments;
|
||||
}
|
||||
|
||||
function toProviderModel(
|
||||
variant: NonNullable<
|
||||
ReturnType<typeof llmResolveModelRegistryVariant>['variant']
|
||||
>
|
||||
): ResolvedProviderModel {
|
||||
return {
|
||||
id: variant.rawModelId,
|
||||
name: variant.displayName,
|
||||
backendKind: variant.backendKind,
|
||||
canonicalKey: variant.canonicalKey,
|
||||
protocol: variant.protocol,
|
||||
requestLayer: variant.requestLayer,
|
||||
routeOverrides: variant.routeOverrides,
|
||||
behaviorFlags: variant.behaviorFlags,
|
||||
capabilities: variant.capabilities.map(capability => ({
|
||||
input: capability.input as ModelInputType[],
|
||||
output: capability.output as ModelOutputType[],
|
||||
attachments: capability.attachments
|
||||
? {
|
||||
kinds: capability.attachments.kinds as PromptAttachmentKind[],
|
||||
sourceKinds: capability.attachments.sourceKinds as
|
||||
| ModelAttachmentCapability['sourceKinds']
|
||||
| undefined,
|
||||
allowRemoteUrls: capability.attachments.allowRemoteUrls,
|
||||
}
|
||||
: undefined,
|
||||
structuredAttachments: capability.structuredAttachments
|
||||
? {
|
||||
kinds: capability.structuredAttachments
|
||||
.kinds as PromptAttachmentKind[],
|
||||
sourceKinds: capability.structuredAttachments.sourceKinds as
|
||||
| ModelAttachmentCapability['sourceKinds']
|
||||
| undefined,
|
||||
allowRemoteUrls: capability.structuredAttachments.allowRemoteUrls,
|
||||
}
|
||||
: undefined,
|
||||
defaultForOutputType: capability.defaultForOutputType,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export type ProviderModelSelection = {
|
||||
kind: 'configured';
|
||||
model: ResolvedProviderModel;
|
||||
};
|
||||
|
||||
export function resolveProviderModelSelection(
|
||||
context: ProviderModelRuntimeContext,
|
||||
cond: ModelFullConditions
|
||||
): ProviderModelSelection | undefined {
|
||||
if (cond.modelId) {
|
||||
const resolved = llmResolveModelRegistryVariant({
|
||||
backendKind: context.backendKind,
|
||||
modelId: cond.modelId,
|
||||
}).variant;
|
||||
if (!resolved) {
|
||||
return;
|
||||
}
|
||||
|
||||
const model = toProviderModel(resolved);
|
||||
const matchedModelId = llmMatchModelCapabilities([model], {
|
||||
...cond,
|
||||
modelId: model.id,
|
||||
});
|
||||
if (!matchedModelId) {
|
||||
return;
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'configured',
|
||||
model,
|
||||
};
|
||||
}
|
||||
|
||||
const resolved = llmMatchModelRegistry({
|
||||
backendKind: context.backendKind,
|
||||
cond,
|
||||
}).variant;
|
||||
if (!resolved) {
|
||||
return;
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'configured',
|
||||
model: toProviderModel(resolved),
|
||||
};
|
||||
}
|
||||
|
||||
function isMultimodal(model: CopilotProviderModel) {
|
||||
return model.capabilities.some(c =>
|
||||
[ModelInputType.Image, ModelInputType.Audio, ModelInputType.File].some(t =>
|
||||
c.input.includes(t)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function handleZodError(ret: z.SafeParseReturnType<any, any>) {
|
||||
if (ret.success) return;
|
||||
const issues = ret.error.issues.map(i => {
|
||||
const path =
|
||||
'root' +
|
||||
(i.path.length
|
||||
? `.${i.path.map(seg => (typeof seg === 'number' ? `[${seg}]` : `.${seg}`)).join('')}`
|
||||
: '');
|
||||
return `${i.message}${path}`;
|
||||
});
|
||||
throw new CopilotPromptInvalid(issues.join('; '));
|
||||
}
|
||||
|
||||
export async function inferModelConditionsFromMessages(
|
||||
messages?: PromptMessage[],
|
||||
withAttachment = true
|
||||
): Promise<Partial<ModelFullConditions>> {
|
||||
if (!messages?.length || !withAttachment) return {};
|
||||
const projectedMessages = messages.map(message => ({
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
...(Array.isArray(message.attachments) && message.attachments.length
|
||||
? {
|
||||
attachments: message.attachments.map(attachment =>
|
||||
applyPromptAttachmentMimeTypeHintForNative(attachment, message)
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
}));
|
||||
const inferredCond = llmInferPromptModelConditions(projectedMessages);
|
||||
|
||||
return {
|
||||
...(inferredCond.attachmentKinds?.length
|
||||
? { attachmentKinds: unique(inferredCond.attachmentKinds) }
|
||||
: {}),
|
||||
...(inferredCond.attachmentSourceKinds?.length
|
||||
? {
|
||||
attachmentSourceKinds: unique(
|
||||
inferredCond.attachmentSourceKinds
|
||||
) as PromptAttachmentSourceKind[],
|
||||
}
|
||||
: {}),
|
||||
...(inferredCond.inputTypes?.length
|
||||
? { inputTypes: unique(inferredCond.inputTypes) as ModelInputType[] }
|
||||
: {}),
|
||||
...(inferredCond.hasRemoteAttachments
|
||||
? { hasRemoteAttachments: true }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function mergeModelConditions(
|
||||
cond: ModelFullConditions,
|
||||
inferredCond: Partial<ModelFullConditions>
|
||||
): ModelFullConditions {
|
||||
return {
|
||||
...inferredCond,
|
||||
...cond,
|
||||
inputTypes: unique([
|
||||
...(inferredCond.inputTypes ?? []),
|
||||
...(cond.inputTypes ?? []),
|
||||
]),
|
||||
attachmentKinds: unique([
|
||||
...(inferredCond.attachmentKinds ?? []),
|
||||
...(cond.attachmentKinds ?? []),
|
||||
]),
|
||||
attachmentSourceKinds: unique([
|
||||
...(inferredCond.attachmentSourceKinds ?? []),
|
||||
...(cond.attachmentSourceKinds ?? []),
|
||||
]),
|
||||
hasRemoteAttachments:
|
||||
cond.hasRemoteAttachments ?? inferredCond.hasRemoteAttachments,
|
||||
};
|
||||
}
|
||||
|
||||
export function getAttachCapability(
|
||||
model: CopilotProviderModel,
|
||||
outputType: ModelOutputType
|
||||
): ModelAttachmentCapability | undefined {
|
||||
const capability =
|
||||
model.capabilities.find(cap => cap.output.includes(outputType)) ??
|
||||
model.capabilities[0];
|
||||
if (!capability) {
|
||||
return;
|
||||
}
|
||||
return resolveAttachmentCapability(capability, outputType);
|
||||
}
|
||||
|
||||
export function matchProviderModel(
|
||||
context: ProviderModelRuntimeContext,
|
||||
cond: ModelFullConditions
|
||||
): boolean {
|
||||
return !!resolveProviderModelSelection(context, cond);
|
||||
}
|
||||
|
||||
export function resolveProviderModel(
|
||||
context: ProviderModelRuntimeContext,
|
||||
modelId: string
|
||||
): ResolvedProviderModel | undefined {
|
||||
return resolveProviderModelSelection(context, {
|
||||
modelId,
|
||||
})?.model;
|
||||
}
|
||||
|
||||
export function hasProviderModelBehaviorFlag(
|
||||
model: CopilotProviderModel,
|
||||
flag: string
|
||||
) {
|
||||
const behaviorFlags = (model as ResolvedProviderModel).behaviorFlags;
|
||||
return Array.isArray(behaviorFlags) && behaviorFlags.includes(flag);
|
||||
}
|
||||
|
||||
export function resolveProviderModelRoute(
|
||||
model: CopilotProviderModel,
|
||||
outputType: ModelOutputType
|
||||
) {
|
||||
const resolved = model as ResolvedProviderModel;
|
||||
const override = resolved.routeOverrides?.[outputType];
|
||||
|
||||
return {
|
||||
protocol: override?.protocol ?? resolved.protocol,
|
||||
requestLayer: override?.requestLayer ?? resolved.requestLayer,
|
||||
};
|
||||
}
|
||||
|
||||
export function requireProviderModelSelection(
|
||||
context: ProviderModelRuntimeContext,
|
||||
cond: ModelFullConditions
|
||||
): ResolvedProviderModel {
|
||||
const selection = resolveProviderModelSelection(context, cond);
|
||||
if (selection) return selection.model;
|
||||
|
||||
const { modelId, outputType, inputTypes } = cond;
|
||||
throw new CopilotPromptInvalid(
|
||||
modelId
|
||||
? `Model ${modelId} does not support ${outputType ?? '<any>'} output with ${inputTypes ?? '<any>'} input`
|
||||
: outputType
|
||||
? `No model supports ${outputType} output with ${inputTypes ?? '<any>'} input for provider ${context.type}`
|
||||
: 'Output type is required when modelId is not provided'
|
||||
);
|
||||
}
|
||||
|
||||
export async function checkProviderParams(
|
||||
context: ProviderModelRuntimeContext,
|
||||
{
|
||||
cond,
|
||||
messages,
|
||||
embeddings,
|
||||
options = {},
|
||||
withAttachment = true,
|
||||
}: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: unknown;
|
||||
}
|
||||
): Promise<ModelFullConditions> {
|
||||
if (messages) {
|
||||
const { requireContent = true, requireAttachment = false } = options;
|
||||
|
||||
const MessageSchema = z
|
||||
.array(
|
||||
PromptMessageSchema.extend({
|
||||
content: requireContent
|
||||
? z.string().trim().min(1)
|
||||
: z.string().optional().nullable(),
|
||||
})
|
||||
.passthrough()
|
||||
.catchall(z.union([z.string(), z.number(), z.date(), z.null()]))
|
||||
)
|
||||
.optional();
|
||||
|
||||
handleZodError(MessageSchema.safeParse(messages));
|
||||
|
||||
const inferredCond = await inferModelConditionsFromMessages(
|
||||
messages,
|
||||
withAttachment
|
||||
);
|
||||
const mergedCond = mergeModelConditions(cond, inferredCond);
|
||||
const model = requireProviderModelSelection(context, mergedCond);
|
||||
const multimodal = isMultimodal(model);
|
||||
|
||||
if (
|
||||
multimodal &&
|
||||
requireAttachment &&
|
||||
!messages.some(
|
||||
message =>
|
||||
message.role === 'user' &&
|
||||
Array.isArray(message.attachments) &&
|
||||
message.attachments.length > 0
|
||||
)
|
||||
) {
|
||||
throw new CopilotPromptInvalid('attachments required in multimodal mode');
|
||||
}
|
||||
|
||||
if (embeddings) {
|
||||
handleZodError(EmbeddingMessage.safeParse(embeddings));
|
||||
}
|
||||
|
||||
return mergedCond;
|
||||
}
|
||||
|
||||
const inferredCond = await inferModelConditionsFromMessages(
|
||||
messages,
|
||||
withAttachment
|
||||
);
|
||||
const mergedCond = mergeModelConditions(cond, inferredCond);
|
||||
|
||||
if (embeddings) {
|
||||
handleZodError(EmbeddingMessage.safeParse(embeddings));
|
||||
}
|
||||
|
||||
return mergedCond;
|
||||
}
|
||||
@@ -1,337 +0,0 @@
|
||||
import type {
|
||||
LlmBackendConfig,
|
||||
LlmEmbeddingRequest,
|
||||
LlmProtocol,
|
||||
LlmRerankRequest,
|
||||
LlmStructuredRequest,
|
||||
} from '../../../native';
|
||||
import {
|
||||
buildLlmImageRequestFromMessages,
|
||||
llmEmbeddingDispatch,
|
||||
llmRerankDispatch,
|
||||
llmStructuredDispatch,
|
||||
} from '../../../native';
|
||||
import type { NodeTextMiddleware, ProviderMiddlewareConfig } from '../config';
|
||||
import {
|
||||
buildToolContracts,
|
||||
projectPromptMessageForNative,
|
||||
} from '../runtime/contracts';
|
||||
import { buildNativeRequest } from '../runtime/native-request-runtime';
|
||||
import type { ToolLoopBackend } from '../runtime/tool/bridge';
|
||||
import type { NativeProviderAdapter } from '../runtime/tool/native-adapter';
|
||||
import type { CopilotToolSet } from '../tools';
|
||||
import type {
|
||||
CopilotProviderExecution,
|
||||
PreparedNativeEmbeddingExecution,
|
||||
PreparedNativeExecution,
|
||||
PreparedNativeImageExecution,
|
||||
PreparedNativeRequestOptions,
|
||||
PreparedNativeRerankExecution,
|
||||
PreparedNativeStructuredExecution,
|
||||
} from './provider-runtime-contract';
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotImageOptions,
|
||||
PromptMessage,
|
||||
} from './types';
|
||||
|
||||
export type CreateToolAdapterOptions = {
|
||||
maxSteps?: number;
|
||||
nodeTextMiddleware?: NodeTextMiddleware[];
|
||||
};
|
||||
|
||||
export type CreateNativeAdapter = (
|
||||
backend: ToolLoopBackend,
|
||||
tools: CopilotToolSet,
|
||||
nodeTextMiddleware?: NodeTextMiddleware[],
|
||||
options?: CreateToolAdapterOptions
|
||||
) => NativeProviderAdapter;
|
||||
|
||||
export type CreatePreparedExecutionRuntimeInput = {
|
||||
resolveProviderId: (execution?: CopilotProviderExecution) => string;
|
||||
getTools: (
|
||||
options: CopilotChatOptions,
|
||||
model: string
|
||||
) => Promise<CopilotToolSet>;
|
||||
getActiveProviderMiddleware: (
|
||||
execution?: CopilotProviderExecution
|
||||
) => ProviderMiddlewareConfig;
|
||||
createNativeAdapter: CreateNativeAdapter;
|
||||
maxSteps: number;
|
||||
};
|
||||
export type PreparedExecutionRuntime = ReturnType<
|
||||
typeof createPreparedExecutionRuntime
|
||||
>;
|
||||
|
||||
export function createPreparedExecutionRuntime(
|
||||
input: CreatePreparedExecutionRuntimeInput
|
||||
) {
|
||||
return {
|
||||
buildPreparedNativeExecution: async (
|
||||
prepared: PreparedNativeRequestOptions
|
||||
) =>
|
||||
await buildPreparedNativeExecution(
|
||||
input.resolveProviderId(prepared.execution),
|
||||
input.getTools,
|
||||
input.getActiveProviderMiddleware,
|
||||
input.maxSteps,
|
||||
prepared
|
||||
),
|
||||
createPreparedExecutionAdapter: (prepared: PreparedNativeExecution) =>
|
||||
createPreparedExecutionAdapter(
|
||||
input.createNativeAdapter,
|
||||
input.maxSteps,
|
||||
prepared
|
||||
),
|
||||
buildPreparedNativeStructuredExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmStructuredRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) =>
|
||||
buildPreparedNativeStructuredExecution(
|
||||
input.resolveProviderId(execution),
|
||||
protocol,
|
||||
backendConfig,
|
||||
model,
|
||||
request
|
||||
),
|
||||
buildPreparedNativeEmbeddingExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmEmbeddingRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) =>
|
||||
buildPreparedNativeEmbeddingExecution(
|
||||
input.resolveProviderId(execution),
|
||||
protocol,
|
||||
backendConfig,
|
||||
model,
|
||||
request
|
||||
),
|
||||
buildPreparedNativeRerankExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmRerankRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) =>
|
||||
buildPreparedNativeRerankExecution(
|
||||
input.resolveProviderId(execution),
|
||||
protocol,
|
||||
backendConfig,
|
||||
model,
|
||||
request
|
||||
),
|
||||
buildPreparedNativeImageExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotImageOptions = {},
|
||||
execution?: CopilotProviderExecution
|
||||
) =>
|
||||
buildPreparedNativeImageExecution(
|
||||
input.resolveProviderId(execution),
|
||||
protocol,
|
||||
backendConfig,
|
||||
model,
|
||||
messages,
|
||||
options
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function createPreparedExecutionAdapter(
|
||||
createNativeAdapter: CreateNativeAdapter,
|
||||
maxSteps: number,
|
||||
prepared: PreparedNativeExecution
|
||||
) {
|
||||
return createNativeAdapter(
|
||||
{
|
||||
protocol: prepared.route.protocol,
|
||||
backendConfig: prepared.route.backendConfig,
|
||||
},
|
||||
prepared.tools,
|
||||
prepared.postprocess?.nodeTextMiddleware,
|
||||
{
|
||||
maxSteps,
|
||||
nodeTextMiddleware: prepared.postprocess?.nodeTextMiddleware,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
export function createNativeStructuredDispatch(
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol
|
||||
) {
|
||||
return (request: LlmStructuredRequest) =>
|
||||
llmStructuredDispatch(protocol, backendConfig, request);
|
||||
}
|
||||
|
||||
export function createNativeEmbeddingDispatch(
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol
|
||||
) {
|
||||
return (request: LlmEmbeddingRequest) =>
|
||||
llmEmbeddingDispatch(protocol, backendConfig, request);
|
||||
}
|
||||
|
||||
export function createNativeRerankDispatch(
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol
|
||||
) {
|
||||
return (request: LlmRerankRequest) =>
|
||||
llmRerankDispatch(protocol, backendConfig, request);
|
||||
}
|
||||
|
||||
function buildPreparedRoute(
|
||||
providerId: string,
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string
|
||||
): PreparedNativeExecution['route'] {
|
||||
return {
|
||||
providerId,
|
||||
protocol,
|
||||
requestLayer: backendConfig.request_layer,
|
||||
model,
|
||||
backendConfig,
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildPreparedNativeExecution(
|
||||
providerId: string,
|
||||
getTools: (
|
||||
options: CopilotChatOptions,
|
||||
model: string
|
||||
) => Promise<CopilotToolSet>,
|
||||
getActiveProviderMiddleware: (
|
||||
execution?: CopilotProviderExecution
|
||||
) => ProviderMiddlewareConfig,
|
||||
maxSteps: number,
|
||||
{
|
||||
protocol,
|
||||
backendConfig,
|
||||
model,
|
||||
messages,
|
||||
options = {},
|
||||
execution,
|
||||
withAttachment = true,
|
||||
attachmentCapability,
|
||||
include,
|
||||
reasoning,
|
||||
tools,
|
||||
middleware,
|
||||
}: PreparedNativeRequestOptions
|
||||
): Promise<PreparedNativeExecution> {
|
||||
const resolvedTools = tools ?? (await getTools(options, model));
|
||||
const resolvedMiddleware =
|
||||
middleware ?? getActiveProviderMiddleware(execution);
|
||||
const { request } = await buildNativeRequest({
|
||||
model,
|
||||
messages,
|
||||
options,
|
||||
toolContracts: buildToolContracts(resolvedTools),
|
||||
withAttachment,
|
||||
attachmentCapability,
|
||||
include,
|
||||
reasoning,
|
||||
middleware: resolvedMiddleware,
|
||||
});
|
||||
|
||||
return {
|
||||
route: buildPreparedRoute(providerId, protocol, backendConfig, model),
|
||||
request,
|
||||
tools: resolvedTools,
|
||||
maxSteps,
|
||||
postprocess: {
|
||||
nodeTextMiddleware: resolvedMiddleware.node?.text,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type BuildPreparedNativeDispatchExecution = <
|
||||
TRequest extends
|
||||
| LlmStructuredRequest
|
||||
| LlmEmbeddingRequest
|
||||
| LlmRerankRequest,
|
||||
>(
|
||||
providerId: string,
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: TRequest
|
||||
) => {
|
||||
route: PreparedNativeExecution['route'];
|
||||
request: TRequest;
|
||||
};
|
||||
|
||||
const buildPreparedNativeDispatchExecution: BuildPreparedNativeDispatchExecution =
|
||||
(providerId, protocol, backendConfig, model, request) => {
|
||||
return {
|
||||
route: buildPreparedRoute(providerId, protocol, backendConfig, model),
|
||||
request,
|
||||
};
|
||||
};
|
||||
|
||||
export const buildPreparedNativeStructuredExecution =
|
||||
buildPreparedNativeDispatchExecution as (
|
||||
providerId: string,
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmStructuredRequest
|
||||
) => PreparedNativeStructuredExecution;
|
||||
|
||||
export const buildPreparedNativeEmbeddingExecution =
|
||||
buildPreparedNativeDispatchExecution as (
|
||||
providerId: string,
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmEmbeddingRequest
|
||||
) => PreparedNativeEmbeddingExecution;
|
||||
|
||||
export const buildPreparedNativeRerankExecution =
|
||||
buildPreparedNativeDispatchExecution as (
|
||||
providerId: string,
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmRerankRequest
|
||||
) => PreparedNativeRerankExecution;
|
||||
|
||||
export function buildPreparedNativeImageExecution(
|
||||
providerId: string,
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotImageOptions = {}
|
||||
): PreparedNativeImageExecution {
|
||||
const nativeMessages = messages.map(
|
||||
message => projectPromptMessageForNative(message).message
|
||||
);
|
||||
|
||||
return {
|
||||
route: buildPreparedRoute(providerId, protocol, backendConfig, model),
|
||||
request: buildLlmImageRequestFromMessages({
|
||||
model,
|
||||
protocol,
|
||||
messages: nativeMessages,
|
||||
options: projectImageRequestOptions(options),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function projectImageRequestOptions(options: CopilotImageOptions = {}) {
|
||||
return {
|
||||
quality: options.quality,
|
||||
seed: options.seed,
|
||||
modelName: options.modelName,
|
||||
loras: options.loras,
|
||||
};
|
||||
}
|
||||
@@ -1,287 +0,0 @@
|
||||
import type {
|
||||
CopilotProviderConfigMap,
|
||||
CopilotProviderDefaults,
|
||||
CopilotProviderProfile,
|
||||
ProviderMiddlewareConfig,
|
||||
} from '../config';
|
||||
import { resolveProviderMiddleware } from './provider-middleware';
|
||||
import { CopilotProviderType, ModelOutputType } from './types';
|
||||
|
||||
const PROVIDER_ID_PATTERN = /^[a-zA-Z0-9-_]+$/;
|
||||
|
||||
const LEGACY_PROVIDER_ORDER: CopilotProviderType[] = [
|
||||
CopilotProviderType.OpenAI,
|
||||
CopilotProviderType.CloudflareWorkersAi,
|
||||
CopilotProviderType.FAL,
|
||||
CopilotProviderType.Gemini,
|
||||
CopilotProviderType.GeminiVertex,
|
||||
CopilotProviderType.Anthropic,
|
||||
CopilotProviderType.AnthropicVertex,
|
||||
];
|
||||
|
||||
const LEGACY_PROVIDER_PRIORITY = LEGACY_PROVIDER_ORDER.reduce(
|
||||
(acc, type, index) => {
|
||||
acc[type] = LEGACY_PROVIDER_ORDER.length - index;
|
||||
return acc;
|
||||
},
|
||||
{} as Record<CopilotProviderType, number>
|
||||
);
|
||||
|
||||
type LegacyProvidersConfig = Partial<
|
||||
Record<CopilotProviderType, CopilotProviderConfigMap[CopilotProviderType]>
|
||||
>;
|
||||
|
||||
export type CopilotProvidersConfigInput = LegacyProvidersConfig & {
|
||||
profiles?: CopilotProviderProfile[] | null;
|
||||
defaults?: CopilotProviderDefaults | null;
|
||||
};
|
||||
|
||||
export type NormalizedCopilotProviderProfile = Omit<
|
||||
CopilotProviderProfile,
|
||||
'enabled' | 'priority' | 'middleware'
|
||||
> & {
|
||||
enabled: boolean;
|
||||
priority: number;
|
||||
middleware: ProviderMiddlewareConfig;
|
||||
};
|
||||
|
||||
export type CopilotProviderRegistry = {
|
||||
profiles: Map<string, NormalizedCopilotProviderProfile>;
|
||||
defaults: CopilotProviderDefaults;
|
||||
order: string[];
|
||||
byType: Map<CopilotProviderType, string[]>;
|
||||
};
|
||||
|
||||
export type ResolveModelResult = {
|
||||
rawModelId?: string;
|
||||
modelId?: string;
|
||||
explicitProviderId?: string;
|
||||
candidateProviderIds: string[];
|
||||
};
|
||||
|
||||
type ResolveModelOptions = {
|
||||
registry: CopilotProviderRegistry;
|
||||
modelId?: string;
|
||||
outputType?: ModelOutputType;
|
||||
availableProviderIds?: Iterable<string>;
|
||||
preferredProviderIds?: Iterable<string>;
|
||||
};
|
||||
|
||||
function unique<T>(list: T[]): T[] {
|
||||
return [...new Set(list)];
|
||||
}
|
||||
|
||||
function asArray<T>(iter?: Iterable<T>): T[] {
|
||||
return iter ? Array.from(iter) : [];
|
||||
}
|
||||
|
||||
function parseModelPrefix(
|
||||
registry: CopilotProviderRegistry,
|
||||
modelId: string
|
||||
): { providerId: string; modelId?: string } | null {
|
||||
const index = modelId.indexOf('/');
|
||||
if (index <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const providerId = modelId.slice(0, index);
|
||||
if (!registry.profiles.has(providerId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const model = modelId.slice(index + 1);
|
||||
return { providerId, modelId: model || undefined };
|
||||
}
|
||||
|
||||
function normalizeProfile(
|
||||
profile: CopilotProviderProfile
|
||||
): NormalizedCopilotProviderProfile {
|
||||
return {
|
||||
...profile,
|
||||
enabled: profile.enabled !== false,
|
||||
priority: profile.priority ?? 0,
|
||||
middleware: resolveProviderMiddleware(profile.type, profile.middleware),
|
||||
};
|
||||
}
|
||||
|
||||
function toLegacyProfiles(
|
||||
config: CopilotProvidersConfigInput
|
||||
): CopilotProviderProfile[] {
|
||||
const legacyProfiles: CopilotProviderProfile[] = [];
|
||||
for (const type of LEGACY_PROVIDER_ORDER) {
|
||||
const legacyConfig = config[type];
|
||||
if (!legacyConfig) {
|
||||
continue;
|
||||
}
|
||||
legacyProfiles.push({
|
||||
id: `${type}-default`,
|
||||
type,
|
||||
priority: LEGACY_PROVIDER_PRIORITY[type],
|
||||
config: legacyConfig,
|
||||
} as CopilotProviderProfile);
|
||||
}
|
||||
return legacyProfiles;
|
||||
}
|
||||
|
||||
function mergeProfiles(
|
||||
explicitProfiles: CopilotProviderProfile[],
|
||||
legacyProfiles: CopilotProviderProfile[]
|
||||
): CopilotProviderProfile[] {
|
||||
const profiles = new Map<string, CopilotProviderProfile>();
|
||||
|
||||
for (const profile of explicitProfiles) {
|
||||
if (!PROVIDER_ID_PATTERN.test(profile.id)) {
|
||||
throw new Error(`Invalid copilot provider profile id: ${profile.id}`);
|
||||
}
|
||||
if (profiles.has(profile.id)) {
|
||||
throw new Error(`Duplicated copilot provider profile id: ${profile.id}`);
|
||||
}
|
||||
profiles.set(profile.id, profile);
|
||||
}
|
||||
|
||||
for (const profile of legacyProfiles) {
|
||||
if (!profiles.has(profile.id)) {
|
||||
profiles.set(profile.id, profile);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(profiles.values());
|
||||
}
|
||||
|
||||
function sortProfiles(profiles: NormalizedCopilotProviderProfile[]) {
|
||||
return profiles.toSorted((a, b) => {
|
||||
if (a.priority !== b.priority) {
|
||||
return b.priority - a.priority;
|
||||
}
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
}
|
||||
|
||||
function assertDefaults(
|
||||
defaults: CopilotProviderDefaults,
|
||||
profiles: Map<string, NormalizedCopilotProviderProfile>
|
||||
) {
|
||||
for (const providerId of Object.values(defaults)) {
|
||||
if (!providerId) {
|
||||
continue;
|
||||
}
|
||||
if (!profiles.has(providerId)) {
|
||||
throw new Error(
|
||||
`Copilot provider defaults references unknown providerId: ${providerId}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildProviderRegistry(
|
||||
config: CopilotProvidersConfigInput
|
||||
): CopilotProviderRegistry {
|
||||
const explicitProfiles = config.profiles ?? [];
|
||||
const legacyProfiles = toLegacyProfiles(config);
|
||||
const mergedProfiles = mergeProfiles(explicitProfiles, legacyProfiles)
|
||||
.map(normalizeProfile)
|
||||
.filter(profile => profile.enabled);
|
||||
const sortedProfiles = sortProfiles(mergedProfiles);
|
||||
|
||||
const profiles = new Map(
|
||||
sortedProfiles.map(profile => [profile.id, profile] as const)
|
||||
);
|
||||
const defaults = config.defaults ?? {};
|
||||
assertDefaults(defaults, profiles);
|
||||
|
||||
const order = sortedProfiles.map(profile => profile.id);
|
||||
const byType = new Map<CopilotProviderType, string[]>();
|
||||
for (const profile of sortedProfiles) {
|
||||
const ids = byType.get(profile.type) ?? [];
|
||||
ids.push(profile.id);
|
||||
byType.set(profile.type, ids);
|
||||
}
|
||||
|
||||
return { profiles, defaults, order, byType };
|
||||
}
|
||||
|
||||
export function resolveModel({
|
||||
registry,
|
||||
modelId,
|
||||
outputType,
|
||||
availableProviderIds,
|
||||
preferredProviderIds,
|
||||
}: ResolveModelOptions): ResolveModelResult {
|
||||
const available = new Set(asArray(availableProviderIds));
|
||||
const preferred = new Set(asArray(preferredProviderIds));
|
||||
const hasAvailableFilter = available.size > 0;
|
||||
const hasPreferredFilter = preferred.size > 0;
|
||||
|
||||
const isAllowed = (providerId: string) => {
|
||||
const profile = registry.profiles.get(providerId);
|
||||
if (!profile?.enabled) {
|
||||
return false;
|
||||
}
|
||||
if (hasAvailableFilter && !available.has(providerId)) {
|
||||
return false;
|
||||
}
|
||||
if (hasPreferredFilter && !preferred.has(providerId)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
const prefixed = modelId ? parseModelPrefix(registry, modelId) : null;
|
||||
if (prefixed) {
|
||||
return {
|
||||
rawModelId: modelId,
|
||||
modelId: prefixed.modelId,
|
||||
explicitProviderId: prefixed.providerId,
|
||||
candidateProviderIds: isAllowed(prefixed.providerId)
|
||||
? [prefixed.providerId]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId) {
|
||||
return {
|
||||
rawModelId: modelId,
|
||||
modelId,
|
||||
candidateProviderIds: registry.order.filter(providerId =>
|
||||
isAllowed(providerId)
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const defaultProviderId =
|
||||
outputType && outputType !== ModelOutputType.Rerank
|
||||
? registry.defaults[outputType]
|
||||
: undefined;
|
||||
|
||||
const fallbackOrder = [
|
||||
...(defaultProviderId ? [defaultProviderId] : []),
|
||||
registry.defaults.fallback,
|
||||
...registry.order,
|
||||
].filter((id): id is string => !!id);
|
||||
|
||||
return {
|
||||
rawModelId: modelId,
|
||||
modelId,
|
||||
candidateProviderIds: unique(
|
||||
fallbackOrder.filter(providerId => isAllowed(providerId))
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function stripProviderPrefix(
|
||||
registry: CopilotProviderRegistry,
|
||||
providerId: string,
|
||||
modelId?: string
|
||||
) {
|
||||
if (!modelId) {
|
||||
return modelId;
|
||||
}
|
||||
const prefixed = parseModelPrefix(registry, modelId);
|
||||
if (!prefixed) {
|
||||
return modelId;
|
||||
}
|
||||
if (prefixed.providerId !== providerId) {
|
||||
return modelId;
|
||||
}
|
||||
return prefixed.modelId;
|
||||
}
|
||||
@@ -1,456 +0,0 @@
|
||||
import type {
|
||||
LlmBackendConfig,
|
||||
LlmEmbeddingRequest,
|
||||
LlmImageRequest,
|
||||
LlmProtocol,
|
||||
LlmRequest,
|
||||
LlmRerankRequest,
|
||||
LlmStructuredRequest,
|
||||
} from '../../../native';
|
||||
import type { NodeTextMiddleware, ProviderMiddlewareConfig } from '../config';
|
||||
import type { CopilotToolSet } from '../tools';
|
||||
import {
|
||||
type ProviderModelRuntimeContext,
|
||||
resolveProviderModelRoute,
|
||||
} from './provider-model-runtime';
|
||||
import type { NormalizedCopilotProviderProfile } from './provider-registry';
|
||||
import {
|
||||
CopilotChatOptions,
|
||||
CopilotImageOptions,
|
||||
CopilotProviderModel,
|
||||
CopilotStructuredOptions,
|
||||
ModelAttachmentCapability,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
ModelOutputType,
|
||||
PromptMessage,
|
||||
} from './types';
|
||||
|
||||
export type NativeExecutionRoute = {
|
||||
protocol: LlmProtocol;
|
||||
requestLayer?: LlmBackendConfig['request_layer'];
|
||||
model: string;
|
||||
backendConfig: LlmBackendConfig;
|
||||
};
|
||||
|
||||
export type CopilotProviderExecution = {
|
||||
providerId: string;
|
||||
profile: NormalizedCopilotProviderProfile;
|
||||
};
|
||||
|
||||
export type PreparedNativeExecution = {
|
||||
route: NativeExecutionRoute & {
|
||||
providerId: string;
|
||||
};
|
||||
request: LlmRequest;
|
||||
tools: CopilotToolSet;
|
||||
maxSteps?: number;
|
||||
postprocess?: {
|
||||
nodeTextMiddleware?: NodeTextMiddleware[];
|
||||
};
|
||||
};
|
||||
|
||||
export type PreparedNativeStructuredExecution = {
|
||||
route: NativeExecutionRoute & {
|
||||
providerId: string;
|
||||
};
|
||||
request: LlmStructuredRequest;
|
||||
};
|
||||
|
||||
export type PreparedNativeEmbeddingExecution = {
|
||||
route: NativeExecutionRoute & {
|
||||
providerId: string;
|
||||
};
|
||||
request: LlmEmbeddingRequest;
|
||||
};
|
||||
|
||||
export type PreparedNativeRerankExecution = {
|
||||
route: NativeExecutionRoute & {
|
||||
providerId: string;
|
||||
};
|
||||
request: LlmRerankRequest;
|
||||
};
|
||||
|
||||
export type PreparedNativeImageExecution = {
|
||||
route: NativeExecutionRoute & {
|
||||
providerId: string;
|
||||
};
|
||||
request: LlmImageRequest;
|
||||
};
|
||||
|
||||
export type PreparedNativeRequestOptions = {
|
||||
protocol: LlmProtocol;
|
||||
backendConfig: LlmBackendConfig;
|
||||
model: string;
|
||||
messages: PromptMessage[];
|
||||
options?: CopilotChatOptions;
|
||||
execution?: CopilotProviderExecution;
|
||||
withAttachment?: boolean;
|
||||
attachmentCapability?: ModelAttachmentCapability;
|
||||
include?: string[];
|
||||
reasoning?: Record<string, unknown>;
|
||||
tools?: CopilotToolSet;
|
||||
middleware?: ProviderMiddlewareConfig;
|
||||
};
|
||||
|
||||
type ProviderChatDriverPrepareResult = Omit<
|
||||
PreparedNativeRequestOptions,
|
||||
'execution' | 'options'
|
||||
>;
|
||||
|
||||
type Awaitable<T> = T | Promise<T>;
|
||||
|
||||
type NativeBackendConfigResolver = (
|
||||
execution?: CopilotProviderExecution
|
||||
) => Awaitable<LlmBackendConfig>;
|
||||
|
||||
export type StructuredProviderDriver = {
|
||||
createBackendConfig: NativeBackendConfigResolver;
|
||||
prepareMessages?: (
|
||||
messages: PromptMessage[],
|
||||
backendConfig: LlmBackendConfig,
|
||||
options: NonNullable<CopilotStructuredOptions>
|
||||
) => Promise<PromptMessage[]>;
|
||||
shouldRetry?: (context: {
|
||||
error: unknown;
|
||||
attempt: number;
|
||||
options: NonNullable<CopilotStructuredOptions>;
|
||||
}) => Awaitable<boolean>;
|
||||
mapError: (error: unknown) => unknown;
|
||||
};
|
||||
|
||||
export type EmbeddingProviderDriver = {
|
||||
createBackendConfig: NativeBackendConfigResolver;
|
||||
defaultDimensions?: number;
|
||||
taskType?: string;
|
||||
mapError: (error: unknown) => unknown;
|
||||
};
|
||||
|
||||
export type RerankProviderDriver = {
|
||||
createBackendConfig: NativeBackendConfigResolver;
|
||||
mapError: (error: unknown) => unknown;
|
||||
};
|
||||
|
||||
export type ImageProviderDriver = {
|
||||
createBackendConfig: NativeBackendConfigResolver;
|
||||
prepareMessages?: (
|
||||
messages: PromptMessage[],
|
||||
backendConfig: LlmBackendConfig,
|
||||
options: NonNullable<CopilotImageOptions>
|
||||
) => Promise<PromptMessage[]>;
|
||||
mapError: (error: unknown) => unknown;
|
||||
};
|
||||
|
||||
export type ProviderMetricLabels = Record<
|
||||
string,
|
||||
string | number | boolean | undefined
|
||||
>;
|
||||
|
||||
export type ProviderExecutionDrivers = {
|
||||
chat?: ProviderChatDriver;
|
||||
structured?: StructuredProviderDriver;
|
||||
embedding?: EmbeddingProviderDriver;
|
||||
rerank?: RerankProviderDriver;
|
||||
image?: ImageProviderDriver;
|
||||
};
|
||||
|
||||
export type ProviderDriverSpec = NativeProviderDriverBase & {
|
||||
chat?: NativeChatDriverOverrides | false;
|
||||
structured?: NativeStructuredDriverOverrides | false;
|
||||
embedding?: NativeEmbeddingDriverOverrides | false;
|
||||
rerank?: NativeRerankDriverOverrides | false;
|
||||
image?: NativeImageDriverOverrides | false;
|
||||
};
|
||||
|
||||
export type ProviderRuntimeHostSeed = {
|
||||
model: ProviderModelRuntimeContext;
|
||||
resolveExecutionDrivers: () => ProviderExecutionDrivers | undefined;
|
||||
selectModel: NativeChatDriverBase['selectModel'];
|
||||
checkParams: NativeChatDriverBase['checkParams'];
|
||||
getAttachCapability: (
|
||||
model: CopilotProviderModel,
|
||||
outputType: ModelOutputType
|
||||
) => ModelAttachmentCapability | undefined;
|
||||
getActiveProviderMiddleware: (
|
||||
execution?: CopilotProviderExecution
|
||||
) => ProviderMiddlewareConfig;
|
||||
getTools: (
|
||||
options: CopilotChatOptions,
|
||||
model: string
|
||||
) => Promise<CopilotToolSet>;
|
||||
metricLabels: (
|
||||
model: string,
|
||||
labels?: ProviderMetricLabels,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ProviderMetricLabels;
|
||||
};
|
||||
|
||||
export type ProviderChatDriverPrepareInput = {
|
||||
kind: 'text' | 'streamText' | 'streamObject';
|
||||
cond: ModelConditions;
|
||||
messages: PromptMessage[];
|
||||
options: CopilotChatOptions;
|
||||
execution?: CopilotProviderExecution;
|
||||
};
|
||||
|
||||
export type ProviderChatDriver = {
|
||||
prepare: (input: {
|
||||
kind: ProviderChatDriverPrepareInput['kind'];
|
||||
cond: ProviderChatDriverPrepareInput['cond'];
|
||||
messages: ProviderChatDriverPrepareInput['messages'];
|
||||
options: ProviderChatDriverPrepareInput['options'];
|
||||
execution?: ProviderChatDriverPrepareInput['execution'];
|
||||
}) => Promise<ProviderChatDriverPrepareResult | null>;
|
||||
mapError: (error: unknown) => unknown;
|
||||
};
|
||||
|
||||
type NativeProviderDriverBase = Pick<
|
||||
StructuredProviderDriver,
|
||||
'createBackendConfig' | 'mapError'
|
||||
>;
|
||||
|
||||
type ChatToolingResult = Pick<
|
||||
ProviderChatDriverPrepareResult,
|
||||
'tools' | 'middleware'
|
||||
>;
|
||||
|
||||
type NativeChatDriverBase = NativeProviderDriverBase & {
|
||||
checkParams: (input: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: CopilotProviderExecution;
|
||||
}) => Promise<ModelFullConditions>;
|
||||
selectModel: (
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => CopilotProviderModel;
|
||||
getTools?: (
|
||||
options: CopilotChatOptions,
|
||||
model: string
|
||||
) => Promise<CopilotToolSet>;
|
||||
getActiveProviderMiddleware?: (
|
||||
execution?: CopilotProviderExecution
|
||||
) => ProviderMiddlewareConfig;
|
||||
};
|
||||
|
||||
type NativeStructuredDriverOverrides = Partial<StructuredProviderDriver>;
|
||||
type NativeEmbeddingDriverOverrides = Partial<EmbeddingProviderDriver>;
|
||||
type NativeRerankDriverOverrides = Partial<RerankProviderDriver>;
|
||||
type NativeImageDriverOverrides = Partial<ImageProviderDriver>;
|
||||
|
||||
type NativeChatDriverContext = {
|
||||
input: ProviderChatDriverPrepareInput;
|
||||
outputType: ModelOutputType;
|
||||
normalizedCond: ModelFullConditions;
|
||||
model: CopilotProviderModel;
|
||||
backendConfig: LlmBackendConfig;
|
||||
protocol: LlmProtocol;
|
||||
messages: PromptMessage[];
|
||||
options: NonNullable<CopilotChatOptions>;
|
||||
execution?: CopilotProviderExecution;
|
||||
};
|
||||
|
||||
type NativeChatDriverOverrides = {
|
||||
resolveOutputType?: (
|
||||
kind: ProviderChatDriverPrepareInput['kind']
|
||||
) => ModelOutputType | null;
|
||||
withAttachment?: boolean;
|
||||
prepareMessages?: (
|
||||
context: Omit<NativeChatDriverContext, 'messages'>
|
||||
) => Awaitable<PromptMessage[]>;
|
||||
resolveTooling?: (
|
||||
context: NativeChatDriverContext
|
||||
) => Awaitable<ChatToolingResult>;
|
||||
resolveRequestOptions?: (
|
||||
context: NativeChatDriverContext
|
||||
) => Awaitable<
|
||||
Partial<
|
||||
Pick<
|
||||
ProviderChatDriverPrepareResult,
|
||||
'withAttachment' | 'attachmentCapability' | 'include' | 'reasoning'
|
||||
>
|
||||
>
|
||||
>;
|
||||
};
|
||||
|
||||
export function createNativeProviderDriverFactory(
|
||||
base: NativeProviderDriverBase
|
||||
) {
|
||||
return {
|
||||
structured(
|
||||
overrides: NativeStructuredDriverOverrides = {}
|
||||
): StructuredProviderDriver {
|
||||
return {
|
||||
createBackendConfig:
|
||||
overrides.createBackendConfig ?? base.createBackendConfig,
|
||||
mapError: overrides.mapError ?? base.mapError,
|
||||
...(overrides.prepareMessages
|
||||
? { prepareMessages: overrides.prepareMessages }
|
||||
: {}),
|
||||
...(overrides.shouldRetry
|
||||
? { shouldRetry: overrides.shouldRetry }
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
embedding(
|
||||
overrides: NativeEmbeddingDriverOverrides = {}
|
||||
): EmbeddingProviderDriver {
|
||||
return {
|
||||
createBackendConfig:
|
||||
overrides.createBackendConfig ?? base.createBackendConfig,
|
||||
mapError: overrides.mapError ?? base.mapError,
|
||||
...(overrides.defaultDimensions !== undefined
|
||||
? { defaultDimensions: overrides.defaultDimensions }
|
||||
: {}),
|
||||
...(overrides.taskType ? { taskType: overrides.taskType } : {}),
|
||||
};
|
||||
},
|
||||
rerank(overrides: NativeRerankDriverOverrides = {}): RerankProviderDriver {
|
||||
return {
|
||||
createBackendConfig:
|
||||
overrides.createBackendConfig ?? base.createBackendConfig,
|
||||
mapError: overrides.mapError ?? base.mapError,
|
||||
};
|
||||
},
|
||||
image(overrides: NativeImageDriverOverrides = {}): ImageProviderDriver {
|
||||
return {
|
||||
createBackendConfig:
|
||||
overrides.createBackendConfig ?? base.createBackendConfig,
|
||||
mapError: overrides.mapError ?? base.mapError,
|
||||
...(overrides.prepareMessages
|
||||
? { prepareMessages: overrides.prepareMessages }
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function compileProviderChatDriver(
|
||||
spec: NativeProviderDriverBase & NativeChatDriverOverrides,
|
||||
base: NativeChatDriverBase
|
||||
): ProviderChatDriver {
|
||||
return {
|
||||
prepare: async (input: ProviderChatDriverPrepareInput) => {
|
||||
const options: NonNullable<CopilotChatOptions> = input.options ?? {};
|
||||
const resolvedOutputType = spec.resolveOutputType?.(input.kind);
|
||||
const outputType =
|
||||
resolvedOutputType === undefined
|
||||
? input.kind === 'streamObject'
|
||||
? ModelOutputType.Object
|
||||
: ModelOutputType.Text
|
||||
: resolvedOutputType;
|
||||
if (!outputType) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalizedCond = await base.checkParams({
|
||||
messages: input.messages,
|
||||
cond: {
|
||||
...input.cond,
|
||||
outputType,
|
||||
},
|
||||
options,
|
||||
execution: input.execution,
|
||||
...(spec.withAttachment !== undefined
|
||||
? { withAttachment: spec.withAttachment }
|
||||
: {}),
|
||||
});
|
||||
const model = base.selectModel(normalizedCond, input.execution);
|
||||
const backendConfig = await spec.createBackendConfig(input.execution);
|
||||
const route = resolveProviderModelRoute(model, outputType);
|
||||
if (!route.protocol) {
|
||||
throw new Error(`Missing native protocol for model ${model.id}`);
|
||||
}
|
||||
const partialContext = {
|
||||
input,
|
||||
outputType,
|
||||
normalizedCond,
|
||||
model,
|
||||
backendConfig:
|
||||
route.requestLayer === backendConfig.request_layer
|
||||
? backendConfig
|
||||
: { ...backendConfig, request_layer: route.requestLayer },
|
||||
protocol: route.protocol,
|
||||
options,
|
||||
execution: input.execution,
|
||||
};
|
||||
const messages = spec.prepareMessages
|
||||
? await spec.prepareMessages(partialContext)
|
||||
: input.messages;
|
||||
const context = {
|
||||
...partialContext,
|
||||
messages,
|
||||
};
|
||||
const tooling = spec.resolveTooling
|
||||
? await spec.resolveTooling(context)
|
||||
: {
|
||||
...(base.getTools
|
||||
? { tools: await base.getTools(options, model.id) }
|
||||
: {}),
|
||||
...(base.getActiveProviderMiddleware
|
||||
? {
|
||||
middleware: base.getActiveProviderMiddleware(input.execution),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const requestOptions = spec.resolveRequestOptions
|
||||
? await spec.resolveRequestOptions(context)
|
||||
: {};
|
||||
|
||||
return {
|
||||
protocol: context.protocol,
|
||||
backendConfig: context.backendConfig,
|
||||
model: model.id,
|
||||
messages,
|
||||
...(spec.withAttachment === false ? { withAttachment: false } : {}),
|
||||
...requestOptions,
|
||||
...tooling,
|
||||
};
|
||||
},
|
||||
mapError: spec.mapError,
|
||||
};
|
||||
}
|
||||
|
||||
export function createNativeExecutionDriverSpec(
|
||||
input: ProviderDriverSpec,
|
||||
runtimeBase: NativeChatDriverBase
|
||||
): ProviderExecutionDrivers {
|
||||
const driverBase = {
|
||||
createBackendConfig: input.createBackendConfig,
|
||||
mapError: input.mapError,
|
||||
};
|
||||
const nativeDrivers = createNativeProviderDriverFactory(driverBase);
|
||||
|
||||
return {
|
||||
...(input.chat !== false
|
||||
? {
|
||||
chat: compileProviderChatDriver(
|
||||
{ ...driverBase, ...input.chat },
|
||||
runtimeBase
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
...(input.structured !== false
|
||||
? {
|
||||
structured: nativeDrivers.structured(input.structured ?? undefined),
|
||||
}
|
||||
: {}),
|
||||
...(input.embedding !== false
|
||||
? {
|
||||
embedding: nativeDrivers.embedding(input.embedding ?? undefined),
|
||||
}
|
||||
: {}),
|
||||
...(input.rerank !== false
|
||||
? { rerank: nativeDrivers.rerank(input.rerank ?? undefined) }
|
||||
: {}),
|
||||
...(input.image !== false
|
||||
? { image: nativeDrivers.image(input.image ?? undefined) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
import {
|
||||
AnthropicOfficialProvider,
|
||||
AnthropicVertexProvider,
|
||||
} from './anthropic';
|
||||
import { CloudflareWorkersAIProvider } from './cloudflare';
|
||||
import { FalProvider } from './fal';
|
||||
import { GeminiGenerativeProvider, GeminiVertexProvider } from './gemini';
|
||||
import { OpenAIProvider } from './openai';
|
||||
|
||||
export const CopilotProviders = [
|
||||
OpenAIProvider,
|
||||
CloudflareWorkersAIProvider,
|
||||
FalProvider,
|
||||
GeminiGenerativeProvider,
|
||||
GeminiVertexProvider,
|
||||
AnthropicOfficialProvider,
|
||||
AnthropicVertexProvider,
|
||||
];
|
||||
@@ -1,249 +0,0 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
import type { NodeTextMiddleware, ProviderMiddlewareConfig } from '../config';
|
||||
import { ToolExecutorHost } from '../runtime/hosts/tool-executor-host';
|
||||
import { mapNativeSemanticError } from '../runtime/native-errors';
|
||||
import type { ToolLoopBackend } from '../runtime/tool/bridge';
|
||||
import type { CopilotTool, CopilotToolSet } from '../tools';
|
||||
import { resolveProviderMiddleware } from './provider-middleware';
|
||||
import {
|
||||
checkProviderParams,
|
||||
getAttachCapability as getAttachCapabilityHelper,
|
||||
matchProviderModel as matchProviderModelHelper,
|
||||
type ProviderModelRuntimeContext,
|
||||
requireProviderModelSelection,
|
||||
resolveProviderModel,
|
||||
} from './provider-model-runtime';
|
||||
import {
|
||||
type CopilotProviderExecution,
|
||||
createNativeExecutionDriverSpec,
|
||||
type ProviderDriverSpec,
|
||||
type ProviderExecutionDrivers,
|
||||
type ProviderRuntimeHostSeed,
|
||||
} from './provider-runtime-contract';
|
||||
import {
|
||||
type CopilotChatOptions,
|
||||
CopilotChatTools,
|
||||
type CopilotImageOptions,
|
||||
type CopilotModelBackendKind,
|
||||
CopilotProviderModel,
|
||||
CopilotProviderType,
|
||||
type CopilotStructuredOptions,
|
||||
type ModelAttachmentCapability,
|
||||
ModelFullConditions,
|
||||
ModelOutputType,
|
||||
type PromptMessage,
|
||||
} from './types';
|
||||
export type {
|
||||
CopilotProviderExecution,
|
||||
ProviderDriverSpec,
|
||||
ProviderExecutionDrivers,
|
||||
ProviderRuntimeHostSeed,
|
||||
} from './provider-runtime-contract';
|
||||
|
||||
@Injectable()
|
||||
export abstract class CopilotProvider<C = any> {
|
||||
protected readonly logger = new Logger(this.constructor.name);
|
||||
protected readonly MAX_STEPS = 20;
|
||||
|
||||
abstract readonly type: CopilotProviderType;
|
||||
protected abstract resolveModelBackendKind(
|
||||
execution?: CopilotProviderExecution
|
||||
): CopilotModelBackendKind;
|
||||
abstract configured(execution?: CopilotProviderExecution): boolean;
|
||||
|
||||
@Inject() protected readonly AFFiNEConfig!: Config;
|
||||
@Inject() protected readonly toolExecutorHost!: ToolExecutorHost;
|
||||
|
||||
get maxSteps() {
|
||||
return this.MAX_STEPS;
|
||||
}
|
||||
|
||||
protected resolveModelRuntimeContext(
|
||||
execution?: CopilotProviderExecution
|
||||
): ProviderModelRuntimeContext {
|
||||
return {
|
||||
type: this.type,
|
||||
backendKind: this.resolveModelBackendKind(execution),
|
||||
};
|
||||
}
|
||||
|
||||
protected get modelRuntimeContext(): ProviderModelRuntimeContext {
|
||||
return this.resolveModelRuntimeContext();
|
||||
}
|
||||
|
||||
getDriverSpec(): ProviderDriverSpec | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
getExecutionDrivers(): ProviderExecutionDrivers | undefined {
|
||||
const spec = this.getDriverSpec();
|
||||
return spec ? this.createDriverSpec(spec) : undefined;
|
||||
}
|
||||
|
||||
protected createDriverSpec(
|
||||
spec: ProviderDriverSpec
|
||||
): ProviderExecutionDrivers {
|
||||
return createNativeExecutionDriverSpec(spec, {
|
||||
createBackendConfig: spec.createBackendConfig,
|
||||
mapError: error => {
|
||||
const mapped = mapNativeSemanticError(error);
|
||||
return mapped === error ? spec.mapError(error) : mapped;
|
||||
},
|
||||
checkParams: input =>
|
||||
checkProviderParams(
|
||||
this.resolveModelRuntimeContext(input.execution),
|
||||
input
|
||||
),
|
||||
selectModel: (cond, execution) =>
|
||||
requireProviderModelSelection(
|
||||
this.resolveModelRuntimeContext(execution),
|
||||
cond
|
||||
),
|
||||
getTools: this.getTools.bind(this),
|
||||
getActiveProviderMiddleware: this.getActiveProviderMiddleware.bind(this),
|
||||
});
|
||||
}
|
||||
|
||||
selectModel(
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
): CopilotProviderModel {
|
||||
return requireProviderModelSelection(
|
||||
this.resolveModelRuntimeContext(execution),
|
||||
cond
|
||||
);
|
||||
}
|
||||
|
||||
checkParams(input: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: CopilotProviderExecution;
|
||||
}) {
|
||||
return checkProviderParams(
|
||||
this.resolveModelRuntimeContext(input.execution),
|
||||
input
|
||||
);
|
||||
}
|
||||
|
||||
getRuntimeHostSeed(): ProviderRuntimeHostSeed {
|
||||
return {
|
||||
model: this.resolveModelRuntimeContext(),
|
||||
resolveExecutionDrivers: () => this.getExecutionDrivers(),
|
||||
selectModel: this.selectModel.bind(this),
|
||||
checkParams: this.checkParams.bind(this),
|
||||
getAttachCapability: this.getAttachCapability.bind(this),
|
||||
getActiveProviderMiddleware: this.getActiveProviderMiddleware.bind(this),
|
||||
getTools: this.getTools.bind(this),
|
||||
metricLabels: this.metricLabels.bind(this),
|
||||
};
|
||||
}
|
||||
|
||||
protected getExecutionProfile(execution?: CopilotProviderExecution) {
|
||||
return execution?.profile?.type === this.type
|
||||
? execution.profile
|
||||
: undefined;
|
||||
}
|
||||
|
||||
getActiveProviderMiddleware(
|
||||
execution?: CopilotProviderExecution
|
||||
): ProviderMiddlewareConfig {
|
||||
return (
|
||||
this.getExecutionProfile(execution)?.middleware ??
|
||||
resolveProviderMiddleware(this.type)
|
||||
);
|
||||
}
|
||||
|
||||
metricLabels(
|
||||
model: string,
|
||||
labels: Record<string, string | number | boolean | undefined> = {},
|
||||
execution?: CopilotProviderExecution
|
||||
) {
|
||||
return {
|
||||
model,
|
||||
providerId: execution?.providerId ?? `${this.type}-default`,
|
||||
...labels,
|
||||
};
|
||||
}
|
||||
|
||||
protected get config(): C {
|
||||
return this.AFFiNEConfig.copilot.providers[this.type] as C;
|
||||
}
|
||||
|
||||
protected getConfig(execution?: CopilotProviderExecution): C {
|
||||
const profile = this.getExecutionProfile(execution);
|
||||
if (profile) {
|
||||
return profile.config as C;
|
||||
}
|
||||
return this.config;
|
||||
}
|
||||
getAttachCapability(
|
||||
model: CopilotProviderModel,
|
||||
outputType: ModelOutputType
|
||||
): ModelAttachmentCapability | undefined {
|
||||
return getAttachCapabilityHelper(model, outputType);
|
||||
}
|
||||
|
||||
// make it async to allow dynamic check available models in some providers
|
||||
async match(
|
||||
cond: ModelFullConditions = {},
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<boolean> {
|
||||
return (
|
||||
this.configured(execution) &&
|
||||
matchProviderModelHelper(this.resolveModelRuntimeContext(execution), cond)
|
||||
);
|
||||
}
|
||||
|
||||
resolveModel(
|
||||
modelId: string,
|
||||
execution?: CopilotProviderExecution
|
||||
): CopilotProviderModel | undefined {
|
||||
return resolveProviderModel(
|
||||
this.resolveModelRuntimeContext(execution),
|
||||
modelId
|
||||
);
|
||||
}
|
||||
|
||||
protected getProviderSpecificTools(
|
||||
_toolName: CopilotChatTools,
|
||||
_model: string
|
||||
): [string, CopilotTool?] | undefined {
|
||||
return;
|
||||
}
|
||||
|
||||
// use for tool use, shared between providers
|
||||
async getTools(
|
||||
options: CopilotChatOptions,
|
||||
model: string
|
||||
): Promise<CopilotToolSet> {
|
||||
this.logger.debug(`getTools: ${JSON.stringify(options?.tools ?? [])}`);
|
||||
return await this.toolExecutorHost.getTools(
|
||||
options,
|
||||
model,
|
||||
this.getProviderSpecificTools.bind(this)
|
||||
);
|
||||
}
|
||||
|
||||
createNativeAdapter(
|
||||
backend: ToolLoopBackend,
|
||||
tools: CopilotToolSet,
|
||||
nodeTextMiddleware?: NodeTextMiddleware[],
|
||||
options: {
|
||||
maxSteps?: number;
|
||||
nodeTextMiddleware?: NodeTextMiddleware[];
|
||||
} = {}
|
||||
) {
|
||||
return this.toolExecutorHost.createNativeAdapter(backend, tools, {
|
||||
...options,
|
||||
nodeTextMiddleware: nodeTextMiddleware ?? options.nodeTextMiddleware,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
import {
|
||||
buildProviderRegistry,
|
||||
type CopilotProviderRegistry,
|
||||
type CopilotProvidersConfigInput,
|
||||
} from './provider-registry';
|
||||
|
||||
@Injectable()
|
||||
export class CopilotProviderRegistryService {
|
||||
private lastConfig?: CopilotProvidersConfigInput;
|
||||
private lastRegistry?: CopilotProviderRegistry;
|
||||
|
||||
constructor(private readonly config: Config) {}
|
||||
|
||||
getRegistry(): CopilotProviderRegistry {
|
||||
const providerConfig = this.config.copilot.providers;
|
||||
if (this.lastConfig === providerConfig && this.lastRegistry) {
|
||||
return this.lastRegistry;
|
||||
}
|
||||
|
||||
const registry = buildProviderRegistry(providerConfig);
|
||||
this.lastConfig = providerConfig;
|
||||
this.lastRegistry = registry;
|
||||
return registry;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { AiPromptRole } from '@prisma/client';
|
||||
import { AiSessionMessageRole } from '@prisma/client';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { JSONSchema } from '../../../base';
|
||||
@@ -7,7 +7,6 @@ import type {
|
||||
CapabilityModelCapability,
|
||||
ModelConditionsContract,
|
||||
} from '../../../native';
|
||||
import type { CopilotModelBackendKind } from '../runtime/contracts';
|
||||
import {
|
||||
type StreamObject,
|
||||
StreamObjectSchema,
|
||||
@@ -97,7 +96,6 @@ export const PromptToolsSchema = z
|
||||
|
||||
export const PromptConfigStrictSchema = z.object({
|
||||
tools: PromptToolsSchema.nullable().optional(),
|
||||
proModels: z.array(z.string()).nullable().optional(),
|
||||
// params requirements
|
||||
requireContent: z.boolean().nullable().optional(),
|
||||
requireAttachment: z.boolean().nullable().optional(),
|
||||
@@ -108,7 +106,7 @@ export const PromptConfigStrictSchema = z.object({
|
||||
presencePenalty: z.number().nullable().optional(),
|
||||
temperature: z.number().nullable().optional(),
|
||||
topP: z.number().nullable().optional(),
|
||||
maxTokens: z.number().nullable().optional(),
|
||||
maxOutputTokens: z.number().nullable().optional(),
|
||||
// fal
|
||||
modelName: z.string().nullable().optional(),
|
||||
loras: z
|
||||
@@ -132,7 +130,7 @@ export type PromptTools = z.infer<typeof PromptToolsSchema>;
|
||||
|
||||
export const EmbeddingMessage = z.array(z.string().trim().min(1)).min(1);
|
||||
|
||||
export const ChatMessageRole = Object.values(AiPromptRole) as [
|
||||
export const ChatMessageRole = Object.values(AiSessionMessageRole) as [
|
||||
'system',
|
||||
'assistant',
|
||||
'user',
|
||||
@@ -268,6 +266,8 @@ const CopilotProviderOptionsSchema = z.object({
|
||||
billingUnitId: z.string().optional(),
|
||||
taskId: z.string().optional(),
|
||||
actionId: z.string().optional(),
|
||||
builtInRouteId: z.string().optional(),
|
||||
managedTargetId: z.string().optional(),
|
||||
quotaBackedRoutesAllowed: z.boolean().optional(),
|
||||
featureKind: z
|
||||
.enum([
|
||||
@@ -380,8 +380,8 @@ export interface CopilotProviderModel {
|
||||
capabilities: ModelCapability[];
|
||||
}
|
||||
|
||||
export type { CopilotModelBackendKind };
|
||||
|
||||
export type ModelConditions = Omit<ModelConditionsContract, 'outputType'>;
|
||||
export type ModelConditions = Omit<ModelConditionsContract, 'outputType'> & {
|
||||
profileId?: string;
|
||||
};
|
||||
|
||||
export type ModelFullConditions = ModelConditionsContract;
|
||||
|
||||
@@ -31,11 +31,12 @@ import { CurrentUser } from '../../core/auth';
|
||||
import { DocAction, PermissionAccess } from '../../core/permission';
|
||||
import { UserType } from '../../core/user';
|
||||
import type { ListSessionOptions, UpdateChatSession } from '../../models';
|
||||
import { llmGetBuiltInRouteOptions } from '../../native';
|
||||
import { ByokEntitlementPolicy } from './byok';
|
||||
import { CompatHistoryProjector } from './compat/history-projector';
|
||||
import { ConversationInboxService } from './conversation/inbox';
|
||||
import { PromptService } from './prompt/service';
|
||||
import { CopilotProviderFactory } from './providers/factory';
|
||||
import { ModelOutputType, type StreamObject } from './providers/types';
|
||||
import { CopilotEnabled } from './feature';
|
||||
import type { StreamObject } from './providers/types';
|
||||
import { ChatSessionService } from './session';
|
||||
import { type ChatHistory, type ChatMessage, SubmittedMessage } from './types';
|
||||
|
||||
@@ -256,12 +257,6 @@ class CopilotHistoriesType implements Omit<ChatHistory, 'userId'> {
|
||||
@Field(() => String)
|
||||
promptName!: string;
|
||||
|
||||
@Field(() => String)
|
||||
model!: string;
|
||||
|
||||
@Field(() => [String])
|
||||
optionalModels!: string[];
|
||||
|
||||
@Field(() => String, {
|
||||
description: 'An mark identifying which view to use to display the session',
|
||||
nullable: true,
|
||||
@@ -274,11 +269,6 @@ class CopilotHistoriesType implements Omit<ChatHistory, 'userId'> {
|
||||
@Field(() => String, { nullable: true })
|
||||
title!: string | null;
|
||||
|
||||
@Field(() => Number, {
|
||||
description: 'The number of tokens used in the session',
|
||||
})
|
||||
tokens!: number;
|
||||
|
||||
@Field(() => [ChatMessageType])
|
||||
messages!: ChatMessageType[];
|
||||
|
||||
@@ -303,27 +293,6 @@ class CopilotQuotaType {
|
||||
used!: number;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class CopilotModelType {
|
||||
@Field(() => String)
|
||||
id!: string;
|
||||
|
||||
@Field(() => String)
|
||||
name!: string;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
export class CopilotModelsType {
|
||||
@Field(() => String)
|
||||
defaultModel!: string;
|
||||
|
||||
@Field(() => [CopilotModelType])
|
||||
optionalModels!: CopilotModelType[];
|
||||
|
||||
@Field(() => [CopilotModelType])
|
||||
proModels!: CopilotModelType[];
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
export class CopilotSessionType {
|
||||
@Field(() => ID)
|
||||
@@ -343,12 +312,33 @@ export class CopilotSessionType {
|
||||
|
||||
@Field(() => String)
|
||||
promptName!: string;
|
||||
}
|
||||
|
||||
@ObjectType('CopilotRouteTarget')
|
||||
class CopilotRouteTargetType {
|
||||
@Field(() => String)
|
||||
id!: string;
|
||||
|
||||
@Field(() => String)
|
||||
model!: string;
|
||||
displayName!: string;
|
||||
|
||||
@Field(() => [String])
|
||||
optionalModels!: string[];
|
||||
@Field(() => String)
|
||||
minimumTier!: string;
|
||||
|
||||
@Field(() => Boolean)
|
||||
available!: boolean;
|
||||
}
|
||||
|
||||
@ObjectType('CopilotRouteOptions')
|
||||
class CopilotRouteOptionsType {
|
||||
@Field(() => String)
|
||||
routeId!: string;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
defaultTargetId!: string | null;
|
||||
|
||||
@Field(() => [CopilotRouteTargetType])
|
||||
choices!: CopilotRouteTargetType[];
|
||||
}
|
||||
|
||||
// ================== Resolver ==================
|
||||
@@ -360,20 +350,47 @@ export class CopilotType {
|
||||
}
|
||||
|
||||
@Throttle()
|
||||
@CopilotEnabled()
|
||||
@Resolver(() => CopilotType)
|
||||
export class CopilotResolver {
|
||||
private readonly modelNames = new Map<string, string>();
|
||||
|
||||
constructor(
|
||||
private readonly ac: PermissionAccess,
|
||||
private readonly mutex: RequestMutex,
|
||||
private readonly prompt: PromptService,
|
||||
private readonly chatSession: ChatSessionService,
|
||||
private readonly historyProjector: CompatHistoryProjector,
|
||||
private readonly inbox: ConversationInboxService,
|
||||
private readonly providerFactory: CopilotProviderFactory
|
||||
private readonly entitlement: ByokEntitlementPolicy
|
||||
) {}
|
||||
|
||||
@ResolveField(() => CopilotRouteOptionsType, {
|
||||
nullable: true,
|
||||
description: 'List native built-in route choices for a prompt',
|
||||
complexity: 2,
|
||||
})
|
||||
async routeOptions(
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('promptName') promptName: string
|
||||
): Promise<CopilotRouteOptionsType | null> {
|
||||
const options = llmGetBuiltInRouteOptions(promptName);
|
||||
if (!options) return null;
|
||||
if (env.selfhosted) {
|
||||
return { routeId: options.routeId, defaultTargetId: null, choices: [] };
|
||||
}
|
||||
const premium = await this.entitlement.hasAiPlan(user.id);
|
||||
return {
|
||||
routeId: options.routeId,
|
||||
defaultTargetId: premium
|
||||
? (options.premiumDefaultTargetId ?? null)
|
||||
: (options.standardDefaultTargetId ?? null),
|
||||
choices: options.choices.map(choice => ({
|
||||
id: choice.id,
|
||||
displayName: choice.displayName,
|
||||
minimumTier: choice.minimumTier,
|
||||
available: premium || choice.minimumTier === 'Standard',
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
@ResolveField(() => CopilotQuotaType, {
|
||||
name: 'quota',
|
||||
description: 'Get the quota of the user in the workspace',
|
||||
@@ -408,51 +425,6 @@ export class CopilotResolver {
|
||||
return { userId: user.id, workspaceId, docId: docId || undefined };
|
||||
}
|
||||
|
||||
@ResolveField(() => CopilotModelsType, {
|
||||
description:
|
||||
'List available models for a prompt, with human-readable names',
|
||||
complexity: 2,
|
||||
})
|
||||
async models(
|
||||
@Args('promptName') promptName: string
|
||||
): Promise<CopilotModelsType> {
|
||||
const prompt = await this.prompt.get(promptName);
|
||||
if (!prompt) {
|
||||
throw new NotFoundException('Prompt not found');
|
||||
}
|
||||
const convertModels = async (ids: string[]) => {
|
||||
const models = await Promise.all(
|
||||
ids.map(async id => {
|
||||
const cachedName = this.modelNames.get(id);
|
||||
if (cachedName) return { id, name: cachedName };
|
||||
|
||||
const resolved = await this.providerFactory.resolveProvider({
|
||||
modelId: id,
|
||||
outputType: ModelOutputType.Text,
|
||||
});
|
||||
const name = resolved?.provider.resolveModel(
|
||||
resolved.modelId ?? id,
|
||||
resolved.execution
|
||||
)?.name;
|
||||
if (name) {
|
||||
this.modelNames.set(id, name);
|
||||
return { id, name };
|
||||
}
|
||||
return null;
|
||||
})
|
||||
);
|
||||
|
||||
return models.filter(model => !!model) as CopilotModelType[];
|
||||
};
|
||||
const proModels = prompt.config?.proModels || [];
|
||||
|
||||
return {
|
||||
defaultModel: prompt.model,
|
||||
optionalModels: await convertModels(prompt.optionalModels),
|
||||
proModels: await convertModels(proModels),
|
||||
};
|
||||
}
|
||||
|
||||
@ResolveField(() => CopilotSessionType, {
|
||||
description: 'Get the session by id',
|
||||
complexity: 2,
|
||||
@@ -813,6 +785,7 @@ export class CopilotResolver {
|
||||
}
|
||||
|
||||
@Throttle()
|
||||
@CopilotEnabled()
|
||||
@Resolver(() => UserType)
|
||||
export class UserCopilotResolver {
|
||||
constructor(private readonly ac: PermissionAccess) {}
|
||||
|
||||
@@ -110,7 +110,7 @@ function isImageAction(actionId: string) {
|
||||
}
|
||||
|
||||
function resolveProjector(actionId: string): ActionResultProjector | null {
|
||||
if (actionId.startsWith('transcript.audio.')) {
|
||||
if (actionId === 'transcript.audio') {
|
||||
return null;
|
||||
}
|
||||
if (isImageAction(actionId)) {
|
||||
|
||||
@@ -1,15 +1,10 @@
|
||||
import { Injectable, Optional } from '@nestjs/common';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Models } from '../../../models';
|
||||
import type { AiActionRunStatus } from '../../../models/copilot-action-run';
|
||||
import {
|
||||
type NativeActionEvent,
|
||||
type NativeActionRuntimeInput,
|
||||
runNativeActionRecipePreparedStream,
|
||||
} from '../../../native';
|
||||
import { type NativeActionEvent } from '../../../native';
|
||||
import type {
|
||||
CopilotImageOptions,
|
||||
CopilotProviderType,
|
||||
CopilotStructuredOptions,
|
||||
PromptMessage,
|
||||
} from '../providers/types';
|
||||
@@ -18,18 +13,10 @@ import {
|
||||
projectActionResultToAssistantTurn,
|
||||
summarizeActionResult,
|
||||
} from './action-output-projector';
|
||||
import {
|
||||
buildStructuredResponseFromSchemaJson,
|
||||
type RequiredStructuredOutputContract,
|
||||
} from './contracts';
|
||||
import { ExecutionPlanBuilder } from './execution-plan';
|
||||
import { CapabilityRuntime } from './capability-runtime';
|
||||
import { type RequiredStructuredOutputContract } from './contracts';
|
||||
import { TurnPersistence } from './hosts/turn-persistence';
|
||||
|
||||
type ActionRuntimeBridgeNativeInput = Omit<
|
||||
NativeActionRuntimeInput,
|
||||
'recipeId' | 'recipeVersion'
|
||||
>;
|
||||
|
||||
export type ActionRuntimeBridgeInput = {
|
||||
userId: string;
|
||||
workspaceId: string;
|
||||
@@ -42,26 +29,18 @@ export type ActionRuntimeBridgeInput = {
|
||||
attempt?: number;
|
||||
retryOf?: string | null;
|
||||
inputSnapshot?: unknown;
|
||||
nativeInput?: ActionRuntimeBridgeNativeInput;
|
||||
onRunCreated?: (
|
||||
context: ActionRuntimeBridgeRunContext
|
||||
) => Promise<void> | void;
|
||||
prepareStructuredRoutes?: {
|
||||
stepId?: string;
|
||||
step: {
|
||||
slot: string;
|
||||
builtInRouteId: string;
|
||||
profileId?: string;
|
||||
modelId?: string;
|
||||
messages: PromptMessage[];
|
||||
options?: CopilotStructuredOptions;
|
||||
prefer?: CopilotProviderType;
|
||||
responseSchemaJson?: Record<string, unknown>;
|
||||
options?: CopilotStructuredOptions | CopilotImageOptions;
|
||||
responseContract?: RequiredStructuredOutputContract;
|
||||
};
|
||||
prepareImageRoutes?: {
|
||||
stepId?: string;
|
||||
modelId?: string;
|
||||
messages: PromptMessage[];
|
||||
options?: CopilotImageOptions;
|
||||
prefer?: CopilotProviderType;
|
||||
};
|
||||
persistAttachment?: (attachment: unknown) => Promise<unknown> | unknown;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
@@ -107,94 +86,41 @@ export class ActionRuntimeBridge {
|
||||
constructor(
|
||||
private readonly models: Models,
|
||||
private readonly turnPersistence: TurnPersistence,
|
||||
@Optional() private readonly plans?: ExecutionPlanBuilder
|
||||
private readonly runtime: CapabilityRuntime
|
||||
) {}
|
||||
|
||||
protected runNativeStream(
|
||||
input: NativeActionRuntimeInput,
|
||||
signal?: AbortSignal
|
||||
) {
|
||||
return runNativeActionRecipePreparedStream(input, signal);
|
||||
}
|
||||
|
||||
private async prepareNativeInput(
|
||||
input: ActionRuntimeBridgeInput
|
||||
): Promise<ActionRuntimeBridgeNativeInput & { input: unknown }> {
|
||||
const nativeInput = {
|
||||
...input.nativeInput,
|
||||
input: input.nativeInput?.input ?? {},
|
||||
};
|
||||
const structured = input.prepareStructuredRoutes;
|
||||
const image = input.prepareImageRoutes;
|
||||
if (!structured && !image) {
|
||||
return nativeInput;
|
||||
}
|
||||
if (!this.plans) {
|
||||
throw new Error('Action route preparation is not available');
|
||||
}
|
||||
const state =
|
||||
nativeInput.input && typeof nativeInput.input === 'object'
|
||||
? { ...(nativeInput.input as Record<string, unknown>) }
|
||||
: {};
|
||||
|
||||
if (structured) {
|
||||
const responseContract =
|
||||
structured.responseContract ??
|
||||
(buildStructuredResponseFromSchemaJson(
|
||||
structured.responseSchemaJson ?? { type: 'object' }
|
||||
) as RequiredStructuredOutputContract);
|
||||
const plan = await this.plans.buildStructuredPlan(
|
||||
{ modelId: structured.modelId },
|
||||
structured.messages,
|
||||
structured.options,
|
||||
structured.prefer ? { prefer: structured.prefer } : undefined,
|
||||
responseContract
|
||||
private async execute(input: ActionRuntimeBridgeInput) {
|
||||
const step = input.step;
|
||||
if (step.responseContract) {
|
||||
const output = await this.runtime.generateStructuredValue(
|
||||
{ profileId: step.profileId, modelId: step.modelId },
|
||||
step.messages,
|
||||
{
|
||||
...(step.options as CopilotStructuredOptions | undefined),
|
||||
builtInRouteId: step.builtInRouteId,
|
||||
},
|
||||
step.responseContract,
|
||||
undefined,
|
||||
step.slot
|
||||
);
|
||||
const preparedRoutes = plan.nativeDispatch?.structured?.routes;
|
||||
if (!preparedRoutes?.length) {
|
||||
throw new Error('No native structured provider route prepared');
|
||||
}
|
||||
|
||||
const existingPreparedRoutes =
|
||||
state.preparedRoutes &&
|
||||
typeof state.preparedRoutes === 'object' &&
|
||||
!Array.isArray(state.preparedRoutes)
|
||||
? (state.preparedRoutes as Record<string, unknown>)
|
||||
: {};
|
||||
state.preparedRoutes = {
|
||||
...existingPreparedRoutes,
|
||||
[structured.stepId ?? 'generate']: preparedRoutes,
|
||||
};
|
||||
return { result: output.value, attachments: [] };
|
||||
}
|
||||
|
||||
if (image) {
|
||||
const plan = await this.plans.buildImagePlan(
|
||||
{ modelId: image.modelId },
|
||||
image.messages,
|
||||
image.options,
|
||||
image.prefer ? { prefer: image.prefer } : undefined
|
||||
);
|
||||
const preparedRoutes = plan.nativeDispatch?.image?.routes;
|
||||
if (!preparedRoutes?.length) {
|
||||
throw new Error('No native image provider route prepared');
|
||||
}
|
||||
|
||||
const existingPreparedRoutes =
|
||||
state.preparedRoutes &&
|
||||
typeof state.preparedRoutes === 'object' &&
|
||||
!Array.isArray(state.preparedRoutes)
|
||||
? (state.preparedRoutes as Record<string, unknown>)
|
||||
: {};
|
||||
state.preparedRoutes = {
|
||||
...existingPreparedRoutes,
|
||||
[image.stepId ?? 'generate-image']: preparedRoutes,
|
||||
};
|
||||
const images = [];
|
||||
for await (const image of this.runtime.streamImageArtifacts(
|
||||
{ profileId: step.profileId, modelId: step.modelId },
|
||||
step.messages,
|
||||
{
|
||||
...(step.options as CopilotImageOptions | undefined),
|
||||
builtInRouteId: step.builtInRouteId,
|
||||
},
|
||||
undefined,
|
||||
step.slot
|
||||
)) {
|
||||
images.push(image);
|
||||
}
|
||||
|
||||
return {
|
||||
...nativeInput,
|
||||
input: state,
|
||||
};
|
||||
const result = images[0];
|
||||
if (!result) throw new Error('Action image generation produced no image');
|
||||
return { result, attachments: [result] };
|
||||
}
|
||||
|
||||
private async projectAssistantResult(
|
||||
@@ -273,28 +199,36 @@ export class ActionRuntimeBridge {
|
||||
let finalEvent: NativeActionEvent | undefined;
|
||||
const attachments: unknown[] = [];
|
||||
try {
|
||||
const nativeInput = await this.prepareNativeInput({
|
||||
...inputWithBillingUnit,
|
||||
});
|
||||
for await (const event of this.runNativeStream(
|
||||
{
|
||||
...nativeInput,
|
||||
recipeId: inputWithBillingUnit.actionId,
|
||||
recipeVersion: inputWithBillingUnit.actionVersion,
|
||||
},
|
||||
inputWithBillingUnit.signal
|
||||
)) {
|
||||
finalEvent = event;
|
||||
let projectedEvent = event;
|
||||
if (event.type === 'attachment') {
|
||||
const attachment = input.persistAttachment
|
||||
? await input.persistAttachment(event.attachment)
|
||||
: event.attachment;
|
||||
attachments.push(attachment);
|
||||
projectedEvent = { ...event, attachment };
|
||||
}
|
||||
yield { ...projectedEvent, runId: run.id };
|
||||
const actionStart: NativeActionEvent = {
|
||||
type: 'action_start',
|
||||
actionId: input.actionId,
|
||||
actionVersion: input.actionVersion,
|
||||
status: 'running',
|
||||
};
|
||||
yield { ...actionStart, runId: run.id };
|
||||
const output = await this.execute(inputWithBillingUnit);
|
||||
for (const artifact of output.attachments) {
|
||||
const attachment = input.persistAttachment
|
||||
? await input.persistAttachment(artifact)
|
||||
: artifact;
|
||||
attachments.push(attachment);
|
||||
yield {
|
||||
type: 'attachment',
|
||||
actionId: input.actionId,
|
||||
actionVersion: input.actionVersion,
|
||||
status: 'running',
|
||||
attachment,
|
||||
runId: run.id,
|
||||
};
|
||||
}
|
||||
finalEvent = {
|
||||
type: 'action_done',
|
||||
actionId: input.actionId,
|
||||
actionVersion: input.actionVersion,
|
||||
status: 'succeeded',
|
||||
result: output.result,
|
||||
};
|
||||
yield { ...finalEvent, runId: run.id };
|
||||
} catch (error) {
|
||||
finalEvent = {
|
||||
type: 'error',
|
||||
@@ -351,33 +285,14 @@ export class ActionRuntimeBridge {
|
||||
): ActionRuntimeBridgeInput {
|
||||
return {
|
||||
...input,
|
||||
prepareStructuredRoutes: input.prepareStructuredRoutes
|
||||
? {
|
||||
...input.prepareStructuredRoutes,
|
||||
options: {
|
||||
...input.prepareStructuredRoutes.options,
|
||||
actionId:
|
||||
input.prepareStructuredRoutes.options?.actionId ??
|
||||
input.actionId,
|
||||
billingUnitId:
|
||||
input.prepareStructuredRoutes.options?.billingUnitId ??
|
||||
billingUnitId,
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
prepareImageRoutes: input.prepareImageRoutes
|
||||
? {
|
||||
...input.prepareImageRoutes,
|
||||
options: {
|
||||
...input.prepareImageRoutes.options,
|
||||
actionId:
|
||||
input.prepareImageRoutes.options?.actionId ?? input.actionId,
|
||||
billingUnitId:
|
||||
input.prepareImageRoutes.options?.billingUnitId ??
|
||||
billingUnitId,
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
step: {
|
||||
...input.step,
|
||||
options: {
|
||||
...input.step.options,
|
||||
actionId: input.step.options?.actionId ?? input.actionId,
|
||||
billingUnitId: input.step.options?.billingUnitId ?? billingUnitId,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,27 @@
|
||||
/* oxlint-disable import/no-cycle -- Tool callbacks can invoke nested Copilot prompts. */
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { CopilotPromptInvalid } from '../../../base';
|
||||
import { ValidatedStructuredValueSchema } from '../core';
|
||||
import { Config } from '../../../base/config';
|
||||
import { CopilotPromptInvalid } from '../../../base/error/errors.gen';
|
||||
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
|
||||
import {
|
||||
buildLlmEmbeddingRequest,
|
||||
buildLlmImageRequestFromMessages,
|
||||
buildLlmRerankRequest,
|
||||
type LlmImageResponse,
|
||||
type LlmToolCallbackRequest,
|
||||
type LlmToolLoopStreamEvent,
|
||||
llmValidateJsonSchema,
|
||||
} from '../../../native';
|
||||
import {
|
||||
getByokSourceCoverage,
|
||||
getCopilotFeatureAccess,
|
||||
} from '../access/feature-coverage';
|
||||
import { assertCopilotEnabled } from '../availability';
|
||||
import { ByokEntitlementPolicy } from '../byok/policy';
|
||||
import type { ByokFeatureKind } from '../byok/types';
|
||||
import { ConversationPolicy } from '../conversation/policy';
|
||||
import { ValidatedStructuredValueSchema } from '../core/types';
|
||||
import {
|
||||
type CopilotChatOptions,
|
||||
type CopilotEmbeddingOptions,
|
||||
@@ -9,112 +29,294 @@ import {
|
||||
type CopilotProviderType,
|
||||
type CopilotRerankRequest,
|
||||
type CopilotStructuredOptions,
|
||||
type ModelAttachmentCapability,
|
||||
type ModelConditions,
|
||||
type PromptMessage,
|
||||
type StreamObject,
|
||||
} from '../providers/types';
|
||||
import {
|
||||
buildToolContracts,
|
||||
type RequiredStructuredOutputContract,
|
||||
requireStructuredOutputContract,
|
||||
} from './contracts';
|
||||
import {
|
||||
ExecutionPlanBuilder,
|
||||
type ExecutionPlanForKind,
|
||||
} from './execution-plan';
|
||||
type CopilotRuntimeEvent,
|
||||
CopilotRuntimeEventConsumer,
|
||||
} from './copilot-runtime-event-consumer';
|
||||
import { mapNativeSemanticError } from './native-errors';
|
||||
import {
|
||||
NativeExecutionEngine,
|
||||
type NativeImageArtifact,
|
||||
} from './native-execution-engine';
|
||||
buildCanonicalNativeRequest,
|
||||
buildCanonicalNativeStructuredRequest,
|
||||
preparePromptMessagesForNativeRequest,
|
||||
} from './native-request-runtime';
|
||||
import { executeToolCall } from './tool/bridge';
|
||||
import { NativeProviderAdapter } from './tool/native-adapter';
|
||||
import { ToolRuntime } from './tool-runtime';
|
||||
|
||||
type ProviderFilter = {
|
||||
prefer?: CopilotProviderType;
|
||||
type ProviderFilter = { prefer?: CopilotProviderType };
|
||||
type RuntimeOptions = NonNullable<CopilotChatOptions> & {
|
||||
dimensions?: number;
|
||||
responseSchemaJson?: Record<string, unknown>;
|
||||
schemaHash?: string;
|
||||
strict?: boolean;
|
||||
profileId?: string;
|
||||
};
|
||||
|
||||
const providerModelId = (modelId?: string) => modelId ?? 'auto';
|
||||
export type NativeImageArtifact = LlmImageResponse['images'][number];
|
||||
|
||||
const attachmentCapability = {
|
||||
kinds: ['image', 'audio', 'file'],
|
||||
sourceKinds: ['url', 'data', 'bytes', 'file_handle'],
|
||||
allowRemoteUrls: true,
|
||||
} satisfies ModelAttachmentCapability;
|
||||
|
||||
@Injectable()
|
||||
export class CapabilityRuntime {
|
||||
constructor(
|
||||
private readonly plans: ExecutionPlanBuilder,
|
||||
private readonly engine: NativeExecutionEngine
|
||||
private readonly backend: BackendRuntimeProvider,
|
||||
private readonly entitlement: ByokEntitlementPolicy,
|
||||
private readonly conversations: ConversationPolicy,
|
||||
private readonly tools: ToolRuntime,
|
||||
private readonly events: CopilotRuntimeEventConsumer,
|
||||
private readonly config: Config
|
||||
) {}
|
||||
|
||||
private async executePlan<TPlan, TResult>(
|
||||
build: () => Promise<TPlan>,
|
||||
execute: (plan: TPlan) => Promise<TResult>
|
||||
) {
|
||||
return await execute(await build());
|
||||
private async access(options: RuntimeOptions) {
|
||||
assertCopilotEnabled(this.config);
|
||||
const workspaceId = options.workspace;
|
||||
const featureKind = (options.featureKind ?? 'chat') as ByokFeatureKind;
|
||||
const coverage = getByokSourceCoverage(featureKind);
|
||||
const [serverByok, localByok, premium] = workspaceId
|
||||
? await Promise.all([
|
||||
coverage.server && this.entitlement.hasServerEntitlement(workspaceId),
|
||||
coverage.local &&
|
||||
this.entitlement.hasLocalEntitlement(workspaceId, options.user),
|
||||
this.entitlement.hasAiPlan(options.user),
|
||||
])
|
||||
: [false, false, await this.entitlement.hasAiPlan(options.user)];
|
||||
const routeAllowed =
|
||||
options.quotaBackedRoutesAllowed ??
|
||||
(!getCopilotFeatureAccess(featureKind).quotaMetered ||
|
||||
!options.user ||
|
||||
(await this.conversations.hasQuota(options.user)));
|
||||
return {
|
||||
routeAllowed,
|
||||
managedTier: premium ? ('Premium' as const) : ('Standard' as const),
|
||||
serverByok,
|
||||
localByok,
|
||||
};
|
||||
}
|
||||
|
||||
private executeStreamPlan<TPlan, TChunk>(
|
||||
build: () => Promise<TPlan>,
|
||||
execute: (plan: TPlan) => AsyncIterableIterator<TChunk>
|
||||
): AsyncIterableIterator<TChunk> {
|
||||
return (async function* () {
|
||||
yield* execute(await build());
|
||||
})();
|
||||
private eventContext(options: RuntimeOptions) {
|
||||
return {
|
||||
workspaceId: options.workspace,
|
||||
userId: options.user,
|
||||
sessionId: options.session,
|
||||
taskId: options.taskId,
|
||||
actionId: options.actionId,
|
||||
billingUnitId: options.billingUnitId,
|
||||
featureKind: (options.featureKind ?? 'chat') as ByokFeatureKind,
|
||||
};
|
||||
}
|
||||
|
||||
private hasNativeDispatch(
|
||||
plan: ExecutionPlanForKind<'embedding'> | ExecutionPlanForKind<'rerank'>,
|
||||
kind: 'embedding' | 'rerank'
|
||||
private targetOverride(cond: ModelConditions) {
|
||||
return cond.profileId && cond.modelId
|
||||
? { profileId: cond.profileId, modelId: cond.modelId }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
async assertRoute(
|
||||
slot: string,
|
||||
cond: ModelConditions,
|
||||
options: CopilotChatOptions = {}
|
||||
) {
|
||||
return !!plan.nativeDispatch?.[kind];
|
||||
try {
|
||||
await this.backend.assertCopilotRoute({
|
||||
slot,
|
||||
builtInRouteId: options.builtInRouteId,
|
||||
workspaceId: options.workspace,
|
||||
userId: options.user,
|
||||
localLeaseId: options.byokLeaseId,
|
||||
access: await this.access(options),
|
||||
managedTargetId: options.managedTargetId,
|
||||
targetOverride: this.targetOverride(cond),
|
||||
});
|
||||
} catch (error) {
|
||||
throw mapNativeSemanticError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private async execute(
|
||||
slot: string,
|
||||
request: unknown,
|
||||
cond: ModelConditions,
|
||||
options: RuntimeOptions
|
||||
) {
|
||||
try {
|
||||
const output = await this.backend.executeCopilot({
|
||||
slot,
|
||||
builtInRouteId: options.builtInRouteId,
|
||||
workspaceId: options.workspace,
|
||||
userId: options.user,
|
||||
localLeaseId: options.byokLeaseId,
|
||||
access: await this.access(options),
|
||||
managedTargetId: options.managedTargetId,
|
||||
targetOverride: this.targetOverride(cond),
|
||||
request,
|
||||
});
|
||||
await this.events.consume(
|
||||
output.events as CopilotRuntimeEvent[],
|
||||
this.eventContext(options)
|
||||
);
|
||||
return output.result;
|
||||
} catch (error) {
|
||||
throw mapNativeSemanticError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private async prepareChat(
|
||||
messages: PromptMessage[],
|
||||
options: RuntimeOptions
|
||||
) {
|
||||
const toolSet = await this.tools.getTools(options, '');
|
||||
const { request } = await buildCanonicalNativeRequest({
|
||||
model: 'route-selected',
|
||||
messages,
|
||||
options,
|
||||
toolContracts: buildToolContracts(toolSet),
|
||||
attachmentCapability,
|
||||
include: options.reasoning ? ['reasoning'] : undefined,
|
||||
reasoning: options.reasoning ? { effort: 'medium' } : undefined,
|
||||
});
|
||||
return { request: { ...request, stream: true }, toolSet };
|
||||
}
|
||||
|
||||
private async stream(
|
||||
slot: string,
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options: RuntimeOptions
|
||||
) {
|
||||
const { request, toolSet } = await this.prepareChat(messages, options);
|
||||
const rawStream = this.backend.streamCopilot<
|
||||
LlmToolLoopStreamEvent | CopilotRuntimeEvent
|
||||
>(
|
||||
{
|
||||
slot,
|
||||
builtInRouteId: options.builtInRouteId,
|
||||
workspaceId: options.workspace,
|
||||
userId: options.user,
|
||||
localLeaseId: options.byokLeaseId,
|
||||
access: await this.access(options),
|
||||
managedTargetId: options.managedTargetId,
|
||||
targetOverride: this.targetOverride(cond),
|
||||
request,
|
||||
},
|
||||
async requestJson => {
|
||||
const toolRequest = JSON.parse(requestJson) as LlmToolCallbackRequest;
|
||||
return JSON.stringify(
|
||||
await executeToolCall(toolSet, toolRequest, {
|
||||
signal: options.signal,
|
||||
messages,
|
||||
})
|
||||
);
|
||||
},
|
||||
{ maxSteps: 20, signal: options.signal }
|
||||
);
|
||||
const runtimeEvents = this.events;
|
||||
const eventContext = this.eventContext(options);
|
||||
async function* productEvents() {
|
||||
for await (const event of rawStream) {
|
||||
if ('route' in event) {
|
||||
await runtimeEvents.consume([event], eventContext);
|
||||
} else if (event.type === 'error') {
|
||||
throw mapNativeSemanticError(
|
||||
new Error(
|
||||
typeof event.message === 'string'
|
||||
? event.message
|
||||
: 'native runtime stream error'
|
||||
)
|
||||
);
|
||||
} else {
|
||||
yield event;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { request, stream: productEvents() };
|
||||
}
|
||||
|
||||
async text(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
filter?: ProviderFilter
|
||||
options: CopilotChatOptions = {},
|
||||
_filter?: ProviderFilter
|
||||
) {
|
||||
return await this.executePlan(
|
||||
() => this.plans.buildTextPlan(cond, messages, options, filter),
|
||||
plan => this.engine.execute(plan)
|
||||
const prepared = await this.stream('prompt.text', cond, messages, options);
|
||||
return await new NativeProviderAdapter(() => prepared.stream).text(
|
||||
prepared.request,
|
||||
options.signal,
|
||||
messages
|
||||
);
|
||||
}
|
||||
|
||||
async *streamText(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
filter?: ProviderFilter
|
||||
options: CopilotChatOptions = {},
|
||||
_filter?: ProviderFilter
|
||||
): AsyncIterableIterator<string> {
|
||||
yield* this.executeStreamPlan(
|
||||
() => this.plans.buildStreamTextPlan(cond, messages, options, filter),
|
||||
plan => this.engine.executeStream(plan)
|
||||
const prepared = await this.stream('chat.default', cond, messages, options);
|
||||
yield* new NativeProviderAdapter(() => prepared.stream).streamText(
|
||||
prepared.request,
|
||||
options.signal,
|
||||
messages
|
||||
);
|
||||
}
|
||||
|
||||
async *streamObject(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
filter?: ProviderFilter
|
||||
options: CopilotChatOptions = {},
|
||||
_filter?: ProviderFilter
|
||||
): AsyncIterableIterator<StreamObject> {
|
||||
yield* this.executeStreamPlan(
|
||||
() => this.plans.buildStreamObjectPlan(cond, messages, options, filter),
|
||||
plan => this.engine.executeStream(plan)
|
||||
const prepared = await this.stream('chat.default', cond, messages, options);
|
||||
yield* new NativeProviderAdapter(() => prepared.stream).streamObject(
|
||||
prepared.request,
|
||||
options.signal,
|
||||
messages
|
||||
);
|
||||
}
|
||||
|
||||
async generateStructured(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotStructuredOptions,
|
||||
filter?: ProviderFilter,
|
||||
responseContract?: RequiredStructuredOutputContract
|
||||
options: CopilotStructuredOptions = {},
|
||||
_filter?: ProviderFilter,
|
||||
responseContract?: RequiredStructuredOutputContract,
|
||||
slot = 'prompt.structured'
|
||||
) {
|
||||
return await this.executePlan(
|
||||
() =>
|
||||
this.plans.buildStructuredPlan(
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
filter,
|
||||
responseContract
|
||||
),
|
||||
plan => this.engine.execute(plan)
|
||||
const contract = requireStructuredOutputContract(responseContract);
|
||||
if (!contract) {
|
||||
throw new CopilotPromptInvalid('Structured schema contract is required');
|
||||
}
|
||||
const { request } = await buildCanonicalNativeStructuredRequest({
|
||||
model: 'route-selected',
|
||||
messages,
|
||||
options,
|
||||
responseContract: contract,
|
||||
attachmentCapability,
|
||||
});
|
||||
const result = (await this.execute(slot, request, cond, options)) as {
|
||||
output_json?: unknown;
|
||||
output_text: string;
|
||||
};
|
||||
if (result.output_json === undefined) {
|
||||
throw new CopilotPromptInvalid(
|
||||
'Structured response is missing output_json'
|
||||
);
|
||||
}
|
||||
return JSON.stringify(
|
||||
llmValidateJsonSchema(request.schema, result.output_json)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -123,87 +325,90 @@ export class CapabilityRuntime {
|
||||
messages: PromptMessage[],
|
||||
options: CopilotStructuredOptions,
|
||||
responseContract?: RequiredStructuredOutputContract,
|
||||
filter?: ProviderFilter
|
||||
filter?: ProviderFilter,
|
||||
slot = 'prompt.structured'
|
||||
) {
|
||||
const validatedResponseContract =
|
||||
requireStructuredOutputContract(responseContract);
|
||||
if (!options || !validatedResponseContract) {
|
||||
const contract = requireStructuredOutputContract(responseContract);
|
||||
if (!contract) {
|
||||
throw new CopilotPromptInvalid('Structured schema contract is required');
|
||||
}
|
||||
|
||||
const output = await this.generateStructured(
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
filter,
|
||||
validatedResponseContract
|
||||
const value = JSON.parse(
|
||||
await this.generateStructured(
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
filter,
|
||||
contract,
|
||||
slot
|
||||
)
|
||||
);
|
||||
const value = JSON.parse(output);
|
||||
return ValidatedStructuredValueSchema.parse({
|
||||
value,
|
||||
schemaHash: validatedResponseContract.schemaHash,
|
||||
schemaHash: contract.schemaHash,
|
||||
schemaValidationVersion: 'json-schema-v1',
|
||||
provider: filter?.prefer ?? 'auto',
|
||||
model: providerModelId(cond.modelId),
|
||||
provider: 'auto',
|
||||
model: 'route-selected',
|
||||
});
|
||||
}
|
||||
|
||||
async embeddingConfigured(modelId: string) {
|
||||
try {
|
||||
return this.hasNativeDispatch(
|
||||
await this.plans.buildEmbeddingPlan(modelId, 'ping'),
|
||||
'embedding'
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
async embeddingConfigured(_modelId: string) {
|
||||
return this.config.copilot.enabled;
|
||||
}
|
||||
|
||||
async embed(
|
||||
modelId: string,
|
||||
_modelId: string,
|
||||
input: string | string[],
|
||||
options?: CopilotEmbeddingOptions
|
||||
options: CopilotEmbeddingOptions = {}
|
||||
) {
|
||||
return await this.executePlan(
|
||||
() => this.plans.buildEmbeddingPlan(modelId, input, options),
|
||||
plan => this.engine.execute(plan)
|
||||
);
|
||||
const result = (await this.execute(
|
||||
'index.embedding',
|
||||
buildLlmEmbeddingRequest({
|
||||
model: 'route-selected',
|
||||
inputs: Array.isArray(input) ? input : [input],
|
||||
dimensions: options.dimensions,
|
||||
}),
|
||||
{},
|
||||
options
|
||||
)) as { embeddings: number[][] };
|
||||
return result.embeddings;
|
||||
}
|
||||
|
||||
async rerankConfigured(modelId: string) {
|
||||
try {
|
||||
return this.hasNativeDispatch(
|
||||
await this.plans.buildRerankPlan(modelId, {
|
||||
query: 'ping',
|
||||
candidates: [{ text: 'ping' }],
|
||||
}),
|
||||
'rerank'
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
async rerankConfigured(_modelId: string) {
|
||||
return this.config.copilot.enabled;
|
||||
}
|
||||
|
||||
async rerank(
|
||||
modelId: string,
|
||||
_modelId: string,
|
||||
request: CopilotRerankRequest,
|
||||
options?: CopilotChatOptions
|
||||
options: CopilotChatOptions = {}
|
||||
) {
|
||||
return await this.executePlan(
|
||||
() => this.plans.buildRerankPlan(modelId, request, options),
|
||||
plan => this.engine.execute(plan)
|
||||
);
|
||||
const result = (await this.execute(
|
||||
'search.rerank',
|
||||
buildLlmRerankRequest('route-selected', request),
|
||||
{},
|
||||
options
|
||||
)) as { scores: number[] };
|
||||
return result.scores;
|
||||
}
|
||||
|
||||
async *streamImageArtifacts(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotImageOptions,
|
||||
filter?: ProviderFilter
|
||||
options: CopilotImageOptions = {},
|
||||
_filter?: ProviderFilter,
|
||||
slot = 'image.generate'
|
||||
): AsyncIterableIterator<NativeImageArtifact> {
|
||||
yield* this.executeStreamPlan(
|
||||
() => this.plans.buildImagePlan(cond, messages, options, filter),
|
||||
plan => this.engine.executeImageArtifacts(plan)
|
||||
);
|
||||
const { quality, seed } = options;
|
||||
const result = (await this.execute(
|
||||
slot,
|
||||
buildLlmImageRequestFromMessages({
|
||||
model: 'route-selected',
|
||||
messages: preparePromptMessagesForNativeRequest(messages, true),
|
||||
options: { quality, seed },
|
||||
}),
|
||||
cond,
|
||||
options
|
||||
)) as LlmImageResponse;
|
||||
yield* result.images;
|
||||
}
|
||||
}
|
||||
|
||||
-104
@@ -1,104 +0,0 @@
|
||||
import {
|
||||
type LlmBackendConfig,
|
||||
llmCompileExecutionPlan,
|
||||
type LlmEmbeddingRequest,
|
||||
type LlmImageRequest,
|
||||
type LlmProtocol,
|
||||
type LlmRequest,
|
||||
type LlmRerankRequest,
|
||||
type LlmStructuredRequest,
|
||||
} from '../../../../native';
|
||||
import type {
|
||||
CopilotProviderType,
|
||||
ModelConditions,
|
||||
PromptMessage,
|
||||
} from '../../providers/types';
|
||||
|
||||
// Owner: runtime core mirror facade.
|
||||
// The semantic source of truth is the native/Rust execution-plan contract
|
||||
// behind llmCompileExecutionPlan(); this file only keeps the TypeScript shape
|
||||
// needed by Node live-plan assembly until generated/native TS types replace it.
|
||||
export type ExecutionRequestKind =
|
||||
| 'text'
|
||||
| 'streamText'
|
||||
| 'streamObject'
|
||||
| 'structured'
|
||||
| 'embedding'
|
||||
| 'rerank'
|
||||
| 'image';
|
||||
|
||||
export type ExecutionRoute = {
|
||||
providerId: string;
|
||||
protocol: LlmProtocol;
|
||||
model: string;
|
||||
backendConfig: LlmBackendConfig;
|
||||
};
|
||||
|
||||
export type ExecutionTransportContract =
|
||||
| { kind: 'chat'; request: LlmRequest }
|
||||
| { kind: 'structured'; request: LlmStructuredRequest }
|
||||
| { kind: 'embedding'; request: LlmEmbeddingRequest }
|
||||
| { kind: 'rerank'; request: LlmRerankRequest }
|
||||
| { kind: 'image'; request: LlmImageRequest };
|
||||
|
||||
export type SerializableExecutionPlanRequest =
|
||||
| {
|
||||
kind: 'text' | 'streamText' | 'streamObject';
|
||||
cond: ModelConditions;
|
||||
messages: PromptMessage[];
|
||||
options?: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
kind: 'structured';
|
||||
cond: ModelConditions;
|
||||
messages: PromptMessage[];
|
||||
options?: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
kind: 'image';
|
||||
cond: ModelConditions;
|
||||
messages: PromptMessage[];
|
||||
options?: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
kind: 'embedding';
|
||||
cond: ModelConditions;
|
||||
modelId: string;
|
||||
input: string | string[];
|
||||
options?: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
kind: 'rerank';
|
||||
cond: ModelConditions;
|
||||
modelId: string;
|
||||
request: {
|
||||
query: string;
|
||||
candidates: { id?: string; text: string }[];
|
||||
topK?: number;
|
||||
};
|
||||
options?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type SerializableExecutionPlan = {
|
||||
routes: ExecutionRoute[];
|
||||
request: SerializableExecutionPlanRequest;
|
||||
transport?: ExecutionTransportContract;
|
||||
routePolicy: { fallbackOrder: string[] };
|
||||
runtimePolicy: {
|
||||
prefer?: CopilotProviderType;
|
||||
maxSteps?: number;
|
||||
};
|
||||
attachmentPolicy: {
|
||||
materializeRemoteAttachments: boolean;
|
||||
};
|
||||
responsePostprocess: {
|
||||
mode: ExecutionRequestKind;
|
||||
};
|
||||
hostContext?: {
|
||||
currentMessages?: PromptMessage[];
|
||||
};
|
||||
};
|
||||
|
||||
export function parseExecutionPlan(value: unknown) {
|
||||
return llmCompileExecutionPlan<SerializableExecutionPlan>(value);
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
export * from './execution-plan-contract';
|
||||
export * from './native-contract';
|
||||
export * from './prompt-contract';
|
||||
export * from './runtime-event-contract';
|
||||
export * from './shared';
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
import serverNativeModule, {
|
||||
type CapabilityMatchRequest,
|
||||
type CapabilityMatchResponse,
|
||||
type ModelRegistryMatchRequest,
|
||||
type ModelRegistryMatchResponse,
|
||||
type ModelRegistryResolveRequest,
|
||||
type ModelRegistryResolveResponse,
|
||||
type ModelRegistryVariantContract,
|
||||
type ProviderDriverSpec,
|
||||
type RequestedModelMatchRequest,
|
||||
type RequestedModelMatchResponse,
|
||||
} from '@affine/server-native';
|
||||
|
||||
// Owner: native/Rust contract facade.
|
||||
// These types and validators intentionally proxy @affine/server-native and
|
||||
// must not grow independent runtime semantics in Node.
|
||||
export type {
|
||||
CapabilityMatchRequest,
|
||||
CapabilityMatchResponse,
|
||||
ProviderDriverSpec,
|
||||
RequestedModelMatchRequest,
|
||||
RequestedModelMatchResponse,
|
||||
};
|
||||
|
||||
export type CopilotModelBackendKind = ModelRegistryMatchRequest['backendKind'];
|
||||
export type ModelRegistryVariant = ModelRegistryVariantContract;
|
||||
export type ResolveModelRegistryVariantRequest = ModelRegistryResolveRequest;
|
||||
export type ResolveModelRegistryVariantResponse = ModelRegistryResolveResponse;
|
||||
export type MatchModelRegistryRequest = ModelRegistryMatchRequest;
|
||||
export type MatchModelRegistryResponse = ModelRegistryMatchResponse;
|
||||
|
||||
function validateNativeContract<T>(name: string, value: unknown): T {
|
||||
return serverNativeModule.llmValidateContract(name, value) as T;
|
||||
}
|
||||
|
||||
export function parseCapabilityMatchRequest(value: unknown) {
|
||||
return validateNativeContract<CapabilityMatchRequest>(
|
||||
'capabilityMatchRequest',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parseCapabilityMatchResponse(value: unknown) {
|
||||
return validateNativeContract<CapabilityMatchResponse>(
|
||||
'capabilityMatchResponse',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parseResolveModelRegistryVariantRequest(value: unknown) {
|
||||
return validateNativeContract<ResolveModelRegistryVariantRequest>(
|
||||
'modelRegistryResolveRequest',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parseResolveModelRegistryVariantResponse(value: unknown) {
|
||||
return validateNativeContract<ResolveModelRegistryVariantResponse>(
|
||||
'modelRegistryResolveResponse',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parseMatchModelRegistryRequest(value: unknown) {
|
||||
return validateNativeContract<MatchModelRegistryRequest>(
|
||||
'modelRegistryMatchRequest',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parseMatchModelRegistryResponse(value: unknown) {
|
||||
return validateNativeContract<MatchModelRegistryResponse>(
|
||||
'modelRegistryMatchResponse',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parseProviderDriverSpec(value: unknown) {
|
||||
return validateNativeContract<ProviderDriverSpec>(
|
||||
'providerDriverSpec',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parseRequestedModelMatchRequest(value: unknown) {
|
||||
return validateNativeContract<RequestedModelMatchRequest>(
|
||||
'requestedModelMatchRequest',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parseRequestedModelMatchResponse(value: unknown) {
|
||||
return validateNativeContract<RequestedModelMatchResponse>(
|
||||
'requestedModelMatchResponse',
|
||||
value
|
||||
);
|
||||
}
|
||||
@@ -1,15 +1,6 @@
|
||||
import {
|
||||
llmValidateContract,
|
||||
type NativePromptCountTokensRequest,
|
||||
type NativePromptCountTokensResponse,
|
||||
type NativePromptMetadataRequest,
|
||||
type NativePromptMetadataResponse,
|
||||
type NativePromptRenderRequest,
|
||||
type NativePromptRenderResponse,
|
||||
type NativePromptSessionRenderRequest,
|
||||
type NativePromptSessionRenderResponse,
|
||||
type PromptMessageContract as NativePromptMessageContract,
|
||||
type PromptStructuredResponseContract as NativePromptStructuredResponseContract,
|
||||
import type {
|
||||
PromptMessageContract as NativePromptMessageContract,
|
||||
PromptStructuredResponseContract as NativePromptStructuredResponseContract,
|
||||
} from '../../../../native';
|
||||
import { normalizePromptResponseFormat } from './structured-output-contract';
|
||||
|
||||
@@ -32,14 +23,6 @@ type PromptMessageInput = {
|
||||
params?: Record<string, unknown> | null;
|
||||
responseFormat?: PromptResponseFormat | null;
|
||||
};
|
||||
export type PromptRenderContract = NativePromptRenderRequest;
|
||||
export type PromptRenderResult = NativePromptRenderResponse;
|
||||
export type PromptTokenCountContract = NativePromptCountTokensRequest;
|
||||
export type PromptTokenCountResult = NativePromptCountTokensResponse;
|
||||
export type PromptMetadataContract = NativePromptMetadataRequest;
|
||||
export type PromptMetadataResult = NativePromptMetadataResponse;
|
||||
export type PromptSessionContract = NativePromptSessionRenderRequest;
|
||||
export type PromptSessionResult = NativePromptSessionRenderResponse;
|
||||
export type NativePromptResponseFormatProjection = {
|
||||
nativeResponseFormat?: PromptStructuredResponseContract;
|
||||
};
|
||||
@@ -82,17 +65,3 @@ export function projectPromptMessageForNative(
|
||||
|
||||
return { message: nativeMessage, nativeResponseFormat };
|
||||
}
|
||||
|
||||
export function parsePromptRenderContract(value: unknown) {
|
||||
return llmValidateContract<PromptRenderContract>(
|
||||
'promptRenderContract',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
export function parsePromptSessionContract(value: unknown) {
|
||||
return llmValidateContract<PromptSessionContract>(
|
||||
'promptSessionContract',
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { metrics } from '../../../base';
|
||||
import { Models } from '../../../models';
|
||||
import { type ByokFeatureKind, ByokProviderSource } from '../byok/types';
|
||||
|
||||
export type CopilotRuntimeRouteIdentity = {
|
||||
profileId: string;
|
||||
source: 'server' | 'local' | 'affine_cloud';
|
||||
provider: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
export type CopilotRuntimeEvent =
|
||||
| { type: 'route_selected'; route: CopilotRuntimeRouteIdentity }
|
||||
| {
|
||||
type: 'route_failed';
|
||||
route: CopilotRuntimeRouteIdentity;
|
||||
errorKind: string;
|
||||
}
|
||||
| {
|
||||
type: 'usage';
|
||||
route: CopilotRuntimeRouteIdentity;
|
||||
usage: {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
total_tokens?: number;
|
||||
cached_tokens?: number;
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type CopilotRuntimeEventContext = {
|
||||
workspaceId?: string;
|
||||
userId?: string;
|
||||
sessionId?: string;
|
||||
taskId?: string;
|
||||
actionId?: string;
|
||||
billingUnitId?: string;
|
||||
featureKind: ByokFeatureKind;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CopilotRuntimeEventConsumer {
|
||||
private readonly logger = new Logger(CopilotRuntimeEventConsumer.name);
|
||||
|
||||
constructor(private readonly models: Models) {}
|
||||
|
||||
async consume(
|
||||
events: CopilotRuntimeEvent[],
|
||||
context: CopilotRuntimeEventContext
|
||||
) {
|
||||
for (const event of events) {
|
||||
try {
|
||||
if (event.type === 'usage') {
|
||||
await this.recordUsage(event, context);
|
||||
} else if (event.type === 'route_failed') {
|
||||
await this.recordFailure(event, context);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to consume copilot runtime event: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async recordUsage(
|
||||
event: Extract<CopilotRuntimeEvent, { type: 'usage' }>,
|
||||
context: CopilotRuntimeEventContext
|
||||
) {
|
||||
if (!context.workspaceId || event.route.source === 'affine_cloud') {
|
||||
return;
|
||||
}
|
||||
const usage = event.usage;
|
||||
metrics.ai.counter('byok_usage').add(1, {
|
||||
provider: event.route.provider,
|
||||
source: event.route.source,
|
||||
feature: context.featureKind,
|
||||
});
|
||||
await this.models.copilotUsage.create({
|
||||
workspaceId: context.workspaceId,
|
||||
userId: context.userId,
|
||||
provider: event.route.provider,
|
||||
providerSource:
|
||||
event.route.source === 'server'
|
||||
? ByokProviderSource.Server
|
||||
: ByokProviderSource.Local,
|
||||
featureKind: context.featureKind,
|
||||
model: event.route.model,
|
||||
sessionId: context.sessionId,
|
||||
taskId: context.taskId,
|
||||
actionId: context.actionId,
|
||||
billingUnitId: context.billingUnitId,
|
||||
promptTokens: usage.prompt_tokens ?? usage.input_tokens ?? 0,
|
||||
completionTokens: usage.completion_tokens ?? usage.output_tokens ?? 0,
|
||||
totalTokens: usage.total_tokens ?? 0,
|
||||
cachedTokens: usage.cached_tokens ?? 0,
|
||||
});
|
||||
if (event.route.source === 'server') {
|
||||
await this.models.copilotWorkspaceByokConfig.touchUsed(
|
||||
context.workspaceId,
|
||||
event.route.profileId
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async recordFailure(
|
||||
event: Extract<CopilotRuntimeEvent, { type: 'route_failed' }>,
|
||||
context: CopilotRuntimeEventContext
|
||||
) {
|
||||
metrics.ai.counter('byok_route_failure').add(1, {
|
||||
provider: event.route.provider,
|
||||
source: event.route.source,
|
||||
feature: context.featureKind,
|
||||
reason: event.errorKind,
|
||||
});
|
||||
if (context.workspaceId && event.route.source === 'server') {
|
||||
await this.models.copilotWorkspaceByokConfig.markFailure(
|
||||
context.workspaceId,
|
||||
event.route.profileId,
|
||||
event.errorKind
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { metrics } from '../../../base';
|
||||
import type { ResolvedCopilotProvider } from '../providers/factory';
|
||||
import type { CopilotProviderType } from '../providers/types';
|
||||
import type { ExecutionRequestKind } from './execution-plan';
|
||||
|
||||
type ExecutionDispatchPath = 'prepared_routes';
|
||||
|
||||
export function summarizePreparedRoutes(
|
||||
routes: Array<Pick<ResolvedCopilotProvider, 'prepared'>>
|
||||
) {
|
||||
const preparedCount = routes.filter(route => !!route.prepared).length;
|
||||
return {
|
||||
routeCount: routes.length,
|
||||
preparedCount,
|
||||
preparedMode:
|
||||
preparedCount === 0
|
||||
? 'none'
|
||||
: preparedCount === routes.length
|
||||
? 'all'
|
||||
: 'partial',
|
||||
} as const;
|
||||
}
|
||||
|
||||
function planAttrs(
|
||||
kind: ExecutionRequestKind,
|
||||
prefer?: CopilotProviderType,
|
||||
routes?: ResolvedCopilotProvider[]
|
||||
) {
|
||||
const summary = summarizePreparedRoutes(routes ?? []);
|
||||
return {
|
||||
kind,
|
||||
prefer: prefer ?? 'auto',
|
||||
prepared: summary.preparedMode,
|
||||
route_count: summary.routeCount,
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class CopilotExecutionMetrics {
|
||||
recordPlan(
|
||||
kind: ExecutionRequestKind,
|
||||
routes: ResolvedCopilotProvider[],
|
||||
prefer?: CopilotProviderType
|
||||
) {
|
||||
const attrs = planAttrs(kind, prefer, routes);
|
||||
metrics.ai.counter('execution_plan_total').add(1, attrs);
|
||||
metrics.ai.histogram('execution_plan_routes').record(attrs.route_count, {
|
||||
kind: attrs.kind,
|
||||
prefer: attrs.prefer,
|
||||
prepared: attrs.prepared,
|
||||
});
|
||||
}
|
||||
|
||||
recordDispatch(
|
||||
kind: ExecutionRequestKind,
|
||||
path: ExecutionDispatchPath,
|
||||
routeCount: number
|
||||
) {
|
||||
const attrs = { kind, path };
|
||||
metrics.ai.counter('execution_dispatch_total').add(1, attrs);
|
||||
metrics.ai.histogram('execution_dispatch_routes').record(routeCount, attrs);
|
||||
}
|
||||
}
|
||||
@@ -1,827 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import type {
|
||||
LlmPreparedDispatchRoute,
|
||||
LlmPreparedEmbeddingDispatchRoute,
|
||||
LlmPreparedImageDispatchRoute,
|
||||
LlmPreparedRerankDispatchRoute,
|
||||
LlmPreparedStructuredDispatchRoute,
|
||||
} from '../../../native';
|
||||
import { llmNormalizePreparedRoutes } from '../../../native';
|
||||
import {
|
||||
CopilotProviderFactory,
|
||||
type ResolvedCopilotProvider,
|
||||
} from '../providers/factory';
|
||||
import type {
|
||||
PreparedNativeEmbeddingExecution,
|
||||
PreparedNativeExecution,
|
||||
PreparedNativeImageExecution,
|
||||
PreparedNativeRerankExecution,
|
||||
PreparedNativeStructuredExecution,
|
||||
} from '../providers/provider-runtime-contract';
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotEmbeddingOptions,
|
||||
CopilotImageOptions,
|
||||
CopilotProviderType,
|
||||
CopilotRerankRequest,
|
||||
CopilotStructuredOptions,
|
||||
ModelConditions,
|
||||
PromptMessage,
|
||||
} from '../providers/types';
|
||||
import { ModelOutputType } from '../providers/types';
|
||||
import type { RequiredStructuredOutputContract } from './contracts';
|
||||
import {
|
||||
type ExecutionRequestKind,
|
||||
type ExecutionRoute,
|
||||
type ExecutionTransportContract,
|
||||
parseExecutionPlan,
|
||||
type SerializableExecutionPlan,
|
||||
type SerializableExecutionPlanRequest,
|
||||
} from './contracts/execution-plan-contract';
|
||||
import { CopilotExecutionMetrics } from './execution-metrics';
|
||||
|
||||
export type { ExecutionRequestKind };
|
||||
|
||||
type ProviderFilter = {
|
||||
prefer?: CopilotProviderType;
|
||||
};
|
||||
|
||||
type BaseExecutionRequest<TKind extends ExecutionRequestKind> = {
|
||||
kind: TKind;
|
||||
cond: ModelConditions;
|
||||
};
|
||||
|
||||
type TextExecutionRequest = BaseExecutionRequest<'text'> & {
|
||||
messages: PromptMessage[];
|
||||
options?: CopilotChatOptions;
|
||||
};
|
||||
|
||||
type StreamTextExecutionRequest = BaseExecutionRequest<'streamText'> & {
|
||||
messages: PromptMessage[];
|
||||
options?: CopilotChatOptions;
|
||||
};
|
||||
|
||||
type StreamObjectExecutionRequest = BaseExecutionRequest<'streamObject'> & {
|
||||
messages: PromptMessage[];
|
||||
options?: CopilotChatOptions;
|
||||
};
|
||||
|
||||
type StructuredExecutionRequest = BaseExecutionRequest<'structured'> & {
|
||||
messages: PromptMessage[];
|
||||
options?: CopilotStructuredOptions;
|
||||
};
|
||||
|
||||
type ImageExecutionRequest = BaseExecutionRequest<'image'> & {
|
||||
messages: PromptMessage[];
|
||||
options?: CopilotImageOptions;
|
||||
};
|
||||
|
||||
type EmbeddingExecutionRequest = BaseExecutionRequest<'embedding'> & {
|
||||
modelId: string;
|
||||
input: string | string[];
|
||||
options?: CopilotEmbeddingOptions;
|
||||
};
|
||||
|
||||
type RerankExecutionRequest = BaseExecutionRequest<'rerank'> & {
|
||||
modelId: string;
|
||||
request: CopilotRerankRequest;
|
||||
options?: CopilotChatOptions;
|
||||
};
|
||||
|
||||
export type ExecutionPlanRequest =
|
||||
| TextExecutionRequest
|
||||
| StreamTextExecutionRequest
|
||||
| StreamObjectExecutionRequest
|
||||
| StructuredExecutionRequest
|
||||
| ImageExecutionRequest
|
||||
| EmbeddingExecutionRequest
|
||||
| RerankExecutionRequest;
|
||||
|
||||
export type ExecutionPlanForKind<TKind extends ExecutionRequestKind> =
|
||||
ExecutionPlan & {
|
||||
request: Extract<ExecutionPlanRequest, { kind: TKind }>;
|
||||
};
|
||||
|
||||
type NativePreparedDispatchPlan<TRoute, TPrepared> = {
|
||||
routes: TRoute[];
|
||||
prepared: TPrepared;
|
||||
};
|
||||
|
||||
export type NativeChatDispatchPlan = NativePreparedDispatchPlan<
|
||||
LlmPreparedDispatchRoute,
|
||||
PreparedNativeExecution
|
||||
> & {
|
||||
hasTools: boolean;
|
||||
};
|
||||
|
||||
export type NativeStructuredDispatchPlan = NativePreparedDispatchPlan<
|
||||
LlmPreparedStructuredDispatchRoute,
|
||||
PreparedNativeStructuredExecution
|
||||
>;
|
||||
|
||||
export type NativeEmbeddingDispatchPlan = NativePreparedDispatchPlan<
|
||||
LlmPreparedEmbeddingDispatchRoute,
|
||||
PreparedNativeEmbeddingExecution
|
||||
>;
|
||||
|
||||
export type NativeRerankDispatchPlan = NativePreparedDispatchPlan<
|
||||
LlmPreparedRerankDispatchRoute,
|
||||
PreparedNativeRerankExecution
|
||||
>;
|
||||
|
||||
export type NativeImageDispatchPlan = NativePreparedDispatchPlan<
|
||||
LlmPreparedImageDispatchRoute,
|
||||
PreparedNativeImageExecution
|
||||
>;
|
||||
|
||||
export type ExecutionPlan = {
|
||||
nativeDispatch?: {
|
||||
chat?: NativeChatDispatchPlan;
|
||||
structured?: NativeStructuredDispatchPlan;
|
||||
embedding?: NativeEmbeddingDispatchPlan;
|
||||
rerank?: NativeRerankDispatchPlan;
|
||||
image?: NativeImageDispatchPlan;
|
||||
};
|
||||
serializable?: SerializableExecutionPlan;
|
||||
transport?: ExecutionTransportContract;
|
||||
request: ExecutionPlanRequest;
|
||||
routePolicy: { fallbackOrder: string[] };
|
||||
runtimePolicy: {
|
||||
prefer?: CopilotProviderType;
|
||||
};
|
||||
attachmentPolicy: {
|
||||
materializeRemoteAttachments: boolean;
|
||||
};
|
||||
responsePostprocess: { mode: ExecutionRequestKind };
|
||||
hostPersistence: {
|
||||
persistAssistantTurn: boolean;
|
||||
outputKind: ExecutionRequestKind;
|
||||
};
|
||||
hostContext: {
|
||||
signal?: AbortSignal;
|
||||
currentMessages?: PromptMessage[];
|
||||
};
|
||||
};
|
||||
|
||||
type PreparedRouteLike<TRequest = unknown> = {
|
||||
route: {
|
||||
providerId: string;
|
||||
protocol: PreparedNativeExecution['route']['protocol'];
|
||||
model: string;
|
||||
backendConfig: PreparedNativeExecution['route']['backendConfig'];
|
||||
};
|
||||
request: TRequest;
|
||||
};
|
||||
|
||||
function buildPreparedTransport<
|
||||
TKind extends ExecutionTransportContract['kind'],
|
||||
TPrepared extends PreparedRouteLike,
|
||||
>(
|
||||
kind: TKind,
|
||||
routes: ResolvedCopilotProvider[],
|
||||
getPrepared: (route: ResolvedCopilotProvider) => TPrepared | undefined
|
||||
): ExecutionTransportContract | undefined {
|
||||
const prepared =
|
||||
routes.length === 1 ? routes[0] && getPrepared(routes[0]) : undefined;
|
||||
if (!prepared) {
|
||||
return;
|
||||
}
|
||||
|
||||
return {
|
||||
kind,
|
||||
request: prepared.request,
|
||||
} as ExecutionTransportContract;
|
||||
}
|
||||
|
||||
function collectPreparedRoutes<TPrepared extends PreparedRouteLike, TRoute>(
|
||||
routes: ResolvedCopilotProvider[],
|
||||
getPrepared: (route: ResolvedCopilotProvider) => TPrepared | undefined,
|
||||
mapPreparedRoute: (prepared: TPrepared) => TRoute
|
||||
): TRoute[] | undefined {
|
||||
if (!routes.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const preparedRoutes: TRoute[] = [];
|
||||
for (const route of routes) {
|
||||
const prepared = getPrepared(route);
|
||||
if (!prepared) {
|
||||
return;
|
||||
}
|
||||
preparedRoutes.push(mapPreparedRoute(prepared));
|
||||
}
|
||||
|
||||
return preparedRoutes;
|
||||
}
|
||||
|
||||
function buildPreparedDispatchPlan<
|
||||
TPrepared extends PreparedRouteLike,
|
||||
TRoute,
|
||||
TDispatch extends NativePreparedDispatchPlan<TRoute, TPrepared>,
|
||||
>(
|
||||
routes: ResolvedCopilotProvider[],
|
||||
getPrepared: (route: ResolvedCopilotProvider) => TPrepared | undefined,
|
||||
mapPreparedRoute: (prepared: TPrepared) => TRoute,
|
||||
buildPreparedDispatchResult?: (
|
||||
preparedRoutes: TRoute[],
|
||||
prepared: TPrepared
|
||||
) => TDispatch
|
||||
): TDispatch | undefined {
|
||||
const preparedRoutes = collectPreparedRoutes(
|
||||
routes,
|
||||
getPrepared,
|
||||
mapPreparedRoute
|
||||
);
|
||||
const prepared = routes[0] && getPrepared(routes[0]);
|
||||
if (!preparedRoutes || !prepared) {
|
||||
return;
|
||||
}
|
||||
|
||||
const normalizedRoutes = llmNormalizePreparedRoutes<TRoute[]>(preparedRoutes);
|
||||
|
||||
return buildPreparedDispatchResult
|
||||
? buildPreparedDispatchResult(normalizedRoutes, prepared)
|
||||
: ({ routes: normalizedRoutes, prepared } as TDispatch);
|
||||
}
|
||||
|
||||
type DispatchPreparedRoute<TRequest> = {
|
||||
provider_id: string;
|
||||
protocol: PreparedNativeExecution['route']['protocol'];
|
||||
model: string;
|
||||
config: PreparedNativeExecution['route']['backendConfig'];
|
||||
request: TRequest;
|
||||
};
|
||||
|
||||
function mapPreparedDispatchRoute<TRequest>(
|
||||
prepared: PreparedRouteLike<TRequest>
|
||||
): DispatchPreparedRoute<TRequest> {
|
||||
return {
|
||||
provider_id: prepared.route.providerId,
|
||||
protocol: prepared.route.protocol,
|
||||
model: prepared.route.model,
|
||||
config: prepared.route.backendConfig,
|
||||
request: prepared.request,
|
||||
};
|
||||
}
|
||||
|
||||
type PreparedExecutionArtifactSpec<
|
||||
TKind extends ExecutionTransportContract['kind'],
|
||||
TPrepared extends PreparedRouteLike,
|
||||
TRoute,
|
||||
TDispatch extends NativePreparedDispatchPlan<TRoute, TPrepared>,
|
||||
> = {
|
||||
transportKind: TKind;
|
||||
getPrepared: (route: ResolvedCopilotProvider) => TPrepared | undefined;
|
||||
mapPreparedRoute: (prepared: TPrepared) => TRoute;
|
||||
buildPreparedDispatch?: (
|
||||
preparedRoutes: TRoute[],
|
||||
prepared: TPrepared
|
||||
) => TDispatch;
|
||||
};
|
||||
|
||||
type PreparedExecutionArtifacts<TDispatch> = {
|
||||
dispatch?: TDispatch;
|
||||
transport?: ExecutionTransportContract;
|
||||
};
|
||||
|
||||
function buildPreparedExecutionArtifacts<
|
||||
TKind extends ExecutionTransportContract['kind'],
|
||||
TPrepared extends PreparedRouteLike,
|
||||
TRoute,
|
||||
TDispatch extends NativePreparedDispatchPlan<TRoute, TPrepared>,
|
||||
>(
|
||||
routes: ResolvedCopilotProvider[],
|
||||
spec: PreparedExecutionArtifactSpec<TKind, TPrepared, TRoute, TDispatch>
|
||||
): PreparedExecutionArtifacts<TDispatch> {
|
||||
return {
|
||||
dispatch: buildPreparedDispatchPlan(
|
||||
routes,
|
||||
spec.getPrepared,
|
||||
spec.mapPreparedRoute,
|
||||
spec.buildPreparedDispatch
|
||||
),
|
||||
transport: buildPreparedTransport(
|
||||
spec.transportKind,
|
||||
routes,
|
||||
spec.getPrepared
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const chatArtifactSpec: PreparedExecutionArtifactSpec<
|
||||
'chat',
|
||||
PreparedNativeExecution,
|
||||
LlmPreparedDispatchRoute,
|
||||
NativeChatDispatchPlan
|
||||
> = {
|
||||
transportKind: 'chat',
|
||||
getPrepared: route => route.prepared,
|
||||
mapPreparedRoute: mapPreparedDispatchRoute,
|
||||
buildPreparedDispatch: (preparedRoutes, prepared) => ({
|
||||
routes: preparedRoutes,
|
||||
prepared,
|
||||
hasTools: Object.keys(prepared.tools).length > 0,
|
||||
}),
|
||||
};
|
||||
|
||||
const structuredArtifactSpec: PreparedExecutionArtifactSpec<
|
||||
'structured',
|
||||
PreparedNativeStructuredExecution,
|
||||
LlmPreparedStructuredDispatchRoute,
|
||||
NativeStructuredDispatchPlan
|
||||
> = {
|
||||
transportKind: 'structured',
|
||||
getPrepared: route => route.preparedStructured,
|
||||
mapPreparedRoute: mapPreparedDispatchRoute,
|
||||
};
|
||||
|
||||
const embeddingArtifactSpec: PreparedExecutionArtifactSpec<
|
||||
'embedding',
|
||||
PreparedNativeEmbeddingExecution,
|
||||
LlmPreparedEmbeddingDispatchRoute,
|
||||
NativeEmbeddingDispatchPlan
|
||||
> = {
|
||||
transportKind: 'embedding',
|
||||
getPrepared: route => route.preparedEmbedding,
|
||||
mapPreparedRoute: mapPreparedDispatchRoute,
|
||||
};
|
||||
|
||||
const rerankArtifactSpec: PreparedExecutionArtifactSpec<
|
||||
'rerank',
|
||||
PreparedNativeRerankExecution,
|
||||
LlmPreparedRerankDispatchRoute,
|
||||
NativeRerankDispatchPlan
|
||||
> = {
|
||||
transportKind: 'rerank',
|
||||
getPrepared: route => route.preparedRerank,
|
||||
mapPreparedRoute: mapPreparedDispatchRoute,
|
||||
};
|
||||
|
||||
const imageArtifactSpec: PreparedExecutionArtifactSpec<
|
||||
'image',
|
||||
PreparedNativeImageExecution,
|
||||
LlmPreparedImageDispatchRoute,
|
||||
NativeImageDispatchPlan
|
||||
> = {
|
||||
transportKind: 'image',
|
||||
getPrepared: route => route.preparedImage,
|
||||
mapPreparedRoute: mapPreparedDispatchRoute,
|
||||
};
|
||||
|
||||
function buildFallbackOrder(routes: ResolvedCopilotProvider[]) {
|
||||
return routes.map(route => route.providerId);
|
||||
}
|
||||
|
||||
function mapExecutionRoute(route: ResolvedCopilotProvider): ExecutionRoute {
|
||||
const preparedRoute =
|
||||
route.prepared?.route ??
|
||||
route.preparedStructured?.route ??
|
||||
route.preparedEmbedding?.route ??
|
||||
route.preparedRerank?.route ??
|
||||
route.preparedImage?.route;
|
||||
|
||||
if (preparedRoute) {
|
||||
return {
|
||||
providerId: preparedRoute.providerId,
|
||||
protocol: preparedRoute.protocol,
|
||||
model: preparedRoute.model,
|
||||
backendConfig: preparedRoute.backendConfig,
|
||||
};
|
||||
}
|
||||
|
||||
const rawRoute = route as unknown as ExecutionRoute;
|
||||
return {
|
||||
providerId: rawRoute.providerId,
|
||||
protocol: rawRoute.protocol,
|
||||
model: rawRoute.model,
|
||||
backendConfig: rawRoute.backendConfig,
|
||||
};
|
||||
}
|
||||
|
||||
function stripHostOnlyOptions<TOptions extends object | undefined>(
|
||||
options: TOptions
|
||||
): Record<string, unknown> | undefined {
|
||||
if (!options) {
|
||||
return;
|
||||
}
|
||||
|
||||
const {
|
||||
signal: _signal,
|
||||
user: _user,
|
||||
session: _session,
|
||||
workspace: _workspace,
|
||||
quotaBackedRoutesAllowed: _quotaBackedRoutesAllowed,
|
||||
...serializable
|
||||
} = options as Record<string, unknown>;
|
||||
|
||||
return Object.keys(serializable).length ? serializable : undefined;
|
||||
}
|
||||
|
||||
function buildSerializableRequest(
|
||||
request: ExecutionPlanRequest
|
||||
): SerializableExecutionPlanRequest {
|
||||
switch (request.kind) {
|
||||
case 'text':
|
||||
case 'streamText':
|
||||
case 'streamObject':
|
||||
case 'structured':
|
||||
case 'image':
|
||||
return {
|
||||
...request,
|
||||
options: stripHostOnlyOptions(request.options),
|
||||
} as SerializableExecutionPlanRequest;
|
||||
case 'embedding':
|
||||
case 'rerank':
|
||||
return {
|
||||
...request,
|
||||
options: stripHostOnlyOptions(request.options),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function buildSerializableExecutionPlan(
|
||||
routes: ResolvedCopilotProvider[],
|
||||
input: Omit<
|
||||
ExecutionPlan,
|
||||
'nativeDispatch' | 'serializable' | 'hostContext'
|
||||
> &
|
||||
Pick<ExecutionPlan, 'hostContext'>
|
||||
): SerializableExecutionPlan {
|
||||
return parseExecutionPlan({
|
||||
routes: routes.map(mapExecutionRoute),
|
||||
request: buildSerializableRequest(input.request),
|
||||
transport: input.transport,
|
||||
routePolicy: input.routePolicy,
|
||||
runtimePolicy: input.runtimePolicy,
|
||||
attachmentPolicy: input.attachmentPolicy,
|
||||
responsePostprocess: input.responsePostprocess,
|
||||
hostContext: input.hostContext.currentMessages
|
||||
? { currentMessages: input.hostContext.currentMessages }
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
|
||||
type MessagePlanArtifacts = Pick<ExecutionPlan, 'nativeDispatch' | 'transport'>;
|
||||
|
||||
function buildMessagePlanArtifacts(
|
||||
kind: Extract<
|
||||
ExecutionRequestKind,
|
||||
'text' | 'streamText' | 'streamObject' | 'structured' | 'image'
|
||||
>,
|
||||
routes: ResolvedCopilotProvider[]
|
||||
): MessagePlanArtifacts {
|
||||
const chatArtifacts =
|
||||
kind === 'text' || kind === 'streamText' || kind === 'streamObject'
|
||||
? buildPreparedExecutionArtifacts(routes, chatArtifactSpec)
|
||||
: undefined;
|
||||
const structuredArtifacts =
|
||||
kind === 'structured'
|
||||
? buildPreparedExecutionArtifacts(routes, structuredArtifactSpec)
|
||||
: undefined;
|
||||
const imageArtifacts =
|
||||
kind === 'image'
|
||||
? buildPreparedExecutionArtifacts(routes, imageArtifactSpec)
|
||||
: undefined;
|
||||
const nativeDispatch = {
|
||||
chat:
|
||||
kind === 'text' || kind === 'streamText' || kind === 'streamObject'
|
||||
? chatArtifacts?.dispatch
|
||||
: undefined,
|
||||
structured:
|
||||
kind === 'structured' ? structuredArtifacts?.dispatch : undefined,
|
||||
image: kind === 'image' ? imageArtifacts?.dispatch : undefined,
|
||||
};
|
||||
|
||||
return {
|
||||
nativeDispatch,
|
||||
transport:
|
||||
kind === 'text' || kind === 'streamText' || kind === 'streamObject'
|
||||
? chatArtifacts?.transport
|
||||
: kind === 'structured'
|
||||
? structuredArtifacts?.transport
|
||||
: kind === 'image'
|
||||
? imageArtifacts?.transport
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildEmbeddingPlanArtifacts(
|
||||
routes: ResolvedCopilotProvider[]
|
||||
): Pick<ExecutionPlan, 'nativeDispatch' | 'transport'> {
|
||||
const embeddingArtifacts = buildPreparedExecutionArtifacts(
|
||||
routes,
|
||||
embeddingArtifactSpec
|
||||
);
|
||||
return {
|
||||
nativeDispatch: {
|
||||
embedding: embeddingArtifacts.dispatch,
|
||||
},
|
||||
transport: embeddingArtifacts.transport,
|
||||
};
|
||||
}
|
||||
|
||||
function buildRerankPlanArtifacts(
|
||||
routes: ResolvedCopilotProvider[]
|
||||
): Pick<ExecutionPlan, 'nativeDispatch' | 'transport'> {
|
||||
const rerankArtifacts = buildPreparedExecutionArtifacts(
|
||||
routes,
|
||||
rerankArtifactSpec
|
||||
);
|
||||
return {
|
||||
nativeDispatch: {
|
||||
rerank: rerankArtifacts.dispatch,
|
||||
},
|
||||
transport: rerankArtifacts.transport,
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ExecutionPlanBuilder {
|
||||
constructor(
|
||||
private readonly providers: CopilotProviderFactory,
|
||||
private readonly executionMetrics: CopilotExecutionMetrics
|
||||
) {}
|
||||
|
||||
private async buildMessagePlan<
|
||||
TKind extends Extract<
|
||||
ExecutionRequestKind,
|
||||
'text' | 'streamText' | 'streamObject' | 'structured' | 'image'
|
||||
>,
|
||||
>(
|
||||
kind: TKind,
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions,
|
||||
filter: ProviderFilter = {}
|
||||
): Promise<ExecutionPlanForKind<TKind>> {
|
||||
const outputType =
|
||||
kind === 'image'
|
||||
? ModelOutputType.Image
|
||||
: kind === 'streamObject'
|
||||
? ModelOutputType.Object
|
||||
: kind === 'structured'
|
||||
? ModelOutputType.Structured
|
||||
: ModelOutputType.Text;
|
||||
|
||||
const routes =
|
||||
kind === 'text' || kind === 'streamText' || kind === 'streamObject'
|
||||
? await this.providers.prepareRoutes(
|
||||
kind,
|
||||
{ ...cond, outputType },
|
||||
messages,
|
||||
(options as CopilotChatOptions | undefined) ?? {},
|
||||
filter
|
||||
)
|
||||
: kind === 'structured'
|
||||
? await this.providers.prepareStructuredRoutes(
|
||||
{ ...cond, outputType },
|
||||
messages,
|
||||
(options as CopilotStructuredOptions | undefined) ?? {},
|
||||
filter
|
||||
)
|
||||
: await this.providers.prepareImageRoutes(
|
||||
{ ...cond, outputType },
|
||||
messages,
|
||||
(options as CopilotImageOptions | undefined) ?? {},
|
||||
filter
|
||||
);
|
||||
this.executionMetrics.recordPlan(kind, routes, filter.prefer);
|
||||
const { nativeDispatch, transport } = buildMessagePlanArtifacts(
|
||||
kind,
|
||||
routes
|
||||
);
|
||||
const plan = {
|
||||
transport,
|
||||
request: {
|
||||
kind,
|
||||
cond: { ...cond, modelId: cond.modelId },
|
||||
messages,
|
||||
options,
|
||||
} as Extract<ExecutionPlanRequest, { kind: TKind }>,
|
||||
routePolicy: {
|
||||
fallbackOrder: buildFallbackOrder(routes),
|
||||
},
|
||||
runtimePolicy: { prefer: filter.prefer },
|
||||
attachmentPolicy: { materializeRemoteAttachments: true },
|
||||
responsePostprocess: { mode: kind },
|
||||
hostPersistence: {
|
||||
persistAssistantTurn: true,
|
||||
outputKind: kind,
|
||||
},
|
||||
hostContext: {
|
||||
signal: options?.signal,
|
||||
currentMessages: messages,
|
||||
},
|
||||
} as Omit<ExecutionPlanForKind<TKind>, 'nativeDispatch' | 'serializable'>;
|
||||
|
||||
return {
|
||||
nativeDispatch,
|
||||
serializable: buildSerializableExecutionPlan(routes, plan),
|
||||
...plan,
|
||||
};
|
||||
}
|
||||
|
||||
async buildTextPlan(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
filter?: ProviderFilter
|
||||
): Promise<ExecutionPlanForKind<'text'>> {
|
||||
return await this.buildMessagePlan('text', cond, messages, options, filter);
|
||||
}
|
||||
|
||||
async buildStreamTextPlan(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
filter?: ProviderFilter
|
||||
): Promise<ExecutionPlanForKind<'streamText'>> {
|
||||
return await this.buildMessagePlan(
|
||||
'streamText',
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
filter
|
||||
);
|
||||
}
|
||||
|
||||
async buildStreamObjectPlan(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
filter?: ProviderFilter
|
||||
): Promise<ExecutionPlanForKind<'streamObject'>> {
|
||||
return await this.buildMessagePlan(
|
||||
'streamObject',
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
filter
|
||||
);
|
||||
}
|
||||
|
||||
async buildStructuredPlan(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotStructuredOptions,
|
||||
filter?: ProviderFilter,
|
||||
responseContract?: RequiredStructuredOutputContract
|
||||
): Promise<ExecutionPlanForKind<'structured'>> {
|
||||
const outputType = ModelOutputType.Structured;
|
||||
const routes = await this.providers.prepareStructuredRoutes(
|
||||
{ ...cond, outputType },
|
||||
messages,
|
||||
options ?? {},
|
||||
filter ?? {},
|
||||
responseContract
|
||||
);
|
||||
this.executionMetrics.recordPlan('structured', routes, filter?.prefer);
|
||||
const { nativeDispatch, transport } = buildMessagePlanArtifacts(
|
||||
'structured',
|
||||
routes
|
||||
);
|
||||
const plan = {
|
||||
transport,
|
||||
request: {
|
||||
kind: 'structured',
|
||||
cond: { ...cond, modelId: cond.modelId },
|
||||
messages,
|
||||
options,
|
||||
},
|
||||
routePolicy: {
|
||||
fallbackOrder: buildFallbackOrder(routes),
|
||||
},
|
||||
runtimePolicy: { prefer: filter?.prefer },
|
||||
attachmentPolicy: { materializeRemoteAttachments: true },
|
||||
responsePostprocess: { mode: 'structured' },
|
||||
hostPersistence: {
|
||||
persistAssistantTurn: true,
|
||||
outputKind: 'structured',
|
||||
},
|
||||
hostContext: {
|
||||
signal: options?.signal,
|
||||
currentMessages: messages,
|
||||
},
|
||||
} as Omit<
|
||||
ExecutionPlanForKind<'structured'>,
|
||||
'nativeDispatch' | 'serializable'
|
||||
>;
|
||||
|
||||
return {
|
||||
nativeDispatch,
|
||||
serializable: buildSerializableExecutionPlan(routes, plan),
|
||||
...plan,
|
||||
};
|
||||
}
|
||||
|
||||
async buildImagePlan(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotImageOptions,
|
||||
filter?: ProviderFilter
|
||||
): Promise<ExecutionPlanForKind<'image'>> {
|
||||
return await this.buildMessagePlan(
|
||||
'image',
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
filter
|
||||
);
|
||||
}
|
||||
|
||||
async buildEmbeddingPlan(
|
||||
modelId: string,
|
||||
input: string | string[],
|
||||
options?: CopilotEmbeddingOptions
|
||||
): Promise<ExecutionPlanForKind<'embedding'>> {
|
||||
const routes = await this.providers.prepareEmbeddingRoutes(
|
||||
modelId,
|
||||
input,
|
||||
options
|
||||
);
|
||||
this.executionMetrics.recordPlan('embedding', routes);
|
||||
const { nativeDispatch, transport } = buildEmbeddingPlanArtifacts(routes);
|
||||
const plan = {
|
||||
transport,
|
||||
request: {
|
||||
kind: 'embedding',
|
||||
cond: { modelId },
|
||||
modelId,
|
||||
input,
|
||||
options,
|
||||
},
|
||||
routePolicy: {
|
||||
fallbackOrder: buildFallbackOrder(routes),
|
||||
},
|
||||
runtimePolicy: {},
|
||||
attachmentPolicy: { materializeRemoteAttachments: false },
|
||||
responsePostprocess: { mode: 'embedding' },
|
||||
hostPersistence: {
|
||||
persistAssistantTurn: false,
|
||||
outputKind: 'embedding',
|
||||
},
|
||||
hostContext: {
|
||||
signal: options?.signal,
|
||||
},
|
||||
} as Omit<
|
||||
ExecutionPlanForKind<'embedding'>,
|
||||
'nativeDispatch' | 'serializable'
|
||||
>;
|
||||
|
||||
return {
|
||||
nativeDispatch,
|
||||
serializable: buildSerializableExecutionPlan(routes, plan),
|
||||
...plan,
|
||||
};
|
||||
}
|
||||
|
||||
async buildRerankPlan(
|
||||
modelId: string,
|
||||
request: CopilotRerankRequest,
|
||||
options?: CopilotChatOptions
|
||||
): Promise<ExecutionPlanForKind<'rerank'>> {
|
||||
const routes = await this.providers.prepareRerankRoutes(
|
||||
modelId,
|
||||
request,
|
||||
options
|
||||
);
|
||||
this.executionMetrics.recordPlan('rerank', routes);
|
||||
const { nativeDispatch, transport } = buildRerankPlanArtifacts(routes);
|
||||
const plan = {
|
||||
transport,
|
||||
request: {
|
||||
kind: 'rerank',
|
||||
cond: { modelId },
|
||||
modelId,
|
||||
request,
|
||||
options,
|
||||
},
|
||||
routePolicy: {
|
||||
fallbackOrder: buildFallbackOrder(routes),
|
||||
},
|
||||
runtimePolicy: {},
|
||||
attachmentPolicy: { materializeRemoteAttachments: false },
|
||||
responsePostprocess: { mode: 'rerank' },
|
||||
hostPersistence: {
|
||||
persistAssistantTurn: false,
|
||||
outputKind: 'rerank',
|
||||
},
|
||||
hostContext: {
|
||||
signal: options?.signal,
|
||||
},
|
||||
} as Omit<
|
||||
ExecutionPlanForKind<'rerank'>,
|
||||
'nativeDispatch' | 'serializable'
|
||||
>;
|
||||
|
||||
return {
|
||||
nativeDispatch,
|
||||
serializable: buildSerializableExecutionPlan(routes, plan),
|
||||
...plan,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import type { LlmImageResponse } from '../../../../native';
|
||||
import {
|
||||
getCopilotActionRecipe,
|
||||
type LlmImageResponse,
|
||||
} from '../../../../native';
|
||||
import { PromptService } from '../../prompt';
|
||||
import type { PromptMessage } from '../../providers/types';
|
||||
import type { ChatSession } from '../../session';
|
||||
@@ -8,6 +11,10 @@ import { ChatQuerySchema } from '../../types';
|
||||
import { projectActionEventToChatEvent } from '../action-output-projector';
|
||||
import type { ActionRuntimeBridgeEvent } from '../action-runtime-bridge';
|
||||
import { ActionRuntimeBridge } from '../action-runtime-bridge';
|
||||
import {
|
||||
buildStructuredResponseFromSchemaJson,
|
||||
requireStructuredOutputContract,
|
||||
} from '../contracts';
|
||||
import { ConversationHost } from './conversation-host';
|
||||
import { ImageResultHost } from './image-result-host';
|
||||
|
||||
@@ -17,32 +24,6 @@ function firstQueryValue(value: string | string[] | undefined) {
|
||||
return Array.isArray(value) ? value[0] : value;
|
||||
}
|
||||
|
||||
const ACTION_PROMPTS: Record<string, string> = {
|
||||
'mindmap.generate': 'mindmap.generate',
|
||||
'slides.outline': 'slides.outline',
|
||||
};
|
||||
|
||||
type ImageActionRoutePreparation = {
|
||||
modelId?: string;
|
||||
messages: PromptMessage[];
|
||||
options: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function isImageAction(id: string) {
|
||||
return id.startsWith('image.filter.');
|
||||
}
|
||||
|
||||
function actionTextResultSchema() {
|
||||
return {
|
||||
type: 'object',
|
||||
properties: {
|
||||
result: { type: 'string' },
|
||||
},
|
||||
required: ['result'],
|
||||
additionalProperties: false,
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ActionStreamHost {
|
||||
constructor(
|
||||
@@ -73,6 +54,7 @@ export class ActionStreamHost {
|
||||
firstQueryValue(query.actionId) ?? prepared.session.config.promptName;
|
||||
const actionId = requestedActionId;
|
||||
const actionVersion = firstQueryValue(query.actionVersion) ?? 'v1';
|
||||
const recipe = getCopilotActionRecipe(actionId, actionVersion);
|
||||
const retryOf = parsedQuery.retry
|
||||
? firstQueryValue(query.runId)
|
||||
: undefined;
|
||||
@@ -81,19 +63,16 @@ export class ActionStreamHost {
|
||||
...this.conversations.buildLatestTurnPromptParams(prepared.latestTurn),
|
||||
};
|
||||
const finalMessage = await this.preparePromptMessages(
|
||||
actionId,
|
||||
recipe.promptRef,
|
||||
prepared.session,
|
||||
params
|
||||
);
|
||||
const imageRoutes = await this.prepareImageRoutes(
|
||||
actionId,
|
||||
prepared.session,
|
||||
params,
|
||||
userId,
|
||||
parsedQuery.byokLeaseId,
|
||||
prepared.quotaBackedRoutesAllowed,
|
||||
signal
|
||||
);
|
||||
const responseContract = recipe.responseContract
|
||||
? requireStructuredOutputContract(
|
||||
buildStructuredResponseFromSchemaJson(recipe.responseContract.schema)
|
||||
)
|
||||
: undefined;
|
||||
const producesImage = recipe.outputProjection === 'first_image';
|
||||
const runStream = this.bridge.runStream({
|
||||
userId,
|
||||
workspaceId: prepared.session.config.workspaceId,
|
||||
@@ -108,7 +87,7 @@ export class ActionStreamHost {
|
||||
params,
|
||||
messageId: prepared.messageId,
|
||||
},
|
||||
persistAttachment: isImageAction(actionId)
|
||||
persistAttachment: producesImage
|
||||
? attachment =>
|
||||
this.persistImageAttachment(
|
||||
userId,
|
||||
@@ -116,35 +95,25 @@ export class ActionStreamHost {
|
||||
attachment
|
||||
)
|
||||
: undefined,
|
||||
prepareStructuredRoutes: isImageAction(actionId)
|
||||
? undefined
|
||||
: {
|
||||
stepId: 'generate',
|
||||
modelId:
|
||||
typeof query.modelId === 'string' && query.modelId
|
||||
? query.modelId
|
||||
: undefined,
|
||||
messages: finalMessage,
|
||||
responseSchemaJson: actionTextResultSchema(),
|
||||
options: {
|
||||
...prepared.session.config.promptConfig,
|
||||
signal,
|
||||
user: userId,
|
||||
workspace: prepared.session.config.workspaceId,
|
||||
session: sessionId,
|
||||
byokLeaseId: parsedQuery.byokLeaseId,
|
||||
quotaBackedRoutesAllowed: prepared.quotaBackedRoutesAllowed,
|
||||
featureKind: 'action',
|
||||
},
|
||||
},
|
||||
prepareImageRoutes: imageRoutes
|
||||
? {
|
||||
stepId: 'generate-image',
|
||||
modelId: imageRoutes.modelId,
|
||||
messages: imageRoutes.messages,
|
||||
options: imageRoutes.options,
|
||||
}
|
||||
: undefined,
|
||||
step: {
|
||||
slot: recipe.slot,
|
||||
builtInRouteId: recipe.promptRef,
|
||||
profileId: parsedQuery.profileId,
|
||||
modelId: parsedQuery.modelId,
|
||||
messages: finalMessage,
|
||||
responseContract,
|
||||
options: {
|
||||
...prepared.session.config.promptConfig,
|
||||
signal,
|
||||
user: userId,
|
||||
workspace: prepared.session.config.workspaceId,
|
||||
session: sessionId,
|
||||
byokLeaseId: parsedQuery.byokLeaseId,
|
||||
managedTargetId: parsedQuery.routeTargetId,
|
||||
quotaBackedRoutesAllowed: prepared.quotaBackedRoutesAllowed,
|
||||
featureKind: producesImage ? 'image' : 'action',
|
||||
},
|
||||
},
|
||||
signal,
|
||||
});
|
||||
|
||||
@@ -157,18 +126,13 @@ export class ActionStreamHost {
|
||||
}
|
||||
|
||||
private async preparePromptMessages(
|
||||
actionId: string,
|
||||
promptRef: string,
|
||||
session: ChatSession,
|
||||
params: Record<string, unknown>
|
||||
): Promise<PromptMessage[]> {
|
||||
const promptName = ACTION_PROMPTS[actionId];
|
||||
if (!promptName) {
|
||||
return session.finish(params);
|
||||
}
|
||||
|
||||
const prompt = await this.prompts.get(promptName);
|
||||
const prompt = await this.prompts.get(promptRef);
|
||||
if (!prompt) {
|
||||
throw new Error(`Prompt ${promptName} not found`);
|
||||
throw new Error(`Prompt ${promptRef} not found`);
|
||||
}
|
||||
return this.prompts.finish(
|
||||
prompt,
|
||||
@@ -177,44 +141,6 @@ export class ActionStreamHost {
|
||||
);
|
||||
}
|
||||
|
||||
private async prepareImageRoutes(
|
||||
actionId: string,
|
||||
session: ChatSession,
|
||||
params: Record<string, unknown>,
|
||||
userId: string,
|
||||
byokLeaseId?: string,
|
||||
quotaBackedRoutesAllowed?: boolean,
|
||||
signal?: AbortSignal
|
||||
): Promise<ImageActionRoutePreparation | undefined> {
|
||||
if (!isImageAction(actionId)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const prompt = await this.prompts.get(actionId);
|
||||
if (!prompt) {
|
||||
throw new Error(`Prompt ${actionId} not found`);
|
||||
}
|
||||
const finalMessage = this.prompts.finish(
|
||||
prompt,
|
||||
params as Record<string, string>,
|
||||
session.config.sessionId
|
||||
);
|
||||
return {
|
||||
modelId: prompt.model,
|
||||
messages: finalMessage,
|
||||
options: {
|
||||
...prompt.config,
|
||||
signal,
|
||||
user: userId,
|
||||
workspace: session.config.workspaceId,
|
||||
session: session.config.sessionId,
|
||||
byokLeaseId,
|
||||
quotaBackedRoutesAllowed,
|
||||
featureKind: 'image',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async persistImageAttachment(
|
||||
userId: string,
|
||||
workspaceId: string,
|
||||
|
||||
-133
@@ -1,133 +0,0 @@
|
||||
import type { LlmBackendConfig, LlmProtocol } from '../../../../native';
|
||||
import { llmPlanAttachmentReference } from '../../../../native';
|
||||
import type { PromptAttachment } from '../../providers/types';
|
||||
import {
|
||||
type AdmittedAttachmentSource,
|
||||
admittedAttachmentToPromptAttachment,
|
||||
} from './attachment-admission';
|
||||
|
||||
export type AdmittedAttachmentMaterializationPlan = {
|
||||
mode: 'inline';
|
||||
reason: 'admitted_bytes';
|
||||
attachment: PromptAttachment;
|
||||
};
|
||||
|
||||
export type HostAttachmentMaterializationRequest = {
|
||||
attachmentId: string;
|
||||
target: 'bytes' | 'data';
|
||||
providerConstraint?: string;
|
||||
maxSize: number;
|
||||
timeoutMs: number;
|
||||
redirectPolicy: 'follow-safe';
|
||||
expectedMime?: string;
|
||||
url: string;
|
||||
};
|
||||
|
||||
type RemoteReferenceReason =
|
||||
| 'generic_remote_reference'
|
||||
| 'gemini_api_file_uri'
|
||||
| 'gemini_api_youtube_url';
|
||||
|
||||
type MaterializationRequestReason =
|
||||
| 'generic_remote_reference'
|
||||
| 'gemini_api_inline_http_url'
|
||||
| 'unsupported_scheme'
|
||||
| 'non_url_source';
|
||||
|
||||
function assertRemoteReferenceReason(
|
||||
reason: string
|
||||
): asserts reason is RemoteReferenceReason {
|
||||
if (
|
||||
reason !== 'generic_remote_reference' &&
|
||||
reason !== 'gemini_api_file_uri' &&
|
||||
reason !== 'gemini_api_youtube_url'
|
||||
) {
|
||||
throw new Error(`Unexpected remote attachment reference reason: ${reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertMaterializationRequestReason(
|
||||
reason: string
|
||||
): asserts reason is MaterializationRequestReason {
|
||||
if (
|
||||
reason !== 'gemini_api_inline_http_url' &&
|
||||
reason !== 'generic_remote_reference' &&
|
||||
reason !== 'unsupported_scheme' &&
|
||||
reason !== 'non_url_source'
|
||||
) {
|
||||
throw new Error(`Unexpected attachment materialization reason: ${reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function planHostUrlAttachmentMaterialization(
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
input: {
|
||||
attachmentId: string;
|
||||
url: string;
|
||||
expectedMime?: string;
|
||||
maxSize: number;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
): Promise<
|
||||
| {
|
||||
mode: 'remote_reference';
|
||||
reason:
|
||||
| 'generic_remote_reference'
|
||||
| 'gemini_api_file_uri'
|
||||
| 'gemini_api_youtube_url';
|
||||
url: string;
|
||||
}
|
||||
| {
|
||||
mode: 'materialization_request';
|
||||
reason:
|
||||
| 'generic_remote_reference'
|
||||
| 'gemini_api_inline_http_url'
|
||||
| 'unsupported_scheme'
|
||||
| 'non_url_source';
|
||||
request: HostAttachmentMaterializationRequest;
|
||||
}
|
||||
> {
|
||||
const plan = await llmPlanAttachmentReference(protocol, backendConfig, {
|
||||
url: input.url,
|
||||
});
|
||||
const forceHostMaterialization =
|
||||
protocol === 'gemini' &&
|
||||
backendConfig.request_layer === 'gemini_vertex' &&
|
||||
plan.reason === 'generic_remote_reference';
|
||||
|
||||
if (plan.mode === 'remote' && !forceHostMaterialization) {
|
||||
assertRemoteReferenceReason(plan.reason);
|
||||
return {
|
||||
mode: 'remote_reference',
|
||||
reason: plan.reason,
|
||||
url: input.url,
|
||||
};
|
||||
}
|
||||
|
||||
assertMaterializationRequestReason(plan.reason);
|
||||
return {
|
||||
mode: 'materialization_request',
|
||||
reason: plan.reason,
|
||||
request: {
|
||||
attachmentId: input.attachmentId,
|
||||
target: 'bytes',
|
||||
providerConstraint: protocol,
|
||||
maxSize: input.maxSize,
|
||||
timeoutMs: input.timeoutMs ?? 15_000,
|
||||
redirectPolicy: 'follow-safe',
|
||||
expectedMime: input.expectedMime,
|
||||
url: input.url,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function planAdmittedAttachmentMaterialization(
|
||||
source: AdmittedAttachmentSource
|
||||
): AdmittedAttachmentMaterializationPlan {
|
||||
return {
|
||||
mode: 'inline',
|
||||
reason: 'admitted_bytes',
|
||||
attachment: admittedAttachmentToPromptAttachment(source),
|
||||
};
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
|
||||
import { ServerFeature, ServerService } from '../../../../core';
|
||||
import { QuotaStateService } from '../../../../core/quota/state';
|
||||
import type { ChatSession } from '../../session';
|
||||
import { type ToolsConfig } from '../../types';
|
||||
import { getTools } from '../../utils';
|
||||
import {
|
||||
ModelSelectionPolicy,
|
||||
type ResolveModelInput,
|
||||
} from '../model-selection-policy';
|
||||
|
||||
export type ChatSelectionOptions = {
|
||||
responseMode: 'text' | 'object' | 'image';
|
||||
modelId?: string;
|
||||
reasoning?: boolean;
|
||||
webSearch?: boolean;
|
||||
toolsConfig?: ToolsConfig;
|
||||
byokLeaseId?: string;
|
||||
billingUnitId?: string;
|
||||
featureKind?: 'chat' | 'action' | 'image';
|
||||
quotaBackedRoutesAllowed?: boolean;
|
||||
};
|
||||
|
||||
type ResolvePolicyModelInput = ResolveModelInput & {
|
||||
proModels?: string[] | null;
|
||||
userId?: string;
|
||||
paymentEnabled?: boolean;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CapabilityPolicyHost {
|
||||
constructor(
|
||||
private readonly server: ServerService,
|
||||
private readonly moduleRef: ModuleRef,
|
||||
private readonly modelSelection: ModelSelectionPolicy
|
||||
) {}
|
||||
|
||||
private async hasAiProAccess(
|
||||
userId: string | undefined,
|
||||
paymentEnabled: boolean | undefined
|
||||
) {
|
||||
if (!paymentEnabled || !userId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const state = await this.moduleRef
|
||||
.get(QuotaStateService, { strict: false })
|
||||
.reconcileUserQuotaState(userId);
|
||||
const flags = state.flags as { unlimitedCopilot?: boolean };
|
||||
return (
|
||||
!!flags.unlimitedCopilot ||
|
||||
['pro', 'lifetime_pro', 'ai'].includes(state.plan)
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async resolveModel(input: ResolvePolicyModelInput) {
|
||||
const resolved = this.modelSelection.resolveRequestedModel(input);
|
||||
if (!resolved.matchedOptionalModel) {
|
||||
return resolved.selectedModel;
|
||||
}
|
||||
|
||||
if (
|
||||
input.paymentEnabled &&
|
||||
this.modelSelection.matchesModelList(
|
||||
input.proModels ?? [],
|
||||
input.requestedModelId
|
||||
) &&
|
||||
!(await this.hasAiProAccess(input.userId, input.paymentEnabled))
|
||||
) {
|
||||
return input.defaultModel;
|
||||
}
|
||||
|
||||
return resolved.selectedModel;
|
||||
}
|
||||
|
||||
async selectChat(session: ChatSession, options: ChatSelectionOptions) {
|
||||
const model = await this.resolveChatModel({
|
||||
userId: session.config.userId,
|
||||
defaultModel: session.model,
|
||||
optionalModels: session.optionalModels,
|
||||
proModels: session.config.promptConfig?.proModels,
|
||||
requestedModelId: options.modelId,
|
||||
paymentEnabled: this.server.features.includes(ServerFeature.Payment),
|
||||
});
|
||||
const tools = getTools(
|
||||
session.config.promptConfig?.tools,
|
||||
options.toolsConfig
|
||||
);
|
||||
return {
|
||||
model,
|
||||
providerOptions: {
|
||||
...session.config.promptConfig,
|
||||
user: session.config.userId,
|
||||
session: session.config.sessionId,
|
||||
workspace: session.config.workspaceId,
|
||||
byokLeaseId: options.byokLeaseId,
|
||||
billingUnitId: options.billingUnitId,
|
||||
featureKind: options.featureKind ?? 'chat',
|
||||
quotaBackedRoutesAllowed: options.quotaBackedRoutesAllowed,
|
||||
reasoning: options.reasoning,
|
||||
webSearch: options.webSearch,
|
||||
tools,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async resolveChatModel(input: ResolvePolicyModelInput) {
|
||||
return await this.resolveModel(input);
|
||||
}
|
||||
|
||||
async resolvePromptModel(input: ResolveModelInput) {
|
||||
return this.modelSelection.resolveRequestedModel(input).selectedModel;
|
||||
}
|
||||
|
||||
async resolveFixedTaskModel(input: ResolveModelInput) {
|
||||
return this.modelSelection.resolveRequestedModel(input).selectedModel;
|
||||
}
|
||||
}
|
||||
@@ -5,8 +5,8 @@ import {
|
||||
CopilotSessionNotFound,
|
||||
Mutex,
|
||||
} from '../../../../base';
|
||||
import { CopilotAccessPolicy } from '../../access';
|
||||
import { CompatSubmissionStore } from '../../compat/submission-store';
|
||||
import { ConversationPolicy } from '../../conversation/policy';
|
||||
import {
|
||||
canonicalizeTurnTrace,
|
||||
type Turn,
|
||||
@@ -35,7 +35,7 @@ export class ConversationHost {
|
||||
private readonly sessions: ChatSessionService,
|
||||
private readonly submissions: CompatSubmissionStore,
|
||||
private readonly mutex: Mutex,
|
||||
private readonly access: CopilotAccessPolicy
|
||||
private readonly policy: ConversationPolicy
|
||||
) {}
|
||||
|
||||
private async loadAcceptedTurn(
|
||||
@@ -109,31 +109,22 @@ export class ConversationHost {
|
||||
session: ChatSession,
|
||||
sessionId: string,
|
||||
messageId?: string,
|
||||
retry = false,
|
||||
byokLeaseId?: string
|
||||
retry = false
|
||||
): Promise<AppendedSessionMessage> {
|
||||
const resolveChatRouteAccess = () =>
|
||||
this.access.resolveTurnRouteAccess({
|
||||
userId,
|
||||
workspaceId: session.config.workspaceId,
|
||||
byokLeaseId,
|
||||
featureKind: 'chat',
|
||||
});
|
||||
const quotaBackedRoutesAllowed = () => this.policy.hasQuota(userId);
|
||||
|
||||
if (!messageId) {
|
||||
await this.sessions.revertLatestMessage(sessionId, false);
|
||||
session.revertLatestMessage(false);
|
||||
if (!session.latestUserTurn) {
|
||||
const routeAccess = await resolveChatRouteAccess();
|
||||
return {
|
||||
turn: session.latestUserTurn,
|
||||
quotaBackedRoutesAllowed: routeAccess.quotaBackedRoutesAllowed,
|
||||
quotaBackedRoutesAllowed: await quotaBackedRoutesAllowed(),
|
||||
};
|
||||
}
|
||||
const routeAccess = await resolveChatRouteAccess();
|
||||
return {
|
||||
turn: session.latestUserTurn,
|
||||
quotaBackedRoutesAllowed: routeAccess.quotaBackedRoutesAllowed,
|
||||
quotaBackedRoutesAllowed: await quotaBackedRoutesAllowed(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -177,7 +168,7 @@ export class ConversationHost {
|
||||
};
|
||||
}
|
||||
|
||||
const routeAccess = await resolveChatRouteAccess();
|
||||
const quotaAllowed = await quotaBackedRoutesAllowed();
|
||||
|
||||
const submission = await this.submissions.get(messageId);
|
||||
if (!submission || submission.sessionId !== sessionId) {
|
||||
@@ -192,7 +183,6 @@ export class ConversationHost {
|
||||
const turn = await this.sessions.appendTurn({
|
||||
sessionId,
|
||||
userId: session.config.userId,
|
||||
prompt: { model: session.model },
|
||||
compatSubmissionId: messageId,
|
||||
turn: {
|
||||
conversationId: sessionId,
|
||||
@@ -213,7 +203,7 @@ export class ConversationHost {
|
||||
session.pushPersistedTurn(turn);
|
||||
return {
|
||||
turn,
|
||||
quotaBackedRoutesAllowed: routeAccess.quotaBackedRoutesAllowed,
|
||||
quotaBackedRoutesAllowed: quotaAllowed,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -222,8 +212,7 @@ export class ConversationHost {
|
||||
sessionId: string,
|
||||
query: Record<string, string | string[]>
|
||||
): Promise<PreparedConversationTurn> {
|
||||
const { messageId, retry, params, byokLeaseId } =
|
||||
ChatQuerySchema.parse(query);
|
||||
const { messageId, retry, params } = ChatQuerySchema.parse(query);
|
||||
const session = await this.sessions.get(sessionId);
|
||||
if (!session || session.config.userId !== userId) {
|
||||
throw new CopilotSessionNotFound();
|
||||
@@ -233,8 +222,7 @@ export class ConversationHost {
|
||||
session,
|
||||
sessionId,
|
||||
messageId,
|
||||
retry,
|
||||
byokLeaseId
|
||||
retry
|
||||
);
|
||||
const currentUserMessage =
|
||||
session.stashTurns.findLast(turn => turn.role === 'user') ??
|
||||
@@ -280,7 +268,6 @@ export class ConversationHost {
|
||||
const persisted = await this.sessions.appendTurn({
|
||||
sessionId: session.config.sessionId,
|
||||
userId: session.config.userId,
|
||||
prompt: { model: session.model },
|
||||
turn: assistantTurn,
|
||||
});
|
||||
session.pushPersistedTurn(persisted);
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import type { NodeTextMiddleware } from '../../config';
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotChatTools,
|
||||
} from '../../providers/types';
|
||||
import type { CopilotTool, CopilotToolSet } from '../../tools';
|
||||
import type { ToolLoopBackend } from '../tool/bridge';
|
||||
import { ToolRuntime } from '../tool-runtime';
|
||||
|
||||
export type ProviderSpecificToolResolver = (
|
||||
toolName: CopilotChatTools,
|
||||
model: string
|
||||
) => [string, CopilotTool?] | undefined;
|
||||
|
||||
@Injectable()
|
||||
export class ToolExecutorHost {
|
||||
constructor(private readonly runtime: ToolRuntime) {}
|
||||
|
||||
async getTools(
|
||||
options: CopilotChatOptions,
|
||||
model: string,
|
||||
resolveProviderSpecificTool?: ProviderSpecificToolResolver
|
||||
): Promise<CopilotToolSet> {
|
||||
return await this.runtime.getTools(
|
||||
options,
|
||||
model,
|
||||
resolveProviderSpecificTool
|
||||
);
|
||||
}
|
||||
|
||||
createNativeAdapter(
|
||||
backend: ToolLoopBackend,
|
||||
tools: CopilotToolSet,
|
||||
options: {
|
||||
maxSteps?: number;
|
||||
nodeTextMiddleware?: NodeTextMiddleware[];
|
||||
} = {}
|
||||
) {
|
||||
return this.runtime.createNativeAdapter(backend, tools, options);
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { CopilotSessionInvalidInput } from '../../../base';
|
||||
import { llmResolveRequestedModelMatch } from '../../../native';
|
||||
import { CopilotProviderRegistryService } from '../providers/registry-service';
|
||||
|
||||
export type ResolveModelInput = {
|
||||
defaultModel: string;
|
||||
optionalModels?: string[] | null;
|
||||
requestedModelId?: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class ModelSelectionPolicy {
|
||||
constructor(private readonly registries: CopilotProviderRegistryService) {}
|
||||
|
||||
private getRegistry() {
|
||||
return this.registries.getRegistry();
|
||||
}
|
||||
|
||||
private matchRequestedModel(
|
||||
optionalModels: string[],
|
||||
requestedModelId?: string,
|
||||
defaultModel?: string
|
||||
) {
|
||||
return llmResolveRequestedModelMatch({
|
||||
providerIds: [...this.getRegistry().profiles.keys()],
|
||||
optionalModels,
|
||||
requestedModelId,
|
||||
defaultModel,
|
||||
});
|
||||
}
|
||||
|
||||
resolveRequestedModel(input: ResolveModelInput): {
|
||||
selectedModel: string;
|
||||
matchedOptionalModel: boolean;
|
||||
} {
|
||||
if (!input.defaultModel) {
|
||||
throw new CopilotSessionInvalidInput('Model is required');
|
||||
}
|
||||
const matched = this.matchRequestedModel(
|
||||
input.optionalModels ?? [],
|
||||
input.requestedModelId,
|
||||
input.defaultModel
|
||||
);
|
||||
return {
|
||||
selectedModel: matched.selectedModel ?? input.defaultModel,
|
||||
matchedOptionalModel: matched.matchedOptionalModel,
|
||||
};
|
||||
}
|
||||
|
||||
matchesModelList(models: string[], modelId?: string) {
|
||||
return this.matchRequestedModel(models, modelId).matchedOptionalModel;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { NetworkError } from '../../../base';
|
||||
import { CopilotQuotaExceeded, NetworkError } from '../../../base';
|
||||
|
||||
const LLM_TIMEOUT_ERROR_PREFIX = 'llm_timeout:';
|
||||
|
||||
@@ -18,6 +18,9 @@ function nativeErrorMessage(error: unknown) {
|
||||
|
||||
export function mapNativeSemanticError(error: unknown): unknown {
|
||||
const message = nativeErrorMessage(error);
|
||||
if (message === 'access_unavailable') {
|
||||
return new CopilotQuotaExceeded();
|
||||
}
|
||||
if (message?.startsWith(LLM_TIMEOUT_ERROR_PREFIX)) {
|
||||
return new NetworkError(
|
||||
message.slice(LLM_TIMEOUT_ERROR_PREFIX.length).trim() ||
|
||||
|
||||
@@ -1,536 +0,0 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { NoCopilotProviderAvailable } from '../../../base';
|
||||
import {
|
||||
llmDispatchPlan,
|
||||
llmDispatchPlanStream,
|
||||
type LlmDispatchResponse,
|
||||
llmEmbeddingDispatchPlan,
|
||||
llmImageDispatchPlan,
|
||||
type LlmImageResponse,
|
||||
llmRerankDispatchPlan,
|
||||
llmStructuredDispatchPlan,
|
||||
llmValidateJsonSchema,
|
||||
parseNativeStructuredOutput,
|
||||
} from '../../../native';
|
||||
import { type ByokFeatureKind, ByokService } from '../byok';
|
||||
import { type StreamObject } from '../providers/types';
|
||||
import { CopilotExecutionMetrics } from './execution-metrics';
|
||||
import {
|
||||
type ExecutionPlan,
|
||||
type ExecutionPlanForKind,
|
||||
type NativeChatDispatchPlan,
|
||||
type NativeImageDispatchPlan,
|
||||
} from './execution-plan';
|
||||
import { mapNativeSemanticError } from './native-errors';
|
||||
import {
|
||||
createNativeToolLoopAdapter,
|
||||
NativeProviderAdapter,
|
||||
type NativeProviderAdapterOptions,
|
||||
} from './tool/native-adapter';
|
||||
|
||||
const logger = new Logger('NativeExecutionEngine');
|
||||
|
||||
function modelIdForError(modelId?: string) {
|
||||
return modelId ?? 'auto';
|
||||
}
|
||||
|
||||
type ExecutionPlanKind = ExecutionPlan['request']['kind'];
|
||||
type ValueExecutionKind = Exclude<
|
||||
ExecutionPlanKind,
|
||||
'streamText' | 'streamObject' | 'image'
|
||||
>;
|
||||
type StreamExecutionKind = Extract<
|
||||
ExecutionPlanKind,
|
||||
'streamText' | 'streamObject'
|
||||
>;
|
||||
export type NativeImageArtifact = LlmImageResponse['images'][number];
|
||||
|
||||
function resolveAbortSignal(
|
||||
signalOrOptions?: AbortSignal | { signal?: AbortSignal }
|
||||
) {
|
||||
return signalOrOptions &&
|
||||
typeof signalOrOptions === 'object' &&
|
||||
'aborted' in signalOrOptions
|
||||
? signalOrOptions
|
||||
: signalOrOptions?.signal;
|
||||
}
|
||||
|
||||
function extractTextResponse(response: LlmDispatchResponse) {
|
||||
return response.message.content
|
||||
.filter(part => part.type === 'text' || part.type === 'reasoning')
|
||||
.map(part => part.text)
|
||||
.join('')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function getUsageContext(plan: ExecutionPlan) {
|
||||
const options = 'options' in plan.request ? plan.request.options : undefined;
|
||||
const requestFeatureKind =
|
||||
plan.request.kind === 'text' ||
|
||||
plan.request.kind === 'streamText' ||
|
||||
plan.request.kind === 'streamObject'
|
||||
? 'chat'
|
||||
: plan.request.kind;
|
||||
return {
|
||||
workspaceId: options?.workspace,
|
||||
userId: options?.user,
|
||||
sessionId: options?.session,
|
||||
taskId: options?.taskId,
|
||||
actionId: options?.actionId,
|
||||
billingUnitId: options?.billingUnitId,
|
||||
featureKind: options?.featureKind ?? requestFeatureKind,
|
||||
};
|
||||
}
|
||||
|
||||
async function recordByokUsage(
|
||||
byok: ByokService,
|
||||
plan: ExecutionPlan,
|
||||
input: {
|
||||
providerId?: string;
|
||||
model?: string | null;
|
||||
usage?: LlmDispatchResponse['usage'];
|
||||
}
|
||||
) {
|
||||
const context = getUsageContext(plan);
|
||||
try {
|
||||
await byok.recordUsage({
|
||||
workspaceId: context.workspaceId,
|
||||
userId: context.userId,
|
||||
sessionId: context.sessionId,
|
||||
taskId: context.taskId,
|
||||
actionId: context.actionId,
|
||||
billingUnitId: context.billingUnitId,
|
||||
featureKind: context.featureKind as ByokFeatureKind,
|
||||
providerId: input.providerId,
|
||||
model: input.model,
|
||||
usage: input.usage,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Failed to record BYOK usage: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function recordSingleByokRouteFailure(
|
||||
byok: ByokService,
|
||||
plan: ExecutionPlan,
|
||||
error: unknown
|
||||
) {
|
||||
const [providerId] = plan.routePolicy.fallbackOrder;
|
||||
if (plan.routePolicy.fallbackOrder.length !== 1 || !providerId) {
|
||||
return;
|
||||
}
|
||||
const context = getUsageContext(plan);
|
||||
try {
|
||||
await byok.recordProviderFailure({
|
||||
workspaceId: context.workspaceId,
|
||||
providerId,
|
||||
featureKind: context.featureKind as ByokFeatureKind,
|
||||
error,
|
||||
});
|
||||
} catch (recordError) {
|
||||
logger.warn(
|
||||
`Failed to record BYOK provider failure: ${
|
||||
recordError instanceof Error ? recordError.message : String(recordError)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function recordPreparedDispatch(
|
||||
executionMetrics: CopilotExecutionMetrics | undefined,
|
||||
plan: ExecutionPlan,
|
||||
routeCount: number
|
||||
) {
|
||||
executionMetrics?.recordDispatch(
|
||||
plan.request.kind,
|
||||
'prepared_routes',
|
||||
routeCount
|
||||
);
|
||||
}
|
||||
|
||||
function createNativeChatAdapter(
|
||||
dispatch: NativeChatDispatchPlan,
|
||||
options?: {
|
||||
onUsage?: NativeProviderAdapterOptions['onUsage'];
|
||||
}
|
||||
) {
|
||||
if (dispatch.hasTools) {
|
||||
return createNativeToolLoopAdapter(
|
||||
{ preparedRoutes: dispatch.routes },
|
||||
dispatch.prepared.tools,
|
||||
{
|
||||
maxSteps: dispatch.prepared.maxSteps,
|
||||
nodeTextMiddleware: dispatch.prepared.postprocess?.nodeTextMiddleware,
|
||||
onUsage: options?.onUsage,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const nativeDispatch = (
|
||||
_nativeRequest: typeof dispatch.prepared.request,
|
||||
signalOrOptions?: AbortSignal | { signal?: AbortSignal }
|
||||
) =>
|
||||
llmDispatchPlanStream({
|
||||
preparedRoutes: dispatch.routes,
|
||||
signal: resolveAbortSignal(signalOrOptions),
|
||||
});
|
||||
|
||||
return new NativeProviderAdapter(nativeDispatch, {
|
||||
nodeTextMiddleware: dispatch.prepared.postprocess?.nodeTextMiddleware,
|
||||
onUsage: options?.onUsage,
|
||||
});
|
||||
}
|
||||
|
||||
async function runPreparedValuePlan<TResult>(
|
||||
plan: ExecutionPlan,
|
||||
routeCount: number,
|
||||
executionMetrics: CopilotExecutionMetrics | undefined,
|
||||
run: () => Promise<TResult>,
|
||||
byok: ByokService
|
||||
) {
|
||||
recordPreparedDispatch(executionMetrics, plan, routeCount);
|
||||
try {
|
||||
return await run();
|
||||
} catch (error) {
|
||||
const mapped = mapNativeSemanticError(error);
|
||||
await recordSingleByokRouteFailure(byok, plan, mapped);
|
||||
throw mapped;
|
||||
}
|
||||
}
|
||||
|
||||
async function* mapPreparedStreamErrors<T>(
|
||||
source: AsyncIterable<T>,
|
||||
plan: ExecutionPlan,
|
||||
byok: ByokService
|
||||
): AsyncIterableIterator<T> {
|
||||
try {
|
||||
yield* source;
|
||||
} catch (error) {
|
||||
const mapped = mapNativeSemanticError(error);
|
||||
await recordSingleByokRouteFailure(byok, plan, mapped);
|
||||
throw mapped;
|
||||
}
|
||||
}
|
||||
|
||||
async function runChatValuePlan(
|
||||
plan: ExecutionPlan,
|
||||
dispatch: NativeChatDispatchPlan,
|
||||
executionMetrics: CopilotExecutionMetrics | undefined,
|
||||
byok: ByokService
|
||||
) {
|
||||
const adapter = createNativeChatAdapter(dispatch);
|
||||
return await runPreparedValuePlan(
|
||||
plan,
|
||||
dispatch.routes.length,
|
||||
executionMetrics,
|
||||
async () => {
|
||||
if (
|
||||
!dispatch.hasTools &&
|
||||
!dispatch.prepared.postprocess?.nodeTextMiddleware?.length
|
||||
) {
|
||||
const result = await llmDispatchPlan({
|
||||
preparedRoutes: dispatch.routes,
|
||||
});
|
||||
await recordByokUsage(byok, plan, {
|
||||
providerId: result.provider_id,
|
||||
model: result.response.model,
|
||||
usage: result.response.usage,
|
||||
});
|
||||
return extractTextResponse(result.response);
|
||||
}
|
||||
|
||||
if (plan.request.kind !== 'text') {
|
||||
throw new Error('chat value dispatch requires text plan');
|
||||
}
|
||||
|
||||
return await adapter.text(
|
||||
dispatch.prepared.request,
|
||||
plan.hostContext.signal,
|
||||
plan.request.messages
|
||||
);
|
||||
},
|
||||
byok
|
||||
);
|
||||
}
|
||||
|
||||
async function* runChatStreamPlan(
|
||||
plan: ExecutionPlan,
|
||||
dispatch: NativeChatDispatchPlan,
|
||||
executionMetrics: CopilotExecutionMetrics | undefined,
|
||||
byok: ByokService
|
||||
): AsyncIterableIterator<string | StreamObject> {
|
||||
const adapter = createNativeChatAdapter(dispatch, {
|
||||
onUsage: async usage => {
|
||||
await recordByokUsage(byok, plan, {
|
||||
providerId: usage.providerId,
|
||||
model: usage.model,
|
||||
usage: usage.usage,
|
||||
});
|
||||
},
|
||||
});
|
||||
recordPreparedDispatch(executionMetrics, plan, dispatch.routes.length);
|
||||
|
||||
if (plan.request.kind === 'streamText') {
|
||||
yield* mapPreparedStreamErrors(
|
||||
adapter.streamText(
|
||||
dispatch.prepared.request,
|
||||
plan.hostContext.signal,
|
||||
plan.request.messages
|
||||
),
|
||||
plan,
|
||||
byok
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (plan.request.kind === 'streamObject') {
|
||||
yield* mapPreparedStreamErrors(
|
||||
adapter.streamObject(
|
||||
dispatch.prepared.request,
|
||||
plan.hostContext.signal,
|
||||
plan.request.messages
|
||||
),
|
||||
plan,
|
||||
byok
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error('chat stream dispatch requires streamText/streamObject plan');
|
||||
}
|
||||
|
||||
async function* runPreparedImageArtifactPlan(
|
||||
dispatch: NativeImageDispatchPlan,
|
||||
plan: ExecutionPlan,
|
||||
executionMetrics: CopilotExecutionMetrics | undefined,
|
||||
byok: ByokService
|
||||
): AsyncIterableIterator<NativeImageArtifact> {
|
||||
if (plan.request.kind !== 'image') {
|
||||
throw new Error('image dispatch requires image plan');
|
||||
}
|
||||
|
||||
recordPreparedDispatch(executionMetrics, plan, dispatch.routes.length);
|
||||
let result;
|
||||
try {
|
||||
result = await llmImageDispatchPlan({
|
||||
preparedRoutes: dispatch.routes,
|
||||
});
|
||||
await recordByokUsage(byok, plan, {
|
||||
providerId: result.provider_id,
|
||||
model: dispatch.prepared.route.model,
|
||||
usage: result.response.usage
|
||||
? {
|
||||
prompt_tokens: result.response.usage.input_tokens ?? 0,
|
||||
completion_tokens: result.response.usage.output_tokens ?? 0,
|
||||
total_tokens: result.response.usage.total_tokens ?? 0,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
} catch (error) {
|
||||
const mapped = mapNativeSemanticError(error);
|
||||
await recordSingleByokRouteFailure(byok, plan, mapped);
|
||||
throw mapped;
|
||||
}
|
||||
for (const artifact of result.response.images) {
|
||||
yield artifact;
|
||||
}
|
||||
}
|
||||
|
||||
async function executePreparedPlan(
|
||||
plan: ExecutionPlan,
|
||||
executionMetrics: CopilotExecutionMetrics | undefined,
|
||||
byok: ByokService
|
||||
): Promise<string | number[][] | number[] | null> {
|
||||
switch (plan.request.kind) {
|
||||
case 'text': {
|
||||
const dispatch = plan.nativeDispatch?.chat;
|
||||
return dispatch
|
||||
? await runChatValuePlan(plan, dispatch, executionMetrics, byok)
|
||||
: null;
|
||||
}
|
||||
case 'structured': {
|
||||
const dispatch = plan.nativeDispatch?.structured;
|
||||
if (!dispatch) {
|
||||
return null;
|
||||
}
|
||||
return await runPreparedValuePlan(
|
||||
plan,
|
||||
dispatch.routes.length,
|
||||
executionMetrics,
|
||||
async () => {
|
||||
const result = await llmStructuredDispatchPlan({
|
||||
preparedRoutes: dispatch.routes,
|
||||
});
|
||||
await recordByokUsage(byok, plan, {
|
||||
providerId: result.provider_id,
|
||||
model: result.response.model,
|
||||
usage: result.response.usage,
|
||||
});
|
||||
const parsed = parseNativeStructuredOutput(result.response);
|
||||
const validated = llmValidateJsonSchema(
|
||||
dispatch.prepared.request.schema,
|
||||
parsed
|
||||
);
|
||||
return JSON.stringify(validated);
|
||||
},
|
||||
byok
|
||||
);
|
||||
}
|
||||
case 'embedding': {
|
||||
const dispatch = plan.nativeDispatch?.embedding;
|
||||
if (!dispatch) {
|
||||
return null;
|
||||
}
|
||||
return await runPreparedValuePlan(
|
||||
plan,
|
||||
dispatch.routes.length,
|
||||
executionMetrics,
|
||||
async () => {
|
||||
const result = await llmEmbeddingDispatchPlan({
|
||||
preparedRoutes: dispatch.routes,
|
||||
});
|
||||
await recordByokUsage(byok, plan, {
|
||||
providerId: result.provider_id,
|
||||
model: result.response.model,
|
||||
usage: result.response.usage
|
||||
? {
|
||||
prompt_tokens: result.response.usage.prompt_tokens,
|
||||
completion_tokens: 0,
|
||||
total_tokens: result.response.usage.total_tokens,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
return result.response.embeddings;
|
||||
},
|
||||
byok
|
||||
);
|
||||
}
|
||||
case 'rerank': {
|
||||
const dispatch = plan.nativeDispatch?.rerank;
|
||||
if (!dispatch) {
|
||||
return null;
|
||||
}
|
||||
return await runPreparedValuePlan(
|
||||
plan,
|
||||
dispatch.routes.length,
|
||||
executionMetrics,
|
||||
async () => {
|
||||
const result = await llmRerankDispatchPlan({
|
||||
preparedRoutes: dispatch.routes,
|
||||
});
|
||||
await recordByokUsage(byok, plan, {
|
||||
providerId: result.provider_id,
|
||||
model: result.response.model,
|
||||
});
|
||||
return result.response.scores;
|
||||
},
|
||||
byok
|
||||
);
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function executePreparedStreamPlan(
|
||||
plan: ExecutionPlan,
|
||||
executionMetrics: CopilotExecutionMetrics | undefined,
|
||||
byok: ByokService
|
||||
): AsyncIterableIterator<string | StreamObject> | null {
|
||||
switch (plan.request.kind) {
|
||||
case 'streamText':
|
||||
case 'streamObject': {
|
||||
const dispatch = plan.nativeDispatch?.chat;
|
||||
return dispatch
|
||||
? runChatStreamPlan(plan, dispatch, executionMetrics, byok)
|
||||
: null;
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function noRouteStream<T>(plan: ExecutionPlan) {
|
||||
return (async function* (): AsyncIterableIterator<T> {
|
||||
yield* [] as T[];
|
||||
throw new NoCopilotProviderAvailable({
|
||||
modelId: modelIdForError(plan.request.cond.modelId),
|
||||
});
|
||||
})();
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class NativeExecutionEngine {
|
||||
constructor(
|
||||
private readonly byok: ByokService,
|
||||
private readonly executionMetrics?: CopilotExecutionMetrics
|
||||
) {}
|
||||
|
||||
private noRoute(plan: ExecutionPlan): never {
|
||||
throw new NoCopilotProviderAvailable({
|
||||
modelId: modelIdForError(plan.request.cond.modelId),
|
||||
});
|
||||
}
|
||||
|
||||
async execute(
|
||||
plan: ExecutionPlanForKind<'text' | 'structured'>
|
||||
): Promise<string>;
|
||||
async execute(plan: ExecutionPlanForKind<'embedding'>): Promise<number[][]>;
|
||||
async execute(plan: ExecutionPlanForKind<'rerank'>): Promise<number[]>;
|
||||
async execute(
|
||||
plan: ExecutionPlanForKind<ValueExecutionKind>
|
||||
): Promise<string | number[][] | number[]> {
|
||||
const result = await executePreparedPlan(
|
||||
plan,
|
||||
this.executionMetrics,
|
||||
this.byok
|
||||
);
|
||||
if (result === null) {
|
||||
return this.noRoute(plan);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
executeStream(
|
||||
plan: ExecutionPlanForKind<'streamText'>
|
||||
): AsyncIterableIterator<string>;
|
||||
executeStream(
|
||||
plan: ExecutionPlanForKind<'streamObject'>
|
||||
): AsyncIterableIterator<StreamObject>;
|
||||
executeStream(
|
||||
plan: ExecutionPlanForKind<StreamExecutionKind>
|
||||
): AsyncIterableIterator<string | StreamObject> {
|
||||
const result = executePreparedStreamPlan(
|
||||
plan,
|
||||
this.executionMetrics,
|
||||
this.byok
|
||||
);
|
||||
if (result) {
|
||||
return result;
|
||||
}
|
||||
|
||||
return noRouteStream(plan);
|
||||
}
|
||||
|
||||
executeImageArtifacts(
|
||||
plan: ExecutionPlanForKind<'image'>
|
||||
): AsyncIterableIterator<NativeImageArtifact> {
|
||||
const dispatch = plan.nativeDispatch?.image;
|
||||
if (dispatch) {
|
||||
return runPreparedImageArtifactPlan(
|
||||
dispatch,
|
||||
plan,
|
||||
this.executionMetrics,
|
||||
this.byok
|
||||
);
|
||||
}
|
||||
|
||||
return noRouteStream(plan);
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,7 @@ export async function buildCanonicalNativeRequest({
|
||||
request = llmBuildCanonicalRequest({
|
||||
model,
|
||||
messages: normalizedMessages,
|
||||
maxTokens: options.maxTokens ?? undefined,
|
||||
maxTokens: options.maxOutputTokens ?? undefined,
|
||||
temperature: options.temperature ?? undefined,
|
||||
tools: toolContracts,
|
||||
include,
|
||||
@@ -171,7 +171,7 @@ export async function buildCanonicalNativeStructuredRequest({
|
||||
model,
|
||||
messages: normalizedMessages,
|
||||
schema: explicitResponseContract?.responseSchemaJson,
|
||||
maxTokens: options.maxTokens ?? undefined,
|
||||
maxTokens: options.maxOutputTokens ?? undefined,
|
||||
temperature: options.temperature ?? undefined,
|
||||
reasoning,
|
||||
strict: options.strict,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
/* oxlint-disable import/no-cycle -- Prompt execution delegates to the capability runtime. */
|
||||
import { forwardRef, Inject, Injectable } from '@nestjs/common';
|
||||
|
||||
import { CopilotPromptNotFound } from '../../../base';
|
||||
import { PromptService } from '../prompt/service';
|
||||
@@ -11,7 +12,6 @@ import {
|
||||
} from '../providers/types';
|
||||
import { CapabilityRuntime } from './capability-runtime';
|
||||
import type { RequiredStructuredOutputContract } from './contracts';
|
||||
import { CapabilityPolicyHost } from './hosts/capability-policy-host';
|
||||
|
||||
type PromptRuntimeStructuredContract = RequiredStructuredOutputContract;
|
||||
|
||||
@@ -24,8 +24,11 @@ type PromptRuntimeStructuredProviderOptions = Omit<
|
||||
export class PromptRuntime {
|
||||
constructor(
|
||||
private readonly prompts: PromptService,
|
||||
private readonly capabilityPolicy: CapabilityPolicyHost,
|
||||
private readonly runtime: CapabilityRuntime
|
||||
@Inject(forwardRef(() => CapabilityRuntime))
|
||||
private readonly runtime: Pick<
|
||||
CapabilityRuntime,
|
||||
'text' | 'generateStructuredValue'
|
||||
>
|
||||
) {}
|
||||
|
||||
private async preparePrompt(
|
||||
@@ -44,11 +47,8 @@ export class PromptRuntime {
|
||||
|
||||
return {
|
||||
prompt,
|
||||
modelId: await this.capabilityPolicy.resolvePromptModel({
|
||||
defaultModel: prompt.model,
|
||||
optionalModels: prompt.optionalModels,
|
||||
requestedModelId: options.modelId,
|
||||
}),
|
||||
builtInRouteId: prompt.name,
|
||||
modelId: 'route-selected',
|
||||
finalMessages: [
|
||||
...this.prompts.finish(prompt, params),
|
||||
...(options.appendMessages ?? []),
|
||||
@@ -75,6 +75,7 @@ export class PromptRuntime {
|
||||
{
|
||||
...prepared.prompt.config,
|
||||
...options.providerOptions,
|
||||
builtInRouteId: prepared.builtInRouteId,
|
||||
},
|
||||
{ prefer: prepared.prefer }
|
||||
);
|
||||
@@ -100,6 +101,7 @@ export class PromptRuntime {
|
||||
{
|
||||
...prepared.prompt.config,
|
||||
...options.providerOptions,
|
||||
builtInRouteId: prepared.builtInRouteId,
|
||||
responseSchemaJson: options.responseContract.responseSchemaJson,
|
||||
schemaHash: options.responseContract.schemaHash,
|
||||
strict: options.strict,
|
||||
|
||||
@@ -1,234 +0,0 @@
|
||||
import type {
|
||||
CopilotProviderExecution,
|
||||
PreparedNativeExecution,
|
||||
PreparedNativeRequestOptions,
|
||||
ProviderChatDriver,
|
||||
ProviderChatDriverPrepareInput,
|
||||
} from '../providers/provider-runtime-contract';
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotProviderModel,
|
||||
CopilotProviderType,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
PromptMessage,
|
||||
StreamObject,
|
||||
} from '../providers/types';
|
||||
import { ModelOutputType } from '../providers/types';
|
||||
import {
|
||||
resolveDriverOrThrow,
|
||||
resolvePreparedModelId,
|
||||
runPreparedExecution,
|
||||
} from './provider-driver-runtime';
|
||||
import type { NativeProviderAdapter } from './tool/native-adapter';
|
||||
|
||||
type MetricLabels = Record<string, string | number | boolean | undefined>;
|
||||
|
||||
export type ChatRuntimeContext = {
|
||||
type: CopilotProviderType;
|
||||
resolveChatDriver: () => ProviderChatDriver | undefined;
|
||||
selectModel: (cond: ModelFullConditions) => CopilotProviderModel;
|
||||
metricLabels: (
|
||||
model: string,
|
||||
labels?: MetricLabels,
|
||||
execution?: CopilotProviderExecution
|
||||
) => MetricLabels;
|
||||
createPreparedExecutionAdapter: (
|
||||
prepared: PreparedNativeExecution
|
||||
) => NativeProviderAdapter;
|
||||
};
|
||||
|
||||
type ChatExecutionMode = {
|
||||
kind: ProviderChatDriverPrepareInput['kind'];
|
||||
outputType: ModelOutputType;
|
||||
unsupportedKind: 'text' | 'object';
|
||||
callMetric: string;
|
||||
errorMetric: string;
|
||||
};
|
||||
|
||||
export async function prepareNativeChatExecution(
|
||||
resolveChatDriver: () => ProviderChatDriver | undefined,
|
||||
buildPreparedNativeExecution: (
|
||||
options: PreparedNativeRequestOptions
|
||||
) => Promise<PreparedNativeExecution>,
|
||||
input: ProviderChatDriverPrepareInput
|
||||
): Promise<PreparedNativeExecution | null> {
|
||||
const driver = resolveChatDriver();
|
||||
if (!driver) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const prepared = await driver.prepare(input);
|
||||
if (!prepared) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return await buildPreparedNativeExecution({
|
||||
...prepared,
|
||||
execution: input.execution,
|
||||
options: input.options,
|
||||
});
|
||||
}
|
||||
|
||||
async function runNativeChat(
|
||||
context: ChatRuntimeContext,
|
||||
prepareNativeExecution: (
|
||||
kind: ProviderChatDriverPrepareInput['kind'],
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => Promise<PreparedNativeExecution | null>,
|
||||
mode: ChatExecutionMode,
|
||||
model: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotChatOptions | undefined,
|
||||
execution: CopilotProviderExecution | undefined,
|
||||
run: (
|
||||
adapter: NativeProviderAdapter,
|
||||
prepared: PreparedNativeExecution,
|
||||
signal: AbortSignal | undefined,
|
||||
promptMessages: PromptMessage[]
|
||||
) =>
|
||||
| Promise<string>
|
||||
| AsyncIterableIterator<string>
|
||||
| AsyncIterableIterator<StreamObject>
|
||||
) {
|
||||
const driver = resolveDriverOrThrow(
|
||||
context.type,
|
||||
mode.unsupportedKind,
|
||||
context.resolveChatDriver
|
||||
);
|
||||
const chatOptions = options ?? {};
|
||||
const prepared = await prepareNativeExecution(
|
||||
mode.kind,
|
||||
model,
|
||||
messages,
|
||||
chatOptions,
|
||||
execution
|
||||
);
|
||||
const modelId = resolvePreparedModelId(
|
||||
context,
|
||||
model,
|
||||
mode.outputType,
|
||||
prepared
|
||||
);
|
||||
|
||||
return await runPreparedExecution({
|
||||
driver,
|
||||
prepared,
|
||||
modelId,
|
||||
execution,
|
||||
metricContext: context,
|
||||
metricsName: {
|
||||
call: mode.callMetric,
|
||||
error: mode.errorMetric,
|
||||
},
|
||||
execute: async preparedExecution =>
|
||||
await run(
|
||||
context.createPreparedExecutionAdapter(preparedExecution),
|
||||
preparedExecution,
|
||||
chatOptions.signal,
|
||||
messages
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
export async function runNativeText(
|
||||
context: ChatRuntimeContext,
|
||||
prepareNativeExecution: (
|
||||
kind: ProviderChatDriverPrepareInput['kind'],
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => Promise<PreparedNativeExecution | null>,
|
||||
model: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) {
|
||||
return (await runNativeChat(
|
||||
context,
|
||||
prepareNativeExecution,
|
||||
{
|
||||
kind: 'text',
|
||||
outputType: ModelOutputType.Text,
|
||||
unsupportedKind: 'text',
|
||||
callMetric: 'chat_text_calls',
|
||||
errorMetric: 'chat_text_errors',
|
||||
},
|
||||
model,
|
||||
messages,
|
||||
options,
|
||||
execution,
|
||||
(adapter, prepared, signal, promptMessages) =>
|
||||
adapter.text(prepared.request, signal, promptMessages)
|
||||
)) as string;
|
||||
}
|
||||
|
||||
export async function* runNativeStreamText(
|
||||
context: ChatRuntimeContext,
|
||||
prepareNativeExecution: (
|
||||
kind: ProviderChatDriverPrepareInput['kind'],
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => Promise<PreparedNativeExecution | null>,
|
||||
model: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
): AsyncIterableIterator<string> {
|
||||
yield* (await runNativeChat(
|
||||
context,
|
||||
prepareNativeExecution,
|
||||
{
|
||||
kind: 'streamText',
|
||||
outputType: ModelOutputType.Text,
|
||||
unsupportedKind: 'text',
|
||||
callMetric: 'chat_text_stream_calls',
|
||||
errorMetric: 'chat_text_stream_errors',
|
||||
},
|
||||
model,
|
||||
messages,
|
||||
options,
|
||||
execution,
|
||||
(adapter, prepared, signal, promptMessages) =>
|
||||
adapter.streamText(prepared.request, signal, promptMessages)
|
||||
)) as AsyncIterableIterator<string>;
|
||||
}
|
||||
|
||||
export async function* runNativeStreamObject(
|
||||
context: ChatRuntimeContext,
|
||||
prepareNativeExecution: (
|
||||
kind: ProviderChatDriverPrepareInput['kind'],
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => Promise<PreparedNativeExecution | null>,
|
||||
model: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
): AsyncIterableIterator<StreamObject> {
|
||||
yield* (await runNativeChat(
|
||||
context,
|
||||
prepareNativeExecution,
|
||||
{
|
||||
kind: 'streamObject',
|
||||
outputType: ModelOutputType.Object,
|
||||
unsupportedKind: 'object',
|
||||
callMetric: 'chat_object_stream_calls',
|
||||
errorMetric: 'chat_object_stream_errors',
|
||||
},
|
||||
model,
|
||||
messages,
|
||||
options,
|
||||
execution,
|
||||
(adapter, prepared, signal, promptMessages) =>
|
||||
adapter.streamObject(prepared.request, signal, promptMessages)
|
||||
)) as AsyncIterableIterator<StreamObject>;
|
||||
}
|
||||
@@ -1,682 +0,0 @@
|
||||
import {
|
||||
CopilotPromptInvalid,
|
||||
CopilotProviderNotSupported,
|
||||
metrics,
|
||||
} from '../../../base';
|
||||
import {
|
||||
buildLlmEmbeddingRequest,
|
||||
buildLlmRerankRequest,
|
||||
type LlmBackendConfig,
|
||||
type LlmEmbeddingRequest,
|
||||
type LlmProtocol,
|
||||
type LlmRerankRequest,
|
||||
type LlmStructuredRequest,
|
||||
type LlmStructuredResponse,
|
||||
llmValidateJsonSchema,
|
||||
parseNativeStructuredOutput,
|
||||
} from '../../../native';
|
||||
import type { ProviderMiddlewareConfig } from '../config';
|
||||
import { resolveProviderModelRoute } from '../providers/provider-model-runtime';
|
||||
import type {
|
||||
CopilotProviderExecution,
|
||||
EmbeddingProviderDriver,
|
||||
ImageProviderDriver,
|
||||
PreparedNativeEmbeddingExecution,
|
||||
PreparedNativeImageExecution,
|
||||
PreparedNativeRerankExecution,
|
||||
PreparedNativeStructuredExecution,
|
||||
RerankProviderDriver,
|
||||
StructuredProviderDriver,
|
||||
} from '../providers/provider-runtime-contract';
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotEmbeddingOptions,
|
||||
CopilotImageOptions,
|
||||
CopilotProviderModel,
|
||||
CopilotProviderType,
|
||||
CopilotRerankRequest,
|
||||
CopilotStructuredOptions,
|
||||
ModelAttachmentCapability,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
PromptMessage,
|
||||
} from '../providers/types';
|
||||
import { ModelOutputType } from '../providers/types';
|
||||
import { type RequiredStructuredOutputContract } from './contracts';
|
||||
import { buildNativeStructuredRequest } from './native-request-runtime';
|
||||
|
||||
const DEFAULT_EMBEDDING_TASK_TYPE = 'RETRIEVAL_DOCUMENT';
|
||||
|
||||
type MetricLabels = Record<string, string | number | boolean | undefined>;
|
||||
type DriverMetricNames = {
|
||||
call: string;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export type StructuredRuntimeContext = {
|
||||
type: CopilotProviderType;
|
||||
resolveStructuredDriver: () => StructuredProviderDriver | undefined;
|
||||
checkParams: (input: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: CopilotProviderExecution;
|
||||
}) => Promise<ModelFullConditions>;
|
||||
selectModel: (
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => CopilotProviderModel;
|
||||
getAttachCapability: (
|
||||
model: CopilotProviderModel,
|
||||
outputType: ModelOutputType
|
||||
) => ModelAttachmentCapability | undefined;
|
||||
getActiveProviderMiddleware: (
|
||||
execution?: CopilotProviderExecution
|
||||
) => ProviderMiddlewareConfig;
|
||||
buildPreparedNativeStructuredExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmStructuredRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) => PreparedNativeStructuredExecution;
|
||||
createNativeStructuredDispatch: (
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol,
|
||||
execution?: CopilotProviderExecution
|
||||
) => (request: LlmStructuredRequest) => Promise<LlmStructuredResponse>;
|
||||
metricLabels: (
|
||||
model: string,
|
||||
labels?: MetricLabels,
|
||||
execution?: CopilotProviderExecution
|
||||
) => MetricLabels;
|
||||
};
|
||||
|
||||
export type EmbeddingRuntimeContext = {
|
||||
type: CopilotProviderType;
|
||||
resolveEmbeddingDriver: () => EmbeddingProviderDriver | undefined;
|
||||
checkParams: (input: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: CopilotProviderExecution;
|
||||
}) => Promise<ModelFullConditions>;
|
||||
selectModel: (
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => CopilotProviderModel;
|
||||
buildPreparedNativeEmbeddingExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmEmbeddingRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) => PreparedNativeEmbeddingExecution;
|
||||
createNativeEmbeddingDispatch: (
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol,
|
||||
execution?: CopilotProviderExecution
|
||||
) => (request: LlmEmbeddingRequest) => Promise<{ embeddings: number[][] }>;
|
||||
metricLabels: (
|
||||
model: string,
|
||||
labels?: MetricLabels,
|
||||
execution?: CopilotProviderExecution
|
||||
) => MetricLabels;
|
||||
};
|
||||
|
||||
export type RerankRuntimeContext = {
|
||||
type: CopilotProviderType;
|
||||
resolveRerankDriver: () => RerankProviderDriver | undefined;
|
||||
checkParams: (input: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: CopilotProviderExecution;
|
||||
}) => Promise<ModelFullConditions>;
|
||||
selectModel: (
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => CopilotProviderModel;
|
||||
buildPreparedNativeRerankExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmRerankRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) => PreparedNativeRerankExecution;
|
||||
createNativeRerankDispatch: (
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol,
|
||||
execution?: CopilotProviderExecution
|
||||
) => (request: LlmRerankRequest) => Promise<{ scores: number[] }>;
|
||||
};
|
||||
|
||||
export type ImageRuntimeContext = {
|
||||
type: CopilotProviderType;
|
||||
resolveImageDriver: () => ImageProviderDriver | undefined;
|
||||
checkParams: (input: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
options?: CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: CopilotProviderExecution;
|
||||
}) => Promise<ModelFullConditions>;
|
||||
selectModel: (
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => CopilotProviderModel;
|
||||
buildPreparedNativeImageExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotImageOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => PreparedNativeImageExecution;
|
||||
};
|
||||
|
||||
type NativeExecutionDriverBase = {
|
||||
createBackendConfig: (
|
||||
execution?: CopilotProviderExecution
|
||||
) => Promise<LlmBackendConfig> | LlmBackendConfig;
|
||||
mapError: (error: unknown) => unknown;
|
||||
};
|
||||
|
||||
type ModelSelectionContext = {
|
||||
selectModel: (
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => CopilotProviderModel;
|
||||
};
|
||||
|
||||
type MetricContext = {
|
||||
metricLabels: (
|
||||
model: string,
|
||||
labels?: MetricLabels,
|
||||
execution?: CopilotProviderExecution
|
||||
) => MetricLabels;
|
||||
};
|
||||
|
||||
type RoutedPreparedExecution = {
|
||||
route: {
|
||||
model: string;
|
||||
backendConfig: LlmBackendConfig;
|
||||
protocol: LlmProtocol;
|
||||
};
|
||||
};
|
||||
|
||||
export function resolveDriverOrThrow<TDriver>(
|
||||
type: CopilotProviderType,
|
||||
kind: string,
|
||||
resolveDriver: () => TDriver | undefined
|
||||
) {
|
||||
const driver = resolveDriver();
|
||||
if (!driver) {
|
||||
throw new CopilotProviderNotSupported({
|
||||
provider: type,
|
||||
kind,
|
||||
});
|
||||
}
|
||||
return driver;
|
||||
}
|
||||
|
||||
export function resolvePreparedModelId(
|
||||
context: ModelSelectionContext,
|
||||
cond: ModelConditions,
|
||||
outputType: ModelOutputType,
|
||||
prepared?: RoutedPreparedExecution | null
|
||||
) {
|
||||
return (
|
||||
prepared?.route.model ??
|
||||
context.selectModel({
|
||||
...cond,
|
||||
outputType,
|
||||
}).id
|
||||
);
|
||||
}
|
||||
|
||||
async function prepareNativeExecutionBase<
|
||||
TDriver extends NativeExecutionDriverBase,
|
||||
TPrepared,
|
||||
>({
|
||||
resolveDriver,
|
||||
cond,
|
||||
outputType,
|
||||
checkParams,
|
||||
selectModel,
|
||||
execution,
|
||||
checkInput,
|
||||
buildPrepared,
|
||||
}: {
|
||||
resolveDriver: () => TDriver | undefined;
|
||||
cond: ModelConditions;
|
||||
outputType: ModelOutputType;
|
||||
checkParams: (input: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: CopilotProviderExecution;
|
||||
}) => Promise<ModelFullConditions>;
|
||||
selectModel: (
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => CopilotProviderModel;
|
||||
execution?: CopilotProviderExecution;
|
||||
checkInput: {
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
};
|
||||
buildPrepared: (args: {
|
||||
driver: TDriver;
|
||||
model: CopilotProviderModel;
|
||||
backendConfig: LlmBackendConfig;
|
||||
protocol: LlmProtocol;
|
||||
}) => Promise<TPrepared> | TPrepared;
|
||||
}): Promise<TPrepared | null> {
|
||||
const driver = resolveDriver();
|
||||
if (!driver) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalizedCond = await checkParams({
|
||||
...checkInput,
|
||||
cond: { ...cond, outputType },
|
||||
execution,
|
||||
});
|
||||
const model = selectModel(normalizedCond, execution);
|
||||
const backendConfig = await driver.createBackendConfig(execution);
|
||||
const route = resolveProviderModelRoute(model, outputType);
|
||||
if (!route.protocol) {
|
||||
throw new Error(`Missing native protocol for model ${model.id}`);
|
||||
}
|
||||
|
||||
return await buildPrepared({
|
||||
driver,
|
||||
model,
|
||||
backendConfig:
|
||||
route.requestLayer === backendConfig.request_layer
|
||||
? backendConfig
|
||||
: { ...backendConfig, request_layer: route.requestLayer },
|
||||
protocol: route.protocol,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runPreparedExecution<
|
||||
TPrepared extends RoutedPreparedExecution,
|
||||
TResult,
|
||||
>({
|
||||
driver,
|
||||
prepared,
|
||||
modelId,
|
||||
execution,
|
||||
metricContext,
|
||||
metricsName,
|
||||
execute,
|
||||
}: {
|
||||
driver: Pick<NativeExecutionDriverBase, 'mapError'>;
|
||||
prepared: TPrepared | null;
|
||||
modelId: string;
|
||||
execution?: CopilotProviderExecution;
|
||||
metricContext?: MetricContext;
|
||||
metricsName?: DriverMetricNames;
|
||||
execute: (prepared: TPrepared) => Promise<TResult>;
|
||||
}): Promise<TResult> {
|
||||
try {
|
||||
if (metricsName && metricContext) {
|
||||
metrics.ai
|
||||
.counter(metricsName.call)
|
||||
.add(1, metricContext.metricLabels(modelId, {}, execution));
|
||||
}
|
||||
if (!prepared) {
|
||||
throw new Error('native route is not available');
|
||||
}
|
||||
return await execute(prepared);
|
||||
} catch (error) {
|
||||
if (metricsName && metricContext) {
|
||||
metrics.ai
|
||||
.counter(metricsName.error)
|
||||
.add(1, metricContext.metricLabels(modelId, {}, execution));
|
||||
}
|
||||
throw driver.mapError(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function prepareNativeStructuredExecution(
|
||||
context: StructuredRuntimeContext,
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotStructuredOptions = {},
|
||||
responseContract?: RequiredStructuredOutputContract,
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<PreparedNativeStructuredExecution | null> {
|
||||
const driver = context.resolveStructuredDriver();
|
||||
if (!driver) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const structuredOptions = options ?? {};
|
||||
const normalizedCond = await context.checkParams({
|
||||
messages,
|
||||
cond: { ...cond, outputType: ModelOutputType.Structured },
|
||||
options: structuredOptions,
|
||||
execution,
|
||||
});
|
||||
const model = context.selectModel(normalizedCond, execution);
|
||||
const backendConfig = await driver.createBackendConfig(execution);
|
||||
const route = resolveProviderModelRoute(model, ModelOutputType.Structured);
|
||||
if (!route.protocol) {
|
||||
throw new Error(`Missing native protocol for model ${model.id}`);
|
||||
}
|
||||
const preparedMessages = driver.prepareMessages
|
||||
? await driver.prepareMessages(messages, backendConfig, structuredOptions)
|
||||
: messages;
|
||||
if (!responseContract) {
|
||||
throw new CopilotPromptInvalid('Schema is required');
|
||||
}
|
||||
const { request } = await buildNativeStructuredRequest({
|
||||
model: model.id,
|
||||
messages: preparedMessages,
|
||||
options: structuredOptions,
|
||||
responseContract,
|
||||
attachmentCapability: context.getAttachCapability(
|
||||
model,
|
||||
ModelOutputType.Structured
|
||||
),
|
||||
middleware: context.getActiveProviderMiddleware(execution),
|
||||
});
|
||||
|
||||
return context.buildPreparedNativeStructuredExecution(
|
||||
route.protocol,
|
||||
route.requestLayer === backendConfig.request_layer
|
||||
? backendConfig
|
||||
: { ...backendConfig, request_layer: route.requestLayer },
|
||||
model.id,
|
||||
request,
|
||||
execution
|
||||
);
|
||||
}
|
||||
|
||||
export async function runNativeStructured(
|
||||
context: StructuredRuntimeContext,
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotStructuredOptions = {},
|
||||
responseContract?: RequiredStructuredOutputContract,
|
||||
execution?: CopilotProviderExecution
|
||||
) {
|
||||
const driver = resolveDriverOrThrow(
|
||||
context.type,
|
||||
'structure',
|
||||
context.resolveStructuredDriver
|
||||
);
|
||||
const structuredOptions = options ?? {};
|
||||
const prepared = await prepareNativeStructuredExecution(
|
||||
context,
|
||||
cond,
|
||||
messages,
|
||||
structuredOptions,
|
||||
responseContract,
|
||||
execution
|
||||
);
|
||||
const modelId = resolvePreparedModelId(
|
||||
context,
|
||||
cond,
|
||||
ModelOutputType.Structured,
|
||||
prepared
|
||||
);
|
||||
|
||||
return await runPreparedExecution({
|
||||
driver,
|
||||
prepared,
|
||||
modelId,
|
||||
execution,
|
||||
metricContext: context,
|
||||
metricsName: {
|
||||
call: 'chat_text_calls',
|
||||
error: 'chat_text_errors',
|
||||
},
|
||||
execute: async preparedExecution => {
|
||||
const dispatch = context.createNativeStructuredDispatch(
|
||||
preparedExecution.route.backendConfig,
|
||||
preparedExecution.route.protocol,
|
||||
execution
|
||||
);
|
||||
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
const response = await dispatch(preparedExecution.request);
|
||||
const parsed = parseNativeStructuredOutput(response);
|
||||
const validated = llmValidateJsonSchema(
|
||||
preparedExecution.request.schema,
|
||||
parsed
|
||||
);
|
||||
return JSON.stringify(validated);
|
||||
} catch (error) {
|
||||
if (
|
||||
!(await driver.shouldRetry?.({
|
||||
error,
|
||||
attempt,
|
||||
options: structuredOptions,
|
||||
}))
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function prepareNativeEmbeddingExecution(
|
||||
context: EmbeddingRuntimeContext,
|
||||
cond: ModelConditions,
|
||||
input: string | string[],
|
||||
options: CopilotEmbeddingOptions = {},
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<PreparedNativeEmbeddingExecution | null> {
|
||||
const values = Array.isArray(input) ? input : [input];
|
||||
return await prepareNativeExecutionBase({
|
||||
resolveDriver: context.resolveEmbeddingDriver,
|
||||
cond,
|
||||
outputType: ModelOutputType.Embedding,
|
||||
checkParams: context.checkParams,
|
||||
selectModel: context.selectModel,
|
||||
execution,
|
||||
checkInput: {
|
||||
embeddings: values,
|
||||
options,
|
||||
},
|
||||
buildPrepared: ({ driver, model, backendConfig, protocol }) =>
|
||||
context.buildPreparedNativeEmbeddingExecution(
|
||||
protocol,
|
||||
backendConfig,
|
||||
model.id,
|
||||
buildLlmEmbeddingRequest({
|
||||
model: model.id,
|
||||
inputs: values,
|
||||
dimensions: options?.dimensions ?? driver.defaultDimensions,
|
||||
taskType: driver.taskType ?? DEFAULT_EMBEDDING_TASK_TYPE,
|
||||
}),
|
||||
execution
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
export async function runNativeEmbedding(
|
||||
context: EmbeddingRuntimeContext,
|
||||
cond: ModelConditions,
|
||||
input: string | string[],
|
||||
options?: CopilotEmbeddingOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) {
|
||||
const driver = resolveDriverOrThrow(
|
||||
context.type,
|
||||
ModelOutputType.Embedding,
|
||||
context.resolveEmbeddingDriver
|
||||
);
|
||||
const prepared = await prepareNativeEmbeddingExecution(
|
||||
context,
|
||||
cond,
|
||||
input,
|
||||
options,
|
||||
execution
|
||||
);
|
||||
const modelId = resolvePreparedModelId(
|
||||
context,
|
||||
cond,
|
||||
ModelOutputType.Embedding,
|
||||
prepared
|
||||
);
|
||||
|
||||
return await runPreparedExecution({
|
||||
driver,
|
||||
prepared,
|
||||
modelId,
|
||||
execution,
|
||||
metricContext: context,
|
||||
metricsName: {
|
||||
call: 'generate_embedding_calls',
|
||||
error: 'generate_embedding_errors',
|
||||
},
|
||||
execute: async preparedExecution => {
|
||||
const response = await context.createNativeEmbeddingDispatch(
|
||||
preparedExecution.route.backendConfig,
|
||||
preparedExecution.route.protocol,
|
||||
execution
|
||||
)(preparedExecution.request);
|
||||
return response.embeddings;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function prepareNativeRerankExecution(
|
||||
context: RerankRuntimeContext,
|
||||
cond: ModelConditions,
|
||||
request: CopilotRerankRequest,
|
||||
options: CopilotChatOptions = {},
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<PreparedNativeRerankExecution | null> {
|
||||
return await prepareNativeExecutionBase({
|
||||
resolveDriver: context.resolveRerankDriver,
|
||||
cond,
|
||||
outputType: ModelOutputType.Rerank,
|
||||
checkParams: context.checkParams,
|
||||
selectModel: context.selectModel,
|
||||
execution,
|
||||
checkInput: {
|
||||
messages: [],
|
||||
options,
|
||||
},
|
||||
buildPrepared: ({ model, backendConfig, protocol }) =>
|
||||
context.buildPreparedNativeRerankExecution(
|
||||
protocol,
|
||||
backendConfig,
|
||||
model.id,
|
||||
buildLlmRerankRequest(model.id, request),
|
||||
execution
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
export async function prepareNativeImageExecution(
|
||||
context: ImageRuntimeContext,
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotImageOptions = {},
|
||||
execution?: CopilotProviderExecution
|
||||
): Promise<PreparedNativeImageExecution | null> {
|
||||
return await prepareNativeExecutionBase({
|
||||
resolveDriver: context.resolveImageDriver,
|
||||
cond,
|
||||
outputType: ModelOutputType.Image,
|
||||
checkParams: context.checkParams,
|
||||
selectModel: context.selectModel,
|
||||
execution,
|
||||
checkInput: {
|
||||
messages,
|
||||
options,
|
||||
},
|
||||
buildPrepared: async ({ driver, model, backendConfig, protocol }) => {
|
||||
const preparedMessages = driver.prepareMessages
|
||||
? await driver.prepareMessages(messages, backendConfig, options)
|
||||
: messages;
|
||||
|
||||
return context.buildPreparedNativeImageExecution(
|
||||
protocol,
|
||||
backendConfig,
|
||||
model.id,
|
||||
preparedMessages,
|
||||
options,
|
||||
execution
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function runNativeRerank(
|
||||
context: RerankRuntimeContext,
|
||||
cond: ModelConditions,
|
||||
request: CopilotRerankRequest,
|
||||
options: CopilotChatOptions = {},
|
||||
execution?: CopilotProviderExecution
|
||||
) {
|
||||
const driver = resolveDriverOrThrow(
|
||||
context.type,
|
||||
ModelOutputType.Rerank,
|
||||
context.resolveRerankDriver
|
||||
);
|
||||
const prepared = await prepareNativeRerankExecution(
|
||||
context,
|
||||
cond,
|
||||
request,
|
||||
options,
|
||||
execution
|
||||
);
|
||||
|
||||
const modelId = resolvePreparedModelId(
|
||||
context,
|
||||
cond,
|
||||
ModelOutputType.Rerank,
|
||||
prepared
|
||||
);
|
||||
|
||||
return await runPreparedExecution({
|
||||
driver,
|
||||
prepared,
|
||||
modelId,
|
||||
execution,
|
||||
execute: async preparedExecution => {
|
||||
const response = await context.createNativeRerankDispatch(
|
||||
preparedExecution.route.backendConfig,
|
||||
preparedExecution.route.protocol,
|
||||
execution
|
||||
)(preparedExecution.request);
|
||||
return response.scores;
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,490 +0,0 @@
|
||||
import type {
|
||||
LlmBackendConfig,
|
||||
LlmEmbeddingRequest,
|
||||
LlmProtocol,
|
||||
LlmRerankRequest,
|
||||
LlmStructuredRequest,
|
||||
LlmStructuredResponse,
|
||||
} from '../../../native';
|
||||
import type { ProviderMiddlewareConfig } from '../config';
|
||||
import type { CopilotProvider } from '../providers/provider';
|
||||
import type { ProviderModelRuntimeContext } from '../providers/provider-model-runtime';
|
||||
import {
|
||||
createNativeEmbeddingDispatch as inputCreateNativeEmbeddingDispatch,
|
||||
createNativeRerankDispatch as inputCreateNativeRerankDispatch,
|
||||
createNativeStructuredDispatch as inputCreateNativeStructuredDispatch,
|
||||
createPreparedExecutionRuntime,
|
||||
type CreatePreparedExecutionRuntimeInput,
|
||||
type PreparedExecutionRuntime,
|
||||
} from '../providers/provider-native-runtime';
|
||||
import type {
|
||||
CopilotProviderExecution,
|
||||
EmbeddingProviderDriver,
|
||||
ImageProviderDriver,
|
||||
PreparedNativeEmbeddingExecution,
|
||||
PreparedNativeExecution,
|
||||
PreparedNativeImageExecution,
|
||||
PreparedNativeRequestOptions,
|
||||
PreparedNativeRerankExecution,
|
||||
PreparedNativeStructuredExecution,
|
||||
ProviderExecutionDrivers,
|
||||
ProviderMetricLabels,
|
||||
ProviderRuntimeHostSeed,
|
||||
RerankProviderDriver,
|
||||
StructuredProviderDriver,
|
||||
} from '../providers/provider-runtime-contract';
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotEmbeddingOptions,
|
||||
CopilotImageOptions,
|
||||
CopilotProviderModel,
|
||||
CopilotRerankRequest,
|
||||
CopilotStructuredOptions,
|
||||
ModelAttachmentCapability,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
ModelOutputType,
|
||||
PromptMessage,
|
||||
} from '../providers/types';
|
||||
import type { CopilotToolSet } from '../tools';
|
||||
import type { RequiredStructuredOutputContract } from './contracts';
|
||||
import type { ChatRuntimeContext } from './provider-chat-runtime';
|
||||
import {
|
||||
prepareNativeChatExecution,
|
||||
runNativeStreamObject,
|
||||
runNativeStreamText,
|
||||
runNativeText,
|
||||
} from './provider-chat-runtime';
|
||||
import type {
|
||||
EmbeddingRuntimeContext,
|
||||
ImageRuntimeContext,
|
||||
RerankRuntimeContext,
|
||||
StructuredRuntimeContext,
|
||||
} from './provider-driver-runtime';
|
||||
import {
|
||||
prepareNativeEmbeddingExecution,
|
||||
prepareNativeImageExecution,
|
||||
prepareNativeRerankExecution,
|
||||
prepareNativeStructuredExecution,
|
||||
runNativeEmbedding,
|
||||
runNativeRerank,
|
||||
runNativeStructured,
|
||||
} from './provider-driver-runtime';
|
||||
import type { NativeProviderAdapter } from './tool/native-adapter';
|
||||
|
||||
type ProviderRuntimeContextInput = {
|
||||
model: ProviderModelRuntimeContext;
|
||||
resolveExecutionDrivers: () => ProviderExecutionDrivers | undefined;
|
||||
selectModel: (
|
||||
cond: ModelFullConditions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => CopilotProviderModel;
|
||||
metricLabels: (
|
||||
model: string,
|
||||
labels?: ProviderMetricLabels,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ProviderMetricLabels;
|
||||
checkParams: (input: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
options?:
|
||||
| CopilotChatOptions
|
||||
| CopilotStructuredOptions
|
||||
| CopilotImageOptions;
|
||||
withAttachment?: boolean;
|
||||
execution?: CopilotProviderExecution;
|
||||
}) => Promise<ModelFullConditions>;
|
||||
getAttachCapability: (
|
||||
model: CopilotProviderModel,
|
||||
outputType: ModelOutputType
|
||||
) => ModelAttachmentCapability | undefined;
|
||||
getActiveProviderMiddleware: (
|
||||
execution?: CopilotProviderExecution
|
||||
) => ProviderMiddlewareConfig;
|
||||
getTools: (
|
||||
options: CopilotChatOptions,
|
||||
model: string
|
||||
) => Promise<CopilotToolSet>;
|
||||
buildPreparedNativeExecution: (
|
||||
options: PreparedNativeRequestOptions
|
||||
) => Promise<PreparedNativeExecution>;
|
||||
createPreparedExecutionAdapter: (
|
||||
prepared: PreparedNativeExecution
|
||||
) => NativeProviderAdapter;
|
||||
buildPreparedNativeStructuredExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmStructuredRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) => PreparedNativeStructuredExecution;
|
||||
createNativeStructuredDispatch: (
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol,
|
||||
execution?: CopilotProviderExecution
|
||||
) => (request: LlmStructuredRequest) => Promise<LlmStructuredResponse>;
|
||||
buildPreparedNativeEmbeddingExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmEmbeddingRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) => PreparedNativeEmbeddingExecution;
|
||||
createNativeEmbeddingDispatch: (
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol,
|
||||
execution?: CopilotProviderExecution
|
||||
) => (request: LlmEmbeddingRequest) => Promise<{ embeddings: number[][] }>;
|
||||
buildPreparedNativeRerankExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
request: LlmRerankRequest,
|
||||
execution?: CopilotProviderExecution
|
||||
) => PreparedNativeRerankExecution;
|
||||
createNativeRerankDispatch: (
|
||||
backendConfig: LlmBackendConfig,
|
||||
protocol: LlmProtocol,
|
||||
execution?: CopilotProviderExecution
|
||||
) => (request: LlmRerankRequest) => Promise<{ scores: number[] }>;
|
||||
buildPreparedNativeImageExecution: (
|
||||
protocol: LlmProtocol,
|
||||
backendConfig: LlmBackendConfig,
|
||||
model: string,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotImageOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => PreparedNativeImageExecution;
|
||||
};
|
||||
|
||||
export type ProviderRuntimeHostInput = Omit<
|
||||
ProviderRuntimeContextInput,
|
||||
| keyof ProviderRuntimeHostSeed
|
||||
| 'buildPreparedNativeExecution'
|
||||
| 'createPreparedExecutionAdapter'
|
||||
| 'buildPreparedNativeStructuredExecution'
|
||||
| 'buildPreparedNativeEmbeddingExecution'
|
||||
| 'buildPreparedNativeRerankExecution'
|
||||
| 'buildPreparedNativeImageExecution'
|
||||
> &
|
||||
ProviderRuntimeHostSeed & {
|
||||
preparedExecutionRuntimeInput: CreatePreparedExecutionRuntimeInput;
|
||||
createNativeStructuredDispatch: ProviderRuntimeContextInput['createNativeStructuredDispatch'];
|
||||
createNativeEmbeddingDispatch: ProviderRuntimeContextInput['createNativeEmbeddingDispatch'];
|
||||
createNativeRerankDispatch: ProviderRuntimeContextInput['createNativeRerankDispatch'];
|
||||
};
|
||||
|
||||
type ProviderRuntimeHostOverride = {
|
||||
overrideRuntimeHost?: (
|
||||
runtimeHost: ProviderRuntimeContexts
|
||||
) => ProviderRuntimeContexts;
|
||||
};
|
||||
|
||||
const runtimeHosts = new WeakMap<CopilotProvider, ProviderRuntimeContexts>();
|
||||
|
||||
export type ProviderRuntimeContexts = {
|
||||
model: ProviderModelRuntimeContext;
|
||||
chat: ChatRuntimeContext;
|
||||
structured: StructuredRuntimeContext;
|
||||
embedding: EmbeddingRuntimeContext;
|
||||
rerank: RerankRuntimeContext;
|
||||
image: ImageRuntimeContext;
|
||||
prepare: {
|
||||
chat: (
|
||||
kind: 'text' | 'streamText' | 'streamObject',
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof prepareNativeChatExecution>;
|
||||
structured: (
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotStructuredOptions,
|
||||
responseContract?: RequiredStructuredOutputContract,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof prepareNativeStructuredExecution>;
|
||||
embedding: (
|
||||
cond: ModelConditions,
|
||||
input: string | string[],
|
||||
options?: CopilotEmbeddingOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof prepareNativeEmbeddingExecution>;
|
||||
rerank: (
|
||||
cond: ModelConditions,
|
||||
request: CopilotRerankRequest,
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof prepareNativeRerankExecution>;
|
||||
image: (
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotImageOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof prepareNativeImageExecution>;
|
||||
};
|
||||
run: {
|
||||
text: (
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof runNativeText>;
|
||||
streamText: (
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof runNativeStreamText>;
|
||||
streamObject: (
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof runNativeStreamObject>;
|
||||
structured: (
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotStructuredOptions,
|
||||
responseContract?: RequiredStructuredOutputContract,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof runNativeStructured>;
|
||||
embedding: (
|
||||
cond: ModelConditions,
|
||||
input: string | string[],
|
||||
options?: CopilotEmbeddingOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof runNativeEmbedding>;
|
||||
rerank: (
|
||||
cond: ModelConditions,
|
||||
request: CopilotRerankRequest,
|
||||
options?: CopilotChatOptions,
|
||||
execution?: CopilotProviderExecution
|
||||
) => ReturnType<typeof runNativeRerank>;
|
||||
};
|
||||
};
|
||||
|
||||
function createProviderRuntimeContexts(
|
||||
input: ProviderRuntimeContextInput
|
||||
): ProviderRuntimeContexts {
|
||||
const resolveDriver = <K extends keyof ProviderExecutionDrivers>(
|
||||
kind: K
|
||||
): ProviderExecutionDrivers[K] | undefined =>
|
||||
input.resolveExecutionDrivers()?.[kind];
|
||||
const chatDriver = resolveDriver('chat');
|
||||
|
||||
const chatContext: ChatRuntimeContext = {
|
||||
type: input.model.type,
|
||||
resolveChatDriver: () => chatDriver,
|
||||
selectModel: input.selectModel,
|
||||
metricLabels: input.metricLabels,
|
||||
createPreparedExecutionAdapter: input.createPreparedExecutionAdapter,
|
||||
};
|
||||
|
||||
const structuredContext: StructuredRuntimeContext = {
|
||||
type: input.model.type,
|
||||
resolveStructuredDriver: () =>
|
||||
resolveDriver('structured') as StructuredProviderDriver | undefined,
|
||||
checkParams: input.checkParams,
|
||||
selectModel: input.selectModel,
|
||||
getAttachCapability: input.getAttachCapability,
|
||||
getActiveProviderMiddleware: input.getActiveProviderMiddleware,
|
||||
buildPreparedNativeStructuredExecution:
|
||||
input.buildPreparedNativeStructuredExecution,
|
||||
createNativeStructuredDispatch: input.createNativeStructuredDispatch,
|
||||
metricLabels: input.metricLabels,
|
||||
};
|
||||
|
||||
const embeddingContext: EmbeddingRuntimeContext = {
|
||||
type: input.model.type,
|
||||
resolveEmbeddingDriver: () =>
|
||||
resolveDriver('embedding') as EmbeddingProviderDriver | undefined,
|
||||
checkParams: input.checkParams,
|
||||
selectModel: input.selectModel,
|
||||
buildPreparedNativeEmbeddingExecution:
|
||||
input.buildPreparedNativeEmbeddingExecution,
|
||||
createNativeEmbeddingDispatch: input.createNativeEmbeddingDispatch,
|
||||
metricLabels: input.metricLabels,
|
||||
};
|
||||
|
||||
const rerankContext: RerankRuntimeContext = {
|
||||
type: input.model.type,
|
||||
resolveRerankDriver: () =>
|
||||
resolveDriver('rerank') as RerankProviderDriver | undefined,
|
||||
checkParams: input.checkParams,
|
||||
selectModel: input.selectModel,
|
||||
buildPreparedNativeRerankExecution:
|
||||
input.buildPreparedNativeRerankExecution,
|
||||
createNativeRerankDispatch: input.createNativeRerankDispatch,
|
||||
};
|
||||
|
||||
const imageContext: ImageRuntimeContext = {
|
||||
type: input.model.type,
|
||||
resolveImageDriver: () =>
|
||||
resolveDriver('image') as ImageProviderDriver | undefined,
|
||||
checkParams: input.checkParams,
|
||||
selectModel: input.selectModel,
|
||||
buildPreparedNativeImageExecution: input.buildPreparedNativeImageExecution,
|
||||
};
|
||||
|
||||
const prepare: ProviderRuntimeContexts['prepare'] = {
|
||||
chat: (kind, cond, messages, options = {}, execution) =>
|
||||
prepareNativeChatExecution(
|
||||
chatContext.resolveChatDriver,
|
||||
input.buildPreparedNativeExecution,
|
||||
{
|
||||
kind,
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
execution,
|
||||
}
|
||||
),
|
||||
structured: (cond, messages, options = {}, responseContract, execution) =>
|
||||
prepareNativeStructuredExecution(
|
||||
structuredContext,
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
responseContract,
|
||||
execution
|
||||
),
|
||||
embedding: (cond, values, options = {}, execution) =>
|
||||
prepareNativeEmbeddingExecution(
|
||||
embeddingContext,
|
||||
cond,
|
||||
values,
|
||||
options,
|
||||
execution
|
||||
),
|
||||
rerank: (cond, request, options = {}, execution) =>
|
||||
prepareNativeRerankExecution(
|
||||
rerankContext,
|
||||
cond,
|
||||
request,
|
||||
options,
|
||||
execution
|
||||
),
|
||||
image: (cond, messages, options = {}, execution) =>
|
||||
prepareNativeImageExecution(
|
||||
imageContext,
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
execution
|
||||
),
|
||||
};
|
||||
|
||||
return {
|
||||
model: input.model,
|
||||
chat: chatContext,
|
||||
structured: structuredContext,
|
||||
embedding: embeddingContext,
|
||||
rerank: rerankContext,
|
||||
image: imageContext,
|
||||
prepare,
|
||||
run: {
|
||||
text: (cond, messages, options, execution) =>
|
||||
runNativeText(
|
||||
chatContext,
|
||||
prepare.chat,
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
execution
|
||||
),
|
||||
streamText: (cond, messages, options, execution) =>
|
||||
runNativeStreamText(
|
||||
chatContext,
|
||||
prepare.chat,
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
execution
|
||||
),
|
||||
streamObject: (cond, messages, options, execution) =>
|
||||
runNativeStreamObject(
|
||||
chatContext,
|
||||
prepare.chat,
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
execution
|
||||
),
|
||||
structured: (cond, messages, options, responseContract, execution) =>
|
||||
runNativeStructured(
|
||||
structuredContext,
|
||||
cond,
|
||||
messages,
|
||||
options,
|
||||
responseContract,
|
||||
execution
|
||||
),
|
||||
embedding: (cond, values, options, execution) =>
|
||||
runNativeEmbedding(embeddingContext, cond, values, options, execution),
|
||||
rerank: (cond, request, options, execution) =>
|
||||
runNativeRerank(rerankContext, cond, request, options, execution),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createProviderRuntimeHost(
|
||||
input: ProviderRuntimeHostInput
|
||||
): ProviderRuntimeContexts {
|
||||
const preparedExecutionRuntime: PreparedExecutionRuntime =
|
||||
createPreparedExecutionRuntime(input.preparedExecutionRuntimeInput);
|
||||
|
||||
return createProviderRuntimeContexts({
|
||||
...input,
|
||||
buildPreparedNativeExecution:
|
||||
preparedExecutionRuntime.buildPreparedNativeExecution,
|
||||
createPreparedExecutionAdapter:
|
||||
preparedExecutionRuntime.createPreparedExecutionAdapter,
|
||||
buildPreparedNativeStructuredExecution:
|
||||
preparedExecutionRuntime.buildPreparedNativeStructuredExecution,
|
||||
buildPreparedNativeEmbeddingExecution:
|
||||
preparedExecutionRuntime.buildPreparedNativeEmbeddingExecution,
|
||||
buildPreparedNativeRerankExecution:
|
||||
preparedExecutionRuntime.buildPreparedNativeRerankExecution,
|
||||
buildPreparedNativeImageExecution:
|
||||
preparedExecutionRuntime.buildPreparedNativeImageExecution,
|
||||
createNativeStructuredDispatch: input.createNativeStructuredDispatch,
|
||||
createNativeEmbeddingDispatch: input.createNativeEmbeddingDispatch,
|
||||
createNativeRerankDispatch: input.createNativeRerankDispatch,
|
||||
});
|
||||
}
|
||||
|
||||
export function getProviderRuntimeHost(
|
||||
provider: CopilotProvider
|
||||
): ProviderRuntimeContexts {
|
||||
const existingRuntimeHost = runtimeHosts.get(provider);
|
||||
if (existingRuntimeHost) {
|
||||
return existingRuntimeHost;
|
||||
}
|
||||
const runtimeHostSeed = provider.getRuntimeHostSeed();
|
||||
const runtimeHost = createProviderRuntimeHost({
|
||||
...runtimeHostSeed,
|
||||
preparedExecutionRuntimeInput: {
|
||||
resolveProviderId: execution =>
|
||||
execution?.providerId ?? `${provider.type}-default`,
|
||||
getTools: runtimeHostSeed.getTools,
|
||||
getActiveProviderMiddleware: runtimeHostSeed.getActiveProviderMiddleware,
|
||||
createNativeAdapter: provider.createNativeAdapter.bind(provider),
|
||||
maxSteps: provider.maxSteps,
|
||||
},
|
||||
createNativeStructuredDispatch: (backendConfig, protocol, _execution) =>
|
||||
inputCreateNativeStructuredDispatch(backendConfig, protocol),
|
||||
createNativeEmbeddingDispatch: (backendConfig, protocol, _execution) =>
|
||||
inputCreateNativeEmbeddingDispatch(backendConfig, protocol),
|
||||
createNativeRerankDispatch: (backendConfig, protocol, _execution) =>
|
||||
inputCreateNativeRerankDispatch(backendConfig, protocol),
|
||||
});
|
||||
const resolvedRuntimeHost =
|
||||
(provider as ProviderRuntimeHostOverride).overrideRuntimeHost?.(
|
||||
runtimeHost
|
||||
) ?? runtimeHost;
|
||||
|
||||
runtimeHosts.set(provider, resolvedRuntimeHost);
|
||||
return resolvedRuntimeHost;
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { QuotaStateService } from '../../../core/quota/state';
|
||||
import { PromptService } from '../prompt/service';
|
||||
|
||||
export const DEFAULT_EMBEDDING_MODEL = 'gemini-embedding-001';
|
||||
export const DEFAULT_RERANK_MODEL = 'gpt-4o-mini';
|
||||
|
||||
@Injectable()
|
||||
export class TaskPolicy {
|
||||
constructor(
|
||||
private readonly quotaState: QuotaStateService,
|
||||
private readonly prompts: PromptService
|
||||
) {}
|
||||
|
||||
resolveEmbeddingModelId() {
|
||||
return DEFAULT_EMBEDDING_MODEL;
|
||||
}
|
||||
|
||||
resolveRerankModelId() {
|
||||
return DEFAULT_RERANK_MODEL;
|
||||
}
|
||||
|
||||
async resolveTranscriptionModel(userId: string) {
|
||||
const prompt = await this.prompts.get('Transcript audio');
|
||||
if (!prompt) return;
|
||||
|
||||
const state = await this.quotaState.reconcileUserQuotaState(userId);
|
||||
const flags = state.flags as { unlimitedCopilot?: boolean };
|
||||
const hasAccess =
|
||||
!!flags.unlimitedCopilot ||
|
||||
['pro', 'lifetime_pro', 'ai'].includes(state.plan);
|
||||
return prompt.optionalModels[hasAccess ? 1 : 0] ?? prompt.model;
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
/* oxlint-disable import/no-cycle -- Tools can invoke nested prompts and semantic search. */
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Config } from '../../../base';
|
||||
@@ -5,7 +6,6 @@ import { DocReader, DocWriter } from '../../../core/doc';
|
||||
import { PermissionAccess } from '../../../core/permission';
|
||||
import { Models } from '../../../models';
|
||||
import { IndexerService } from '../../indexer';
|
||||
import type { NodeTextMiddleware } from '../config';
|
||||
import { CopilotContextService } from '../context/service';
|
||||
import {
|
||||
type CopilotChatOptions,
|
||||
@@ -36,8 +36,6 @@ import {
|
||||
createSectionEditTool,
|
||||
} from '../tools';
|
||||
import { PromptRuntime } from './prompt-runtime';
|
||||
import type { ToolLoopBackend } from './tool/bridge';
|
||||
import { createNativeToolLoopAdapter } from './tool/native-adapter';
|
||||
|
||||
export type ProviderSpecificToolResolver = (
|
||||
toolName: CopilotChatTools,
|
||||
@@ -192,15 +190,4 @@ export class ToolRuntime {
|
||||
|
||||
return tools;
|
||||
}
|
||||
|
||||
createNativeAdapter(
|
||||
backend: ToolLoopBackend,
|
||||
tools: CopilotToolSet,
|
||||
options: {
|
||||
maxSteps?: number;
|
||||
nodeTextMiddleware?: NodeTextMiddleware[];
|
||||
} = {}
|
||||
) {
|
||||
return createNativeToolLoopAdapter(backend, tools, options);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +1,8 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
import {
|
||||
type LlmBackendConfig,
|
||||
llmDispatchToolLoopStream,
|
||||
llmDispatchToolLoopStreamPrepared,
|
||||
llmDispatchToolLoopStreamRouted,
|
||||
type LlmPreparedDispatchRoute,
|
||||
type LlmProtocol,
|
||||
type LlmRequest,
|
||||
type LlmRoutedBackend,
|
||||
type LlmToolCallbackRequest,
|
||||
type LlmToolCallbackResponse,
|
||||
type LlmToolLoopStreamEvent,
|
||||
} from '../../../../native';
|
||||
import type {
|
||||
CopilotTool,
|
||||
@@ -19,71 +10,11 @@ import type {
|
||||
CopilotToolSet,
|
||||
} from '../../tools';
|
||||
|
||||
export type ToolLoopDispatch = (
|
||||
request: LlmRequest,
|
||||
signalOrOptions?: AbortSignal | CopilotToolExecuteOptions,
|
||||
maybeMessages?: CopilotToolExecuteOptions['messages']
|
||||
) => AsyncIterableIterator<LlmToolLoopStreamEvent>;
|
||||
|
||||
export type ToolLoopBackend =
|
||||
| { protocol: LlmProtocol; backendConfig: LlmBackendConfig }
|
||||
| { routes: LlmRoutedBackend[] }
|
||||
| { preparedRoutes: LlmPreparedDispatchRoute[] };
|
||||
|
||||
function normalizeToolExecuteOptions(
|
||||
signalOrOptions?: AbortSignal | CopilotToolExecuteOptions,
|
||||
maybeMessages?: CopilotToolExecuteOptions['messages']
|
||||
): CopilotToolExecuteOptions {
|
||||
if (
|
||||
signalOrOptions &&
|
||||
typeof signalOrOptions === 'object' &&
|
||||
'aborted' in signalOrOptions
|
||||
) {
|
||||
return {
|
||||
signal: signalOrOptions,
|
||||
messages: maybeMessages,
|
||||
};
|
||||
}
|
||||
|
||||
if (!signalOrOptions) {
|
||||
return maybeMessages ? { messages: maybeMessages } : {};
|
||||
}
|
||||
|
||||
return {
|
||||
...signalOrOptions,
|
||||
signal: signalOrOptions.signal,
|
||||
messages: signalOrOptions.messages ?? maybeMessages,
|
||||
};
|
||||
}
|
||||
|
||||
export function createToolExecutionCallback(
|
||||
tools: CopilotToolSet,
|
||||
options: CopilotToolExecuteOptions = {}
|
||||
) {
|
||||
return async (request: LlmToolCallbackRequest) => {
|
||||
return await executeToolCall(tools, request, options);
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeToolCall(
|
||||
tools: CopilotToolSet,
|
||||
request: LlmToolCallbackRequest,
|
||||
options: CopilotToolExecuteOptions
|
||||
): Promise<LlmToolCallbackResponse> {
|
||||
const tool = tools[request.name] as CopilotTool | undefined;
|
||||
|
||||
if (!tool?.execute) {
|
||||
return {
|
||||
callId: request.callId,
|
||||
name: request.name,
|
||||
args: request.args,
|
||||
rawArgumentsText: request.rawArgumentsText,
|
||||
argumentParseError: request.argumentParseError,
|
||||
isError: true,
|
||||
output: { message: `Tool not found: ${request.name}` },
|
||||
};
|
||||
}
|
||||
|
||||
if (request.argumentParseError) {
|
||||
return {
|
||||
callId: request.callId,
|
||||
@@ -104,6 +35,19 @@ export async function executeToolCall(
|
||||
};
|
||||
}
|
||||
|
||||
const tool = tools[request.name] as CopilotTool | undefined;
|
||||
if (!tool?.execute) {
|
||||
return {
|
||||
callId: request.callId,
|
||||
name: request.name,
|
||||
args: request.args,
|
||||
rawArgumentsText: request.rawArgumentsText,
|
||||
argumentParseError: request.argumentParseError,
|
||||
isError: true,
|
||||
output: { message: `Tool not found: ${request.name}` },
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const args =
|
||||
tool.inputSchema instanceof z.ZodType
|
||||
@@ -133,53 +77,5 @@ export async function executeToolCall(
|
||||
}
|
||||
}
|
||||
|
||||
export function createToolLoopBridge(
|
||||
backend: ToolLoopBackend,
|
||||
tools: CopilotToolSet,
|
||||
maxSteps = 20
|
||||
): ToolLoopDispatch {
|
||||
return (
|
||||
request: LlmRequest,
|
||||
signalOrOptions?: AbortSignal | CopilotToolExecuteOptions,
|
||||
maybeMessages?: CopilotToolExecuteOptions['messages']
|
||||
) => {
|
||||
const toolExecuteOptions = normalizeToolExecuteOptions(
|
||||
signalOrOptions,
|
||||
maybeMessages
|
||||
);
|
||||
const execute = createToolExecutionCallback(tools, toolExecuteOptions);
|
||||
const toolLoopRequest = { ...request, stream: true };
|
||||
|
||||
if ('routes' in backend) {
|
||||
return llmDispatchToolLoopStreamRouted(
|
||||
backend.routes,
|
||||
toolLoopRequest,
|
||||
execute,
|
||||
maxSteps,
|
||||
toolExecuteOptions.signal
|
||||
);
|
||||
}
|
||||
|
||||
if ('preparedRoutes' in backend) {
|
||||
return llmDispatchToolLoopStreamPrepared(
|
||||
backend.preparedRoutes,
|
||||
execute,
|
||||
maxSteps,
|
||||
toolExecuteOptions.signal
|
||||
);
|
||||
}
|
||||
|
||||
return llmDispatchToolLoopStream(
|
||||
backend.protocol,
|
||||
backend.backendConfig,
|
||||
toolLoopRequest,
|
||||
execute,
|
||||
maxSteps,
|
||||
toolExecuteOptions.signal
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
// re-export for test consumers
|
||||
export type { LlmToolCallbackRequest } from '../../../../native';
|
||||
export type { CopilotToolExecuteOptions, CopilotToolSet } from '../../tools';
|
||||
|
||||
@@ -7,9 +7,7 @@ import {
|
||||
CitationFootnoteFormatter,
|
||||
TextStreamParser,
|
||||
} from '../../providers/utils';
|
||||
import type { CopilotToolSet } from '../../tools';
|
||||
import { projectRuntimeEventToStreamObject } from '../contracts/runtime-event-contract';
|
||||
import { createToolLoopBridge, type ToolLoopBackend } from './bridge';
|
||||
import {
|
||||
type EnrichedToolCallEvent,
|
||||
type EnrichedToolResultEvent,
|
||||
@@ -425,14 +423,3 @@ export class NativeProviderAdapter {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createNativeToolLoopAdapter(
|
||||
backend: ToolLoopBackend,
|
||||
tools: CopilotToolSet,
|
||||
options: NativeProviderAdapterOptions = {}
|
||||
) {
|
||||
return new NativeProviderAdapter(
|
||||
createToolLoopBridge(backend, tools, options.maxSteps),
|
||||
options
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3,14 +3,15 @@ import { Injectable } from '@nestjs/common';
|
||||
import { CopilotContextService } from '../context/service';
|
||||
import { type Turn } from '../core';
|
||||
import {
|
||||
type ModelConditions,
|
||||
ModelInputType,
|
||||
type PromptParams,
|
||||
type StreamObject,
|
||||
} from '../providers/types';
|
||||
import { ChatSession } from '../session';
|
||||
import { ChatQuerySchema } from '../types';
|
||||
import { getTools } from '../utils';
|
||||
import { CapabilityRuntime } from './capability-runtime';
|
||||
import { CapabilityPolicyHost } from './hosts/capability-policy-host';
|
||||
import { ConversationHost } from './hosts/conversation-host';
|
||||
import { ImageResultHost } from './hosts/image-result-host';
|
||||
import { TurnPersistence } from './hosts/turn-persistence';
|
||||
@@ -20,7 +21,6 @@ export class TurnOrchestrator {
|
||||
constructor(
|
||||
private readonly conversations: ConversationHost,
|
||||
private readonly context: CopilotContextService,
|
||||
private readonly capabilityPolicy: CapabilityPolicyHost,
|
||||
private readonly runtime: CapabilityRuntime,
|
||||
private readonly imageResults: ImageResultHost,
|
||||
private readonly turnPersistence: TurnPersistence
|
||||
@@ -62,8 +62,15 @@ export class TurnOrchestrator {
|
||||
sessionId,
|
||||
query
|
||||
);
|
||||
const { modelId, reasoning, webSearch, toolsConfig, byokLeaseId } =
|
||||
ChatQuerySchema.parse(query);
|
||||
const {
|
||||
profileId,
|
||||
modelId,
|
||||
routeTargetId,
|
||||
reasoning,
|
||||
webSearch,
|
||||
toolsConfig,
|
||||
byokLeaseId,
|
||||
} = ChatQuerySchema.parse(query);
|
||||
const promptParams = await this.buildPromptParams(sessionId, {
|
||||
latestTurn: prepared.latestTurn,
|
||||
includeContextFiles: selection.includeContextFiles,
|
||||
@@ -76,22 +83,34 @@ export class TurnOrchestrator {
|
||||
return {
|
||||
prepared,
|
||||
finalMessage,
|
||||
selection: await this.capabilityPolicy.selectChat(prepared.session, {
|
||||
responseMode: selection.responseMode,
|
||||
modelId,
|
||||
reasoning,
|
||||
webSearch,
|
||||
toolsConfig,
|
||||
byokLeaseId,
|
||||
billingUnitId: prepared.latestTurn?.id,
|
||||
quotaBackedRoutesAllowed: prepared.quotaBackedRoutesAllowed,
|
||||
featureKind:
|
||||
selection.responseMode === 'image'
|
||||
? 'image'
|
||||
: selection.responseMode === 'object'
|
||||
? 'action'
|
||||
: 'chat',
|
||||
}),
|
||||
selection: {
|
||||
model: profileId && modelId ? modelId : 'route-selected',
|
||||
conditions: { profileId, modelId },
|
||||
providerOptions: {
|
||||
...prepared.session.config.promptConfig,
|
||||
user: prepared.session.config.userId,
|
||||
session: prepared.session.config.sessionId,
|
||||
workspace: prepared.session.config.workspaceId,
|
||||
profileId,
|
||||
byokLeaseId,
|
||||
billingUnitId: prepared.latestTurn?.id,
|
||||
builtInRouteId: prepared.session.config.promptName,
|
||||
managedTargetId: routeTargetId,
|
||||
quotaBackedRoutesAllowed: prepared.quotaBackedRoutesAllowed,
|
||||
featureKind:
|
||||
selection.responseMode === 'image'
|
||||
? 'image'
|
||||
: selection.responseMode === 'object'
|
||||
? 'action'
|
||||
: 'chat',
|
||||
reasoning,
|
||||
webSearch,
|
||||
tools: getTools(
|
||||
prepared.session.config.promptConfig?.tools,
|
||||
toolsConfig
|
||||
),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -110,7 +129,7 @@ export class TurnOrchestrator {
|
||||
|
||||
const stream = this.streamTextResult(
|
||||
prepared.session,
|
||||
selection.model,
|
||||
selection.conditions,
|
||||
finalMessage,
|
||||
{
|
||||
...selection.providerOptions,
|
||||
@@ -129,14 +148,14 @@ export class TurnOrchestrator {
|
||||
|
||||
private async *streamTextResult(
|
||||
session: ChatSession,
|
||||
model: string,
|
||||
conditions: ModelConditions,
|
||||
finalMessage: ReturnType<ChatSession['finish']>,
|
||||
options: Record<string, unknown>,
|
||||
wasAborted: () => boolean
|
||||
) {
|
||||
let buffer = '';
|
||||
for await (const chunk of this.runtime.streamText(
|
||||
{ modelId: model },
|
||||
conditions,
|
||||
finalMessage,
|
||||
options
|
||||
)) {
|
||||
@@ -165,7 +184,7 @@ export class TurnOrchestrator {
|
||||
finalMessage,
|
||||
stream: this.streamObjectResult(
|
||||
prepared.session,
|
||||
selection.model,
|
||||
selection.conditions,
|
||||
finalMessage,
|
||||
{
|
||||
...selection.providerOptions,
|
||||
@@ -178,14 +197,14 @@ export class TurnOrchestrator {
|
||||
|
||||
private async *streamObjectResult(
|
||||
session: ChatSession,
|
||||
model: string,
|
||||
conditions: ModelConditions,
|
||||
finalMessage: ReturnType<ChatSession['finish']>,
|
||||
options: Record<string, unknown>,
|
||||
wasAborted: () => boolean
|
||||
): AsyncIterableIterator<StreamObject> {
|
||||
const chunks: StreamObject[] = [];
|
||||
for await (const chunk of this.runtime.streamObject(
|
||||
{ modelId: model },
|
||||
conditions,
|
||||
finalMessage,
|
||||
options
|
||||
)) {
|
||||
@@ -223,7 +242,7 @@ export class TurnOrchestrator {
|
||||
userId,
|
||||
sessionId,
|
||||
prepared.session,
|
||||
undefined,
|
||||
selection.conditions,
|
||||
hasAttachment,
|
||||
finalMessage,
|
||||
{
|
||||
@@ -244,7 +263,7 @@ export class TurnOrchestrator {
|
||||
userId: string,
|
||||
sessionId: string,
|
||||
session: ChatSession,
|
||||
model: string | undefined,
|
||||
conditions: ModelConditions,
|
||||
hasAttachment: boolean,
|
||||
finalMessage: ReturnType<ChatSession['finish']>,
|
||||
options: Record<string, unknown>,
|
||||
@@ -253,7 +272,7 @@ export class TurnOrchestrator {
|
||||
const attachments: string[] = [];
|
||||
for await (const artifact of this.runtime.streamImageArtifacts(
|
||||
{
|
||||
modelId: model,
|
||||
...conditions,
|
||||
inputTypes: hasAttachment
|
||||
? [ModelInputType.Image]
|
||||
: [ModelInputType.Text],
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
|
||||
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Transactional } from '@nestjs-cls/transactional';
|
||||
import { AiPromptRole } from '@prisma/client';
|
||||
import { AiSessionMessageRole } from '@prisma/client';
|
||||
|
||||
import {
|
||||
CopilotActionTaken,
|
||||
@@ -20,7 +20,6 @@ import {
|
||||
type UpdateChatSession,
|
||||
UpdateChatSessionOptions,
|
||||
} from '../../models';
|
||||
import { CopilotAccessPolicy } from './access';
|
||||
import { ConversationPolicy } from './conversation/policy';
|
||||
import { ConversationStore } from './conversation/store';
|
||||
import { type Conversation, promptMessageFromTurn, type Turn } from './core';
|
||||
@@ -50,7 +49,6 @@ export class ChatSession implements AsyncDisposable {
|
||||
prompt: ResolvedPrompt,
|
||||
turns: PromptMessage[],
|
||||
params: PromptParams,
|
||||
maxTokenSize: number,
|
||||
sessionId?: string
|
||||
) => PromptMessage[];
|
||||
constructor(
|
||||
@@ -59,23 +57,13 @@ export class ChatSession implements AsyncDisposable {
|
||||
prompt: ResolvedPrompt,
|
||||
turns: PromptMessage[],
|
||||
params: PromptParams,
|
||||
maxTokenSize: number,
|
||||
sessionId?: string
|
||||
) => PromptMessage[],
|
||||
private readonly dispose?: (state: ChatSessionState) => Promise<void>,
|
||||
private readonly maxTokenSize = state.prompt.config?.maxTokens || 128 * 1024
|
||||
private readonly dispose?: (state: ChatSessionState) => Promise<void>
|
||||
) {
|
||||
this.renderPromptSession = renderPromptSession;
|
||||
}
|
||||
|
||||
get model() {
|
||||
return this.state.prompt.model;
|
||||
}
|
||||
|
||||
get optionalModels() {
|
||||
return this.state.prompt.optionalModels;
|
||||
}
|
||||
|
||||
get config() {
|
||||
const {
|
||||
sessionId,
|
||||
@@ -126,7 +114,7 @@ export class ChatSession implements AsyncDisposable {
|
||||
revertLatestMessage(removeLatestUserMessage: boolean) {
|
||||
const turns = this.state.turns;
|
||||
turns.splice(
|
||||
turns.findLastIndex(({ role }) => role === AiPromptRole.user) +
|
||||
turns.findLastIndex(({ role }) => role === AiSessionMessageRole.user) +
|
||||
(removeLatestUserMessage ? 0 : 1)
|
||||
);
|
||||
}
|
||||
@@ -136,7 +124,6 @@ export class ChatSession implements AsyncDisposable {
|
||||
this.state.prompt,
|
||||
this.state.turns.map(turn => promptMessageFromTurn(turn)),
|
||||
params,
|
||||
this.maxTokenSize,
|
||||
this.state.sessionId
|
||||
);
|
||||
}
|
||||
@@ -158,13 +145,11 @@ export type ConversationState = {
|
||||
conversation: Conversation;
|
||||
turns: Turn[];
|
||||
prompt: ResolvedPrompt;
|
||||
tokenCost: number;
|
||||
};
|
||||
|
||||
export type ConversationMetaState = {
|
||||
conversation: Conversation;
|
||||
prompt: ResolvedPrompt;
|
||||
tokenCost: number;
|
||||
};
|
||||
|
||||
type StoredConversation = NonNullable<
|
||||
@@ -183,7 +168,6 @@ export class ChatSessionService {
|
||||
private readonly models: Models,
|
||||
private readonly jobs: JobQueue,
|
||||
private readonly store: ConversationStore,
|
||||
private readonly access: CopilotAccessPolicy,
|
||||
private readonly conversationPolicy: ConversationPolicy,
|
||||
private readonly prompts: PromptService,
|
||||
private readonly promptRuntime: PromptRuntime
|
||||
@@ -206,14 +190,13 @@ export class ChatSessionService {
|
||||
private async toConversationState(
|
||||
session: StoredConversation
|
||||
): Promise<ConversationState> {
|
||||
const { conversation, prompt, tokenCost } =
|
||||
const { conversation, prompt } =
|
||||
await this.toConversationMetaState(session);
|
||||
|
||||
return {
|
||||
conversation,
|
||||
turns: session.turns,
|
||||
prompt,
|
||||
tokenCost,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -226,7 +209,6 @@ export class ChatSessionService {
|
||||
return {
|
||||
conversation: session.conversation,
|
||||
prompt,
|
||||
tokenCost: session.tokenCost,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -296,11 +278,11 @@ export class ChatSessionService {
|
||||
}
|
||||
|
||||
async getQuota(userId: string) {
|
||||
return await this.access.getQuota(userId);
|
||||
return await this.conversationPolicy.getQuota(userId);
|
||||
}
|
||||
|
||||
async checkQuota(userId: string) {
|
||||
await this.access.checkQuota(userId);
|
||||
await this.conversationPolicy.checkQuota(userId);
|
||||
}
|
||||
|
||||
async create(options: ChatSessionOptions): Promise<string> {
|
||||
@@ -360,7 +342,6 @@ export class ChatSessionService {
|
||||
);
|
||||
finalData.promptName = prompt.name;
|
||||
finalData.promptAction = prompt.action ?? null;
|
||||
finalData.promptModel = prompt.model;
|
||||
}
|
||||
finalData.pinned = options.pinned;
|
||||
finalData.docId = options.docId;
|
||||
@@ -389,7 +370,8 @@ export class ChatSessionService {
|
||||
if (options.latestMessageId) {
|
||||
const lastMessageIdx = state.turns.findLastIndex(
|
||||
({ id, role }) =>
|
||||
role === AiPromptRole.assistant && id === options.latestMessageId
|
||||
role === AiSessionMessageRole.assistant &&
|
||||
id === options.latestMessageId
|
||||
);
|
||||
if (lastMessageIdx < 0) {
|
||||
throw new CopilotMessageNotFound({
|
||||
@@ -410,7 +392,6 @@ export class ChatSessionService {
|
||||
prompt: {
|
||||
name: state.prompt.name,
|
||||
action: state.prompt.action,
|
||||
model: state.prompt.model,
|
||||
},
|
||||
turns,
|
||||
});
|
||||
@@ -434,7 +415,6 @@ export class ChatSessionService {
|
||||
async appendTurn(input: {
|
||||
sessionId: string;
|
||||
userId: string;
|
||||
prompt: { model: string };
|
||||
turn: Turn;
|
||||
compatSubmissionId?: string;
|
||||
}) {
|
||||
@@ -485,14 +465,8 @@ export class ChatSessionService {
|
||||
turns: state.turns,
|
||||
prompt: state.prompt,
|
||||
},
|
||||
(prompt, turns, params, maxTokenSize, sessionId) =>
|
||||
this.prompts.renderSession(
|
||||
prompt,
|
||||
turns,
|
||||
params,
|
||||
maxTokenSize,
|
||||
sessionId
|
||||
),
|
||||
(prompt, turns, params, sessionId) =>
|
||||
this.prompts.renderSession(prompt, turns, params, sessionId),
|
||||
async state => {
|
||||
await this.store.appendTurns(state);
|
||||
if (this.conversationPolicy.shouldScheduleTitle(state.prompt)) {
|
||||
@@ -538,9 +512,18 @@ export class ChatSessionService {
|
||||
const promptContent =
|
||||
this.conversationPolicy.buildTitlePromptContent(turns);
|
||||
const generatedTitle = this.stripNullBytes(
|
||||
await this.promptRuntime.runText('Summary as title', {
|
||||
content: promptContent,
|
||||
})
|
||||
await this.promptRuntime.runText(
|
||||
'Summary as title',
|
||||
{ content: promptContent },
|
||||
{
|
||||
providerOptions: {
|
||||
user: conversation.userId,
|
||||
workspace: conversation.workspaceId,
|
||||
featureKind: 'chat',
|
||||
quotaBackedRoutesAllowed: true,
|
||||
},
|
||||
}
|
||||
)
|
||||
).trim();
|
||||
|
||||
if (!generatedTitle) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
/* oxlint-disable import/no-cycle -- Semantic search uses the shared embedding runtime. */
|
||||
import { omit } from 'lodash-es';
|
||||
import { z } from 'zod';
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
/* oxlint-disable import/no-cycle -- Tool exports include semantic search runtime dependencies. */
|
||||
export * from './blob-read';
|
||||
export * from './code-artifact';
|
||||
export * from './conversation-summary';
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { CopilotTranscriptionJobNotFound } from '../../../base';
|
||||
import { Config } from '../../../base/config';
|
||||
import { CopilotTranscriptionJobNotFound } from '../../../base/error/errors.gen';
|
||||
import { PermissionAccess } from '../../../core/permission';
|
||||
import {
|
||||
RealtimeRegistry,
|
||||
realtimeTranscriptTaskRoom,
|
||||
registerRealtimeLiveQuery,
|
||||
} from '../../../core/realtime';
|
||||
import { assertCopilotEnabled } from '../availability';
|
||||
import { CopilotTranscriptionReader } from './reader';
|
||||
|
||||
@Injectable()
|
||||
@@ -15,7 +17,8 @@ export class CopilotTranscriptRealtimeProvider implements OnModuleInit {
|
||||
constructor(
|
||||
private readonly ac: PermissionAccess,
|
||||
private readonly transcript: CopilotTranscriptionReader,
|
||||
private readonly registry: RealtimeRegistry
|
||||
private readonly registry: RealtimeRegistry,
|
||||
private readonly config: Config
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
@@ -68,6 +71,7 @@ export class CopilotTranscriptRealtimeProvider implements OnModuleInit {
|
||||
}
|
||||
|
||||
private async assertCopilot(userId: string, workspaceId: string) {
|
||||
assertCopilotEnabled(this.config);
|
||||
await this.ac
|
||||
.user(userId)
|
||||
.workspace(workspaceId)
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
} from '../../../base';
|
||||
import { CurrentUser } from '../../../core/auth';
|
||||
import { PermissionAccess } from '../../../core/permission';
|
||||
import { CopilotEnabled } from '../feature';
|
||||
import { CopilotType } from '../resolver';
|
||||
import type { TranscriptionJob } from './job';
|
||||
import { buildLegacyProjection } from './projection';
|
||||
@@ -36,7 +37,6 @@ import type {
|
||||
TranscriptionQuality,
|
||||
TranscriptionSourceAudio,
|
||||
TranscriptionSubmitInput,
|
||||
TranscriptProviderMeta,
|
||||
} from './types';
|
||||
|
||||
registerEnumType(AiJobStatus, {
|
||||
@@ -166,15 +166,6 @@ class TranscriptionQualityType implements TranscriptionQuality {
|
||||
overflowCount!: number | null;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
class TranscriptProviderMetaType implements TranscriptProviderMeta {
|
||||
@Field(() => String, { nullable: true })
|
||||
provider!: string | null;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
model!: string | null;
|
||||
}
|
||||
|
||||
@InputType()
|
||||
class AudioSliceManifestItemInput implements AudioSliceManifestItem {
|
||||
@Field(() => Int)
|
||||
@@ -233,9 +224,6 @@ class SubmitAudioTranscriptionInput implements TranscriptionSubmitInput {
|
||||
|
||||
@Field(() => [AudioSliceManifestItemInput], { nullable: true })
|
||||
sliceManifest?: AudioSliceManifestItemInput[];
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
strategy?: string | null;
|
||||
}
|
||||
|
||||
@ObjectType()
|
||||
@@ -273,15 +261,9 @@ class TranscriptionResultType {
|
||||
@Field(() => MeetingSummaryV2Type, { nullable: true })
|
||||
summaryJson!: TranscriptionPayload['summaryJson'] | null;
|
||||
|
||||
@Field(() => TranscriptProviderMetaType, { nullable: true })
|
||||
providerMeta!: TranscriptionPayload['providerMeta'] | null;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
version!: string | null;
|
||||
|
||||
@Field(() => String, { nullable: true })
|
||||
strategy!: string | null;
|
||||
|
||||
@Field(() => AiJobStatus)
|
||||
status!: AiJobStatus;
|
||||
}
|
||||
@@ -292,6 +274,7 @@ const FinishedStatus: Set<AiJobStatus> = new Set([
|
||||
]);
|
||||
|
||||
@Injectable()
|
||||
@CopilotEnabled()
|
||||
@Resolver(() => CopilotType)
|
||||
export class CopilotTranscriptionResolver {
|
||||
constructor(
|
||||
@@ -318,9 +301,7 @@ export class CopilotTranscriptionResolver {
|
||||
normalizedSegments: null,
|
||||
normalizedTranscript: null,
|
||||
summaryJson: null,
|
||||
providerMeta: null,
|
||||
version: null,
|
||||
strategy: null,
|
||||
};
|
||||
if (FinishedStatus.has(finalJob.status)) {
|
||||
finalJob.title = legacy?.title ?? null;
|
||||
@@ -333,9 +314,7 @@ export class CopilotTranscriptionResolver {
|
||||
finalJob.normalizedSegments = ret?.normalizedSegments ?? null;
|
||||
finalJob.normalizedTranscript = ret?.normalizedTranscript ?? null;
|
||||
finalJob.summaryJson = ret?.summaryJson ?? null;
|
||||
finalJob.providerMeta = ret?.providerMeta ?? null;
|
||||
finalJob.version = ret?.version ?? null;
|
||||
finalJob.strategy = ret?.strategy ?? null;
|
||||
}
|
||||
return finalJob;
|
||||
}
|
||||
|
||||
@@ -73,11 +73,6 @@ export const TranscriptionQualitySchema = z.object({
|
||||
overflowCount: z.number().nullable().optional(),
|
||||
});
|
||||
|
||||
export const TranscriptProviderMetaSchema = z.object({
|
||||
provider: z.string().nullable().optional(),
|
||||
model: z.string().nullable().optional(),
|
||||
});
|
||||
|
||||
export const TranscriptionLegacyProjectionSchema = z.object({
|
||||
title: z.string().nullable().optional(),
|
||||
summary: z.string().nullable().optional(),
|
||||
@@ -96,9 +91,7 @@ export const TranscriptionPayloadV2Schema = z.object({
|
||||
.optional(),
|
||||
normalizedTranscript: z.string().nullable().optional(),
|
||||
summaryJson: MeetingSummaryV2Schema.nullable().optional(),
|
||||
providerMeta: TranscriptProviderMetaSchema.nullable().optional(),
|
||||
version: z.string().optional(),
|
||||
strategy: z.string().optional(),
|
||||
});
|
||||
|
||||
export const TranscriptionSubmitInputSchema = TranscriptionPayloadV2Schema.pick(
|
||||
@@ -135,9 +128,7 @@ const CanonicalTranscriptPayloadSchema = TranscriptionPayloadV2Schema.refine(
|
||||
payload.normalizedSegments !== undefined ||
|
||||
payload.normalizedTranscript !== undefined ||
|
||||
payload.summaryJson !== undefined ||
|
||||
payload.providerMeta !== undefined ||
|
||||
payload.version !== undefined ||
|
||||
payload.strategy !== undefined,
|
||||
payload.version !== undefined,
|
||||
{
|
||||
message:
|
||||
'canonical transcript payload must contain canonical transcript fields',
|
||||
|
||||
@@ -14,11 +14,9 @@ import {
|
||||
realtimeTranscriptTaskRoom,
|
||||
} from '../../../core/realtime';
|
||||
import { Models } from '../../../models';
|
||||
import { CopilotAccessPolicy } from '../access';
|
||||
import { PromptService } from '../prompt';
|
||||
import { CopilotProviderType } from '../providers/types';
|
||||
import { ActionRuntimeBridge } from '../runtime/action-runtime-bridge';
|
||||
import { TaskPolicy } from '../runtime/task-policy';
|
||||
import { CapabilityRuntime } from '../runtime/capability-runtime';
|
||||
import { CopilotStorage } from '../storage';
|
||||
import { taskToJob, type TranscriptionJob } from './job';
|
||||
import {
|
||||
@@ -32,9 +30,9 @@ import type {
|
||||
} from './types';
|
||||
import { readStream } from './utils';
|
||||
|
||||
const TRANSCRIPT_ACTION_ID = 'transcript.audio.gemini';
|
||||
const TRANSCRIPT_ACTION_ID = 'transcript.audio';
|
||||
const TRANSCRIPT_PROMPT_REF = 'Transcript audio structured';
|
||||
const TRANSCRIPT_ACTION_VERSION = 'v1';
|
||||
const TRANSCRIPT_STRATEGY = 'gemini';
|
||||
|
||||
@Injectable()
|
||||
export class CopilotTranscriptionService {
|
||||
@@ -42,10 +40,9 @@ export class CopilotTranscriptionService {
|
||||
private readonly models: Models,
|
||||
private readonly job: JobQueue,
|
||||
private readonly storage: CopilotStorage,
|
||||
private readonly tasks: TaskPolicy,
|
||||
private readonly prompts: PromptService,
|
||||
private readonly actionBridge: ActionRuntimeBridge,
|
||||
private readonly access: CopilotAccessPolicy,
|
||||
private readonly runtime: CapabilityRuntime,
|
||||
private readonly realtime: RealtimePublisher
|
||||
) {}
|
||||
|
||||
@@ -58,27 +55,10 @@ export class CopilotTranscriptionService {
|
||||
sourceAudio: payload.sourceAudio,
|
||||
quality: payload.quality,
|
||||
sliceManifest: payload.sliceManifest,
|
||||
providerMeta: payload.providerMeta,
|
||||
version: 'transcript-result-v1',
|
||||
strategy: TRANSCRIPT_STRATEGY,
|
||||
};
|
||||
}
|
||||
|
||||
private async resolveTranscriptStrategy(userId: string, strategy?: string) {
|
||||
if (strategy && strategy !== TRANSCRIPT_STRATEGY) {
|
||||
throw new BadRequestException(
|
||||
`Transcript strategy ${strategy} is not available`
|
||||
);
|
||||
}
|
||||
const model = await this.tasks.resolveTranscriptionModel(userId);
|
||||
if (!model) {
|
||||
throw new BadRequestException(
|
||||
'Transcript strategy gemini is not available'
|
||||
);
|
||||
}
|
||||
return { model, strategy: TRANSCRIPT_STRATEGY };
|
||||
}
|
||||
|
||||
private async persistUploads(
|
||||
userId: string,
|
||||
workspaceId: string,
|
||||
@@ -123,11 +103,8 @@ export class CopilotTranscriptionService {
|
||||
} satisfies TranscriptionPayloadV2;
|
||||
}
|
||||
|
||||
private async buildTranscriptActionMessages(
|
||||
payload: TranscriptionPayloadV2,
|
||||
modelId?: string
|
||||
) {
|
||||
const prompt = await this.prompts.get('Transcript audio structured');
|
||||
private async buildTranscriptActionMessages(payload: TranscriptionPayloadV2) {
|
||||
const prompt = await this.prompts.get(TRANSCRIPT_PROMPT_REF);
|
||||
if (!prompt) {
|
||||
throw new Error('Transcript action prompt not found');
|
||||
}
|
||||
@@ -140,10 +117,6 @@ export class CopilotTranscriptionService {
|
||||
mimeType: info.mimeType,
|
||||
index: info.index ?? null,
|
||||
})) ?? null,
|
||||
providerMeta: {
|
||||
provider: CopilotProviderType.Gemini,
|
||||
model: modelId ?? payload.providerMeta?.model ?? null,
|
||||
},
|
||||
};
|
||||
const attachments =
|
||||
payload.infos?.map(info => ({
|
||||
@@ -165,7 +138,7 @@ export class CopilotTranscriptionService {
|
||||
workspaceId: string,
|
||||
blobId: string,
|
||||
blobs: FileUpload[],
|
||||
input?: TranscriptionSubmitInput & { strategy?: string | null }
|
||||
input?: TranscriptionSubmitInput
|
||||
): Promise<TranscriptionJob> {
|
||||
const existingTask = await this.models.copilotTranscriptTask.getWithUser(
|
||||
userId,
|
||||
@@ -180,15 +153,15 @@ export class CopilotTranscriptionService {
|
||||
throw new CopilotTranscriptionJobExists();
|
||||
}
|
||||
|
||||
await this.access.assertQuotaOrByok({
|
||||
userId,
|
||||
workspaceId,
|
||||
featureKind: 'transcript',
|
||||
});
|
||||
|
||||
const { model, strategy } = await this.resolveTranscriptStrategy(
|
||||
userId,
|
||||
input?.strategy ?? undefined
|
||||
await this.runtime.assertRoute(
|
||||
'transcript.audio',
|
||||
{},
|
||||
{
|
||||
user: userId,
|
||||
workspace: workspaceId,
|
||||
featureKind: 'transcript',
|
||||
builtInRouteId: TRANSCRIPT_PROMPT_REF,
|
||||
}
|
||||
);
|
||||
const infos = await this.persistUploads(userId, workspaceId, blobId, blobs);
|
||||
const payload = this.createCanonicalPayload(blobId, infos, input);
|
||||
@@ -196,7 +169,6 @@ export class CopilotTranscriptionService {
|
||||
userId,
|
||||
workspaceId,
|
||||
blobId,
|
||||
strategy,
|
||||
recipeId: TRANSCRIPT_ACTION_ID,
|
||||
recipeVersion: TRANSCRIPT_ACTION_VERSION,
|
||||
inputSnapshot: payload,
|
||||
@@ -206,7 +178,6 @@ export class CopilotTranscriptionService {
|
||||
await this.job.add('copilot.transcript.task.submit', {
|
||||
taskId: task.id,
|
||||
payload,
|
||||
modelId: model,
|
||||
});
|
||||
await this.models.copilotTranscriptTask.markRunning(task.id);
|
||||
this.publishTaskChanged(workspaceId, task.id, AiJobStatus.running);
|
||||
@@ -234,21 +205,20 @@ export class CopilotTranscriptionService {
|
||||
);
|
||||
}
|
||||
|
||||
await this.access.assertQuotaOrByok({
|
||||
userId,
|
||||
workspaceId,
|
||||
featureKind: 'transcript',
|
||||
});
|
||||
|
||||
const payload = this.parseTaskPayload(task.protectedResult);
|
||||
const { model } = await this.resolveTranscriptStrategy(
|
||||
userId,
|
||||
task.strategy
|
||||
await this.runtime.assertRoute(
|
||||
'transcript.audio',
|
||||
{},
|
||||
{
|
||||
user: userId,
|
||||
workspace: workspaceId,
|
||||
featureKind: 'transcript',
|
||||
builtInRouteId: TRANSCRIPT_PROMPT_REF,
|
||||
}
|
||||
);
|
||||
await this.job.add('copilot.transcript.task.submit', {
|
||||
taskId,
|
||||
payload,
|
||||
modelId: model,
|
||||
retryOf: task.actionRunId ?? undefined,
|
||||
});
|
||||
await this.models.copilotTranscriptTask.markRunning(taskId);
|
||||
@@ -282,12 +252,6 @@ export class CopilotTranscriptionService {
|
||||
return taskToJob(task);
|
||||
}
|
||||
|
||||
await this.access.assertQuotaOrByok({
|
||||
userId,
|
||||
workspaceId,
|
||||
featureKind: 'transcript',
|
||||
});
|
||||
|
||||
const settled = await this.models.copilotTranscriptTask.settle(task.id);
|
||||
return taskToJob(settled);
|
||||
}
|
||||
@@ -311,7 +275,6 @@ export class CopilotTranscriptionService {
|
||||
async transcriptTask({
|
||||
taskId,
|
||||
payload,
|
||||
modelId,
|
||||
retryOf,
|
||||
}: Jobs['copilot.transcript.task.submit']) {
|
||||
const task = await this.models.copilotTranscriptTask.get(taskId);
|
||||
@@ -324,10 +287,7 @@ export class CopilotTranscriptionService {
|
||||
let bridgeFailed = false;
|
||||
let bridgeError = 'transcript native recipe failed';
|
||||
let finalResult: unknown = null;
|
||||
const messages = await this.buildTranscriptActionMessages(
|
||||
payload,
|
||||
modelId
|
||||
);
|
||||
const messages = await this.buildTranscriptActionMessages(payload);
|
||||
for await (const event of this.actionBridge.runStream({
|
||||
userId: task.userId,
|
||||
workspaceId: task.workspaceId,
|
||||
@@ -335,14 +295,6 @@ export class CopilotTranscriptionService {
|
||||
actionVersion: TRANSCRIPT_ACTION_VERSION,
|
||||
retryOf: retryOf ?? null,
|
||||
inputSnapshot: payload,
|
||||
nativeInput: {
|
||||
input: {
|
||||
sourceAudio: payload.sourceAudio ?? null,
|
||||
quality: payload.quality ?? null,
|
||||
infos: payload.infos ?? null,
|
||||
sliceManifest: payload.sliceManifest ?? null,
|
||||
},
|
||||
},
|
||||
onRunCreated: async ({ runId }) => {
|
||||
await this.models.copilotTranscriptTask.markRunning(taskId, runId);
|
||||
this.publishTaskChanged(
|
||||
@@ -351,9 +303,9 @@ export class CopilotTranscriptionService {
|
||||
AiJobStatus.running
|
||||
);
|
||||
},
|
||||
prepareStructuredRoutes: {
|
||||
stepId: 'transcribe',
|
||||
modelId,
|
||||
step: {
|
||||
slot: 'transcript.audio',
|
||||
builtInRouteId: TRANSCRIPT_PROMPT_REF,
|
||||
messages,
|
||||
options: {
|
||||
user: task.userId,
|
||||
@@ -362,7 +314,6 @@ export class CopilotTranscriptionService {
|
||||
billingUnitId: taskId,
|
||||
featureKind: 'transcript',
|
||||
},
|
||||
prefer: CopilotProviderType.Gemini,
|
||||
responseContract: TranscriptActionResultContract,
|
||||
},
|
||||
})) {
|
||||
|
||||
@@ -15,7 +15,6 @@ import {
|
||||
TranscriptionQualitySchema,
|
||||
TranscriptionSourceAudioSchema,
|
||||
TranscriptionSubmitInputSchema,
|
||||
TranscriptProviderMetaSchema,
|
||||
} from './schema';
|
||||
|
||||
export type LegacyTranscriptionSegment = z.infer<
|
||||
@@ -36,9 +35,6 @@ export type TranscriptionSourceAudio = z.infer<
|
||||
typeof TranscriptionSourceAudioSchema
|
||||
>;
|
||||
export type TranscriptionQuality = z.infer<typeof TranscriptionQualitySchema>;
|
||||
export type TranscriptProviderMeta = z.infer<
|
||||
typeof TranscriptProviderMetaSchema
|
||||
>;
|
||||
export type TranscriptionLegacyProjection = z.infer<
|
||||
typeof TranscriptionLegacyProjectionSchema
|
||||
>;
|
||||
@@ -57,7 +53,6 @@ declare global {
|
||||
'copilot.transcript.task.submit': {
|
||||
taskId: string;
|
||||
payload: TranscriptionPayloadV2;
|
||||
modelId?: string;
|
||||
retryOf?: string;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -36,7 +36,9 @@ export type ToolsConfig = z.infer<typeof ToolsConfigSchema>;
|
||||
export const ChatQuerySchema = z
|
||||
.object({
|
||||
messageId: zMaybeString,
|
||||
profileId: zMaybeString,
|
||||
modelId: zMaybeString,
|
||||
routeTargetId: zMaybeString,
|
||||
byokLeaseId: zMaybeString,
|
||||
retry: zBool,
|
||||
reasoning: zBool,
|
||||
@@ -44,10 +46,29 @@ export const ChatQuerySchema = z
|
||||
toolsConfig: ToolsConfigSchema,
|
||||
})
|
||||
.catchall(z.string())
|
||||
.superRefine((value, context) => {
|
||||
if (!!value.profileId !== !!value.modelId) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: 'profileId and modelId must be provided together',
|
||||
});
|
||||
}
|
||||
for (const field of ['requirements', 'deployment', 'profiles', 'presets']) {
|
||||
if (Object.hasOwn(value, field)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
path: [field],
|
||||
message: `${field} is owned by the native route policy`,
|
||||
});
|
||||
}
|
||||
}
|
||||
})
|
||||
.transform(
|
||||
({
|
||||
messageId,
|
||||
profileId,
|
||||
modelId,
|
||||
routeTargetId,
|
||||
byokLeaseId,
|
||||
retry,
|
||||
reasoning,
|
||||
@@ -56,7 +77,9 @@ export const ChatQuerySchema = z
|
||||
...params
|
||||
}) => ({
|
||||
messageId,
|
||||
profileId,
|
||||
modelId,
|
||||
routeTargetId,
|
||||
byokLeaseId,
|
||||
retry,
|
||||
reasoning,
|
||||
@@ -85,11 +108,7 @@ export const ChatHistorySchema = z
|
||||
title: z.string().nullable(),
|
||||
|
||||
action: z.string().nullable(),
|
||||
model: z.string(),
|
||||
optionalModels: z.array(z.string()),
|
||||
promptName: z.string(),
|
||||
|
||||
tokens: z.number(),
|
||||
messages: z.array(ChatMessageSchema),
|
||||
createdAt: z.date(),
|
||||
updatedAt: z.date(),
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
import { CurrentUser } from '../../../core/auth';
|
||||
import { PermissionAccess } from '../../../core/permission';
|
||||
import { WorkspaceType } from '../../../core/workspaces';
|
||||
import { CopilotEnabled } from '../feature';
|
||||
import { COPILOT_LOCKER } from '../resolver';
|
||||
import { MAX_EMBEDDABLE_SIZE } from '../utils';
|
||||
import { CopilotWorkspaceService } from './service';
|
||||
@@ -47,6 +48,7 @@ export class CopilotWorkspaceConfigType {
|
||||
* Public apis rate limit: 10 req/m
|
||||
* Other rate limit: 120 req/m
|
||||
*/
|
||||
@CopilotEnabled()
|
||||
@Resolver(() => WorkspaceType)
|
||||
export class CopilotWorkspaceEmbeddingResolver {
|
||||
constructor(private readonly ac: PermissionAccess) {}
|
||||
@@ -67,6 +69,7 @@ export class CopilotWorkspaceEmbeddingResolver {
|
||||
}
|
||||
}
|
||||
|
||||
@CopilotEnabled()
|
||||
@Resolver(() => CopilotWorkspaceConfigType)
|
||||
export class CopilotWorkspaceEmbeddingConfigResolver {
|
||||
constructor(
|
||||
|
||||
Reference in New Issue
Block a user