feat(server): converge legacy compatibility (#15426)

#### PR Dependency Tree


* **PR #15426** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added workspace BYOK profiles with provider/model catalogs, capability
validation, connection probing, credential rotation, reordering, and
secure local leases.
* Added Copilot route options, selectable targets, managed tiers,
explicit profile/model overrides, and improved streaming with tool
callbacks and abort support.
* Added Copilot availability controls to prevent access when the feature
is disabled.
* **Changes**
* Simplified Copilot configuration and removed legacy provider-specific
settings.
* Removed obsolete model, token-cost, transcript strategy, and provider
metadata fields from public responses.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-08-05 00:40:13 +08:00
committed by GitHub
parent fdfb6df826
commit 965f4590ff
272 changed files with 12430 additions and 33824 deletions
@@ -1,2 +1 @@
export * from './feature-coverage';
export * from './policy';
@@ -1,106 +0,0 @@
import { Injectable } from '@nestjs/common';
import { CopilotQuotaExceeded } from '../../../base';
import { ByokService } from '../byok/service';
import type { ByokFeatureKind } from '../byok/types';
import type { CopilotProviderProfile } from '../config';
import { ConversationPolicy } from '../conversation/policy';
import {
getByokSourceCoverage,
getCopilotFeatureAccess,
} from './feature-coverage';
export type CopilotAccessContext = {
userId?: string;
workspaceId?: string;
byokLeaseId?: string;
featureKind?: ByokFeatureKind;
quotaBackedRoutesAllowed?: boolean;
};
export type CopilotRouteAccess = {
byokProfiles: CopilotProviderProfile[];
quotaBackedRoutesAvailable: boolean;
};
export type CopilotTurnRouteAccess = {
byokProfiles: CopilotProviderProfile[];
quotaBackedRoutesAllowed?: boolean;
};
@Injectable()
export class CopilotAccessPolicy {
constructor(
private readonly conversationPolicy: ConversationPolicy,
private readonly byok: ByokService
) {}
async getByokProfiles(context: CopilotAccessContext = {}) {
const coverage = getByokSourceCoverage(context.featureKind);
return await this.byok.getProfiles(context, coverage);
}
async canUseQuotaBackedRoutes(context: CopilotAccessContext = {}) {
if (context.quotaBackedRoutesAllowed !== undefined) {
return context.quotaBackedRoutesAllowed;
}
if (!getCopilotFeatureAccess(context.featureKind).quotaMetered) {
return true;
}
if (!context.userId) {
return true;
}
return await this.conversationPolicy.hasQuota(context.userId);
}
async getQuota(userId: string) {
return await this.conversationPolicy.getQuota(userId);
}
async checkQuota(userId: string) {
await this.conversationPolicy.checkQuota(userId);
}
async resolveRouteAccess(
context: CopilotAccessContext = {}
): Promise<CopilotRouteAccess> {
const [byokProfiles, quotaBackedRoutesAvailable] = await Promise.all([
this.getByokProfiles(context),
this.canUseQuotaBackedRoutes(context),
]);
return { byokProfiles, quotaBackedRoutesAvailable };
}
async resolveTurnRouteAccess(
context: CopilotAccessContext
): Promise<CopilotTurnRouteAccess> {
const byokProfiles = await this.getByokProfiles(context);
if (context.quotaBackedRoutesAllowed === false) {
return { byokProfiles, quotaBackedRoutesAllowed: false };
}
const featureAccess = getCopilotFeatureAccess(context.featureKind);
if (!byokProfiles.length && context.userId && featureAccess.quotaMetered) {
await this.conversationPolicy.checkQuota(context.userId);
}
const quotaBackedRoutesAllowed = byokProfiles.length
? context.quotaBackedRoutesAllowed
: true;
return { byokProfiles, quotaBackedRoutesAllowed };
}
async assertQuotaOrByok(context: CopilotAccessContext) {
const byokProfiles = await this.getByokProfiles(context);
if (context.quotaBackedRoutesAllowed === false) {
if (!byokProfiles.length) {
throw new CopilotQuotaExceeded();
}
return;
}
const featureAccess = getCopilotFeatureAccess(context.featureKind);
if (!byokProfiles.length && context.userId && featureAccess.quotaMetered) {
await this.conversationPolicy.checkQuota(context.userId);
}
}
}
@@ -0,0 +1,8 @@
import type { Config } from '../../base/config';
import { ActionForbidden } from '../../base/error/errors.gen';
export function assertCopilotEnabled(config: Config) {
if (!config.copilot.enabled) {
throw new ActionForbidden('Copilot is disabled.');
}
}
@@ -1,4 +1,3 @@
export { ByokEntitlementPolicy } from './policy';
export { WorkspaceByokResolver } from './resolver';
export { type ByokProviderRequestContext, ByokService } from './service';
export * from './types';
@@ -103,6 +103,16 @@ export class ByokEntitlementPolicy {
}
}
async assertEntitled(workspaceId: string, userId?: string) {
const [serverEntitled, localEntitled] = await this.hasEntitlement(
workspaceId,
userId
);
if (!serverEntitled && !localEntitled) {
throw new ActionForbidden('BYOK requires Pro, Team, or Believer.');
}
}
private async hasWorkspaceTeamPlan(workspaceId: string) {
try {
const state =
@@ -1,94 +0,0 @@
import { BadRequestException } from '@nestjs/common';
import type { safeFetch } from '../../../base';
import { ByokProvider } from './types';
const TEST_TIMEOUT_MS = 10_000;
export const PROVIDER_PROBE_MAX_BYTES = 1024 * 1024;
type ProbeFetch = typeof safeFetch;
export async function runProviderProbe(
probeFetch: ProbeFetch,
provider: ByokProvider,
apiKey: string,
endpoint: string | null,
allowPrivateEndpoint: boolean
) {
const request = buildProbeRequest(provider, apiKey, endpoint);
const response = await probeFetch(
request.url,
{
method: request.method,
headers: request.headers,
},
{
timeoutMs: TEST_TIMEOUT_MS,
maxRedirects: 3,
maxBytes: PROVIDER_PROBE_MAX_BYTES,
allowedHeaders: Object.keys(request.headers),
allowHttp: endpoint?.startsWith('http:') ?? false,
allowPrivateTargetOrigin: allowPrivateEndpoint,
}
);
if (!response.ok) {
throw new BadRequestException(providerProbeFailureMessage(response.status));
}
}
function buildProbeRequest(
provider: ByokProvider,
apiKey: string,
endpoint: string | null
): {
method: 'GET';
url: string;
headers: Record<string, string>;
} {
switch (provider) {
case ByokProvider.openai:
return {
method: 'GET',
url: `${endpoint ?? 'https://api.openai.com/v1'}/models`,
headers: { Authorization: `Bearer ${apiKey}` },
};
case ByokProvider.anthropic:
return {
method: 'GET',
url: `${endpoint ?? 'https://api.anthropic.com/v1'}/models`,
headers: {
'x-api-key': apiKey,
'anthropic-version': '2023-06-01',
},
};
case ByokProvider.gemini:
return {
method: 'GET',
url: `${endpoint ?? 'https://generativelanguage.googleapis.com/v1beta'}/models`,
headers: { 'x-goog-api-key': apiKey },
};
case ByokProvider.fal:
return {
method: 'GET',
url: 'https://api.fal.ai/v1/models?limit=10',
headers: { Authorization: `Key ${apiKey}` },
};
}
}
function providerProbeFailureMessage(status: number) {
switch (status) {
case 401:
return 'Provider rejected the BYOK key.';
case 403:
return 'Provider rejected the BYOK key permissions.';
case 404:
return 'Provider probe endpoint was not found.';
case 429:
return 'Provider rate limit exceeded while testing the key.';
default:
return status >= 500
? 'Provider service is unavailable.'
: `Provider key test failed with HTTP ${status}.`;
}
}
@@ -1,3 +1,4 @@
import { BadRequestException } from '@nestjs/common';
import {
Args,
Field,
@@ -11,18 +12,139 @@ import {
} from '@nestjs/graphql';
import { SafeIntResolver } from 'graphql-scalars';
import { Throttle } from '../../../base';
import { Config, Throttle } from '../../../base';
import { CurrentUser } from '../../../core/auth';
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
import { PermissionAccess } from '../../../core/permission';
import { WorkspaceType } from '../../../core/workspaces';
import { Models } from '../../../models';
import { llmGetByokCatalog } from '../../../native';
import { CopilotEnabled } from '../feature';
import { ByokEntitlementPolicy } from './policy';
import { ByokKeyConfig, ByokLocalLeaseProvider, ByokService } from './service';
import { ByokKeyStorage, ByokKeyTestStatus, ByokProvider } from './types';
import {
BYOK_ALLOWED_PROVIDERS,
ByokProvider,
ByokProviderSource,
} from './types';
@ObjectType()
export class WorkspaceByokKeyConfigType implements ByokKeyConfig {
class WorkspaceByokCapabilityType {
@Field(() => [String])
input!: string[];
@Field(() => [String])
output!: string[];
@Field(() => [String])
features!: string[];
@Field(() => [String])
attachmentKinds!: string[];
@Field(() => [String])
attachmentSources!: string[];
}
@ObjectType()
class WorkspaceByokModelDeclarationType {
@Field(() => String)
modelId!: string;
@Field(() => Boolean)
enabled!: boolean;
@Field(() => [WorkspaceByokCapabilityType])
capabilities!: WorkspaceByokCapabilityType[];
}
@ObjectType()
class WorkspaceByokEndpointType {
@Field(() => String)
kind!: string;
@Field(() => String, { nullable: true })
url!: string | null;
}
@ObjectType()
class WorkspaceByokProfileDefinitionType {
@Field(() => SafeIntResolver)
version!: number;
@Field(() => WorkspaceByokEndpointType)
endpoint!: WorkspaceByokEndpointType;
@Field(() => [WorkspaceByokModelDeclarationType])
models!: WorkspaceByokModelDeclarationType[];
}
@ObjectType()
class WorkspaceByokProbeStatusType {
@Field(() => String)
kind!: string;
@Field(() => Date, { nullable: true })
testedAt!: Date | null;
@Field(() => String, { nullable: true })
errorKind!: string | null;
}
@ObjectType()
class WorkspaceByokModelProbeCheckType {
@Field(() => String)
operation!: string;
@Field(() => WorkspaceByokProbeStatusType)
status!: WorkspaceByokProbeStatusType;
}
@ObjectType()
class WorkspaceByokModelProbeType {
@Field(() => String)
modelId!: string;
@Field(() => [WorkspaceByokModelProbeCheckType])
checks!: WorkspaceByokModelProbeCheckType[];
}
@ObjectType()
class WorkspaceByokValidationType {
@Field(() => String)
definitionFingerprint!: string;
@Field(() => SafeIntResolver)
credentialGeneration!: number;
@Field(() => WorkspaceByokProbeStatusType)
connection!: WorkspaceByokProbeStatusType;
@Field(() => [WorkspaceByokModelProbeType])
models!: WorkspaceByokModelProbeType[];
}
@ObjectType()
class WorkspaceByokProbeResultType {
@Field(() => String)
definitionFingerprint!: string;
@Field(() => Boolean)
stale!: boolean;
@Field(() => WorkspaceByokProbeStatusType)
connection!: WorkspaceByokProbeStatusType;
@Field(() => [WorkspaceByokModelProbeType])
models!: WorkspaceByokModelProbeType[];
}
@ObjectType()
export class WorkspaceByokProfileType {
@Field(() => ID)
id!: string;
profileId!: string;
@Field(() => String)
workspaceId!: string;
@Field(() => ByokProvider)
provider!: ByokProvider;
@@ -33,59 +155,53 @@ export class WorkspaceByokKeyConfigType implements ByokKeyConfig {
@Field(() => String, { nullable: true })
description!: string | null;
@Field(() => ByokKeyStorage)
storage!: ByokKeyStorage;
@Field(() => Boolean)
configured!: boolean;
@Field(() => WorkspaceByokProfileDefinitionType)
definition!: WorkspaceByokProfileDefinitionType;
@Field(() => Boolean)
enabled!: boolean;
@Field(() => String, { nullable: true })
endpoint!: string | null;
@Field(() => Boolean)
endpointEditable!: boolean;
@Field(() => SafeIntResolver)
sortOrder!: number;
@Field(() => [String])
capabilities!: string[];
@Field(() => SafeIntResolver)
revision!: number;
@Field(() => ByokKeyTestStatus)
testStatus!: ByokKeyTestStatus;
@Field(() => String, { nullable: true })
disabledReason!: string | null;
@Field(() => Date, { nullable: true })
lastTestedAt!: Date | null;
@Field(() => String, { nullable: true })
lastTestError!: string | null;
@Field(() => Date, { nullable: true })
lastUsedAt!: Date | null;
@Field(() => Date, { nullable: true })
lastErrorAt!: Date | null;
@Field(() => String, { nullable: true })
lastError!: string | null;
@Field(() => WorkspaceByokValidationType, { nullable: true })
validation!: WorkspaceByokValidationType | null;
}
@ObjectType()
class WorkspaceByokCapabilityWarningType {
class WorkspaceByokCatalogModelType {
@Field(() => String)
featureKind!: string;
modelId!: string;
@Field(() => String)
reason!: string;
displayName!: string;
@Field(() => [ByokProvider])
requiredProviders!: ByokProvider[];
@Field(() => Boolean)
recommended!: boolean;
@Field(() => [WorkspaceByokCapabilityType])
capabilities!: WorkspaceByokCapabilityType[];
}
@ObjectType()
class WorkspaceByokCatalogProviderType {
@Field(() => ByokProvider)
provider!: ByokProvider;
@Field(() => [WorkspaceByokCatalogModelType])
models!: WorkspaceByokCatalogModelType[];
}
@ObjectType()
class WorkspaceByokCatalogType {
@Field(() => String)
version!: string;
@Field(() => [WorkspaceByokCatalogProviderType])
providers!: WorkspaceByokCatalogProviderType[];
}
@ObjectType()
@@ -102,29 +218,20 @@ class WorkspaceByokSettingsType {
@Field(() => Boolean)
localEntitled!: boolean;
@Field(() => [String])
entitlementRequired!: string[];
@Field(() => [WorkspaceByokKeyConfigType])
keys!: WorkspaceByokKeyConfigType[];
@Field(() => [WorkspaceByokProfileType])
profiles!: WorkspaceByokProfileType[];
@Field(() => [ByokProvider])
allowedProviders!: ByokProvider[];
@Field(() => Boolean)
localStorageSupported!: boolean;
@Field(() => Boolean)
customEndpointSupported!: boolean;
@Field(() => Boolean)
privateEndpointSupported!: boolean;
@Field(() => Boolean)
hasAiPlan!: boolean;
@Field(() => [WorkspaceByokCapabilityWarningType])
warnings!: WorkspaceByokCapabilityWarningType[];
@Field(() => WorkspaceByokCatalogType)
catalog!: WorkspaceByokCatalogType;
}
@ObjectType()
@@ -139,18 +246,6 @@ class WorkspaceByokUsagePointType {
totalTokens!: number;
}
@ObjectType()
class TestWorkspaceByokConfigResultType {
@Field(() => Boolean)
ok!: boolean;
@Field(() => ByokKeyTestStatus)
status!: ByokKeyTestStatus;
@Field(() => String, { nullable: true })
message!: string | null;
}
@ObjectType()
class CreateWorkspaceByokLocalLeaseResultType {
@Field(() => String)
@@ -161,10 +256,58 @@ class CreateWorkspaceByokLocalLeaseResultType {
}
@InputType()
class UpsertWorkspaceByokConfigInput {
@Field(() => ID, { nullable: true })
id?: string;
class WorkspaceByokCapabilityInput {
@Field(() => [String])
input!: string[];
@Field(() => [String])
output!: string[];
@Field(() => [String])
features!: string[];
@Field(() => [String])
attachmentKinds!: string[];
@Field(() => [String])
attachmentSources!: string[];
}
@InputType()
class WorkspaceByokModelDeclarationInput {
@Field(() => String)
modelId!: string;
@Field(() => Boolean)
enabled!: boolean;
@Field(() => [WorkspaceByokCapabilityInput])
capabilities!: WorkspaceByokCapabilityInput[];
}
@InputType()
class WorkspaceByokEndpointInput {
@Field(() => String)
kind!: string;
@Field(() => String, { nullable: true })
url!: string | null;
}
@InputType()
class WorkspaceByokProfileDefinitionInput {
@Field(() => SafeIntResolver)
version!: number;
@Field(() => WorkspaceByokEndpointInput)
endpoint!: WorkspaceByokEndpointInput;
@Field(() => [WorkspaceByokModelDeclarationInput])
models!: WorkspaceByokModelDeclarationInput[];
}
@InputType()
class CreateWorkspaceByokProfileInput {
@Field(() => String)
workspaceId!: string;
@@ -175,59 +318,125 @@ class UpsertWorkspaceByokConfigInput {
name!: string;
@Field(() => String, { nullable: true })
description?: string | null;
description!: string | null;
@Field(() => ByokKeyStorage)
storage!: ByokKeyStorage;
@Field(() => String)
credential!: string;
@Field(() => String, { nullable: true })
apiKey?: string | null;
@Field(() => WorkspaceByokProfileDefinitionInput)
definition!: WorkspaceByokProfileDefinitionInput;
@Field(() => String, { nullable: true })
endpoint?: string | null;
@Field(() => SafeIntResolver, { nullable: true })
sortOrder?: number | null;
@Field(() => Boolean, { nullable: true })
enabled?: boolean | null;
@Field(() => Boolean)
enabled!: boolean;
}
@InputType()
class TestWorkspaceByokConfigInput {
class ReplaceWorkspaceByokProfileInput {
@Field(() => String)
workspaceId!: string;
@Field(() => ID)
profileId!: string;
@Field(() => SafeIntResolver)
expectedRevision!: number;
@Field(() => String)
name!: string;
@Field(() => String, { nullable: true })
description!: string | null;
@Field(() => WorkspaceByokProfileDefinitionInput)
definition!: WorkspaceByokProfileDefinitionInput;
@Field(() => String, { nullable: true })
credential!: string | null;
@Field(() => Boolean)
enabled!: boolean;
}
@InputType()
class RotateWorkspaceByokCredentialInput {
@Field(() => String)
workspaceId!: string;
@Field(() => ID)
profileId!: string;
@Field(() => SafeIntResolver)
expectedRevision!: number;
@Field(() => String)
credential!: string;
}
@InputType()
class WorkspaceByokProbeCheckInput {
@Field(() => String)
modelId!: string;
@Field(() => String)
operation!: string;
}
@InputType()
class ProbeWorkspaceByokProfileInput {
@Field(() => String)
workspaceId!: string;
@Field(() => ID)
profileId!: string;
@Field(() => [WorkspaceByokProbeCheckInput])
checks!: WorkspaceByokProbeCheckInput[];
}
@InputType()
class ProbeWorkspaceByokDraftInput {
@Field(() => String)
workspaceId!: string;
@Field(() => ByokProvider)
provider!: ByokProvider;
@Field(() => ByokKeyStorage)
storage!: ByokKeyStorage;
@Field(() => String, { nullable: true })
apiKey?: string | null;
@Field(() => String, { nullable: true })
endpoint?: string | null;
credential!: string | null;
@Field(() => ID, { nullable: true })
configId?: string | null;
profileId!: string | null;
@Field(() => SafeIntResolver, { nullable: true })
expectedRevision!: number | null;
@Field(() => WorkspaceByokProfileDefinitionInput)
definition!: WorkspaceByokProfileDefinitionInput;
@Field(() => [WorkspaceByokProbeCheckInput])
checks!: WorkspaceByokProbeCheckInput[];
}
@InputType()
class ReorderWorkspaceByokConfigsInput {
class WorkspaceByokProfileOrderInput {
@Field(() => ID)
profileId!: string;
@Field(() => SafeIntResolver)
expectedRevision!: number;
}
@InputType()
class ReorderWorkspaceByokProfilesInput {
@Field(() => String)
workspaceId!: string;
@Field(() => ByokKeyStorage)
storage!: ByokKeyStorage;
@Field(() => [ID])
ids!: string[];
@Field(() => [WorkspaceByokProfileOrderInput])
profiles!: WorkspaceByokProfileOrderInput[];
}
@InputType()
class CreateWorkspaceByokLocalLeaseProviderInput implements ByokLocalLeaseProvider {
class CreateWorkspaceByokLocalLeaseProviderInput {
@Field(() => ByokProvider)
provider!: ByokProvider;
@@ -235,19 +444,16 @@ class CreateWorkspaceByokLocalLeaseProviderInput implements ByokLocalLeaseProvid
name!: string;
@Field(() => String, { nullable: true })
description?: string | null;
description!: string | null;
@Field(() => String)
apiKey!: string;
credential!: string;
@Field(() => String, { nullable: true })
endpoint?: string | null;
@Field(() => WorkspaceByokProfileDefinitionInput)
definition!: WorkspaceByokProfileDefinitionInput;
@Field(() => SafeIntResolver, { nullable: true })
sortOrder?: number | null;
@Field(() => Boolean, { nullable: true })
enabled?: boolean | null;
@Field(() => Boolean)
enabled!: boolean;
}
@InputType()
@@ -259,12 +465,15 @@ class CreateWorkspaceByokLocalLeaseInput {
providers!: CreateWorkspaceByokLocalLeaseProviderInput[];
}
@CopilotEnabled()
@Resolver(() => WorkspaceType)
export class WorkspaceByokResolver {
constructor(
private readonly ac: PermissionAccess,
private readonly entitlement: ByokEntitlementPolicy,
private readonly byok: ByokService
private readonly runtime: BackendRuntimeProvider,
private readonly models: Models,
private readonly config: Config
) {}
@ResolveField(() => WorkspaceByokSettingsType, {
@@ -275,13 +484,35 @@ export class WorkspaceByokResolver {
@CurrentUser() user: CurrentUser,
@Parent() workspace: WorkspaceType
) {
await this.ac
.user(user.id)
.workspace(workspace.id)
.allowLocal()
.assert('Workspace.Settings.Read');
await this.assertRead(user.id, workspace.id);
await this.entitlement.assertManagementAccess(workspace.id, user.id);
return await this.byok.getSettings(workspace.id, user.id);
const [serverEntitled, localEntitled] =
await this.entitlement.hasEntitlement(workspace.id, user.id);
const profiles = serverEntitled
? await this.runtime.listByokProfiles(workspace.id)
: [];
const customEndpointSupported =
this.config.copilot.byok.allowCustomEndpoint;
const catalog = llmGetByokCatalog();
return {
workspaceId: workspace.id,
entitled: serverEntitled || localEntitled,
serverEntitled,
localEntitled,
profiles: profiles.map(profile => projectProfile(profile)),
allowedProviders: [...BYOK_ALLOWED_PROVIDERS],
customEndpointSupported,
privateEndpointSupported:
customEndpointSupported &&
this.config.copilot.byok.allowPrivateEndpoint,
catalog: {
...catalog,
providers: catalog.providers.map(provider => ({
...provider,
provider: provider.provider as ByokProvider,
})),
},
};
}
@ResolveField(() => [WorkspaceByokUsagePointType], {
@@ -294,100 +525,131 @@ export class WorkspaceByokResolver {
@Args('from', { type: () => Date }) from: Date,
@Args('to', { type: () => Date }) to: Date
) {
await this.ac
.user(user.id)
.workspace(workspace.id)
.allowLocal()
.assert('Workspace.Settings.Read');
await this.assertRead(user.id, workspace.id);
await this.entitlement.assertManagementAccess(workspace.id, user.id);
return await this.byok.getUsage(workspace.id, from, to);
return await this.models.copilotUsage.aggregateByDay({
workspaceId: workspace.id,
from,
to,
providerSources: [ByokProviderSource.Server, ByokProviderSource.Local],
});
}
@Mutation(() => WorkspaceByokProfileType)
@Throttle('strict')
@Mutation(() => TestWorkspaceByokConfigResultType)
async testWorkspaceByokConfig(
async createWorkspaceByokProfile(
@CurrentUser() user: CurrentUser,
@Args('input') input: TestWorkspaceByokConfigInput
@Args('input') input: CreateWorkspaceByokProfileInput
) {
await this.ac
.user(user.id)
.workspace(input.workspaceId)
.allowLocal()
.assert('Workspace.Settings.Update');
await this.entitlement.assertManagementAccess(input.workspaceId, user.id);
if (input.storage === ByokKeyStorage.server) {
await this.assertUpdate(user.id, input.workspaceId);
await this.entitlement.assertServerEntitled(input.workspaceId);
requireExplicitDescription(input);
return projectProfile(
await this.runtime.createByokProfile({
...input,
description: input.description ?? undefined,
definition: nativeDefinition(input.definition),
actorUserId: user.id,
})
);
}
@Mutation(() => WorkspaceByokProfileType)
@Throttle('strict')
async replaceWorkspaceByokProfile(
@CurrentUser() user: CurrentUser,
@Args('input') input: ReplaceWorkspaceByokProfileInput
) {
await this.assertUpdate(user.id, input.workspaceId);
await this.entitlement.assertServerEntitled(input.workspaceId);
requireExplicitDescription(input);
return projectProfile(
await this.runtime.replaceByokProfile({
...input,
description: input.description ?? undefined,
credential: input.credential ?? undefined,
definition: nativeDefinition(input.definition),
actorUserId: user.id,
})
);
}
@Mutation(() => WorkspaceByokProfileType)
@Throttle('strict')
async rotateWorkspaceByokCredential(
@CurrentUser() user: CurrentUser,
@Args('input') input: RotateWorkspaceByokCredentialInput
) {
await this.assertUpdate(user.id, input.workspaceId);
await this.entitlement.assertServerEntitled(input.workspaceId);
return projectProfile(
await this.runtime.rotateByokCredential({
...input,
actorUserId: user.id,
})
);
}
@Mutation(() => WorkspaceByokProbeResultType)
@Throttle('strict')
async probeWorkspaceByokProfile(
@CurrentUser() user: CurrentUser,
@Args('input') input: ProbeWorkspaceByokProfileInput
) {
await this.assertUpdate(user.id, input.workspaceId);
await this.entitlement.assertServerEntitled(input.workspaceId);
return projectProbeResult(await this.runtime.probeByokProfile(input));
}
@Mutation(() => WorkspaceByokProbeResultType)
@Throttle('strict')
async probeWorkspaceByokDraft(
@CurrentUser() user: CurrentUser,
@Args('input') input: ProbeWorkspaceByokDraftInput
) {
await this.assertUpdate(user.id, input.workspaceId);
if (input.profileId) {
await this.entitlement.assertServerEntitled(input.workspaceId);
} else {
await this.entitlement.assertLocalEntitled(input.workspaceId, user.id);
await this.entitlement.assertEntitled(input.workspaceId, user.id);
}
return await this.byok.testConfig({ ...input, userId: user.id });
}
@Mutation(() => WorkspaceByokKeyConfigType)
@Throttle('strict')
async upsertWorkspaceByokConfig(
@CurrentUser() user: CurrentUser,
@Args('input') input: UpsertWorkspaceByokConfigInput
) {
await this.ac
.user(user.id)
.workspace(input.workspaceId)
.allowLocal()
.assert('Workspace.Settings.Update');
await this.entitlement.assertManagementAccess(input.workspaceId, user.id);
await this.entitlement.assertServerEntitled(input.workspaceId);
return await this.byok.upsertConfig({ ...input, userId: user.id });
}
@Mutation(() => [WorkspaceByokKeyConfigType])
@Throttle('strict')
async reorderWorkspaceByokConfigs(
@CurrentUser() user: CurrentUser,
@Args('input') input: ReorderWorkspaceByokConfigsInput
) {
await this.ac
.user(user.id)
.workspace(input.workspaceId)
.allowLocal()
.assert('Workspace.Settings.Update');
await this.entitlement.assertManagementAccess(input.workspaceId, user.id);
await this.entitlement.assertServerEntitled(input.workspaceId);
return await this.byok.reorderConfigs({ ...input, userId: user.id });
return projectProbeResult(
await this.runtime.probeByokDraft({
...input,
credential: input.credential ?? undefined,
profileId: input.profileId ?? undefined,
expectedRevision: input.expectedRevision ?? undefined,
definition: nativeDefinition(input.definition),
})
);
}
@Mutation(() => Boolean)
@Throttle('strict')
async deleteWorkspaceByokConfig(
async deleteWorkspaceByokProfile(
@CurrentUser() user: CurrentUser,
@Args('id', { type: () => ID }) id: string,
@Args('profileId', { type: () => ID }) profileId: string,
@Args('workspaceId', { type: () => String }) workspaceId: string
) {
await this.ac
.user(user.id)
.workspace(workspaceId)
.allowLocal()
.assert('Workspace.Settings.Update');
await this.entitlement.assertManagementAccess(workspaceId, user.id);
await this.assertUpdate(user.id, workspaceId);
await this.entitlement.assertServerEntitled(workspaceId);
return await this.byok.deleteConfig(workspaceId, id, user.id);
return await this.runtime.deleteByokProfile(workspaceId, profileId);
}
@Mutation(() => Boolean)
@Mutation(() => [WorkspaceByokProfileType])
@Throttle('strict')
async clearWorkspaceByokConfigs(
async reorderWorkspaceByokProfiles(
@CurrentUser() user: CurrentUser,
@Args('workspaceId', { type: () => String }) workspaceId: string,
@Args('provider', { type: () => ByokProvider, nullable: true })
provider?: ByokProvider | null
@Args('input') input: ReorderWorkspaceByokProfilesInput
) {
await this.ac
.user(user.id)
.workspace(workspaceId)
.allowLocal()
.assert('Workspace.Settings.Update');
await this.entitlement.assertManagementAccess(workspaceId, user.id);
await this.entitlement.assertServerEntitled(workspaceId);
return await this.byok.clearConfigs(workspaceId, provider, user.id);
await this.assertUpdate(user.id, input.workspaceId);
await this.entitlement.assertServerEntitled(input.workspaceId);
return (
await this.runtime.reorderByokProfiles({
...input,
actorUserId: user.id,
})
).map(profile => projectProfile(profile));
}
@Mutation(() => CreateWorkspaceByokLocalLeaseResultType)
@@ -403,6 +665,119 @@ export class WorkspaceByokResolver {
.assert('Workspace.Copilot');
await this.entitlement.assertManagementAccess(input.workspaceId, user.id);
await this.entitlement.assertLocalEntitled(input.workspaceId, user.id);
return await this.byok.createLocalLease({ ...input, userId: user.id });
input.providers.forEach(requireExplicitDescription);
const result = await this.runtime.createByokLocalLease({
...input,
providers: input.providers.map(provider => ({
...provider,
description: provider.description ?? undefined,
definition: nativeDefinition(provider.definition),
})),
userId: user.id,
});
return {
leaseId: result.leaseId,
expiresAt: new Date(result.expiresAtMs),
};
}
private async assertRead(userId: string, workspaceId: string) {
await this.ac
.user(userId)
.workspace(workspaceId)
.allowLocal()
.assert('Workspace.Settings.Read');
}
private async assertUpdate(userId: string, workspaceId: string) {
await this.ac
.user(userId)
.workspace(workspaceId)
.allowLocal()
.assert('Workspace.Settings.Update');
await this.entitlement.assertManagementAccess(workspaceId, userId);
}
}
function requireExplicitDescription(input: { description: string | null }) {
if (!Object.hasOwn(input, 'description')) {
throw new BadRequestException('description must be provided explicitly.');
}
}
function nativeDefinition(input: WorkspaceByokProfileDefinitionInput) {
return {
...input,
endpoint: {
...input.endpoint,
url: input.endpoint.url ?? undefined,
},
};
}
function projectProbe(probe: {
kind: string;
testedAtMs?: number;
errorKind?: string;
}) {
return {
kind: probe.kind,
testedAt: probe.testedAtMs ? new Date(probe.testedAtMs) : null,
errorKind: probe.errorKind ?? null,
};
}
function projectProbeResult(result: {
definitionFingerprint: string;
stale: boolean;
connection: {
kind: string;
testedAtMs?: number;
errorKind?: string;
};
models: Array<{
modelId: string;
checks: Array<{
operation: string;
status: {
kind: string;
testedAtMs?: number;
errorKind?: string;
};
}>;
}>;
}) {
return {
...result,
connection: projectProbe(result.connection),
models: result.models.map(model => ({
...model,
checks: model.checks.map(check => ({
...check,
status: projectProbe(check.status),
})),
})),
};
}
function projectProfile(
profile: Awaited<ReturnType<BackendRuntimeProvider['createByokProfile']>>
) {
return {
...profile,
provider: profile.provider as ByokProvider,
validation: profile.validation
? {
...profile.validation,
connection: projectProbe(profile.validation.connection),
models: profile.validation.models.map(model => ({
...model,
checks: model.checks.map(check => ({
...check,
status: projectProbe(check.status),
})),
})),
}
: null,
};
}
@@ -1,833 +0,0 @@
import { createHash, createHmac, randomUUID } from 'node:crypto';
import { BadRequestException, Injectable } from '@nestjs/common';
import {
BadRequest,
Cache,
Config,
CryptoHelper,
metrics,
safeFetch,
} from '../../../base';
import { Models } from '../../../models';
import type { CopilotProviderProfile } from '../config';
import { ByokEntitlementPolicy } from './policy';
import { runProviderProbe } from './probe';
import {
BYOK_ALLOWED_PROVIDERS,
type ByokFeatureKind,
ByokKeyStorage,
ByokKeyTestStatus,
ByokProvider,
ByokProviderSource,
byokProviderToCopilotType,
isByokProvider,
} from './types';
const LOCAL_LEASE_TTL_MS = 10 * 60 * 1000;
const BYOK_PROFILE_PRIORITY_BASE = 10_000;
const SERVER_PROFILE_PRIORITY_OFFSET = 2_000;
export type ByokProviderRequestContext = {
userId?: string;
workspaceId?: string;
byokLeaseId?: string;
};
export type ByokProfileSourceFilter = {
local?: boolean;
server?: boolean;
};
export type ByokKeyConfig = {
id: string;
provider: ByokProvider;
name: string;
description: string | null;
storage: ByokKeyStorage;
configured: boolean;
enabled: boolean;
endpoint: string | null;
endpointEditable: boolean;
sortOrder: number;
capabilities: string[];
testStatus: ByokKeyTestStatus;
disabledReason: string | null;
lastTestedAt: Date | null;
lastTestError: string | null;
lastUsedAt: Date | null;
lastErrorAt: Date | null;
lastError: string | null;
};
export type ByokSettings = {
workspaceId: string;
entitled: boolean;
serverEntitled: boolean;
localEntitled: boolean;
entitlementRequired: string[];
keys: ByokKeyConfig[];
allowedProviders: ByokProvider[];
localStorageSupported: boolean;
customEndpointSupported: boolean;
privateEndpointSupported: boolean;
hasAiPlan: boolean;
warnings: Array<{
featureKind: string;
reason: string;
requiredProviders: ByokProvider[];
}>;
};
export type ByokLocalLeaseProvider = {
provider: ByokProvider;
name: string;
description?: string | null;
apiKey: string;
endpoint?: string | null;
sortOrder?: number | null;
enabled?: boolean | null;
};
type LocalLeasePayload = {
workspaceId: string;
userId: string;
providers: Array<
Omit<ByokLocalLeaseProvider, 'apiKey'> & { encryptedApiKey: string }
>;
};
type LocalLeaseActive = {
leaseId: string;
expiresAt: string;
};
type ByokProfileMeta = {
source: ByokProviderSource.Server | ByokProviderSource.Local;
keyId?: string;
provider: ByokProvider;
};
@Injectable()
export class ByokService {
private readonly probeFetch = safeFetch;
constructor(
private readonly models: Models,
private readonly crypto: CryptoHelper,
private readonly cache: Cache,
private readonly entitlement: ByokEntitlementPolicy,
private readonly config: Config
) {}
get customEndpointSupported() {
return env.selfhosted && this.config.copilot.byok.allowCustomEndpoint;
}
get privateEndpointSupported() {
return (
this.customEndpointSupported &&
this.config.copilot.byok.allowPrivateEndpoint
);
}
async getSettings(
workspaceId: string,
userId?: string
): Promise<ByokSettings> {
if (!(await this.entitlement.hasManagementAccess(workspaceId, userId))) {
return {
workspaceId,
entitled: false,
serverEntitled: false,
localEntitled: false,
entitlementRequired: ['Workspace owner or admin'],
keys: [],
allowedProviders: [...BYOK_ALLOWED_PROVIDERS],
localStorageSupported: false,
customEndpointSupported: this.customEndpointSupported,
privateEndpointSupported: this.privateEndpointSupported,
hasAiPlan: await this.entitlement.hasAiPlan(userId),
warnings: [],
};
}
const [serverEntitled, localEntitled] =
await this.entitlement.hasEntitlement(workspaceId, userId);
const entitled = serverEntitled || localEntitled;
if (!entitled) {
return {
workspaceId,
entitled: false,
serverEntitled: false,
localEntitled: false,
entitlementRequired: ['Pro', 'Team', 'Believer'],
keys: [],
allowedProviders: [...BYOK_ALLOWED_PROVIDERS],
localStorageSupported: false,
customEndpointSupported: this.customEndpointSupported,
privateEndpointSupported: this.privateEndpointSupported,
hasAiPlan: await this.entitlement.hasAiPlan(userId),
warnings: [],
};
}
const rows = serverEntitled
? await this.models.copilotWorkspaceByokConfig.list(workspaceId)
: [];
const keys = rows.map(row => this.toKeyConfig(row));
return {
workspaceId,
entitled: true,
serverEntitled,
localEntitled,
entitlementRequired: ['Pro', 'Team', 'Believer'],
keys,
allowedProviders: [...BYOK_ALLOWED_PROVIDERS],
localStorageSupported: false,
customEndpointSupported: this.customEndpointSupported,
privateEndpointSupported: this.privateEndpointSupported,
hasAiPlan: await this.entitlement.hasAiPlan(userId),
warnings: this.buildWarnings(keys),
};
}
async upsertConfig(input: {
id?: string | null;
workspaceId: string;
provider: ByokProvider;
name: string;
description?: string | null;
storage: ByokKeyStorage;
apiKey?: string | null;
endpoint?: string | null;
sortOrder?: number | null;
enabled?: boolean | null;
userId?: string;
}): Promise<ByokKeyConfig> {
await this.entitlement.assertManagementAccess(
input.workspaceId,
input.userId
);
await this.entitlement.assertServerEntitled(input.workspaceId);
this.assertProvider(input.provider);
if (input.storage !== ByokKeyStorage.server) {
throw new BadRequestException('Only server BYOK keys are persisted.');
}
const existing = input.id
? await this.models.copilotWorkspaceByokConfig.get(input.id)
: null;
if (input.id && (!existing || existing.workspaceId !== input.workspaceId)) {
throw new BadRequest('BYOK config not found.');
}
const encryptedApiKey = input.apiKey
? this.crypto.encrypt(input.apiKey)
: undefined;
if (!input.id && !encryptedApiKey) {
throw new BadRequestException('apiKey is required.');
}
const description =
input.description !== undefined
? input.description?.trim() || null
: (existing?.description ?? null);
const endpoint =
input.endpoint !== undefined
? this.normalizeEndpoint(input.endpoint)
: (existing?.endpoint ?? null);
const sortOrder = input.sortOrder ?? existing?.sortOrder ?? 0;
const enabled = input.enabled ?? existing?.enabled ?? true;
const row = await this.models.copilotWorkspaceByokConfig.upsert({
id: input.id,
workspaceId: input.workspaceId,
provider: input.provider,
name: input.name.trim(),
description,
encryptedApiKey,
endpoint,
sortOrder,
enabled,
userId: input.userId,
});
return this.toKeyConfig(row);
}
async reorderConfigs(input: {
workspaceId: string;
storage: ByokKeyStorage;
ids: string[];
userId?: string;
}) {
await this.entitlement.assertManagementAccess(
input.workspaceId,
input.userId
);
await this.entitlement.assertServerEntitled(input.workspaceId);
if (input.storage !== ByokKeyStorage.server) {
throw new BadRequestException('Only server BYOK keys are persisted.');
}
await this.models.copilotWorkspaceByokConfig.reorder(
input.workspaceId,
input.ids,
input.userId
);
return (await this.getSettings(input.workspaceId, input.userId)).keys;
}
async deleteConfig(workspaceId: string, id: string, _userId?: string) {
await this.entitlement.assertManagementAccess(workspaceId, _userId);
await this.entitlement.assertServerEntitled(workspaceId);
await this.models.copilotWorkspaceByokConfig.delete(workspaceId, id);
return true;
}
async clearConfigs(
workspaceId: string,
provider: ByokProvider | null | undefined,
_userId?: string
) {
await this.entitlement.assertManagementAccess(workspaceId, _userId);
await this.entitlement.assertServerEntitled(workspaceId);
await this.models.copilotWorkspaceByokConfig.clear(workspaceId, provider);
return true;
}
async testConfig(input: {
workspaceId: string;
provider: ByokProvider;
storage: ByokKeyStorage;
apiKey?: string | null;
endpoint?: string | null;
configId?: string | null;
userId?: string;
}) {
await this.entitlement.assertManagementAccess(
input.workspaceId,
input.userId
);
if (input.storage === ByokKeyStorage.server) {
await this.entitlement.assertServerEntitled(input.workspaceId);
} else {
await this.entitlement.assertLocalEntitled(
input.workspaceId,
input.userId
);
}
this.assertProvider(input.provider);
let apiKey = input.apiKey;
let endpoint = this.normalizeEndpoint(input.endpoint);
if (!apiKey && input.configId && input.storage === ByokKeyStorage.server) {
const config = await this.models.copilotWorkspaceByokConfig.get(
input.configId
);
if (
!config ||
config.workspaceId !== input.workspaceId ||
config.provider !== input.provider
) {
throw new BadRequestException('BYOK config not found.');
}
apiKey = this.crypto.decrypt(config.encryptedApiKey);
endpoint =
input.endpoint !== undefined
? endpoint
: this.normalizeEndpoint(config.endpoint);
}
if (!apiKey) {
throw new BadRequestException('apiKey is required.');
}
try {
await runProviderProbe(
this.probeFetch,
input.provider,
apiKey,
endpoint,
this.privateEndpointSupported
);
if (input.configId && input.storage === ByokKeyStorage.server) {
await this.models.copilotWorkspaceByokConfig.markValidated(
input.workspaceId,
input.configId,
input.userId
);
}
metrics.ai.counter('byok_test_key').add(1, {
workspace: input.workspaceId,
provider: input.provider,
storage: input.storage,
result: 'passed',
});
return { ok: true, status: ByokKeyTestStatus.passed, message: null };
} catch (error) {
const message = this.sanitizeError(error);
if (input.configId && input.storage === ByokKeyStorage.server) {
await this.models.copilotWorkspaceByokConfig.markFailure(
input.workspaceId,
input.configId,
message
);
}
metrics.ai.counter('byok_test_key').add(1, {
workspace: input.workspaceId,
provider: input.provider,
storage: input.storage,
result: 'failed',
});
return { ok: false, status: ByokKeyTestStatus.failed, message };
}
}
async createLocalLease(input: {
workspaceId: string;
providers: ByokLocalLeaseProvider[];
userId: string;
}) {
await this.entitlement.assertManagementAccess(
input.workspaceId,
input.userId
);
await this.entitlement.assertLocalEntitled(input.workspaceId, input.userId);
const providers = input.providers.map(provider => {
this.assertProvider(provider.provider);
const endpoint = this.normalizeEndpoint(provider.endpoint);
return { ...provider, endpoint };
});
const activeCacheKey = this.localLeaseActiveCacheKey({
...input,
providers,
});
const activeLease = await this.getActiveLocalLease(activeCacheKey);
if (activeLease) return activeLease;
const leaseId = randomUUID();
const expiresAt = new Date(Date.now() + LOCAL_LEASE_TTL_MS);
const payload: LocalLeasePayload = {
workspaceId: input.workspaceId,
userId: input.userId,
providers: providers.map(provider => ({
provider: provider.provider,
name: provider.name,
description: provider.description,
encryptedApiKey: this.crypto.encrypt(provider.apiKey),
endpoint: provider.endpoint,
sortOrder: provider.sortOrder,
enabled: provider.enabled,
})),
};
await this.cache.set(this.leaseCacheKey(leaseId), payload, {
ttl: LOCAL_LEASE_TTL_MS,
});
const registered = await this.cache.setnx<LocalLeaseActive>(
activeCacheKey,
{ leaseId, expiresAt: expiresAt.toISOString() },
{ ttl: LOCAL_LEASE_TTL_MS }
);
if (!registered) {
const current = await this.getActiveLocalLease(activeCacheKey);
if (current) {
await this.cache.delete(this.leaseCacheKey(leaseId));
return current;
}
}
return { leaseId, expiresAt };
}
async getProfiles(
context: ByokProviderRequestContext = {},
sources: ByokProfileSourceFilter = { local: true, server: true }
): Promise<CopilotProviderProfile[]> {
if (!context.workspaceId) {
return [];
}
const [localEntitled, serverEntitled] = await Promise.all([
this.entitlement.hasLocalEntitlement(context.workspaceId, context.userId),
this.entitlement.hasServerEntitlement(context.workspaceId),
]);
const [localProfiles, serverProfiles] = await Promise.all([
sources.local && localEntitled
? this.getLocalProfiles(context)
: Promise.resolve([]),
sources.server && serverEntitled
? this.getServerProfiles(context.workspaceId)
: Promise.resolve([]),
]);
return [...localProfiles, ...serverProfiles];
}
async recordUsage(input: {
workspaceId?: string;
userId?: string;
providerId?: string;
model?: string | null;
featureKind: ByokFeatureKind;
sessionId?: string;
taskId?: string;
actionId?: string;
billingUnitId?: string;
usage?: {
prompt_tokens?: number;
completion_tokens?: number;
total_tokens?: number;
cached_tokens?: number;
};
}) {
if (!input.workspaceId || !input.providerId) return;
const meta = this.parseProfileMeta(input.providerId, input.workspaceId);
if (!meta) return;
metrics.ai.counter('byok_usage').add(1, {
workspace: input.workspaceId,
provider: meta.provider,
source: meta.source,
feature: input.featureKind,
});
await this.models.copilotUsage.create({
workspaceId: input.workspaceId,
userId: input.userId,
provider: meta.provider,
providerSource: meta.source,
featureKind: input.featureKind,
model: input.model ?? null,
sessionId: input.sessionId,
taskId: input.taskId,
actionId: input.actionId,
billingUnitId: input.billingUnitId,
promptTokens: input.usage?.prompt_tokens ?? 0,
completionTokens: input.usage?.completion_tokens ?? 0,
totalTokens: input.usage?.total_tokens ?? 0,
cachedTokens: input.usage?.cached_tokens ?? 0,
});
if (meta.source === ByokProviderSource.Server && meta.keyId) {
await this.models.copilotWorkspaceByokConfig.touchUsed(
input.workspaceId,
meta.keyId
);
}
}
async recordProviderFailure(input: {
workspaceId?: string;
providerId?: string;
featureKind: ByokFeatureKind;
error: unknown;
}) {
if (!input.workspaceId || !input.providerId) return;
const meta = this.parseProfileMeta(input.providerId, input.workspaceId);
if (!meta) return;
const message = this.sanitizeError(input.error);
metrics.ai.counter('byok_route_failure').add(1, {
workspace: input.workspaceId,
provider: meta.provider,
source: meta.source,
feature: input.featureKind,
});
if (meta.source === ByokProviderSource.Server && meta.keyId) {
await this.models.copilotWorkspaceByokConfig.markFailure(
input.workspaceId,
meta.keyId,
message
);
}
}
async getUsage(workspaceId: string, from: Date, to: Date) {
return await this.models.copilotUsage.aggregateByDay({
workspaceId,
from,
to,
providerSources: [ByokProviderSource.Server, ByokProviderSource.Local],
});
}
private async getServerProfiles(workspaceId: string) {
const rows =
await this.models.copilotWorkspaceByokConfig.listEnabled(workspaceId);
return rows
.filter(row => isByokProvider(row.provider))
.map((row, index): CopilotProviderProfile => {
const provider = row.provider as ByokProvider;
return {
id: this.profileId(workspaceId, provider, row.id, 'server'),
type: byokProviderToCopilotType(provider),
priority:
BYOK_PROFILE_PRIORITY_BASE - SERVER_PROFILE_PRIORITY_OFFSET - index,
config: this.providerConfig(
provider,
row.encryptedApiKey,
row.endpoint
),
} as CopilotProviderProfile;
});
}
private async getLocalProfiles(context: ByokProviderRequestContext) {
if (!context.byokLeaseId || !context.workspaceId || !context.userId) {
return [];
}
if (
!(await this.entitlement.hasManagementAccess(
context.workspaceId,
context.userId
))
) {
return [];
}
const lease = await this.cache.get<LocalLeasePayload>(
this.leaseCacheKey(context.byokLeaseId)
);
if (
!lease ||
lease.workspaceId !== context.workspaceId ||
lease.userId !== context.userId
) {
return [];
}
return lease.providers
.filter(provider => provider.enabled !== false)
.map((provider, index): CopilotProviderProfile => {
return {
id: this.profileId(
context.workspaceId ?? lease.workspaceId,
provider.provider,
`${index}`,
'local'
),
type: byokProviderToCopilotType(provider.provider),
priority: BYOK_PROFILE_PRIORITY_BASE - index,
config: this.providerConfig(
provider.provider,
provider.encryptedApiKey,
provider.endpoint ?? null
),
} as CopilotProviderProfile;
});
}
private providerConfig(
provider: ByokProvider,
encryptedApiKey: string,
endpoint: string | null
) {
const apiKey = this.crypto.decrypt(encryptedApiKey);
switch (provider) {
case ByokProvider.openai:
case ByokProvider.gemini:
case ByokProvider.anthropic:
return { apiKey, ...(endpoint ? { baseURL: endpoint } : {}) };
case ByokProvider.fal:
return { apiKey };
}
}
private profileId(
workspaceId: string,
provider: ByokProvider,
keyId: string,
storage: 'server' | 'local'
) {
const hash = this.workspaceHash(workspaceId);
const sanitizedKeyId = keyId.replaceAll(/[^a-zA-Z0-9-_]/g, '');
return storage === 'local'
? `byok-${hash}-${provider}-local-${sanitizedKeyId}`
: `byok-${hash}-${provider}-${sanitizedKeyId}`;
}
parseProfileMeta(
providerId: string,
workspaceId?: string
): ByokProfileMeta | null {
const match =
/^byok-([a-f0-9]{12})-(openai|anthropic|gemini|fal)-(.+)$/.exec(
providerId
);
if (!match) return null;
if (workspaceId && match[1] !== this.workspaceHash(workspaceId)) {
return null;
}
const keyId = match[3];
return {
provider: match[2] as ByokProvider,
source: keyId.startsWith('local-')
? ByokProviderSource.Local
: ByokProviderSource.Server,
keyId: keyId.startsWith('local-') ? undefined : keyId,
};
}
private toKeyConfig(row: {
id: string;
provider: string;
name: string;
description: string | null;
endpoint: string | null;
sortOrder: number;
enabled: boolean;
disabledReason: string | null;
lastValidatedAt: Date | null;
lastValidationError: string | null;
lastUsedAt: Date | null;
lastErrorAt: Date | null;
lastError: string | null;
}): ByokKeyConfig {
const provider = row.provider as ByokProvider;
return {
id: row.id,
provider,
name: row.name,
description: row.description,
storage: ByokKeyStorage.server,
configured: true,
enabled: row.enabled,
endpoint: row.endpoint,
endpointEditable: this.customEndpointSupported,
sortOrder: row.sortOrder,
capabilities: this.capabilities(provider, 'server'),
testStatus: row.lastValidationError
? ByokKeyTestStatus.failed
: row.lastValidatedAt
? ByokKeyTestStatus.passed
: ByokKeyTestStatus.untested,
disabledReason: row.disabledReason,
lastTestedAt: row.lastValidatedAt,
lastTestError: row.lastValidationError,
lastUsedAt: row.lastUsedAt,
lastErrorAt: row.lastErrorAt,
lastError: row.lastError,
};
}
private capabilities(provider: ByokProvider, storage: 'server' | 'local') {
switch (provider) {
case ByokProvider.openai:
return ['Text', 'Image input', 'Actions', 'Image generate'];
case ByokProvider.anthropic:
return ['Text', 'Image input'];
case ByokProvider.gemini:
return storage === 'server'
? [
'Text',
'Image input',
'Actions',
'Image generate',
'Transcript',
'Indexing',
]
: ['Text', 'Image input', 'Actions', 'Image generate'];
case ByokProvider.fal:
return ['Image generate'];
}
}
private buildWarnings(keys: ByokKeyConfig[]) {
const activeServerGemini = keys.some(
key =>
key.provider === ByokProvider.gemini &&
key.storage === ByokKeyStorage.server &&
key.enabled
);
if (activeServerGemini) {
return [];
}
return [
{
featureKind: 'transcript',
reason:
'Transcript and workspace indexing require a server Gemini BYOK key or AFFiNE AI plan fallback.',
requiredProviders: [ByokProvider.gemini],
},
{
featureKind: 'workspace_indexing',
reason:
'Workspace indexing requires a server Gemini BYOK key or AFFiNE AI plan fallback.',
requiredProviders: [ByokProvider.gemini],
},
];
}
private normalizeEndpoint(endpoint?: string | null) {
if (!endpoint) return null;
if (!this.customEndpointSupported) {
throw new BadRequestException('Custom BYOK endpoint is not supported.');
}
let parsed: URL;
try {
parsed = new URL(endpoint);
} catch {
throw new BadRequestException('Invalid BYOK endpoint.');
}
if (!['https:', 'http:'].includes(parsed.protocol)) {
throw new BadRequestException('BYOK endpoint must use HTTP or HTTPS.');
}
return parsed.toString().replace(/\/$/, '');
}
private assertProvider(provider: ByokProvider) {
if (!BYOK_ALLOWED_PROVIDERS.includes(provider)) {
throw new BadRequestException('Unsupported BYOK provider.');
}
}
private sanitizeError(error: unknown) {
if (error instanceof Error && error.name === 'AbortError') {
return 'Provider key test timed out.';
}
if (error instanceof BadRequestException && error.message) {
return error.message.slice(0, 300);
}
return 'Provider request failed.';
}
private workspaceHash(workspaceId: string) {
return createHash('sha256').update(workspaceId).digest('hex').slice(0, 12);
}
private leaseCacheKey(leaseId: string) {
return `copilot:byok:lease:${leaseId}`;
}
private async getActiveLocalLease(activeCacheKey: string) {
const active = await this.cache.get<LocalLeaseActive>(activeCacheKey);
if (!active) return null;
if (await this.cache.has(this.leaseCacheKey(active.leaseId))) {
return { leaseId: active.leaseId, expiresAt: new Date(active.expiresAt) };
}
await this.cache.delete(activeCacheKey);
return null;
}
private localLeaseActiveCacheKey(input: {
workspaceId: string;
userId: string;
providers: ByokLocalLeaseProvider[];
}) {
const fingerprint = createHmac(
'sha256',
this.crypto.keyPair.sha256.privateKey
)
.update(
JSON.stringify(
input.providers.map(provider => ({
provider: provider.provider,
name: provider.name,
description: provider.description ?? null,
apiKey: provider.apiKey,
endpoint: provider.endpoint ?? null,
sortOrder: provider.sortOrder ?? 0,
enabled: provider.enabled ?? true,
}))
)
)
.digest('hex');
return `copilot:byok:lease:active:${input.workspaceId}:${input.userId}:${fingerprint}`;
}
}
@@ -1,5 +1,5 @@
import { Injectable } from '@nestjs/common';
import { AiPromptRole } from '@prisma/client';
import { AiSessionMessageRole } from '@prisma/client';
import type { Conversation, Turn } from '../core';
import { chatMessageFromTurn } from '../core';
@@ -16,7 +16,6 @@ export type CanonicalConversationHistory = {
conversation: Conversation;
turns: Turn[];
prompt: ResolvedPrompt;
tokenCost: number;
};
export type CanonicalConversationMeta = Omit<
@@ -35,7 +34,7 @@ export class CompatHistoryProjector {
private projectSessionBase(
history: CanonicalConversationMeta
): Omit<ChatHistory, 'messages'> {
const { conversation, prompt, tokenCost } = history;
const { conversation, prompt } = history;
return {
userId: conversation.userId,
sessionId: conversation.id,
@@ -45,10 +44,7 @@ export class CompatHistoryProjector {
pinned: conversation.pinned,
title: conversation.title,
action: prompt.action || null,
model: prompt.model,
optionalModels: prompt.optionalModels || [],
promptName: prompt.name,
tokens: tokenCost,
createdAt: conversation.createdAt,
updatedAt: conversation.updatedAt,
};
@@ -84,7 +80,7 @@ export class CompatHistoryProjector {
.concat(messages)
.filter(
message =>
message.role !== AiPromptRole.user ||
message.role !== AiSessionMessageRole.user ||
!!message.content.trim() ||
!!message.attachments?.length
)
@@ -1,5 +1,5 @@
import { Injectable } from '@nestjs/common';
import { AiPromptRole } from '@prisma/client';
import { AiSessionMessageRole } from '@prisma/client';
import { PromptService } from '../prompt/service';
import type { ChatMessage } from '../types';
@@ -24,7 +24,9 @@ export class HistoryPromptPreloadProjector {
history.turns[0] ? history.turns[0].metadata : {},
history.conversation.id
)
.filter(({ role }) => role !== AiPromptRole.system) as ChatMessage[];
.filter(
({ role }) => role !== AiSessionMessageRole.system
) as ChatMessage[];
preload.forEach((message, index) => {
message.createdAt = new Date(
@@ -5,32 +5,9 @@ import {
StorageJSONSchema,
StorageProviderConfig,
} from '../../base';
import {
AnthropicOfficialConfig,
AnthropicVertexConfig,
} from './providers/anthropic';
import { CloudflareWorkersAIConfig } from './providers/cloudflare';
import type { FalConfig } from './providers/fal';
import { GeminiGenerativeConfig, GeminiVertexConfig } from './providers/gemini';
import { OpenAIConfig } from './providers/openai';
import {
CopilotProviderType,
ModelOutputType,
VertexSchema,
} from './providers/types';
import { CopilotProviderType } from './providers/types';
export type CopilotProviderConfigMap = {
[CopilotProviderType.OpenAI]: OpenAIConfig;
[CopilotProviderType.CloudflareWorkersAi]: CloudflareWorkersAIConfig;
[CopilotProviderType.FAL]: FalConfig;
[CopilotProviderType.Gemini]: GeminiGenerativeConfig;
[CopilotProviderType.GeminiVertex]: GeminiVertexConfig;
[CopilotProviderType.Anthropic]: AnthropicOfficialConfig;
[CopilotProviderType.AnthropicVertex]: AnthropicVertexConfig;
};
export type ProviderSpecificConfig =
CopilotProviderConfigMap[keyof CopilotProviderConfigMap];
export type ProviderSpecificConfig = Record<string, unknown>;
export const RustRequestMiddlewareValues = [
'normalize_messages',
@@ -65,24 +42,13 @@ type CopilotProviderProfileCommon = {
displayName?: string;
priority?: number;
enabled?: boolean;
models?: string[];
models: string[];
middleware?: ProviderMiddlewareConfig;
};
type CopilotProviderProfileVariant<T extends CopilotProviderType> = {
type: T;
config: CopilotProviderConfigMap[T];
};
export type CopilotProviderProfile = CopilotProviderProfileCommon &
{
[Type in CopilotProviderType]: CopilotProviderProfileVariant<Type>;
}[CopilotProviderType];
export type CopilotProviderDefaults = Partial<
Record<Exclude<ModelOutputType, typeof ModelOutputType.Rerank>, string>
> & {
fallback?: string;
export type CopilotProviderProfile = CopilotProviderProfileCommon & {
type: CopilotProviderType;
config: ProviderSpecificConfig;
};
const CopilotProviderProfileBaseShape = z.object({
@@ -90,7 +56,7 @@ const CopilotProviderProfileBaseShape = z.object({
displayName: z.string().optional(),
priority: z.number().optional(),
enabled: z.boolean().optional(),
models: z.array(z.string()).optional(),
models: z.array(z.string().min(1)).min(1),
middleware: z
.object({
rust: z
@@ -106,79 +72,9 @@ const CopilotProviderProfileBaseShape = z.object({
.optional(),
});
const OpenAIConfigShape = z.object({
apiKey: z.string(),
baseURL: z.string().optional(),
oldApiStyle: z.boolean().optional(),
});
const FalConfigShape = z.object({
apiKey: z.string(),
});
const CloudflareWorkersAIConfigShape = z.object({
apiToken: z.string(),
accountId: z.string().optional(),
baseURL: z.string().optional(),
});
const GeminiGenerativeConfigShape = z.object({
apiKey: z.string(),
baseURL: z.string().optional(),
});
const VertexProviderConfigShape = z.object({
location: z.string().optional(),
project: z.string().optional(),
baseURL: z.string().optional(),
googleAuthOptions: z.any().optional(),
fetch: z.any().optional(),
});
const AnthropicOfficialConfigShape = z.object({
apiKey: z.string(),
baseURL: z.string().optional(),
});
const CopilotProviderProfileShape = z.discriminatedUnion('type', [
CopilotProviderProfileBaseShape.extend({
type: z.literal(CopilotProviderType.OpenAI),
config: OpenAIConfigShape,
}),
CopilotProviderProfileBaseShape.extend({
type: z.literal(CopilotProviderType.FAL),
config: FalConfigShape,
}),
CopilotProviderProfileBaseShape.extend({
type: z.literal(CopilotProviderType.CloudflareWorkersAi),
config: CloudflareWorkersAIConfigShape,
}),
CopilotProviderProfileBaseShape.extend({
type: z.literal(CopilotProviderType.Gemini),
config: GeminiGenerativeConfigShape,
}),
CopilotProviderProfileBaseShape.extend({
type: z.literal(CopilotProviderType.GeminiVertex),
config: VertexProviderConfigShape,
}),
CopilotProviderProfileBaseShape.extend({
type: z.literal(CopilotProviderType.Anthropic),
config: AnthropicOfficialConfigShape,
}),
CopilotProviderProfileBaseShape.extend({
type: z.literal(CopilotProviderType.AnthropicVertex),
config: VertexProviderConfigShape,
}),
]);
const CopilotProviderDefaultsShape = z.object({
[ModelOutputType.Text]: z.string().optional(),
[ModelOutputType.Object]: z.string().optional(),
[ModelOutputType.Embedding]: z.string().optional(),
[ModelOutputType.Image]: z.string().optional(),
[ModelOutputType.Rerank]: z.string().optional(),
[ModelOutputType.Structured]: z.string().optional(),
fallback: z.string().optional(),
const CopilotProviderProfileShape = CopilotProviderProfileBaseShape.extend({
type: z.nativeEnum(CopilotProviderType),
config: z.record(z.string(), z.unknown()),
});
declare global {
@@ -202,14 +98,6 @@ declare global {
storage: ConfigItem<StorageProviderConfig>;
providers: {
profiles: ConfigItem<CopilotProviderProfile[]>;
defaults: ConfigItem<CopilotProviderDefaults>;
openai: ConfigItem<OpenAIConfig>;
cloudflareWorkersAi: ConfigItem<CloudflareWorkersAIConfig>;
fal: ConfigItem<FalConfig>;
gemini: ConfigItem<GeminiGenerativeConfig>;
geminiVertex: ConfigItem<GeminiVertexConfig>;
anthropic: ConfigItem<AnthropicOfficialConfig>;
anthropicVertex: ConfigItem<AnthropicVertexConfig>;
};
};
}
@@ -245,56 +133,6 @@ defineModuleConfig('copilot', {
default: [],
shape: z.array(CopilotProviderProfileShape),
},
'providers.defaults': {
desc: 'The default provider ids for model output types and global fallback.',
default: {},
shape: CopilotProviderDefaultsShape,
},
'providers.openai': {
desc: 'The config for the openai provider.',
default: {
apiKey: '',
baseURL: 'https://api.openai.com/v1',
},
link: 'https://github.com/openai/openai-node',
},
'providers.cloudflareWorkersAi': {
desc: 'The config for the Cloudflare Workers AI provider.',
default: {
apiToken: '',
accountId: '',
},
},
'providers.fal': {
desc: 'The config for the fal provider.',
default: {
apiKey: '',
},
},
'providers.gemini': {
desc: 'The config for the gemini provider.',
default: {
apiKey: '',
baseURL: 'https://generativelanguage.googleapis.com/v1beta',
},
},
'providers.geminiVertex': {
desc: 'The config for the gemini provider in Google Vertex AI.',
default: {},
schema: VertexSchema,
},
'providers.anthropic': {
desc: 'The config for the anthropic provider.',
default: {
apiKey: '',
baseURL: 'https://api.anthropic.com/v1',
},
},
'providers.anthropicVertex': {
desc: 'The config for the anthropic provider in Google Vertex AI.',
default: {},
schema: VertexSchema,
},
unsplash: {
desc: 'The config for the unsplash key.',
default: {
@@ -1,7 +1,8 @@
import { Injectable, OnModuleInit } from '@nestjs/common';
import { z } from 'zod';
import { OnEvent } from '../../../base';
import { Config } from '../../../base/config';
import { OnEvent } from '../../../base/event';
import { PermissionAccess } from '../../../core/permission';
import {
RealtimePublisher,
@@ -10,6 +11,7 @@ import {
registerRealtimeLiveQuery,
} from '../../../core/realtime';
import { Models } from '../../../models';
import { assertCopilotEnabled } from '../availability';
export function workspaceEmbeddingRoom(workspaceId: string) {
return realtimeWorkspaceEmbeddingProgressRoom(workspaceId);
@@ -21,7 +23,8 @@ export class CopilotEmbeddingRealtimeProvider implements OnModuleInit {
private readonly ac: PermissionAccess,
private readonly models: Models,
private readonly registry: RealtimeRegistry,
private readonly publisher: RealtimePublisher
private readonly publisher: RealtimePublisher,
private readonly config: Config
) {}
onModuleInit() {
@@ -118,6 +121,7 @@ export class CopilotEmbeddingRealtimeProvider implements OnModuleInit {
}
private async assertCopilot(userId: string, workspaceId: string) {
assertCopilotEnabled(this.config);
await this.ac
.user(userId)
.workspace(workspaceId)
@@ -50,6 +50,7 @@ import {
Models,
} from '../../../models';
import { CopilotEmbeddingJob } from '../embedding/job';
import { CopilotEnabled } from '../feature';
import { COPILOT_LOCKER, CopilotType } from '../resolver';
import { ChatSessionService } from '../session';
import { CopilotStorage } from '../storage';
@@ -286,6 +287,7 @@ class ContextMatchedDocChunk implements DocChunkSimilarity {
}
@Throttle()
@CopilotEnabled()
@Resolver(() => CopilotType)
export class CopilotContextRootResolver {
constructor(
@@ -435,6 +437,7 @@ export class CopilotContextRootResolver {
}
@Throttle()
@CopilotEnabled()
@Resolver(() => CopilotContextType)
export class CopilotContextResolver {
constructor(
@@ -1,3 +1,4 @@
/* oxlint-disable import/no-cycle -- Context embedding reuses the shared capability runtime. */
import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
import {
@@ -36,6 +36,7 @@ import {
UnsplashIsNotConfigured,
} from '../../base';
import { CurrentUser, Public } from '../../core/auth';
import { CopilotEnabled } from './feature';
import {
ActionStreamHost,
projectActionEventToChatEvent,
@@ -52,6 +53,7 @@ export interface ChatEvent {
const PING_INTERVAL = 5000;
@CopilotEnabled()
@Controller('/api/copilot')
export class CopilotController implements BeforeApplicationShutdown {
private readonly logger = new Logger(CopilotController.name);
@@ -83,7 +83,6 @@ export class ConversationStore {
conversation: Conversation;
turns: Turn[];
promptName: string;
tokenCost: number;
}
| undefined
> {
@@ -96,7 +95,6 @@ export class ConversationStore {
conversation: this.toConversation(session),
turns: this.toTurns(session),
promptName: session.promptName,
tokenCost: session.tokenCost,
};
}
@@ -104,7 +102,6 @@ export class ConversationStore {
| {
conversation: Conversation;
promptName: string;
tokenCost: number;
}
| undefined
> {
@@ -124,7 +121,6 @@ export class ConversationStore {
updatedAt: session.updatedAt,
},
promptName: session.promptName,
tokenCost: session.tokenCost,
};
}
@@ -146,7 +142,6 @@ export class ConversationStore {
turnFromChatMessage(message, session.id)
),
promptName: session.promptName,
tokenCost: session.tokenCost,
}));
}
@@ -168,14 +163,12 @@ export class ConversationStore {
updatedAt: session.updatedAt,
} satisfies Conversation,
promptName: session.promptName,
tokenCost: session.tokenCost,
}));
}
async appendTurns(input: {
sessionId: string;
userId: string;
prompt: { model: string };
turns: Turn[];
}) {
return await this.models.copilotSession.updateMessages({
@@ -190,14 +183,12 @@ export class ConversationStore {
async appendTurn(input: {
sessionId: string;
userId: string;
prompt: { model: string };
turn: Turn;
compatSubmissionId?: string;
}) {
const message = await this.models.copilotSession.appendMessage({
sessionId: input.sessionId,
userId: input.userId,
prompt: input.prompt,
message: (() => {
const { id: _id, ...message } = chatMessageFromTurn(input.turn);
return { ...message, compatSubmissionId: input.compatSubmissionId };
@@ -5,6 +5,7 @@ import { JOB_SIGNAL, JobQueue, OneDay, OnJob } from '../../base';
import { Models } from '../../models';
const CLEANUP_EMBEDDING_JOB_BATCH_SIZE = 100;
const BACKGROUND_COPILOT_JOB_PRIORITY = 100;
declare global {
interface Jobs {
@@ -71,9 +72,11 @@ export class CopilotCronJobs {
const sessions = await this.models.copilotSession.toBeGenerateTitle();
for (const session of sessions) {
await this.jobs.add('copilot.session.generateTitle', {
sessionId: session.id,
});
await this.jobs.add(
'copilot.session.generateTitle',
{ sessionId: session.id },
{ priority: BACKGROUND_COPILOT_JOB_PRIORITY }
);
}
this.logger.log(
`Scheduled title generation for ${sessions.length} sessions`
@@ -1,6 +1,7 @@
/* oxlint-disable import/no-cycle -- Embedding delegates to the shared capability runtime. */
import { createHash } from 'node:crypto';
import { Injectable, Logger } from '@nestjs/common';
import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common';
import { CopilotFailedToGenerateEmbedding } from '../../../base/error/errors.gen';
import {
@@ -10,7 +11,6 @@ import {
} from '../../../models';
import { type CopilotRerankRequest } from '../providers/types';
import { CapabilityRuntime } from '../runtime/capability-runtime';
import { TaskPolicy } from '../runtime/task-policy';
import {
type EmbeddingCallOptionsInput,
EmbeddingClient,
@@ -18,20 +18,20 @@ import {
type ReRankResult,
} from './types';
type EmbeddingRuntime = Pick<
CapabilityRuntime,
'embeddingConfigured' | 'embed' | 'rerank'
>;
class ProductionEmbeddingClient extends EmbeddingClient {
private readonly logger = new Logger(ProductionEmbeddingClient.name);
constructor(
private readonly taskPolicy: TaskPolicy,
private readonly runtime: CapabilityRuntime
) {
constructor(private readonly runtime: EmbeddingRuntime) {
super();
}
override async configured(): Promise<boolean> {
const result = await this.runtime.embeddingConfigured(
this.taskPolicy.resolveEmbeddingModelId()
);
const result = await this.runtime.embeddingConfigured('route-selected');
if (!result) {
this.logger.warn(
'Copilot embedding client is not configured properly, please check your configuration.'
@@ -45,7 +45,7 @@ class ProductionEmbeddingClient extends EmbeddingClient {
options?: EmbeddingCallOptionsInput
): Promise<Embedding[]> {
const normalizedOptions = normalizeEmbeddingCallOptions(options);
const modelId = this.taskPolicy.resolveEmbeddingModelId();
const modelId = 'route-selected';
const embeddings = await this.runtime.embed(modelId, input, {
dimensions: EMBEDDING_DIMENSIONS,
signal: normalizedOptions.signal,
@@ -94,17 +94,13 @@ class ProductionEmbeddingClient extends EmbeddingClient {
})),
};
const ranks = await this.runtime.rerank(
this.taskPolicy.resolveRerankModelId(),
rerankRequest,
{
signal: normalizedOptions.signal,
user: normalizedOptions.userId,
workspace: normalizedOptions.workspaceId,
byokLeaseId: normalizedOptions.byokLeaseId,
featureKind: 'rerank',
}
);
const ranks = await this.runtime.rerank('route-selected', rerankRequest, {
signal: normalizedOptions.signal,
user: normalizedOptions.userId,
workspace: normalizedOptions.workspaceId,
byokLeaseId: normalizedOptions.byokLeaseId,
featureKind: 'rerank',
});
try {
return ranks.map((score, i) => {
@@ -206,12 +202,12 @@ export class CopilotEmbeddingClientService {
private client: EmbeddingClient | undefined;
constructor(
private readonly taskPolicy: TaskPolicy,
private readonly runtime: CapabilityRuntime
@Inject(forwardRef(() => CapabilityRuntime))
private readonly runtime: EmbeddingRuntime
) {}
async refresh() {
const client = new ProductionEmbeddingClient(this.taskPolicy, this.runtime);
const client = new ProductionEmbeddingClient(this.runtime);
await client.configured();
this.client = client;
return this.client;
@@ -0,0 +1,54 @@
import { CanActivate, Injectable, UseGuards } from '@nestjs/common';
import { Config } from '../../base/config';
import { OnEvent } from '../../base/event';
import { ServerFeature, ServerService } from '../../core/config';
import { assertCopilotEnabled } from './availability';
@Injectable()
export class CopilotFeatureService {
constructor(
private readonly config: Config,
private readonly server: ServerService
) {}
get enabled() {
return this.config.copilot.enabled;
}
@OnEvent('config.init')
onConfigInit() {
this.syncServerFeature();
}
@OnEvent('config.changed')
onConfigChanged(event: Events['config.changed']) {
if ('copilot' in event.updates) {
this.syncServerFeature();
}
}
assertEnabled() {
assertCopilotEnabled(this.config);
}
private syncServerFeature() {
if (this.enabled) {
this.server.enableFeature(ServerFeature.Copilot);
} else {
this.server.disableFeature(ServerFeature.Copilot);
}
}
}
@Injectable()
export class CopilotFeatureGuard implements CanActivate {
constructor(private readonly feature: CopilotFeatureService) {}
canActivate() {
this.feature.assertEnabled();
return true;
}
}
export const CopilotEnabled = () => UseGuards(CopilotFeatureGuard);
@@ -11,6 +11,7 @@ import { StorageModule } from '../../core/storage';
import { WorkspaceModule } from '../../core/workspaces';
import { IndexerModule } from '../indexer';
import { CopilotController } from './controller';
import { CopilotFeatureGuard, CopilotFeatureService } from './feature';
import { WorkspaceMcpController } from './mcp/controller';
import { McpCredentialService } from './mcp/credential';
import { McpCredentialResolver } from './mcp/resolver';
@@ -34,20 +35,27 @@ const COPILOT_SHARED_IMPORTS = [
];
@Module({
imports: [...COPILOT_SHARED_IMPORTS],
imports: [ServerConfigModule],
providers: [CopilotFeatureService, CopilotFeatureGuard],
exports: [CopilotFeatureService, CopilotFeatureGuard],
})
export class CopilotAvailabilityModule {}
@Module({
imports: [...COPILOT_SHARED_IMPORTS, CopilotAvailabilityModule],
providers: [...COPILOT_KERNEL_PROVIDERS],
exports: [...COPILOT_KERNEL_PROVIDERS],
exports: [CopilotAvailabilityModule, ...COPILOT_KERNEL_PROVIDERS],
})
export class CopilotKernelModule {}
@Module({
imports: [PermissionModule],
imports: [PermissionModule, CopilotAvailabilityModule],
providers: [...COPILOT_TRANSCRIPT_REALTIME_PROVIDERS],
})
export class CopilotRealtimeModule {}
@Module({
imports: [PermissionModule],
imports: [PermissionModule, CopilotAvailabilityModule],
providers: [...COPILOT_CONTEXT_REALTIME_PROVIDERS],
})
export class CopilotEmbeddingRealtimeModule {}
@@ -16,6 +16,7 @@ import type { Request, Response } from 'express';
import { ActionForbidden, Throttle } from '../../../base';
import { Public } from '../../../core/auth';
import { extractTokenFromHeader } from '../../../core/auth/input';
import { CopilotEnabled } from '../feature';
import { McpCredentialService } from './credential';
import { WorkspaceMcpProvider, type WorkspaceMcpServer } from './provider';
@@ -46,6 +47,7 @@ const SUPPORTED_PROTOCOL_VERSIONS = new Set([
'2024-10-07',
]);
@CopilotEnabled()
@Controller('/api/workspaces/:workspaceId/mcp')
export class WorkspaceMcpController {
private readonly logger = new Logger(WorkspaceMcpController.name);
@@ -15,6 +15,7 @@ import { McpAccessMode } from '@prisma/client';
import { CurrentUser } from '../../../core/auth';
import { PermissionAccess } from '../../../core/permission';
import { CopilotEnabled } from '../feature';
import { McpCredentialService } from './credential';
registerEnumType(McpAccessMode, { name: 'McpAccessMode' });
@@ -89,6 +90,7 @@ class CreateMcpCredentialInput {
expirationDays!: number;
}
@CopilotEnabled()
@Resolver()
export class McpCredentialResolver {
constructor(
@@ -1,9 +1,4 @@
import { CopilotAccessPolicy } from './access';
import {
ByokEntitlementPolicy,
ByokService,
WorkspaceByokResolver,
} from './byok';
import { ByokEntitlementPolicy, WorkspaceByokResolver } from './byok';
import { HistoryAttachmentUrlProjector } from './compat/history-attachment-url-projector';
import { CompatHistoryProjector } from './compat/history-projector';
import { HistoryPromptPreloadProjector } from './compat/history-prompt-preload-projector';
@@ -25,30 +20,18 @@ import {
} from './embedding';
import { WorkspaceMcpProvider } from './mcp/provider';
import { PromptService } from './prompt';
import {
CopilotProviderFactory,
CopilotProviderLifecycleService,
CopilotProviderRegistryService,
CopilotProviders,
} from './providers';
import { CopilotResolver, UserCopilotResolver } from './resolver';
import { ActionRuntimeBridge } from './runtime/action-runtime-bridge';
import { CapabilityRuntime } from './runtime/capability-runtime';
import { CopilotExecutionMetrics } from './runtime/execution-metrics';
import { ExecutionPlanBuilder } from './runtime/execution-plan';
import { CopilotRuntimeEventConsumer } from './runtime/copilot-runtime-event-consumer';
import { ActionStreamHost } from './runtime/hosts/action-stream-host';
import { AttachmentAdmissionHost } from './runtime/hosts/attachment-admission';
import { AttachmentMaterializer } from './runtime/hosts/attachment-materializer';
import { CapabilityPolicyHost } from './runtime/hosts/capability-policy-host';
import { ConversationHost } from './runtime/hosts/conversation-host';
import { ImageResultHost } from './runtime/hosts/image-result-host';
import { ResponsePostprocessor } from './runtime/hosts/response-postprocessor';
import { ToolExecutorHost } from './runtime/hosts/tool-executor-host';
import { TurnPersistence } from './runtime/hosts/turn-persistence';
import { ModelSelectionPolicy } from './runtime/model-selection-policy';
import { NativeExecutionEngine } from './runtime/native-execution-engine';
import { PromptRuntime } from './runtime/prompt-runtime';
import { TaskPolicy } from './runtime/task-policy';
import { ToolRuntime } from './runtime/tool-runtime';
import { TurnOrchestrator } from './runtime/turn-orchestrator';
import { ChatSessionService } from './session';
@@ -65,21 +48,14 @@ import {
CopilotWorkspaceService,
} from './workspace';
export const COPILOT_PROVIDER_PROVIDERS = [
...CopilotProviders,
CopilotProviderRegistryService,
CopilotProviderFactory,
CopilotProviderLifecycleService,
];
export const COPILOT_PROVIDER_PROVIDERS: [] = [];
export const COPILOT_RUNTIME_PROVIDERS = [
ByokEntitlementPolicy,
ByokService,
ChatSessionService,
ConversationStore,
ConversationInboxService,
ConversationPolicy,
CopilotAccessPolicy,
HistoryAttachmentUrlProjector,
CompatHistoryProjector,
HistoryPromptPreloadProjector,
@@ -88,18 +64,12 @@ export const COPILOT_RUNTIME_PROVIDERS = [
CopilotContextService,
CopilotEmbeddingClientService,
PromptService,
ModelSelectionPolicy,
ActionRuntimeBridge,
CopilotExecutionMetrics,
ExecutionPlanBuilder,
CopilotRuntimeEventConsumer,
PromptRuntime,
CapabilityPolicyHost,
ConversationHost,
CapabilityRuntime,
NativeExecutionEngine,
TaskPolicy,
ToolRuntime,
ToolExecutorHost,
AttachmentMaterializer,
AttachmentAdmissionHost,
ActionStreamHost,
@@ -1,36 +1,17 @@
import {
llmCollectPromptMetadata,
llmCountPromptTokens,
llmGetBuiltInPromptSpec,
llmListBuiltInPromptSpecs,
llmRenderBuiltInPrompt,
llmRenderBuiltInSessionPrompt,
llmRenderPrompt,
llmRenderSessionPrompt,
type NativeBuiltInPromptRenderRequest as NativeBuiltInPromptRenderContract,
type NativeBuiltInPromptSessionRenderRequest as NativeBuiltInPromptSessionContract,
type NativePromptCountTokensRequest as NativePromptTokenCountContract,
type NativePromptCountTokensResponse as NativePromptTokenCountResult,
type NativePromptMetadataRequest as NativePromptMetadataContract,
type NativePromptMetadataResponse as NativePromptMetadataResult,
type NativePromptRenderRequest as NativePromptRenderContract,
type NativePromptRenderResponse as NativePromptRenderResult,
type NativePromptSessionRenderRequest as NativePromptSessionContract,
type NativePromptSessionRenderResponse as NativePromptSessionResult,
} from '../../../native';
import type { PromptMessage, PromptParams } from '../providers/types';
import { projectPromptMessageForNative } from '../runtime/contracts';
import type { PromptSpec } from './spec';
export type NativePromptRenderRequest = Omit<
NativePromptRenderContract,
'messages' | 'templateParams' | 'renderParams'
> & {
messages: PromptMessage[];
templateParams: PromptParams;
renderParams: PromptParams;
};
export type NativePromptRenderResponse = Omit<
NativePromptRenderResult,
'messages'
@@ -45,46 +26,6 @@ export type NativeBuiltInPromptRenderRequest = Omit<
renderParams: PromptParams;
};
export type NativePromptCountTokensRequest = Omit<
NativePromptTokenCountContract,
'messages' | 'model'
> & {
model?: string | null;
messages: Pick<PromptMessage, 'content'>[];
};
export type NativePromptCountTokensResponse = NativePromptTokenCountResult;
export type NativePromptMetadataRequest = Omit<
NativePromptMetadataContract,
'messages'
> & {
messages: PromptMessage[];
};
export type NativePromptMetadataResponse = Omit<
NativePromptMetadataResult,
'templateParams'
> & {
templateParams: PromptParams;
};
export type NativePromptSessionRenderRequest = Omit<
NativePromptSessionContract,
'prompt' | 'turns' | 'renderParams'
> & {
prompt: Omit<
NativePromptSessionContract['prompt'],
'templateParams' | 'messages' | 'model'
> & {
model?: string | null;
templateParams: PromptParams;
messages: PromptMessage[];
};
turns: PromptMessage[];
renderParams: PromptParams;
};
export type NativePromptSessionRenderResponse = Omit<
NativePromptSessionResult,
'messages'
@@ -100,8 +41,7 @@ export type NativeBuiltInPromptSessionRenderRequest = Omit<
renderParams: PromptParams;
};
type NativePromptContractMessage =
NativePromptRenderContract['messages'][number];
type NativePromptContractMessage = NativePromptRenderResult['messages'][number];
function toNativePromptMessage(
message: PromptMessage
@@ -123,22 +63,6 @@ function fromNativePromptMessage(
};
}
export function renderPromptNative(
request: NativePromptRenderRequest
): NativePromptRenderResponse {
const normalizedMessages = request.messages.map(toNativePromptMessage);
const rendered = llmRenderPrompt({
messages: normalizedMessages,
templateParams: request.templateParams,
renderParams: request.renderParams,
});
return {
...rendered,
messages: rendered.messages.map(fromNativePromptMessage),
};
}
export function renderBuiltInPromptNative(
request: NativeBuiltInPromptRenderRequest
): NativePromptRenderResponse {
@@ -153,25 +77,6 @@ export function renderBuiltInPromptNative(
};
}
export function renderPromptSessionNative(
request: NativePromptSessionRenderRequest
): NativePromptSessionRenderResponse {
const rendered = llmRenderSessionPrompt({
...request,
prompt: {
...request.prompt,
messages: request.prompt.messages.map(toNativePromptMessage),
model: request.prompt.model ?? undefined,
},
turns: request.turns.map(toNativePromptMessage),
renderParams: request.renderParams,
});
return {
...rendered,
messages: rendered.messages.map(fromNativePromptMessage),
};
}
export function renderBuiltInPromptSessionNative(
request: NativeBuiltInPromptSessionRenderRequest
): NativePromptSessionRenderResponse {
@@ -187,29 +92,10 @@ export function renderBuiltInPromptSessionNative(
};
}
export function countPromptTokensNative(
request: NativePromptCountTokensRequest
): NativePromptCountTokensResponse {
return llmCountPromptTokens({
...request,
model: request.model ?? undefined,
});
}
export function collectPromptMetadataNative(
request: NativePromptMetadataRequest
): NativePromptMetadataResponse {
return llmCollectPromptMetadata({
messages: request.messages.map(toNativePromptMessage),
});
}
export function listBuiltInPromptSpecsNative(): PromptSpec[] {
return llmListBuiltInPromptSpecs().map(spec => ({
name: spec.name,
action: spec.action,
model: spec.model,
optionalModels: spec.optionalModels,
config: spec.config,
params: spec.params
? Object.fromEntries(
@@ -238,8 +124,6 @@ export function getBuiltInPromptSpecNative(name: string): PromptSpec | null {
return {
name: spec.name,
action: spec.action,
model: spec.model,
optionalModels: spec.optionalModels,
config: spec.config,
params: spec.params
? Object.fromEntries(
@@ -2,15 +2,11 @@ import { Injectable, Logger } from '@nestjs/common';
import type { PromptMessage, PromptParams } from '../providers/types';
import {
collectPromptMetadataNative,
countPromptTokensNative,
getBuiltInPromptSpecNative,
renderBuiltInPromptNative,
renderBuiltInPromptSessionNative,
renderPromptNative,
renderPromptSessionNative,
} from './native-contract';
import type { Prompt, PromptSpec, ResolvedPrompt } from './spec';
import type { PromptSpec, ResolvedPrompt } from './spec';
@Injectable()
export class PromptService {
@@ -20,11 +16,6 @@ export class PromptService {
}
async get(name: string): Promise<ResolvedPrompt | null> {
const compatPrompt = this.lookupCompatPrompt(name);
if (compatPrompt) {
return this.describeCompatPrompt(this.clonePrompt(compatPrompt));
}
const builtInPromptSpec = this.lookupBuiltInPromptSpec(name);
if (!builtInPromptSpec) return null;
@@ -36,17 +27,10 @@ export class PromptService {
params: PromptParams,
sessionId?: string
): PromptMessage[] {
const rendered =
prompt.source === 'built_in'
? renderBuiltInPromptNative({
name: prompt.name,
renderParams: params,
})
: renderPromptNative({
messages: this.requireCompatMessages(prompt),
templateParams: prompt.params,
renderParams: params,
});
const rendered = renderBuiltInPromptNative({
name: prompt.name,
renderParams: params,
});
this.logWarnings(rendered.warnings, sessionId);
return rendered.messages;
@@ -56,38 +40,18 @@ export class PromptService {
prompt: ResolvedPrompt,
turns: PromptMessage[],
params: PromptParams,
maxTokenSize = prompt.config?.maxTokens || 128 * 1024,
sessionId?: string
): PromptMessage[] {
const rendered =
prompt.source === 'built_in'
? renderBuiltInPromptSessionNative({
name: prompt.name,
turns,
renderParams: params,
maxTokenSize,
})
: renderPromptSessionNative({
prompt: {
action: prompt.action,
model: prompt.model,
promptTokens: this.countCompatPromptTokens(prompt),
templateParams: prompt.params,
messages: this.requireCompatMessages(prompt),
},
turns,
renderParams: params,
maxTokenSize,
});
const rendered = renderBuiltInPromptSessionNative({
name: prompt.name,
turns,
renderParams: params,
});
this.logWarnings(rendered.warnings, sessionId);
return rendered.messages;
}
protected lookupCompatPrompt(_name: string): Prompt | null {
return null;
}
protected lookupBuiltInPromptSpec(name: string): PromptSpec | null {
const spec = getBuiltInPromptSpecNative(name);
return spec ? this.clonePromptSpec(spec) : null;
@@ -104,23 +68,9 @@ export class PromptService {
}));
}
protected clonePrompt(prompt: Prompt): Prompt {
return {
...prompt,
optionalModels: prompt.optionalModels
? [...prompt.optionalModels]
: undefined,
config: prompt.config ? structuredClone(prompt.config) : undefined,
messages: this.cloneMessages(prompt.messages),
};
}
protected clonePromptSpec(spec: PromptSpec): PromptSpec {
return {
...spec,
optionalModels: spec.optionalModels
? [...spec.optionalModels]
: undefined,
config: spec.config ? structuredClone(spec.config) : undefined,
params: spec.params ? structuredClone(spec.params) : undefined,
messages: spec.messages.map(message => ({ ...message })),
@@ -132,27 +82,9 @@ export class PromptService {
return {
name: spec.name,
action: spec.action,
model: spec.model,
optionalModels: spec.optionalModels ?? [],
config: spec.config ? structuredClone(spec.config) : undefined,
paramKeys: Object.keys(params),
params,
source: 'built_in',
};
}
private describeCompatPrompt(prompt: Prompt): ResolvedPrompt {
const metadata = collectPromptMetadataNative({ messages: prompt.messages });
return {
name: prompt.name,
action: prompt.action,
model: prompt.model,
optionalModels: prompt.optionalModels ?? [],
config: prompt.config ? structuredClone(prompt.config) : undefined,
paramKeys: metadata.paramKeys,
params: metadata.templateParams,
source: 'compat',
messages: prompt.messages,
};
}
@@ -178,23 +110,6 @@ export class PromptService {
);
}
private countCompatPromptTokens(prompt: ResolvedPrompt): number {
return countPromptTokensNative({
model: prompt.model,
messages: this.requireCompatMessages(prompt).map(message => ({
content: message.content,
})),
}).tokens;
}
private requireCompatMessages(prompt: ResolvedPrompt): PromptMessage[] {
if (prompt.source === 'compat' && prompt.messages) {
return this.cloneMessages(prompt.messages);
}
throw new Error(`Prompt ${prompt.name} does not expose compat messages`);
}
private logWarnings(warnings: string[], sessionId?: string) {
if (!sessionId) {
return;
@@ -1,28 +1,11 @@
import type {
PromptConfig,
PromptMessage,
PromptParams,
} from '../providers/types';
export type Prompt = {
name: string;
model: string;
optionalModels?: string[];
action?: string;
messages: PromptMessage[];
config?: PromptConfig;
};
import type { PromptConfig, PromptParams } from '../providers/types';
export type ResolvedPrompt = {
name: string;
model: string;
optionalModels: string[];
action?: string;
config?: PromptConfig;
paramKeys: string[];
params: PromptParams;
source: 'built_in' | 'compat';
messages?: PromptMessage[];
};
type PromptParamSpec = {
@@ -38,8 +21,6 @@ type PromptSpecMessage = {
export type PromptSpec = {
name: string;
action?: string;
model: string;
optionalModels?: string[];
config?: PromptConfig;
params?: Record<string, PromptParamSpec>;
messages: PromptSpecMessage[];
@@ -1,100 +0,0 @@
import { CopilotProviderSideError, UserFriendlyError } from '../../../../base';
import {
type LlmBackendConfig,
llmResolveRequestIntentOptions,
} from '../../../../native';
import { CopilotProvider } from '../provider';
import { hasProviderModelBehaviorFlag } from '../provider-model-runtime';
import {
type CopilotProviderExecution,
type ProviderDriverSpec,
} from '../provider-runtime-contract';
import { CopilotProviderType } from '../types';
import {
getGoogleAuth,
getVertexAnthropicBaseUrl,
type VertexAnthropicProviderConfig,
} from '../utils';
export abstract class AnthropicProvider<T> extends CopilotProvider<T> {
protected resolveModelBackendKind() {
return this.type === CopilotProviderType.AnthropicVertex
? ('anthropic_vertex' as const)
: ('anthropic' as const);
}
override getDriverSpec(): ProviderDriverSpec {
return {
createBackendConfig: execution => this.createNativeConfig(execution),
mapError: error => this.handleError(error),
chat: {
resolveRequestOptions: async context => {
const requestIntent = await llmResolveRequestIntentOptions({
protocol: context.protocol,
backendConfig: context.backendConfig,
reasoning: {
enabled: context.options.reasoning,
supported: hasProviderModelBehaviorFlag(
context.model,
'reasoning_budget_12000'
),
budgetTokens: hasProviderModelBehaviorFlag(
context.model,
'reasoning_budget_12000'
)
? 12000
: undefined,
},
});
return {
attachmentCapability: this.getAttachCapability(
context.model,
context.outputType
),
reasoning: requestIntent.reasoning,
};
},
},
structured: false,
embedding: false,
rerank: false,
};
}
private handleError(e: any) {
if (e instanceof UserFriendlyError) {
return e;
}
return new CopilotProviderSideError({
provider: this.type,
kind: 'unexpected_response',
message: e?.message || 'Unexpected anthropic response',
});
}
private async createNativeConfig(
execution?: CopilotProviderExecution
): Promise<LlmBackendConfig> {
const config = this.getConfig(execution);
if (this.type === CopilotProviderType.AnthropicVertex) {
const vertexConfig = config as VertexAnthropicProviderConfig;
const auth = await getGoogleAuth(vertexConfig, 'anthropic');
const { Authorization: authHeader } = auth.headers();
const token = authHeader.replace(/^Bearer\s+/i, '');
const baseUrl = getVertexAnthropicBaseUrl(vertexConfig) || auth.baseUrl;
return {
base_url: baseUrl || '',
auth_token: token,
headers: { Authorization: authHeader },
};
}
const officialConfig = config as { apiKey: string; baseURL?: string };
const baseUrl = officialConfig.baseURL || 'https://api.anthropic.com/v1';
return {
base_url: baseUrl.replace(/\/v1\/?$/, ''),
auth_token: officialConfig.apiKey,
};
}
}
@@ -1,2 +0,0 @@
export * from './official';
export * from './vertex';
@@ -1,16 +0,0 @@
import type { CopilotProviderExecution } from '../provider-runtime-contract';
import { CopilotProviderType } from '../types';
import { AnthropicProvider } from './anthropic';
export type AnthropicOfficialConfig = {
apiKey: string;
baseURL?: string;
};
export class AnthropicOfficialProvider extends AnthropicProvider<AnthropicOfficialConfig> {
override readonly type = CopilotProviderType.Anthropic;
override configured(execution?: CopilotProviderExecution): boolean {
return !!this.getConfig(execution).apiKey;
}
}
@@ -1,16 +0,0 @@
import type { CopilotProviderExecution } from '../provider-runtime-contract';
import { CopilotProviderType } from '../types';
import { getVertexAnthropicBaseUrl, type VertexProviderConfig } from '../utils';
import { AnthropicProvider } from './anthropic';
export type AnthropicVertexConfig = VertexProviderConfig;
export class AnthropicVertexProvider extends AnthropicProvider<AnthropicVertexConfig> {
override readonly type = CopilotProviderType.AnthropicVertex;
override configured(execution?: CopilotProviderExecution): boolean {
const config = this.getConfig(execution);
if (!config.location || !config.googleAuthOptions) return false;
return !!config.project || !!getVertexAnthropicBaseUrl(config);
}
}
@@ -1,42 +1,4 @@
import type {
ModelAttachmentCapability,
PromptAttachment,
PromptMessage,
} from './types';
export const IMAGE_ATTACHMENT_CAPABILITY: ModelAttachmentCapability = {
kinds: ['image'],
sourceKinds: ['url', 'data'],
allowRemoteUrls: true,
};
export const GEMINI_ATTACHMENT_CAPABILITY: ModelAttachmentCapability = {
kinds: ['image', 'audio', 'file'],
sourceKinds: ['url', 'data', 'bytes', 'file_handle'],
allowRemoteUrls: true,
};
export function promptAttachmentHasSource(
attachment: PromptAttachment
): boolean {
if (typeof attachment === 'string') {
return !!attachment.trim();
}
if ('attachment' in attachment) {
return !!attachment.attachment;
}
switch (attachment.kind) {
case 'url':
return !!attachment.url;
case 'data':
case 'bytes':
return !!attachment.data;
case 'file_handle':
return !!attachment.fileHandle;
}
}
import type { PromptAttachment, PromptMessage } from './types';
export function applyPromptAttachmentMimeTypeHintForNative(
attachment: PromptAttachment,
@@ -1,65 +0,0 @@
import { CopilotProviderSideError, UserFriendlyError } from '../../../base';
import { type LlmBackendConfig } from '../../../native';
import { CopilotProvider } from './provider';
import {
type CopilotProviderExecution,
type ProviderDriverSpec,
} from './provider-runtime-contract';
import { CopilotProviderType } from './types';
export type CloudflareWorkersAIConfig = {
apiToken: string;
accountId?: string;
baseURL?: string;
};
export class CloudflareWorkersAIProvider extends CopilotProvider<CloudflareWorkersAIConfig> {
override readonly type = CopilotProviderType.CloudflareWorkersAi;
protected resolveModelBackendKind() {
return 'cloudflare_workers_ai' as const;
}
override configured(execution?: CopilotProviderExecution): boolean {
const config = this.getConfig(execution);
return !!config.apiToken && (!!config.accountId || !!config.baseURL);
}
private handleError(e: any) {
if (e instanceof UserFriendlyError) {
return e;
}
return new CopilotProviderSideError({
provider: this.type,
kind: 'unexpected_response',
message: e?.message || 'Unexpected cloudflare workers ai response',
});
}
private createNativeConfig(
execution?: CopilotProviderExecution
): LlmBackendConfig {
const config = this.getConfig(execution);
return {
base_url: this.resolveBaseUrl(execution),
auth_token: config.apiToken,
};
}
private resolveBaseUrl(execution?: CopilotProviderExecution) {
const config = this.getConfig(execution);
if (config.baseURL) {
return config.baseURL.replace(/\/v1\/?$/, '').replace(/\/$/, '');
}
const accountId = config.accountId ?? '';
return `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai`;
}
override getDriverSpec(): ProviderDriverSpec {
return {
createBackendConfig: execution => this.createNativeConfig(execution),
mapError: error => this.handleError(error),
structured: false,
embedding: false,
};
}
}
@@ -1,527 +0,0 @@
import { Injectable, Logger } from '@nestjs/common';
import { CopilotQuotaExceeded } from '../../../base';
import { ServerFeature, ServerService } from '../../../core';
import { type CopilotAccessContext, CopilotAccessPolicy } from '../access';
import type { RequiredStructuredOutputContract } from '../runtime/contracts';
import { getProviderRuntimeHost } from '../runtime/provider-runtime-context';
import type { CopilotProvider } from './provider';
import {
buildProviderRegistry,
type CopilotProviderRegistry,
type NormalizedCopilotProviderProfile,
resolveModel,
stripProviderPrefix,
} from './provider-registry';
import type {
CopilotProviderExecution,
PreparedNativeEmbeddingExecution,
PreparedNativeExecution,
PreparedNativeImageExecution,
PreparedNativeRerankExecution,
PreparedNativeStructuredExecution,
} from './provider-runtime-contract';
import { CopilotProviderRegistryService } from './registry-service';
import {
type CopilotChatOptions,
type CopilotEmbeddingOptions,
type CopilotImageOptions,
CopilotProviderType,
type CopilotRerankRequest,
type CopilotStructuredOptions,
ModelFullConditions,
ModelOutputType,
type PromptMessage,
} from './types';
export type ResolvedCopilotProvider = {
providerId: string;
provider: CopilotProvider;
execution: CopilotProviderExecution;
profile: NormalizedCopilotProviderProfile;
rawModelId?: string;
modelId?: string;
explicitProviderId?: string;
prepared?: PreparedNativeExecution;
preparedStructured?: PreparedNativeStructuredExecution;
preparedEmbedding?: PreparedNativeEmbeddingExecution;
preparedRerank?: PreparedNativeRerankExecution;
preparedImage?: PreparedNativeImageExecution;
};
type RoutePreparationResult = Partial<
Pick<
ResolvedCopilotProvider,
| 'prepared'
| 'preparedStructured'
| 'preparedEmbedding'
| 'preparedRerank'
| 'preparedImage'
| 'modelId'
>
>;
type EffectiveProviderRegistry = {
byokRegistry: CopilotProviderRegistry;
quotaBackedRegistry: CopilotProviderRegistry;
quotaBackedRoutesAvailable: boolean;
};
@Injectable()
export class CopilotProviderFactory {
constructor(
private readonly server: ServerService,
private readonly registries: CopilotProviderRegistryService,
private readonly access: CopilotAccessPolicy
) {}
private readonly logger = new Logger(CopilotProviderFactory.name);
readonly #providers = new Map<string, CopilotProvider>();
readonly #providerIdsByType = new Map<CopilotProviderType, Set<string>>();
private getRegistry() {
return this.registries.getRegistry();
}
private getProviderByProfile(
providerId: string,
profile: NormalizedCopilotProviderProfile
) {
return (
this.#providers.get(providerId) ??
Array.from(this.#providerIdsByType.get(profile.type) ?? [])
.map(id => this.#providers.get(id))
.find((provider): provider is CopilotProvider => !!provider)
);
}
private providerAvailable(
providerId: string,
profile: NormalizedCopilotProviderProfile
) {
return !!this.getProviderByProfile(providerId, profile);
}
private getAvailableProviderIds(registry: CopilotProviderRegistry) {
return Array.from(registry.profiles.entries())
.filter(([providerId, profile]) =>
this.providerAvailable(providerId, profile)
)
.map(([providerId]) => providerId);
}
private getPreferredProviderIds(
registry: CopilotProviderRegistry,
type?: CopilotProviderType
) {
if (!type) return undefined;
return registry.byType.get(type)?.filter(providerId => {
const profile = registry.profiles.get(providerId);
return profile ? this.providerAvailable(providerId, profile) : false;
});
}
private normalizeCond(
registry: CopilotProviderRegistry,
providerId: string,
cond: ModelFullConditions
): ModelFullConditions {
const modelId = stripProviderPrefix(registry, providerId, cond.modelId);
return { ...cond, modelId };
}
private async getEffectiveRegistry(
context: CopilotAccessContext = {}
): Promise<EffectiveProviderRegistry> {
const quotaBackedRegistry = this.getRegistry();
const routeAccess = await this.access.resolveRouteAccess(context);
return {
byokRegistry: buildProviderRegistry({
profiles: routeAccess.byokProfiles,
defaults: {},
}),
quotaBackedRegistry,
quotaBackedRoutesAvailable: routeAccess.quotaBackedRoutesAvailable,
};
}
private getRequestContext(
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions
): CopilotAccessContext {
return {
userId: options?.user,
workspaceId: options?.workspace,
byokLeaseId: options?.byokLeaseId,
featureKind: options?.featureKind,
quotaBackedRoutesAllowed: options?.quotaBackedRoutesAllowed,
};
}
private filterPreparedRoutes(routes: Array<ResolvedCopilotProvider | null>) {
return routes.filter(
(route): route is ResolvedCopilotProvider => route !== null
);
}
private async prepareResolvedRoutes(
routes: ResolvedCopilotProvider[],
prepare: (
route: ResolvedCopilotProvider
) => Promise<RoutePreparationResult | null | undefined>
) {
const preparedRoutes = await Promise.all(
routes.map(async route => {
const prepared = await prepare(route);
return prepared ? { ...route, ...prepared } : null;
})
);
return this.filterPreparedRoutes(preparedRoutes);
}
async resolveProvider(
cond: ModelFullConditions,
filter: {
prefer?: CopilotProviderType;
} = {},
context: CopilotAccessContext = {}
): Promise<ResolvedCopilotProvider | null> {
return (await this.resolveRoutes(cond, filter, context))[0] ?? null;
}
async resolveRoutes(
cond: ModelFullConditions,
filter: {
prefer?: CopilotProviderType;
} = {},
context: CopilotAccessContext = {}
): Promise<ResolvedCopilotProvider[]> {
this.logger.debug(
`Resolving copilot provider for output type: ${cond.outputType}`
);
const { byokRegistry, quotaBackedRegistry, quotaBackedRoutesAvailable } =
await this.getEffectiveRegistry(context);
const byokRoutes = await this.resolveRoutesFromRegistry(
byokRegistry,
cond,
filter
);
const resolved = byokRoutes.length
? byokRoutes
: quotaBackedRoutesAvailable
? await this.resolveRoutesFromRegistry(
quotaBackedRegistry,
cond,
filter
)
: [];
for (const route of resolved) {
this.logger.debug(
`Copilot provider candidate found: ${route.provider.type} (${route.providerId})`
);
}
if (
!resolved.length &&
!quotaBackedRoutesAvailable &&
context.quotaBackedRoutesAllowed !== false
) {
const quotaBackedRoutes = await this.resolveRoutesFromRegistry(
quotaBackedRegistry,
cond,
filter
);
if (quotaBackedRoutes.length) {
throw new CopilotQuotaExceeded();
}
}
return resolved;
}
private async resolveRoutesFromRegistry(
registry: CopilotProviderRegistry,
cond: ModelFullConditions,
filter: {
prefer?: CopilotProviderType;
} = {}
): Promise<ResolvedCopilotProvider[]> {
const route = resolveModel({
registry,
modelId: cond.modelId,
outputType: cond.outputType,
availableProviderIds: this.getAvailableProviderIds(registry),
preferredProviderIds: this.getPreferredProviderIds(
registry,
filter.prefer
),
});
const resolved: ResolvedCopilotProvider[] = [];
for (const providerId of route.candidateProviderIds) {
const profile = registry.profiles.get(providerId);
const provider = profile
? this.getProviderByProfile(providerId, profile)
: undefined;
if (!provider || !profile) continue;
const normalizedCond = this.normalizeCond(registry, providerId, cond);
if (
normalizedCond.modelId &&
profile.models?.length &&
!profile.models.includes(normalizedCond.modelId)
) {
continue;
}
const execution = { providerId, profile };
const matched = await provider.match(normalizedCond, execution);
if (!matched) continue;
resolved.push({
providerId,
provider,
execution,
profile,
rawModelId: route.rawModelId,
modelId: normalizedCond.modelId,
explicitProviderId: route.explicitProviderId,
});
}
return resolved;
}
async prepareRoutes(
kind: 'text' | 'streamText' | 'streamObject',
cond: ModelFullConditions,
messages: PromptMessage[],
options: CopilotChatOptions = {},
filter: {
prefer?: CopilotProviderType;
} = {}
): Promise<ResolvedCopilotProvider[]> {
const routes = await this.resolveRoutes(
cond,
filter,
this.getRequestContext(options)
);
return await this.prepareResolvedRoutes(routes, async route => {
const prepared = await getProviderRuntimeHost(
route.provider
).prepare.chat(
kind,
{ ...cond, modelId: route.modelId },
messages,
options,
route.execution
);
const normalizedPrepared = prepared?.route ? prepared : undefined;
if (!normalizedPrepared) {
return null;
}
return {
modelId: normalizedPrepared.route.model,
prepared: normalizedPrepared,
};
});
}
async prepareStructuredRoutes(
cond: ModelFullConditions,
messages: PromptMessage[],
options: CopilotStructuredOptions = {},
filter: {
prefer?: CopilotProviderType;
} = {},
responseContract?: RequiredStructuredOutputContract
): Promise<ResolvedCopilotProvider[]> {
const routes = await this.resolveRoutes(
cond,
filter,
this.getRequestContext(options)
);
return await this.prepareResolvedRoutes(routes, async route => {
const preparedStructured =
(await getProviderRuntimeHost(route.provider).prepare.structured(
{ ...cond, modelId: route.modelId },
messages,
options,
responseContract,
route.execution
)) ?? undefined;
if (!preparedStructured) {
return null;
}
return {
modelId: preparedStructured.route.model,
preparedStructured,
};
});
}
async prepareEmbeddingRoutes(
modelId: string,
input: string | string[],
options: CopilotEmbeddingOptions = {}
): Promise<ResolvedCopilotProvider[]> {
const routes = await this.resolveRoutes(
{ modelId, outputType: ModelOutputType.Embedding },
{},
{
...this.getRequestContext(options),
featureKind: options?.featureKind ?? 'embedding',
}
);
return await this.prepareResolvedRoutes(routes, async route => {
const preparedEmbedding =
(await getProviderRuntimeHost(route.provider).prepare.embedding(
{ modelId: route.modelId },
input,
options,
route.execution
)) ?? undefined;
if (!preparedEmbedding) {
return null;
}
return {
modelId: preparedEmbedding.route.model,
preparedEmbedding,
};
});
}
async prepareRerankRoutes(
modelId: string,
request: CopilotRerankRequest,
options: CopilotChatOptions = {}
): Promise<ResolvedCopilotProvider[]> {
const routes = await this.resolveRoutes(
{
modelId,
outputType: ModelOutputType.Rerank,
},
{},
{ ...this.getRequestContext(options), featureKind: 'rerank' }
);
return await this.prepareResolvedRoutes(routes, async route => {
const preparedRerank =
(await getProviderRuntimeHost(route.provider).prepare.rerank(
{ modelId: route.modelId },
request,
options,
route.execution
)) ?? undefined;
if (!preparedRerank) {
return null;
}
return {
modelId: preparedRerank.route.model,
preparedRerank,
};
});
}
async prepareImageRoutes(
cond: ModelFullConditions,
messages: PromptMessage[],
options: CopilotImageOptions = {},
filter: {
prefer?: CopilotProviderType;
} = {}
): Promise<ResolvedCopilotProvider[]> {
const routes = await this.resolveRoutes(cond, filter, {
...this.getRequestContext(options),
featureKind: options?.featureKind ?? 'image',
});
return await this.prepareResolvedRoutes(routes, async route => {
const preparedImage =
(await getProviderRuntimeHost(route.provider).prepare.image(
{ ...cond, modelId: route.modelId },
messages,
options,
route.execution
)) ?? undefined;
if (!preparedImage) {
return null;
}
return {
modelId: preparedImage.route.model,
preparedImage,
};
});
}
async getProvider(
cond: ModelFullConditions,
filter: {
prefer?: CopilotProviderType;
} = {}
): Promise<CopilotProvider | null> {
return (await this.resolveProvider(cond, filter))?.provider ?? null;
}
async getProviderByModel(
modelId: string,
filter: {
prefer?: CopilotProviderType;
} = {}
): Promise<CopilotProvider | null> {
this.logger.debug(`Resolving copilot provider for model: ${modelId}`);
return this.getProvider({ modelId }, filter);
}
register(providerId: string, provider: CopilotProvider) {
const existed = this.#providers.get(providerId);
if (existed?.type && existed.type !== provider.type) {
const ids = this.#providerIdsByType.get(existed.type);
ids?.delete(providerId);
if (!ids?.size) {
this.#providerIdsByType.delete(existed.type);
}
}
this.#providers.set(providerId, provider);
const ids = this.#providerIdsByType.get(provider.type) ?? new Set<string>();
ids.add(providerId);
this.#providerIdsByType.set(provider.type, ids);
this.logger.log(
`Copilot provider [${provider.type}] registered as [${providerId}].`
);
this.server.enableFeature(ServerFeature.Copilot);
}
unregister(providerId: string, provider: CopilotProvider) {
const existed = this.#providers.get(providerId);
if (!existed || existed !== provider) {
return;
}
this.#providers.delete(providerId);
const ids = this.#providerIdsByType.get(provider.type);
ids?.delete(providerId);
if (!ids?.size) {
this.#providerIdsByType.delete(provider.type);
}
this.logger.log(
`Copilot provider [${provider.type}] unregistered from [${providerId}].`
);
if (this.#providers.size === 0) {
this.server.disableFeature(ServerFeature.Copilot);
}
}
}
@@ -1,59 +0,0 @@
import { Injectable } from '@nestjs/common';
import { CopilotProviderSideError, UserFriendlyError } from '../../../base';
import { CopilotProvider } from './provider';
import type {
CopilotProviderExecution,
ProviderDriverSpec,
} from './provider-runtime-contract';
import { CopilotProviderType } from './types';
export type FalConfig = {
apiKey: string;
};
@Injectable()
export class FalProvider extends CopilotProvider<FalConfig> {
override type = CopilotProviderType.FAL;
protected resolveModelBackendKind() {
return 'fal' as const;
}
override configured(execution?: CopilotProviderExecution): boolean {
return !!this.getConfig(execution).apiKey;
}
private createNativeConfig(execution?: CopilotProviderExecution) {
return {
base_url: 'https://fal.run',
auth_token: this.getConfig(execution).apiKey,
};
}
override getDriverSpec(): ProviderDriverSpec {
return {
createBackendConfig: execution => this.createNativeConfig(execution),
mapError: error => this.handleError(error),
chat: false,
structured: false,
embedding: false,
rerank: false,
image: {},
};
}
private handleError(e: any) {
if (e instanceof UserFriendlyError) {
// pass through user friendly errors
return e;
} else {
const error = new CopilotProviderSideError({
provider: this.type,
kind: 'unexpected_response',
message: e?.message || 'Unexpected fal response',
});
return error;
}
}
}
@@ -1,250 +0,0 @@
import { setTimeout as delay } from 'node:timers/promises';
import { Inject } from '@nestjs/common';
import { ZodError } from 'zod';
import {
CopilotProviderSideError,
OneMB,
UserFriendlyError,
} from '../../../../base';
import {
isInvalidStructuredOutputError,
type LlmBackendConfig,
llmResolveRequestIntentOptions,
} from '../../../../native';
import {
admittedAttachmentToPromptAttachment,
AttachmentAdmissionHost,
} from '../../runtime/hosts/attachment-admission';
import {
planAdmittedAttachmentMaterialization,
planHostUrlAttachmentMaterialization,
} from '../../runtime/hosts/attachment-materialization-planner';
import { AttachmentMaterializer } from '../../runtime/hosts/attachment-materializer';
import { CopilotProvider } from '../provider';
import { hasProviderModelBehaviorFlag } from '../provider-model-runtime';
import {
type CopilotProviderExecution,
type ProviderDriverSpec,
} from '../provider-runtime-contract';
import type { PromptAttachment, PromptMessage } from '../types';
import { promptAttachmentMimeType, promptAttachmentToUrl } from '../utils';
export const DEFAULT_DIMENSIONS = 256;
const GEMINI_REMOTE_ATTACHMENT_MAX_BYTES = 64 * OneMB;
const TRUSTED_ATTACHMENT_HOST_SUFFIXES = ['cdn.affine.pro'];
const GEMINI_RETRY_INITIAL_DELAY_MS = 2_000;
function normalizeMimeType(mediaType?: string) {
return mediaType?.split(';', 1)[0]?.trim() || 'application/octet-stream';
}
export abstract class GeminiProvider<T> extends CopilotProvider<T> {
@Inject() protected readonly attachmentMaterializer!: AttachmentMaterializer;
@Inject()
protected readonly attachmentAdmissionHost?: AttachmentAdmissionHost;
protected resolveModelBackendKind() {
return this.type === 'geminiVertex'
? ('gemini_vertex' as const)
: ('gemini_api' as const);
}
protected abstract createNativeConfig(
execution?: CopilotProviderExecution
): Promise<LlmBackendConfig>;
private handleError(e: any) {
if (e instanceof UserFriendlyError) {
return e;
} else {
return new CopilotProviderSideError({
provider: this.type,
kind: 'unexpected_response',
message: e?.message || 'Unexpected google response',
});
}
}
private getAttachmentAdmissionHost() {
return (
this.attachmentAdmissionHost ??
new AttachmentAdmissionHost(this.attachmentMaterializer)
);
}
protected async prepareMessages(
messages: PromptMessage[],
backendConfig: LlmBackendConfig,
options?: {
signal?: AbortSignal;
user?: string;
workspace?: string;
session?: string;
}
): Promise<PromptMessage[]> {
const prepared: PromptMessage[] = [];
for (const message of messages) {
options?.signal?.throwIfAborted();
if (!Array.isArray(message.attachments) || !message.attachments.length) {
prepared.push(message);
continue;
}
const attachments: PromptAttachment[] = [];
let changed = false;
for (const attachment of message.attachments) {
options?.signal?.throwIfAborted();
const rawUrl = promptAttachmentToUrl(attachment);
if (!rawUrl || rawUrl.startsWith('data:')) {
attachments.push(attachment);
continue;
}
try {
new URL(rawUrl);
} catch {
attachments.push(attachment);
continue;
}
const declaredMimeType = promptAttachmentMimeType(
attachment,
typeof message.params?.mimetype === 'string'
? message.params.mimetype
: undefined
);
const referencePlan = await planHostUrlAttachmentMaterialization(
'gemini',
backendConfig,
{
attachmentId: rawUrl,
url: rawUrl,
expectedMime: declaredMimeType
? normalizeMimeType(declaredMimeType)
: undefined,
maxSize: GEMINI_REMOTE_ATTACHMENT_MAX_BYTES,
}
);
if (referencePlan.mode === 'remote_reference') {
attachments.push(attachment);
continue;
}
const admitted =
await this.getAttachmentAdmissionHost().admitPromptAttachment(
attachment,
{
userId: options?.user ?? 'provider-runtime',
workspaceId: options?.workspace ?? 'provider-runtime',
sessionId: options?.session,
signal: options?.signal,
maxBytes: referencePlan.request.maxSize,
trustedHostSuffixes: TRUSTED_ATTACHMENT_HOST_SUFFIXES,
}
);
const materialization = planAdmittedAttachmentMaterialization(admitted);
attachments.push(
materialization.mode === 'inline'
? materialization.attachment
: admittedAttachmentToPromptAttachment(admitted)
);
changed = true;
}
prepared.push(changed ? { ...message, attachments } : message);
}
return prepared;
}
protected async waitForStructuredRetry(
delayMs: number,
signal?: AbortSignal
) {
await delay(delayMs, undefined, signal ? { signal } : undefined);
}
override getDriverSpec(): ProviderDriverSpec {
return {
createBackendConfig: execution => this.createNativeConfig(execution),
mapError: error => this.handleError(error),
chat: {
prepareMessages: async context =>
await this.prepareMessages(
context.input.messages,
context.backendConfig,
context.options
),
resolveRequestOptions: async context => {
const requestIntent = await llmResolveRequestIntentOptions({
protocol: context.protocol,
backendConfig: context.backendConfig,
reasoning: {
enabled: context.options.reasoning,
supported:
hasProviderModelBehaviorFlag(
context.model,
'reasoning_medium'
) ||
hasProviderModelBehaviorFlag(context.model, 'reasoning_high'),
effort: hasProviderModelBehaviorFlag(
context.model,
'reasoning_high'
)
? 'high'
: 'medium',
includeReasoning:
hasProviderModelBehaviorFlag(
context.model,
'reasoning_medium'
) ||
hasProviderModelBehaviorFlag(context.model, 'reasoning_high'),
},
});
return {
attachmentCapability: this.getAttachCapability(
context.model,
context.outputType
),
include: requestIntent.include,
reasoning: requestIntent.reasoning,
};
},
},
structured: {
prepareMessages: (inputMessages, backendConfig, structuredOptions) =>
this.prepareMessages(inputMessages, backendConfig, structuredOptions),
shouldRetry: async ({ error, attempt, options: structuredOptions }) => {
const isParsingError =
isInvalidStructuredOutputError(error) || error instanceof ZodError;
const retryableError =
isParsingError || !(error instanceof UserFriendlyError);
const maxRetries = Math.max(structuredOptions.maxRetries ?? 3, 0);
if (!retryableError || attempt >= maxRetries) {
return false;
}
if (!isParsingError) {
await this.waitForStructuredRetry(
GEMINI_RETRY_INITIAL_DELAY_MS * 2 ** attempt,
structuredOptions.signal
);
}
return true;
},
},
embedding: {
defaultDimensions: DEFAULT_DIMENSIONS,
taskType: 'RETRIEVAL_DOCUMENT',
},
rerank: false,
image: {
prepareMessages: (inputMessages, backendConfig, imageOptions) =>
this.prepareMessages(inputMessages, backendConfig, imageOptions),
},
};
}
}
@@ -1,28 +0,0 @@
import type { LlmBackendConfig } from '../../../../native';
import type { CopilotProviderExecution } from '../provider-runtime-contract';
import { CopilotProviderType } from '../types';
import { GeminiProvider } from './gemini';
export type GeminiGenerativeConfig = {
apiKey: string;
baseURL?: string;
};
export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeConfig> {
override readonly type = CopilotProviderType.Gemini;
override configured(execution?: CopilotProviderExecution): boolean {
return !!this.getConfig(execution).apiKey;
}
protected override async createNativeConfig(
execution?: CopilotProviderExecution
): Promise<LlmBackendConfig> {
const config = this.getConfig(execution);
return {
base_url: (
config.baseURL || 'https://generativelanguage.googleapis.com/v1beta'
).replace(/\/$/, ''),
auth_token: config.apiKey,
};
}
}
@@ -1,2 +0,0 @@
export * from './generative';
export * from './vertex';
@@ -1,34 +0,0 @@
import type { LlmBackendConfig } from '../../../../native';
import type { CopilotProviderExecution } from '../provider-runtime-contract';
import { CopilotProviderType } from '../types';
import {
getGoogleAuth,
getVertexGoogleBaseUrl,
type VertexProviderConfig,
} from '../utils';
import { GeminiProvider } from './gemini';
export type GeminiVertexConfig = VertexProviderConfig;
export class GeminiVertexProvider extends GeminiProvider<GeminiVertexConfig> {
override readonly type = CopilotProviderType.GeminiVertex;
override configured(execution?: CopilotProviderExecution): boolean {
const config = this.getConfig(execution);
return !!getVertexGoogleBaseUrl(config) && !!config.googleAuthOptions;
}
protected async resolveVertexAuth(execution?: CopilotProviderExecution) {
return await getGoogleAuth(this.getConfig(execution), 'google');
}
protected override async createNativeConfig(
execution?: CopilotProviderExecution
): Promise<LlmBackendConfig> {
const auth = await this.resolveVertexAuth(execution);
const { Authorization: authHeader } = auth.headers();
return {
base_url: auth.baseUrl || '',
auth_token: authHeader.replace(/^Bearer\s+/i, ''),
};
}
}
@@ -1,14 +0,0 @@
export {
AnthropicOfficialProvider,
AnthropicVertexProvider,
} from './anthropic';
export { CloudflareWorkersAIProvider } from './cloudflare';
export { CopilotProviderFactory } from './factory';
export { FalProvider } from './fal';
export { GeminiGenerativeProvider, GeminiVertexProvider } from './gemini';
export { CopilotProviderLifecycleService } from './lifecycle-service';
export { OpenAIProvider } from './openai';
export type { CopilotProvider } from './provider';
export { CopilotProviders } from './provider-tokens';
export { CopilotProviderRegistryService } from './registry-service';
export * from './types';
@@ -1,90 +0,0 @@
import { Injectable, Type } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { OnEvent } from '../../../base';
import { CopilotProviderFactory } from './factory';
import type { CopilotProvider } from './provider';
import type { CopilotProviderExecution } from './provider-runtime-contract';
import { CopilotProviders } from './provider-tokens';
import { CopilotProviderRegistryService } from './registry-service';
@Injectable()
export class CopilotProviderLifecycleService {
private readonly registeredByProvider = new WeakMap<
CopilotProvider,
Set<string>
>();
constructor(
private readonly moduleRef: ModuleRef,
private readonly factory: CopilotProviderFactory,
private readonly registries: CopilotProviderRegistryService
) {}
private getProviders(): CopilotProvider[] {
return CopilotProviders.flatMap(token => {
const provider = this.moduleRef.get(token as Type<CopilotProvider>, {
strict: false,
});
return provider ? [provider] : [];
});
}
private getRegisteredProviderIds(provider: CopilotProvider) {
const current = this.registeredByProvider.get(provider);
if (current) {
return current;
}
const next = new Set<string>();
this.registeredByProvider.set(provider, next);
return next;
}
private async syncProvider(provider: CopilotProvider) {
const registry = this.registries.getRegistry();
const configuredIds = new Set<string>();
for (const providerId of registry.byType.get(provider.type) ?? []) {
const profile = registry.profiles.get(providerId);
if (!profile) {
continue;
}
const execution: CopilotProviderExecution = { providerId, profile };
if (!provider.configured(execution)) {
this.factory.unregister(providerId, provider);
continue;
}
configuredIds.add(providerId);
this.factory.register(providerId, provider);
}
const previous = this.getRegisteredProviderIds(provider);
for (const providerId of previous) {
if (!configuredIds.has(providerId)) {
this.factory.unregister(providerId, provider);
}
}
this.registeredByProvider.set(provider, configuredIds);
}
async syncProviders() {
for (const provider of this.getProviders()) {
await this.syncProvider(provider);
}
}
@OnEvent('config.init')
async onConfigInit() {
await this.syncProviders();
}
@OnEvent('config.changed')
async onConfigChanged(event: Events['config.changed']) {
if ('copilot' in event.updates) {
await this.syncProviders();
}
}
}
@@ -1,172 +0,0 @@
import { Inject } from '@nestjs/common';
import {
CopilotProviderSideError,
OneMB,
UserFriendlyError,
} from '../../../base';
import {
type LlmBackendConfig,
llmResolveRequestIntentOptions,
} from '../../../native';
import {
admittedAttachmentToPromptAttachment,
AttachmentAdmissionHost,
} from '../runtime/hosts/attachment-admission';
import { AttachmentMaterializer } from '../runtime/hosts/attachment-materializer';
import { CopilotProvider } from './provider';
import { hasProviderModelBehaviorFlag } from './provider-model-runtime';
import type {
CopilotProviderExecution,
ProviderDriverSpec,
} from './provider-runtime-contract';
import {
CopilotProviderType,
type PromptAttachment,
type PromptMessage,
} from './types';
import { promptAttachmentToUrl } from './utils';
export const DEFAULT_DIMENSIONS = 256;
export type OpenAIConfig = {
apiKey: string;
baseURL?: string;
oldApiStyle?: boolean;
};
export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
readonly type = CopilotProviderType.OpenAI;
@Inject() protected readonly attachmentMaterializer!: AttachmentMaterializer;
@Inject()
protected readonly attachmentAdmissionHost?: AttachmentAdmissionHost;
protected resolveModelBackendKind(execution?: CopilotProviderExecution) {
return this.getConfig(execution).oldApiStyle
? ('openai_chat' as const)
: ('openai_responses' as const);
}
override configured(execution?: CopilotProviderExecution): boolean {
return !!this.getConfig(execution).apiKey;
}
private handleError(e: any) {
if (e instanceof UserFriendlyError) {
return e;
}
return new CopilotProviderSideError({
provider: this.type,
kind: 'unexpected_response',
message: e?.message || 'Unexpected openai response',
});
}
protected createNativeConfig(
execution?: CopilotProviderExecution
): LlmBackendConfig {
const config = this.getConfig(execution);
const baseUrl = config.baseURL || 'https://api.openai.com/v1';
return {
base_url: baseUrl.replace(/\/v1\/?$/, ''),
auth_token: config.apiKey,
};
}
private getAttachmentAdmissionHost() {
return (
this.attachmentAdmissionHost ??
new AttachmentAdmissionHost(this.attachmentMaterializer)
);
}
private async prepareImageMessages(
messages: PromptMessage[],
options: {
signal?: AbortSignal;
user?: string;
workspace?: string;
session?: string;
}
) {
const prepared: PromptMessage[] = [];
for (const message of messages) {
options.signal?.throwIfAborted();
if (!Array.isArray(message.attachments) || !message.attachments.length) {
prepared.push(message);
continue;
}
let changed = false;
const attachments: PromptAttachment[] = [];
for (const attachment of message.attachments) {
options.signal?.throwIfAborted();
const url = promptAttachmentToUrl(attachment);
if (!url || url.startsWith('data:')) {
attachments.push(attachment);
continue;
}
const admitted =
await this.getAttachmentAdmissionHost().admitPromptAttachment(
attachment,
{
userId: options.user ?? 'provider-runtime',
workspaceId: options.workspace ?? 'provider-runtime',
sessionId: options.session,
signal: options.signal,
maxBytes: 50 * OneMB,
}
);
attachments.push(admittedAttachmentToPromptAttachment(admitted));
changed = true;
}
prepared.push(changed ? { ...message, attachments } : message);
}
return prepared;
}
override getDriverSpec(): ProviderDriverSpec {
return {
createBackendConfig: execution => this.createNativeConfig(execution),
mapError: error => this.handleError(error),
chat: {
resolveRequestOptions: async context => {
const requestIntent = await llmResolveRequestIntentOptions({
protocol: context.protocol,
backendConfig: context.backendConfig,
include: context.options.webSearch ? ['citations'] : undefined,
reasoning: {
enabled: context.options.reasoning,
supported: hasProviderModelBehaviorFlag(
context.model,
'reasoning_supported'
),
},
});
return {
attachmentCapability: this.getAttachCapability(
context.model,
context.outputType
),
include: requestIntent.include,
reasoning: requestIntent.reasoning,
};
},
},
structured: {},
embedding: {
defaultDimensions: DEFAULT_DIMENSIONS,
taskType: 'RETRIEVAL_DOCUMENT',
},
image: {
prepareMessages: async (messages, _backendConfig, options) =>
await this.prepareImageMessages(messages, options),
},
};
}
}
@@ -1,73 +0,0 @@
import type { ProviderMiddlewareConfig } from '../config';
import { CopilotProviderType } from './types';
const DEFAULT_NODE_TEXT_MIDDLEWARE: NonNullable<
NonNullable<ProviderMiddlewareConfig['node']>['text']
> = ['citation_footnote', 'callout'];
const DEFAULT_MIDDLEWARE_BY_TYPE: Record<
CopilotProviderType,
ProviderMiddlewareConfig
> = {
[CopilotProviderType.OpenAI]: {
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
},
[CopilotProviderType.CloudflareWorkersAi]: {
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
},
[CopilotProviderType.Anthropic]: {
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
},
[CopilotProviderType.AnthropicVertex]: {
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
},
[CopilotProviderType.Gemini]: {
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
},
[CopilotProviderType.GeminiVertex]: {
node: { text: DEFAULT_NODE_TEXT_MIDDLEWARE },
},
[CopilotProviderType.FAL]: {},
};
function unique<T>(items: T[]) {
return [...new Set(items)];
}
function mergeArray<T>(base: T[] | undefined, override: T[] | undefined) {
if (!base?.length && !override?.length) {
return undefined;
}
return unique([...(base ?? []), ...(override ?? [])]);
}
function compactMiddlewareSection<T extends Record<string, unknown>>(
section: T
): T | undefined {
return Object.values(section).some(value => value !== undefined)
? section
: undefined;
}
export function mergeProviderMiddleware(
defaults: ProviderMiddlewareConfig,
override?: ProviderMiddlewareConfig
): ProviderMiddlewareConfig {
return {
rust: compactMiddlewareSection({
request: mergeArray(defaults.rust?.request, override?.rust?.request),
stream: mergeArray(defaults.rust?.stream, override?.rust?.stream),
}),
node: compactMiddlewareSection({
text: mergeArray(defaults.node?.text, override?.node?.text),
}),
};
}
export function resolveProviderMiddleware(
type: CopilotProviderType,
override?: ProviderMiddlewareConfig
): ProviderMiddlewareConfig {
const defaults = DEFAULT_MIDDLEWARE_BY_TYPE[type] ?? {};
return mergeProviderMiddleware(defaults, override);
}
@@ -1,385 +0,0 @@
import { z } from 'zod';
import { CopilotPromptInvalid } from '../../../base';
import {
type LlmBackendConfig,
llmInferPromptModelConditions,
llmMatchModelCapabilities,
llmMatchModelRegistry,
type LlmProtocol,
llmResolveModelRegistryVariant,
} from '../../../native';
import { applyPromptAttachmentMimeTypeHintForNative } from './attachments';
import {
type CopilotChatOptions,
type CopilotImageOptions,
type CopilotModelBackendKind,
type CopilotProviderModel,
type CopilotProviderType,
type CopilotStructuredOptions,
EmbeddingMessage,
type ModelAttachmentCapability,
type ModelCapability,
type ModelFullConditions,
ModelInputType,
ModelOutputType,
type PromptAttachmentKind,
type PromptAttachmentSourceKind,
type PromptMessage,
PromptMessageSchema,
} from './types';
// Owner: backend host model-selection glue.
// Capability matching and catalog lookup are delegated to native/adapter; this
// file keeps provider prefix/default/prefer behavior and Node prompt checks.
export type ProviderModelRuntimeContext = {
type: CopilotProviderType;
backendKind: CopilotModelBackendKind;
};
export type ResolvedProviderModel = CopilotProviderModel & {
backendKind: CopilotModelBackendKind;
canonicalKey: string;
protocol?: LlmProtocol;
requestLayer?: LlmBackendConfig['request_layer'];
routeOverrides?: Partial<
Record<
ModelOutputType,
{
protocol?: LlmProtocol;
requestLayer?: LlmBackendConfig['request_layer'];
}
>
>;
behaviorFlags?: string[];
};
function unique<T>(values: Iterable<T>) {
return Array.from(new Set(values));
}
function resolveAttachmentCapability(
cap: ModelCapability,
outputType?: ModelOutputType
): ModelAttachmentCapability | undefined {
if (outputType === ModelOutputType.Structured) {
return cap.structuredAttachments ?? cap.attachments;
}
return cap.attachments;
}
function toProviderModel(
variant: NonNullable<
ReturnType<typeof llmResolveModelRegistryVariant>['variant']
>
): ResolvedProviderModel {
return {
id: variant.rawModelId,
name: variant.displayName,
backendKind: variant.backendKind,
canonicalKey: variant.canonicalKey,
protocol: variant.protocol,
requestLayer: variant.requestLayer,
routeOverrides: variant.routeOverrides,
behaviorFlags: variant.behaviorFlags,
capabilities: variant.capabilities.map(capability => ({
input: capability.input as ModelInputType[],
output: capability.output as ModelOutputType[],
attachments: capability.attachments
? {
kinds: capability.attachments.kinds as PromptAttachmentKind[],
sourceKinds: capability.attachments.sourceKinds as
| ModelAttachmentCapability['sourceKinds']
| undefined,
allowRemoteUrls: capability.attachments.allowRemoteUrls,
}
: undefined,
structuredAttachments: capability.structuredAttachments
? {
kinds: capability.structuredAttachments
.kinds as PromptAttachmentKind[],
sourceKinds: capability.structuredAttachments.sourceKinds as
| ModelAttachmentCapability['sourceKinds']
| undefined,
allowRemoteUrls: capability.structuredAttachments.allowRemoteUrls,
}
: undefined,
defaultForOutputType: capability.defaultForOutputType,
})),
};
}
export type ProviderModelSelection = {
kind: 'configured';
model: ResolvedProviderModel;
};
export function resolveProviderModelSelection(
context: ProviderModelRuntimeContext,
cond: ModelFullConditions
): ProviderModelSelection | undefined {
if (cond.modelId) {
const resolved = llmResolveModelRegistryVariant({
backendKind: context.backendKind,
modelId: cond.modelId,
}).variant;
if (!resolved) {
return;
}
const model = toProviderModel(resolved);
const matchedModelId = llmMatchModelCapabilities([model], {
...cond,
modelId: model.id,
});
if (!matchedModelId) {
return;
}
return {
kind: 'configured',
model,
};
}
const resolved = llmMatchModelRegistry({
backendKind: context.backendKind,
cond,
}).variant;
if (!resolved) {
return;
}
return {
kind: 'configured',
model: toProviderModel(resolved),
};
}
function isMultimodal(model: CopilotProviderModel) {
return model.capabilities.some(c =>
[ModelInputType.Image, ModelInputType.Audio, ModelInputType.File].some(t =>
c.input.includes(t)
)
);
}
function handleZodError(ret: z.SafeParseReturnType<any, any>) {
if (ret.success) return;
const issues = ret.error.issues.map(i => {
const path =
'root' +
(i.path.length
? `.${i.path.map(seg => (typeof seg === 'number' ? `[${seg}]` : `.${seg}`)).join('')}`
: '');
return `${i.message}${path}`;
});
throw new CopilotPromptInvalid(issues.join('; '));
}
export async function inferModelConditionsFromMessages(
messages?: PromptMessage[],
withAttachment = true
): Promise<Partial<ModelFullConditions>> {
if (!messages?.length || !withAttachment) return {};
const projectedMessages = messages.map(message => ({
role: message.role,
content: message.content,
...(Array.isArray(message.attachments) && message.attachments.length
? {
attachments: message.attachments.map(attachment =>
applyPromptAttachmentMimeTypeHintForNative(attachment, message)
),
}
: {}),
}));
const inferredCond = llmInferPromptModelConditions(projectedMessages);
return {
...(inferredCond.attachmentKinds?.length
? { attachmentKinds: unique(inferredCond.attachmentKinds) }
: {}),
...(inferredCond.attachmentSourceKinds?.length
? {
attachmentSourceKinds: unique(
inferredCond.attachmentSourceKinds
) as PromptAttachmentSourceKind[],
}
: {}),
...(inferredCond.inputTypes?.length
? { inputTypes: unique(inferredCond.inputTypes) as ModelInputType[] }
: {}),
...(inferredCond.hasRemoteAttachments
? { hasRemoteAttachments: true }
: {}),
};
}
export function mergeModelConditions(
cond: ModelFullConditions,
inferredCond: Partial<ModelFullConditions>
): ModelFullConditions {
return {
...inferredCond,
...cond,
inputTypes: unique([
...(inferredCond.inputTypes ?? []),
...(cond.inputTypes ?? []),
]),
attachmentKinds: unique([
...(inferredCond.attachmentKinds ?? []),
...(cond.attachmentKinds ?? []),
]),
attachmentSourceKinds: unique([
...(inferredCond.attachmentSourceKinds ?? []),
...(cond.attachmentSourceKinds ?? []),
]),
hasRemoteAttachments:
cond.hasRemoteAttachments ?? inferredCond.hasRemoteAttachments,
};
}
export function getAttachCapability(
model: CopilotProviderModel,
outputType: ModelOutputType
): ModelAttachmentCapability | undefined {
const capability =
model.capabilities.find(cap => cap.output.includes(outputType)) ??
model.capabilities[0];
if (!capability) {
return;
}
return resolveAttachmentCapability(capability, outputType);
}
export function matchProviderModel(
context: ProviderModelRuntimeContext,
cond: ModelFullConditions
): boolean {
return !!resolveProviderModelSelection(context, cond);
}
export function resolveProviderModel(
context: ProviderModelRuntimeContext,
modelId: string
): ResolvedProviderModel | undefined {
return resolveProviderModelSelection(context, {
modelId,
})?.model;
}
export function hasProviderModelBehaviorFlag(
model: CopilotProviderModel,
flag: string
) {
const behaviorFlags = (model as ResolvedProviderModel).behaviorFlags;
return Array.isArray(behaviorFlags) && behaviorFlags.includes(flag);
}
export function resolveProviderModelRoute(
model: CopilotProviderModel,
outputType: ModelOutputType
) {
const resolved = model as ResolvedProviderModel;
const override = resolved.routeOverrides?.[outputType];
return {
protocol: override?.protocol ?? resolved.protocol,
requestLayer: override?.requestLayer ?? resolved.requestLayer,
};
}
export function requireProviderModelSelection(
context: ProviderModelRuntimeContext,
cond: ModelFullConditions
): ResolvedProviderModel {
const selection = resolveProviderModelSelection(context, cond);
if (selection) return selection.model;
const { modelId, outputType, inputTypes } = cond;
throw new CopilotPromptInvalid(
modelId
? `Model ${modelId} does not support ${outputType ?? '<any>'} output with ${inputTypes ?? '<any>'} input`
: outputType
? `No model supports ${outputType} output with ${inputTypes ?? '<any>'} input for provider ${context.type}`
: 'Output type is required when modelId is not provided'
);
}
export async function checkProviderParams(
context: ProviderModelRuntimeContext,
{
cond,
messages,
embeddings,
options = {},
withAttachment = true,
}: {
cond: ModelFullConditions;
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
execution?: unknown;
}
): Promise<ModelFullConditions> {
if (messages) {
const { requireContent = true, requireAttachment = false } = options;
const MessageSchema = z
.array(
PromptMessageSchema.extend({
content: requireContent
? z.string().trim().min(1)
: z.string().optional().nullable(),
})
.passthrough()
.catchall(z.union([z.string(), z.number(), z.date(), z.null()]))
)
.optional();
handleZodError(MessageSchema.safeParse(messages));
const inferredCond = await inferModelConditionsFromMessages(
messages,
withAttachment
);
const mergedCond = mergeModelConditions(cond, inferredCond);
const model = requireProviderModelSelection(context, mergedCond);
const multimodal = isMultimodal(model);
if (
multimodal &&
requireAttachment &&
!messages.some(
message =>
message.role === 'user' &&
Array.isArray(message.attachments) &&
message.attachments.length > 0
)
) {
throw new CopilotPromptInvalid('attachments required in multimodal mode');
}
if (embeddings) {
handleZodError(EmbeddingMessage.safeParse(embeddings));
}
return mergedCond;
}
const inferredCond = await inferModelConditionsFromMessages(
messages,
withAttachment
);
const mergedCond = mergeModelConditions(cond, inferredCond);
if (embeddings) {
handleZodError(EmbeddingMessage.safeParse(embeddings));
}
return mergedCond;
}
@@ -1,337 +0,0 @@
import type {
LlmBackendConfig,
LlmEmbeddingRequest,
LlmProtocol,
LlmRerankRequest,
LlmStructuredRequest,
} from '../../../native';
import {
buildLlmImageRequestFromMessages,
llmEmbeddingDispatch,
llmRerankDispatch,
llmStructuredDispatch,
} from '../../../native';
import type { NodeTextMiddleware, ProviderMiddlewareConfig } from '../config';
import {
buildToolContracts,
projectPromptMessageForNative,
} from '../runtime/contracts';
import { buildNativeRequest } from '../runtime/native-request-runtime';
import type { ToolLoopBackend } from '../runtime/tool/bridge';
import type { NativeProviderAdapter } from '../runtime/tool/native-adapter';
import type { CopilotToolSet } from '../tools';
import type {
CopilotProviderExecution,
PreparedNativeEmbeddingExecution,
PreparedNativeExecution,
PreparedNativeImageExecution,
PreparedNativeRequestOptions,
PreparedNativeRerankExecution,
PreparedNativeStructuredExecution,
} from './provider-runtime-contract';
import type {
CopilotChatOptions,
CopilotImageOptions,
PromptMessage,
} from './types';
export type CreateToolAdapterOptions = {
maxSteps?: number;
nodeTextMiddleware?: NodeTextMiddleware[];
};
export type CreateNativeAdapter = (
backend: ToolLoopBackend,
tools: CopilotToolSet,
nodeTextMiddleware?: NodeTextMiddleware[],
options?: CreateToolAdapterOptions
) => NativeProviderAdapter;
export type CreatePreparedExecutionRuntimeInput = {
resolveProviderId: (execution?: CopilotProviderExecution) => string;
getTools: (
options: CopilotChatOptions,
model: string
) => Promise<CopilotToolSet>;
getActiveProviderMiddleware: (
execution?: CopilotProviderExecution
) => ProviderMiddlewareConfig;
createNativeAdapter: CreateNativeAdapter;
maxSteps: number;
};
export type PreparedExecutionRuntime = ReturnType<
typeof createPreparedExecutionRuntime
>;
export function createPreparedExecutionRuntime(
input: CreatePreparedExecutionRuntimeInput
) {
return {
buildPreparedNativeExecution: async (
prepared: PreparedNativeRequestOptions
) =>
await buildPreparedNativeExecution(
input.resolveProviderId(prepared.execution),
input.getTools,
input.getActiveProviderMiddleware,
input.maxSteps,
prepared
),
createPreparedExecutionAdapter: (prepared: PreparedNativeExecution) =>
createPreparedExecutionAdapter(
input.createNativeAdapter,
input.maxSteps,
prepared
),
buildPreparedNativeStructuredExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmStructuredRequest,
execution?: CopilotProviderExecution
) =>
buildPreparedNativeStructuredExecution(
input.resolveProviderId(execution),
protocol,
backendConfig,
model,
request
),
buildPreparedNativeEmbeddingExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmEmbeddingRequest,
execution?: CopilotProviderExecution
) =>
buildPreparedNativeEmbeddingExecution(
input.resolveProviderId(execution),
protocol,
backendConfig,
model,
request
),
buildPreparedNativeRerankExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmRerankRequest,
execution?: CopilotProviderExecution
) =>
buildPreparedNativeRerankExecution(
input.resolveProviderId(execution),
protocol,
backendConfig,
model,
request
),
buildPreparedNativeImageExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
messages: PromptMessage[],
options: CopilotImageOptions = {},
execution?: CopilotProviderExecution
) =>
buildPreparedNativeImageExecution(
input.resolveProviderId(execution),
protocol,
backendConfig,
model,
messages,
options
),
};
}
export function createPreparedExecutionAdapter(
createNativeAdapter: CreateNativeAdapter,
maxSteps: number,
prepared: PreparedNativeExecution
) {
return createNativeAdapter(
{
protocol: prepared.route.protocol,
backendConfig: prepared.route.backendConfig,
},
prepared.tools,
prepared.postprocess?.nodeTextMiddleware,
{
maxSteps,
nodeTextMiddleware: prepared.postprocess?.nodeTextMiddleware,
}
);
}
export function createNativeStructuredDispatch(
backendConfig: LlmBackendConfig,
protocol: LlmProtocol
) {
return (request: LlmStructuredRequest) =>
llmStructuredDispatch(protocol, backendConfig, request);
}
export function createNativeEmbeddingDispatch(
backendConfig: LlmBackendConfig,
protocol: LlmProtocol
) {
return (request: LlmEmbeddingRequest) =>
llmEmbeddingDispatch(protocol, backendConfig, request);
}
export function createNativeRerankDispatch(
backendConfig: LlmBackendConfig,
protocol: LlmProtocol
) {
return (request: LlmRerankRequest) =>
llmRerankDispatch(protocol, backendConfig, request);
}
function buildPreparedRoute(
providerId: string,
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string
): PreparedNativeExecution['route'] {
return {
providerId,
protocol,
requestLayer: backendConfig.request_layer,
model,
backendConfig,
};
}
export async function buildPreparedNativeExecution(
providerId: string,
getTools: (
options: CopilotChatOptions,
model: string
) => Promise<CopilotToolSet>,
getActiveProviderMiddleware: (
execution?: CopilotProviderExecution
) => ProviderMiddlewareConfig,
maxSteps: number,
{
protocol,
backendConfig,
model,
messages,
options = {},
execution,
withAttachment = true,
attachmentCapability,
include,
reasoning,
tools,
middleware,
}: PreparedNativeRequestOptions
): Promise<PreparedNativeExecution> {
const resolvedTools = tools ?? (await getTools(options, model));
const resolvedMiddleware =
middleware ?? getActiveProviderMiddleware(execution);
const { request } = await buildNativeRequest({
model,
messages,
options,
toolContracts: buildToolContracts(resolvedTools),
withAttachment,
attachmentCapability,
include,
reasoning,
middleware: resolvedMiddleware,
});
return {
route: buildPreparedRoute(providerId, protocol, backendConfig, model),
request,
tools: resolvedTools,
maxSteps,
postprocess: {
nodeTextMiddleware: resolvedMiddleware.node?.text,
},
};
}
type BuildPreparedNativeDispatchExecution = <
TRequest extends
| LlmStructuredRequest
| LlmEmbeddingRequest
| LlmRerankRequest,
>(
providerId: string,
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: TRequest
) => {
route: PreparedNativeExecution['route'];
request: TRequest;
};
const buildPreparedNativeDispatchExecution: BuildPreparedNativeDispatchExecution =
(providerId, protocol, backendConfig, model, request) => {
return {
route: buildPreparedRoute(providerId, protocol, backendConfig, model),
request,
};
};
export const buildPreparedNativeStructuredExecution =
buildPreparedNativeDispatchExecution as (
providerId: string,
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmStructuredRequest
) => PreparedNativeStructuredExecution;
export const buildPreparedNativeEmbeddingExecution =
buildPreparedNativeDispatchExecution as (
providerId: string,
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmEmbeddingRequest
) => PreparedNativeEmbeddingExecution;
export const buildPreparedNativeRerankExecution =
buildPreparedNativeDispatchExecution as (
providerId: string,
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmRerankRequest
) => PreparedNativeRerankExecution;
export function buildPreparedNativeImageExecution(
providerId: string,
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
messages: PromptMessage[],
options: CopilotImageOptions = {}
): PreparedNativeImageExecution {
const nativeMessages = messages.map(
message => projectPromptMessageForNative(message).message
);
return {
route: buildPreparedRoute(providerId, protocol, backendConfig, model),
request: buildLlmImageRequestFromMessages({
model,
protocol,
messages: nativeMessages,
options: projectImageRequestOptions(options),
}),
};
}
function projectImageRequestOptions(options: CopilotImageOptions = {}) {
return {
quality: options.quality,
seed: options.seed,
modelName: options.modelName,
loras: options.loras,
};
}
@@ -1,287 +0,0 @@
import type {
CopilotProviderConfigMap,
CopilotProviderDefaults,
CopilotProviderProfile,
ProviderMiddlewareConfig,
} from '../config';
import { resolveProviderMiddleware } from './provider-middleware';
import { CopilotProviderType, ModelOutputType } from './types';
const PROVIDER_ID_PATTERN = /^[a-zA-Z0-9-_]+$/;
const LEGACY_PROVIDER_ORDER: CopilotProviderType[] = [
CopilotProviderType.OpenAI,
CopilotProviderType.CloudflareWorkersAi,
CopilotProviderType.FAL,
CopilotProviderType.Gemini,
CopilotProviderType.GeminiVertex,
CopilotProviderType.Anthropic,
CopilotProviderType.AnthropicVertex,
];
const LEGACY_PROVIDER_PRIORITY = LEGACY_PROVIDER_ORDER.reduce(
(acc, type, index) => {
acc[type] = LEGACY_PROVIDER_ORDER.length - index;
return acc;
},
{} as Record<CopilotProviderType, number>
);
type LegacyProvidersConfig = Partial<
Record<CopilotProviderType, CopilotProviderConfigMap[CopilotProviderType]>
>;
export type CopilotProvidersConfigInput = LegacyProvidersConfig & {
profiles?: CopilotProviderProfile[] | null;
defaults?: CopilotProviderDefaults | null;
};
export type NormalizedCopilotProviderProfile = Omit<
CopilotProviderProfile,
'enabled' | 'priority' | 'middleware'
> & {
enabled: boolean;
priority: number;
middleware: ProviderMiddlewareConfig;
};
export type CopilotProviderRegistry = {
profiles: Map<string, NormalizedCopilotProviderProfile>;
defaults: CopilotProviderDefaults;
order: string[];
byType: Map<CopilotProviderType, string[]>;
};
export type ResolveModelResult = {
rawModelId?: string;
modelId?: string;
explicitProviderId?: string;
candidateProviderIds: string[];
};
type ResolveModelOptions = {
registry: CopilotProviderRegistry;
modelId?: string;
outputType?: ModelOutputType;
availableProviderIds?: Iterable<string>;
preferredProviderIds?: Iterable<string>;
};
function unique<T>(list: T[]): T[] {
return [...new Set(list)];
}
function asArray<T>(iter?: Iterable<T>): T[] {
return iter ? Array.from(iter) : [];
}
function parseModelPrefix(
registry: CopilotProviderRegistry,
modelId: string
): { providerId: string; modelId?: string } | null {
const index = modelId.indexOf('/');
if (index <= 0) {
return null;
}
const providerId = modelId.slice(0, index);
if (!registry.profiles.has(providerId)) {
return null;
}
const model = modelId.slice(index + 1);
return { providerId, modelId: model || undefined };
}
function normalizeProfile(
profile: CopilotProviderProfile
): NormalizedCopilotProviderProfile {
return {
...profile,
enabled: profile.enabled !== false,
priority: profile.priority ?? 0,
middleware: resolveProviderMiddleware(profile.type, profile.middleware),
};
}
function toLegacyProfiles(
config: CopilotProvidersConfigInput
): CopilotProviderProfile[] {
const legacyProfiles: CopilotProviderProfile[] = [];
for (const type of LEGACY_PROVIDER_ORDER) {
const legacyConfig = config[type];
if (!legacyConfig) {
continue;
}
legacyProfiles.push({
id: `${type}-default`,
type,
priority: LEGACY_PROVIDER_PRIORITY[type],
config: legacyConfig,
} as CopilotProviderProfile);
}
return legacyProfiles;
}
function mergeProfiles(
explicitProfiles: CopilotProviderProfile[],
legacyProfiles: CopilotProviderProfile[]
): CopilotProviderProfile[] {
const profiles = new Map<string, CopilotProviderProfile>();
for (const profile of explicitProfiles) {
if (!PROVIDER_ID_PATTERN.test(profile.id)) {
throw new Error(`Invalid copilot provider profile id: ${profile.id}`);
}
if (profiles.has(profile.id)) {
throw new Error(`Duplicated copilot provider profile id: ${profile.id}`);
}
profiles.set(profile.id, profile);
}
for (const profile of legacyProfiles) {
if (!profiles.has(profile.id)) {
profiles.set(profile.id, profile);
}
}
return Array.from(profiles.values());
}
function sortProfiles(profiles: NormalizedCopilotProviderProfile[]) {
return profiles.toSorted((a, b) => {
if (a.priority !== b.priority) {
return b.priority - a.priority;
}
return a.id.localeCompare(b.id);
});
}
function assertDefaults(
defaults: CopilotProviderDefaults,
profiles: Map<string, NormalizedCopilotProviderProfile>
) {
for (const providerId of Object.values(defaults)) {
if (!providerId) {
continue;
}
if (!profiles.has(providerId)) {
throw new Error(
`Copilot provider defaults references unknown providerId: ${providerId}`
);
}
}
}
export function buildProviderRegistry(
config: CopilotProvidersConfigInput
): CopilotProviderRegistry {
const explicitProfiles = config.profiles ?? [];
const legacyProfiles = toLegacyProfiles(config);
const mergedProfiles = mergeProfiles(explicitProfiles, legacyProfiles)
.map(normalizeProfile)
.filter(profile => profile.enabled);
const sortedProfiles = sortProfiles(mergedProfiles);
const profiles = new Map(
sortedProfiles.map(profile => [profile.id, profile] as const)
);
const defaults = config.defaults ?? {};
assertDefaults(defaults, profiles);
const order = sortedProfiles.map(profile => profile.id);
const byType = new Map<CopilotProviderType, string[]>();
for (const profile of sortedProfiles) {
const ids = byType.get(profile.type) ?? [];
ids.push(profile.id);
byType.set(profile.type, ids);
}
return { profiles, defaults, order, byType };
}
export function resolveModel({
registry,
modelId,
outputType,
availableProviderIds,
preferredProviderIds,
}: ResolveModelOptions): ResolveModelResult {
const available = new Set(asArray(availableProviderIds));
const preferred = new Set(asArray(preferredProviderIds));
const hasAvailableFilter = available.size > 0;
const hasPreferredFilter = preferred.size > 0;
const isAllowed = (providerId: string) => {
const profile = registry.profiles.get(providerId);
if (!profile?.enabled) {
return false;
}
if (hasAvailableFilter && !available.has(providerId)) {
return false;
}
if (hasPreferredFilter && !preferred.has(providerId)) {
return false;
}
return true;
};
const prefixed = modelId ? parseModelPrefix(registry, modelId) : null;
if (prefixed) {
return {
rawModelId: modelId,
modelId: prefixed.modelId,
explicitProviderId: prefixed.providerId,
candidateProviderIds: isAllowed(prefixed.providerId)
? [prefixed.providerId]
: [],
};
}
if (modelId) {
return {
rawModelId: modelId,
modelId,
candidateProviderIds: registry.order.filter(providerId =>
isAllowed(providerId)
),
};
}
const defaultProviderId =
outputType && outputType !== ModelOutputType.Rerank
? registry.defaults[outputType]
: undefined;
const fallbackOrder = [
...(defaultProviderId ? [defaultProviderId] : []),
registry.defaults.fallback,
...registry.order,
].filter((id): id is string => !!id);
return {
rawModelId: modelId,
modelId,
candidateProviderIds: unique(
fallbackOrder.filter(providerId => isAllowed(providerId))
),
};
}
export function stripProviderPrefix(
registry: CopilotProviderRegistry,
providerId: string,
modelId?: string
) {
if (!modelId) {
return modelId;
}
const prefixed = parseModelPrefix(registry, modelId);
if (!prefixed) {
return modelId;
}
if (prefixed.providerId !== providerId) {
return modelId;
}
return prefixed.modelId;
}
@@ -1,456 +0,0 @@
import type {
LlmBackendConfig,
LlmEmbeddingRequest,
LlmImageRequest,
LlmProtocol,
LlmRequest,
LlmRerankRequest,
LlmStructuredRequest,
} from '../../../native';
import type { NodeTextMiddleware, ProviderMiddlewareConfig } from '../config';
import type { CopilotToolSet } from '../tools';
import {
type ProviderModelRuntimeContext,
resolveProviderModelRoute,
} from './provider-model-runtime';
import type { NormalizedCopilotProviderProfile } from './provider-registry';
import {
CopilotChatOptions,
CopilotImageOptions,
CopilotProviderModel,
CopilotStructuredOptions,
ModelAttachmentCapability,
ModelConditions,
ModelFullConditions,
ModelOutputType,
PromptMessage,
} from './types';
export type NativeExecutionRoute = {
protocol: LlmProtocol;
requestLayer?: LlmBackendConfig['request_layer'];
model: string;
backendConfig: LlmBackendConfig;
};
export type CopilotProviderExecution = {
providerId: string;
profile: NormalizedCopilotProviderProfile;
};
export type PreparedNativeExecution = {
route: NativeExecutionRoute & {
providerId: string;
};
request: LlmRequest;
tools: CopilotToolSet;
maxSteps?: number;
postprocess?: {
nodeTextMiddleware?: NodeTextMiddleware[];
};
};
export type PreparedNativeStructuredExecution = {
route: NativeExecutionRoute & {
providerId: string;
};
request: LlmStructuredRequest;
};
export type PreparedNativeEmbeddingExecution = {
route: NativeExecutionRoute & {
providerId: string;
};
request: LlmEmbeddingRequest;
};
export type PreparedNativeRerankExecution = {
route: NativeExecutionRoute & {
providerId: string;
};
request: LlmRerankRequest;
};
export type PreparedNativeImageExecution = {
route: NativeExecutionRoute & {
providerId: string;
};
request: LlmImageRequest;
};
export type PreparedNativeRequestOptions = {
protocol: LlmProtocol;
backendConfig: LlmBackendConfig;
model: string;
messages: PromptMessage[];
options?: CopilotChatOptions;
execution?: CopilotProviderExecution;
withAttachment?: boolean;
attachmentCapability?: ModelAttachmentCapability;
include?: string[];
reasoning?: Record<string, unknown>;
tools?: CopilotToolSet;
middleware?: ProviderMiddlewareConfig;
};
type ProviderChatDriverPrepareResult = Omit<
PreparedNativeRequestOptions,
'execution' | 'options'
>;
type Awaitable<T> = T | Promise<T>;
type NativeBackendConfigResolver = (
execution?: CopilotProviderExecution
) => Awaitable<LlmBackendConfig>;
export type StructuredProviderDriver = {
createBackendConfig: NativeBackendConfigResolver;
prepareMessages?: (
messages: PromptMessage[],
backendConfig: LlmBackendConfig,
options: NonNullable<CopilotStructuredOptions>
) => Promise<PromptMessage[]>;
shouldRetry?: (context: {
error: unknown;
attempt: number;
options: NonNullable<CopilotStructuredOptions>;
}) => Awaitable<boolean>;
mapError: (error: unknown) => unknown;
};
export type EmbeddingProviderDriver = {
createBackendConfig: NativeBackendConfigResolver;
defaultDimensions?: number;
taskType?: string;
mapError: (error: unknown) => unknown;
};
export type RerankProviderDriver = {
createBackendConfig: NativeBackendConfigResolver;
mapError: (error: unknown) => unknown;
};
export type ImageProviderDriver = {
createBackendConfig: NativeBackendConfigResolver;
prepareMessages?: (
messages: PromptMessage[],
backendConfig: LlmBackendConfig,
options: NonNullable<CopilotImageOptions>
) => Promise<PromptMessage[]>;
mapError: (error: unknown) => unknown;
};
export type ProviderMetricLabels = Record<
string,
string | number | boolean | undefined
>;
export type ProviderExecutionDrivers = {
chat?: ProviderChatDriver;
structured?: StructuredProviderDriver;
embedding?: EmbeddingProviderDriver;
rerank?: RerankProviderDriver;
image?: ImageProviderDriver;
};
export type ProviderDriverSpec = NativeProviderDriverBase & {
chat?: NativeChatDriverOverrides | false;
structured?: NativeStructuredDriverOverrides | false;
embedding?: NativeEmbeddingDriverOverrides | false;
rerank?: NativeRerankDriverOverrides | false;
image?: NativeImageDriverOverrides | false;
};
export type ProviderRuntimeHostSeed = {
model: ProviderModelRuntimeContext;
resolveExecutionDrivers: () => ProviderExecutionDrivers | undefined;
selectModel: NativeChatDriverBase['selectModel'];
checkParams: NativeChatDriverBase['checkParams'];
getAttachCapability: (
model: CopilotProviderModel,
outputType: ModelOutputType
) => ModelAttachmentCapability | undefined;
getActiveProviderMiddleware: (
execution?: CopilotProviderExecution
) => ProviderMiddlewareConfig;
getTools: (
options: CopilotChatOptions,
model: string
) => Promise<CopilotToolSet>;
metricLabels: (
model: string,
labels?: ProviderMetricLabels,
execution?: CopilotProviderExecution
) => ProviderMetricLabels;
};
export type ProviderChatDriverPrepareInput = {
kind: 'text' | 'streamText' | 'streamObject';
cond: ModelConditions;
messages: PromptMessage[];
options: CopilotChatOptions;
execution?: CopilotProviderExecution;
};
export type ProviderChatDriver = {
prepare: (input: {
kind: ProviderChatDriverPrepareInput['kind'];
cond: ProviderChatDriverPrepareInput['cond'];
messages: ProviderChatDriverPrepareInput['messages'];
options: ProviderChatDriverPrepareInput['options'];
execution?: ProviderChatDriverPrepareInput['execution'];
}) => Promise<ProviderChatDriverPrepareResult | null>;
mapError: (error: unknown) => unknown;
};
type NativeProviderDriverBase = Pick<
StructuredProviderDriver,
'createBackendConfig' | 'mapError'
>;
type ChatToolingResult = Pick<
ProviderChatDriverPrepareResult,
'tools' | 'middleware'
>;
type NativeChatDriverBase = NativeProviderDriverBase & {
checkParams: (input: {
cond: ModelFullConditions;
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
execution?: CopilotProviderExecution;
}) => Promise<ModelFullConditions>;
selectModel: (
cond: ModelFullConditions,
execution?: CopilotProviderExecution
) => CopilotProviderModel;
getTools?: (
options: CopilotChatOptions,
model: string
) => Promise<CopilotToolSet>;
getActiveProviderMiddleware?: (
execution?: CopilotProviderExecution
) => ProviderMiddlewareConfig;
};
type NativeStructuredDriverOverrides = Partial<StructuredProviderDriver>;
type NativeEmbeddingDriverOverrides = Partial<EmbeddingProviderDriver>;
type NativeRerankDriverOverrides = Partial<RerankProviderDriver>;
type NativeImageDriverOverrides = Partial<ImageProviderDriver>;
type NativeChatDriverContext = {
input: ProviderChatDriverPrepareInput;
outputType: ModelOutputType;
normalizedCond: ModelFullConditions;
model: CopilotProviderModel;
backendConfig: LlmBackendConfig;
protocol: LlmProtocol;
messages: PromptMessage[];
options: NonNullable<CopilotChatOptions>;
execution?: CopilotProviderExecution;
};
type NativeChatDriverOverrides = {
resolveOutputType?: (
kind: ProviderChatDriverPrepareInput['kind']
) => ModelOutputType | null;
withAttachment?: boolean;
prepareMessages?: (
context: Omit<NativeChatDriverContext, 'messages'>
) => Awaitable<PromptMessage[]>;
resolveTooling?: (
context: NativeChatDriverContext
) => Awaitable<ChatToolingResult>;
resolveRequestOptions?: (
context: NativeChatDriverContext
) => Awaitable<
Partial<
Pick<
ProviderChatDriverPrepareResult,
'withAttachment' | 'attachmentCapability' | 'include' | 'reasoning'
>
>
>;
};
export function createNativeProviderDriverFactory(
base: NativeProviderDriverBase
) {
return {
structured(
overrides: NativeStructuredDriverOverrides = {}
): StructuredProviderDriver {
return {
createBackendConfig:
overrides.createBackendConfig ?? base.createBackendConfig,
mapError: overrides.mapError ?? base.mapError,
...(overrides.prepareMessages
? { prepareMessages: overrides.prepareMessages }
: {}),
...(overrides.shouldRetry
? { shouldRetry: overrides.shouldRetry }
: {}),
};
},
embedding(
overrides: NativeEmbeddingDriverOverrides = {}
): EmbeddingProviderDriver {
return {
createBackendConfig:
overrides.createBackendConfig ?? base.createBackendConfig,
mapError: overrides.mapError ?? base.mapError,
...(overrides.defaultDimensions !== undefined
? { defaultDimensions: overrides.defaultDimensions }
: {}),
...(overrides.taskType ? { taskType: overrides.taskType } : {}),
};
},
rerank(overrides: NativeRerankDriverOverrides = {}): RerankProviderDriver {
return {
createBackendConfig:
overrides.createBackendConfig ?? base.createBackendConfig,
mapError: overrides.mapError ?? base.mapError,
};
},
image(overrides: NativeImageDriverOverrides = {}): ImageProviderDriver {
return {
createBackendConfig:
overrides.createBackendConfig ?? base.createBackendConfig,
mapError: overrides.mapError ?? base.mapError,
...(overrides.prepareMessages
? { prepareMessages: overrides.prepareMessages }
: {}),
};
},
};
}
function compileProviderChatDriver(
spec: NativeProviderDriverBase & NativeChatDriverOverrides,
base: NativeChatDriverBase
): ProviderChatDriver {
return {
prepare: async (input: ProviderChatDriverPrepareInput) => {
const options: NonNullable<CopilotChatOptions> = input.options ?? {};
const resolvedOutputType = spec.resolveOutputType?.(input.kind);
const outputType =
resolvedOutputType === undefined
? input.kind === 'streamObject'
? ModelOutputType.Object
: ModelOutputType.Text
: resolvedOutputType;
if (!outputType) {
return null;
}
const normalizedCond = await base.checkParams({
messages: input.messages,
cond: {
...input.cond,
outputType,
},
options,
execution: input.execution,
...(spec.withAttachment !== undefined
? { withAttachment: spec.withAttachment }
: {}),
});
const model = base.selectModel(normalizedCond, input.execution);
const backendConfig = await spec.createBackendConfig(input.execution);
const route = resolveProviderModelRoute(model, outputType);
if (!route.protocol) {
throw new Error(`Missing native protocol for model ${model.id}`);
}
const partialContext = {
input,
outputType,
normalizedCond,
model,
backendConfig:
route.requestLayer === backendConfig.request_layer
? backendConfig
: { ...backendConfig, request_layer: route.requestLayer },
protocol: route.protocol,
options,
execution: input.execution,
};
const messages = spec.prepareMessages
? await spec.prepareMessages(partialContext)
: input.messages;
const context = {
...partialContext,
messages,
};
const tooling = spec.resolveTooling
? await spec.resolveTooling(context)
: {
...(base.getTools
? { tools: await base.getTools(options, model.id) }
: {}),
...(base.getActiveProviderMiddleware
? {
middleware: base.getActiveProviderMiddleware(input.execution),
}
: {}),
};
const requestOptions = spec.resolveRequestOptions
? await spec.resolveRequestOptions(context)
: {};
return {
protocol: context.protocol,
backendConfig: context.backendConfig,
model: model.id,
messages,
...(spec.withAttachment === false ? { withAttachment: false } : {}),
...requestOptions,
...tooling,
};
},
mapError: spec.mapError,
};
}
export function createNativeExecutionDriverSpec(
input: ProviderDriverSpec,
runtimeBase: NativeChatDriverBase
): ProviderExecutionDrivers {
const driverBase = {
createBackendConfig: input.createBackendConfig,
mapError: input.mapError,
};
const nativeDrivers = createNativeProviderDriverFactory(driverBase);
return {
...(input.chat !== false
? {
chat: compileProviderChatDriver(
{ ...driverBase, ...input.chat },
runtimeBase
),
}
: {}),
...(input.structured !== false
? {
structured: nativeDrivers.structured(input.structured ?? undefined),
}
: {}),
...(input.embedding !== false
? {
embedding: nativeDrivers.embedding(input.embedding ?? undefined),
}
: {}),
...(input.rerank !== false
? { rerank: nativeDrivers.rerank(input.rerank ?? undefined) }
: {}),
...(input.image !== false
? { image: nativeDrivers.image(input.image ?? undefined) }
: {}),
};
}
@@ -1,18 +0,0 @@
import {
AnthropicOfficialProvider,
AnthropicVertexProvider,
} from './anthropic';
import { CloudflareWorkersAIProvider } from './cloudflare';
import { FalProvider } from './fal';
import { GeminiGenerativeProvider, GeminiVertexProvider } from './gemini';
import { OpenAIProvider } from './openai';
export const CopilotProviders = [
OpenAIProvider,
CloudflareWorkersAIProvider,
FalProvider,
GeminiGenerativeProvider,
GeminiVertexProvider,
AnthropicOfficialProvider,
AnthropicVertexProvider,
];
@@ -1,249 +0,0 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { Config } from '../../../base';
import type { NodeTextMiddleware, ProviderMiddlewareConfig } from '../config';
import { ToolExecutorHost } from '../runtime/hosts/tool-executor-host';
import { mapNativeSemanticError } from '../runtime/native-errors';
import type { ToolLoopBackend } from '../runtime/tool/bridge';
import type { CopilotTool, CopilotToolSet } from '../tools';
import { resolveProviderMiddleware } from './provider-middleware';
import {
checkProviderParams,
getAttachCapability as getAttachCapabilityHelper,
matchProviderModel as matchProviderModelHelper,
type ProviderModelRuntimeContext,
requireProviderModelSelection,
resolveProviderModel,
} from './provider-model-runtime';
import {
type CopilotProviderExecution,
createNativeExecutionDriverSpec,
type ProviderDriverSpec,
type ProviderExecutionDrivers,
type ProviderRuntimeHostSeed,
} from './provider-runtime-contract';
import {
type CopilotChatOptions,
CopilotChatTools,
type CopilotImageOptions,
type CopilotModelBackendKind,
CopilotProviderModel,
CopilotProviderType,
type CopilotStructuredOptions,
type ModelAttachmentCapability,
ModelFullConditions,
ModelOutputType,
type PromptMessage,
} from './types';
export type {
CopilotProviderExecution,
ProviderDriverSpec,
ProviderExecutionDrivers,
ProviderRuntimeHostSeed,
} from './provider-runtime-contract';
@Injectable()
export abstract class CopilotProvider<C = any> {
protected readonly logger = new Logger(this.constructor.name);
protected readonly MAX_STEPS = 20;
abstract readonly type: CopilotProviderType;
protected abstract resolveModelBackendKind(
execution?: CopilotProviderExecution
): CopilotModelBackendKind;
abstract configured(execution?: CopilotProviderExecution): boolean;
@Inject() protected readonly AFFiNEConfig!: Config;
@Inject() protected readonly toolExecutorHost!: ToolExecutorHost;
get maxSteps() {
return this.MAX_STEPS;
}
protected resolveModelRuntimeContext(
execution?: CopilotProviderExecution
): ProviderModelRuntimeContext {
return {
type: this.type,
backendKind: this.resolveModelBackendKind(execution),
};
}
protected get modelRuntimeContext(): ProviderModelRuntimeContext {
return this.resolveModelRuntimeContext();
}
getDriverSpec(): ProviderDriverSpec | undefined {
return undefined;
}
getExecutionDrivers(): ProviderExecutionDrivers | undefined {
const spec = this.getDriverSpec();
return spec ? this.createDriverSpec(spec) : undefined;
}
protected createDriverSpec(
spec: ProviderDriverSpec
): ProviderExecutionDrivers {
return createNativeExecutionDriverSpec(spec, {
createBackendConfig: spec.createBackendConfig,
mapError: error => {
const mapped = mapNativeSemanticError(error);
return mapped === error ? spec.mapError(error) : mapped;
},
checkParams: input =>
checkProviderParams(
this.resolveModelRuntimeContext(input.execution),
input
),
selectModel: (cond, execution) =>
requireProviderModelSelection(
this.resolveModelRuntimeContext(execution),
cond
),
getTools: this.getTools.bind(this),
getActiveProviderMiddleware: this.getActiveProviderMiddleware.bind(this),
});
}
selectModel(
cond: ModelFullConditions,
execution?: CopilotProviderExecution
): CopilotProviderModel {
return requireProviderModelSelection(
this.resolveModelRuntimeContext(execution),
cond
);
}
checkParams(input: {
cond: ModelFullConditions;
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
execution?: CopilotProviderExecution;
}) {
return checkProviderParams(
this.resolveModelRuntimeContext(input.execution),
input
);
}
getRuntimeHostSeed(): ProviderRuntimeHostSeed {
return {
model: this.resolveModelRuntimeContext(),
resolveExecutionDrivers: () => this.getExecutionDrivers(),
selectModel: this.selectModel.bind(this),
checkParams: this.checkParams.bind(this),
getAttachCapability: this.getAttachCapability.bind(this),
getActiveProviderMiddleware: this.getActiveProviderMiddleware.bind(this),
getTools: this.getTools.bind(this),
metricLabels: this.metricLabels.bind(this),
};
}
protected getExecutionProfile(execution?: CopilotProviderExecution) {
return execution?.profile?.type === this.type
? execution.profile
: undefined;
}
getActiveProviderMiddleware(
execution?: CopilotProviderExecution
): ProviderMiddlewareConfig {
return (
this.getExecutionProfile(execution)?.middleware ??
resolveProviderMiddleware(this.type)
);
}
metricLabels(
model: string,
labels: Record<string, string | number | boolean | undefined> = {},
execution?: CopilotProviderExecution
) {
return {
model,
providerId: execution?.providerId ?? `${this.type}-default`,
...labels,
};
}
protected get config(): C {
return this.AFFiNEConfig.copilot.providers[this.type] as C;
}
protected getConfig(execution?: CopilotProviderExecution): C {
const profile = this.getExecutionProfile(execution);
if (profile) {
return profile.config as C;
}
return this.config;
}
getAttachCapability(
model: CopilotProviderModel,
outputType: ModelOutputType
): ModelAttachmentCapability | undefined {
return getAttachCapabilityHelper(model, outputType);
}
// make it async to allow dynamic check available models in some providers
async match(
cond: ModelFullConditions = {},
execution?: CopilotProviderExecution
): Promise<boolean> {
return (
this.configured(execution) &&
matchProviderModelHelper(this.resolveModelRuntimeContext(execution), cond)
);
}
resolveModel(
modelId: string,
execution?: CopilotProviderExecution
): CopilotProviderModel | undefined {
return resolveProviderModel(
this.resolveModelRuntimeContext(execution),
modelId
);
}
protected getProviderSpecificTools(
_toolName: CopilotChatTools,
_model: string
): [string, CopilotTool?] | undefined {
return;
}
// use for tool use, shared between providers
async getTools(
options: CopilotChatOptions,
model: string
): Promise<CopilotToolSet> {
this.logger.debug(`getTools: ${JSON.stringify(options?.tools ?? [])}`);
return await this.toolExecutorHost.getTools(
options,
model,
this.getProviderSpecificTools.bind(this)
);
}
createNativeAdapter(
backend: ToolLoopBackend,
tools: CopilotToolSet,
nodeTextMiddleware?: NodeTextMiddleware[],
options: {
maxSteps?: number;
nodeTextMiddleware?: NodeTextMiddleware[];
} = {}
) {
return this.toolExecutorHost.createNativeAdapter(backend, tools, {
...options,
nodeTextMiddleware: nodeTextMiddleware ?? options.nodeTextMiddleware,
});
}
}
@@ -1,28 +0,0 @@
import { Injectable } from '@nestjs/common';
import { Config } from '../../../base';
import {
buildProviderRegistry,
type CopilotProviderRegistry,
type CopilotProvidersConfigInput,
} from './provider-registry';
@Injectable()
export class CopilotProviderRegistryService {
private lastConfig?: CopilotProvidersConfigInput;
private lastRegistry?: CopilotProviderRegistry;
constructor(private readonly config: Config) {}
getRegistry(): CopilotProviderRegistry {
const providerConfig = this.config.copilot.providers;
if (this.lastConfig === providerConfig && this.lastRegistry) {
return this.lastRegistry;
}
const registry = buildProviderRegistry(providerConfig);
this.lastConfig = providerConfig;
this.lastRegistry = registry;
return registry;
}
}
@@ -1,4 +1,4 @@
import { AiPromptRole } from '@prisma/client';
import { AiSessionMessageRole } from '@prisma/client';
import { z } from 'zod';
import { JSONSchema } from '../../../base';
@@ -7,7 +7,6 @@ import type {
CapabilityModelCapability,
ModelConditionsContract,
} from '../../../native';
import type { CopilotModelBackendKind } from '../runtime/contracts';
import {
type StreamObject,
StreamObjectSchema,
@@ -97,7 +96,6 @@ export const PromptToolsSchema = z
export const PromptConfigStrictSchema = z.object({
tools: PromptToolsSchema.nullable().optional(),
proModels: z.array(z.string()).nullable().optional(),
// params requirements
requireContent: z.boolean().nullable().optional(),
requireAttachment: z.boolean().nullable().optional(),
@@ -108,7 +106,7 @@ export const PromptConfigStrictSchema = z.object({
presencePenalty: z.number().nullable().optional(),
temperature: z.number().nullable().optional(),
topP: z.number().nullable().optional(),
maxTokens: z.number().nullable().optional(),
maxOutputTokens: z.number().nullable().optional(),
// fal
modelName: z.string().nullable().optional(),
loras: z
@@ -132,7 +130,7 @@ export type PromptTools = z.infer<typeof PromptToolsSchema>;
export const EmbeddingMessage = z.array(z.string().trim().min(1)).min(1);
export const ChatMessageRole = Object.values(AiPromptRole) as [
export const ChatMessageRole = Object.values(AiSessionMessageRole) as [
'system',
'assistant',
'user',
@@ -268,6 +266,8 @@ const CopilotProviderOptionsSchema = z.object({
billingUnitId: z.string().optional(),
taskId: z.string().optional(),
actionId: z.string().optional(),
builtInRouteId: z.string().optional(),
managedTargetId: z.string().optional(),
quotaBackedRoutesAllowed: z.boolean().optional(),
featureKind: z
.enum([
@@ -380,8 +380,8 @@ export interface CopilotProviderModel {
capabilities: ModelCapability[];
}
export type { CopilotModelBackendKind };
export type ModelConditions = Omit<ModelConditionsContract, 'outputType'>;
export type ModelConditions = Omit<ModelConditionsContract, 'outputType'> & {
profileId?: string;
};
export type ModelFullConditions = ModelConditionsContract;
@@ -31,11 +31,12 @@ import { CurrentUser } from '../../core/auth';
import { DocAction, PermissionAccess } from '../../core/permission';
import { UserType } from '../../core/user';
import type { ListSessionOptions, UpdateChatSession } from '../../models';
import { llmGetBuiltInRouteOptions } from '../../native';
import { ByokEntitlementPolicy } from './byok';
import { CompatHistoryProjector } from './compat/history-projector';
import { ConversationInboxService } from './conversation/inbox';
import { PromptService } from './prompt/service';
import { CopilotProviderFactory } from './providers/factory';
import { ModelOutputType, type StreamObject } from './providers/types';
import { CopilotEnabled } from './feature';
import type { StreamObject } from './providers/types';
import { ChatSessionService } from './session';
import { type ChatHistory, type ChatMessage, SubmittedMessage } from './types';
@@ -256,12 +257,6 @@ class CopilotHistoriesType implements Omit<ChatHistory, 'userId'> {
@Field(() => String)
promptName!: string;
@Field(() => String)
model!: string;
@Field(() => [String])
optionalModels!: string[];
@Field(() => String, {
description: 'An mark identifying which view to use to display the session',
nullable: true,
@@ -274,11 +269,6 @@ class CopilotHistoriesType implements Omit<ChatHistory, 'userId'> {
@Field(() => String, { nullable: true })
title!: string | null;
@Field(() => Number, {
description: 'The number of tokens used in the session',
})
tokens!: number;
@Field(() => [ChatMessageType])
messages!: ChatMessageType[];
@@ -303,27 +293,6 @@ class CopilotQuotaType {
used!: number;
}
@ObjectType()
class CopilotModelType {
@Field(() => String)
id!: string;
@Field(() => String)
name!: string;
}
@ObjectType()
export class CopilotModelsType {
@Field(() => String)
defaultModel!: string;
@Field(() => [CopilotModelType])
optionalModels!: CopilotModelType[];
@Field(() => [CopilotModelType])
proModels!: CopilotModelType[];
}
@ObjectType()
export class CopilotSessionType {
@Field(() => ID)
@@ -343,12 +312,33 @@ export class CopilotSessionType {
@Field(() => String)
promptName!: string;
}
@ObjectType('CopilotRouteTarget')
class CopilotRouteTargetType {
@Field(() => String)
id!: string;
@Field(() => String)
model!: string;
displayName!: string;
@Field(() => [String])
optionalModels!: string[];
@Field(() => String)
minimumTier!: string;
@Field(() => Boolean)
available!: boolean;
}
@ObjectType('CopilotRouteOptions')
class CopilotRouteOptionsType {
@Field(() => String)
routeId!: string;
@Field(() => String, { nullable: true })
defaultTargetId!: string | null;
@Field(() => [CopilotRouteTargetType])
choices!: CopilotRouteTargetType[];
}
// ================== Resolver ==================
@@ -360,20 +350,47 @@ export class CopilotType {
}
@Throttle()
@CopilotEnabled()
@Resolver(() => CopilotType)
export class CopilotResolver {
private readonly modelNames = new Map<string, string>();
constructor(
private readonly ac: PermissionAccess,
private readonly mutex: RequestMutex,
private readonly prompt: PromptService,
private readonly chatSession: ChatSessionService,
private readonly historyProjector: CompatHistoryProjector,
private readonly inbox: ConversationInboxService,
private readonly providerFactory: CopilotProviderFactory
private readonly entitlement: ByokEntitlementPolicy
) {}
@ResolveField(() => CopilotRouteOptionsType, {
nullable: true,
description: 'List native built-in route choices for a prompt',
complexity: 2,
})
async routeOptions(
@CurrentUser() user: CurrentUser,
@Args('promptName') promptName: string
): Promise<CopilotRouteOptionsType | null> {
const options = llmGetBuiltInRouteOptions(promptName);
if (!options) return null;
if (env.selfhosted) {
return { routeId: options.routeId, defaultTargetId: null, choices: [] };
}
const premium = await this.entitlement.hasAiPlan(user.id);
return {
routeId: options.routeId,
defaultTargetId: premium
? (options.premiumDefaultTargetId ?? null)
: (options.standardDefaultTargetId ?? null),
choices: options.choices.map(choice => ({
id: choice.id,
displayName: choice.displayName,
minimumTier: choice.minimumTier,
available: premium || choice.minimumTier === 'Standard',
})),
};
}
@ResolveField(() => CopilotQuotaType, {
name: 'quota',
description: 'Get the quota of the user in the workspace',
@@ -408,51 +425,6 @@ export class CopilotResolver {
return { userId: user.id, workspaceId, docId: docId || undefined };
}
@ResolveField(() => CopilotModelsType, {
description:
'List available models for a prompt, with human-readable names',
complexity: 2,
})
async models(
@Args('promptName') promptName: string
): Promise<CopilotModelsType> {
const prompt = await this.prompt.get(promptName);
if (!prompt) {
throw new NotFoundException('Prompt not found');
}
const convertModels = async (ids: string[]) => {
const models = await Promise.all(
ids.map(async id => {
const cachedName = this.modelNames.get(id);
if (cachedName) return { id, name: cachedName };
const resolved = await this.providerFactory.resolveProvider({
modelId: id,
outputType: ModelOutputType.Text,
});
const name = resolved?.provider.resolveModel(
resolved.modelId ?? id,
resolved.execution
)?.name;
if (name) {
this.modelNames.set(id, name);
return { id, name };
}
return null;
})
);
return models.filter(model => !!model) as CopilotModelType[];
};
const proModels = prompt.config?.proModels || [];
return {
defaultModel: prompt.model,
optionalModels: await convertModels(prompt.optionalModels),
proModels: await convertModels(proModels),
};
}
@ResolveField(() => CopilotSessionType, {
description: 'Get the session by id',
complexity: 2,
@@ -813,6 +785,7 @@ export class CopilotResolver {
}
@Throttle()
@CopilotEnabled()
@Resolver(() => UserType)
export class UserCopilotResolver {
constructor(private readonly ac: PermissionAccess) {}
@@ -110,7 +110,7 @@ function isImageAction(actionId: string) {
}
function resolveProjector(actionId: string): ActionResultProjector | null {
if (actionId.startsWith('transcript.audio.')) {
if (actionId === 'transcript.audio') {
return null;
}
if (isImageAction(actionId)) {
@@ -1,15 +1,10 @@
import { Injectable, Optional } from '@nestjs/common';
import { Injectable } from '@nestjs/common';
import { Models } from '../../../models';
import type { AiActionRunStatus } from '../../../models/copilot-action-run';
import {
type NativeActionEvent,
type NativeActionRuntimeInput,
runNativeActionRecipePreparedStream,
} from '../../../native';
import { type NativeActionEvent } from '../../../native';
import type {
CopilotImageOptions,
CopilotProviderType,
CopilotStructuredOptions,
PromptMessage,
} from '../providers/types';
@@ -18,18 +13,10 @@ import {
projectActionResultToAssistantTurn,
summarizeActionResult,
} from './action-output-projector';
import {
buildStructuredResponseFromSchemaJson,
type RequiredStructuredOutputContract,
} from './contracts';
import { ExecutionPlanBuilder } from './execution-plan';
import { CapabilityRuntime } from './capability-runtime';
import { type RequiredStructuredOutputContract } from './contracts';
import { TurnPersistence } from './hosts/turn-persistence';
type ActionRuntimeBridgeNativeInput = Omit<
NativeActionRuntimeInput,
'recipeId' | 'recipeVersion'
>;
export type ActionRuntimeBridgeInput = {
userId: string;
workspaceId: string;
@@ -42,26 +29,18 @@ export type ActionRuntimeBridgeInput = {
attempt?: number;
retryOf?: string | null;
inputSnapshot?: unknown;
nativeInput?: ActionRuntimeBridgeNativeInput;
onRunCreated?: (
context: ActionRuntimeBridgeRunContext
) => Promise<void> | void;
prepareStructuredRoutes?: {
stepId?: string;
step: {
slot: string;
builtInRouteId: string;
profileId?: string;
modelId?: string;
messages: PromptMessage[];
options?: CopilotStructuredOptions;
prefer?: CopilotProviderType;
responseSchemaJson?: Record<string, unknown>;
options?: CopilotStructuredOptions | CopilotImageOptions;
responseContract?: RequiredStructuredOutputContract;
};
prepareImageRoutes?: {
stepId?: string;
modelId?: string;
messages: PromptMessage[];
options?: CopilotImageOptions;
prefer?: CopilotProviderType;
};
persistAttachment?: (attachment: unknown) => Promise<unknown> | unknown;
signal?: AbortSignal;
};
@@ -107,94 +86,41 @@ export class ActionRuntimeBridge {
constructor(
private readonly models: Models,
private readonly turnPersistence: TurnPersistence,
@Optional() private readonly plans?: ExecutionPlanBuilder
private readonly runtime: CapabilityRuntime
) {}
protected runNativeStream(
input: NativeActionRuntimeInput,
signal?: AbortSignal
) {
return runNativeActionRecipePreparedStream(input, signal);
}
private async prepareNativeInput(
input: ActionRuntimeBridgeInput
): Promise<ActionRuntimeBridgeNativeInput & { input: unknown }> {
const nativeInput = {
...input.nativeInput,
input: input.nativeInput?.input ?? {},
};
const structured = input.prepareStructuredRoutes;
const image = input.prepareImageRoutes;
if (!structured && !image) {
return nativeInput;
}
if (!this.plans) {
throw new Error('Action route preparation is not available');
}
const state =
nativeInput.input && typeof nativeInput.input === 'object'
? { ...(nativeInput.input as Record<string, unknown>) }
: {};
if (structured) {
const responseContract =
structured.responseContract ??
(buildStructuredResponseFromSchemaJson(
structured.responseSchemaJson ?? { type: 'object' }
) as RequiredStructuredOutputContract);
const plan = await this.plans.buildStructuredPlan(
{ modelId: structured.modelId },
structured.messages,
structured.options,
structured.prefer ? { prefer: structured.prefer } : undefined,
responseContract
private async execute(input: ActionRuntimeBridgeInput) {
const step = input.step;
if (step.responseContract) {
const output = await this.runtime.generateStructuredValue(
{ profileId: step.profileId, modelId: step.modelId },
step.messages,
{
...(step.options as CopilotStructuredOptions | undefined),
builtInRouteId: step.builtInRouteId,
},
step.responseContract,
undefined,
step.slot
);
const preparedRoutes = plan.nativeDispatch?.structured?.routes;
if (!preparedRoutes?.length) {
throw new Error('No native structured provider route prepared');
}
const existingPreparedRoutes =
state.preparedRoutes &&
typeof state.preparedRoutes === 'object' &&
!Array.isArray(state.preparedRoutes)
? (state.preparedRoutes as Record<string, unknown>)
: {};
state.preparedRoutes = {
...existingPreparedRoutes,
[structured.stepId ?? 'generate']: preparedRoutes,
};
return { result: output.value, attachments: [] };
}
if (image) {
const plan = await this.plans.buildImagePlan(
{ modelId: image.modelId },
image.messages,
image.options,
image.prefer ? { prefer: image.prefer } : undefined
);
const preparedRoutes = plan.nativeDispatch?.image?.routes;
if (!preparedRoutes?.length) {
throw new Error('No native image provider route prepared');
}
const existingPreparedRoutes =
state.preparedRoutes &&
typeof state.preparedRoutes === 'object' &&
!Array.isArray(state.preparedRoutes)
? (state.preparedRoutes as Record<string, unknown>)
: {};
state.preparedRoutes = {
...existingPreparedRoutes,
[image.stepId ?? 'generate-image']: preparedRoutes,
};
const images = [];
for await (const image of this.runtime.streamImageArtifacts(
{ profileId: step.profileId, modelId: step.modelId },
step.messages,
{
...(step.options as CopilotImageOptions | undefined),
builtInRouteId: step.builtInRouteId,
},
undefined,
step.slot
)) {
images.push(image);
}
return {
...nativeInput,
input: state,
};
const result = images[0];
if (!result) throw new Error('Action image generation produced no image');
return { result, attachments: [result] };
}
private async projectAssistantResult(
@@ -273,28 +199,36 @@ export class ActionRuntimeBridge {
let finalEvent: NativeActionEvent | undefined;
const attachments: unknown[] = [];
try {
const nativeInput = await this.prepareNativeInput({
...inputWithBillingUnit,
});
for await (const event of this.runNativeStream(
{
...nativeInput,
recipeId: inputWithBillingUnit.actionId,
recipeVersion: inputWithBillingUnit.actionVersion,
},
inputWithBillingUnit.signal
)) {
finalEvent = event;
let projectedEvent = event;
if (event.type === 'attachment') {
const attachment = input.persistAttachment
? await input.persistAttachment(event.attachment)
: event.attachment;
attachments.push(attachment);
projectedEvent = { ...event, attachment };
}
yield { ...projectedEvent, runId: run.id };
const actionStart: NativeActionEvent = {
type: 'action_start',
actionId: input.actionId,
actionVersion: input.actionVersion,
status: 'running',
};
yield { ...actionStart, runId: run.id };
const output = await this.execute(inputWithBillingUnit);
for (const artifact of output.attachments) {
const attachment = input.persistAttachment
? await input.persistAttachment(artifact)
: artifact;
attachments.push(attachment);
yield {
type: 'attachment',
actionId: input.actionId,
actionVersion: input.actionVersion,
status: 'running',
attachment,
runId: run.id,
};
}
finalEvent = {
type: 'action_done',
actionId: input.actionId,
actionVersion: input.actionVersion,
status: 'succeeded',
result: output.result,
};
yield { ...finalEvent, runId: run.id };
} catch (error) {
finalEvent = {
type: 'error',
@@ -351,33 +285,14 @@ export class ActionRuntimeBridge {
): ActionRuntimeBridgeInput {
return {
...input,
prepareStructuredRoutes: input.prepareStructuredRoutes
? {
...input.prepareStructuredRoutes,
options: {
...input.prepareStructuredRoutes.options,
actionId:
input.prepareStructuredRoutes.options?.actionId ??
input.actionId,
billingUnitId:
input.prepareStructuredRoutes.options?.billingUnitId ??
billingUnitId,
},
}
: undefined,
prepareImageRoutes: input.prepareImageRoutes
? {
...input.prepareImageRoutes,
options: {
...input.prepareImageRoutes.options,
actionId:
input.prepareImageRoutes.options?.actionId ?? input.actionId,
billingUnitId:
input.prepareImageRoutes.options?.billingUnitId ??
billingUnitId,
},
}
: undefined,
step: {
...input.step,
options: {
...input.step.options,
actionId: input.step.options?.actionId ?? input.actionId,
billingUnitId: input.step.options?.billingUnitId ?? billingUnitId,
},
},
};
}
}
@@ -1,7 +1,27 @@
/* oxlint-disable import/no-cycle -- Tool callbacks can invoke nested Copilot prompts. */
import { Injectable } from '@nestjs/common';
import { CopilotPromptInvalid } from '../../../base';
import { ValidatedStructuredValueSchema } from '../core';
import { Config } from '../../../base/config';
import { CopilotPromptInvalid } from '../../../base/error/errors.gen';
import { BackendRuntimeProvider } from '../../../core/backend-runtime';
import {
buildLlmEmbeddingRequest,
buildLlmImageRequestFromMessages,
buildLlmRerankRequest,
type LlmImageResponse,
type LlmToolCallbackRequest,
type LlmToolLoopStreamEvent,
llmValidateJsonSchema,
} from '../../../native';
import {
getByokSourceCoverage,
getCopilotFeatureAccess,
} from '../access/feature-coverage';
import { assertCopilotEnabled } from '../availability';
import { ByokEntitlementPolicy } from '../byok/policy';
import type { ByokFeatureKind } from '../byok/types';
import { ConversationPolicy } from '../conversation/policy';
import { ValidatedStructuredValueSchema } from '../core/types';
import {
type CopilotChatOptions,
type CopilotEmbeddingOptions,
@@ -9,112 +29,294 @@ import {
type CopilotProviderType,
type CopilotRerankRequest,
type CopilotStructuredOptions,
type ModelAttachmentCapability,
type ModelConditions,
type PromptMessage,
type StreamObject,
} from '../providers/types';
import {
buildToolContracts,
type RequiredStructuredOutputContract,
requireStructuredOutputContract,
} from './contracts';
import {
ExecutionPlanBuilder,
type ExecutionPlanForKind,
} from './execution-plan';
type CopilotRuntimeEvent,
CopilotRuntimeEventConsumer,
} from './copilot-runtime-event-consumer';
import { mapNativeSemanticError } from './native-errors';
import {
NativeExecutionEngine,
type NativeImageArtifact,
} from './native-execution-engine';
buildCanonicalNativeRequest,
buildCanonicalNativeStructuredRequest,
preparePromptMessagesForNativeRequest,
} from './native-request-runtime';
import { executeToolCall } from './tool/bridge';
import { NativeProviderAdapter } from './tool/native-adapter';
import { ToolRuntime } from './tool-runtime';
type ProviderFilter = {
prefer?: CopilotProviderType;
type ProviderFilter = { prefer?: CopilotProviderType };
type RuntimeOptions = NonNullable<CopilotChatOptions> & {
dimensions?: number;
responseSchemaJson?: Record<string, unknown>;
schemaHash?: string;
strict?: boolean;
profileId?: string;
};
const providerModelId = (modelId?: string) => modelId ?? 'auto';
export type NativeImageArtifact = LlmImageResponse['images'][number];
const attachmentCapability = {
kinds: ['image', 'audio', 'file'],
sourceKinds: ['url', 'data', 'bytes', 'file_handle'],
allowRemoteUrls: true,
} satisfies ModelAttachmentCapability;
@Injectable()
export class CapabilityRuntime {
constructor(
private readonly plans: ExecutionPlanBuilder,
private readonly engine: NativeExecutionEngine
private readonly backend: BackendRuntimeProvider,
private readonly entitlement: ByokEntitlementPolicy,
private readonly conversations: ConversationPolicy,
private readonly tools: ToolRuntime,
private readonly events: CopilotRuntimeEventConsumer,
private readonly config: Config
) {}
private async executePlan<TPlan, TResult>(
build: () => Promise<TPlan>,
execute: (plan: TPlan) => Promise<TResult>
) {
return await execute(await build());
private async access(options: RuntimeOptions) {
assertCopilotEnabled(this.config);
const workspaceId = options.workspace;
const featureKind = (options.featureKind ?? 'chat') as ByokFeatureKind;
const coverage = getByokSourceCoverage(featureKind);
const [serverByok, localByok, premium] = workspaceId
? await Promise.all([
coverage.server && this.entitlement.hasServerEntitlement(workspaceId),
coverage.local &&
this.entitlement.hasLocalEntitlement(workspaceId, options.user),
this.entitlement.hasAiPlan(options.user),
])
: [false, false, await this.entitlement.hasAiPlan(options.user)];
const routeAllowed =
options.quotaBackedRoutesAllowed ??
(!getCopilotFeatureAccess(featureKind).quotaMetered ||
!options.user ||
(await this.conversations.hasQuota(options.user)));
return {
routeAllowed,
managedTier: premium ? ('Premium' as const) : ('Standard' as const),
serverByok,
localByok,
};
}
private executeStreamPlan<TPlan, TChunk>(
build: () => Promise<TPlan>,
execute: (plan: TPlan) => AsyncIterableIterator<TChunk>
): AsyncIterableIterator<TChunk> {
return (async function* () {
yield* execute(await build());
})();
private eventContext(options: RuntimeOptions) {
return {
workspaceId: options.workspace,
userId: options.user,
sessionId: options.session,
taskId: options.taskId,
actionId: options.actionId,
billingUnitId: options.billingUnitId,
featureKind: (options.featureKind ?? 'chat') as ByokFeatureKind,
};
}
private hasNativeDispatch(
plan: ExecutionPlanForKind<'embedding'> | ExecutionPlanForKind<'rerank'>,
kind: 'embedding' | 'rerank'
private targetOverride(cond: ModelConditions) {
return cond.profileId && cond.modelId
? { profileId: cond.profileId, modelId: cond.modelId }
: undefined;
}
async assertRoute(
slot: string,
cond: ModelConditions,
options: CopilotChatOptions = {}
) {
return !!plan.nativeDispatch?.[kind];
try {
await this.backend.assertCopilotRoute({
slot,
builtInRouteId: options.builtInRouteId,
workspaceId: options.workspace,
userId: options.user,
localLeaseId: options.byokLeaseId,
access: await this.access(options),
managedTargetId: options.managedTargetId,
targetOverride: this.targetOverride(cond),
});
} catch (error) {
throw mapNativeSemanticError(error);
}
}
private async execute(
slot: string,
request: unknown,
cond: ModelConditions,
options: RuntimeOptions
) {
try {
const output = await this.backend.executeCopilot({
slot,
builtInRouteId: options.builtInRouteId,
workspaceId: options.workspace,
userId: options.user,
localLeaseId: options.byokLeaseId,
access: await this.access(options),
managedTargetId: options.managedTargetId,
targetOverride: this.targetOverride(cond),
request,
});
await this.events.consume(
output.events as CopilotRuntimeEvent[],
this.eventContext(options)
);
return output.result;
} catch (error) {
throw mapNativeSemanticError(error);
}
}
private async prepareChat(
messages: PromptMessage[],
options: RuntimeOptions
) {
const toolSet = await this.tools.getTools(options, '');
const { request } = await buildCanonicalNativeRequest({
model: 'route-selected',
messages,
options,
toolContracts: buildToolContracts(toolSet),
attachmentCapability,
include: options.reasoning ? ['reasoning'] : undefined,
reasoning: options.reasoning ? { effort: 'medium' } : undefined,
});
return { request: { ...request, stream: true }, toolSet };
}
private async stream(
slot: string,
cond: ModelConditions,
messages: PromptMessage[],
options: RuntimeOptions
) {
const { request, toolSet } = await this.prepareChat(messages, options);
const rawStream = this.backend.streamCopilot<
LlmToolLoopStreamEvent | CopilotRuntimeEvent
>(
{
slot,
builtInRouteId: options.builtInRouteId,
workspaceId: options.workspace,
userId: options.user,
localLeaseId: options.byokLeaseId,
access: await this.access(options),
managedTargetId: options.managedTargetId,
targetOverride: this.targetOverride(cond),
request,
},
async requestJson => {
const toolRequest = JSON.parse(requestJson) as LlmToolCallbackRequest;
return JSON.stringify(
await executeToolCall(toolSet, toolRequest, {
signal: options.signal,
messages,
})
);
},
{ maxSteps: 20, signal: options.signal }
);
const runtimeEvents = this.events;
const eventContext = this.eventContext(options);
async function* productEvents() {
for await (const event of rawStream) {
if ('route' in event) {
await runtimeEvents.consume([event], eventContext);
} else if (event.type === 'error') {
throw mapNativeSemanticError(
new Error(
typeof event.message === 'string'
? event.message
: 'native runtime stream error'
)
);
} else {
yield event;
}
}
}
return { request, stream: productEvents() };
}
async text(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
filter?: ProviderFilter
options: CopilotChatOptions = {},
_filter?: ProviderFilter
) {
return await this.executePlan(
() => this.plans.buildTextPlan(cond, messages, options, filter),
plan => this.engine.execute(plan)
const prepared = await this.stream('prompt.text', cond, messages, options);
return await new NativeProviderAdapter(() => prepared.stream).text(
prepared.request,
options.signal,
messages
);
}
async *streamText(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
filter?: ProviderFilter
options: CopilotChatOptions = {},
_filter?: ProviderFilter
): AsyncIterableIterator<string> {
yield* this.executeStreamPlan(
() => this.plans.buildStreamTextPlan(cond, messages, options, filter),
plan => this.engine.executeStream(plan)
const prepared = await this.stream('chat.default', cond, messages, options);
yield* new NativeProviderAdapter(() => prepared.stream).streamText(
prepared.request,
options.signal,
messages
);
}
async *streamObject(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
filter?: ProviderFilter
options: CopilotChatOptions = {},
_filter?: ProviderFilter
): AsyncIterableIterator<StreamObject> {
yield* this.executeStreamPlan(
() => this.plans.buildStreamObjectPlan(cond, messages, options, filter),
plan => this.engine.executeStream(plan)
const prepared = await this.stream('chat.default', cond, messages, options);
yield* new NativeProviderAdapter(() => prepared.stream).streamObject(
prepared.request,
options.signal,
messages
);
}
async generateStructured(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotStructuredOptions,
filter?: ProviderFilter,
responseContract?: RequiredStructuredOutputContract
options: CopilotStructuredOptions = {},
_filter?: ProviderFilter,
responseContract?: RequiredStructuredOutputContract,
slot = 'prompt.structured'
) {
return await this.executePlan(
() =>
this.plans.buildStructuredPlan(
cond,
messages,
options,
filter,
responseContract
),
plan => this.engine.execute(plan)
const contract = requireStructuredOutputContract(responseContract);
if (!contract) {
throw new CopilotPromptInvalid('Structured schema contract is required');
}
const { request } = await buildCanonicalNativeStructuredRequest({
model: 'route-selected',
messages,
options,
responseContract: contract,
attachmentCapability,
});
const result = (await this.execute(slot, request, cond, options)) as {
output_json?: unknown;
output_text: string;
};
if (result.output_json === undefined) {
throw new CopilotPromptInvalid(
'Structured response is missing output_json'
);
}
return JSON.stringify(
llmValidateJsonSchema(request.schema, result.output_json)
);
}
@@ -123,87 +325,90 @@ export class CapabilityRuntime {
messages: PromptMessage[],
options: CopilotStructuredOptions,
responseContract?: RequiredStructuredOutputContract,
filter?: ProviderFilter
filter?: ProviderFilter,
slot = 'prompt.structured'
) {
const validatedResponseContract =
requireStructuredOutputContract(responseContract);
if (!options || !validatedResponseContract) {
const contract = requireStructuredOutputContract(responseContract);
if (!contract) {
throw new CopilotPromptInvalid('Structured schema contract is required');
}
const output = await this.generateStructured(
cond,
messages,
options,
filter,
validatedResponseContract
const value = JSON.parse(
await this.generateStructured(
cond,
messages,
options,
filter,
contract,
slot
)
);
const value = JSON.parse(output);
return ValidatedStructuredValueSchema.parse({
value,
schemaHash: validatedResponseContract.schemaHash,
schemaHash: contract.schemaHash,
schemaValidationVersion: 'json-schema-v1',
provider: filter?.prefer ?? 'auto',
model: providerModelId(cond.modelId),
provider: 'auto',
model: 'route-selected',
});
}
async embeddingConfigured(modelId: string) {
try {
return this.hasNativeDispatch(
await this.plans.buildEmbeddingPlan(modelId, 'ping'),
'embedding'
);
} catch {
return false;
}
async embeddingConfigured(_modelId: string) {
return this.config.copilot.enabled;
}
async embed(
modelId: string,
_modelId: string,
input: string | string[],
options?: CopilotEmbeddingOptions
options: CopilotEmbeddingOptions = {}
) {
return await this.executePlan(
() => this.plans.buildEmbeddingPlan(modelId, input, options),
plan => this.engine.execute(plan)
);
const result = (await this.execute(
'index.embedding',
buildLlmEmbeddingRequest({
model: 'route-selected',
inputs: Array.isArray(input) ? input : [input],
dimensions: options.dimensions,
}),
{},
options
)) as { embeddings: number[][] };
return result.embeddings;
}
async rerankConfigured(modelId: string) {
try {
return this.hasNativeDispatch(
await this.plans.buildRerankPlan(modelId, {
query: 'ping',
candidates: [{ text: 'ping' }],
}),
'rerank'
);
} catch {
return false;
}
async rerankConfigured(_modelId: string) {
return this.config.copilot.enabled;
}
async rerank(
modelId: string,
_modelId: string,
request: CopilotRerankRequest,
options?: CopilotChatOptions
options: CopilotChatOptions = {}
) {
return await this.executePlan(
() => this.plans.buildRerankPlan(modelId, request, options),
plan => this.engine.execute(plan)
);
const result = (await this.execute(
'search.rerank',
buildLlmRerankRequest('route-selected', request),
{},
options
)) as { scores: number[] };
return result.scores;
}
async *streamImageArtifacts(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotImageOptions,
filter?: ProviderFilter
options: CopilotImageOptions = {},
_filter?: ProviderFilter,
slot = 'image.generate'
): AsyncIterableIterator<NativeImageArtifact> {
yield* this.executeStreamPlan(
() => this.plans.buildImagePlan(cond, messages, options, filter),
plan => this.engine.executeImageArtifacts(plan)
);
const { quality, seed } = options;
const result = (await this.execute(
slot,
buildLlmImageRequestFromMessages({
model: 'route-selected',
messages: preparePromptMessagesForNativeRequest(messages, true),
options: { quality, seed },
}),
cond,
options
)) as LlmImageResponse;
yield* result.images;
}
}
@@ -1,104 +0,0 @@
import {
type LlmBackendConfig,
llmCompileExecutionPlan,
type LlmEmbeddingRequest,
type LlmImageRequest,
type LlmProtocol,
type LlmRequest,
type LlmRerankRequest,
type LlmStructuredRequest,
} from '../../../../native';
import type {
CopilotProviderType,
ModelConditions,
PromptMessage,
} from '../../providers/types';
// Owner: runtime core mirror facade.
// The semantic source of truth is the native/Rust execution-plan contract
// behind llmCompileExecutionPlan(); this file only keeps the TypeScript shape
// needed by Node live-plan assembly until generated/native TS types replace it.
export type ExecutionRequestKind =
| 'text'
| 'streamText'
| 'streamObject'
| 'structured'
| 'embedding'
| 'rerank'
| 'image';
export type ExecutionRoute = {
providerId: string;
protocol: LlmProtocol;
model: string;
backendConfig: LlmBackendConfig;
};
export type ExecutionTransportContract =
| { kind: 'chat'; request: LlmRequest }
| { kind: 'structured'; request: LlmStructuredRequest }
| { kind: 'embedding'; request: LlmEmbeddingRequest }
| { kind: 'rerank'; request: LlmRerankRequest }
| { kind: 'image'; request: LlmImageRequest };
export type SerializableExecutionPlanRequest =
| {
kind: 'text' | 'streamText' | 'streamObject';
cond: ModelConditions;
messages: PromptMessage[];
options?: Record<string, unknown>;
}
| {
kind: 'structured';
cond: ModelConditions;
messages: PromptMessage[];
options?: Record<string, unknown>;
}
| {
kind: 'image';
cond: ModelConditions;
messages: PromptMessage[];
options?: Record<string, unknown>;
}
| {
kind: 'embedding';
cond: ModelConditions;
modelId: string;
input: string | string[];
options?: Record<string, unknown>;
}
| {
kind: 'rerank';
cond: ModelConditions;
modelId: string;
request: {
query: string;
candidates: { id?: string; text: string }[];
topK?: number;
};
options?: Record<string, unknown>;
};
export type SerializableExecutionPlan = {
routes: ExecutionRoute[];
request: SerializableExecutionPlanRequest;
transport?: ExecutionTransportContract;
routePolicy: { fallbackOrder: string[] };
runtimePolicy: {
prefer?: CopilotProviderType;
maxSteps?: number;
};
attachmentPolicy: {
materializeRemoteAttachments: boolean;
};
responsePostprocess: {
mode: ExecutionRequestKind;
};
hostContext?: {
currentMessages?: PromptMessage[];
};
};
export function parseExecutionPlan(value: unknown) {
return llmCompileExecutionPlan<SerializableExecutionPlan>(value);
}
@@ -1,5 +1,3 @@
export * from './execution-plan-contract';
export * from './native-contract';
export * from './prompt-contract';
export * from './runtime-event-contract';
export * from './shared';
@@ -1,97 +0,0 @@
import serverNativeModule, {
type CapabilityMatchRequest,
type CapabilityMatchResponse,
type ModelRegistryMatchRequest,
type ModelRegistryMatchResponse,
type ModelRegistryResolveRequest,
type ModelRegistryResolveResponse,
type ModelRegistryVariantContract,
type ProviderDriverSpec,
type RequestedModelMatchRequest,
type RequestedModelMatchResponse,
} from '@affine/server-native';
// Owner: native/Rust contract facade.
// These types and validators intentionally proxy @affine/server-native and
// must not grow independent runtime semantics in Node.
export type {
CapabilityMatchRequest,
CapabilityMatchResponse,
ProviderDriverSpec,
RequestedModelMatchRequest,
RequestedModelMatchResponse,
};
export type CopilotModelBackendKind = ModelRegistryMatchRequest['backendKind'];
export type ModelRegistryVariant = ModelRegistryVariantContract;
export type ResolveModelRegistryVariantRequest = ModelRegistryResolveRequest;
export type ResolveModelRegistryVariantResponse = ModelRegistryResolveResponse;
export type MatchModelRegistryRequest = ModelRegistryMatchRequest;
export type MatchModelRegistryResponse = ModelRegistryMatchResponse;
function validateNativeContract<T>(name: string, value: unknown): T {
return serverNativeModule.llmValidateContract(name, value) as T;
}
export function parseCapabilityMatchRequest(value: unknown) {
return validateNativeContract<CapabilityMatchRequest>(
'capabilityMatchRequest',
value
);
}
export function parseCapabilityMatchResponse(value: unknown) {
return validateNativeContract<CapabilityMatchResponse>(
'capabilityMatchResponse',
value
);
}
export function parseResolveModelRegistryVariantRequest(value: unknown) {
return validateNativeContract<ResolveModelRegistryVariantRequest>(
'modelRegistryResolveRequest',
value
);
}
export function parseResolveModelRegistryVariantResponse(value: unknown) {
return validateNativeContract<ResolveModelRegistryVariantResponse>(
'modelRegistryResolveResponse',
value
);
}
export function parseMatchModelRegistryRequest(value: unknown) {
return validateNativeContract<MatchModelRegistryRequest>(
'modelRegistryMatchRequest',
value
);
}
export function parseMatchModelRegistryResponse(value: unknown) {
return validateNativeContract<MatchModelRegistryResponse>(
'modelRegistryMatchResponse',
value
);
}
export function parseProviderDriverSpec(value: unknown) {
return validateNativeContract<ProviderDriverSpec>(
'providerDriverSpec',
value
);
}
export function parseRequestedModelMatchRequest(value: unknown) {
return validateNativeContract<RequestedModelMatchRequest>(
'requestedModelMatchRequest',
value
);
}
export function parseRequestedModelMatchResponse(value: unknown) {
return validateNativeContract<RequestedModelMatchResponse>(
'requestedModelMatchResponse',
value
);
}
@@ -1,15 +1,6 @@
import {
llmValidateContract,
type NativePromptCountTokensRequest,
type NativePromptCountTokensResponse,
type NativePromptMetadataRequest,
type NativePromptMetadataResponse,
type NativePromptRenderRequest,
type NativePromptRenderResponse,
type NativePromptSessionRenderRequest,
type NativePromptSessionRenderResponse,
type PromptMessageContract as NativePromptMessageContract,
type PromptStructuredResponseContract as NativePromptStructuredResponseContract,
import type {
PromptMessageContract as NativePromptMessageContract,
PromptStructuredResponseContract as NativePromptStructuredResponseContract,
} from '../../../../native';
import { normalizePromptResponseFormat } from './structured-output-contract';
@@ -32,14 +23,6 @@ type PromptMessageInput = {
params?: Record<string, unknown> | null;
responseFormat?: PromptResponseFormat | null;
};
export type PromptRenderContract = NativePromptRenderRequest;
export type PromptRenderResult = NativePromptRenderResponse;
export type PromptTokenCountContract = NativePromptCountTokensRequest;
export type PromptTokenCountResult = NativePromptCountTokensResponse;
export type PromptMetadataContract = NativePromptMetadataRequest;
export type PromptMetadataResult = NativePromptMetadataResponse;
export type PromptSessionContract = NativePromptSessionRenderRequest;
export type PromptSessionResult = NativePromptSessionRenderResponse;
export type NativePromptResponseFormatProjection = {
nativeResponseFormat?: PromptStructuredResponseContract;
};
@@ -82,17 +65,3 @@ export function projectPromptMessageForNative(
return { message: nativeMessage, nativeResponseFormat };
}
export function parsePromptRenderContract(value: unknown) {
return llmValidateContract<PromptRenderContract>(
'promptRenderContract',
value
);
}
export function parsePromptSessionContract(value: unknown) {
return llmValidateContract<PromptSessionContract>(
'promptSessionContract',
value
);
}
@@ -0,0 +1,129 @@
import { Injectable, Logger } from '@nestjs/common';
import { metrics } from '../../../base';
import { Models } from '../../../models';
import { type ByokFeatureKind, ByokProviderSource } from '../byok/types';
export type CopilotRuntimeRouteIdentity = {
profileId: string;
source: 'server' | 'local' | 'affine_cloud';
provider: string;
model: string;
};
export type CopilotRuntimeEvent =
| { type: 'route_selected'; route: CopilotRuntimeRouteIdentity }
| {
type: 'route_failed';
route: CopilotRuntimeRouteIdentity;
errorKind: string;
}
| {
type: 'usage';
route: CopilotRuntimeRouteIdentity;
usage: {
prompt_tokens?: number;
completion_tokens?: number;
total_tokens?: number;
cached_tokens?: number;
input_tokens?: number;
output_tokens?: number;
};
};
export type CopilotRuntimeEventContext = {
workspaceId?: string;
userId?: string;
sessionId?: string;
taskId?: string;
actionId?: string;
billingUnitId?: string;
featureKind: ByokFeatureKind;
};
@Injectable()
export class CopilotRuntimeEventConsumer {
private readonly logger = new Logger(CopilotRuntimeEventConsumer.name);
constructor(private readonly models: Models) {}
async consume(
events: CopilotRuntimeEvent[],
context: CopilotRuntimeEventContext
) {
for (const event of events) {
try {
if (event.type === 'usage') {
await this.recordUsage(event, context);
} else if (event.type === 'route_failed') {
await this.recordFailure(event, context);
}
} catch (error) {
this.logger.warn(
`Failed to consume copilot runtime event: ${
error instanceof Error ? error.message : String(error)
}`
);
}
}
}
private async recordUsage(
event: Extract<CopilotRuntimeEvent, { type: 'usage' }>,
context: CopilotRuntimeEventContext
) {
if (!context.workspaceId || event.route.source === 'affine_cloud') {
return;
}
const usage = event.usage;
metrics.ai.counter('byok_usage').add(1, {
provider: event.route.provider,
source: event.route.source,
feature: context.featureKind,
});
await this.models.copilotUsage.create({
workspaceId: context.workspaceId,
userId: context.userId,
provider: event.route.provider,
providerSource:
event.route.source === 'server'
? ByokProviderSource.Server
: ByokProviderSource.Local,
featureKind: context.featureKind,
model: event.route.model,
sessionId: context.sessionId,
taskId: context.taskId,
actionId: context.actionId,
billingUnitId: context.billingUnitId,
promptTokens: usage.prompt_tokens ?? usage.input_tokens ?? 0,
completionTokens: usage.completion_tokens ?? usage.output_tokens ?? 0,
totalTokens: usage.total_tokens ?? 0,
cachedTokens: usage.cached_tokens ?? 0,
});
if (event.route.source === 'server') {
await this.models.copilotWorkspaceByokConfig.touchUsed(
context.workspaceId,
event.route.profileId
);
}
}
private async recordFailure(
event: Extract<CopilotRuntimeEvent, { type: 'route_failed' }>,
context: CopilotRuntimeEventContext
) {
metrics.ai.counter('byok_route_failure').add(1, {
provider: event.route.provider,
source: event.route.source,
feature: context.featureKind,
reason: event.errorKind,
});
if (context.workspaceId && event.route.source === 'server') {
await this.models.copilotWorkspaceByokConfig.markFailure(
context.workspaceId,
event.route.profileId,
event.errorKind
);
}
}
}
@@ -1,65 +0,0 @@
import { Injectable } from '@nestjs/common';
import { metrics } from '../../../base';
import type { ResolvedCopilotProvider } from '../providers/factory';
import type { CopilotProviderType } from '../providers/types';
import type { ExecutionRequestKind } from './execution-plan';
type ExecutionDispatchPath = 'prepared_routes';
export function summarizePreparedRoutes(
routes: Array<Pick<ResolvedCopilotProvider, 'prepared'>>
) {
const preparedCount = routes.filter(route => !!route.prepared).length;
return {
routeCount: routes.length,
preparedCount,
preparedMode:
preparedCount === 0
? 'none'
: preparedCount === routes.length
? 'all'
: 'partial',
} as const;
}
function planAttrs(
kind: ExecutionRequestKind,
prefer?: CopilotProviderType,
routes?: ResolvedCopilotProvider[]
) {
const summary = summarizePreparedRoutes(routes ?? []);
return {
kind,
prefer: prefer ?? 'auto',
prepared: summary.preparedMode,
route_count: summary.routeCount,
};
}
@Injectable()
export class CopilotExecutionMetrics {
recordPlan(
kind: ExecutionRequestKind,
routes: ResolvedCopilotProvider[],
prefer?: CopilotProviderType
) {
const attrs = planAttrs(kind, prefer, routes);
metrics.ai.counter('execution_plan_total').add(1, attrs);
metrics.ai.histogram('execution_plan_routes').record(attrs.route_count, {
kind: attrs.kind,
prefer: attrs.prefer,
prepared: attrs.prepared,
});
}
recordDispatch(
kind: ExecutionRequestKind,
path: ExecutionDispatchPath,
routeCount: number
) {
const attrs = { kind, path };
metrics.ai.counter('execution_dispatch_total').add(1, attrs);
metrics.ai.histogram('execution_dispatch_routes').record(routeCount, attrs);
}
}
@@ -1,827 +0,0 @@
import { Injectable } from '@nestjs/common';
import type {
LlmPreparedDispatchRoute,
LlmPreparedEmbeddingDispatchRoute,
LlmPreparedImageDispatchRoute,
LlmPreparedRerankDispatchRoute,
LlmPreparedStructuredDispatchRoute,
} from '../../../native';
import { llmNormalizePreparedRoutes } from '../../../native';
import {
CopilotProviderFactory,
type ResolvedCopilotProvider,
} from '../providers/factory';
import type {
PreparedNativeEmbeddingExecution,
PreparedNativeExecution,
PreparedNativeImageExecution,
PreparedNativeRerankExecution,
PreparedNativeStructuredExecution,
} from '../providers/provider-runtime-contract';
import type {
CopilotChatOptions,
CopilotEmbeddingOptions,
CopilotImageOptions,
CopilotProviderType,
CopilotRerankRequest,
CopilotStructuredOptions,
ModelConditions,
PromptMessage,
} from '../providers/types';
import { ModelOutputType } from '../providers/types';
import type { RequiredStructuredOutputContract } from './contracts';
import {
type ExecutionRequestKind,
type ExecutionRoute,
type ExecutionTransportContract,
parseExecutionPlan,
type SerializableExecutionPlan,
type SerializableExecutionPlanRequest,
} from './contracts/execution-plan-contract';
import { CopilotExecutionMetrics } from './execution-metrics';
export type { ExecutionRequestKind };
type ProviderFilter = {
prefer?: CopilotProviderType;
};
type BaseExecutionRequest<TKind extends ExecutionRequestKind> = {
kind: TKind;
cond: ModelConditions;
};
type TextExecutionRequest = BaseExecutionRequest<'text'> & {
messages: PromptMessage[];
options?: CopilotChatOptions;
};
type StreamTextExecutionRequest = BaseExecutionRequest<'streamText'> & {
messages: PromptMessage[];
options?: CopilotChatOptions;
};
type StreamObjectExecutionRequest = BaseExecutionRequest<'streamObject'> & {
messages: PromptMessage[];
options?: CopilotChatOptions;
};
type StructuredExecutionRequest = BaseExecutionRequest<'structured'> & {
messages: PromptMessage[];
options?: CopilotStructuredOptions;
};
type ImageExecutionRequest = BaseExecutionRequest<'image'> & {
messages: PromptMessage[];
options?: CopilotImageOptions;
};
type EmbeddingExecutionRequest = BaseExecutionRequest<'embedding'> & {
modelId: string;
input: string | string[];
options?: CopilotEmbeddingOptions;
};
type RerankExecutionRequest = BaseExecutionRequest<'rerank'> & {
modelId: string;
request: CopilotRerankRequest;
options?: CopilotChatOptions;
};
export type ExecutionPlanRequest =
| TextExecutionRequest
| StreamTextExecutionRequest
| StreamObjectExecutionRequest
| StructuredExecutionRequest
| ImageExecutionRequest
| EmbeddingExecutionRequest
| RerankExecutionRequest;
export type ExecutionPlanForKind<TKind extends ExecutionRequestKind> =
ExecutionPlan & {
request: Extract<ExecutionPlanRequest, { kind: TKind }>;
};
type NativePreparedDispatchPlan<TRoute, TPrepared> = {
routes: TRoute[];
prepared: TPrepared;
};
export type NativeChatDispatchPlan = NativePreparedDispatchPlan<
LlmPreparedDispatchRoute,
PreparedNativeExecution
> & {
hasTools: boolean;
};
export type NativeStructuredDispatchPlan = NativePreparedDispatchPlan<
LlmPreparedStructuredDispatchRoute,
PreparedNativeStructuredExecution
>;
export type NativeEmbeddingDispatchPlan = NativePreparedDispatchPlan<
LlmPreparedEmbeddingDispatchRoute,
PreparedNativeEmbeddingExecution
>;
export type NativeRerankDispatchPlan = NativePreparedDispatchPlan<
LlmPreparedRerankDispatchRoute,
PreparedNativeRerankExecution
>;
export type NativeImageDispatchPlan = NativePreparedDispatchPlan<
LlmPreparedImageDispatchRoute,
PreparedNativeImageExecution
>;
export type ExecutionPlan = {
nativeDispatch?: {
chat?: NativeChatDispatchPlan;
structured?: NativeStructuredDispatchPlan;
embedding?: NativeEmbeddingDispatchPlan;
rerank?: NativeRerankDispatchPlan;
image?: NativeImageDispatchPlan;
};
serializable?: SerializableExecutionPlan;
transport?: ExecutionTransportContract;
request: ExecutionPlanRequest;
routePolicy: { fallbackOrder: string[] };
runtimePolicy: {
prefer?: CopilotProviderType;
};
attachmentPolicy: {
materializeRemoteAttachments: boolean;
};
responsePostprocess: { mode: ExecutionRequestKind };
hostPersistence: {
persistAssistantTurn: boolean;
outputKind: ExecutionRequestKind;
};
hostContext: {
signal?: AbortSignal;
currentMessages?: PromptMessage[];
};
};
type PreparedRouteLike<TRequest = unknown> = {
route: {
providerId: string;
protocol: PreparedNativeExecution['route']['protocol'];
model: string;
backendConfig: PreparedNativeExecution['route']['backendConfig'];
};
request: TRequest;
};
function buildPreparedTransport<
TKind extends ExecutionTransportContract['kind'],
TPrepared extends PreparedRouteLike,
>(
kind: TKind,
routes: ResolvedCopilotProvider[],
getPrepared: (route: ResolvedCopilotProvider) => TPrepared | undefined
): ExecutionTransportContract | undefined {
const prepared =
routes.length === 1 ? routes[0] && getPrepared(routes[0]) : undefined;
if (!prepared) {
return;
}
return {
kind,
request: prepared.request,
} as ExecutionTransportContract;
}
function collectPreparedRoutes<TPrepared extends PreparedRouteLike, TRoute>(
routes: ResolvedCopilotProvider[],
getPrepared: (route: ResolvedCopilotProvider) => TPrepared | undefined,
mapPreparedRoute: (prepared: TPrepared) => TRoute
): TRoute[] | undefined {
if (!routes.length) {
return;
}
const preparedRoutes: TRoute[] = [];
for (const route of routes) {
const prepared = getPrepared(route);
if (!prepared) {
return;
}
preparedRoutes.push(mapPreparedRoute(prepared));
}
return preparedRoutes;
}
function buildPreparedDispatchPlan<
TPrepared extends PreparedRouteLike,
TRoute,
TDispatch extends NativePreparedDispatchPlan<TRoute, TPrepared>,
>(
routes: ResolvedCopilotProvider[],
getPrepared: (route: ResolvedCopilotProvider) => TPrepared | undefined,
mapPreparedRoute: (prepared: TPrepared) => TRoute,
buildPreparedDispatchResult?: (
preparedRoutes: TRoute[],
prepared: TPrepared
) => TDispatch
): TDispatch | undefined {
const preparedRoutes = collectPreparedRoutes(
routes,
getPrepared,
mapPreparedRoute
);
const prepared = routes[0] && getPrepared(routes[0]);
if (!preparedRoutes || !prepared) {
return;
}
const normalizedRoutes = llmNormalizePreparedRoutes<TRoute[]>(preparedRoutes);
return buildPreparedDispatchResult
? buildPreparedDispatchResult(normalizedRoutes, prepared)
: ({ routes: normalizedRoutes, prepared } as TDispatch);
}
type DispatchPreparedRoute<TRequest> = {
provider_id: string;
protocol: PreparedNativeExecution['route']['protocol'];
model: string;
config: PreparedNativeExecution['route']['backendConfig'];
request: TRequest;
};
function mapPreparedDispatchRoute<TRequest>(
prepared: PreparedRouteLike<TRequest>
): DispatchPreparedRoute<TRequest> {
return {
provider_id: prepared.route.providerId,
protocol: prepared.route.protocol,
model: prepared.route.model,
config: prepared.route.backendConfig,
request: prepared.request,
};
}
type PreparedExecutionArtifactSpec<
TKind extends ExecutionTransportContract['kind'],
TPrepared extends PreparedRouteLike,
TRoute,
TDispatch extends NativePreparedDispatchPlan<TRoute, TPrepared>,
> = {
transportKind: TKind;
getPrepared: (route: ResolvedCopilotProvider) => TPrepared | undefined;
mapPreparedRoute: (prepared: TPrepared) => TRoute;
buildPreparedDispatch?: (
preparedRoutes: TRoute[],
prepared: TPrepared
) => TDispatch;
};
type PreparedExecutionArtifacts<TDispatch> = {
dispatch?: TDispatch;
transport?: ExecutionTransportContract;
};
function buildPreparedExecutionArtifacts<
TKind extends ExecutionTransportContract['kind'],
TPrepared extends PreparedRouteLike,
TRoute,
TDispatch extends NativePreparedDispatchPlan<TRoute, TPrepared>,
>(
routes: ResolvedCopilotProvider[],
spec: PreparedExecutionArtifactSpec<TKind, TPrepared, TRoute, TDispatch>
): PreparedExecutionArtifacts<TDispatch> {
return {
dispatch: buildPreparedDispatchPlan(
routes,
spec.getPrepared,
spec.mapPreparedRoute,
spec.buildPreparedDispatch
),
transport: buildPreparedTransport(
spec.transportKind,
routes,
spec.getPrepared
),
};
}
const chatArtifactSpec: PreparedExecutionArtifactSpec<
'chat',
PreparedNativeExecution,
LlmPreparedDispatchRoute,
NativeChatDispatchPlan
> = {
transportKind: 'chat',
getPrepared: route => route.prepared,
mapPreparedRoute: mapPreparedDispatchRoute,
buildPreparedDispatch: (preparedRoutes, prepared) => ({
routes: preparedRoutes,
prepared,
hasTools: Object.keys(prepared.tools).length > 0,
}),
};
const structuredArtifactSpec: PreparedExecutionArtifactSpec<
'structured',
PreparedNativeStructuredExecution,
LlmPreparedStructuredDispatchRoute,
NativeStructuredDispatchPlan
> = {
transportKind: 'structured',
getPrepared: route => route.preparedStructured,
mapPreparedRoute: mapPreparedDispatchRoute,
};
const embeddingArtifactSpec: PreparedExecutionArtifactSpec<
'embedding',
PreparedNativeEmbeddingExecution,
LlmPreparedEmbeddingDispatchRoute,
NativeEmbeddingDispatchPlan
> = {
transportKind: 'embedding',
getPrepared: route => route.preparedEmbedding,
mapPreparedRoute: mapPreparedDispatchRoute,
};
const rerankArtifactSpec: PreparedExecutionArtifactSpec<
'rerank',
PreparedNativeRerankExecution,
LlmPreparedRerankDispatchRoute,
NativeRerankDispatchPlan
> = {
transportKind: 'rerank',
getPrepared: route => route.preparedRerank,
mapPreparedRoute: mapPreparedDispatchRoute,
};
const imageArtifactSpec: PreparedExecutionArtifactSpec<
'image',
PreparedNativeImageExecution,
LlmPreparedImageDispatchRoute,
NativeImageDispatchPlan
> = {
transportKind: 'image',
getPrepared: route => route.preparedImage,
mapPreparedRoute: mapPreparedDispatchRoute,
};
function buildFallbackOrder(routes: ResolvedCopilotProvider[]) {
return routes.map(route => route.providerId);
}
function mapExecutionRoute(route: ResolvedCopilotProvider): ExecutionRoute {
const preparedRoute =
route.prepared?.route ??
route.preparedStructured?.route ??
route.preparedEmbedding?.route ??
route.preparedRerank?.route ??
route.preparedImage?.route;
if (preparedRoute) {
return {
providerId: preparedRoute.providerId,
protocol: preparedRoute.protocol,
model: preparedRoute.model,
backendConfig: preparedRoute.backendConfig,
};
}
const rawRoute = route as unknown as ExecutionRoute;
return {
providerId: rawRoute.providerId,
protocol: rawRoute.protocol,
model: rawRoute.model,
backendConfig: rawRoute.backendConfig,
};
}
function stripHostOnlyOptions<TOptions extends object | undefined>(
options: TOptions
): Record<string, unknown> | undefined {
if (!options) {
return;
}
const {
signal: _signal,
user: _user,
session: _session,
workspace: _workspace,
quotaBackedRoutesAllowed: _quotaBackedRoutesAllowed,
...serializable
} = options as Record<string, unknown>;
return Object.keys(serializable).length ? serializable : undefined;
}
function buildSerializableRequest(
request: ExecutionPlanRequest
): SerializableExecutionPlanRequest {
switch (request.kind) {
case 'text':
case 'streamText':
case 'streamObject':
case 'structured':
case 'image':
return {
...request,
options: stripHostOnlyOptions(request.options),
} as SerializableExecutionPlanRequest;
case 'embedding':
case 'rerank':
return {
...request,
options: stripHostOnlyOptions(request.options),
};
}
}
function buildSerializableExecutionPlan(
routes: ResolvedCopilotProvider[],
input: Omit<
ExecutionPlan,
'nativeDispatch' | 'serializable' | 'hostContext'
> &
Pick<ExecutionPlan, 'hostContext'>
): SerializableExecutionPlan {
return parseExecutionPlan({
routes: routes.map(mapExecutionRoute),
request: buildSerializableRequest(input.request),
transport: input.transport,
routePolicy: input.routePolicy,
runtimePolicy: input.runtimePolicy,
attachmentPolicy: input.attachmentPolicy,
responsePostprocess: input.responsePostprocess,
hostContext: input.hostContext.currentMessages
? { currentMessages: input.hostContext.currentMessages }
: undefined,
});
}
type MessagePlanArtifacts = Pick<ExecutionPlan, 'nativeDispatch' | 'transport'>;
function buildMessagePlanArtifacts(
kind: Extract<
ExecutionRequestKind,
'text' | 'streamText' | 'streamObject' | 'structured' | 'image'
>,
routes: ResolvedCopilotProvider[]
): MessagePlanArtifacts {
const chatArtifacts =
kind === 'text' || kind === 'streamText' || kind === 'streamObject'
? buildPreparedExecutionArtifacts(routes, chatArtifactSpec)
: undefined;
const structuredArtifacts =
kind === 'structured'
? buildPreparedExecutionArtifacts(routes, structuredArtifactSpec)
: undefined;
const imageArtifacts =
kind === 'image'
? buildPreparedExecutionArtifacts(routes, imageArtifactSpec)
: undefined;
const nativeDispatch = {
chat:
kind === 'text' || kind === 'streamText' || kind === 'streamObject'
? chatArtifacts?.dispatch
: undefined,
structured:
kind === 'structured' ? structuredArtifacts?.dispatch : undefined,
image: kind === 'image' ? imageArtifacts?.dispatch : undefined,
};
return {
nativeDispatch,
transport:
kind === 'text' || kind === 'streamText' || kind === 'streamObject'
? chatArtifacts?.transport
: kind === 'structured'
? structuredArtifacts?.transport
: kind === 'image'
? imageArtifacts?.transport
: undefined,
};
}
function buildEmbeddingPlanArtifacts(
routes: ResolvedCopilotProvider[]
): Pick<ExecutionPlan, 'nativeDispatch' | 'transport'> {
const embeddingArtifacts = buildPreparedExecutionArtifacts(
routes,
embeddingArtifactSpec
);
return {
nativeDispatch: {
embedding: embeddingArtifacts.dispatch,
},
transport: embeddingArtifacts.transport,
};
}
function buildRerankPlanArtifacts(
routes: ResolvedCopilotProvider[]
): Pick<ExecutionPlan, 'nativeDispatch' | 'transport'> {
const rerankArtifacts = buildPreparedExecutionArtifacts(
routes,
rerankArtifactSpec
);
return {
nativeDispatch: {
rerank: rerankArtifacts.dispatch,
},
transport: rerankArtifacts.transport,
};
}
@Injectable()
export class ExecutionPlanBuilder {
constructor(
private readonly providers: CopilotProviderFactory,
private readonly executionMetrics: CopilotExecutionMetrics
) {}
private async buildMessagePlan<
TKind extends Extract<
ExecutionRequestKind,
'text' | 'streamText' | 'streamObject' | 'structured' | 'image'
>,
>(
kind: TKind,
cond: ModelConditions,
messages: PromptMessage[],
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions,
filter: ProviderFilter = {}
): Promise<ExecutionPlanForKind<TKind>> {
const outputType =
kind === 'image'
? ModelOutputType.Image
: kind === 'streamObject'
? ModelOutputType.Object
: kind === 'structured'
? ModelOutputType.Structured
: ModelOutputType.Text;
const routes =
kind === 'text' || kind === 'streamText' || kind === 'streamObject'
? await this.providers.prepareRoutes(
kind,
{ ...cond, outputType },
messages,
(options as CopilotChatOptions | undefined) ?? {},
filter
)
: kind === 'structured'
? await this.providers.prepareStructuredRoutes(
{ ...cond, outputType },
messages,
(options as CopilotStructuredOptions | undefined) ?? {},
filter
)
: await this.providers.prepareImageRoutes(
{ ...cond, outputType },
messages,
(options as CopilotImageOptions | undefined) ?? {},
filter
);
this.executionMetrics.recordPlan(kind, routes, filter.prefer);
const { nativeDispatch, transport } = buildMessagePlanArtifacts(
kind,
routes
);
const plan = {
transport,
request: {
kind,
cond: { ...cond, modelId: cond.modelId },
messages,
options,
} as Extract<ExecutionPlanRequest, { kind: TKind }>,
routePolicy: {
fallbackOrder: buildFallbackOrder(routes),
},
runtimePolicy: { prefer: filter.prefer },
attachmentPolicy: { materializeRemoteAttachments: true },
responsePostprocess: { mode: kind },
hostPersistence: {
persistAssistantTurn: true,
outputKind: kind,
},
hostContext: {
signal: options?.signal,
currentMessages: messages,
},
} as Omit<ExecutionPlanForKind<TKind>, 'nativeDispatch' | 'serializable'>;
return {
nativeDispatch,
serializable: buildSerializableExecutionPlan(routes, plan),
...plan,
};
}
async buildTextPlan(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
filter?: ProviderFilter
): Promise<ExecutionPlanForKind<'text'>> {
return await this.buildMessagePlan('text', cond, messages, options, filter);
}
async buildStreamTextPlan(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
filter?: ProviderFilter
): Promise<ExecutionPlanForKind<'streamText'>> {
return await this.buildMessagePlan(
'streamText',
cond,
messages,
options,
filter
);
}
async buildStreamObjectPlan(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
filter?: ProviderFilter
): Promise<ExecutionPlanForKind<'streamObject'>> {
return await this.buildMessagePlan(
'streamObject',
cond,
messages,
options,
filter
);
}
async buildStructuredPlan(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotStructuredOptions,
filter?: ProviderFilter,
responseContract?: RequiredStructuredOutputContract
): Promise<ExecutionPlanForKind<'structured'>> {
const outputType = ModelOutputType.Structured;
const routes = await this.providers.prepareStructuredRoutes(
{ ...cond, outputType },
messages,
options ?? {},
filter ?? {},
responseContract
);
this.executionMetrics.recordPlan('structured', routes, filter?.prefer);
const { nativeDispatch, transport } = buildMessagePlanArtifacts(
'structured',
routes
);
const plan = {
transport,
request: {
kind: 'structured',
cond: { ...cond, modelId: cond.modelId },
messages,
options,
},
routePolicy: {
fallbackOrder: buildFallbackOrder(routes),
},
runtimePolicy: { prefer: filter?.prefer },
attachmentPolicy: { materializeRemoteAttachments: true },
responsePostprocess: { mode: 'structured' },
hostPersistence: {
persistAssistantTurn: true,
outputKind: 'structured',
},
hostContext: {
signal: options?.signal,
currentMessages: messages,
},
} as Omit<
ExecutionPlanForKind<'structured'>,
'nativeDispatch' | 'serializable'
>;
return {
nativeDispatch,
serializable: buildSerializableExecutionPlan(routes, plan),
...plan,
};
}
async buildImagePlan(
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotImageOptions,
filter?: ProviderFilter
): Promise<ExecutionPlanForKind<'image'>> {
return await this.buildMessagePlan(
'image',
cond,
messages,
options,
filter
);
}
async buildEmbeddingPlan(
modelId: string,
input: string | string[],
options?: CopilotEmbeddingOptions
): Promise<ExecutionPlanForKind<'embedding'>> {
const routes = await this.providers.prepareEmbeddingRoutes(
modelId,
input,
options
);
this.executionMetrics.recordPlan('embedding', routes);
const { nativeDispatch, transport } = buildEmbeddingPlanArtifacts(routes);
const plan = {
transport,
request: {
kind: 'embedding',
cond: { modelId },
modelId,
input,
options,
},
routePolicy: {
fallbackOrder: buildFallbackOrder(routes),
},
runtimePolicy: {},
attachmentPolicy: { materializeRemoteAttachments: false },
responsePostprocess: { mode: 'embedding' },
hostPersistence: {
persistAssistantTurn: false,
outputKind: 'embedding',
},
hostContext: {
signal: options?.signal,
},
} as Omit<
ExecutionPlanForKind<'embedding'>,
'nativeDispatch' | 'serializable'
>;
return {
nativeDispatch,
serializable: buildSerializableExecutionPlan(routes, plan),
...plan,
};
}
async buildRerankPlan(
modelId: string,
request: CopilotRerankRequest,
options?: CopilotChatOptions
): Promise<ExecutionPlanForKind<'rerank'>> {
const routes = await this.providers.prepareRerankRoutes(
modelId,
request,
options
);
this.executionMetrics.recordPlan('rerank', routes);
const { nativeDispatch, transport } = buildRerankPlanArtifacts(routes);
const plan = {
transport,
request: {
kind: 'rerank',
cond: { modelId },
modelId,
request,
options,
},
routePolicy: {
fallbackOrder: buildFallbackOrder(routes),
},
runtimePolicy: {},
attachmentPolicy: { materializeRemoteAttachments: false },
responsePostprocess: { mode: 'rerank' },
hostPersistence: {
persistAssistantTurn: false,
outputKind: 'rerank',
},
hostContext: {
signal: options?.signal,
},
} as Omit<
ExecutionPlanForKind<'rerank'>,
'nativeDispatch' | 'serializable'
>;
return {
nativeDispatch,
serializable: buildSerializableExecutionPlan(routes, plan),
...plan,
};
}
}
@@ -1,6 +1,9 @@
import { Injectable } from '@nestjs/common';
import type { LlmImageResponse } from '../../../../native';
import {
getCopilotActionRecipe,
type LlmImageResponse,
} from '../../../../native';
import { PromptService } from '../../prompt';
import type { PromptMessage } from '../../providers/types';
import type { ChatSession } from '../../session';
@@ -8,6 +11,10 @@ import { ChatQuerySchema } from '../../types';
import { projectActionEventToChatEvent } from '../action-output-projector';
import type { ActionRuntimeBridgeEvent } from '../action-runtime-bridge';
import { ActionRuntimeBridge } from '../action-runtime-bridge';
import {
buildStructuredResponseFromSchemaJson,
requireStructuredOutputContract,
} from '../contracts';
import { ConversationHost } from './conversation-host';
import { ImageResultHost } from './image-result-host';
@@ -17,32 +24,6 @@ function firstQueryValue(value: string | string[] | undefined) {
return Array.isArray(value) ? value[0] : value;
}
const ACTION_PROMPTS: Record<string, string> = {
'mindmap.generate': 'mindmap.generate',
'slides.outline': 'slides.outline',
};
type ImageActionRoutePreparation = {
modelId?: string;
messages: PromptMessage[];
options: Record<string, unknown>;
};
function isImageAction(id: string) {
return id.startsWith('image.filter.');
}
function actionTextResultSchema() {
return {
type: 'object',
properties: {
result: { type: 'string' },
},
required: ['result'],
additionalProperties: false,
};
}
@Injectable()
export class ActionStreamHost {
constructor(
@@ -73,6 +54,7 @@ export class ActionStreamHost {
firstQueryValue(query.actionId) ?? prepared.session.config.promptName;
const actionId = requestedActionId;
const actionVersion = firstQueryValue(query.actionVersion) ?? 'v1';
const recipe = getCopilotActionRecipe(actionId, actionVersion);
const retryOf = parsedQuery.retry
? firstQueryValue(query.runId)
: undefined;
@@ -81,19 +63,16 @@ export class ActionStreamHost {
...this.conversations.buildLatestTurnPromptParams(prepared.latestTurn),
};
const finalMessage = await this.preparePromptMessages(
actionId,
recipe.promptRef,
prepared.session,
params
);
const imageRoutes = await this.prepareImageRoutes(
actionId,
prepared.session,
params,
userId,
parsedQuery.byokLeaseId,
prepared.quotaBackedRoutesAllowed,
signal
);
const responseContract = recipe.responseContract
? requireStructuredOutputContract(
buildStructuredResponseFromSchemaJson(recipe.responseContract.schema)
)
: undefined;
const producesImage = recipe.outputProjection === 'first_image';
const runStream = this.bridge.runStream({
userId,
workspaceId: prepared.session.config.workspaceId,
@@ -108,7 +87,7 @@ export class ActionStreamHost {
params,
messageId: prepared.messageId,
},
persistAttachment: isImageAction(actionId)
persistAttachment: producesImage
? attachment =>
this.persistImageAttachment(
userId,
@@ -116,35 +95,25 @@ export class ActionStreamHost {
attachment
)
: undefined,
prepareStructuredRoutes: isImageAction(actionId)
? undefined
: {
stepId: 'generate',
modelId:
typeof query.modelId === 'string' && query.modelId
? query.modelId
: undefined,
messages: finalMessage,
responseSchemaJson: actionTextResultSchema(),
options: {
...prepared.session.config.promptConfig,
signal,
user: userId,
workspace: prepared.session.config.workspaceId,
session: sessionId,
byokLeaseId: parsedQuery.byokLeaseId,
quotaBackedRoutesAllowed: prepared.quotaBackedRoutesAllowed,
featureKind: 'action',
},
},
prepareImageRoutes: imageRoutes
? {
stepId: 'generate-image',
modelId: imageRoutes.modelId,
messages: imageRoutes.messages,
options: imageRoutes.options,
}
: undefined,
step: {
slot: recipe.slot,
builtInRouteId: recipe.promptRef,
profileId: parsedQuery.profileId,
modelId: parsedQuery.modelId,
messages: finalMessage,
responseContract,
options: {
...prepared.session.config.promptConfig,
signal,
user: userId,
workspace: prepared.session.config.workspaceId,
session: sessionId,
byokLeaseId: parsedQuery.byokLeaseId,
managedTargetId: parsedQuery.routeTargetId,
quotaBackedRoutesAllowed: prepared.quotaBackedRoutesAllowed,
featureKind: producesImage ? 'image' : 'action',
},
},
signal,
});
@@ -157,18 +126,13 @@ export class ActionStreamHost {
}
private async preparePromptMessages(
actionId: string,
promptRef: string,
session: ChatSession,
params: Record<string, unknown>
): Promise<PromptMessage[]> {
const promptName = ACTION_PROMPTS[actionId];
if (!promptName) {
return session.finish(params);
}
const prompt = await this.prompts.get(promptName);
const prompt = await this.prompts.get(promptRef);
if (!prompt) {
throw new Error(`Prompt ${promptName} not found`);
throw new Error(`Prompt ${promptRef} not found`);
}
return this.prompts.finish(
prompt,
@@ -177,44 +141,6 @@ export class ActionStreamHost {
);
}
private async prepareImageRoutes(
actionId: string,
session: ChatSession,
params: Record<string, unknown>,
userId: string,
byokLeaseId?: string,
quotaBackedRoutesAllowed?: boolean,
signal?: AbortSignal
): Promise<ImageActionRoutePreparation | undefined> {
if (!isImageAction(actionId)) {
return undefined;
}
const prompt = await this.prompts.get(actionId);
if (!prompt) {
throw new Error(`Prompt ${actionId} not found`);
}
const finalMessage = this.prompts.finish(
prompt,
params as Record<string, string>,
session.config.sessionId
);
return {
modelId: prompt.model,
messages: finalMessage,
options: {
...prompt.config,
signal,
user: userId,
workspace: session.config.workspaceId,
session: session.config.sessionId,
byokLeaseId,
quotaBackedRoutesAllowed,
featureKind: 'image',
},
};
}
private async persistImageAttachment(
userId: string,
workspaceId: string,
@@ -1,133 +0,0 @@
import type { LlmBackendConfig, LlmProtocol } from '../../../../native';
import { llmPlanAttachmentReference } from '../../../../native';
import type { PromptAttachment } from '../../providers/types';
import {
type AdmittedAttachmentSource,
admittedAttachmentToPromptAttachment,
} from './attachment-admission';
export type AdmittedAttachmentMaterializationPlan = {
mode: 'inline';
reason: 'admitted_bytes';
attachment: PromptAttachment;
};
export type HostAttachmentMaterializationRequest = {
attachmentId: string;
target: 'bytes' | 'data';
providerConstraint?: string;
maxSize: number;
timeoutMs: number;
redirectPolicy: 'follow-safe';
expectedMime?: string;
url: string;
};
type RemoteReferenceReason =
| 'generic_remote_reference'
| 'gemini_api_file_uri'
| 'gemini_api_youtube_url';
type MaterializationRequestReason =
| 'generic_remote_reference'
| 'gemini_api_inline_http_url'
| 'unsupported_scheme'
| 'non_url_source';
function assertRemoteReferenceReason(
reason: string
): asserts reason is RemoteReferenceReason {
if (
reason !== 'generic_remote_reference' &&
reason !== 'gemini_api_file_uri' &&
reason !== 'gemini_api_youtube_url'
) {
throw new Error(`Unexpected remote attachment reference reason: ${reason}`);
}
}
function assertMaterializationRequestReason(
reason: string
): asserts reason is MaterializationRequestReason {
if (
reason !== 'gemini_api_inline_http_url' &&
reason !== 'generic_remote_reference' &&
reason !== 'unsupported_scheme' &&
reason !== 'non_url_source'
) {
throw new Error(`Unexpected attachment materialization reason: ${reason}`);
}
}
export async function planHostUrlAttachmentMaterialization(
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
input: {
attachmentId: string;
url: string;
expectedMime?: string;
maxSize: number;
timeoutMs?: number;
}
): Promise<
| {
mode: 'remote_reference';
reason:
| 'generic_remote_reference'
| 'gemini_api_file_uri'
| 'gemini_api_youtube_url';
url: string;
}
| {
mode: 'materialization_request';
reason:
| 'generic_remote_reference'
| 'gemini_api_inline_http_url'
| 'unsupported_scheme'
| 'non_url_source';
request: HostAttachmentMaterializationRequest;
}
> {
const plan = await llmPlanAttachmentReference(protocol, backendConfig, {
url: input.url,
});
const forceHostMaterialization =
protocol === 'gemini' &&
backendConfig.request_layer === 'gemini_vertex' &&
plan.reason === 'generic_remote_reference';
if (plan.mode === 'remote' && !forceHostMaterialization) {
assertRemoteReferenceReason(plan.reason);
return {
mode: 'remote_reference',
reason: plan.reason,
url: input.url,
};
}
assertMaterializationRequestReason(plan.reason);
return {
mode: 'materialization_request',
reason: plan.reason,
request: {
attachmentId: input.attachmentId,
target: 'bytes',
providerConstraint: protocol,
maxSize: input.maxSize,
timeoutMs: input.timeoutMs ?? 15_000,
redirectPolicy: 'follow-safe',
expectedMime: input.expectedMime,
url: input.url,
},
};
}
export function planAdmittedAttachmentMaterialization(
source: AdmittedAttachmentSource
): AdmittedAttachmentMaterializationPlan {
return {
mode: 'inline',
reason: 'admitted_bytes',
attachment: admittedAttachmentToPromptAttachment(source),
};
}
@@ -1,124 +0,0 @@
import { Injectable } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { ServerFeature, ServerService } from '../../../../core';
import { QuotaStateService } from '../../../../core/quota/state';
import type { ChatSession } from '../../session';
import { type ToolsConfig } from '../../types';
import { getTools } from '../../utils';
import {
ModelSelectionPolicy,
type ResolveModelInput,
} from '../model-selection-policy';
export type ChatSelectionOptions = {
responseMode: 'text' | 'object' | 'image';
modelId?: string;
reasoning?: boolean;
webSearch?: boolean;
toolsConfig?: ToolsConfig;
byokLeaseId?: string;
billingUnitId?: string;
featureKind?: 'chat' | 'action' | 'image';
quotaBackedRoutesAllowed?: boolean;
};
type ResolvePolicyModelInput = ResolveModelInput & {
proModels?: string[] | null;
userId?: string;
paymentEnabled?: boolean;
};
@Injectable()
export class CapabilityPolicyHost {
constructor(
private readonly server: ServerService,
private readonly moduleRef: ModuleRef,
private readonly modelSelection: ModelSelectionPolicy
) {}
private async hasAiProAccess(
userId: string | undefined,
paymentEnabled: boolean | undefined
) {
if (!paymentEnabled || !userId) {
return false;
}
try {
const state = await this.moduleRef
.get(QuotaStateService, { strict: false })
.reconcileUserQuotaState(userId);
const flags = state.flags as { unlimitedCopilot?: boolean };
return (
!!flags.unlimitedCopilot ||
['pro', 'lifetime_pro', 'ai'].includes(state.plan)
);
} catch {
return false;
}
}
private async resolveModel(input: ResolvePolicyModelInput) {
const resolved = this.modelSelection.resolveRequestedModel(input);
if (!resolved.matchedOptionalModel) {
return resolved.selectedModel;
}
if (
input.paymentEnabled &&
this.modelSelection.matchesModelList(
input.proModels ?? [],
input.requestedModelId
) &&
!(await this.hasAiProAccess(input.userId, input.paymentEnabled))
) {
return input.defaultModel;
}
return resolved.selectedModel;
}
async selectChat(session: ChatSession, options: ChatSelectionOptions) {
const model = await this.resolveChatModel({
userId: session.config.userId,
defaultModel: session.model,
optionalModels: session.optionalModels,
proModels: session.config.promptConfig?.proModels,
requestedModelId: options.modelId,
paymentEnabled: this.server.features.includes(ServerFeature.Payment),
});
const tools = getTools(
session.config.promptConfig?.tools,
options.toolsConfig
);
return {
model,
providerOptions: {
...session.config.promptConfig,
user: session.config.userId,
session: session.config.sessionId,
workspace: session.config.workspaceId,
byokLeaseId: options.byokLeaseId,
billingUnitId: options.billingUnitId,
featureKind: options.featureKind ?? 'chat',
quotaBackedRoutesAllowed: options.quotaBackedRoutesAllowed,
reasoning: options.reasoning,
webSearch: options.webSearch,
tools,
},
};
}
async resolveChatModel(input: ResolvePolicyModelInput) {
return await this.resolveModel(input);
}
async resolvePromptModel(input: ResolveModelInput) {
return this.modelSelection.resolveRequestedModel(input).selectedModel;
}
async resolveFixedTaskModel(input: ResolveModelInput) {
return this.modelSelection.resolveRequestedModel(input).selectedModel;
}
}
@@ -5,8 +5,8 @@ import {
CopilotSessionNotFound,
Mutex,
} from '../../../../base';
import { CopilotAccessPolicy } from '../../access';
import { CompatSubmissionStore } from '../../compat/submission-store';
import { ConversationPolicy } from '../../conversation/policy';
import {
canonicalizeTurnTrace,
type Turn,
@@ -35,7 +35,7 @@ export class ConversationHost {
private readonly sessions: ChatSessionService,
private readonly submissions: CompatSubmissionStore,
private readonly mutex: Mutex,
private readonly access: CopilotAccessPolicy
private readonly policy: ConversationPolicy
) {}
private async loadAcceptedTurn(
@@ -109,31 +109,22 @@ export class ConversationHost {
session: ChatSession,
sessionId: string,
messageId?: string,
retry = false,
byokLeaseId?: string
retry = false
): Promise<AppendedSessionMessage> {
const resolveChatRouteAccess = () =>
this.access.resolveTurnRouteAccess({
userId,
workspaceId: session.config.workspaceId,
byokLeaseId,
featureKind: 'chat',
});
const quotaBackedRoutesAllowed = () => this.policy.hasQuota(userId);
if (!messageId) {
await this.sessions.revertLatestMessage(sessionId, false);
session.revertLatestMessage(false);
if (!session.latestUserTurn) {
const routeAccess = await resolveChatRouteAccess();
return {
turn: session.latestUserTurn,
quotaBackedRoutesAllowed: routeAccess.quotaBackedRoutesAllowed,
quotaBackedRoutesAllowed: await quotaBackedRoutesAllowed(),
};
}
const routeAccess = await resolveChatRouteAccess();
return {
turn: session.latestUserTurn,
quotaBackedRoutesAllowed: routeAccess.quotaBackedRoutesAllowed,
quotaBackedRoutesAllowed: await quotaBackedRoutesAllowed(),
};
}
@@ -177,7 +168,7 @@ export class ConversationHost {
};
}
const routeAccess = await resolveChatRouteAccess();
const quotaAllowed = await quotaBackedRoutesAllowed();
const submission = await this.submissions.get(messageId);
if (!submission || submission.sessionId !== sessionId) {
@@ -192,7 +183,6 @@ export class ConversationHost {
const turn = await this.sessions.appendTurn({
sessionId,
userId: session.config.userId,
prompt: { model: session.model },
compatSubmissionId: messageId,
turn: {
conversationId: sessionId,
@@ -213,7 +203,7 @@ export class ConversationHost {
session.pushPersistedTurn(turn);
return {
turn,
quotaBackedRoutesAllowed: routeAccess.quotaBackedRoutesAllowed,
quotaBackedRoutesAllowed: quotaAllowed,
};
}
@@ -222,8 +212,7 @@ export class ConversationHost {
sessionId: string,
query: Record<string, string | string[]>
): Promise<PreparedConversationTurn> {
const { messageId, retry, params, byokLeaseId } =
ChatQuerySchema.parse(query);
const { messageId, retry, params } = ChatQuerySchema.parse(query);
const session = await this.sessions.get(sessionId);
if (!session || session.config.userId !== userId) {
throw new CopilotSessionNotFound();
@@ -233,8 +222,7 @@ export class ConversationHost {
session,
sessionId,
messageId,
retry,
byokLeaseId
retry
);
const currentUserMessage =
session.stashTurns.findLast(turn => turn.role === 'user') ??
@@ -280,7 +268,6 @@ export class ConversationHost {
const persisted = await this.sessions.appendTurn({
sessionId: session.config.sessionId,
userId: session.config.userId,
prompt: { model: session.model },
turn: assistantTurn,
});
session.pushPersistedTurn(persisted);
@@ -1,43 +0,0 @@
import { Injectable } from '@nestjs/common';
import type { NodeTextMiddleware } from '../../config';
import type {
CopilotChatOptions,
CopilotChatTools,
} from '../../providers/types';
import type { CopilotTool, CopilotToolSet } from '../../tools';
import type { ToolLoopBackend } from '../tool/bridge';
import { ToolRuntime } from '../tool-runtime';
export type ProviderSpecificToolResolver = (
toolName: CopilotChatTools,
model: string
) => [string, CopilotTool?] | undefined;
@Injectable()
export class ToolExecutorHost {
constructor(private readonly runtime: ToolRuntime) {}
async getTools(
options: CopilotChatOptions,
model: string,
resolveProviderSpecificTool?: ProviderSpecificToolResolver
): Promise<CopilotToolSet> {
return await this.runtime.getTools(
options,
model,
resolveProviderSpecificTool
);
}
createNativeAdapter(
backend: ToolLoopBackend,
tools: CopilotToolSet,
options: {
maxSteps?: number;
nodeTextMiddleware?: NodeTextMiddleware[];
} = {}
) {
return this.runtime.createNativeAdapter(backend, tools, options);
}
}
@@ -1,55 +0,0 @@
import { Injectable } from '@nestjs/common';
import { CopilotSessionInvalidInput } from '../../../base';
import { llmResolveRequestedModelMatch } from '../../../native';
import { CopilotProviderRegistryService } from '../providers/registry-service';
export type ResolveModelInput = {
defaultModel: string;
optionalModels?: string[] | null;
requestedModelId?: string;
};
@Injectable()
export class ModelSelectionPolicy {
constructor(private readonly registries: CopilotProviderRegistryService) {}
private getRegistry() {
return this.registries.getRegistry();
}
private matchRequestedModel(
optionalModels: string[],
requestedModelId?: string,
defaultModel?: string
) {
return llmResolveRequestedModelMatch({
providerIds: [...this.getRegistry().profiles.keys()],
optionalModels,
requestedModelId,
defaultModel,
});
}
resolveRequestedModel(input: ResolveModelInput): {
selectedModel: string;
matchedOptionalModel: boolean;
} {
if (!input.defaultModel) {
throw new CopilotSessionInvalidInput('Model is required');
}
const matched = this.matchRequestedModel(
input.optionalModels ?? [],
input.requestedModelId,
input.defaultModel
);
return {
selectedModel: matched.selectedModel ?? input.defaultModel,
matchedOptionalModel: matched.matchedOptionalModel,
};
}
matchesModelList(models: string[], modelId?: string) {
return this.matchRequestedModel(models, modelId).matchedOptionalModel;
}
}
@@ -1,4 +1,4 @@
import { NetworkError } from '../../../base';
import { CopilotQuotaExceeded, NetworkError } from '../../../base';
const LLM_TIMEOUT_ERROR_PREFIX = 'llm_timeout:';
@@ -18,6 +18,9 @@ function nativeErrorMessage(error: unknown) {
export function mapNativeSemanticError(error: unknown): unknown {
const message = nativeErrorMessage(error);
if (message === 'access_unavailable') {
return new CopilotQuotaExceeded();
}
if (message?.startsWith(LLM_TIMEOUT_ERROR_PREFIX)) {
return new NetworkError(
message.slice(LLM_TIMEOUT_ERROR_PREFIX.length).trim() ||
@@ -1,536 +0,0 @@
import { Injectable, Logger } from '@nestjs/common';
import { NoCopilotProviderAvailable } from '../../../base';
import {
llmDispatchPlan,
llmDispatchPlanStream,
type LlmDispatchResponse,
llmEmbeddingDispatchPlan,
llmImageDispatchPlan,
type LlmImageResponse,
llmRerankDispatchPlan,
llmStructuredDispatchPlan,
llmValidateJsonSchema,
parseNativeStructuredOutput,
} from '../../../native';
import { type ByokFeatureKind, ByokService } from '../byok';
import { type StreamObject } from '../providers/types';
import { CopilotExecutionMetrics } from './execution-metrics';
import {
type ExecutionPlan,
type ExecutionPlanForKind,
type NativeChatDispatchPlan,
type NativeImageDispatchPlan,
} from './execution-plan';
import { mapNativeSemanticError } from './native-errors';
import {
createNativeToolLoopAdapter,
NativeProviderAdapter,
type NativeProviderAdapterOptions,
} from './tool/native-adapter';
const logger = new Logger('NativeExecutionEngine');
function modelIdForError(modelId?: string) {
return modelId ?? 'auto';
}
type ExecutionPlanKind = ExecutionPlan['request']['kind'];
type ValueExecutionKind = Exclude<
ExecutionPlanKind,
'streamText' | 'streamObject' | 'image'
>;
type StreamExecutionKind = Extract<
ExecutionPlanKind,
'streamText' | 'streamObject'
>;
export type NativeImageArtifact = LlmImageResponse['images'][number];
function resolveAbortSignal(
signalOrOptions?: AbortSignal | { signal?: AbortSignal }
) {
return signalOrOptions &&
typeof signalOrOptions === 'object' &&
'aborted' in signalOrOptions
? signalOrOptions
: signalOrOptions?.signal;
}
function extractTextResponse(response: LlmDispatchResponse) {
return response.message.content
.filter(part => part.type === 'text' || part.type === 'reasoning')
.map(part => part.text)
.join('')
.trim();
}
function getUsageContext(plan: ExecutionPlan) {
const options = 'options' in plan.request ? plan.request.options : undefined;
const requestFeatureKind =
plan.request.kind === 'text' ||
plan.request.kind === 'streamText' ||
plan.request.kind === 'streamObject'
? 'chat'
: plan.request.kind;
return {
workspaceId: options?.workspace,
userId: options?.user,
sessionId: options?.session,
taskId: options?.taskId,
actionId: options?.actionId,
billingUnitId: options?.billingUnitId,
featureKind: options?.featureKind ?? requestFeatureKind,
};
}
async function recordByokUsage(
byok: ByokService,
plan: ExecutionPlan,
input: {
providerId?: string;
model?: string | null;
usage?: LlmDispatchResponse['usage'];
}
) {
const context = getUsageContext(plan);
try {
await byok.recordUsage({
workspaceId: context.workspaceId,
userId: context.userId,
sessionId: context.sessionId,
taskId: context.taskId,
actionId: context.actionId,
billingUnitId: context.billingUnitId,
featureKind: context.featureKind as ByokFeatureKind,
providerId: input.providerId,
model: input.model,
usage: input.usage,
});
} catch (error) {
logger.warn(
`Failed to record BYOK usage: ${
error instanceof Error ? error.message : String(error)
}`
);
}
}
async function recordSingleByokRouteFailure(
byok: ByokService,
plan: ExecutionPlan,
error: unknown
) {
const [providerId] = plan.routePolicy.fallbackOrder;
if (plan.routePolicy.fallbackOrder.length !== 1 || !providerId) {
return;
}
const context = getUsageContext(plan);
try {
await byok.recordProviderFailure({
workspaceId: context.workspaceId,
providerId,
featureKind: context.featureKind as ByokFeatureKind,
error,
});
} catch (recordError) {
logger.warn(
`Failed to record BYOK provider failure: ${
recordError instanceof Error ? recordError.message : String(recordError)
}`
);
}
}
function recordPreparedDispatch(
executionMetrics: CopilotExecutionMetrics | undefined,
plan: ExecutionPlan,
routeCount: number
) {
executionMetrics?.recordDispatch(
plan.request.kind,
'prepared_routes',
routeCount
);
}
function createNativeChatAdapter(
dispatch: NativeChatDispatchPlan,
options?: {
onUsage?: NativeProviderAdapterOptions['onUsage'];
}
) {
if (dispatch.hasTools) {
return createNativeToolLoopAdapter(
{ preparedRoutes: dispatch.routes },
dispatch.prepared.tools,
{
maxSteps: dispatch.prepared.maxSteps,
nodeTextMiddleware: dispatch.prepared.postprocess?.nodeTextMiddleware,
onUsage: options?.onUsage,
}
);
}
const nativeDispatch = (
_nativeRequest: typeof dispatch.prepared.request,
signalOrOptions?: AbortSignal | { signal?: AbortSignal }
) =>
llmDispatchPlanStream({
preparedRoutes: dispatch.routes,
signal: resolveAbortSignal(signalOrOptions),
});
return new NativeProviderAdapter(nativeDispatch, {
nodeTextMiddleware: dispatch.prepared.postprocess?.nodeTextMiddleware,
onUsage: options?.onUsage,
});
}
async function runPreparedValuePlan<TResult>(
plan: ExecutionPlan,
routeCount: number,
executionMetrics: CopilotExecutionMetrics | undefined,
run: () => Promise<TResult>,
byok: ByokService
) {
recordPreparedDispatch(executionMetrics, plan, routeCount);
try {
return await run();
} catch (error) {
const mapped = mapNativeSemanticError(error);
await recordSingleByokRouteFailure(byok, plan, mapped);
throw mapped;
}
}
async function* mapPreparedStreamErrors<T>(
source: AsyncIterable<T>,
plan: ExecutionPlan,
byok: ByokService
): AsyncIterableIterator<T> {
try {
yield* source;
} catch (error) {
const mapped = mapNativeSemanticError(error);
await recordSingleByokRouteFailure(byok, plan, mapped);
throw mapped;
}
}
async function runChatValuePlan(
plan: ExecutionPlan,
dispatch: NativeChatDispatchPlan,
executionMetrics: CopilotExecutionMetrics | undefined,
byok: ByokService
) {
const adapter = createNativeChatAdapter(dispatch);
return await runPreparedValuePlan(
plan,
dispatch.routes.length,
executionMetrics,
async () => {
if (
!dispatch.hasTools &&
!dispatch.prepared.postprocess?.nodeTextMiddleware?.length
) {
const result = await llmDispatchPlan({
preparedRoutes: dispatch.routes,
});
await recordByokUsage(byok, plan, {
providerId: result.provider_id,
model: result.response.model,
usage: result.response.usage,
});
return extractTextResponse(result.response);
}
if (plan.request.kind !== 'text') {
throw new Error('chat value dispatch requires text plan');
}
return await adapter.text(
dispatch.prepared.request,
plan.hostContext.signal,
plan.request.messages
);
},
byok
);
}
async function* runChatStreamPlan(
plan: ExecutionPlan,
dispatch: NativeChatDispatchPlan,
executionMetrics: CopilotExecutionMetrics | undefined,
byok: ByokService
): AsyncIterableIterator<string | StreamObject> {
const adapter = createNativeChatAdapter(dispatch, {
onUsage: async usage => {
await recordByokUsage(byok, plan, {
providerId: usage.providerId,
model: usage.model,
usage: usage.usage,
});
},
});
recordPreparedDispatch(executionMetrics, plan, dispatch.routes.length);
if (plan.request.kind === 'streamText') {
yield* mapPreparedStreamErrors(
adapter.streamText(
dispatch.prepared.request,
plan.hostContext.signal,
plan.request.messages
),
plan,
byok
);
return;
}
if (plan.request.kind === 'streamObject') {
yield* mapPreparedStreamErrors(
adapter.streamObject(
dispatch.prepared.request,
plan.hostContext.signal,
plan.request.messages
),
plan,
byok
);
return;
}
throw new Error('chat stream dispatch requires streamText/streamObject plan');
}
async function* runPreparedImageArtifactPlan(
dispatch: NativeImageDispatchPlan,
plan: ExecutionPlan,
executionMetrics: CopilotExecutionMetrics | undefined,
byok: ByokService
): AsyncIterableIterator<NativeImageArtifact> {
if (plan.request.kind !== 'image') {
throw new Error('image dispatch requires image plan');
}
recordPreparedDispatch(executionMetrics, plan, dispatch.routes.length);
let result;
try {
result = await llmImageDispatchPlan({
preparedRoutes: dispatch.routes,
});
await recordByokUsage(byok, plan, {
providerId: result.provider_id,
model: dispatch.prepared.route.model,
usage: result.response.usage
? {
prompt_tokens: result.response.usage.input_tokens ?? 0,
completion_tokens: result.response.usage.output_tokens ?? 0,
total_tokens: result.response.usage.total_tokens ?? 0,
}
: undefined,
});
} catch (error) {
const mapped = mapNativeSemanticError(error);
await recordSingleByokRouteFailure(byok, plan, mapped);
throw mapped;
}
for (const artifact of result.response.images) {
yield artifact;
}
}
async function executePreparedPlan(
plan: ExecutionPlan,
executionMetrics: CopilotExecutionMetrics | undefined,
byok: ByokService
): Promise<string | number[][] | number[] | null> {
switch (plan.request.kind) {
case 'text': {
const dispatch = plan.nativeDispatch?.chat;
return dispatch
? await runChatValuePlan(plan, dispatch, executionMetrics, byok)
: null;
}
case 'structured': {
const dispatch = plan.nativeDispatch?.structured;
if (!dispatch) {
return null;
}
return await runPreparedValuePlan(
plan,
dispatch.routes.length,
executionMetrics,
async () => {
const result = await llmStructuredDispatchPlan({
preparedRoutes: dispatch.routes,
});
await recordByokUsage(byok, plan, {
providerId: result.provider_id,
model: result.response.model,
usage: result.response.usage,
});
const parsed = parseNativeStructuredOutput(result.response);
const validated = llmValidateJsonSchema(
dispatch.prepared.request.schema,
parsed
);
return JSON.stringify(validated);
},
byok
);
}
case 'embedding': {
const dispatch = plan.nativeDispatch?.embedding;
if (!dispatch) {
return null;
}
return await runPreparedValuePlan(
plan,
dispatch.routes.length,
executionMetrics,
async () => {
const result = await llmEmbeddingDispatchPlan({
preparedRoutes: dispatch.routes,
});
await recordByokUsage(byok, plan, {
providerId: result.provider_id,
model: result.response.model,
usage: result.response.usage
? {
prompt_tokens: result.response.usage.prompt_tokens,
completion_tokens: 0,
total_tokens: result.response.usage.total_tokens,
}
: undefined,
});
return result.response.embeddings;
},
byok
);
}
case 'rerank': {
const dispatch = plan.nativeDispatch?.rerank;
if (!dispatch) {
return null;
}
return await runPreparedValuePlan(
plan,
dispatch.routes.length,
executionMetrics,
async () => {
const result = await llmRerankDispatchPlan({
preparedRoutes: dispatch.routes,
});
await recordByokUsage(byok, plan, {
providerId: result.provider_id,
model: result.response.model,
});
return result.response.scores;
},
byok
);
}
default:
return null;
}
}
function executePreparedStreamPlan(
plan: ExecutionPlan,
executionMetrics: CopilotExecutionMetrics | undefined,
byok: ByokService
): AsyncIterableIterator<string | StreamObject> | null {
switch (plan.request.kind) {
case 'streamText':
case 'streamObject': {
const dispatch = plan.nativeDispatch?.chat;
return dispatch
? runChatStreamPlan(plan, dispatch, executionMetrics, byok)
: null;
}
default:
return null;
}
}
function noRouteStream<T>(plan: ExecutionPlan) {
return (async function* (): AsyncIterableIterator<T> {
yield* [] as T[];
throw new NoCopilotProviderAvailable({
modelId: modelIdForError(plan.request.cond.modelId),
});
})();
}
@Injectable()
export class NativeExecutionEngine {
constructor(
private readonly byok: ByokService,
private readonly executionMetrics?: CopilotExecutionMetrics
) {}
private noRoute(plan: ExecutionPlan): never {
throw new NoCopilotProviderAvailable({
modelId: modelIdForError(plan.request.cond.modelId),
});
}
async execute(
plan: ExecutionPlanForKind<'text' | 'structured'>
): Promise<string>;
async execute(plan: ExecutionPlanForKind<'embedding'>): Promise<number[][]>;
async execute(plan: ExecutionPlanForKind<'rerank'>): Promise<number[]>;
async execute(
plan: ExecutionPlanForKind<ValueExecutionKind>
): Promise<string | number[][] | number[]> {
const result = await executePreparedPlan(
plan,
this.executionMetrics,
this.byok
);
if (result === null) {
return this.noRoute(plan);
}
return result;
}
executeStream(
plan: ExecutionPlanForKind<'streamText'>
): AsyncIterableIterator<string>;
executeStream(
plan: ExecutionPlanForKind<'streamObject'>
): AsyncIterableIterator<StreamObject>;
executeStream(
plan: ExecutionPlanForKind<StreamExecutionKind>
): AsyncIterableIterator<string | StreamObject> {
const result = executePreparedStreamPlan(
plan,
this.executionMetrics,
this.byok
);
if (result) {
return result;
}
return noRouteStream(plan);
}
executeImageArtifacts(
plan: ExecutionPlanForKind<'image'>
): AsyncIterableIterator<NativeImageArtifact> {
const dispatch = plan.nativeDispatch?.image;
if (dispatch) {
return runPreparedImageArtifactPlan(
dispatch,
plan,
this.executionMetrics,
this.byok
);
}
return noRouteStream(plan);
}
}
@@ -114,7 +114,7 @@ export async function buildCanonicalNativeRequest({
request = llmBuildCanonicalRequest({
model,
messages: normalizedMessages,
maxTokens: options.maxTokens ?? undefined,
maxTokens: options.maxOutputTokens ?? undefined,
temperature: options.temperature ?? undefined,
tools: toolContracts,
include,
@@ -171,7 +171,7 @@ export async function buildCanonicalNativeStructuredRequest({
model,
messages: normalizedMessages,
schema: explicitResponseContract?.responseSchemaJson,
maxTokens: options.maxTokens ?? undefined,
maxTokens: options.maxOutputTokens ?? undefined,
temperature: options.temperature ?? undefined,
reasoning,
strict: options.strict,
@@ -1,4 +1,5 @@
import { Injectable } from '@nestjs/common';
/* oxlint-disable import/no-cycle -- Prompt execution delegates to the capability runtime. */
import { forwardRef, Inject, Injectable } from '@nestjs/common';
import { CopilotPromptNotFound } from '../../../base';
import { PromptService } from '../prompt/service';
@@ -11,7 +12,6 @@ import {
} from '../providers/types';
import { CapabilityRuntime } from './capability-runtime';
import type { RequiredStructuredOutputContract } from './contracts';
import { CapabilityPolicyHost } from './hosts/capability-policy-host';
type PromptRuntimeStructuredContract = RequiredStructuredOutputContract;
@@ -24,8 +24,11 @@ type PromptRuntimeStructuredProviderOptions = Omit<
export class PromptRuntime {
constructor(
private readonly prompts: PromptService,
private readonly capabilityPolicy: CapabilityPolicyHost,
private readonly runtime: CapabilityRuntime
@Inject(forwardRef(() => CapabilityRuntime))
private readonly runtime: Pick<
CapabilityRuntime,
'text' | 'generateStructuredValue'
>
) {}
private async preparePrompt(
@@ -44,11 +47,8 @@ export class PromptRuntime {
return {
prompt,
modelId: await this.capabilityPolicy.resolvePromptModel({
defaultModel: prompt.model,
optionalModels: prompt.optionalModels,
requestedModelId: options.modelId,
}),
builtInRouteId: prompt.name,
modelId: 'route-selected',
finalMessages: [
...this.prompts.finish(prompt, params),
...(options.appendMessages ?? []),
@@ -75,6 +75,7 @@ export class PromptRuntime {
{
...prepared.prompt.config,
...options.providerOptions,
builtInRouteId: prepared.builtInRouteId,
},
{ prefer: prepared.prefer }
);
@@ -100,6 +101,7 @@ export class PromptRuntime {
{
...prepared.prompt.config,
...options.providerOptions,
builtInRouteId: prepared.builtInRouteId,
responseSchemaJson: options.responseContract.responseSchemaJson,
schemaHash: options.responseContract.schemaHash,
strict: options.strict,
@@ -1,234 +0,0 @@
import type {
CopilotProviderExecution,
PreparedNativeExecution,
PreparedNativeRequestOptions,
ProviderChatDriver,
ProviderChatDriverPrepareInput,
} from '../providers/provider-runtime-contract';
import type {
CopilotChatOptions,
CopilotProviderModel,
CopilotProviderType,
ModelConditions,
ModelFullConditions,
PromptMessage,
StreamObject,
} from '../providers/types';
import { ModelOutputType } from '../providers/types';
import {
resolveDriverOrThrow,
resolvePreparedModelId,
runPreparedExecution,
} from './provider-driver-runtime';
import type { NativeProviderAdapter } from './tool/native-adapter';
type MetricLabels = Record<string, string | number | boolean | undefined>;
export type ChatRuntimeContext = {
type: CopilotProviderType;
resolveChatDriver: () => ProviderChatDriver | undefined;
selectModel: (cond: ModelFullConditions) => CopilotProviderModel;
metricLabels: (
model: string,
labels?: MetricLabels,
execution?: CopilotProviderExecution
) => MetricLabels;
createPreparedExecutionAdapter: (
prepared: PreparedNativeExecution
) => NativeProviderAdapter;
};
type ChatExecutionMode = {
kind: ProviderChatDriverPrepareInput['kind'];
outputType: ModelOutputType;
unsupportedKind: 'text' | 'object';
callMetric: string;
errorMetric: string;
};
export async function prepareNativeChatExecution(
resolveChatDriver: () => ProviderChatDriver | undefined,
buildPreparedNativeExecution: (
options: PreparedNativeRequestOptions
) => Promise<PreparedNativeExecution>,
input: ProviderChatDriverPrepareInput
): Promise<PreparedNativeExecution | null> {
const driver = resolveChatDriver();
if (!driver) {
return null;
}
const prepared = await driver.prepare(input);
if (!prepared) {
return null;
}
return await buildPreparedNativeExecution({
...prepared,
execution: input.execution,
options: input.options,
});
}
async function runNativeChat(
context: ChatRuntimeContext,
prepareNativeExecution: (
kind: ProviderChatDriverPrepareInput['kind'],
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => Promise<PreparedNativeExecution | null>,
mode: ChatExecutionMode,
model: ModelConditions,
messages: PromptMessage[],
options: CopilotChatOptions | undefined,
execution: CopilotProviderExecution | undefined,
run: (
adapter: NativeProviderAdapter,
prepared: PreparedNativeExecution,
signal: AbortSignal | undefined,
promptMessages: PromptMessage[]
) =>
| Promise<string>
| AsyncIterableIterator<string>
| AsyncIterableIterator<StreamObject>
) {
const driver = resolveDriverOrThrow(
context.type,
mode.unsupportedKind,
context.resolveChatDriver
);
const chatOptions = options ?? {};
const prepared = await prepareNativeExecution(
mode.kind,
model,
messages,
chatOptions,
execution
);
const modelId = resolvePreparedModelId(
context,
model,
mode.outputType,
prepared
);
return await runPreparedExecution({
driver,
prepared,
modelId,
execution,
metricContext: context,
metricsName: {
call: mode.callMetric,
error: mode.errorMetric,
},
execute: async preparedExecution =>
await run(
context.createPreparedExecutionAdapter(preparedExecution),
preparedExecution,
chatOptions.signal,
messages
),
});
}
export async function runNativeText(
context: ChatRuntimeContext,
prepareNativeExecution: (
kind: ProviderChatDriverPrepareInput['kind'],
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => Promise<PreparedNativeExecution | null>,
model: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) {
return (await runNativeChat(
context,
prepareNativeExecution,
{
kind: 'text',
outputType: ModelOutputType.Text,
unsupportedKind: 'text',
callMetric: 'chat_text_calls',
errorMetric: 'chat_text_errors',
},
model,
messages,
options,
execution,
(adapter, prepared, signal, promptMessages) =>
adapter.text(prepared.request, signal, promptMessages)
)) as string;
}
export async function* runNativeStreamText(
context: ChatRuntimeContext,
prepareNativeExecution: (
kind: ProviderChatDriverPrepareInput['kind'],
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => Promise<PreparedNativeExecution | null>,
model: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
): AsyncIterableIterator<string> {
yield* (await runNativeChat(
context,
prepareNativeExecution,
{
kind: 'streamText',
outputType: ModelOutputType.Text,
unsupportedKind: 'text',
callMetric: 'chat_text_stream_calls',
errorMetric: 'chat_text_stream_errors',
},
model,
messages,
options,
execution,
(adapter, prepared, signal, promptMessages) =>
adapter.streamText(prepared.request, signal, promptMessages)
)) as AsyncIterableIterator<string>;
}
export async function* runNativeStreamObject(
context: ChatRuntimeContext,
prepareNativeExecution: (
kind: ProviderChatDriverPrepareInput['kind'],
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => Promise<PreparedNativeExecution | null>,
model: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
): AsyncIterableIterator<StreamObject> {
yield* (await runNativeChat(
context,
prepareNativeExecution,
{
kind: 'streamObject',
outputType: ModelOutputType.Object,
unsupportedKind: 'object',
callMetric: 'chat_object_stream_calls',
errorMetric: 'chat_object_stream_errors',
},
model,
messages,
options,
execution,
(adapter, prepared, signal, promptMessages) =>
adapter.streamObject(prepared.request, signal, promptMessages)
)) as AsyncIterableIterator<StreamObject>;
}
@@ -1,682 +0,0 @@
import {
CopilotPromptInvalid,
CopilotProviderNotSupported,
metrics,
} from '../../../base';
import {
buildLlmEmbeddingRequest,
buildLlmRerankRequest,
type LlmBackendConfig,
type LlmEmbeddingRequest,
type LlmProtocol,
type LlmRerankRequest,
type LlmStructuredRequest,
type LlmStructuredResponse,
llmValidateJsonSchema,
parseNativeStructuredOutput,
} from '../../../native';
import type { ProviderMiddlewareConfig } from '../config';
import { resolveProviderModelRoute } from '../providers/provider-model-runtime';
import type {
CopilotProviderExecution,
EmbeddingProviderDriver,
ImageProviderDriver,
PreparedNativeEmbeddingExecution,
PreparedNativeImageExecution,
PreparedNativeRerankExecution,
PreparedNativeStructuredExecution,
RerankProviderDriver,
StructuredProviderDriver,
} from '../providers/provider-runtime-contract';
import type {
CopilotChatOptions,
CopilotEmbeddingOptions,
CopilotImageOptions,
CopilotProviderModel,
CopilotProviderType,
CopilotRerankRequest,
CopilotStructuredOptions,
ModelAttachmentCapability,
ModelConditions,
ModelFullConditions,
PromptMessage,
} from '../providers/types';
import { ModelOutputType } from '../providers/types';
import { type RequiredStructuredOutputContract } from './contracts';
import { buildNativeStructuredRequest } from './native-request-runtime';
const DEFAULT_EMBEDDING_TASK_TYPE = 'RETRIEVAL_DOCUMENT';
type MetricLabels = Record<string, string | number | boolean | undefined>;
type DriverMetricNames = {
call: string;
error: string;
};
export type StructuredRuntimeContext = {
type: CopilotProviderType;
resolveStructuredDriver: () => StructuredProviderDriver | undefined;
checkParams: (input: {
cond: ModelFullConditions;
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
execution?: CopilotProviderExecution;
}) => Promise<ModelFullConditions>;
selectModel: (
cond: ModelFullConditions,
execution?: CopilotProviderExecution
) => CopilotProviderModel;
getAttachCapability: (
model: CopilotProviderModel,
outputType: ModelOutputType
) => ModelAttachmentCapability | undefined;
getActiveProviderMiddleware: (
execution?: CopilotProviderExecution
) => ProviderMiddlewareConfig;
buildPreparedNativeStructuredExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmStructuredRequest,
execution?: CopilotProviderExecution
) => PreparedNativeStructuredExecution;
createNativeStructuredDispatch: (
backendConfig: LlmBackendConfig,
protocol: LlmProtocol,
execution?: CopilotProviderExecution
) => (request: LlmStructuredRequest) => Promise<LlmStructuredResponse>;
metricLabels: (
model: string,
labels?: MetricLabels,
execution?: CopilotProviderExecution
) => MetricLabels;
};
export type EmbeddingRuntimeContext = {
type: CopilotProviderType;
resolveEmbeddingDriver: () => EmbeddingProviderDriver | undefined;
checkParams: (input: {
cond: ModelFullConditions;
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
execution?: CopilotProviderExecution;
}) => Promise<ModelFullConditions>;
selectModel: (
cond: ModelFullConditions,
execution?: CopilotProviderExecution
) => CopilotProviderModel;
buildPreparedNativeEmbeddingExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmEmbeddingRequest,
execution?: CopilotProviderExecution
) => PreparedNativeEmbeddingExecution;
createNativeEmbeddingDispatch: (
backendConfig: LlmBackendConfig,
protocol: LlmProtocol,
execution?: CopilotProviderExecution
) => (request: LlmEmbeddingRequest) => Promise<{ embeddings: number[][] }>;
metricLabels: (
model: string,
labels?: MetricLabels,
execution?: CopilotProviderExecution
) => MetricLabels;
};
export type RerankRuntimeContext = {
type: CopilotProviderType;
resolveRerankDriver: () => RerankProviderDriver | undefined;
checkParams: (input: {
cond: ModelFullConditions;
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
execution?: CopilotProviderExecution;
}) => Promise<ModelFullConditions>;
selectModel: (
cond: ModelFullConditions,
execution?: CopilotProviderExecution
) => CopilotProviderModel;
buildPreparedNativeRerankExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmRerankRequest,
execution?: CopilotProviderExecution
) => PreparedNativeRerankExecution;
createNativeRerankDispatch: (
backendConfig: LlmBackendConfig,
protocol: LlmProtocol,
execution?: CopilotProviderExecution
) => (request: LlmRerankRequest) => Promise<{ scores: number[] }>;
};
export type ImageRuntimeContext = {
type: CopilotProviderType;
resolveImageDriver: () => ImageProviderDriver | undefined;
checkParams: (input: {
cond: ModelFullConditions;
messages?: PromptMessage[];
options?: CopilotImageOptions;
withAttachment?: boolean;
execution?: CopilotProviderExecution;
}) => Promise<ModelFullConditions>;
selectModel: (
cond: ModelFullConditions,
execution?: CopilotProviderExecution
) => CopilotProviderModel;
buildPreparedNativeImageExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
messages: PromptMessage[],
options?: CopilotImageOptions,
execution?: CopilotProviderExecution
) => PreparedNativeImageExecution;
};
type NativeExecutionDriverBase = {
createBackendConfig: (
execution?: CopilotProviderExecution
) => Promise<LlmBackendConfig> | LlmBackendConfig;
mapError: (error: unknown) => unknown;
};
type ModelSelectionContext = {
selectModel: (
cond: ModelFullConditions,
execution?: CopilotProviderExecution
) => CopilotProviderModel;
};
type MetricContext = {
metricLabels: (
model: string,
labels?: MetricLabels,
execution?: CopilotProviderExecution
) => MetricLabels;
};
type RoutedPreparedExecution = {
route: {
model: string;
backendConfig: LlmBackendConfig;
protocol: LlmProtocol;
};
};
export function resolveDriverOrThrow<TDriver>(
type: CopilotProviderType,
kind: string,
resolveDriver: () => TDriver | undefined
) {
const driver = resolveDriver();
if (!driver) {
throw new CopilotProviderNotSupported({
provider: type,
kind,
});
}
return driver;
}
export function resolvePreparedModelId(
context: ModelSelectionContext,
cond: ModelConditions,
outputType: ModelOutputType,
prepared?: RoutedPreparedExecution | null
) {
return (
prepared?.route.model ??
context.selectModel({
...cond,
outputType,
}).id
);
}
async function prepareNativeExecutionBase<
TDriver extends NativeExecutionDriverBase,
TPrepared,
>({
resolveDriver,
cond,
outputType,
checkParams,
selectModel,
execution,
checkInput,
buildPrepared,
}: {
resolveDriver: () => TDriver | undefined;
cond: ModelConditions;
outputType: ModelOutputType;
checkParams: (input: {
cond: ModelFullConditions;
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
execution?: CopilotProviderExecution;
}) => Promise<ModelFullConditions>;
selectModel: (
cond: ModelFullConditions,
execution?: CopilotProviderExecution
) => CopilotProviderModel;
execution?: CopilotProviderExecution;
checkInput: {
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
};
buildPrepared: (args: {
driver: TDriver;
model: CopilotProviderModel;
backendConfig: LlmBackendConfig;
protocol: LlmProtocol;
}) => Promise<TPrepared> | TPrepared;
}): Promise<TPrepared | null> {
const driver = resolveDriver();
if (!driver) {
return null;
}
const normalizedCond = await checkParams({
...checkInput,
cond: { ...cond, outputType },
execution,
});
const model = selectModel(normalizedCond, execution);
const backendConfig = await driver.createBackendConfig(execution);
const route = resolveProviderModelRoute(model, outputType);
if (!route.protocol) {
throw new Error(`Missing native protocol for model ${model.id}`);
}
return await buildPrepared({
driver,
model,
backendConfig:
route.requestLayer === backendConfig.request_layer
? backendConfig
: { ...backendConfig, request_layer: route.requestLayer },
protocol: route.protocol,
});
}
export async function runPreparedExecution<
TPrepared extends RoutedPreparedExecution,
TResult,
>({
driver,
prepared,
modelId,
execution,
metricContext,
metricsName,
execute,
}: {
driver: Pick<NativeExecutionDriverBase, 'mapError'>;
prepared: TPrepared | null;
modelId: string;
execution?: CopilotProviderExecution;
metricContext?: MetricContext;
metricsName?: DriverMetricNames;
execute: (prepared: TPrepared) => Promise<TResult>;
}): Promise<TResult> {
try {
if (metricsName && metricContext) {
metrics.ai
.counter(metricsName.call)
.add(1, metricContext.metricLabels(modelId, {}, execution));
}
if (!prepared) {
throw new Error('native route is not available');
}
return await execute(prepared);
} catch (error) {
if (metricsName && metricContext) {
metrics.ai
.counter(metricsName.error)
.add(1, metricContext.metricLabels(modelId, {}, execution));
}
throw driver.mapError(error);
}
}
export async function prepareNativeStructuredExecution(
context: StructuredRuntimeContext,
cond: ModelConditions,
messages: PromptMessage[],
options: CopilotStructuredOptions = {},
responseContract?: RequiredStructuredOutputContract,
execution?: CopilotProviderExecution
): Promise<PreparedNativeStructuredExecution | null> {
const driver = context.resolveStructuredDriver();
if (!driver) {
return null;
}
const structuredOptions = options ?? {};
const normalizedCond = await context.checkParams({
messages,
cond: { ...cond, outputType: ModelOutputType.Structured },
options: structuredOptions,
execution,
});
const model = context.selectModel(normalizedCond, execution);
const backendConfig = await driver.createBackendConfig(execution);
const route = resolveProviderModelRoute(model, ModelOutputType.Structured);
if (!route.protocol) {
throw new Error(`Missing native protocol for model ${model.id}`);
}
const preparedMessages = driver.prepareMessages
? await driver.prepareMessages(messages, backendConfig, structuredOptions)
: messages;
if (!responseContract) {
throw new CopilotPromptInvalid('Schema is required');
}
const { request } = await buildNativeStructuredRequest({
model: model.id,
messages: preparedMessages,
options: structuredOptions,
responseContract,
attachmentCapability: context.getAttachCapability(
model,
ModelOutputType.Structured
),
middleware: context.getActiveProviderMiddleware(execution),
});
return context.buildPreparedNativeStructuredExecution(
route.protocol,
route.requestLayer === backendConfig.request_layer
? backendConfig
: { ...backendConfig, request_layer: route.requestLayer },
model.id,
request,
execution
);
}
export async function runNativeStructured(
context: StructuredRuntimeContext,
cond: ModelConditions,
messages: PromptMessage[],
options: CopilotStructuredOptions = {},
responseContract?: RequiredStructuredOutputContract,
execution?: CopilotProviderExecution
) {
const driver = resolveDriverOrThrow(
context.type,
'structure',
context.resolveStructuredDriver
);
const structuredOptions = options ?? {};
const prepared = await prepareNativeStructuredExecution(
context,
cond,
messages,
structuredOptions,
responseContract,
execution
);
const modelId = resolvePreparedModelId(
context,
cond,
ModelOutputType.Structured,
prepared
);
return await runPreparedExecution({
driver,
prepared,
modelId,
execution,
metricContext: context,
metricsName: {
call: 'chat_text_calls',
error: 'chat_text_errors',
},
execute: async preparedExecution => {
const dispatch = context.createNativeStructuredDispatch(
preparedExecution.route.backendConfig,
preparedExecution.route.protocol,
execution
);
for (let attempt = 0; ; attempt++) {
try {
const response = await dispatch(preparedExecution.request);
const parsed = parseNativeStructuredOutput(response);
const validated = llmValidateJsonSchema(
preparedExecution.request.schema,
parsed
);
return JSON.stringify(validated);
} catch (error) {
if (
!(await driver.shouldRetry?.({
error,
attempt,
options: structuredOptions,
}))
) {
throw error;
}
}
}
},
});
}
export async function prepareNativeEmbeddingExecution(
context: EmbeddingRuntimeContext,
cond: ModelConditions,
input: string | string[],
options: CopilotEmbeddingOptions = {},
execution?: CopilotProviderExecution
): Promise<PreparedNativeEmbeddingExecution | null> {
const values = Array.isArray(input) ? input : [input];
return await prepareNativeExecutionBase({
resolveDriver: context.resolveEmbeddingDriver,
cond,
outputType: ModelOutputType.Embedding,
checkParams: context.checkParams,
selectModel: context.selectModel,
execution,
checkInput: {
embeddings: values,
options,
},
buildPrepared: ({ driver, model, backendConfig, protocol }) =>
context.buildPreparedNativeEmbeddingExecution(
protocol,
backendConfig,
model.id,
buildLlmEmbeddingRequest({
model: model.id,
inputs: values,
dimensions: options?.dimensions ?? driver.defaultDimensions,
taskType: driver.taskType ?? DEFAULT_EMBEDDING_TASK_TYPE,
}),
execution
),
});
}
export async function runNativeEmbedding(
context: EmbeddingRuntimeContext,
cond: ModelConditions,
input: string | string[],
options?: CopilotEmbeddingOptions,
execution?: CopilotProviderExecution
) {
const driver = resolveDriverOrThrow(
context.type,
ModelOutputType.Embedding,
context.resolveEmbeddingDriver
);
const prepared = await prepareNativeEmbeddingExecution(
context,
cond,
input,
options,
execution
);
const modelId = resolvePreparedModelId(
context,
cond,
ModelOutputType.Embedding,
prepared
);
return await runPreparedExecution({
driver,
prepared,
modelId,
execution,
metricContext: context,
metricsName: {
call: 'generate_embedding_calls',
error: 'generate_embedding_errors',
},
execute: async preparedExecution => {
const response = await context.createNativeEmbeddingDispatch(
preparedExecution.route.backendConfig,
preparedExecution.route.protocol,
execution
)(preparedExecution.request);
return response.embeddings;
},
});
}
export async function prepareNativeRerankExecution(
context: RerankRuntimeContext,
cond: ModelConditions,
request: CopilotRerankRequest,
options: CopilotChatOptions = {},
execution?: CopilotProviderExecution
): Promise<PreparedNativeRerankExecution | null> {
return await prepareNativeExecutionBase({
resolveDriver: context.resolveRerankDriver,
cond,
outputType: ModelOutputType.Rerank,
checkParams: context.checkParams,
selectModel: context.selectModel,
execution,
checkInput: {
messages: [],
options,
},
buildPrepared: ({ model, backendConfig, protocol }) =>
context.buildPreparedNativeRerankExecution(
protocol,
backendConfig,
model.id,
buildLlmRerankRequest(model.id, request),
execution
),
});
}
export async function prepareNativeImageExecution(
context: ImageRuntimeContext,
cond: ModelConditions,
messages: PromptMessage[],
options: CopilotImageOptions = {},
execution?: CopilotProviderExecution
): Promise<PreparedNativeImageExecution | null> {
return await prepareNativeExecutionBase({
resolveDriver: context.resolveImageDriver,
cond,
outputType: ModelOutputType.Image,
checkParams: context.checkParams,
selectModel: context.selectModel,
execution,
checkInput: {
messages,
options,
},
buildPrepared: async ({ driver, model, backendConfig, protocol }) => {
const preparedMessages = driver.prepareMessages
? await driver.prepareMessages(messages, backendConfig, options)
: messages;
return context.buildPreparedNativeImageExecution(
protocol,
backendConfig,
model.id,
preparedMessages,
options,
execution
);
},
});
}
export async function runNativeRerank(
context: RerankRuntimeContext,
cond: ModelConditions,
request: CopilotRerankRequest,
options: CopilotChatOptions = {},
execution?: CopilotProviderExecution
) {
const driver = resolveDriverOrThrow(
context.type,
ModelOutputType.Rerank,
context.resolveRerankDriver
);
const prepared = await prepareNativeRerankExecution(
context,
cond,
request,
options,
execution
);
const modelId = resolvePreparedModelId(
context,
cond,
ModelOutputType.Rerank,
prepared
);
return await runPreparedExecution({
driver,
prepared,
modelId,
execution,
execute: async preparedExecution => {
const response = await context.createNativeRerankDispatch(
preparedExecution.route.backendConfig,
preparedExecution.route.protocol,
execution
)(preparedExecution.request);
return response.scores;
},
});
}
@@ -1,490 +0,0 @@
import type {
LlmBackendConfig,
LlmEmbeddingRequest,
LlmProtocol,
LlmRerankRequest,
LlmStructuredRequest,
LlmStructuredResponse,
} from '../../../native';
import type { ProviderMiddlewareConfig } from '../config';
import type { CopilotProvider } from '../providers/provider';
import type { ProviderModelRuntimeContext } from '../providers/provider-model-runtime';
import {
createNativeEmbeddingDispatch as inputCreateNativeEmbeddingDispatch,
createNativeRerankDispatch as inputCreateNativeRerankDispatch,
createNativeStructuredDispatch as inputCreateNativeStructuredDispatch,
createPreparedExecutionRuntime,
type CreatePreparedExecutionRuntimeInput,
type PreparedExecutionRuntime,
} from '../providers/provider-native-runtime';
import type {
CopilotProviderExecution,
EmbeddingProviderDriver,
ImageProviderDriver,
PreparedNativeEmbeddingExecution,
PreparedNativeExecution,
PreparedNativeImageExecution,
PreparedNativeRequestOptions,
PreparedNativeRerankExecution,
PreparedNativeStructuredExecution,
ProviderExecutionDrivers,
ProviderMetricLabels,
ProviderRuntimeHostSeed,
RerankProviderDriver,
StructuredProviderDriver,
} from '../providers/provider-runtime-contract';
import type {
CopilotChatOptions,
CopilotEmbeddingOptions,
CopilotImageOptions,
CopilotProviderModel,
CopilotRerankRequest,
CopilotStructuredOptions,
ModelAttachmentCapability,
ModelConditions,
ModelFullConditions,
ModelOutputType,
PromptMessage,
} from '../providers/types';
import type { CopilotToolSet } from '../tools';
import type { RequiredStructuredOutputContract } from './contracts';
import type { ChatRuntimeContext } from './provider-chat-runtime';
import {
prepareNativeChatExecution,
runNativeStreamObject,
runNativeStreamText,
runNativeText,
} from './provider-chat-runtime';
import type {
EmbeddingRuntimeContext,
ImageRuntimeContext,
RerankRuntimeContext,
StructuredRuntimeContext,
} from './provider-driver-runtime';
import {
prepareNativeEmbeddingExecution,
prepareNativeImageExecution,
prepareNativeRerankExecution,
prepareNativeStructuredExecution,
runNativeEmbedding,
runNativeRerank,
runNativeStructured,
} from './provider-driver-runtime';
import type { NativeProviderAdapter } from './tool/native-adapter';
type ProviderRuntimeContextInput = {
model: ProviderModelRuntimeContext;
resolveExecutionDrivers: () => ProviderExecutionDrivers | undefined;
selectModel: (
cond: ModelFullConditions,
execution?: CopilotProviderExecution
) => CopilotProviderModel;
metricLabels: (
model: string,
labels?: ProviderMetricLabels,
execution?: CopilotProviderExecution
) => ProviderMetricLabels;
checkParams: (input: {
cond: ModelFullConditions;
messages?: PromptMessage[];
embeddings?: string[];
options?:
| CopilotChatOptions
| CopilotStructuredOptions
| CopilotImageOptions;
withAttachment?: boolean;
execution?: CopilotProviderExecution;
}) => Promise<ModelFullConditions>;
getAttachCapability: (
model: CopilotProviderModel,
outputType: ModelOutputType
) => ModelAttachmentCapability | undefined;
getActiveProviderMiddleware: (
execution?: CopilotProviderExecution
) => ProviderMiddlewareConfig;
getTools: (
options: CopilotChatOptions,
model: string
) => Promise<CopilotToolSet>;
buildPreparedNativeExecution: (
options: PreparedNativeRequestOptions
) => Promise<PreparedNativeExecution>;
createPreparedExecutionAdapter: (
prepared: PreparedNativeExecution
) => NativeProviderAdapter;
buildPreparedNativeStructuredExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmStructuredRequest,
execution?: CopilotProviderExecution
) => PreparedNativeStructuredExecution;
createNativeStructuredDispatch: (
backendConfig: LlmBackendConfig,
protocol: LlmProtocol,
execution?: CopilotProviderExecution
) => (request: LlmStructuredRequest) => Promise<LlmStructuredResponse>;
buildPreparedNativeEmbeddingExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmEmbeddingRequest,
execution?: CopilotProviderExecution
) => PreparedNativeEmbeddingExecution;
createNativeEmbeddingDispatch: (
backendConfig: LlmBackendConfig,
protocol: LlmProtocol,
execution?: CopilotProviderExecution
) => (request: LlmEmbeddingRequest) => Promise<{ embeddings: number[][] }>;
buildPreparedNativeRerankExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
request: LlmRerankRequest,
execution?: CopilotProviderExecution
) => PreparedNativeRerankExecution;
createNativeRerankDispatch: (
backendConfig: LlmBackendConfig,
protocol: LlmProtocol,
execution?: CopilotProviderExecution
) => (request: LlmRerankRequest) => Promise<{ scores: number[] }>;
buildPreparedNativeImageExecution: (
protocol: LlmProtocol,
backendConfig: LlmBackendConfig,
model: string,
messages: PromptMessage[],
options?: CopilotImageOptions,
execution?: CopilotProviderExecution
) => PreparedNativeImageExecution;
};
export type ProviderRuntimeHostInput = Omit<
ProviderRuntimeContextInput,
| keyof ProviderRuntimeHostSeed
| 'buildPreparedNativeExecution'
| 'createPreparedExecutionAdapter'
| 'buildPreparedNativeStructuredExecution'
| 'buildPreparedNativeEmbeddingExecution'
| 'buildPreparedNativeRerankExecution'
| 'buildPreparedNativeImageExecution'
> &
ProviderRuntimeHostSeed & {
preparedExecutionRuntimeInput: CreatePreparedExecutionRuntimeInput;
createNativeStructuredDispatch: ProviderRuntimeContextInput['createNativeStructuredDispatch'];
createNativeEmbeddingDispatch: ProviderRuntimeContextInput['createNativeEmbeddingDispatch'];
createNativeRerankDispatch: ProviderRuntimeContextInput['createNativeRerankDispatch'];
};
type ProviderRuntimeHostOverride = {
overrideRuntimeHost?: (
runtimeHost: ProviderRuntimeContexts
) => ProviderRuntimeContexts;
};
const runtimeHosts = new WeakMap<CopilotProvider, ProviderRuntimeContexts>();
export type ProviderRuntimeContexts = {
model: ProviderModelRuntimeContext;
chat: ChatRuntimeContext;
structured: StructuredRuntimeContext;
embedding: EmbeddingRuntimeContext;
rerank: RerankRuntimeContext;
image: ImageRuntimeContext;
prepare: {
chat: (
kind: 'text' | 'streamText' | 'streamObject',
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof prepareNativeChatExecution>;
structured: (
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotStructuredOptions,
responseContract?: RequiredStructuredOutputContract,
execution?: CopilotProviderExecution
) => ReturnType<typeof prepareNativeStructuredExecution>;
embedding: (
cond: ModelConditions,
input: string | string[],
options?: CopilotEmbeddingOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof prepareNativeEmbeddingExecution>;
rerank: (
cond: ModelConditions,
request: CopilotRerankRequest,
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof prepareNativeRerankExecution>;
image: (
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotImageOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof prepareNativeImageExecution>;
};
run: {
text: (
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof runNativeText>;
streamText: (
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof runNativeStreamText>;
streamObject: (
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof runNativeStreamObject>;
structured: (
cond: ModelConditions,
messages: PromptMessage[],
options?: CopilotStructuredOptions,
responseContract?: RequiredStructuredOutputContract,
execution?: CopilotProviderExecution
) => ReturnType<typeof runNativeStructured>;
embedding: (
cond: ModelConditions,
input: string | string[],
options?: CopilotEmbeddingOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof runNativeEmbedding>;
rerank: (
cond: ModelConditions,
request: CopilotRerankRequest,
options?: CopilotChatOptions,
execution?: CopilotProviderExecution
) => ReturnType<typeof runNativeRerank>;
};
};
function createProviderRuntimeContexts(
input: ProviderRuntimeContextInput
): ProviderRuntimeContexts {
const resolveDriver = <K extends keyof ProviderExecutionDrivers>(
kind: K
): ProviderExecutionDrivers[K] | undefined =>
input.resolveExecutionDrivers()?.[kind];
const chatDriver = resolveDriver('chat');
const chatContext: ChatRuntimeContext = {
type: input.model.type,
resolveChatDriver: () => chatDriver,
selectModel: input.selectModel,
metricLabels: input.metricLabels,
createPreparedExecutionAdapter: input.createPreparedExecutionAdapter,
};
const structuredContext: StructuredRuntimeContext = {
type: input.model.type,
resolveStructuredDriver: () =>
resolveDriver('structured') as StructuredProviderDriver | undefined,
checkParams: input.checkParams,
selectModel: input.selectModel,
getAttachCapability: input.getAttachCapability,
getActiveProviderMiddleware: input.getActiveProviderMiddleware,
buildPreparedNativeStructuredExecution:
input.buildPreparedNativeStructuredExecution,
createNativeStructuredDispatch: input.createNativeStructuredDispatch,
metricLabels: input.metricLabels,
};
const embeddingContext: EmbeddingRuntimeContext = {
type: input.model.type,
resolveEmbeddingDriver: () =>
resolveDriver('embedding') as EmbeddingProviderDriver | undefined,
checkParams: input.checkParams,
selectModel: input.selectModel,
buildPreparedNativeEmbeddingExecution:
input.buildPreparedNativeEmbeddingExecution,
createNativeEmbeddingDispatch: input.createNativeEmbeddingDispatch,
metricLabels: input.metricLabels,
};
const rerankContext: RerankRuntimeContext = {
type: input.model.type,
resolveRerankDriver: () =>
resolveDriver('rerank') as RerankProviderDriver | undefined,
checkParams: input.checkParams,
selectModel: input.selectModel,
buildPreparedNativeRerankExecution:
input.buildPreparedNativeRerankExecution,
createNativeRerankDispatch: input.createNativeRerankDispatch,
};
const imageContext: ImageRuntimeContext = {
type: input.model.type,
resolveImageDriver: () =>
resolveDriver('image') as ImageProviderDriver | undefined,
checkParams: input.checkParams,
selectModel: input.selectModel,
buildPreparedNativeImageExecution: input.buildPreparedNativeImageExecution,
};
const prepare: ProviderRuntimeContexts['prepare'] = {
chat: (kind, cond, messages, options = {}, execution) =>
prepareNativeChatExecution(
chatContext.resolveChatDriver,
input.buildPreparedNativeExecution,
{
kind,
cond,
messages,
options,
execution,
}
),
structured: (cond, messages, options = {}, responseContract, execution) =>
prepareNativeStructuredExecution(
structuredContext,
cond,
messages,
options,
responseContract,
execution
),
embedding: (cond, values, options = {}, execution) =>
prepareNativeEmbeddingExecution(
embeddingContext,
cond,
values,
options,
execution
),
rerank: (cond, request, options = {}, execution) =>
prepareNativeRerankExecution(
rerankContext,
cond,
request,
options,
execution
),
image: (cond, messages, options = {}, execution) =>
prepareNativeImageExecution(
imageContext,
cond,
messages,
options,
execution
),
};
return {
model: input.model,
chat: chatContext,
structured: structuredContext,
embedding: embeddingContext,
rerank: rerankContext,
image: imageContext,
prepare,
run: {
text: (cond, messages, options, execution) =>
runNativeText(
chatContext,
prepare.chat,
cond,
messages,
options,
execution
),
streamText: (cond, messages, options, execution) =>
runNativeStreamText(
chatContext,
prepare.chat,
cond,
messages,
options,
execution
),
streamObject: (cond, messages, options, execution) =>
runNativeStreamObject(
chatContext,
prepare.chat,
cond,
messages,
options,
execution
),
structured: (cond, messages, options, responseContract, execution) =>
runNativeStructured(
structuredContext,
cond,
messages,
options,
responseContract,
execution
),
embedding: (cond, values, options, execution) =>
runNativeEmbedding(embeddingContext, cond, values, options, execution),
rerank: (cond, request, options, execution) =>
runNativeRerank(rerankContext, cond, request, options, execution),
},
};
}
export function createProviderRuntimeHost(
input: ProviderRuntimeHostInput
): ProviderRuntimeContexts {
const preparedExecutionRuntime: PreparedExecutionRuntime =
createPreparedExecutionRuntime(input.preparedExecutionRuntimeInput);
return createProviderRuntimeContexts({
...input,
buildPreparedNativeExecution:
preparedExecutionRuntime.buildPreparedNativeExecution,
createPreparedExecutionAdapter:
preparedExecutionRuntime.createPreparedExecutionAdapter,
buildPreparedNativeStructuredExecution:
preparedExecutionRuntime.buildPreparedNativeStructuredExecution,
buildPreparedNativeEmbeddingExecution:
preparedExecutionRuntime.buildPreparedNativeEmbeddingExecution,
buildPreparedNativeRerankExecution:
preparedExecutionRuntime.buildPreparedNativeRerankExecution,
buildPreparedNativeImageExecution:
preparedExecutionRuntime.buildPreparedNativeImageExecution,
createNativeStructuredDispatch: input.createNativeStructuredDispatch,
createNativeEmbeddingDispatch: input.createNativeEmbeddingDispatch,
createNativeRerankDispatch: input.createNativeRerankDispatch,
});
}
export function getProviderRuntimeHost(
provider: CopilotProvider
): ProviderRuntimeContexts {
const existingRuntimeHost = runtimeHosts.get(provider);
if (existingRuntimeHost) {
return existingRuntimeHost;
}
const runtimeHostSeed = provider.getRuntimeHostSeed();
const runtimeHost = createProviderRuntimeHost({
...runtimeHostSeed,
preparedExecutionRuntimeInput: {
resolveProviderId: execution =>
execution?.providerId ?? `${provider.type}-default`,
getTools: runtimeHostSeed.getTools,
getActiveProviderMiddleware: runtimeHostSeed.getActiveProviderMiddleware,
createNativeAdapter: provider.createNativeAdapter.bind(provider),
maxSteps: provider.maxSteps,
},
createNativeStructuredDispatch: (backendConfig, protocol, _execution) =>
inputCreateNativeStructuredDispatch(backendConfig, protocol),
createNativeEmbeddingDispatch: (backendConfig, protocol, _execution) =>
inputCreateNativeEmbeddingDispatch(backendConfig, protocol),
createNativeRerankDispatch: (backendConfig, protocol, _execution) =>
inputCreateNativeRerankDispatch(backendConfig, protocol),
});
const resolvedRuntimeHost =
(provider as ProviderRuntimeHostOverride).overrideRuntimeHost?.(
runtimeHost
) ?? runtimeHost;
runtimeHosts.set(provider, resolvedRuntimeHost);
return resolvedRuntimeHost;
}
@@ -1,35 +0,0 @@
import { Injectable } from '@nestjs/common';
import { QuotaStateService } from '../../../core/quota/state';
import { PromptService } from '../prompt/service';
export const DEFAULT_EMBEDDING_MODEL = 'gemini-embedding-001';
export const DEFAULT_RERANK_MODEL = 'gpt-4o-mini';
@Injectable()
export class TaskPolicy {
constructor(
private readonly quotaState: QuotaStateService,
private readonly prompts: PromptService
) {}
resolveEmbeddingModelId() {
return DEFAULT_EMBEDDING_MODEL;
}
resolveRerankModelId() {
return DEFAULT_RERANK_MODEL;
}
async resolveTranscriptionModel(userId: string) {
const prompt = await this.prompts.get('Transcript audio');
if (!prompt) return;
const state = await this.quotaState.reconcileUserQuotaState(userId);
const flags = state.flags as { unlimitedCopilot?: boolean };
const hasAccess =
!!flags.unlimitedCopilot ||
['pro', 'lifetime_pro', 'ai'].includes(state.plan);
return prompt.optionalModels[hasAccess ? 1 : 0] ?? prompt.model;
}
}
@@ -1,3 +1,4 @@
/* oxlint-disable import/no-cycle -- Tools can invoke nested prompts and semantic search. */
import { Injectable } from '@nestjs/common';
import { Config } from '../../../base';
@@ -5,7 +6,6 @@ import { DocReader, DocWriter } from '../../../core/doc';
import { PermissionAccess } from '../../../core/permission';
import { Models } from '../../../models';
import { IndexerService } from '../../indexer';
import type { NodeTextMiddleware } from '../config';
import { CopilotContextService } from '../context/service';
import {
type CopilotChatOptions,
@@ -36,8 +36,6 @@ import {
createSectionEditTool,
} from '../tools';
import { PromptRuntime } from './prompt-runtime';
import type { ToolLoopBackend } from './tool/bridge';
import { createNativeToolLoopAdapter } from './tool/native-adapter';
export type ProviderSpecificToolResolver = (
toolName: CopilotChatTools,
@@ -192,15 +190,4 @@ export class ToolRuntime {
return tools;
}
createNativeAdapter(
backend: ToolLoopBackend,
tools: CopilotToolSet,
options: {
maxSteps?: number;
nodeTextMiddleware?: NodeTextMiddleware[];
} = {}
) {
return createNativeToolLoopAdapter(backend, tools, options);
}
}
@@ -1,17 +1,8 @@
import { z } from 'zod';
import {
type LlmBackendConfig,
llmDispatchToolLoopStream,
llmDispatchToolLoopStreamPrepared,
llmDispatchToolLoopStreamRouted,
type LlmPreparedDispatchRoute,
type LlmProtocol,
type LlmRequest,
type LlmRoutedBackend,
type LlmToolCallbackRequest,
type LlmToolCallbackResponse,
type LlmToolLoopStreamEvent,
} from '../../../../native';
import type {
CopilotTool,
@@ -19,71 +10,11 @@ import type {
CopilotToolSet,
} from '../../tools';
export type ToolLoopDispatch = (
request: LlmRequest,
signalOrOptions?: AbortSignal | CopilotToolExecuteOptions,
maybeMessages?: CopilotToolExecuteOptions['messages']
) => AsyncIterableIterator<LlmToolLoopStreamEvent>;
export type ToolLoopBackend =
| { protocol: LlmProtocol; backendConfig: LlmBackendConfig }
| { routes: LlmRoutedBackend[] }
| { preparedRoutes: LlmPreparedDispatchRoute[] };
function normalizeToolExecuteOptions(
signalOrOptions?: AbortSignal | CopilotToolExecuteOptions,
maybeMessages?: CopilotToolExecuteOptions['messages']
): CopilotToolExecuteOptions {
if (
signalOrOptions &&
typeof signalOrOptions === 'object' &&
'aborted' in signalOrOptions
) {
return {
signal: signalOrOptions,
messages: maybeMessages,
};
}
if (!signalOrOptions) {
return maybeMessages ? { messages: maybeMessages } : {};
}
return {
...signalOrOptions,
signal: signalOrOptions.signal,
messages: signalOrOptions.messages ?? maybeMessages,
};
}
export function createToolExecutionCallback(
tools: CopilotToolSet,
options: CopilotToolExecuteOptions = {}
) {
return async (request: LlmToolCallbackRequest) => {
return await executeToolCall(tools, request, options);
};
}
export async function executeToolCall(
tools: CopilotToolSet,
request: LlmToolCallbackRequest,
options: CopilotToolExecuteOptions
): Promise<LlmToolCallbackResponse> {
const tool = tools[request.name] as CopilotTool | undefined;
if (!tool?.execute) {
return {
callId: request.callId,
name: request.name,
args: request.args,
rawArgumentsText: request.rawArgumentsText,
argumentParseError: request.argumentParseError,
isError: true,
output: { message: `Tool not found: ${request.name}` },
};
}
if (request.argumentParseError) {
return {
callId: request.callId,
@@ -104,6 +35,19 @@ export async function executeToolCall(
};
}
const tool = tools[request.name] as CopilotTool | undefined;
if (!tool?.execute) {
return {
callId: request.callId,
name: request.name,
args: request.args,
rawArgumentsText: request.rawArgumentsText,
argumentParseError: request.argumentParseError,
isError: true,
output: { message: `Tool not found: ${request.name}` },
};
}
try {
const args =
tool.inputSchema instanceof z.ZodType
@@ -133,53 +77,5 @@ export async function executeToolCall(
}
}
export function createToolLoopBridge(
backend: ToolLoopBackend,
tools: CopilotToolSet,
maxSteps = 20
): ToolLoopDispatch {
return (
request: LlmRequest,
signalOrOptions?: AbortSignal | CopilotToolExecuteOptions,
maybeMessages?: CopilotToolExecuteOptions['messages']
) => {
const toolExecuteOptions = normalizeToolExecuteOptions(
signalOrOptions,
maybeMessages
);
const execute = createToolExecutionCallback(tools, toolExecuteOptions);
const toolLoopRequest = { ...request, stream: true };
if ('routes' in backend) {
return llmDispatchToolLoopStreamRouted(
backend.routes,
toolLoopRequest,
execute,
maxSteps,
toolExecuteOptions.signal
);
}
if ('preparedRoutes' in backend) {
return llmDispatchToolLoopStreamPrepared(
backend.preparedRoutes,
execute,
maxSteps,
toolExecuteOptions.signal
);
}
return llmDispatchToolLoopStream(
backend.protocol,
backend.backendConfig,
toolLoopRequest,
execute,
maxSteps,
toolExecuteOptions.signal
);
};
}
// re-export for test consumers
export type { LlmToolCallbackRequest } from '../../../../native';
export type { CopilotToolExecuteOptions, CopilotToolSet } from '../../tools';
@@ -7,9 +7,7 @@ import {
CitationFootnoteFormatter,
TextStreamParser,
} from '../../providers/utils';
import type { CopilotToolSet } from '../../tools';
import { projectRuntimeEventToStreamObject } from '../contracts/runtime-event-contract';
import { createToolLoopBridge, type ToolLoopBackend } from './bridge';
import {
type EnrichedToolCallEvent,
type EnrichedToolResultEvent,
@@ -425,14 +423,3 @@ export class NativeProviderAdapter {
}
}
}
export function createNativeToolLoopAdapter(
backend: ToolLoopBackend,
tools: CopilotToolSet,
options: NativeProviderAdapterOptions = {}
) {
return new NativeProviderAdapter(
createToolLoopBridge(backend, tools, options.maxSteps),
options
);
}
@@ -3,14 +3,15 @@ import { Injectable } from '@nestjs/common';
import { CopilotContextService } from '../context/service';
import { type Turn } from '../core';
import {
type ModelConditions,
ModelInputType,
type PromptParams,
type StreamObject,
} from '../providers/types';
import { ChatSession } from '../session';
import { ChatQuerySchema } from '../types';
import { getTools } from '../utils';
import { CapabilityRuntime } from './capability-runtime';
import { CapabilityPolicyHost } from './hosts/capability-policy-host';
import { ConversationHost } from './hosts/conversation-host';
import { ImageResultHost } from './hosts/image-result-host';
import { TurnPersistence } from './hosts/turn-persistence';
@@ -20,7 +21,6 @@ export class TurnOrchestrator {
constructor(
private readonly conversations: ConversationHost,
private readonly context: CopilotContextService,
private readonly capabilityPolicy: CapabilityPolicyHost,
private readonly runtime: CapabilityRuntime,
private readonly imageResults: ImageResultHost,
private readonly turnPersistence: TurnPersistence
@@ -62,8 +62,15 @@ export class TurnOrchestrator {
sessionId,
query
);
const { modelId, reasoning, webSearch, toolsConfig, byokLeaseId } =
ChatQuerySchema.parse(query);
const {
profileId,
modelId,
routeTargetId,
reasoning,
webSearch,
toolsConfig,
byokLeaseId,
} = ChatQuerySchema.parse(query);
const promptParams = await this.buildPromptParams(sessionId, {
latestTurn: prepared.latestTurn,
includeContextFiles: selection.includeContextFiles,
@@ -76,22 +83,34 @@ export class TurnOrchestrator {
return {
prepared,
finalMessage,
selection: await this.capabilityPolicy.selectChat(prepared.session, {
responseMode: selection.responseMode,
modelId,
reasoning,
webSearch,
toolsConfig,
byokLeaseId,
billingUnitId: prepared.latestTurn?.id,
quotaBackedRoutesAllowed: prepared.quotaBackedRoutesAllowed,
featureKind:
selection.responseMode === 'image'
? 'image'
: selection.responseMode === 'object'
? 'action'
: 'chat',
}),
selection: {
model: profileId && modelId ? modelId : 'route-selected',
conditions: { profileId, modelId },
providerOptions: {
...prepared.session.config.promptConfig,
user: prepared.session.config.userId,
session: prepared.session.config.sessionId,
workspace: prepared.session.config.workspaceId,
profileId,
byokLeaseId,
billingUnitId: prepared.latestTurn?.id,
builtInRouteId: prepared.session.config.promptName,
managedTargetId: routeTargetId,
quotaBackedRoutesAllowed: prepared.quotaBackedRoutesAllowed,
featureKind:
selection.responseMode === 'image'
? 'image'
: selection.responseMode === 'object'
? 'action'
: 'chat',
reasoning,
webSearch,
tools: getTools(
prepared.session.config.promptConfig?.tools,
toolsConfig
),
},
},
};
}
@@ -110,7 +129,7 @@ export class TurnOrchestrator {
const stream = this.streamTextResult(
prepared.session,
selection.model,
selection.conditions,
finalMessage,
{
...selection.providerOptions,
@@ -129,14 +148,14 @@ export class TurnOrchestrator {
private async *streamTextResult(
session: ChatSession,
model: string,
conditions: ModelConditions,
finalMessage: ReturnType<ChatSession['finish']>,
options: Record<string, unknown>,
wasAborted: () => boolean
) {
let buffer = '';
for await (const chunk of this.runtime.streamText(
{ modelId: model },
conditions,
finalMessage,
options
)) {
@@ -165,7 +184,7 @@ export class TurnOrchestrator {
finalMessage,
stream: this.streamObjectResult(
prepared.session,
selection.model,
selection.conditions,
finalMessage,
{
...selection.providerOptions,
@@ -178,14 +197,14 @@ export class TurnOrchestrator {
private async *streamObjectResult(
session: ChatSession,
model: string,
conditions: ModelConditions,
finalMessage: ReturnType<ChatSession['finish']>,
options: Record<string, unknown>,
wasAborted: () => boolean
): AsyncIterableIterator<StreamObject> {
const chunks: StreamObject[] = [];
for await (const chunk of this.runtime.streamObject(
{ modelId: model },
conditions,
finalMessage,
options
)) {
@@ -223,7 +242,7 @@ export class TurnOrchestrator {
userId,
sessionId,
prepared.session,
undefined,
selection.conditions,
hasAttachment,
finalMessage,
{
@@ -244,7 +263,7 @@ export class TurnOrchestrator {
userId: string,
sessionId: string,
session: ChatSession,
model: string | undefined,
conditions: ModelConditions,
hasAttachment: boolean,
finalMessage: ReturnType<ChatSession['finish']>,
options: Record<string, unknown>,
@@ -253,7 +272,7 @@ export class TurnOrchestrator {
const attachments: string[] = [];
for await (const artifact of this.runtime.streamImageArtifacts(
{
modelId: model,
...conditions,
inputTypes: hasAttachment
? [ModelInputType.Image]
: [ModelInputType.Text],
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
import { Injectable, Logger } from '@nestjs/common';
import { Transactional } from '@nestjs-cls/transactional';
import { AiPromptRole } from '@prisma/client';
import { AiSessionMessageRole } from '@prisma/client';
import {
CopilotActionTaken,
@@ -20,7 +20,6 @@ import {
type UpdateChatSession,
UpdateChatSessionOptions,
} from '../../models';
import { CopilotAccessPolicy } from './access';
import { ConversationPolicy } from './conversation/policy';
import { ConversationStore } from './conversation/store';
import { type Conversation, promptMessageFromTurn, type Turn } from './core';
@@ -50,7 +49,6 @@ export class ChatSession implements AsyncDisposable {
prompt: ResolvedPrompt,
turns: PromptMessage[],
params: PromptParams,
maxTokenSize: number,
sessionId?: string
) => PromptMessage[];
constructor(
@@ -59,23 +57,13 @@ export class ChatSession implements AsyncDisposable {
prompt: ResolvedPrompt,
turns: PromptMessage[],
params: PromptParams,
maxTokenSize: number,
sessionId?: string
) => PromptMessage[],
private readonly dispose?: (state: ChatSessionState) => Promise<void>,
private readonly maxTokenSize = state.prompt.config?.maxTokens || 128 * 1024
private readonly dispose?: (state: ChatSessionState) => Promise<void>
) {
this.renderPromptSession = renderPromptSession;
}
get model() {
return this.state.prompt.model;
}
get optionalModels() {
return this.state.prompt.optionalModels;
}
get config() {
const {
sessionId,
@@ -126,7 +114,7 @@ export class ChatSession implements AsyncDisposable {
revertLatestMessage(removeLatestUserMessage: boolean) {
const turns = this.state.turns;
turns.splice(
turns.findLastIndex(({ role }) => role === AiPromptRole.user) +
turns.findLastIndex(({ role }) => role === AiSessionMessageRole.user) +
(removeLatestUserMessage ? 0 : 1)
);
}
@@ -136,7 +124,6 @@ export class ChatSession implements AsyncDisposable {
this.state.prompt,
this.state.turns.map(turn => promptMessageFromTurn(turn)),
params,
this.maxTokenSize,
this.state.sessionId
);
}
@@ -158,13 +145,11 @@ export type ConversationState = {
conversation: Conversation;
turns: Turn[];
prompt: ResolvedPrompt;
tokenCost: number;
};
export type ConversationMetaState = {
conversation: Conversation;
prompt: ResolvedPrompt;
tokenCost: number;
};
type StoredConversation = NonNullable<
@@ -183,7 +168,6 @@ export class ChatSessionService {
private readonly models: Models,
private readonly jobs: JobQueue,
private readonly store: ConversationStore,
private readonly access: CopilotAccessPolicy,
private readonly conversationPolicy: ConversationPolicy,
private readonly prompts: PromptService,
private readonly promptRuntime: PromptRuntime
@@ -206,14 +190,13 @@ export class ChatSessionService {
private async toConversationState(
session: StoredConversation
): Promise<ConversationState> {
const { conversation, prompt, tokenCost } =
const { conversation, prompt } =
await this.toConversationMetaState(session);
return {
conversation,
turns: session.turns,
prompt,
tokenCost,
};
}
@@ -226,7 +209,6 @@ export class ChatSessionService {
return {
conversation: session.conversation,
prompt,
tokenCost: session.tokenCost,
};
}
@@ -296,11 +278,11 @@ export class ChatSessionService {
}
async getQuota(userId: string) {
return await this.access.getQuota(userId);
return await this.conversationPolicy.getQuota(userId);
}
async checkQuota(userId: string) {
await this.access.checkQuota(userId);
await this.conversationPolicy.checkQuota(userId);
}
async create(options: ChatSessionOptions): Promise<string> {
@@ -360,7 +342,6 @@ export class ChatSessionService {
);
finalData.promptName = prompt.name;
finalData.promptAction = prompt.action ?? null;
finalData.promptModel = prompt.model;
}
finalData.pinned = options.pinned;
finalData.docId = options.docId;
@@ -389,7 +370,8 @@ export class ChatSessionService {
if (options.latestMessageId) {
const lastMessageIdx = state.turns.findLastIndex(
({ id, role }) =>
role === AiPromptRole.assistant && id === options.latestMessageId
role === AiSessionMessageRole.assistant &&
id === options.latestMessageId
);
if (lastMessageIdx < 0) {
throw new CopilotMessageNotFound({
@@ -410,7 +392,6 @@ export class ChatSessionService {
prompt: {
name: state.prompt.name,
action: state.prompt.action,
model: state.prompt.model,
},
turns,
});
@@ -434,7 +415,6 @@ export class ChatSessionService {
async appendTurn(input: {
sessionId: string;
userId: string;
prompt: { model: string };
turn: Turn;
compatSubmissionId?: string;
}) {
@@ -485,14 +465,8 @@ export class ChatSessionService {
turns: state.turns,
prompt: state.prompt,
},
(prompt, turns, params, maxTokenSize, sessionId) =>
this.prompts.renderSession(
prompt,
turns,
params,
maxTokenSize,
sessionId
),
(prompt, turns, params, sessionId) =>
this.prompts.renderSession(prompt, turns, params, sessionId),
async state => {
await this.store.appendTurns(state);
if (this.conversationPolicy.shouldScheduleTitle(state.prompt)) {
@@ -538,9 +512,18 @@ export class ChatSessionService {
const promptContent =
this.conversationPolicy.buildTitlePromptContent(turns);
const generatedTitle = this.stripNullBytes(
await this.promptRuntime.runText('Summary as title', {
content: promptContent,
})
await this.promptRuntime.runText(
'Summary as title',
{ content: promptContent },
{
providerOptions: {
user: conversation.userId,
workspace: conversation.workspaceId,
featureKind: 'chat',
quotaBackedRoutesAllowed: true,
},
}
)
).trim();
if (!generatedTitle) {
@@ -1,3 +1,4 @@
/* oxlint-disable import/no-cycle -- Semantic search uses the shared embedding runtime. */
import { omit } from 'lodash-es';
import { z } from 'zod';
@@ -1,3 +1,4 @@
/* oxlint-disable import/no-cycle -- Tool exports include semantic search runtime dependencies. */
export * from './blob-read';
export * from './code-artifact';
export * from './conversation-summary';
@@ -1,13 +1,15 @@
import { Injectable, OnModuleInit } from '@nestjs/common';
import { z } from 'zod';
import { CopilotTranscriptionJobNotFound } from '../../../base';
import { Config } from '../../../base/config';
import { CopilotTranscriptionJobNotFound } from '../../../base/error/errors.gen';
import { PermissionAccess } from '../../../core/permission';
import {
RealtimeRegistry,
realtimeTranscriptTaskRoom,
registerRealtimeLiveQuery,
} from '../../../core/realtime';
import { assertCopilotEnabled } from '../availability';
import { CopilotTranscriptionReader } from './reader';
@Injectable()
@@ -15,7 +17,8 @@ export class CopilotTranscriptRealtimeProvider implements OnModuleInit {
constructor(
private readonly ac: PermissionAccess,
private readonly transcript: CopilotTranscriptionReader,
private readonly registry: RealtimeRegistry
private readonly registry: RealtimeRegistry,
private readonly config: Config
) {}
onModuleInit() {
@@ -68,6 +71,7 @@ export class CopilotTranscriptRealtimeProvider implements OnModuleInit {
}
private async assertCopilot(userId: string, workspaceId: string) {
assertCopilotEnabled(this.config);
await this.ac
.user(userId)
.workspace(workspaceId)
@@ -22,6 +22,7 @@ import {
} from '../../../base';
import { CurrentUser } from '../../../core/auth';
import { PermissionAccess } from '../../../core/permission';
import { CopilotEnabled } from '../feature';
import { CopilotType } from '../resolver';
import type { TranscriptionJob } from './job';
import { buildLegacyProjection } from './projection';
@@ -36,7 +37,6 @@ import type {
TranscriptionQuality,
TranscriptionSourceAudio,
TranscriptionSubmitInput,
TranscriptProviderMeta,
} from './types';
registerEnumType(AiJobStatus, {
@@ -166,15 +166,6 @@ class TranscriptionQualityType implements TranscriptionQuality {
overflowCount!: number | null;
}
@ObjectType()
class TranscriptProviderMetaType implements TranscriptProviderMeta {
@Field(() => String, { nullable: true })
provider!: string | null;
@Field(() => String, { nullable: true })
model!: string | null;
}
@InputType()
class AudioSliceManifestItemInput implements AudioSliceManifestItem {
@Field(() => Int)
@@ -233,9 +224,6 @@ class SubmitAudioTranscriptionInput implements TranscriptionSubmitInput {
@Field(() => [AudioSliceManifestItemInput], { nullable: true })
sliceManifest?: AudioSliceManifestItemInput[];
@Field(() => String, { nullable: true })
strategy?: string | null;
}
@ObjectType()
@@ -273,15 +261,9 @@ class TranscriptionResultType {
@Field(() => MeetingSummaryV2Type, { nullable: true })
summaryJson!: TranscriptionPayload['summaryJson'] | null;
@Field(() => TranscriptProviderMetaType, { nullable: true })
providerMeta!: TranscriptionPayload['providerMeta'] | null;
@Field(() => String, { nullable: true })
version!: string | null;
@Field(() => String, { nullable: true })
strategy!: string | null;
@Field(() => AiJobStatus)
status!: AiJobStatus;
}
@@ -292,6 +274,7 @@ const FinishedStatus: Set<AiJobStatus> = new Set([
]);
@Injectable()
@CopilotEnabled()
@Resolver(() => CopilotType)
export class CopilotTranscriptionResolver {
constructor(
@@ -318,9 +301,7 @@ export class CopilotTranscriptionResolver {
normalizedSegments: null,
normalizedTranscript: null,
summaryJson: null,
providerMeta: null,
version: null,
strategy: null,
};
if (FinishedStatus.has(finalJob.status)) {
finalJob.title = legacy?.title ?? null;
@@ -333,9 +314,7 @@ export class CopilotTranscriptionResolver {
finalJob.normalizedSegments = ret?.normalizedSegments ?? null;
finalJob.normalizedTranscript = ret?.normalizedTranscript ?? null;
finalJob.summaryJson = ret?.summaryJson ?? null;
finalJob.providerMeta = ret?.providerMeta ?? null;
finalJob.version = ret?.version ?? null;
finalJob.strategy = ret?.strategy ?? null;
}
return finalJob;
}
@@ -73,11 +73,6 @@ export const TranscriptionQualitySchema = z.object({
overflowCount: z.number().nullable().optional(),
});
export const TranscriptProviderMetaSchema = z.object({
provider: z.string().nullable().optional(),
model: z.string().nullable().optional(),
});
export const TranscriptionLegacyProjectionSchema = z.object({
title: z.string().nullable().optional(),
summary: z.string().nullable().optional(),
@@ -96,9 +91,7 @@ export const TranscriptionPayloadV2Schema = z.object({
.optional(),
normalizedTranscript: z.string().nullable().optional(),
summaryJson: MeetingSummaryV2Schema.nullable().optional(),
providerMeta: TranscriptProviderMetaSchema.nullable().optional(),
version: z.string().optional(),
strategy: z.string().optional(),
});
export const TranscriptionSubmitInputSchema = TranscriptionPayloadV2Schema.pick(
@@ -135,9 +128,7 @@ const CanonicalTranscriptPayloadSchema = TranscriptionPayloadV2Schema.refine(
payload.normalizedSegments !== undefined ||
payload.normalizedTranscript !== undefined ||
payload.summaryJson !== undefined ||
payload.providerMeta !== undefined ||
payload.version !== undefined ||
payload.strategy !== undefined,
payload.version !== undefined,
{
message:
'canonical transcript payload must contain canonical transcript fields',
@@ -14,11 +14,9 @@ import {
realtimeTranscriptTaskRoom,
} from '../../../core/realtime';
import { Models } from '../../../models';
import { CopilotAccessPolicy } from '../access';
import { PromptService } from '../prompt';
import { CopilotProviderType } from '../providers/types';
import { ActionRuntimeBridge } from '../runtime/action-runtime-bridge';
import { TaskPolicy } from '../runtime/task-policy';
import { CapabilityRuntime } from '../runtime/capability-runtime';
import { CopilotStorage } from '../storage';
import { taskToJob, type TranscriptionJob } from './job';
import {
@@ -32,9 +30,9 @@ import type {
} from './types';
import { readStream } from './utils';
const TRANSCRIPT_ACTION_ID = 'transcript.audio.gemini';
const TRANSCRIPT_ACTION_ID = 'transcript.audio';
const TRANSCRIPT_PROMPT_REF = 'Transcript audio structured';
const TRANSCRIPT_ACTION_VERSION = 'v1';
const TRANSCRIPT_STRATEGY = 'gemini';
@Injectable()
export class CopilotTranscriptionService {
@@ -42,10 +40,9 @@ export class CopilotTranscriptionService {
private readonly models: Models,
private readonly job: JobQueue,
private readonly storage: CopilotStorage,
private readonly tasks: TaskPolicy,
private readonly prompts: PromptService,
private readonly actionBridge: ActionRuntimeBridge,
private readonly access: CopilotAccessPolicy,
private readonly runtime: CapabilityRuntime,
private readonly realtime: RealtimePublisher
) {}
@@ -58,27 +55,10 @@ export class CopilotTranscriptionService {
sourceAudio: payload.sourceAudio,
quality: payload.quality,
sliceManifest: payload.sliceManifest,
providerMeta: payload.providerMeta,
version: 'transcript-result-v1',
strategy: TRANSCRIPT_STRATEGY,
};
}
private async resolveTranscriptStrategy(userId: string, strategy?: string) {
if (strategy && strategy !== TRANSCRIPT_STRATEGY) {
throw new BadRequestException(
`Transcript strategy ${strategy} is not available`
);
}
const model = await this.tasks.resolveTranscriptionModel(userId);
if (!model) {
throw new BadRequestException(
'Transcript strategy gemini is not available'
);
}
return { model, strategy: TRANSCRIPT_STRATEGY };
}
private async persistUploads(
userId: string,
workspaceId: string,
@@ -123,11 +103,8 @@ export class CopilotTranscriptionService {
} satisfies TranscriptionPayloadV2;
}
private async buildTranscriptActionMessages(
payload: TranscriptionPayloadV2,
modelId?: string
) {
const prompt = await this.prompts.get('Transcript audio structured');
private async buildTranscriptActionMessages(payload: TranscriptionPayloadV2) {
const prompt = await this.prompts.get(TRANSCRIPT_PROMPT_REF);
if (!prompt) {
throw new Error('Transcript action prompt not found');
}
@@ -140,10 +117,6 @@ export class CopilotTranscriptionService {
mimeType: info.mimeType,
index: info.index ?? null,
})) ?? null,
providerMeta: {
provider: CopilotProviderType.Gemini,
model: modelId ?? payload.providerMeta?.model ?? null,
},
};
const attachments =
payload.infos?.map(info => ({
@@ -165,7 +138,7 @@ export class CopilotTranscriptionService {
workspaceId: string,
blobId: string,
blobs: FileUpload[],
input?: TranscriptionSubmitInput & { strategy?: string | null }
input?: TranscriptionSubmitInput
): Promise<TranscriptionJob> {
const existingTask = await this.models.copilotTranscriptTask.getWithUser(
userId,
@@ -180,15 +153,15 @@ export class CopilotTranscriptionService {
throw new CopilotTranscriptionJobExists();
}
await this.access.assertQuotaOrByok({
userId,
workspaceId,
featureKind: 'transcript',
});
const { model, strategy } = await this.resolveTranscriptStrategy(
userId,
input?.strategy ?? undefined
await this.runtime.assertRoute(
'transcript.audio',
{},
{
user: userId,
workspace: workspaceId,
featureKind: 'transcript',
builtInRouteId: TRANSCRIPT_PROMPT_REF,
}
);
const infos = await this.persistUploads(userId, workspaceId, blobId, blobs);
const payload = this.createCanonicalPayload(blobId, infos, input);
@@ -196,7 +169,6 @@ export class CopilotTranscriptionService {
userId,
workspaceId,
blobId,
strategy,
recipeId: TRANSCRIPT_ACTION_ID,
recipeVersion: TRANSCRIPT_ACTION_VERSION,
inputSnapshot: payload,
@@ -206,7 +178,6 @@ export class CopilotTranscriptionService {
await this.job.add('copilot.transcript.task.submit', {
taskId: task.id,
payload,
modelId: model,
});
await this.models.copilotTranscriptTask.markRunning(task.id);
this.publishTaskChanged(workspaceId, task.id, AiJobStatus.running);
@@ -234,21 +205,20 @@ export class CopilotTranscriptionService {
);
}
await this.access.assertQuotaOrByok({
userId,
workspaceId,
featureKind: 'transcript',
});
const payload = this.parseTaskPayload(task.protectedResult);
const { model } = await this.resolveTranscriptStrategy(
userId,
task.strategy
await this.runtime.assertRoute(
'transcript.audio',
{},
{
user: userId,
workspace: workspaceId,
featureKind: 'transcript',
builtInRouteId: TRANSCRIPT_PROMPT_REF,
}
);
await this.job.add('copilot.transcript.task.submit', {
taskId,
payload,
modelId: model,
retryOf: task.actionRunId ?? undefined,
});
await this.models.copilotTranscriptTask.markRunning(taskId);
@@ -282,12 +252,6 @@ export class CopilotTranscriptionService {
return taskToJob(task);
}
await this.access.assertQuotaOrByok({
userId,
workspaceId,
featureKind: 'transcript',
});
const settled = await this.models.copilotTranscriptTask.settle(task.id);
return taskToJob(settled);
}
@@ -311,7 +275,6 @@ export class CopilotTranscriptionService {
async transcriptTask({
taskId,
payload,
modelId,
retryOf,
}: Jobs['copilot.transcript.task.submit']) {
const task = await this.models.copilotTranscriptTask.get(taskId);
@@ -324,10 +287,7 @@ export class CopilotTranscriptionService {
let bridgeFailed = false;
let bridgeError = 'transcript native recipe failed';
let finalResult: unknown = null;
const messages = await this.buildTranscriptActionMessages(
payload,
modelId
);
const messages = await this.buildTranscriptActionMessages(payload);
for await (const event of this.actionBridge.runStream({
userId: task.userId,
workspaceId: task.workspaceId,
@@ -335,14 +295,6 @@ export class CopilotTranscriptionService {
actionVersion: TRANSCRIPT_ACTION_VERSION,
retryOf: retryOf ?? null,
inputSnapshot: payload,
nativeInput: {
input: {
sourceAudio: payload.sourceAudio ?? null,
quality: payload.quality ?? null,
infos: payload.infos ?? null,
sliceManifest: payload.sliceManifest ?? null,
},
},
onRunCreated: async ({ runId }) => {
await this.models.copilotTranscriptTask.markRunning(taskId, runId);
this.publishTaskChanged(
@@ -351,9 +303,9 @@ export class CopilotTranscriptionService {
AiJobStatus.running
);
},
prepareStructuredRoutes: {
stepId: 'transcribe',
modelId,
step: {
slot: 'transcript.audio',
builtInRouteId: TRANSCRIPT_PROMPT_REF,
messages,
options: {
user: task.userId,
@@ -362,7 +314,6 @@ export class CopilotTranscriptionService {
billingUnitId: taskId,
featureKind: 'transcript',
},
prefer: CopilotProviderType.Gemini,
responseContract: TranscriptActionResultContract,
},
})) {
@@ -15,7 +15,6 @@ import {
TranscriptionQualitySchema,
TranscriptionSourceAudioSchema,
TranscriptionSubmitInputSchema,
TranscriptProviderMetaSchema,
} from './schema';
export type LegacyTranscriptionSegment = z.infer<
@@ -36,9 +35,6 @@ export type TranscriptionSourceAudio = z.infer<
typeof TranscriptionSourceAudioSchema
>;
export type TranscriptionQuality = z.infer<typeof TranscriptionQualitySchema>;
export type TranscriptProviderMeta = z.infer<
typeof TranscriptProviderMetaSchema
>;
export type TranscriptionLegacyProjection = z.infer<
typeof TranscriptionLegacyProjectionSchema
>;
@@ -57,7 +53,6 @@ declare global {
'copilot.transcript.task.submit': {
taskId: string;
payload: TranscriptionPayloadV2;
modelId?: string;
retryOf?: string;
};
}
@@ -36,7 +36,9 @@ export type ToolsConfig = z.infer<typeof ToolsConfigSchema>;
export const ChatQuerySchema = z
.object({
messageId: zMaybeString,
profileId: zMaybeString,
modelId: zMaybeString,
routeTargetId: zMaybeString,
byokLeaseId: zMaybeString,
retry: zBool,
reasoning: zBool,
@@ -44,10 +46,29 @@ export const ChatQuerySchema = z
toolsConfig: ToolsConfigSchema,
})
.catchall(z.string())
.superRefine((value, context) => {
if (!!value.profileId !== !!value.modelId) {
context.addIssue({
code: 'custom',
message: 'profileId and modelId must be provided together',
});
}
for (const field of ['requirements', 'deployment', 'profiles', 'presets']) {
if (Object.hasOwn(value, field)) {
context.addIssue({
code: 'custom',
path: [field],
message: `${field} is owned by the native route policy`,
});
}
}
})
.transform(
({
messageId,
profileId,
modelId,
routeTargetId,
byokLeaseId,
retry,
reasoning,
@@ -56,7 +77,9 @@ export const ChatQuerySchema = z
...params
}) => ({
messageId,
profileId,
modelId,
routeTargetId,
byokLeaseId,
retry,
reasoning,
@@ -85,11 +108,7 @@ export const ChatHistorySchema = z
title: z.string().nullable(),
action: z.string().nullable(),
model: z.string(),
optionalModels: z.array(z.string()),
promptName: z.string(),
tokens: z.number(),
messages: z.array(ChatMessageSchema),
createdAt: z.date(),
updatedAt: z.date(),
@@ -26,6 +26,7 @@ import {
import { CurrentUser } from '../../../core/auth';
import { PermissionAccess } from '../../../core/permission';
import { WorkspaceType } from '../../../core/workspaces';
import { CopilotEnabled } from '../feature';
import { COPILOT_LOCKER } from '../resolver';
import { MAX_EMBEDDABLE_SIZE } from '../utils';
import { CopilotWorkspaceService } from './service';
@@ -47,6 +48,7 @@ export class CopilotWorkspaceConfigType {
* Public apis rate limit: 10 req/m
* Other rate limit: 120 req/m
*/
@CopilotEnabled()
@Resolver(() => WorkspaceType)
export class CopilotWorkspaceEmbeddingResolver {
constructor(private readonly ac: PermissionAccess) {}
@@ -67,6 +69,7 @@ export class CopilotWorkspaceEmbeddingResolver {
}
}
@CopilotEnabled()
@Resolver(() => CopilotWorkspaceConfigType)
export class CopilotWorkspaceEmbeddingConfigResolver {
constructor(