fix(server): index generation error handle (#15532)

<!--
Thank you for contributing to AFFiNE!

The PR title must follow Conventional Commits (enforced by CI):
type(scope): description e.g. fix(editor): keep selection after paste
Types: feat fix docs style refactor perf test build ci chore revert
-->

## Description

<!-- What does this PR do? Link related issues, e.g. "Closes #1234".
Screenshots or recordings are welcome for UI changes. -->

## Checklist

- [ ] I have signed the [AFFiNE Contributor License
Agreement](https://cla-assistant.io/toeverything/AFFiNE) — required
before merge; the `license/cla` check must be green ([how it
works](https://github.com/toeverything/AFFiNE/blob/canary/docs/BUILDING.md#sign-the-cla-first))
- [ ] The PR targets the `canary` branch and its title follows
[Conventional Commits](https://www.conventionalcommits.org/)
- [ ] Tests are added or updated where it makes sense
- [ ] `yarn lint` and `yarn typecheck` pass locally



#### PR Dependency Tree


* **PR #15532** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Bug Fixes**
* Improved search reconciliation and publication status handling when
workspace reconciliation fails.
* Prevented failed workspace reconciliation from incorrectly blocking
generation completion.
* Preserved active generation state so reconciliation can retry and
complete pending publications.
* Improved managed provider profile migration, including legacy
configurations, unavailable models, conflicting assignments, and missing
defaults.

* **Tests**
* Expanded coverage for workspace recovery and managed provider profile
migration scenarios.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-08-26 19:57:26 +08:00
committed by GitHub
parent 88585a2024
commit c91b1810bd
9 changed files with 251 additions and 87 deletions
@@ -3,7 +3,7 @@ use sha2::{Digest, Sha256};
use sqlx::{PgPool, Row};
use uuid::Uuid;
use super::{SCHEMA_FINGERPRINT, SearchProvider, SearchTable};
use super::{SCHEMA_FINGERPRINT, SearchProvider, SearchTable, WORKSPACE_RECONCILE_FAILED};
use crate::{
runtime::{RuntimeError, RuntimeResult, SearchRuntimeConfig},
search_index::EmbeddedSearchIndex,
@@ -311,6 +311,7 @@ pub(super) async fn activate(
SELECT 1
FROM search_projection.workspace_states state
WHERE state.generation_id=generation.id
AND state.last_error IS DISTINCT FROM $2
AND (
NOT state.covered
OR state.required_permission_version > state.applied_permission_version
@@ -331,6 +332,12 @@ pub(super) async fn activate(
NOT EXISTS (
SELECT 1 FROM search_projection.document_states state
WHERE state.generation_id=generation.id
AND NOT EXISTS (
SELECT 1 FROM search_projection.workspace_states workspace
WHERE workspace.generation_id=state.generation_id
AND workspace.workspace_id=state.workspace_id
AND workspace.last_error=$2
)
AND (state.target_source_version <> state.published_source_version
OR state.target_source_exists <> state.published_source_exists
OR state.target_permission_version <> state.published_permission_version)
@@ -340,6 +347,7 @@ pub(super) async fn activate(
FOR UPDATE"#,
)
.bind(generation.id)
.bind(WORKSPACE_RECONCILE_FAILED)
.fetch_optional(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("load search generation activation state", error))?;
@@ -24,6 +24,7 @@ use super::{
};
const SCHEMA_FINGERPRINT: i32 = 1;
const WORKSPACE_RECONCILE_FAILED: &str = "search_workspace_reconcile_failed";
#[cfg(test)]
pub(crate) static SEARCH_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
@@ -8,8 +8,8 @@ use tokio::{
use uuid::Uuid;
use super::{
ActiveGeneration, PermissionAuthorizer, SearchProvider, activate, cleanup_retired_generation, config_hash, ensure,
load_active, reconcile_workspace, sweep_generation_orphans,
ActiveGeneration, PermissionAuthorizer, SearchProvider, WORKSPACE_RECONCILE_FAILED, activate,
cleanup_retired_generation, config_hash, ensure, load_active, reconcile_workspace, sweep_generation_orphans,
};
use crate::{
runtime::{RuntimeError, RuntimeResult, SearchRuntimeConfig},
@@ -212,12 +212,16 @@ impl SearchRuntime {
.await
.map_err(|error| RuntimeError::database("count search workspaces", error))?;
let pending: bool = sqlx::query_scalar(
"SELECT EXISTS (SELECT 1 FROM search_projection.workspace_states WHERE generation_id=$1 AND (NOT covered OR \
pending_scope <> 'none')) OR EXISTS (SELECT 1 FROM search_projection.document_states WHERE generation_id=$1 \
AND (target_source_version <> published_source_version OR target_source_exists <> published_source_exists OR \
target_permission_version <> published_permission_version))",
"SELECT EXISTS (SELECT 1 FROM search_projection.workspace_states WHERE generation_id=$1 AND last_error IS \
DISTINCT FROM $2 AND (NOT covered OR pending_scope <> 'none')) OR EXISTS (SELECT 1 FROM \
search_projection.document_states document WHERE generation_id=$1 AND NOT EXISTS (SELECT 1 FROM \
search_projection.workspace_states workspace WHERE workspace.generation_id=document.generation_id AND \
workspace.workspace_id=document.workspace_id AND workspace.last_error=$2) AND (target_source_version <> \
published_source_version OR target_source_exists <> published_source_exists OR target_permission_version <> \
published_permission_version))",
)
.bind(generation.id)
.bind(WORKSPACE_RECONCILE_FAILED)
.fetch_one(&self.pool)
.await
.map_err(|error| RuntimeError::database("check pending search projection", error))?;
@@ -318,6 +322,12 @@ impl SearchRuntime {
r#"SELECT id,provider,state,scan_cursor_sid,scan_high_water_sid,
(SELECT count(*) FROM search_projection.document_states document
WHERE document.generation_id=generations.id
AND NOT EXISTS (
SELECT 1 FROM search_projection.workspace_states workspace
WHERE workspace.generation_id=document.generation_id
AND workspace.workspace_id=document.workspace_id
AND workspace.last_error=$4
)
AND (document.target_source_version <> document.published_source_version
OR document.target_source_exists <> document.published_source_exists
OR document.target_permission_version <> document.published_permission_version)) AS pending_publications,
@@ -332,6 +342,7 @@ impl SearchRuntime {
.bind(&self.config.provider)
.bind(config_hash(&self.config))
.bind(super::SCHEMA_FINGERPRINT)
.bind(WORKSPACE_RECONCILE_FAILED)
.fetch_optional(&self.pool)
.await
.map_err(|error| RuntimeError::database("load search projection status", error))?;
@@ -36,6 +36,6 @@ use workspace_state::{
};
use super::{
ActiveGeneration, ProjectionInput, SearchChange, SearchProvider, SearchTable, project_document,
projection_external_id, provider_payload,
ActiveGeneration, ProjectionInput, SearchChange, SearchProvider, SearchTable, WORKSPACE_RECONCILE_FAILED,
project_document, projection_external_id, provider_payload,
};
@@ -3,7 +3,7 @@ use serde_json::Value;
use sqlx::{PgPool, Row};
use uuid::Uuid;
use super::{ActiveGeneration, LEASE_SECONDS, SearchTable};
use super::{ActiveGeneration, LEASE_SECONDS, SearchTable, WORKSPACE_RECONCILE_FAILED};
use crate::runtime::{RuntimeError, RuntimeResult};
const ANTI_ENTROPY_INTERVAL_SECONDS: i64 = 3600;
@@ -350,29 +350,9 @@ pub(super) async fn mark_workspace_failed(
.begin()
.await
.map_err(|error| RuntimeError::database("begin failed search workspace update", error))?;
let generation_state: Option<String> = sqlx::query_scalar(
r#"SELECT generation.state
FROM search_projection.workspace_states state
JOIN search_projection.generations generation ON generation.id=state.generation_id
WHERE state.generation_id=$1 AND state.workspace_id=$2 AND state.claim_fence=$3
FOR UPDATE"#,
)
.bind(generation_id)
.bind(workspace_id)
.bind(fence)
.fetch_optional(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("load failed search workspace claim", error))?;
let Some(generation_state) = generation_state else {
transaction
.commit()
.await
.map_err(|error| RuntimeError::database("commit empty search workspace update", error))?;
return Ok(());
};
sqlx::query(
let updated = sqlx::query(
r#"UPDATE search_projection.workspace_states
SET covered=false, pending_scope='workspace', last_error='search_workspace_reconcile_failed',
SET covered=false, pending_scope='workspace', last_error=$4,
progress=NULL, available_at='infinity'::timestamptz,
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE generation_id=$1 AND workspace_id=$2 AND claim_fence=$3"#,
@@ -380,9 +360,17 @@ pub(super) async fn mark_workspace_failed(
.bind(generation_id)
.bind(workspace_id)
.bind(fence)
.bind(WORKSPACE_RECONCILE_FAILED)
.execute(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("mark search workspace failed", error))?;
if updated.rows_affected() == 0 {
transaction
.commit()
.await
.map_err(|error| RuntimeError::database("commit empty search workspace update", error))?;
return Ok(());
}
sqlx::query(
r#"UPDATE search_projection.document_states
SET available_at='infinity'::timestamptz, lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
@@ -393,17 +381,6 @@ pub(super) async fn mark_workspace_failed(
.execute(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("pause failed search document publications", error))?;
if generation_state != "active" {
sqlx::query(
r#"UPDATE search_projection.generations
SET state='failed', last_error='search_workspace_reconcile_failed'
WHERE id=$1 AND state='building'"#,
)
.bind(generation_id)
.execute(&mut *transaction)
.await
.map_err(|error| RuntimeError::database("mark search generation failed", error))?;
}
transaction
.commit()
.await
@@ -416,7 +393,11 @@ mod tests {
use serde_json::json;
use super::*;
use crate::runtime::{backend_runtime::search::SEARCH_TEST_LOCK, migrations::migrate_search_tables};
use crate::runtime::{
SearchRuntimeConfig,
backend_runtime::search::{SEARCH_TEST_LOCK, SearchRuntime, config_hash},
migrations::migrate_search_tables,
};
#[test]
fn progress_round_trips_versioned_context_for_each_phase() {
@@ -465,6 +446,7 @@ mod tests {
migrate_search_tables(&pool).await.unwrap();
let generation_id = Uuid::new_v4();
let workspace_id = format!("claim-workspace-{}", Uuid::new_v4().simple());
let config = SearchRuntimeConfig::default();
sqlx::query("INSERT INTO workspaces(id) VALUES($1)")
.bind(&workspace_id)
.execute(&pool)
@@ -485,10 +467,12 @@ mod tests {
.await
.unwrap();
sqlx::query(
r#"INSERT INTO search_projection.generations(id,provider,state,config_hash,schema_version)
VALUES($1,'embedded','failed',decode(repeat('00',32),'hex'),1)"#,
r#"INSERT INTO search_projection.generations(
id,provider,state,config_hash,schema_version,scan_high_water_sid,scan_cursor_sid
) VALUES($1,'embedded','building',$2,1,0,0)"#,
)
.bind(generation_id)
.bind(config_hash(&config))
.execute(&pool)
.await
.unwrap();
@@ -536,14 +520,9 @@ mod tests {
checkpoint_workspace(&pool, generation_id, &workspace_id, first.fence, checkpoint)
.await
.unwrap();
sqlx::query("UPDATE snapshots SET updated_at='2026-01-02 UTC' WHERE workspace_id=$1 AND guid=$1")
.bind(&workspace_id)
.execute(&pool)
.await
.unwrap();
sqlx::query(
"UPDATE search_projection.workspace_states SET required_permission_version=4 WHERE generation_id=$1 AND \
workspace_id=$2",
"UPDATE search_projection.workspace_states SET target_root_revision=target_root_revision+1, \
required_permission_version=4 WHERE generation_id=$1 AND workspace_id=$2",
)
.bind(generation_id)
.bind(&workspace_id)
@@ -557,12 +536,25 @@ mod tests {
.unwrap();
assert_eq!(second.progress.captured_root_revision, initial_root);
assert_eq!(second.progress.captured_permission_version, 3);
sqlx::query(
r#"INSERT INTO search_projection.document_states(
generation_id,workspace_id,doc_id,target_source_version,target_source_exists
) VALUES($1,$2,'pending-doc',1,true)"#,
)
.bind(generation_id)
.bind(&workspace_id)
.execute(&pool)
.await
.unwrap();
mark_workspace_failed(&pool, generation_id, &workspace_id, second.fence)
.await
.unwrap();
let failed: (bool, Option<String>, Option<Value>, bool) = sqlx::query_as(
r#"SELECT covered,last_error,progress,available_at='infinity'::timestamptz
FROM search_projection.workspace_states WHERE generation_id=$1 AND workspace_id=$2"#,
let failed: (bool, Option<String>, Option<Value>, bool, String) = sqlx::query_as(
r#"SELECT state.covered,state.last_error,state.progress,state.available_at='infinity'::timestamptz,
generation.state
FROM search_projection.workspace_states state
JOIN search_projection.generations generation ON generation.id=state.generation_id
WHERE state.generation_id=$1 AND state.workspace_id=$2"#,
)
.bind(generation_id)
.bind(&workspace_id)
@@ -571,9 +563,28 @@ mod tests {
.unwrap();
assert_eq!(
failed,
(false, Some("search_workspace_reconcile_failed".to_string()), None, true)
(
false,
Some(WORKSPACE_RECONCILE_FAILED.to_string()),
None,
true,
"building".to_string()
)
);
let runtime = SearchRuntime::new(pool.clone(), config).unwrap();
runtime.embedded.prepare_generation(generation_id).await;
assert_eq!(runtime.reconcile_pending(1).await.unwrap(), 0);
assert_eq!(
sqlx::query_scalar::<_, String>("SELECT state FROM search_projection.generations WHERE id=$1")
.bind(generation_id)
.fetch_one(&pool)
.await
.unwrap(),
"active"
);
assert_eq!(runtime.status().await.unwrap()["metrics"]["pendingPublications"], 0);
sqlx::query("DELETE FROM search_projection.generations WHERE id=$1")
.bind(generation_id)
.execute(&pool)
@@ -293,6 +293,7 @@ mod tests {
assert!(!SEARCH_PROJECTION_MIGRATION.contains("clock_timestamp()"));
assert!(!SEARCH_PROJECTION_MIGRATION.contains("CREATE TABLE search_runtime_projections"));
assert!(!SEARCH_PROJECTION_MIGRATION.contains("payload JSONB NOT NULL"));
assert!(SEARCH_PROJECTION_MIGRATION.contains("search_workspace_reconcile_failed"));
}
#[tokio::test]
@@ -266,16 +266,23 @@ RETURNS boolean LANGUAGE SQL STABLE AS $$
AND NOT EXISTS (
SELECT 1 FROM search_projection.workspace_states
WHERE generation_id = target_generation
AND last_error IS DISTINCT FROM 'search_workspace_reconcile_failed'
AND (NOT covered OR pending_scope <> 'none'
OR required_permission_version > applied_permission_version
OR last_error IS NOT NULL)
)
AND NOT EXISTS (
SELECT 1 FROM search_projection.document_states
WHERE generation_id = target_generation
AND (target_source_version <> published_source_version
OR target_source_exists <> published_source_exists
OR target_permission_version <> published_permission_version)
SELECT 1 FROM search_projection.document_states document
WHERE document.generation_id = target_generation
AND NOT EXISTS (
SELECT 1 FROM search_projection.workspace_states workspace
WHERE workspace.generation_id = document.generation_id
AND workspace.workspace_id = document.workspace_id
AND workspace.last_error = 'search_workspace_reconcile_failed'
)
AND (document.target_source_version <> document.published_source_version
OR document.target_source_exists <> document.published_source_exists
OR document.target_permission_version <> document.published_permission_version)
)
$$;
@@ -143,6 +143,24 @@ test('managed provider migration preserves explicit profiles and converts legacy
models: ['@cf/baai/bge-reranker-base'],
config: { apiKey: 'profile-key' },
},
{
id: 'fal-default',
type: 'fal',
priority: 5,
config: { apiKey: 'existing-fal-key' },
},
{
id: 'anthropic-default',
type: 'anthropic',
priority: 2,
config: { apiKey: 'existing-anthropic-key' },
},
{
id: 'anthropicVertex-default',
type: 'anthropicVertex',
priority: 1,
config: { projectId: 'existing-anthropic-vertex-project' },
},
];
await t.context.db.appConfig.createMany({
data: [
@@ -151,6 +169,10 @@ test('managed provider migration preserves explicit profiles and converts legacy
id: 'copilot.providers.openai',
value: { apiKey: 'openai-key' },
},
{
id: 'copilot.providers.cloudflareWorkersAi',
value: { apiKey: 'legacy-cloudflare-key' },
},
{
id: 'copilot.providers.gemini',
value: { apiKey: 'gemini-key' },
@@ -159,6 +181,18 @@ test('managed provider migration preserves explicit profiles and converts legacy
id: 'copilot.providers.geminiVertex',
value: { projectId: 'gemini-vertex-project' },
},
{
id: 'copilot.providers.fal',
value: { apiKey: 'legacy-fal-key' },
},
{
id: 'copilot.providers.anthropic',
value: { apiKey: 'legacy-anthropic-key' },
},
{
id: 'copilot.providers.anthropicVertex',
value: { projectId: 'legacy-anthropic-vertex-project' },
},
{
id: 'copilot.providers.defaults',
value: { fallback: 'openai-default' },
@@ -173,7 +207,20 @@ test('managed provider migration preserves explicit profiles and converts legacy
where: { id: 'copilot.providers.profiles' },
});
t.deepEqual(migrated.value, [
...profiles,
profiles[0],
{
...profiles[1],
models: ['lora/image-to-image', 'workflowutils/teed'],
},
{
...profiles[2],
models: ['claude-sonnet-4-6'],
},
{
...profiles[3],
models: ['claude-sonnet-4-6'],
enabled: false,
},
{
id: 'openai-default',
type: 'openai',
@@ -181,6 +228,14 @@ test('managed provider migration preserves explicit profiles and converts legacy
models: ['gpt-5.6-luna', 'gpt-5.6-terra', 'gpt-image-1', 'gpt-4o-mini'],
config: { apiKey: 'openai-key' },
},
{
id: 'cloudflareWorkersAi-default',
type: 'cloudflareWorkersAi',
priority: 6,
models: ['@cf/baai/bge-reranker-base'],
config: { apiKey: 'legacy-cloudflare-key' },
enabled: false,
},
{
id: 'gemini-default',
type: 'gemini',
@@ -203,8 +258,12 @@ test('managed provider migration preserves explicit profiles and converts legacy
id: {
in: [
'copilot.providers.openai',
'copilot.providers.cloudflareWorkersAi',
'copilot.providers.gemini',
'copilot.providers.geminiVertex',
'copilot.providers.fal',
'copilot.providers.anthropic',
'copilot.providers.anthropicVertex',
],
},
},
@@ -216,6 +275,38 @@ test('managed provider migration preserves explicit profiles and converts legacy
where: { id: 'copilot.providers.defaults' },
})
);
await t.context.db.appConfig.delete({
where: { id: 'copilot.providers.defaults' },
});
const defaultOnlyProfiles = profiles.slice(1);
await t.context.db.appConfig.update({
where: { id: 'copilot.providers.profiles' },
data: { value: defaultOnlyProfiles },
});
await ConvergeManagedProviderProfiles1786810000000.up(t.context.db);
t.deepEqual(
(
await t.context.db.appConfig.findUniqueOrThrow({
where: { id: 'copilot.providers.profiles' },
})
).value,
[
{
...defaultOnlyProfiles[0],
models: ['lora/image-to-image', 'workflowutils/teed'],
},
{
...defaultOnlyProfiles[1],
models: ['claude-sonnet-4-6'],
},
{
...defaultOnlyProfiles[2],
models: ['claude-sonnet-4-6'],
enabled: false,
},
]
);
await t.context.db.appConfig.update({
where: { id: 'copilot.providers.profiles' },
@@ -13,6 +13,9 @@ const PROVIDERS = [
] as const;
const PROVIDER_IDS = PROVIDERS.map(provider => `copilot.providers.${provider}`);
const DEFAULT_PROFILE_IDS = new Set(
PROVIDERS.map(provider => `${provider}-default`)
);
const PROVIDER_MODELS: Record<(typeof PROVIDERS)[number], string[]> = {
openai: ['gpt-5.6-luna', 'gpt-5.6-terra', 'gpt-image-1', 'gpt-4o-mini'],
cloudflareWorkersAi: ['@cf/baai/bge-reranker-base'],
@@ -60,16 +63,11 @@ export class ConvergeManagedProviderProfiles1786810000000 {
const byId = new Map(rows.map(row => [row.id, row]));
const profileRow = byId.get(PROFILE_KEY);
const profiles = readProfiles(profileRow?.value);
const profileIds = new Set(
profiles.flatMap(profile =>
isRecord(profile) && typeof profile.id === 'string'
? [profile.id]
: []
)
);
const assignedModels = new Set(
profiles.flatMap(profile =>
isRecord(profile) &&
typeof profile.id === 'string' &&
!DEFAULT_PROFILE_IDS.has(profile.id) &&
profile.enabled !== false &&
Array.isArray(profile.models)
? profile.models.filter(
@@ -78,33 +76,69 @@ export class ConvergeManagedProviderProfiles1786810000000 {
: []
)
);
let converged = false;
for (const [index, provider] of PROVIDERS.entries()) {
const legacy = byId.get(`copilot.providers.${provider}`);
if (!legacy) continue;
if (!isRecord(legacy.value)) {
if (legacy && !isRecord(legacy.value)) {
throw new Error(`copilot.providers.${provider} must be an object`);
}
const id = `${provider}-default`;
if (!profileIds.has(id)) {
const models = PROVIDER_MODELS[provider].filter(
model => !assignedModels.has(model)
);
const enabled = models.length > 0;
profiles.push({
id,
type: provider,
priority: PROVIDERS.length - index,
models: enabled ? models : PROVIDER_MODELS[provider],
config: legacy.value,
...(enabled ? {} : { enabled: false }),
const profileIndex = profiles.findIndex(
profile => isRecord(profile) && profile.id === id
);
const existing =
profileIndex === -1 ? undefined : profiles[profileIndex];
if (!legacy && !existing) continue;
converged = true;
const configuredModels =
isRecord(existing) &&
Array.isArray(existing.models) &&
existing.models.length > 0
? existing.models
: PROVIDER_MODELS[provider];
const canEnable =
!isRecord(existing) ||
existing.enabled === undefined ||
existing.enabled === true;
const modelsAreValid = configuredModels.every(
model => typeof model === 'string'
);
const availableModels =
canEnable && modelsAreValid
? configuredModels.filter(model => !assignedModels.has(model))
: configuredModels;
const hasConflict = canEnable && availableModels.length === 0;
const models = hasConflict ? configuredModels : availableModels;
const profile: Prisma.JsonObject = {
...(legacy
? {
id,
type: provider,
priority: PROVIDERS.length - index,
config: legacy.value,
}
: {}),
...(isRecord(existing) ? existing : {}),
id,
type: provider,
models,
...(hasConflict ? { enabled: false } : {}),
};
if (profileIndex === -1) {
profiles.push(profile);
} else {
profiles[profileIndex] = profile;
}
if (canEnable && !hasConflict) {
models.forEach(model => {
if (typeof model === 'string') assignedModels.add(model);
});
models.forEach(model => assignedModels.add(model));
profileIds.add(id);
}
}
if (PROVIDER_IDS.some(id => byId.has(id))) {
if (converged) {
validateProfiles(profiles);
await tx.appConfig.upsert({
where: { id: PROFILE_KEY },