feat(server): faster reranking based on confidence (#12957)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Improved document reranking with a more streamlined and accurate
scoring system.
* Enhanced support for binary ("yes"/"no") document relevance judgments.

* **Improvements**
* Simplified user prompts and output formats for reranking tasks, making
results easier to interpret.
  * Increased reliability and consistency in document ranking results.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -440,6 +440,60 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
     }
   }
 
+  override async rerank(
+    cond: ModelConditions,
+    chunkMessages: PromptMessage[][],
+    options: CopilotChatOptions = {}
+  ): Promise<number[]> {
+    const fullCond = { ...cond, outputType: ModelOutputType.Text };
+    await this.checkParams({ messages: [], cond: fullCond, options });
+    const model = this.selectModel(fullCond);
+    const instance = this.#instance.responses(model.id);
+
+    const scores = await Promise.all(
+      chunkMessages.map(async messages => {
+        const [system, msgs] = await chatToGPTMessage(messages);
+
+        const { logprobs } = await generateText({
+          model: instance,
+          system,
+          messages: msgs,
+          temperature: 0,
+          maxTokens: 1,
+          providerOptions: {
+            openai: {
+              ...this.getOpenAIOptions(options, model.id),
+              // get the log probability of "yes"/"no"
+              logprobs: 2,
+            },
+          },
+          maxSteps: 1,
+          abortSignal: options.signal,
+        });
+
+        const top = (logprobs?.[0]?.topLogprobs ?? []).reduce(
+          (acc, item) => {
+            acc[item.token] = item.logprob;
+            return acc;
+          },
+          {} as Record<string, number>
+        );
+
+        // OpenAI often includes a leading space, so try matching both ' yes' and 'yes'
+        const logYes = top[' yes'] ?? top['yes'] ?? Number.NEGATIVE_INFINITY;
+        const logNo = top[' no'] ?? top['no'] ?? Number.NEGATIVE_INFINITY;
+
+        const pYes = Math.exp(logYes);
+        const pNo = Math.exp(logNo);
+        const prob = pYes + pNo === 0 ? 0 : pYes / (pYes + pNo);
+
+        return prob;
+      })
+    );
+
+    return scores;
+  }
+
   private async getFullStream(
     model: CopilotProviderModel,
     messages: PromptMessage[],

packages/backend/server/src/plugins/copilot/providers/provider.ts

@@ -295,4 +295,15 @@ export abstract class CopilotProvider<C = any> {
       kind: 'embedding',
     });
   }
+
+  async rerank(
+    _model: ModelConditions,
+    _messages: PromptMessage[][],
+    _options?: CopilotChatOptions
+  ): Promise<number[]> {
+    throw new CopilotProviderNotSupported({
+      provider: this.type,
+      kind: 'rerank',
+    });
+  }
 }