fix: copilot not working (#3425)

apps/core/src/bootstrap/register-plugins.ts

@@ -2,6 +2,7 @@
 import 'ses';
 
 import * as AFFiNEComponent from '@affine/component';
+import { FormatQuickBar } from '@blocksuite/blocks';
 import * as BlockSuiteBlocksStd from '@blocksuite/blocks/std';
 import { DisposableGroup } from '@blocksuite/global/utils';
 import * as BlockSuiteGlobalUtils from '@blocksuite/global/utils';
@@ -12,6 +13,7 @@ import {
   headerItemsAtom,
   registeredPluginAtom,
   rootStore,
+  settingItemsAtom,
   windowItemsAtom,
 } from '@toeverything/plugin-infra/atom';
 import type {
@@ -96,8 +98,28 @@ const createGlobalThis = () => {
     document,
     navigator,
     userAgent: navigator.userAgent,
-    // todo: permission control
+    // todo(himself65): permission control
-    fetch: globalThis.fetch,
+    fetch: function (input: RequestInfo, init?: RequestInit) {
+      return globalThis.fetch(input, init);
+    },
+    setTimeout: function (callback: () => void, timeout: number) {
+      return globalThis.setTimeout(callback, timeout);
+    },
+    clearTimeout: function (id: number) {
+      return globalThis.clearTimeout(id);
+    },
+    // copilot uses these
+    crypto: globalThis.crypto,
+    CustomEvent: globalThis.CustomEvent,
+    Date: globalThis.Date,
+    Math: globalThis.Math,
+    URL: globalThis.URL,
+    URLSearchParams: globalThis.URLSearchParams,
+    Headers: globalThis.Headers,
+    TextEncoder: globalThis.TextEncoder,
+    TextDecoder: globalThis.TextDecoder,
+    Request: globalThis.Request,
+    Error: globalThis.Error,
 
     // fixme: use our own db api
     indexedDB: globalThis.indexedDB,
@@ -177,6 +199,20 @@ await Promise.all(
               ...items,
               [plugin]: callback as CallbackMap['window'],
             }));
+          } else if (part === 'setting') {
+            console.log('setting');
+            rootStore.set(settingItemsAtom, items => ({
+              ...items,
+              [plugin]: callback as CallbackMap['setting'],
+            }));
+          } else if (part === 'formatBar') {
+            console.log('1');
+            FormatQuickBar.customElements.push((page, getBlockRange) => {
+              console.log('2');
+              const div = document.createElement('div');
+              (callback as CallbackMap['formatBar'])(div, page, getBlockRange);
+              return div;
+            });
           } else {
             throw new Error(`Unknown part: ${part}`);
           }

apps/core/src/components/affine/setting-modal/general-setting/plugins/index.tsx

@@ -1,10 +1,38 @@
-import {
+import { SettingHeader } from '@affine/component/setting-components';
-  SettingHeader,
-  SettingWrapper,
-} from '@affine/component/setting-components';
 import { useAFFiNEI18N } from '@affine/i18n/hooks';
-import { registeredPluginAtom } from '@toeverything/plugin-infra/atom';
+import {
+  registeredPluginAtom,
+  settingItemsAtom,
+} from '@toeverything/plugin-infra/atom';
 import { useAtomValue } from 'jotai';
+import type { FC, ReactNode } from 'react';
+import { useRef } from 'react';
+
+const PluginSettingWrapper: FC<{
+  id: string;
+  title?: ReactNode;
+}> = ({ title, id }) => {
+  const Setting = useAtomValue(settingItemsAtom)[id];
+  const disposeRef = useRef<(() => void) | null>(null);
+  return (
+    <div>
+      {title ? <div className="title">{title}</div> : null}
+      <div
+        ref={ref => {
+          if (ref && Setting) {
+            setTimeout(() => {
+              disposeRef.current = Setting(ref);
+            });
+          } else if (ref === null) {
+            setTimeout(() => {
+              disposeRef.current?.();
+            });
+          }
+        }}
+      />
+    </div>
+  );
+};
 
 export const Plugins = () => {
   const t = useAFFiNEI18N();
@@ -17,7 +45,7 @@ export const Plugins = () => {
         data-testid="plugins-title"
       />
       {allowedPlugins.map(plugin => (
-        <SettingWrapper key={plugin} title={plugin}></SettingWrapper>
+        <PluginSettingWrapper key={plugin} id={plugin} title={plugin} />
       ))}
     </>
   );

packages/cli/src/bin/dev-plugin.ts

@@ -167,7 +167,7 @@ await build({
         if (!existsSync(outDir)) {
           await mkdir(outDir, { recursive: true });
         }
-        const file = await open(pluginListJsonPath, 'w+', 0o777);
+        const file = await open(pluginListJsonPath, 'as+', 0o777);
         const txt = await file.readFile({
           encoding: 'utf-8',
         });

packages/plugin-infra/src/atom.ts

@@ -16,6 +16,12 @@ export const headerItemsAtom = atom<Record<string, CallbackMap['headerItem']>>(
 export const editorItemsAtom = atom<Record<string, CallbackMap['editor']>>({});
 export const registeredPluginAtom = atom<string[]>([]);
 export const windowItemsAtom = atom<Record<string, CallbackMap['window']>>({});
+export const settingItemsAtom = atom<Record<string, CallbackMap['setting']>>(
+  {}
+);
+export const formatBarItemsAtom = atom<
+  Record<string, CallbackMap['formatBar']>
+>({});
 
 export const currentWorkspaceIdAtom = atom<string | null>(null);
 export const currentPageIdAtom = atom<string | null>(null);

packages/plugin-infra/src/entry.ts

@@ -1,12 +1,20 @@
+import type { getCurrentBlockRange } from '@blocksuite/blocks';
 import type { EditorContainer } from '@blocksuite/editor';
+import type { Page } from '@blocksuite/store';
 import type { FC } from 'react';
 
-export type Part = 'headerItem' | 'editor' | 'window';
+export type Part = 'headerItem' | 'editor' | 'window' | 'setting' | 'formatBar';
 
 export type CallbackMap = {
   headerItem: (root: HTMLElement) => () => void;
   window: (root: HTMLElement) => () => void;
   editor: (root: HTMLElement, editor: EditorContainer) => () => void;
+  setting: (root: HTMLElement) => () => void;
+  formatBar: (
+    root: HTMLElement,
+    page: Page,
+    getBlockRange: () => ReturnType<typeof getCurrentBlockRange>
+  ) => () => void;
 };
 
 export interface PluginContext {

plugins/copilot/package.json

@@ -11,7 +11,7 @@
     "@affine/component": "workspace:*",
     "@toeverything/plugin-infra": "workspace:*",
     "idb": "^7.1.1",
-    "langchain": "^0.0.107",
+    "langchain": "^0.0.118",
     "marked": "^5.1.1",
     "marked-gfm-heading-id": "^3.0.4",
     "marked-mangle": "^1.1.0",

plugins/copilot/src/UI/debug-content.tsx

@@ -1,6 +1,4 @@
 import { Button, FlexWrapper, Input } from '@affine/component';
-import { SettingRow } from '@affine/component/setting-components';
-import { SettingWrapper } from '@affine/component/setting-components';
 import { useAtom } from 'jotai';
 import { type ReactElement, useCallback } from 'react';
 
@@ -9,43 +7,28 @@ import { conversationHistoryDBName } from '../core/langchain/message-history';
 
 export const DebugContent = (): ReactElement => {
   const [key, setKey] = useAtom(openAIApiKeyAtom);
-  const desc = (
-    <>
-      <span>You can get your API key from </span>
-      <a
-        target="_blank"
-        rel="noreferrer"
-        href="https://beta.openai.com/account/api-keys"
-      >
-        here.
-      </a>
-    </>
-  );
   return (
     <div>
-      <SettingWrapper title={'Ai Copilot'}>
+      <FlexWrapper justifyContent="space-between">
-        <SettingRow name={'openAI API key'} desc={desc}></SettingRow>
+        <Input
-        <FlexWrapper justifyContent="space-between">
+          defaultValue={key ?? undefined}
-          <Input
+          onChange={useCallback(
-            defaultValue={key ?? undefined}
+            (newValue: string) => {
-            onChange={useCallback(
+              setKey(newValue);
-              (newValue: string) => {
+            },
-                setKey(newValue);
+            [setKey]
-              },
+          )}
-              [setKey]
+        />
-            )}
+        <Button
-          />
+          size="large"
-          <Button
+          onClick={() => {
-            size="large"
+            indexedDB.deleteDatabase(conversationHistoryDBName);
-            onClick={() => {
+            location.reload();
-              indexedDB.deleteDatabase(conversationHistoryDBName);
+          }}
-              location.reload();
+        >
-            }}
+          {'Clean conversations'}
-          >
+        </Button>
-            {'Clean conversations'}
+      </FlexWrapper>
-          </Button>
-        </FlexWrapper>
-      </SettingWrapper>
     </div>
   );
 };

plugins/copilot/src/core/langchain/vector-store.ts

@@ -1,118 +0,0 @@
-// fixme: vector store has not finished
-import type { DBSchema } from 'idb';
-import { Document } from 'langchain/document';
-import { Embeddings } from 'langchain/embeddings';
-import { VectorStore } from 'langchain/vectorstores';
-import { similarity as ml_distance_similarity } from 'ml-distance';
-
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-interface VectorDBV1 extends DBSchema {
-  vector: {
-    key: string;
-    value: Vector;
-  };
-}
-
-interface Vector {
-  id: string;
-
-  content: string;
-  embedding: number[];
-  metadata: Record<string, unknown>;
-}
-
-export interface MemoryVectorStoreArgs {
-  similarity?: typeof ml_distance_similarity.cosine;
-}
-
-export class IndexedDBVectorStore extends VectorStore {
-  memoryVectors: any[] = [];
-
-  similarity: typeof ml_distance_similarity.cosine;
-
-  constructor(
-    embeddings: Embeddings,
-    { similarity, ...rest }: MemoryVectorStoreArgs = {}
-  ) {
-    super(embeddings, rest);
-
-    this.similarity = similarity ?? ml_distance_similarity.cosine;
-  }
-
-  async addDocuments(documents: Document[]): Promise<void> {
-    const texts = documents.map(({ pageContent }) => pageContent);
-    return this.addVectors(
-      await this.embeddings.embedDocuments(texts),
-      documents
-    );
-  }
-
-  async addVectors(vectors: number[][], documents: Document[]): Promise<void> {
-    const memoryVectors = vectors.map((embedding, idx) => ({
-      content: documents[idx].pageContent,
-      embedding,
-      metadata: documents[idx].metadata,
-    }));
-
-    this.memoryVectors = this.memoryVectors.concat(memoryVectors);
-  }
-
-  async similaritySearchVectorWithScore(
-    query: number[],
-    k: number
-  ): Promise<[Document, number][]> {
-    const searches = this.memoryVectors
-      .map((vector, index) => ({
-        similarity: this.similarity(query, vector.embedding),
-        index,
-      }))
-      .sort((a, b) => (a.similarity > b.similarity ? -1 : 0))
-      .slice(0, k);
-
-    const result: [Document, number][] = searches.map(search => [
-      new Document({
-        metadata: this.memoryVectors[search.index].metadata,
-        pageContent: this.memoryVectors[search.index].content,
-      }),
-      search.similarity,
-    ]);
-
-    return result;
-  }
-
-  static override async fromTexts(
-    texts: string[],
-    metadatas: object[] | object,
-    embeddings: Embeddings,
-    dbConfig?: MemoryVectorStoreArgs
-  ): Promise<IndexedDBVectorStore> {
-    const docs: Document[] = [];
-    for (let i = 0; i < texts.length; i += 1) {
-      const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-      const newDoc = new Document({
-        pageContent: texts[i],
-        metadata,
-      });
-      docs.push(newDoc);
-    }
-    return IndexedDBVectorStore.fromDocuments(docs, embeddings, dbConfig);
-  }
-
-  static override async fromDocuments(
-    docs: Document[],
-    embeddings: Embeddings,
-    dbConfig?: MemoryVectorStoreArgs
-  ): Promise<IndexedDBVectorStore> {
-    const instance = new this(embeddings, dbConfig);
-    await instance.addDocuments(docs);
-    return instance;
-  }
-
-  static async fromExistingIndex(
-    embeddings: Embeddings,
-    dbConfig?: MemoryVectorStoreArgs
-  ): Promise<IndexedDBVectorStore> {
-    const instance = new this(embeddings, dbConfig);
-    return instance;
-  }
-}

plugins/copilot/src/index.ts

@@ -2,6 +2,7 @@ import type { PluginContext } from '@toeverything/plugin-infra/entry';
 import { createElement } from 'react';
 import { createRoot } from 'react-dom/client';
 
+import { DebugContent } from './UI/debug-content';
 import { DetailContent } from './UI/detail-content';
 import { HeaderItem } from './UI/header-item';
 
@@ -29,5 +30,19 @@ export const entry = (context: PluginContext) => {
       root.unmount();
     };
   });
+
+  context.register('setting', div => {
+    const root = createRoot(div);
+    root.render(
+      createElement(
+        context.utils.PluginProvider,
+        {},
+        createElement(DebugContent)
+      )
+    );
+    return () => {
+      root.unmount();
+    };
+  });
   return () => {};
 };

plugins/hello-world/src/index.ts

@@ -20,6 +20,14 @@ export const entry = (context: PluginContext) => {
     };
   });
 
+  context.register('formatBar', div => {
+    const root = createRoot(div);
+    root.render(createElement(HeaderItem));
+    return () => {
+      root.unmount();
+    };
+  });
+
   return () => {
     console.log('unregister');
   };

yarn.lock

@@ -164,7 +164,7 @@ __metadata:
     "@types/marked": ^5.0.1
     idb: ^7.1.1
     jotai: ^2.2.2
-    langchain: ^0.0.107
+    langchain: ^0.0.118
     marked: ^5.1.1
     marked-gfm-heading-id: ^3.0.4
     marked-mangle: ^1.1.0
@@ -657,7 +657,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@anthropic-ai/sdk@npm:^0.5.3":
+"@anthropic-ai/sdk@npm:^0.5.7":
   version: 0.5.8
   resolution: "@anthropic-ai/sdk@npm:0.5.8"
   dependencies:
@@ -22258,11 +22258,11 @@ __metadata:
   languageName: node
   linkType: hard
 
-"langchain@npm:^0.0.107":
+"langchain@npm:^0.0.118":
-  version: 0.0.107
+  version: 0.0.118
-  resolution: "langchain@npm:0.0.107"
+  resolution: "langchain@npm:0.0.118"
   dependencies:
-    "@anthropic-ai/sdk": ^0.5.3
+    "@anthropic-ai/sdk": ^0.5.7
     ansi-styles: ^5.0.0
     binary-extensions: ^2.2.0
     camelcase: 6
@@ -22272,7 +22272,7 @@ __metadata:
     js-tiktoken: ^1.0.7
     js-yaml: ^4.1.0
     jsonpointer: ^5.0.1
-    langchainplus-sdk: ^0.0.19
+    langsmith: ~0.0.11
     ml-distance: ^4.0.0
     object-hash: ^3.0.0
     openai: ^3.3.0
@@ -22285,6 +22285,7 @@ __metadata:
     zod-to-json-schema: ^3.20.4
   peerDependencies:
     "@aws-sdk/client-dynamodb": ^3.310.0
+    "@aws-sdk/client-kendra": ^3.352.0
     "@aws-sdk/client-lambda": ^3.310.0
     "@aws-sdk/client-s3": ^3.310.0
     "@aws-sdk/client-sagemaker-runtime": ^3.310.0
@@ -22294,11 +22295,13 @@ __metadata:
     "@getmetal/metal-sdk": "*"
     "@getzep/zep-js": ^0.4.1
     "@gomomento/sdk": ^1.23.0
+    "@google-ai/generativelanguage": ^0.2.1
     "@google-cloud/storage": ^6.10.1
     "@huggingface/inference": ^1.5.1
     "@notionhq/client": ^2.2.5
     "@opensearch-project/opensearch": "*"
     "@pinecone-database/pinecone": "*"
+    "@planetscale/database": ^1.8.0
     "@qdrant/js-client-rest": ^1.2.0
     "@supabase/postgrest-js": ^1.1.1
     "@supabase/supabase-js": ^2.10.0
@@ -22316,10 +22319,12 @@ __metadata:
     d3-dsv: ^2.0.0
     epub2: ^3.0.1
     faiss-node: ^0.2.1
-    google-auth-library: ^8.8.0
+    firebase-admin: ^11.9.0
+    google-auth-library: ^8.9.0
     hnswlib-node: ^1.4.2
     html-to-text: ^9.0.5
     ignore: ^5.2.0
+    ioredis: ^5.3.2
     mammoth: "*"
     mongodb: ^5.2.0
     mysql2: ^3.3.3
@@ -22327,11 +22332,12 @@ __metadata:
     pdf-parse: 1.1.1
     peggy: ^3.0.2
     pg: ^8.11.0
+    pg-copy-streams: ^6.0.5
     pickleparser: ^0.1.0
     playwright: ^1.32.1
     puppeteer: ^19.7.2
     redis: ^4.6.4
-    replicate: ^0.9.0
+    replicate: ^0.12.3
     sonix-speech-recognition: ^2.1.1
     srt-parser-2: ^1.2.2
     typeorm: ^0.3.12
@@ -22341,6 +22347,8 @@ __metadata:
   peerDependenciesMeta:
     "@aws-sdk/client-dynamodb":
       optional: true
+    "@aws-sdk/client-kendra":
+      optional: true
     "@aws-sdk/client-lambda":
       optional: true
     "@aws-sdk/client-s3":
@@ -22359,6 +22367,8 @@ __metadata:
       optional: true
     "@gomomento/sdk":
       optional: true
+    "@google-ai/generativelanguage":
+      optional: true
     "@google-cloud/storage":
       optional: true
     "@huggingface/inference":
@@ -22369,6 +22379,8 @@ __metadata:
       optional: true
     "@pinecone-database/pinecone":
       optional: true
+    "@planetscale/database":
+      optional: true
     "@qdrant/js-client-rest":
       optional: true
     "@supabase/postgrest-js":
@@ -22403,6 +22415,8 @@ __metadata:
       optional: true
     faiss-node:
       optional: true
+    firebase-admin:
+      optional: true
     google-auth-library:
       optional: true
     hnswlib-node:
@@ -22411,6 +22425,8 @@ __metadata:
       optional: true
     ignore:
       optional: true
+    ioredis:
+      optional: true
     mammoth:
       optional: true
     mongodb:
@@ -22425,6 +22441,8 @@ __metadata:
       optional: true
     pg:
       optional: true
+    pg-copy-streams:
+      optional: true
     pickleparser:
       optional: true
     playwright:
@@ -22447,13 +22465,13 @@ __metadata:
       optional: true
     weaviate-ts-client:
       optional: true
-  checksum: d1f01321db83aebae5b619ade2060b2416459e1b55723793c0e3244a9ec8fe57775f8c159f406b85eb58b66761a59705e17e9a2efb522876e81e881056711227
+  checksum: 080e56911adf4e869d30ad92228356468d21ebcfb2b67e88d1625faa2fc9c9b267b0fd5c0776a5de704794b7db5ae255d3a8204b52c09b5798d233ee226b7288
   languageName: node
   linkType: hard
 
-"langchainplus-sdk@npm:^0.0.19":
+"langsmith@npm:~0.0.11":
-  version: 0.0.19
+  version: 0.0.15
-  resolution: "langchainplus-sdk@npm:0.0.19"
+  resolution: "langsmith@npm:0.0.15"
   dependencies:
     "@types/uuid": ^9.0.1
     commander: ^10.0.1
@@ -22461,8 +22479,8 @@ __metadata:
     p-retry: 4
     uuid: ^9.0.0
   bin:
-    langchain: dist/cli/main.cjs
+    langsmith: dist/cli/main.cjs
-  checksum: f0174c1e248e4bc5034a7dd182f703b895a485b6408aa518c91ff12b3f015febd5546eeb7f821c82b63a9d2b67a7ea903a1a57ad196049743f0934ff1c524ae8
+  checksum: d23111aa0e65d5c28ebe08dd846b12fd788cc08a9de068c76f4ba362a0e5009939380efd7662d154375809d549eaba3a6658890f2676fea9b94b95de36ce62f1
   languageName: node
   linkType: hard