feat(infra): better orm (#7502)

packages/common/infra/src/orm/core/__tests__/yjs.spec.ts

@@ -26,6 +26,7 @@ const TEST_SCHEMA = {
   users: {
     id: f.number().primaryKey().default(incremental()),
     name: f.string(),
+    email: f.string().optional(),
   },
 } satisfies DBSchemaBuilder;
 
@@ -211,9 +212,11 @@ describe('ORM entity CRUD', () => {
   test('should be able to subscribe to entity key list', t => {
     const { client } = t;
 
+    let callbackCount = 0;
     let keys: string[] = [];
     const subscription = client.tags.keys$().subscribe(data => {
       keys = data;
+      callbackCount++;
     });
 
     client.tags.create({
@@ -229,6 +232,7 @@ describe('ORM entity CRUD', () => {
 
     client.tags.delete('test');
     expect(keys).toStrictEqual([]);
+    expect(callbackCount).toStrictEqual(3); // init, create, delete
 
     subscription.unsubscribe();
   });
@@ -236,9 +240,11 @@ describe('ORM entity CRUD', () => {
   test('should be able to subscribe to filtered entity changes', t => {
     const { client } = t;
 
+    let callbackCount = 0;
     let entities: any[] = [];
     const subscription = client.tags.find$({ name: 'test' }).subscribe(data => {
       entities = data;
+      callbackCount++;
     });
 
     const tag1 = client.tags.create({
@@ -257,6 +263,24 @@ describe('ORM entity CRUD', () => {
 
     expect(entities).toStrictEqual([tag1, tag2]);
 
+    client.tags.create({
+      id: '3',
+      name: 'not-test',
+      color: 'yellow',
+    });
+
+    expect(entities).toStrictEqual([tag1, tag2]);
+    expect(callbackCount).toStrictEqual(3);
+
+    client.tags.update('1', { color: 'green' });
+    expect(entities).toStrictEqual([{ ...tag1, color: 'green' }, tag2]);
+
+    client.tags.delete('1');
+    expect(entities).toStrictEqual([tag2]);
+
+    client.tags.delete('2');
+    expect(entities).toStrictEqual([]);
+
     subscription.unsubscribe();
   });
 
@@ -264,7 +288,7 @@ describe('ORM entity CRUD', () => {
     const { client } = t;
 
     let entities: any[] = [];
-    const subscription = client.tags.find$({}).subscribe(data => {
+    const subscription = client.tags.find$().subscribe(data => {
       entities = data;
     });
 
@@ -302,4 +326,82 @@ describe('ORM entity CRUD', () => {
       "[Table(tags)]: Field '$$DELETED' is reserved keyword and can't be used"
     );
   });
+
+  test('should be able to validate entity data', t => {
+    const { client } = t;
+
+    expect(() => {
+      client.users.create({
+        // @ts-expect-error
+        name: null,
+      });
+    }).toThrowError("Field 'name' is required but not set.");
+
+    expect(() => {
+      // @ts-expect-error
+      client.users.create({});
+    }).toThrowError("Field 'name' is required but not set.");
+
+    expect(() => {
+      client.users.update(1, {
+        // @ts-expect-error
+        name: null,
+      });
+    }).toThrowError("Field 'name' is required but not set.");
+  });
+
+  test('should be able to set optional field to null', t => {
+    const { client } = t;
+
+    {
+      const user = client.users.create({
+        name: 'test',
+      });
+
+      expect(user.email).toBe(null);
+    }
+
+    {
+      const user = client.users.create({
+        name: 'test',
+        email: null,
+      });
+
+      expect(user.email).toBe(null);
+    }
+
+    {
+      const user = client.users.create({
+        name: 'test',
+        email: 'test@example.com',
+      });
+
+      client.users.update(user.id, {
+        email: null,
+      });
+
+      expect(client.users.get(user.id)!.email).toBe(null);
+    }
+  });
+
+  test('should be able to find entity by optional field', t => {
+    const { client } = t;
+
+    const user = client.users.create({
+      name: 'test',
+      email: null,
+    });
+
+    {
+      const found = client.users.find({ email: null });
+
+      expect(found).toEqual([user]);
+    }
+
+    {
+      const found = client.users.find({ email: undefined });
+
+      expect(found).toEqual([]);
+    }
+  });
 });

packages/common/infra/src/orm/core/adapters/yjs/table.ts

@@ -104,36 +104,47 @@ export class YjsTableAdapter implements TableAdapter {
     const { where, select, callback } = query;
 
     let listeningOnAll = false;
-    const obKeys = new Set<any>();
-    const results = [];
+    const results = new Map<string, any>();
 
     if (!where) {
       listeningOnAll = true;
-    } else if ('byKey' in where) {
-      obKeys.add(where.byKey.toString());
     }
 
     for (const record of this.iterate(where)) {
-      if (!listeningOnAll) {
-        obKeys.add(this.keyof(record));
-      }
-      results.push(this.value(record, select));
+      results.set(this.keyof(record), this.value(record, select));
     }
 
-    callback(results);
+    callback(Array.from(results.values()));
 
     const ob = (tx: Transaction) => {
+      let hasChanged = false;
       for (const [ty] of tx.changed) {
-        const record = ty as unknown as AbstractType<any>;
-        if (
-          listeningOnAll ||
-          obKeys.has(this.keyof(record)) ||
-          (where && this.match(record, where))
-        ) {
-          callback(this.find({ where, select }));
-          return;
+        const record = ty;
+        const key = this.keyof(record);
+        const isMatch =
+          (listeningOnAll || (where && this.match(record, where))) &&
+          !this.isDeleted(record);
+        const prevMatch = results.get(key);
+        const isPrevMatched = results.has(key);
+
+        if (isMatch && isPrevMatched) {
+          const newValue = this.value(record, select);
+          if (prevMatch !== newValue) {
+            results.set(key, newValue);
+            hasChanged = true;
+          }
+        } else if (isMatch && !isPrevMatched) {
+          results.set(this.keyof(record), this.value(record, select));
+          hasChanged = true;
+        } else if (!isMatch && isPrevMatched) {
+          results.delete(key);
+          hasChanged = true;
         }
       }
+
+      if (hasChanged) {
+        callback(Array.from(results.values()));
+      }
     };
 
     this.doc.on('afterTransaction', ob);
@@ -165,9 +176,16 @@ export class YjsTableAdapter implements TableAdapter {
     return null;
   }
 
-  private *iterate(where: WhereCondition = []) {
+  private *iterate(where?: WhereCondition) {
+    if (!where) {
+      for (const map of this.doc.share.values()) {
+        if (!this.isDeleted(map)) {
+          yield map;
+        }
+      }
+    }
     // fast pass for key lookup without iterating the whole table
-    if ('byKey' in where) {
+    else if ('byKey' in where) {
       const record = this.recordByKey(where.byKey.toString());
       if (record) {
         yield record;
@@ -202,7 +220,9 @@ export class YjsTableAdapter implements TableAdapter {
     return (
       !this.isDeleted(record) &&
       (Array.isArray(where)
-        ? where.every(c => this.field(record, c.field) === c.value)
+        ? where.length === 0
+          ? false
+          : where.every(c => this.field(record, c.field) === c.value)
         : where.byKey === this.keyof(record))
     );
   }

packages/common/infra/src/orm/core/table.ts

@@ -30,7 +30,9 @@ type OptionalFields<T extends TableSchemaBuilder> = {
     ? Optional extends true
       ? K
       : never
-    : never]?: T[K] extends FieldSchemaBuilder<infer Type> ? Type : never;
+    : never]?: T[K] extends FieldSchemaBuilder<infer Type>
+    ? Type | null
+    : never;
 };
 
 type PrimaryKeyField<T extends TableSchemaBuilder> = {
@@ -68,17 +70,13 @@ export type Entity<T extends TableSchemaBuilder> = Pretty<
 >;
 
 export type UpdateEntityInput<T extends TableSchemaBuilder> = Pretty<{
-  [key in NonPrimaryKeyFields<T>]?: T[key] extends FieldSchemaBuilder<
-    infer Type
-  >
-    ? Type
+  [key in NonPrimaryKeyFields<T>]?: key extends keyof Entity<T>
+    ? Entity<T>[key]
     : never;
 }>;
 
 export type FindEntityInput<T extends TableSchemaBuilder> = Pretty<{
-  [key in keyof T]?: T[key] extends FieldSchemaBuilder<infer Type>
-    ? Type
-    : never;
+  [key in keyof T]?: key extends keyof Entity<T> ? Entity<T>[key] : never;
 }>;
 
 export class Table<T extends TableSchemaBuilder> {
@@ -192,22 +190,30 @@ export class Table<T extends TableSchemaBuilder> {
     return ob$;
   }
 
-  find(where: FindEntityInput<T>): Entity<T>[] {
+  find(where?: FindEntityInput<T>): Entity<T>[] {
     return this.adapter.find({
-      where: Object.entries(where).map(([field, value]) => ({
-        field,
-        value,
-      })),
+      where: !where
+        ? undefined
+        : Object.entries(where)
+            .map(([field, value]) => ({
+              field,
+              value,
+            }))
+            .filter(({ value }) => value !== undefined),
     });
   }
 
-  find$(where: FindEntityInput<T>): Observable<Entity<T>[]> {
+  find$(where?: FindEntityInput<T>): Observable<Entity<T>[]> {
     return new Observable<Entity<T>[]>(subscriber => {
       const unsubscribe = this.adapter.observe({
-        where: Object.entries(where).map(([field, value]) => ({
-          field,
-          value,
-        })),
+        where: !where
+          ? undefined
+          : Object.entries(where)
+              .map(([field, value]) => ({
+                field,
+                value,
+              }))
+              .filter(({ value }) => value !== undefined),
         callback: data => {
           subscriber.next(data);
         },

packages/common/infra/src/orm/core/validators/data.ts

@@ -65,14 +65,13 @@ export const dataValidators = {
           continue;
         }
 
-        if (
-          val === null &&
-          (!field.optional ||
-            field.optional) /* say 'null' can be stored as 'json' */
-        ) {
-          throw new Error(
-            `[Table(${table.name})]: Field '${key}' is required but set as null.`
-          );
+        if (val === null) {
+          if (!field.optional) {
+            throw new Error(
+              `[Table(${table.name})]: Field '${key}' is required but not set.`
+            );
+          }
+          continue;
         }
 
         const typeGet = inputType(val);
@@ -97,10 +96,13 @@ export const dataValidators = {
 
         const val = data[key];
 
-        if ((val === undefined || val === null) && !field.optional) {
-          throw new Error(
-            `[Table(${table.name})]: Field '${key}' is required but not set.`
-          );
+        if (val === undefined || val === null) {
+          if (!field.optional) {
+            throw new Error(
+              `[Table(${table.name})]: Field '${key}' is required but not set.`
+            );
+          }
+          continue;
         }
 
         const typeGet = inputType(val);