perf: table domain s cache (#2105)

* perf: table domain s cache

* perf: optimize link recursive sql perf

Author: nichenqin

apps/nestjs-backend/src/features/field/field-calculate/field-calculate.module.ts

@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
 import { DbProvider } from '../../../db-provider/db.provider';
 import { CalculationModule } from '../../calculation/calculation.module';
 import { CollaboratorModule } from '../../collaborator/collaborator.module';
+import { ComputedModule } from '../../record/computed/computed.module';
 import { TableIndexService } from '../../table/table-index.service';
 import { TableDomainQueryModule } from '../../table-domain';
 import { ViewModule } from '../../view/view.module';
@@ -16,7 +17,14 @@ import { FormulaFieldService } from './formula-field.service';
 import { LinkFieldQueryService } from './link-field-query.service';
 
 @Module({
-  imports: [FieldModule, CalculationModule, ViewModule, CollaboratorModule, TableDomainQueryModule],
+  imports: [
+    FieldModule,
+    CalculationModule,
+    ViewModule,
+    CollaboratorModule,
+    TableDomainQueryModule,
+    ComputedModule,
+  ],
   providers: [
     DbProvider,
     FieldDeletingService,

apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts

@@ -41,6 +41,7 @@ import type { IOpsMap } from '../../calculation/utils/compose-maps';
 import { composeOpMaps } from '../../calculation/utils/compose-maps';
 import { isLinkCellValue } from '../../calculation/utils/detect-link';
 import { CollaboratorService } from '../../collaborator/collaborator.service';
+import { ComputedOrchestratorService } from '../../record/computed/services/computed-orchestrator.service';
 import { TableIndexService } from '../../table/table-index.service';
 import { FieldService } from '../field.service';
 import type { IFieldInstance, IFieldMap } from '../model/factory';
@@ -71,6 +72,7 @@ export class FieldConvertingService {
     private readonly fieldCalculationService: FieldCalculationService,
     private readonly collaboratorService: CollaboratorService,
     private readonly tableIndexService: TableIndexService,
+    private readonly computedOrchestrator: ComputedOrchestratorService,
     @InjectModel('CUSTOM_KNEX') private readonly knex: Knex
   ) {}
 
@@ -955,6 +957,37 @@ export class FieldConvertingService {
     };
   }
 
+  private buildCellContextsFromOps(opsMap: IOpsMap[string] | undefined) {
+    const contexts: ICellContext[] = [];
+    if (!opsMap) {
+      return contexts;
+    }
+    for (const [recordId, ops] of Object.entries(opsMap)) {
+      for (const op of ops) {
+        const context = RecordOpBuilder.editor.setRecord.detect(op);
+        if (!context) {
+          continue;
+        }
+        contexts.push({
+          recordId,
+          fieldId: context.fieldId,
+          oldValue: context.oldCellValue,
+          newValue: context.newCellValue,
+        });
+      }
+    }
+    return contexts;
+  }
+
+  private buildComputedSources(recordOpsMap: IOpsMap) {
+    return Object.entries(recordOpsMap)
+      .map(([tableId, ops]) => ({
+        tableId,
+        cellContexts: this.buildCellContextsFromOps(ops),
+      }))
+      .filter((source) => source.cellContexts.length);
+  }
+
   // eslint-disable-next-line sonarjs/cognitive-complexity
   private async calculateAndSaveRecords(
     tableId: string,
@@ -985,7 +1018,17 @@ export class FieldConvertingService {
       }
     }
 
-    await this.batchService.updateRecords(recordOpsMap);
+    const computedSources = this.buildComputedSources(recordOpsMap);
+    if (computedSources.length) {
+      await this.computedOrchestrator.computeCellChangesForRecordsMulti(
+        computedSources,
+        async () => {
+          await this.batchService.updateRecords(recordOpsMap);
+        }
+      );
+    } else {
+      await this.batchService.updateRecords(recordOpsMap);
+    }
   }
 
   private async getExistRecords(tableId: string, newField: IFieldInstance) {
@@ -1295,7 +1338,35 @@ export class FieldConvertingService {
       return false;
     }
 
-    return majorFieldKeysChanged(oldField, newField);
+    if (majorFieldKeysChanged(oldField, newField)) {
+      return true;
+    }
+
+    if (this.hasConditionalLookupDiff(newField, oldField)) {
+      return true;
+    }
+
+    if (this.hasConditionalRollupDiff(newField, oldField)) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private hasConditionalLookupDiff(newField: IFieldInstance, oldField: IFieldInstance) {
+    if (!newField.isConditionalLookup) {
+      return false;
+    }
+
+    return !isEqual(newField.lookupOptions, oldField.lookupOptions);
+  }
+
+  private hasConditionalRollupDiff(newField: IFieldInstance, oldField: IFieldInstance) {
+    if (newField.type !== FieldType.ConditionalRollup) {
+      return false;
+    }
+
+    return !isEqual(newField.options, oldField.options);
   }
 
   private async calculateField(
@@ -1307,7 +1378,11 @@ export class FieldConvertingService {
       return;
     }
 
-    if (!majorFieldKeysChanged(oldField, newField)) {
+    const hasMajorChange = majorFieldKeysChanged(oldField, newField);
+    const conditionalLookupDiff = this.hasConditionalLookupDiff(newField, oldField);
+    const conditionalRollupDiff = this.hasConditionalRollupDiff(newField, oldField);
+
+    if (!hasMajorChange && !conditionalLookupDiff && !conditionalRollupDiff) {
       return;
     }
 

apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts

@@ -947,6 +947,7 @@ export class FieldOpenApiService {
     oldField,
     modifiedOps,
     supplementChange,
+    dependentFieldIds,
   }: {
     tableId: string;
     newField: IFieldInstance;
@@ -957,6 +958,7 @@ export class FieldOpenApiService {
       newField: IFieldInstance;
       oldField: IFieldInstance;
     };
+    dependentFieldIds?: string[];
   }): Promise<{ compatibilityIssue: boolean }> {
     let encounteredCompatibilityIssue = false;
 
@@ -983,19 +985,47 @@ export class FieldOpenApiService {
       }
     };
 
+    const sourceMap = new Map<string, Set<string>>();
+    const shouldRecomputeSelf = this.fieldConvertingService.needCalculate(newField, oldField);
+    const addSource = (tid: string, fieldIds: string[]) => {
+      const set = sourceMap.get(tid) ?? new Set<string>();
+      fieldIds.forEach((id) => set.add(id));
+      sourceMap.set(tid, set);
+    };
+
+    if (shouldRecomputeSelf) {
+      addSource(tableId, [newField.id]);
+    }
+
+    if (dependentFieldIds?.length) {
+      const dependentFields = await this.prismaService.field.findMany({
+        where: { id: { in: dependentFieldIds }, deletedTime: null },
+        select: { id: true, tableId: true },
+      });
+      dependentFields
+        .filter(
+          ({ id, tableId: depTableId }) =>
+            shouldRecomputeSelf || id !== newField.id || depTableId !== tableId
+        )
+        .forEach(({ id, tableId: depTableId }) => addSource(depTableId, [id]));
+    }
+
+    if (supplementChange) {
+      addSource(supplementChange.tableId, [supplementChange.newField.id]);
+    }
+
+    const sources = Array.from(sourceMap.entries()).map(([tid, ids]) => ({
+      tableId: tid,
+      fieldIds: Array.from(ids),
+    }));
+    const hasSources = sources.length > 0;
+
     // 1. stage close constraint
     await this.fieldConvertingService.closeConstraint(tableId, newField, oldField);
 
     // 2. stage alter + apply record changes and calculate field with computed publishing (atomic)
     await this.prismaService.$tx(
       async () => {
-        const sources = [{ tableId, fieldIds: [newField.id] }];
-        if (supplementChange)
-          sources.push({
-            tableId: supplementChange.tableId,
-            fieldIds: [supplementChange.newField.id],
-          });
-
         const runCompute = async () => {
           // Update dependencies and schema first so evaluate() sees new schema
           await this.fieldViewSyncService.convertDependenciesByFieldIds(
@@ -1027,14 +1057,19 @@ export class FieldOpenApiService {
           }
         };
 
-        try {
-          await this.computedOrchestrator.computeCellChangesForFields(sources, runCompute);
-        } catch (error) {
-          if (this.isFieldReferenceCompatibilityError(error)) {
-            encounteredCompatibilityIssue = true;
-            return;
+        if (hasSources) {
+          try {
+            await this.computedOrchestrator.computeCellChangesForFields(sources, runCompute);
+          } catch (error) {
+            if (this.isFieldReferenceCompatibilityError(error)) {
+              encounteredCompatibilityIssue = true;
+              return;
+            }
+
+            throw error;
           }
-          throw error;
+        } else {
+          await runCompute();
         }
       },
       { timeout: this.thresholdConfig.bigTransactionTimeout }
@@ -1086,12 +1121,26 @@ export class FieldOpenApiService {
     const { newField, oldField, modifiedOps, supplementChange, references } =
       await this.fieldConvertingService.stageAnalysis(tableId, fieldId, updateFieldRo);
 
+    const dependentRefs = await this.prismaService.reference.findMany({
+      where: { fromFieldId: fieldId },
+      select: { toFieldId: true },
+    });
+    const dependentFieldIds = Array.from(
+      new Set([...(references ?? []), ...dependentRefs.map((ref) => ref.toFieldId)])
+    );
+
+    const shouldRecomputeSelf = this.fieldConvertingService.needCalculate(newField, oldField);
+    const filteredDependentFieldIds = shouldRecomputeSelf
+      ? dependentFieldIds
+      : dependentFieldIds.filter((id) => id !== newField.id);
+
     const { compatibilityIssue } = await this.performConvertField({
       tableId,
       newField,
       oldField,
       modifiedOps,
       supplementChange,
+      dependentFieldIds: filteredDependentFieldIds,
     });
 
     const shouldForceLookupError =
@@ -1102,20 +1151,37 @@ export class FieldOpenApiService {
         ((newField.options as ILinkFieldOptions | undefined)?.foreignTableId ?? null) !==
           ((oldField.options as ILinkFieldOptions | undefined)?.foreignTableId ?? null));
 
-    const dependentRefs = await this.prismaService.reference.findMany({
-      where: { fromFieldId: fieldId },
-      select: { toFieldId: true },
-    });
-    const dependentFieldIds = Array.from(
-      new Set([...(references ?? []), ...dependentRefs.map((ref) => ref.toFieldId)])
-    );
-
-    if (dependentFieldIds.length) {
+    if (filteredDependentFieldIds.length) {
       try {
-        await this.restoreReference(dependentFieldIds);
+        await this.restoreReference(filteredDependentFieldIds);
         const dependentFieldRaws = await this.prismaService.field.findMany({
-          where: { id: { in: dependentFieldIds }, deletedTime: null },
+          where: { id: { in: filteredDependentFieldIds }, deletedTime: null },
         });
+
+        if (dependentFieldRaws.length) {
+          const dependentSourceMap = dependentFieldRaws.reduce<Record<string, Set<string>>>(
+            (acc, field) => {
+              const set = acc[field.tableId] ?? new Set<string>();
+              set.add(field.id);
+              acc[field.tableId] = set;
+              return acc;
+            },
+            {}
+          );
+          const dependentSources = Object.entries(dependentSourceMap).map(([tid, ids]) => ({
+            tableId: tid,
+            fieldIds: Array.from(ids),
+          }));
+          if (dependentSources.length) {
+            await this.computedOrchestrator.computeCellChangesForFields(
+              dependentSources,
+              async () => {
+                // schema/meta already up to date; nothing additional to run here
+              }
+            );
+          }
+        }
+
         for (const raw of dependentFieldRaws) {
           const instance = createFieldInstanceByRaw(raw);
           const isValid = await this.isFieldConfigurationValid(raw.tableId, instance);