Merge pull request #1106 from refly-ai/fix/agent-media-config

fix: properly handle missing media model in agent mode

Author: mrcfps

apps/api/src/modules/provider/provider.service.ts

@@ -430,49 +430,45 @@ export class ProviderService implements OnModuleInit {
       const userPreferences = await this.getUserPreferences(user.uid);
       const userDefaultModel = userPreferences?.defaultModel;
 
-      if (!userDefaultModel) {
-        this.logger.log(
-          `No default model configuration found for user ${user.uid} for ${mediaType}`,
-        );
-        return null;
-      }
-
       // Get the specific media model configuration based on mediaType
-      const mediaModelConfig = userDefaultModel[mediaType];
-
-      if (!mediaModelConfig?.itemId) {
-        this.logger.log(`No ${mediaType} model configured for user ${user.uid}`);
-        return null;
-      }
+      const mediaModelConfig = userDefaultModel?.[mediaType];
 
       // Find the provider item for this configured model
       const providerItems = await this.listProviderItems(user, {
         category: 'mediaGeneration',
         enabled: true,
       });
 
-      const configuredProviderItem = providerItems.find(
-        (item) => item.itemId === mediaModelConfig.itemId,
+      let configuredProviderItem = providerItems.find(
+        (item) => item.itemId === mediaModelConfig?.itemId,
       );
 
       if (!configuredProviderItem) {
-        this.logger.warn(
-          `Configured ${mediaType} model ${mediaModelConfig.itemId} not found in user's provider items`,
-        );
-        return null;
+        // Fallback: find an enabled model that supports the requested mediaType
+        const fallbackProviderItem = providerItems.find((item) => {
+          try {
+            const config: MediaGenerationModelConfig = JSON.parse(item.config || '{}');
+
+            return config.capabilities?.[mediaType];
+          } catch (error) {
+            this.logger.warn(
+              `Failed to parse config for provider item ${item.itemId}: ${error?.message}`,
+            );
+            return false;
+          }
+        });
+
+        if (!fallbackProviderItem) {
+          this.logger.warn(`No enabled ${mediaType} model found in user's provider items`);
+          return null;
+        }
+
+        configuredProviderItem = fallbackProviderItem;
       }
 
       // Parse the model configuration
       const config: MediaGenerationModelConfig = JSON.parse(configuredProviderItem.config || '{}');
 
-      // Verify that this model actually supports the requested media type
-      if (!config.capabilities?.[mediaType]) {
-        this.logger.warn(
-          `Configured ${mediaType} model ${config.modelId} does not support ${mediaType} generation`,
-        );
-        return null;
-      }
-
       this.logger.log(
         `Using user configured ${mediaType} model: ${config.modelId} from provider: ${configuredProviderItem.provider?.providerKey}`,
       );

packages/errors/src/errors.ts

@@ -120,6 +120,22 @@ export class EmbeddingNotConfiguredError extends BaseError {
   };
 }
 
+export class MediaProviderNotConfiguredError extends BaseError {
+  code = 'E0016';
+  messageDict = {
+    en: 'Media provider not configured, please configure a media provider in the settings',
+    'zh-CN': '未配置媒体提供方，请先在设置中进行配置',
+  };
+}
+
+export class MediaModelNotConfiguredError extends BaseError {
+  code = 'E0017';
+  messageDict = {
+    en: 'Media model not configured, please configure a media model in the settings',
+    'zh-CN': '未配置媒体模型，请先在设置中进行配置',
+  };
+}
+
 export class CanvasNotFoundError extends BaseError {
   code = 'E1000';
   messageDict = {
@@ -353,6 +369,8 @@ const errorMap = {
   E0013: EmbeddingNotAllowedToChangeError,
   E0014: ChatModelNotConfiguredError,
   E0015: EmbeddingNotConfiguredError,
+  E0016: MediaProviderNotConfiguredError,
+  E0017: MediaModelNotConfiguredError,
   E1000: CanvasNotFoundError,
   E1002: ResourceNotFoundError,
   E1003: DocumentNotFoundError,

packages/skill-template/src/skills/generate-media.ts

@@ -11,6 +11,7 @@ import { StateGraphArgs, StateGraph, START, END } from '@langchain/langgraph';
 import { GraphState } from '../scheduler/types';
 import { Runnable } from '@langchain/core/runnables';
 import { SystemMessage } from '@langchain/core/messages';
+import { MediaProviderNotConfiguredError, MediaModelNotConfiguredError } from '@refly/errors';
 
 /**
  * Interface for log argument objects used in processLogArgs
@@ -138,7 +139,16 @@ export class GenerateMedia extends BaseSkill {
     );
     const quality = parsedParams.quality || String(tplConfig?.quality?.value ?? 'high');
 
-    const { provider, model } = await this.engine.service.getUserMediaConfig(user, mediaType);
+    const { provider, model } =
+      (await this.engine.service.getUserMediaConfig(user, mediaType)) || {};
+
+    if (!provider) {
+      throw new MediaProviderNotConfiguredError();
+    }
+
+    if (!model) {
+      throw new MediaModelNotConfiguredError();
+    }
 
     // Clean the query by removing parameter specifications
     const cleanedQuery = this.cleanQueryFromParameters(query);