feat: mix search weight (#4170)

c121914yu · web-flow · commit 93f774790492 · 2025-03-14T18:31:37.000+08:00
* feat: mix search weight

* feat: svg render
diff --git a/docSite/content/zh-cn/docs/development/upgrading/491.md b/docSite/content/zh-cn/docs/development/upgrading/491.md
@@ -13,7 +13,8 @@ weight: 799
 2. 知识库分块阅读器。
 3. API 知识库支持 PDF 增强解析。
 4. 邀请团队成员，改为邀请链接模式。
-5. 支持重排模型选择和权重设置，同时调整了知识库搜索权重计算方式，改成 搜索权重 + 重排权重，而不是向量检索权重+全文检索权重+重排权重。
+5. 支持混合检索权重设置。
+6. 支持重排模型选择和权重设置，同时调整了知识库搜索权重计算方式，改成 搜索权重 + 重排权重，而不是向量检索权重+全文检索权重+重排权重。
 
 ## ⚙️ 优化
 
diff --git a/packages/global/core/app/type.d.ts b/packages/global/core/app/type.d.ts
@@ -75,6 +75,7 @@ export type AppDatasetSearchParamsType = {
   searchMode: `${DatasetSearchModeEnum}`;
   limit?: number; // limit max tokens
   similarity?: number;
+  embeddingWeight?: number; // embedding weight, fullText weight = 1 - embeddingWeight
 
   usingReRank?: boolean;
   rerankModel?: string;
diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts
@@ -108,6 +108,10 @@ export const appWorkflow2Form = ({
       defaultAppForm.dataset.searchMode =
         findInputValueByKey(node.inputs, NodeInputKeyEnum.datasetSearchMode) ||
         DatasetSearchModeEnum.embedding;
+      defaultAppForm.dataset.embeddingWeight = findInputValueByKey(
+        node.inputs,
+        NodeInputKeyEnum.datasetSearchEmbeddingWeight
+      );
       // Rerank
       defaultAppForm.dataset.usingReRank = !!findInputValueByKey(
         node.inputs,
diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts
@@ -185,7 +185,7 @@ export enum SearchScoreTypeEnum {
 }
 export const SearchScoreTypeMap = {
   [SearchScoreTypeEnum.embedding]: {
-    label: i18nT('common:core.dataset.search.score.embedding'),
+    label: i18nT('common:core.dataset.search.mode.embedding'),
     desc: i18nT('common:core.dataset.search.score.embedding desc'),
     showScore: true
   },
diff --git a/packages/global/core/workflow/constants.ts b/packages/global/core/workflow/constants.ts
@@ -154,9 +154,12 @@ export enum NodeInputKeyEnum {
   datasetSimilarity = 'similarity',
   datasetMaxTokens = 'limit',
   datasetSearchMode = 'searchMode',
+  datasetSearchEmbeddingWeight = 'embeddingWeight',
+
   datasetSearchUsingReRank = 'usingReRank',
   datasetSearchRerankWeight = 'rerankWeight',
   datasetSearchRerankModel = 'rerankModel',
+
   datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
   datasetSearchExtensionModel = 'datasetSearchExtensionModel',
   datasetSearchExtensionBg = 'datasetSearchExtensionBg',
diff --git a/packages/global/core/workflow/runtime/type.d.ts b/packages/global/core/workflow/runtime/type.d.ts
@@ -133,6 +133,9 @@ export type DispatchNodeResponseType = {
   similarity?: number;
   limit?: number;
   searchMode?: `${DatasetSearchModeEnum}`;
+  embeddingWeight?: number;
+  rerankModel?: string;
+  rerankWeight?: number;
   searchUsingReRank?: boolean;
   queryExtensionResult?: {
     model: string;
diff --git a/packages/global/core/workflow/template/system/datasetSearch.ts b/packages/global/core/workflow/template/system/datasetSearch.ts
@@ -64,6 +64,13 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
       valueType: WorkflowIOValueTypeEnum.string,
       value: DatasetSearchModeEnum.embedding
     },
+    {
+      key: NodeInputKeyEnum.datasetSearchEmbeddingWeight,
+      renderTypeList: [FlowNodeInputTypeEnum.hidden],
+      label: '',
+      valueType: WorkflowIOValueTypeEnum.number,
+      value: 0.5
+    },
     // Rerank
     {
       key: NodeInputKeyEnum.datasetSearchUsingReRank,
diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts
@@ -40,6 +40,7 @@ export type SearchDatasetDataProps = {
   [NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
   [NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
   [NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
+  [NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
 
   [NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
   [NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
@@ -161,6 +162,7 @@ export async function searchDatasetData(
     similarity = 0,
     limit: maxTokens,
     searchMode = DatasetSearchModeEnum.embedding,
+    embeddingWeight = 0.5,
     usingReRank = false,
     rerankModel,
     rerankWeight = 0.5,
@@ -731,16 +733,20 @@ export async function searchDatasetData(
   })();
 
   // embedding recall and fullText recall rrf concat
+  const baseK = 120;
+  const embK = Math.round(baseK * (1 - embeddingWeight)); // 搜索结果的 k 值
+  const fullTextK = Math.round(baseK * embeddingWeight); // rerank 结果的 k 值
+
   const rrfSearchResult = datasetSearchResultConcat([
-    { k: 60, list: embeddingRecallResults },
-    { k: 60, list: fullTextRecallResults }
+    { k: embK, list: embeddingRecallResults },
+    { k: fullTextK, list: fullTextRecallResults }
   ]);
   const rrfConcatResults = (() => {
+    if (reRankResults.length === 0) return rrfSearchResult;
     if (rerankWeight === 1) return reRankResults;
 
-    const baseK = 30;
-    const searchK = Math.round(baseK / (1 - rerankWeight)); // 搜索结果的 k 值
-    const rerankK = Math.round(baseK / rerankWeight); // rerank 结果的 k 值
+    const searchK = Math.round(baseK * rerankWeight); // 搜索结果的 k 值
+    const rerankK = Math.round(baseK * (1 - rerankWeight)); // rerank 结果的 k 值
 
     return datasetSearchResultConcat([
       { k: searchK, list: rrfSearchResult },
diff --git a/packages/service/core/workflow/dispatch/dataset/search.ts b/packages/service/core/workflow/dispatch/dataset/search.ts
@@ -22,8 +22,9 @@ type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
   [NodeInputKeyEnum.datasetSimilarity]: number;
   [NodeInputKeyEnum.datasetMaxTokens]: number;
-  [NodeInputKeyEnum.datasetSearchMode]: `${DatasetSearchModeEnum}`;
   [NodeInputKeyEnum.userChatInput]?: string;
+  [NodeInputKeyEnum.datasetSearchMode]: `${DatasetSearchModeEnum}`;
+  [NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
 
   [NodeInputKeyEnum.datasetSearchUsingReRank]: boolean;
   [NodeInputKeyEnum.datasetSearchRerankModel]?: string;
@@ -57,11 +58,11 @@ export async function dispatchDatasetSearch(
       datasets = [],
       similarity,
       limit = 1500,
-      searchMode,
       userChatInput = '',
       authTmbId = false,
       collectionFilterMatch,
-
+      searchMode,
+      embeddingWeight,
       usingReRank,
       rerankModel,
       rerankWeight,
@@ -129,6 +130,7 @@ export async function dispatchDatasetSearch(
     limit,
     datasetIds,
     searchMode,
+    embeddingWeight,
     usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
     rerankModel: getRerankModel(rerankModel),
     rerankWeight,
@@ -228,6 +230,9 @@ export async function dispatchDatasetSearch(
     similarity: usingSimilarityFilter ? similarity : undefined,
     limit,
     searchMode,
+    embeddingWeight: searchMode === DatasetSearchModeEnum.mixedRecall ? embeddingWeight : undefined,
+    rerankModel: usingReRank ? getRerankModel(rerankModel)?.name : undefined,
+    rerankWeight: usingReRank ? rerankWeight : undefined,
     searchUsingReRank: searchUsingReRank,
     quoteList: searchRes,
     queryExtensionResult,
diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts
@@ -429,6 +429,7 @@ export const iconPaths = {
   'price/bg': () => import('./icons/price/bg.svg'),
   'price/right': () => import('./icons/price/right.svg'),
   save: () => import('./icons/save.svg'),
+  sliderTag: () => import('./icons/sliderTag.svg'),
   stop: () => import('./icons/stop.svg'),
   'support/account/laf': () => import('./icons/support/account/laf.svg'),
   'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'),
diff --git a/packages/web/components/common/Icon/icons/sliderTag.svg b/packages/web/components/common/Icon/icons/sliderTag.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 11 11" >
+  <path d="M5.04123 0.144501L9.47821 4.82132C9.83075 5.19292 10.0273 5.68562 10.0273 6.19784V8.65565C10.0273 9.76022 9.13185 10.6557 8.02728 10.6557H2.05518C0.950606 10.6557 0.0551758 9.76022 0.0551758 8.65565V6.19785C0.0551758 5.68562 0.251705 5.19292 0.604247 4.82132L5.04123 0.144501Z" fill="#3370FF"/>
+</svg>
diff --git a/packages/web/i18n/en/app.json b/packages/web/i18n/en/app.json
@@ -120,7 +120,6 @@
   "publish_success": "Publish Successful",
   "question_guide_tip": "After the conversation, 3 guiding questions will be generated for you.",
   "reasoning_response": "Output thinking",
-  "rerank_weight": "Rearrange weights",
   "response_format": "Response format",
   "saved_success": "Saved successfully! \nTo use this version externally, click Save and Publish",
   "search_app": "Search apps",
diff --git a/packages/web/i18n/en/chat.json b/packages/web/i18n/en/chat.json
@@ -49,6 +49,7 @@
   "response.child total points": "Sub-workflow point consumption",
   "response.dataset_concat_length": "Combined total",
   "response.node_inputs": "Node Inputs",
+  "response_hybrid_weight": "Embedding : Full text = {{emb}} : {{text}}",
   "select": "Select",
   "select_file": "Upload File",
   "select_file_img": "Upload file / image",
diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json
@@ -1025,6 +1025,7 @@
   "question_feedback": "Work order",
   "read_quote": "View citations",
   "required": "Required",
+  "rerank_weight": "Rearrange weights",
   "resume_failed": "Resume Failed",
   "select_reference_variable": "Select Reference Variable",
   "share_link": "Share Link",
diff --git a/packages/web/i18n/zh-CN/app.json b/packages/web/i18n/zh-CN/app.json
@@ -120,7 +120,6 @@
   "publish_success": "发布成功",
   "question_guide_tip": "对话结束后，会为你生成 3 个引导性问题。",
   "reasoning_response": "输出思考",
-  "rerank_weight": "重排权重",
   "response_format": "回复格式",
   "saved_success": "保存成功！如需在外部使用该版本，请点击“保存并发布”",
   "search_app": "搜索应用",
diff --git a/packages/web/i18n/zh-CN/chat.json b/packages/web/i18n/zh-CN/chat.json
@@ -49,6 +49,7 @@
   "response.child total points": "子工作流积分消耗",
   "response.dataset_concat_length": "合并后总数",
   "response.node_inputs": "节点输入",
+  "response_hybrid_weight": "语义检索 : 全文检索 = {{emb}} : {{text}}",
   "select": "选择",
   "select_file": "上传文件",
   "select_file_img": "上传文件/图片",
diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json
@@ -623,7 +623,6 @@
   "core.dataset.search.mode.fullTextRecall desc": "使用传统的全文检索，适合查找一些关键词和主谓语特殊的数据",
   "core.dataset.search.mode.mixedRecall": "混合检索",
   "core.dataset.search.mode.mixedRecall desc": "使用向量检索与全文检索的综合结果返回，使用 RRF 算法进行排序。",
-  "core.dataset.search.score.embedding": "语义检索",
   "core.dataset.search.score.embedding desc": "通过计算向量之间的距离获取得分，范围为 0~1。",
   "core.dataset.search.score.fullText": "全文检索",
   "core.dataset.search.score.fullText desc": "计算相同关键词的得分，范围为 0~无穷。",
@@ -1029,6 +1028,7 @@
   "question_feedback": "工单咨询",
   "read_quote": "查看引用",
   "required": "必须",
+  "rerank_weight": "重排权重",
   "resume_failed": "恢复失败",
   "select_reference_variable": "选择引用变量",
   "share_link": "分享链接",
diff --git a/packages/web/i18n/zh-Hant/app.json b/packages/web/i18n/zh-Hant/app.json
@@ -120,7 +120,6 @@
   "publish_success": "發布成功",
   "question_guide_tip": "對話結束後，會為你產生 3 個引導性問題。",
   "reasoning_response": "輸出思考",
-  "rerank_weight": "重排權重",
   "response_format": "回复格式",
   "saved_success": "保存成功！\n如需在外部使用該版本，請點擊“儲存並發布”",
   "search_app": "搜尋應用程式",
diff --git a/packages/web/i18n/zh-Hant/chat.json b/packages/web/i18n/zh-Hant/chat.json
@@ -48,6 +48,7 @@
   "response.child total points": "子工作流程點數消耗",
   "response.dataset_concat_length": "合併總數",
   "response.node_inputs": "節點輸入",
+  "response_hybrid_weight": "語義檢索 : 全文檢索 = {{emb}} : {{text}}",
   "select": "選取",
   "select_file": "上傳檔案",
   "select_file_img": "上傳檔案 / 圖片",
diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json
@@ -1024,6 +1024,7 @@
   "question_feedback": "工單諮詢",
   "read_quote": "查看引用",
   "required": "必填",
+  "rerank_weight": "重排權重",
   "resume_failed": "恢復失敗",
   "select_reference_variable": "選擇引用變數",
   "share_link": "分享連結",
diff --git a/projects/app/src/components/Markdown/index.tsx b/projects/app/src/components/Markdown/index.tsx
@@ -85,7 +85,7 @@ export default React.memo(Markdown);
 function Code(e: any) {
   const { className, codeBlock, children } = e;
   const match = /language-(\w+)/.exec(className || '');
-  const codeType = match?.[1];
+  const codeType = match?.[1]?.toLowerCase();
 
   const strChildren = String(children);
 
@@ -105,7 +105,7 @@ function Code(e: any) {
     if (codeType === CodeClassNameEnum.iframe) {
       return <IframeCodeBlock code={strChildren} />;
     }
-    if (codeType && codeType.toLowerCase() === CodeClassNameEnum.html) {
+    if (codeType === CodeClassNameEnum.html || codeType === CodeClassNameEnum.svg) {
       return (
         <IframeHtmlCodeBlock className={className} codeBlock={codeBlock} match={match}>
           {children}
diff --git a/projects/app/src/components/Markdown/utils.ts b/projects/app/src/components/Markdown/utils.ts
@@ -8,6 +8,7 @@ export enum CodeClassNameEnum {
   latex = 'latex',
   iframe = 'iframe',
   html = 'html',
+  svg = 'svg',
   video = 'video',
   audio = 'audio'
 }
diff --git a/projects/app/src/components/core/app/DatasetParamsModal.tsx b/projects/app/src/components/core/app/DatasetParamsModal.tsx
@@ -1,5 +1,17 @@
 import React, { useEffect, useMemo, useState } from 'react';
-import { Box, Button, Flex, HStack, ModalBody, ModalFooter, Switch } from '@chakra-ui/react';
+import {
+  Box,
+  Button,
+  Flex,
+  HStack,
+  ModalBody,
+  ModalFooter,
+  Switch,
+  Slider,
+  SliderTrack,
+  SliderFilledTrack,
+  SliderThumb
+} from '@chakra-ui/react';
 import { useForm } from 'react-hook-form';
 import MyModal from '@fastgpt/web/components/common/MyModal';
 import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
@@ -17,6 +29,7 @@ import { defaultDatasetMaxTokens } from '@fastgpt/global/core/app/constants';
 import InputSlider from '@fastgpt/web/components/common/MySlider/InputSlider';
 import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
 import { AppDatasetSearchParamsType } from '@fastgpt/global/core/app/type';
+import MyIcon from '@fastgpt/web/components/common/Icon';
 
 enum SearchSettingTabEnum {
   searchMode = 'searchMode',
@@ -28,6 +41,7 @@ const DatasetParamsModal = ({
   searchMode = DatasetSearchModeEnum.embedding,
   limit,
   similarity,
+  embeddingWeight,
   usingReRank,
   rerankModel,
   rerankWeight,
@@ -62,19 +76,26 @@ const DatasetParamsModal = ({
   const { register, setValue, getValues, handleSubmit, watch } =
     useForm<AppDatasetSearchParamsType>({
       defaultValues: {
-        limit,
-        similarity,
         searchMode,
+        embeddingWeight: embeddingWeight || 0.5,
         usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
         rerankModel: rerankModel || defaultModels?.rerank?.model,
         rerankWeight: rerankWeight || 0.5,
+        limit,
+        similarity,
         datasetSearchUsingExtensionQuery,
         datasetSearchExtensionModel: datasetSearchExtensionModel || defaultModels.llm?.model,
         datasetSearchExtensionBg
       }
     });
 
   const searchModeWatch = watch('searchMode');
+  const embeddingWeightWatch = watch('embeddingWeight');
+  const fullTextWeightWatch = useMemo(() => {
+    const val = 1 - (embeddingWeightWatch || 0.5);
+    return Number(val.toFixed(2));
+  }, [embeddingWeightWatch]);
+
   const datasetSearchUsingCfrForm = watch('datasetSearchUsingExtensionQuery');
   const queryExtensionModel = watch('datasetSearchExtensionModel');
 
@@ -168,8 +189,45 @@ const DatasetParamsModal = ({
                 {
                   title: t('common:core.dataset.search.mode.mixedRecall'),
                   desc: t('common:core.dataset.search.mode.mixedRecall desc'),
-                  value: DatasetSearchModeEnum.mixedRecall
-                  // children: searchModeWatch === DatasetSearchModeEnum.mixedRecall && <Box>111</Box>
+                  value: DatasetSearchModeEnum.mixedRecall,
+                  children: searchModeWatch === DatasetSearchModeEnum.mixedRecall && (
+                    <Box mt={3}>
+                      <HStack justifyContent={'space-between'}>
+                        <Flex alignItems={'center'}>
+                          <Box fontSize={'sm'} color={'myGray.900'}>
+                            {t('common:core.dataset.search.mode.embedding')}
+                          </Box>
+                          <Box fontSize={'xs'} color={'myGray.500'}>
+                            {embeddingWeightWatch}
+                          </Box>
+                        </Flex>
+                        <Flex alignItems={'center'}>
+                          <Box fontSize={'sm'} color={'myGray.900'}>
+                            {t('common:core.dataset.search.score.fullText')}
+                          </Box>
+                          <Box fontSize={'xs'} color={'myGray.500'}>
+                            {fullTextWeightWatch}
+                          </Box>
+                        </Flex>
+                      </HStack>
+                      <Slider
+                        defaultValue={embeddingWeightWatch}
+                        min={0.1}
+                        max={0.9}
+                        step={0.01}
+                        onChange={(e) => {
+                          setValue('embeddingWeight', Number(e.toFixed(2)));
+                        }}
+                      >
+                        <SliderTrack bg={'#F9518E'}>
+                          <SliderFilledTrack bg={'#3370FF'} />
+                        </SliderTrack>
+                        <SliderThumb boxShadow={'none'} bg={'none'}>
+                          <MyIcon transform={'translateY(10px)'} name={'sliderTag'} w={'1rem'} />
+                        </SliderThumb>
+                      </Slider>
+                    </Box>
+                  )
                 }
               ]}
               value={searchModeWatch}
@@ -201,7 +259,7 @@ const DatasetParamsModal = ({
                 <>
                   <HStack mt={3} justifyContent={'space-between'}>
                     <Box fontSize={'sm'} flex={'0 0 100px'} color={'myGray.700'}>
-                      {t('app:rerank_weight')}
+                      {t('common:rerank_weight')}
                     </Box>
                     <Box flex={'1 0 0'}>
                       <InputSlider
diff --git a/projects/app/src/components/core/chat/components/WholeResponseModal.tsx b/projects/app/src/components/core/chat/components/WholeResponseModal.tsx
diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts
diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/SelectDatasetParams.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/SelectDatasetParams.tsx
diff --git a/projects/app/src/pageComponents/dataset/detail/Test.tsx b/projects/app/src/pageComponents/dataset/detail/Test.tsx
diff --git a/projects/app/src/pages/api/core/dataset/searchTest.ts b/projects/app/src/pages/api/core/dataset/searchTest.ts
diff --git a/projects/app/src/web/core/app/utils.ts b/projects/app/src/web/core/app/utils.ts

Original file line number	Diff line number	Diff line change
`@@ -185,7 +185,7 @@ export enum SearchScoreTypeEnum {`
`185`	`185`	`}`
`186`	`186`	`export const SearchScoreTypeMap = {`
`187`	`187`	`[SearchScoreTypeEnum.embedding]: {`
`188`		`- label: i18nT('common:core.dataset.search.score.embedding'),`
	`188`	`+ label: i18nT('common:core.dataset.search.mode.embedding'),`
`189`	`189`	`desc: i18nT('common:core.dataset.search.score.embedding desc'),`
`190`	`190`	`showScore: true`
`191`	`191`	`},`