diff --git a/app/web_ui/src/lib/utils/formatters.ts b/app/web_ui/src/lib/utils/formatters.ts index 436d12773..a8abb3a70 100644 --- a/app/web_ui/src/lib/utils/formatters.ts +++ b/app/web_ui/src/lib/utils/formatters.ts @@ -1,4 +1,5 @@ import { + type ChunkerConfig, type ChunkerType, type EvalConfigType, type OutputFormat, @@ -164,6 +165,22 @@ export function chunker_type_format(chunker_type: ChunkerType): string { } } +export function format_chunker_config_overview(config: ChunkerConfig) { + const props = config.properties + switch (config.chunker_type) { + case "fixed_window": + return `${chunker_type_format(config.chunker_type)} • Size: ${props.chunk_size || "N/A"} words • Overlap: ${props.chunk_overlap || "N/A"} words` + case "semantic": + return `${chunker_type_format(config.chunker_type)} • Buffer: ${props.buffer_size || "N/A"} • Threshold: ${props.breakpoint_percentile_threshold || "N/A"}` + default: { + // type check will catch missing cases + const unknownChunkerType: never = config.chunker_type + console.error(`Invalid chunker type: ${unknownChunkerType}`) + return "unknown" + } + } +} + export function capitalize(str: string | undefined | null): string { if (!str) { return "" diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/+page.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/+page.svelte index 16e163b80..99b313792 100644 --- a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/+page.svelte +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/+page.svelte @@ -29,6 +29,7 @@ import Warning from "$lib/ui/warning.svelte" import posthog from "posthog-js" import { uncache_available_tools } from "$lib/stores" + import { goto } from "$app/navigation" $: project_id = $page.params.project_id $: rag_config_id = $page.params.rag_config_id @@ -241,6 +242,14 @@ }, }, ]), + { + label: "Clone", + handler: () => { + goto( + `/docs/rag_configs/${project_id}/${rag_config_id}/rag_config/clone`, + ) + }, + }, { label: rag_config?.is_archived ? "Unarchive" : "Archive", primary: rag_config?.is_archived, diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/clone/+page.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/clone/+page.svelte new file mode 100644 index 000000000..1296c83b5 --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/clone/+page.svelte @@ -0,0 +1,92 @@ + + +
+ + {#if loading} +
+
+
+ {:else if error} +
+
{error.message}
+
+ {:else if !rag_config} +
+
Search Tool not found
+
+ {:else} + { + goto(`/docs/rag_configs/${project_id}`) + }} + /> + {/if} +
+
diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/clone/+page.ts b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/clone/+page.ts new file mode 100644 index 000000000..9786e09d9 --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/[rag_config_id]/rag_config/clone/+page.ts @@ -0,0 +1 @@ +export const prerender = false diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/+page.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/+page.svelte index 1b671e3ca..d3b98e04f 100644 --- a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/+page.svelte +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/+page.svelte @@ -1,941 +1,49 @@ - - {#if loading} -
-
-
- {:else} -
- - -
Part 1: Tool Properties
- - - - -
- (selected_tags = e.detail.selected_tags)} - /> -
- -
-
Part 2: Search Configuration
-
- This configuration controls how the search tool will extract, index, - and search your documents. - {#if template && !customize_template_mode} - - {/if} -
-
- {#if template && !customize_template_mode} -
-
- -
- -
Advanced
-
-
-
- {:else} -
- -
- {#if loading_extractor_configs} -
-
- Loading extractors... -
- {:else} - - {/if} -
- - -
- {#if loading_chunker_configs} -
-
- Loading chunkers... -
- {:else} - - {/if} -
- - -
- {#if loading_embedding_configs} -
-
- Loading embedding models... -
- {:else} - - {/if} -
- - -
- {#if loading_vector_store_configs} -
-
- Loading vector stores... -
- {:else} - - {/if} -
-
- - - - - - - {/if} -
-
- {/if} -
- - { - handle_modal_close() - if (selected_extractor_config_id === "create_new") { - selected_extractor_config_id = null - } - }} -> - { - await loadExtractorConfigs() - selected_extractor_config_id = e.detail.extractor_config_id - }} - /> - - - { - handle_modal_close() - if (selected_chunker_config_id === "create_new") { - selected_chunker_config_id = null - } - }} -> - { - await loadChunkerConfigs() - selected_chunker_config_id = e.detail.chunker_config_id - }} - /> - - - { - handle_modal_close() - if (selected_embedding_config_id === "create_new") { - selected_embedding_config_id = null - } - }} -> - { - await loadEmbeddingConfigs() - selected_embedding_config_id = e.detail.embedding_config_id - }} - /> - - - { - handle_modal_close() - if (selected_vector_store_config_id === "create_new") { - selected_vector_store_config_id = null - } - }} -> - { - await loadVectorStoreConfigs() - selected_vector_store_config_id = e.detail.vector_store_config_id - }} - /> - +
+ + {#if loading} +
+
+
+ {:else} + { + goto(`/docs/rag_configs/${project_id}`) + }} + /> + {/if} +
+
diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_chunker_dialog.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_chunker_dialog.svelte new file mode 100644 index 000000000..584db9155 --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_chunker_dialog.svelte @@ -0,0 +1,30 @@ + + + { + dispatch("close") + }} +> + { + dispatch("success", { chunker_config_id: e.detail.chunker_config_id }) + }} + /> + diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_embedding_dialog.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_embedding_dialog.svelte new file mode 100644 index 000000000..421b6b059 --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_embedding_dialog.svelte @@ -0,0 +1,30 @@ + + + { + dispatch("close") + }} +> + { + dispatch("success", { embedding_config_id: e.detail.embedding_config_id }) + }} + /> + diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_embedding_form.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_embedding_form.svelte index 104f334db..6f2700baa 100644 --- a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_embedding_form.svelte +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_embedding_form.svelte @@ -18,13 +18,14 @@ $: project_id = $page.params.project_id let loading: boolean = false + let loadingModels = true let error: KilnError | null = null + let name: string = "" let description: string = "" let selectedModel: EmbeddingOptionValue | null = null let customDimensions: number | null = null let embeddingModels: OptionGroup[] = [] - let loadingModels = true export let keyboard_submit: boolean = false type EmbeddingOptionValue = { @@ -46,13 +47,12 @@ async function loadEmbeddingModels() { try { loadingModels = true - const { error: modelsError, data } = await client.GET( + const { error: load_models_error, data } = await client.GET( "/api/available_embedding_models", ) - if (modelsError) { - error = createKilnError(modelsError) - return + if (load_models_error) { + throw load_models_error } // Transform the API response into OptionGroup format @@ -85,14 +85,13 @@ } async function create_embedding_config() { - if (!selectedModel) { - error = createKilnError(new Error("Please select an embedding model")) - return - } - try { loading = true + if (!selectedModel) { + throw new Error("Please select an embedding model") + } + const properties: Record = {} if (customDimensions && selectedModel.supports_custom_dimensions) { properties.dimensions = customDimensions @@ -118,40 +117,41 @@ ) if (create_embedding_error) { - error = createKilnError(create_embedding_error) - return + throw create_embedding_error } if (!data.id) { - error = createKilnError(new Error("Failed to create embedding config")) - return + throw new Error("Failed to create embedding config") } dispatch("success", { embedding_config_id: data.id }) + } catch (err) { + error = createKilnError(err) } finally { loading = false } } - { - await create_embedding_config() - }} - {error} - gap={4} - bind:submitting={loading} - {keyboard_submit} -> +{#if loading || loadingModels}
- {#if loadingModels} -
-
- Loading embedding models... -
- {:else} +
+
+
+
+{:else} + { + await create_embedding_config() + }} + {error} + gap={4} + bind:submitting={loading} + {keyboard_submit} + > +
- {/if} -
- - - - - {#if selectedModel && selectedModel.supports_custom_dimensions} + + - {/if} - -
+ + + {#if selectedModel && selectedModel.supports_custom_dimensions} + + {/if} + +
+{/if} diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_extractor_dialog.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_extractor_dialog.svelte new file mode 100644 index 000000000..3724023d9 --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_extractor_dialog.svelte @@ -0,0 +1,30 @@ + + + { + dispatch("close") + }} +> + { + dispatch("success", { extractor_config_id: e.detail.extractor_config_id }) + }} + /> + diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_vector_store_dialog.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_vector_store_dialog.svelte new file mode 100644 index 000000000..515924f4f --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/create_vector_store_dialog.svelte @@ -0,0 +1,32 @@ + + + { + dispatch("close") + }} +> + { + dispatch("success", { + vector_store_config_id: e.detail.vector_store_config_id, + }) + }} + /> + diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/edit_rag_config_form.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/edit_rag_config_form.svelte new file mode 100644 index 000000000..09381d064 --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/edit_rag_config_form.svelte @@ -0,0 +1,728 @@ + + +{#if loading || loading_subconfig_options} +
+
+
+{:else} + { + if (template && !customize_template_mode) { + await save_template() + } else { + await create_rag_config() + } + }} + {error} + gap={4} + bind:submitting={loading} + keyboard_submit={!modal_opened} + > + +
Part 1: Tool Properties
+ + + + +
+ (selected_tags = e.detail.selected_tags)} + /> +
+ +
+
Part 2: Search Configuration
+
+ This configuration controls how the search tool will extract, index, and + search your documents. + {#if template && !customize_template_mode} + + {/if} +
+
+ + {#if template && !customize_template_mode} + + {:else} +
+ +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+
+ + + + + + + {/if} +
+{/if} + + { + await loadExtractorConfigs() + selected_extractor_config_id = e.detail.extractor_config_id + }} + on:close={() => { + handle_modal_close() + if (selected_extractor_config_id === "create_new") { + selected_extractor_config_id = null + } + show_create_extractor_dialog?.close() + }} +/> + + { + await loadChunkerConfigs() + selected_chunker_config_id = e.detail.chunker_config_id + }} + on:close={() => { + handle_modal_close() + if (selected_chunker_config_id === "create_new") { + selected_chunker_config_id = null + } + show_create_chunker_dialog?.close() + }} +/> + + { + await loadEmbeddingConfigs() + selected_embedding_config_id = e.detail.embedding_config_id + }} + on:close={() => { + handle_modal_close() + if (selected_embedding_config_id === "create_new") { + selected_embedding_config_id = null + } + show_create_embedding_dialog?.close() + }} +/> + + { + await loadVectorStoreConfigs() + selected_vector_store_config_id = e.detail.vector_store_config_id + }} + on:close={() => { + handle_modal_close() + if (selected_vector_store_config_id === "create_new") { + selected_vector_store_config_id = null + } + show_create_vector_store_dialog?.close() + }} +/> diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/options_groups.ts b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/options_groups.ts new file mode 100644 index 000000000..9f0707633 --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/options_groups.ts @@ -0,0 +1,168 @@ +import { + embedding_model_name, + get_model_friendly_name, + provider_name_from_id, +} from "$lib/stores" +import type { + ChunkerConfig, + EmbeddingConfig, + ExtractorConfig, + VectorStoreConfig, +} from "$lib/types" +import type { OptionGroup } from "$lib/ui/fancy_select_types" +import { + extractor_output_format, + format_chunker_config_overview, +} from "$lib/utils/formatters" + +function fmt_extractor_label(extractor: ExtractorConfig) { + return `${get_model_friendly_name(extractor.model_name)} (${provider_name_from_id(extractor.model_provider_name)}) • ${extractor_output_format(extractor.output_format)}` +} + +export function build_extractor_options(extractor_configs: ExtractorConfig[]) { + return [ + { + options: [ + { + label: "New Extractor Configuration", + value: "create_new", + badge: "New", + badge_color: "primary", + }, + ], + }, + ...(extractor_configs.length > 0 + ? [ + { + label: "Extractors", + options: extractor_configs + .filter((config) => !config.is_archived) + .map((config) => ({ + label: fmt_extractor_label(config), + value: config.id, + description: + config.name + + (config.description ? " - " + config.description : ""), + })), + }, + ] + : []), + ] as OptionGroup[] +} + +export function build_chunker_options(chunker_configs: ChunkerConfig[]) { + return [ + { + options: [ + { + label: "New Chunker Configuration", + value: "create_new", + badge: "New", + badge_color: "primary", + }, + ], + }, + ...(chunker_configs.length > 0 + ? [ + { + label: "Chunkers", + options: chunker_configs.map((config) => { + return { + label: format_chunker_config_overview(config), + value: config.id, + description: + config.name + + (config.description ? ` • ${config.description}` : ""), + } + }), + }, + ] + : []), + ] as OptionGroup[] +} + +function fmt_embedding_label(config: EmbeddingConfig) { + return ( + `${embedding_model_name(config.model_name, config.model_provider_name)} (${provider_name_from_id(config.model_provider_name)})` + + (config.properties?.dimensions + ? `• ${config.properties?.dimensions} dimensions` + : "") + ) +} + +export function build_embedding_options(embedding_configs: EmbeddingConfig[]) { + return [ + { + options: [ + { + label: "New Embedding Configuration", + value: "create_new", + badge: "New", + badge_color: "primary", + }, + ], + }, + ...(embedding_configs.length > 0 + ? [ + { + label: "Embedding Models", + options: embedding_configs.map((config) => ({ + label: fmt_embedding_label(config), + value: config.id, + description: + config.name + + (config.description ? " • " + config.description : ""), + })), + }, + ] + : []), + ] as OptionGroup[] +} + +function fmt_vector_store_label(config: VectorStoreConfig) { + switch (config.store_type) { + case "lancedb_fts": + return "Full Text Search" + case "lancedb_vector": + return "Vector Search" + case "lancedb_hybrid": + return "Hybrid Search" + default: { + // type check will catch missing cases + const unknownVectorStoreType: never = config.store_type + console.error(`Invalid vector store type: ${unknownVectorStoreType}`) + return "unknown" + } + } +} + +export function build_vector_store_options( + vector_store_configs: VectorStoreConfig[], +) { + return [ + { + options: [ + { + label: "New Search Index Configuration", + value: "create_new", + badge: "New", + badge_color: "primary", + }, + ], + }, + ...(vector_store_configs.length > 0 + ? [ + { + label: "Search Index Configurations", + options: vector_store_configs.map((config) => ({ + label: fmt_vector_store_label(config), + value: config.id, + description: + config.name + + ` • ${config.properties.similarity_top_k ?? 10} results`, + })), + }, + ] + : []), + ] as OptionGroup[] +} diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/template_property_overview.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/template_property_overview.svelte new file mode 100644 index 000000000..a3978e4c1 --- /dev/null +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/create_rag_config/template_property_overview.svelte @@ -0,0 +1,66 @@ + + +
+
+ +
+ +
Advanced
+
+
+
diff --git a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/table_rag_config_row.svelte b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/table_rag_config_row.svelte index f4a9a1d5f..8f48750f3 100644 --- a/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/table_rag_config_row.svelte +++ b/app/web_ui/src/routes/(app)/docs/rag_configs/[project_id]/table_rag_config_row.svelte @@ -13,6 +13,7 @@ type RagConfigurationStatus, } from "$lib/stores/rag_progress_store" import { goto } from "$app/navigation" + import { format_chunker_config_overview } from "$lib/utils/formatters" $: projectStateStore = getProjectRagStateStore(project_id) $: ragProgressState = $projectStateStore @@ -80,21 +81,6 @@ function open() { goto(`/docs/rag_configs/${project_id}/${rag_config.id}/rag_config`) } - - $: chunk_size = rag_config.chunker_config.properties.chunk_size - $: chunk_overlap = rag_config.chunker_config.properties.chunk_overlap - - function format_chunking(chunk_size: unknown, chunk_overlap: unknown) { - // we expect a non-nullable number for both, but we validate because we - // do not have typing on the properties object - const is_chunk_size_valid = typeof chunk_size === "number" - const is_chunk_overlap_valid = typeof chunk_overlap === "number" - if (!is_chunk_size_valid || !is_chunk_overlap_valid) { - return "Invalid chunk size or overlap, not a number" - } - - return `${chunk_size} words, ${chunk_overlap} overlap` - } {#if rag_progress && rag_config} @@ -121,7 +107,9 @@ ) || ""})
- Chunking: {format_chunking(chunk_size, chunk_overlap) || "N/A"} + Chunking: {format_chunker_config_overview( + rag_config.chunker_config, + )}
Embedding: {embedding_model_name(