add support for warnings

Author: remorses

docs-website/src/lib/edit-tool.ts

@@ -71,13 +71,14 @@ export function isStrReplaceParameterComplete(
 
 // Export callback argument types
 export type ValidateNewContentArgs = { githubPath: string; content: string }
+export type ValidateNewContentResult = { error?: string; warning?: string } | undefined
 export type GetPageContentArgs = { githubPath: string }
 
 export function createEditExecute({
     fileSystem,
     validateNewContent,
 }: {
-    validateNewContent?: (x: ValidateNewContentArgs) => any
+    validateNewContent?: (x: ValidateNewContentArgs) => Promise<ValidateNewContentResult>
     fileSystem: FileSystemEmulator
 }) {
     const previousEdits = [] as FileUpdate[]
@@ -158,16 +159,20 @@ export function createEditExecute({
                             githubPath: path,
                             content: file_text,
                         })
-                        if (result && result.error) {
+                        if (result?.error) {
                             return {
                                 success: false,
-                                error: `result content is invalid: create: ${result.error}`,
+                                error: `Page not created! The page creation failed with the following error:\n${result.error}`,
                             }
                         }
+                        if (result?.warning) {
+                            await fileSystem.write(path, file_text)
+                            return `Page created with following warning, fix it:\n${result.warning}\n\nFile content:\n\n${file_text}`
+                        }
                     } catch (e: any) {
                         return {
                             success: false,
-                            error: e && e.message ? e.message : String(e),
+                            error: `Page not created! The page creation failed with the following error:\n${e && e.message ? e.message : String(e)}`,
                         }
                     }
                 }
@@ -220,18 +225,20 @@ export function createEditExecute({
                 //     }
                 // }
                 const replacedContent = currentContent.replace(old_str, new_str)
+                let warning: string | undefined
                 if (validateNewContent) {
                     try {
                         const result = await validateNewContent({
                             githubPath: path,
                             content: replacedContent,
                         })
-                        if (result && result.error) {
+                        if (result?.error) {
                             return {
                                 success: false,
                                 error: `result content is invalid: str_replace: ${result.error}`,
                             }
                         }
+                        warning = result?.warning
                     } catch (e: any) {
                         return {
                             success: false,
@@ -256,6 +263,9 @@ export function createEditExecute({
                 if (occurrences > 1) {
                     result += `, notice that you replaced more than one match, if that was not desired undo the change or add back the old content you want to keep`
                 }
+                if (warning) {
+                    result += `\n\nWarning: ${warning}`
+                }
                 result += `\n\n${cleanPatch}`
                 return result
             }
@@ -295,18 +305,20 @@ export function createEditExecute({
                 const insertAt = Math.min(insert_line, lines.length)
                 lines.splice(insertAt, 0, new_str)
                 const newContent = lines.join('\n')
+                let warning: string | undefined
                 if (validateNewContent) {
                     try {
                         const result = await validateNewContent({
                             githubPath: path,
                             content: newContent,
                         })
-                        if (result && result.error) {
+                        if (result?.error) {
                             return {
                                 success: false,
                                 error: `result content is invalid: insert: ${result.error}`,
                             }
                         }
+                        warning = result?.warning
                     } catch (e: any) {
                         return {
                             success: false,
@@ -327,7 +339,12 @@ export function createEditExecute({
                     /\\ No newline at end of file\n?/g,
                     '',
                 )
-                return `Here is the diff of the changes made:\n\n${cleanPatch}`
+                let result = `Here is the diff of the changes made`
+                if (warning) {
+                    result += `\n\nWarning: ${warning}`
+                }
+                result += `:\n\n${cleanPatch}`
+                return result
             }
             case 'undo_edit': {
                 const previous = previousEdits.pop()
@@ -345,7 +362,7 @@ export function createEditExecute({
                             githubPath: path,
                             content: previous.content,
                         })
-                        if (result && result.error) {
+                        if (result?.error) {
                             return {
                                 success: false,
                                 error: `result content is invalid: undo_edit: ${result.error}`,
@@ -387,7 +404,7 @@ export function createEditTool({
     model,
 }: {
     fileSystem: FileSystemEmulator
-    validateNewContent?: (x: ValidateNewContentArgs) => any
+    validateNewContent?: (x: ValidateNewContentArgs) => Promise<ValidateNewContentResult>
     model?: { provider?: string }
 }) {
     const execute = createEditExecute({

website/src/lib/spiceflow-generate-message.tsx

@@ -44,6 +44,7 @@ import {
     editToolParamsSchema,
     fileUpdateSchema,
     type FileUpdate,
+    type ValidateNewContentResult,
 } from 'docs-website/src/lib/edit-tool'
 import { FileSystemEmulator } from './file-system-emulator'
 import { notifyError } from './errors'
@@ -91,6 +92,7 @@ export async function generateSystemMessage({
         await import('../prompts/css-variables.md?raw').then((x) => x.default),
         await import('../prompts/frontmatter.md?raw').then((x) => x.default),
         await import('../prompts/gitchamber.md?raw').then((x) => x.default),
+        await import('../prompts/migrating.md?raw').then((x) => x.default),
         dedent`
         ## fumabase.jsonc
 
@@ -347,22 +349,23 @@ export async function* generateMessageStream({
     const strReplaceEditor = createEditTool({
         fileSystem,
         model: { provider: model.provider },
-        async validateNewContent(x) {
+        async validateNewContent(x): Promise<ValidateNewContentResult> {
             if (githubFolder && !x.githubPath.startsWith(githubFolder)) {
-                throw new Error(
-                    `githubPath should always start with ${githubFolder}. This site is in ${githubFolder} base directory`,
-                )
+                return {
+                    error: `githubPath should always start with ${githubFolder}. This site is in ${githubFolder} base directory`,
+                }
             }
+
+            let warning: string | undefined
+
             if (mdxOrMdRegex.test(x.githubPath)) {
                 const parsed = fm(x.content)
                 const frontmatter = parsed.attributes as any
 
                 if (frontmatter && frontmatter.icon) {
                     const iconName = String(frontmatter.icon)
                     if (!isValidLucideIconName(iconName)) {
-                        throw new Error(
-                            `you used an invalid icon "${iconName}", to see the possible icons fetch the url https://fumabase.com/lucide-icons.json`,
-                        )
+                        warning = `you used an invalid icon "${iconName}", to see the possible icons fetch the url https://fumabase.com/lucide-icons.json`
                     }
                 }
             }
@@ -414,27 +417,29 @@ export async function* generateMessageStream({
                         If you want to reference a page you plan to create later, first create it with empty content and only frontmatter
                         `
 
-                        throw createFormattedError(
+                        const errorMessage = createFormattedError(
                             linkError,
                             x.content,
                             'Link Validation Error',
                             additionalMessage,
                         )
+                        return { error: errorMessage.message }
                     }
                 } catch (error: any) {
                     if (
                         error?.line != null ||
                         error.position?.start?.line != null
                     ) {
                         // Format MDX compilation errors
-                        throw createFormattedError(
+                        const errorMessage = createFormattedError(
                             error as ErrorWithPosition,
                             x.content,
                             'MDX Compilation Error',
                             'Please fix the MDX syntax error and submit the tool call again.',
                         )
+                        return { error: errorMessage.message }
                     }
-                    throw error
+                    return { error: error.message || String(error) }
                 }
             }
             if (x.githubPath.endsWith('.json')) {
@@ -460,14 +465,18 @@ export async function* generateMessageStream({
                     jsonError.line = line
                     jsonError.column = column
 
-                    throw createFormattedError(
+                    const errorMessage = createFormattedError(
                         jsonError,
                         x.content,
                         'JSON Parse Error',
                         'Please fix the JSON syntax error and submit the tool call again.',
                     )
+                    return { error: errorMessage.message }
                 }
             }
+
+            // Return warning if we found one, or undefined if everything is ok
+            return warning ? { warning } : undefined
         },
     })
 
@@ -503,7 +512,7 @@ export async function* generateMessageStream({
                         githubFolder || '.',
                         'fumabase.jsonc',
                     ),
-                    title: 'Use the renderForm tool to update these values',
+                    title: 'Use the updateFumabaseJsonc tool to update these values',
                 })
                 // filePaths.push({ path: 'styles.css', title: 'The CSS styles for the website. Only update this file for advanced CSS customisations' })
                 return printDirectoryTree({

website/src/prompts/agent.md

@@ -2,7 +2,9 @@ You are a professional docs content writer
 
 You have access to tools to edit and create files in the project.
 
-DO NOT THINK too much
+DO NOT THINK.
+
+when the user asks you to do something never reply with `ok, i will do ..., proceed?`. JUST do the thing the user asked you without waiting for confirmation.
 
 You do not have access to terminal commands. You are not in a sandbox terminal environment, you cannot run bash commands or use a filesystem. You instead have to use the tools available to edit files. DO NOT USE bash commands.
 

website/src/prompts/migrating.md

@@ -0,0 +1,28 @@
+## migrating existing websites to fumabase
+
+the user will sometimes ask to use an existing website as the source of a new fumabase website
+
+you will have to migrate the pages from the existing website to fumabase. create a new .mdx page for each page from the old website.
+
+when migrating existing websites, when the user provides you a url of existing website, here is the procedure to follow
+
+- try to fetch the docs website sitemap. this can usually be done fetching /sitemap.xml and following the references with more fetch calls. only consider the results related to the docs website
+- sometimes docs websites providers like Mintlify or GitBook expose the website markdown directly in the website if you append .md to the url query. Try doing this to get up to date markdown
+- Some docs websites are stored in a GitHub repository using markdown files. You can find the GitHub repository by reading the page HTML and looking for "Edit this page on GitHub" links, these will point to the github repository and file of the current markdown file there. If you find links like these you can use gitchamber to read the files in the repository and read them to migrate them to fumabase.
+- If you can't find the source markdown for the pages your only way to migrate the pages will be to manually convert the html to markdown and migrate the pages this way, this should be done as a last resort
+
+NEVER leave comments like `(content continues, converted fully)`. ALWAYS migrate the full content of a page! If the content is too long for a single tool call split the `strReplaceEditor` `insert` tool calls into many calls that append a section of valid markdown each.
+
+when migrating an existing website never add the pages in a folder like `gitbook` or `migrated-website`. Replicate the website structure exactly like the old website, putting pages in the root level folder. NEVER put the migrated pages in a single folder.
+
+### migrating existing .md to fumabase mdx
+
+you may need to fix the pages being migrated, sometimes it's not possible to just copy them as is directly into a fumabase mdx document.
+
+here are the things you may need to do when migrating old website pages to fumabase
+- convert missing mdx components to ones that are available in fumabase (Notice fumabase implements basically all Mintlify components so this should not be necessary in that case)
+- remove html style comments when creating .mdx pages. mdx uses js style comments like `{/* comment */}` instead of `<-- comment -->`
+- if a page is using `<iframe>` elements to display media or videos (urls ends with .mp4 for example) use the video html tag instead of iframe.
+
+
+IMPORTANT: when migrating a page from an existing website do not add this information in the prompt in the frontmatter. Leave the frontmatter prompt in that case

website/src/routes/org.$orgId.site.$siteId.chat.$chatId.create-pr.tsx

@@ -141,7 +141,6 @@ export async function loader({
 }: Route.LoaderArgs) {
     const { userId } = await getSession({ request })
 
-    // Return a promise that will resolve with the PR URL
     const prPromise = createPrSuggestionForChat({
         chatId,
         userId,

website/src/routes/org.$orgId.site.$siteId.tsx

@@ -23,7 +23,6 @@ export async function loader({
         throw new Error('Request aborted')
     }
 
-    // TODO change params to be branchId instead of siteId! then get the branch here
     const [site, chatHistory, siteBranches] = await Promise.all([
         prisma.site.findUnique({
             where: {