fix: improve llms.txt path generation for docs embedded in the NPM packages

CLAUDE.md

@@ -11,15 +11,16 @@ View AppKit API reference (docs only, NOT for scaffolding):
 
 ```bash
 # ONLY for viewing documentation - do NOT use for init/scaffold
-npx @databricks/appkit docs <path>
+npx @databricks/appkit docs <query>
 ```
 
-**IMPORTANT**: ALWAYS run `npx @databricks/appkit docs` (no path) FIRST to see available pages. DO NOT guess paths - use the index to find correct paths.
+**IMPORTANT**: ALWAYS run `npx @databricks/appkit docs` (no query) FIRST to see the documentation index. DO NOT guess paths - use the index to find correct paths.
 
-Examples of known paths:
-- Root index: `npx @databricks/appkit docs`
-- API reference: `npx @databricks/appkit docs ./docs/docs/api.md`
-- Component docs: `npx @databricks/appkit docs ./docs/docs/api/appkit-ui/components/Sidebar.md`
+Examples:
+- Documentation index: `npx @databricks/appkit docs`
+- View a section: `npx @databricks/appkit docs "appkit-ui API reference"`
+- Full index (all API entries): `npx @databricks/appkit docs --full`
+- View specific doc: `npx @databricks/appkit docs ./docs/plugins/analytics.md`
 
 ## Repository Structure
 
@@ -183,7 +184,7 @@ pnpm release:ci
 
 ### Plugin System
 
-For full props API, see: `npx @databricks/appkit docs ./docs/docs/plugins.md`.
+For full props API, see: `npx @databricks/appkit docs ./docs/plugins.md`.
 
 ### Execution Interceptor Pattern
 

docs/docs/development/ai-assisted-development.mdx

@@ -22,7 +22,9 @@ There are two ways to access the documentation:
 2. Use the `npx @databricks/appkit docs` command to view the documentation in the terminal.
 
   ```bash
-  npx @databricks/appkit docs # returns the llms.txt file content with all available documentation pages
-  npx @databricks/appkit docs <path> # for specific documentation file
+  npx @databricks/appkit docs            # documentation index with section overview
+  npx @databricks/appkit docs "appkit-ui API reference"  # view a specific section
+  npx @databricks/appkit docs --full     # full index with all API entries
+  npx @databricks/appkit docs ./docs.md  # view a specific documentation file
   ```
 

docs/docs/development/llm-guide.md

@@ -54,15 +54,16 @@ View API reference (docs only, NOT for scaffolding):
 
 ```bash
 # ONLY for viewing documentation - do NOT use for init/scaffold
-npx @databricks/appkit docs <path>
+npx @databricks/appkit docs <query>
 ```
 
-**IMPORTANT**: ALWAYS run `npx @databricks/appkit docs` (no path) FIRST to see available pages. DO NOT guess paths - use the index to find correct paths.
+**IMPORTANT**: ALWAYS run `npx @databricks/appkit docs` (no query) FIRST to see the documentation index. DO NOT guess paths - use the index to find correct paths.
 
-Examples of known paths:
-- Root index: `npx @databricks/appkit docs`
-- API reference: `npx @databricks/appkit docs ./docs/docs/api.md`
-- Component docs: `npx @databricks/appkit docs ./docs/docs/api/appkit-ui/components/Sidebar.md`
+Examples:
+- Documentation index: `npx @databricks/appkit docs`
+- View a section: `npx @databricks/appkit docs "appkit-ui API reference"`
+- Full index (all API entries): `npx @databricks/appkit docs --full`
+- View specific doc: `npx @databricks/appkit docs ./docs/plugins/analytics.md`
 
 ## LLM checklist (before finalizing code)
 

docs/docusaurus.config.ts

@@ -164,6 +164,28 @@ const config: Config = {
           siteDescription:
             "Node.js + React SDK for Databricks Apps. Built for humans and AI.",
           enableLlmsFullTxt: true,
+          autoSectionDepth: 3,
+          autoSectionPosition: 2,
+          sections: [
+            {
+              id: "docs",
+              name: "General docs",
+              position: 1,
+              routes: [{ route: "/appkit/docs/*" }],
+            },
+            {
+              id: "appkit-api",
+              name: "appkit API reference [collapsed]",
+              position: 100,
+              routes: [{ route: "/appkit/docs/api/appkit/**" }],
+            },
+            {
+              id: "appkit-ui-api",
+              name: "appkit-ui API reference [collapsed]",
+              position: 101,
+              routes: [{ route: "/appkit/docs/api/appkit-ui/**" }],
+            },
+          ],
         },
         ui: {
           copyPageContent: {

packages/appkit-ui/package.json

@@ -12,6 +12,7 @@
     "bin",
     "scripts",
     "docs",
+    "docs.md",
     "CLAUDE.md",
     "llms.txt",
     "README.md",

packages/appkit/package.json

@@ -14,6 +14,7 @@
     "bin",
     "scripts",
     "docs",
+    "docs.md",
     "CLAUDE.md",
     "llms.txt",
     "README.md",

packages/shared/src/cli/commands/docs.ts

@@ -6,6 +6,14 @@ import { Command } from "commander";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
+const COLLAPSED_MARKER = "[collapsed]";
+
+interface Section {
+  name: string;
+  body: string;
+  collapsed: boolean;
+}
+
 function findPackageRoot(): string {
   let dir = __dirname;
   while (dir !== path.parse(dir).root) {
@@ -17,50 +25,123 @@ function findPackageRoot(): string {
   throw new Error("Could not find package root");
 }
 
-function runDocs(docPath?: string) {
-  const packageRoot = findPackageRoot();
+function parseSections(content: string): {
+  header: string;
+  sections: Section[];
+} {
+  const parts = content.split(/^(## .+)$/m);
+  const header = parts[0];
+  const sections: Section[] = [];
 
-  if (!docPath) {
-    // Display llms.txt by default
-    const llmsPath = path.join(packageRoot, "llms.txt");
+  for (let i = 1; i < parts.length; i += 2) {
+    const rawName = parts[i].replace(/^## /, "");
+    const body = parts[i + 1] ?? "";
+    const collapsed = rawName.includes(COLLAPSED_MARKER);
+    const name = rawName.replace(COLLAPSED_MARKER, "").trim();
+    sections.push({ name, body, collapsed });
+  }
 
-    if (!fs.existsSync(llmsPath)) {
-      console.error("Error: llms.txt not found in package");
-      process.exit(1);
-    }
+  return { header, sections };
+}
 
-    const content = fs.readFileSync(llmsPath, "utf-8");
-    console.log(content);
-    return;
+function countPages(body: string): number {
+  return (body.match(/^- \[/gm) || []).length;
+}
+
+function findSections(sections: Section[], query: string): Section[] {
+  const q = query.toLowerCase();
+  return sections.filter((s) => s.name.toLowerCase().includes(q));
+}
+
+function isFilePath(arg: string): boolean {
+  return arg.startsWith("./") && arg.endsWith(".md");
+}
+
+function readLlmsTxt(packageRoot: string): string {
+  const llmsPath = path.join(packageRoot, "llms.txt");
+  if (!fs.existsSync(llmsPath)) {
+    console.error("Error: llms.txt not found in package");
+    process.exit(1);
   }
+  return fs.readFileSync(llmsPath, "utf-8");
+}
 
-  // Handle path - remove leading ./ and / first, then strip prefixes
+function readDocFile(packageRoot: string, docPath: string): void {
   let normalizedPath = docPath;
-
-  // Strip leading ./ or /
   normalizedPath = normalizedPath.replace(/^\.\//, "");
   normalizedPath = normalizedPath.replace(/^\//, "");
+  normalizedPath = normalizedPath.replace(/^appkit\//, "");
 
-  // Remove /appkit/docs/ or docs/ prefix since files are in packageRoot/docs/
-  normalizedPath = normalizedPath.replace(/^appkit\/docs\//, "");
-  normalizedPath = normalizedPath.replace(/^docs\//, "");
-
-  const fullPath = path.join(packageRoot, "docs", normalizedPath);
+  const fullPath = path.join(packageRoot, normalizedPath);
 
   if (!fs.existsSync(fullPath)) {
     console.error(`Error: Documentation file not found: ${docPath}`);
     console.error(`Tried: ${fullPath}`);
     process.exit(1);
   }
 
-  const content = fs.readFileSync(fullPath, "utf-8");
-  console.log(content);
+  console.log(fs.readFileSync(fullPath, "utf-8"));
+}
+
+function formatCollapsedSection(section: Section): string {
+  const pages = countPages(section.body);
+  return [
+    `## ${section.name} (${pages} pages)`,
+    "",
+    `> Use \`appkit docs "${section.name}"\` to expand, or \`appkit docs --full\` to expand all sections.`,
+    "",
+  ].join("\n");
+}
+
+function formatExpandedSection(section: Section): string {
+  return `## ${section.name}${section.body}`;
+}
+
+function runDocs(query: string | undefined, options: { full?: boolean }) {
+  const packageRoot = findPackageRoot();
+
+  if (query && isFilePath(query)) {
+    readDocFile(packageRoot, query);
+    return;
+  }
+
+  const content = readLlmsTxt(packageRoot);
+
+  if (options.full) {
+    console.log(content.replaceAll(` ${COLLAPSED_MARKER}`, ""));
+    return;
+  }
+
+  const { header, sections } = parseSections(content);
+
+  if (query) {
+    const matched = findSections(sections, query);
+    if (matched.length === 0) {
+      const available = sections.map((s) => `  - ${s.name}`).join("\n");
+      console.error(
+        `No section matching "${query}". Available sections:\n${available}`,
+      );
+      process.exit(1);
+    }
+    console.log(matched.map(formatExpandedSection).join("\n"));
+    return;
+  }
+
+  const output =
+    header +
+    sections
+      .map((s) =>
+        s.collapsed ? formatCollapsedSection(s) : formatExpandedSection(s),
+      )
+      .join("\n");
+  console.log(output);
 }
 
 export const docsCommand = new Command("docs")
   .description("Display embedded documentation")
   .argument(
-    "[path]",
-    "Path to specific documentation file (e.g., /appkit/docs/api/appkit-ui/components/Sidebar.md)",
+    "[query]",
+    "Section name (e.g. 'plugins') or path to a doc file (e.g. './docs.md')",
   )
+  .option("--full", "Show complete index including all API reference entries")
   .action(runDocs);

tools/dist-appkit.ts

@@ -102,49 +102,84 @@ if (fs.existsSync(sharedPostinstall)) {
 // Copy documentation from docs/build into tmp/docs/
 const docsBuildPath = path.join(__dirname, "../docs/build");
 
-// Copy all .md files and docs/ subdirectory from docs/build to tmp/docs
+function copyMdFilesRecursive(src: string, dest: string) {
+  for (const entry of fs.readdirSync(src)) {
+    const srcPath = path.join(src, entry);
+    const destPath = path.join(dest, entry);
+    if (fs.statSync(srcPath).isDirectory()) {
+      copyMdFilesRecursive(srcPath, destPath);
+    } else if (entry.endsWith(".md")) {
+      fs.mkdirSync(dest, { recursive: true });
+      fs.copyFileSync(srcPath, destPath);
+    }
+  }
+}
+
+// Flatten docs/build/docs/ .md files into tmp/docs/, plus top-level .md files
 fs.mkdirSync("tmp/docs", { recursive: true });
 
-// Copy all files and directories we want, preserving structure
 const itemsToCopy = fs.readdirSync(docsBuildPath);
 for (const item of itemsToCopy) {
   const sourcePath = path.join(docsBuildPath, item);
   const stat = fs.statSync(sourcePath);
 
-  // Copy .md files and docs directory
-  if (item.endsWith(".md") || item === "docs") {
-    const destPath = path.join("tmp/docs", item);
-    if (stat.isDirectory()) {
-      fs.cpSync(sourcePath, destPath, { recursive: true });
-    } else {
-      fs.copyFileSync(sourcePath, destPath);
+  if (item === "docs" && stat.isDirectory()) {
+    copyMdFilesRecursive(sourcePath, "tmp/docs");
+  } else if (item.endsWith(".md")) {
+    fs.copyFileSync(sourcePath, path.join("tmp", item));
+  }
+}
+
+// Replace Docusaurus URL paths with local relative paths in markdown links.
+function replaceDocPaths(content: string): string {
+  // Matches /appkit/docs/ or /appkit/docs.md after "(" (markdown link position),
+  // captures the "docs/" or "docs.md" portion and rewrites to "./$1".
+  return content.replace(/(?<=\()\/appkit\/(docs(?:\/|\.md))/g, "./$1");
+}
+
+function processDocFile(filePath: string) {
+  fs.writeFileSync(
+    filePath,
+    replaceDocPaths(fs.readFileSync(filePath, "utf-8")),
+  );
+}
+
+function processDocsLinks(dir: string) {
+  for (const entry of fs.readdirSync(dir)) {
+    const fullPath = path.join(dir, entry);
+    if (fs.statSync(fullPath).isDirectory()) {
+      processDocsLinks(fullPath);
+    } else if (entry.endsWith(".md")) {
+      processDocFile(fullPath);
     }
   }
 }
 
-// Process llms.txt (keep existing logic but update path replacement)
-const llmsSourcePath = path.join(docsBuildPath, "llms.txt");
-let llmsContent = fs.readFileSync(llmsSourcePath, "utf-8");
+// Process links in all copied .md files
+processDocsLinks("tmp/docs");
+for (const entry of fs.readdirSync("tmp")) {
+  if (entry.endsWith(".md")) {
+    processDocFile(path.join("tmp", entry));
+  }
+}
 
-// Replace /appkit/ with ./docs/ to match new structure
-llmsContent = llmsContent.replace(/\/appkit\//g, "./docs/");
+// Process llms.txt
+const llmsSourcePath = path.join(docsBuildPath, "llms.txt");
+let llmsContent = replaceDocPaths(fs.readFileSync(llmsSourcePath, "utf-8"));
 
 // Prepend AI agent guidance for navigating documentation
 const agentGuidance = `## For AI Agents/Assistants
 
-To view specific documentation files referenced below, use the appkit CLI:
+The section names and doc paths below can be passed as the \`<query>\` argument:
 
 \`\`\`bash
-npx @databricks/appkit docs <path>
+npx @databricks/appkit docs <query>
 \`\`\`
 
-Examples:
-- View main documentation: \`npx @databricks/appkit docs\`
-- View specific file: \`npx @databricks/appkit docs ./docs/docs.md\`
-- View API reference: \`npx @databricks/appkit docs ./docs/docs/api.md\`
-- View component docs: \`npx @databricks/appkit docs ./docs/docs/api/appkit-ui/components/Sidebar.md\`
-
-The CLI will display the documentation content directly in the terminal.
+- View documentation index: \`npx @databricks/appkit docs\`
+- View a section: \`npx @databricks/appkit docs "appkit-ui API reference"\`
+- Full index (all API entries): \`npx @databricks/appkit docs --full\`
+- View specific doc: \`npx @databricks/appkit docs ./docs/plugins/analytics.md\`
 
 ---