improve analyze references cmd and remove orphaned file usage

Author: cbullinger

audit-cli/README.md

@@ -308,7 +308,6 @@ This command helps writers:
 - Understand the impact of changes to a file (what pages will be affected)
 - Find all usages of an include file across the documentation
 - Track where code examples are referenced
-- Identify orphaned files (files with no references from content inclusion directives)
 - Plan refactoring by understanding file dependencies
 
 **Basic Usage:**
@@ -336,8 +335,10 @@ This command helps writers:
 - `-v, --verbose` - Show detailed information including line numbers and reference paths
 - `-c, --count-only` - Only show the count of references (useful for quick checks and scripting)
 - `--paths-only` - Only show the file paths, one per line (useful for piping to other commands)
+- `--summary` - Only show summary statistics (total files and references by type, without file list)
 - `-t, --directive-type <type>` - Filter by directive type: `include`, `literalinclude`, `io-code-block`, or `toctree`
 - `--include-toctree` - Include toctree entries (navigation links) in addition to content inclusion directives
+- `--exclude <pattern>` - Exclude paths matching this glob pattern (e.g., `*/archive/*` or `*/deprecated/*`)
 
 **Understanding the Counts:**
 
@@ -388,7 +389,8 @@ With `--include-toctree`, also tracks:
       getting-started
    ```
 
-**Note:** Only file-based references are tracked. Inline content (e.g., `.. input::` with `:language:` but no file path) aly
+**Note:** Only file-based references are tracked. Inline content (e.g., `.. input::` with `:language:` but no file path) is not tracked since it doesn't reference external files.
+
 **Output Formats:**
 
 **Text** (default):
@@ -440,22 +442,22 @@ include             : 3 files, 4 references
   "total_references": 4,
   "referencing_files": [
     {
-      "FilePath": "/path/to/duplicate-include-test.rst",
-      "DirectiveType": "include",
-      "ReferencePath": "/includes/intro.rst",
-      "LineNumber": 6
+      "file_path": "/path/to/duplicate-include-test.rst",
+      "directive_type": "include",
+      "reference_path": "/includes/intro.rst",
+      "line_number": 6
     },
     {
-      "FilePath": "/path/to/duplicate-include-test.rst",
-      "DirectiveType": "include",
-      "ReferencePath": "/includes/intro.rst",
-      "LineNumber": 13
+      "file_path": "/path/to/duplicate-include-test.rst",
+      "directive_type": "include",
+      "reference_path": "/includes/intro.rst",
+      "line_number": 13
     },
     {
-      "FilePath": "/path/to/include-test.rst",
-      "DirectiveType": "include",
-      "ReferencePath": "/includes/intro.rst",
-      "LineNumber": 6
+      "file_path": "/path/to/include-test.rst",
+      "directive_type": "include",
+      "reference_path": "/includes/intro.rst",
+      "line_number": 6
     }
   ]
 }
@@ -480,6 +482,15 @@ include             : 3 files, 4 references
 ./audit-cli analyze file-references ~/docs/source/includes/fact.rst --count-only
 # Output: 5
 
+# Show summary statistics only
+./audit-cli analyze file-references ~/docs/source/includes/fact.rst --summary
+# Output:
+# Total Files: 3
+# Total References: 5
+#
+# By Type:
+#   include             : 3 files, 5 references
+
 # Get list of files for piping to other commands
 ./audit-cli analyze file-references ~/docs/source/includes/fact.rst --paths-only
 # Output:
@@ -498,6 +509,10 @@ include             : 3 files, 4 references
 
 # Combine filters: list files that use this as an io-code-block
 ./audit-cli analyze file-references ~/docs/source/code-examples/query.js -t io-code-block --paths-only
+
+# Exclude archived or deprecated files from search
+./audit-cli analyze file-references ~/docs/source/includes/fact.rst --exclude "*/archive/*"
+./audit-cli analyze file-references ~/docs/source/includes/fact.rst --exclude "*/deprecated/*"
 ```
 
 ### Compare Commands

audit-cli/commands/analyze/file-references/analyzer.go

@@ -25,11 +25,18 @@ import (
 // Parameters:
 //   - targetFile: Absolute path to the file to analyze
 //   - includeToctree: If true, include toctree entries in the search
+//   - verbose: If true, show progress information
+//   - excludePattern: Glob pattern for paths to exclude (empty string means no exclusion)
 //
 // Returns:
 //   - *ReferenceAnalysis: The analysis results
 //   - error: Any error encountered during analysis
-func AnalyzeReferences(targetFile string, includeToctree bool) (*ReferenceAnalysis, error) {
+func AnalyzeReferences(targetFile string, includeToctree bool, verbose bool, excludePattern string) (*ReferenceAnalysis, error) {
+	// Check if target file exists
+	if _, err := os.Stat(targetFile); os.IsNotExist(err) {
+		return nil, fmt.Errorf("target file does not exist: %s\n\nPlease check:\n  - The file path is correct\n  - The file hasn't been moved or deleted\n  - You have permission to access the file", targetFile)
+	}
+
 	// Get absolute path
 	absTargetFile, err := filepath.Abs(targetFile)
 	if err != nil {
@@ -39,7 +46,7 @@ func AnalyzeReferences(targetFile string, includeToctree bool) (*ReferenceAnalys
 	// Find the source directory
 	sourceDir, err := pathresolver.FindSourceDirectory(absTargetFile)
 	if err != nil {
-		return nil, fmt.Errorf("failed to find source directory: %w", err)
+		return nil, fmt.Errorf("failed to find source directory: %w\n\nThe source directory is detected by looking for a 'source' directory in the file's path.\nMake sure the target file is within a documentation repository with a 'source' directory.", err)
 	}
 
 	// Initialize analysis result
@@ -49,6 +56,15 @@ func AnalyzeReferences(targetFile string, includeToctree bool) (*ReferenceAnalys
 		ReferencingFiles: []FileReference{},
 	}
 
+	// Track if we found any RST/YAML files
+	foundAnyFiles := false
+	filesProcessed := 0
+
+	// Show progress message if verbose
+	if verbose {
+		fmt.Fprintf(os.Stderr, "Scanning for references in %s...\n", sourceDir)
+	}
+
 	// Walk through all RST and YAML files in the source directory
 	err = filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -67,6 +83,26 @@ func AnalyzeReferences(targetFile string, includeToctree bool) (*ReferenceAnalys
 			return nil
 		}
 
+		// Check if path should be excluded
+		if excludePattern != "" {
+			matched, err := filepath.Match(excludePattern, path)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: invalid exclude pattern: %v\n", err)
+			} else if matched {
+				// Skip this file
+				return nil
+			}
+		}
+
+		// Mark that we found at least one file
+		foundAnyFiles = true
+		filesProcessed++
+
+		// Show progress every 100 files if verbose
+		if verbose && filesProcessed%100 == 0 {
+			fmt.Fprintf(os.Stderr, "Processed %d files...\n", filesProcessed)
+		}
+
 		// Search for references in this file
 		refs, err := findReferencesInFile(path, absTargetFile, sourceDir, includeToctree)
 		if err != nil {
@@ -85,6 +121,16 @@ func AnalyzeReferences(targetFile string, includeToctree bool) (*ReferenceAnalys
 		return nil, fmt.Errorf("failed to walk source directory: %w", err)
 	}
 
+	// Check if we found any RST/YAML files
+	if !foundAnyFiles {
+		return nil, fmt.Errorf("no RST or YAML files found in source directory: %s\n\nThis might not be a documentation repository.\nExpected to find files with extensions: .rst, .txt, .yaml, .yml", sourceDir)
+	}
+
+	// Show completion message if verbose
+	if verbose {
+		fmt.Fprintf(os.Stderr, "Scan complete. Processed %d files.\n", filesProcessed)
+	}
+
 	// Update total counts
 	analysis.TotalReferences = len(analysis.ReferencingFiles)
 	analysis.TotalFiles = countUniqueFiles(analysis.ReferencingFiles)

audit-cli/commands/analyze/file-references/file_references.go

@@ -11,7 +11,6 @@
 //   - Understanding the impact of changes to a file
 //   - Finding all usages of an include file
 //   - Tracking code example references
-//   - Identifying orphaned files (files with no references, including toctree entries)
 package filereferences
 
 import (
@@ -41,8 +40,10 @@ func NewFileReferencesCommand() *cobra.Command {
 		verbose        bool
 		countOnly      bool
 		pathsOnly      bool
+		summaryOnly    bool
 		directiveType  string
 		includeToctree bool
+		excludePattern string
 	)
 
 	cmd := &cobra.Command{
@@ -69,7 +70,6 @@ This is useful for:
   - Understanding the impact of changes to a file
   - Finding all usages of an include file
   - Tracking code example references
-  - Identifying orphaned files (files with no references from content inclusion directives)
 
 Examples:
   # Find what references an include file
@@ -93,20 +93,28 @@ Examples:
   # Just show the file paths
   analyze file-references /path/to/file.rst --paths-only
 
+  # Show summary statistics only
+  analyze file-references /path/to/file.rst --summary
+
+  # Exclude certain paths from search
+  analyze file-references /path/to/file.rst --exclude "*/archive/*"
+
   # Filter by directive type
   analyze file-references /path/to/file.rst --directive-type include`,
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return runReferences(args[0], format, verbose, countOnly, pathsOnly, directiveType, includeToctree)
+			return runReferences(args[0], format, verbose, countOnly, pathsOnly, summaryOnly, directiveType, includeToctree, excludePattern)
 		},
 	}
 
 	cmd.Flags().StringVar(&format, "format", "text", "Output format (text or json)")
 	cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Show detailed information including line numbers")
 	cmd.Flags().BoolVarP(&countOnly, "count-only", "c", false, "Only show the count of references")
 	cmd.Flags().BoolVar(&pathsOnly, "paths-only", false, "Only show the file paths (one per line)")
+	cmd.Flags().BoolVar(&summaryOnly, "summary", false, "Only show summary statistics (total files and references by type)")
 	cmd.Flags().StringVarP(&directiveType, "directive-type", "t", "", "Filter by directive type (include, literalinclude, io-code-block, toctree)")
 	cmd.Flags().BoolVar(&includeToctree, "include-toctree", false, "Include toctree entries (navigation links) in addition to content inclusion directives")
+	cmd.Flags().StringVar(&excludePattern, "exclude", "", "Exclude paths matching this glob pattern (e.g., '*/archive/*' or '*/deprecated/*')")
 
 	return cmd
 }
@@ -121,12 +129,14 @@ Examples:
 //   - verbose: If true, show detailed information
 //   - countOnly: If true, only show the count
 //   - pathsOnly: If true, only show the file paths
+//   - summaryOnly: If true, only show summary statistics
 //   - directiveType: Filter by directive type (empty string means all types)
 //   - includeToctree: If true, include toctree entries in the search
+//   - excludePattern: Glob pattern for paths to exclude (empty string means no exclusion)
 //
 // Returns:
 //   - error: Any error encountered during analysis
-func runReferences(targetFile, format string, verbose, countOnly, pathsOnly bool, directiveType string, includeToctree bool) error {
+func runReferences(targetFile, format string, verbose, countOnly, pathsOnly, summaryOnly bool, directiveType string, includeToctree bool, excludePattern string) error {
 	// Validate directive type if specified
 	if directiveType != "" {
 		validTypes := map[string]bool{
@@ -147,15 +157,25 @@ func runReferences(targetFile, format string, verbose, countOnly, pathsOnly bool
 	}
 
 	// Validate flag combinations
-	if countOnly && pathsOnly {
-		return fmt.Errorf("cannot use --count-only and --paths-only together")
+	exclusiveFlags := 0
+	if countOnly {
+		exclusiveFlags++
+	}
+	if pathsOnly {
+		exclusiveFlags++
 	}
-	if (countOnly || pathsOnly) && outputFormat == FormatJSON {
-		return fmt.Errorf("--count-only and --paths-only are not compatible with --format json")
+	if summaryOnly {
+		exclusiveFlags++
+	}
+	if exclusiveFlags > 1 {
+		return fmt.Errorf("cannot use --count-only, --paths-only, and --summary together")
+	}
+	if (countOnly || pathsOnly || summaryOnly) && outputFormat == FormatJSON {
+		return fmt.Errorf("--count-only, --paths-only, and --summary are not compatible with --format json")
 	}
 
 	// Perform analysis
-	analysis, err := AnalyzeReferences(targetFile, includeToctree)
+	analysis, err := AnalyzeReferences(targetFile, includeToctree, verbose, excludePattern)
 	if err != nil {
 		return fmt.Errorf("failed to analyze references: %w", err)
 	}
@@ -176,6 +196,11 @@ func runReferences(targetFile, format string, verbose, countOnly, pathsOnly bool
 		return PrintPathsOnly(analysis)
 	}
 
+	// Handle summary-only output
+	if summaryOnly {
+		return PrintSummary(analysis)
+	}
+
 	// Print full results
 	return PrintAnalysis(analysis, outputFormat, verbose)
 }

audit-cli/commands/analyze/file-references/file_references_test.go

@@ -51,8 +51,8 @@ func TestAnalyzeReferences(t *testing.T) {
 				t.Fatalf("failed to get absolute path: %v", err)
 			}
 
-			// Run analysis (without toctree by default)
-			analysis, err := AnalyzeReferences(absTargetPath, false)
+			// Run analysis (without toctree by default, not verbose, no exclude pattern)
+			analysis, err := AnalyzeReferences(absTargetPath, false, false, "")
 			if err != nil {
 				t.Fatalf("AnalyzeReferences failed: %v", err)
 			}

audit-cli/commands/analyze/file-references/output.go

@@ -51,14 +51,23 @@ func printText(analysis *ReferenceAnalysis, verbose bool) {
 	if analysis.TotalReferences == 0 {
 		fmt.Println("No files reference this file.")
 		fmt.Println()
+		fmt.Println("This could mean:")
+		fmt.Println("  - The file is not included in any documentation pages")
+		fmt.Println("  - The file might be orphaned (not used)")
+		fmt.Println("  - The file is referenced using a different path")
+		fmt.Println()
+		fmt.Println("Note: By default, only content inclusion directives are searched.")
+		fmt.Println("Use --include-toctree to also search for toctree navigation links.")
+		fmt.Println()
 		return
 	}
 
 	// Group references by directive type
 	byDirectiveType := groupByDirectiveType(analysis.ReferencingFiles)
 
 	// Print breakdown by directive type with file and reference counts
-	for _, directiveType := range []string{"include", "literalinclude", "io-code-block"} {
+	directiveTypes := []string{"include", "literalinclude", "io-code-block", "toctree"}
+	for _, directiveType := range directiveTypes {
 		if refs, ok := byDirectiveType[directiveType]; ok {
 			uniqueFiles := countUniqueFiles(refs)
 			totalRefs := len(refs)
@@ -213,3 +222,35 @@ func PrintPathsOnly(analysis *ReferenceAnalysis) error {
 	return nil
 }
 
+// PrintSummary prints only summary statistics without the file list.
+//
+// This is useful for getting a quick overview of reference counts.
+//
+// Parameters:
+//   - analysis: The analysis results
+//
+// Returns:
+//   - error: Any error encountered during printing
+func PrintSummary(analysis *ReferenceAnalysis) error {
+	fmt.Printf("Total Files: %d\n", analysis.TotalFiles)
+	fmt.Printf("Total References: %d\n", analysis.TotalReferences)
+
+	if analysis.TotalReferences > 0 {
+		// Group by directive type
+		byDirectiveType := groupByDirectiveType(analysis.ReferencingFiles)
+
+		// Print breakdown by type
+		fmt.Println("\nBy Type:")
+		directiveTypes := []string{"include", "literalinclude", "io-code-block", "toctree"}
+		for _, directiveType := range directiveTypes {
+			if refs, ok := byDirectiveType[directiveType]; ok {
+				uniqueFiles := countUniqueFiles(refs)
+				totalRefs := len(refs)
+				fmt.Printf("  %-20s: %d files, %d references\n", directiveType, uniqueFiles, totalRefs)
+			}
+		}
+	}
+
+	return nil
+}
+

audit-cli/commands/analyze/file-references/types.go

@@ -30,17 +30,17 @@ type ReferenceAnalysis struct {
 // This structure captures details about how and where the reference occurs.
 type FileReference struct {
 	// FilePath is the absolute path to the file that references the target
-	FilePath string
+	FilePath string `json:"file_path"`
 
 	// DirectiveType is the type of directive used to reference the file
 	// Possible values: "include", "literalinclude", "io-code-block", "toctree"
-	DirectiveType string
+	DirectiveType string `json:"directive_type"`
 
 	// ReferencePath is the path used in the directive (as written in the file)
-	ReferencePath string
+	ReferencePath string `json:"reference_path"`
 
 	// LineNumber is the line number where the reference occurs
-	LineNumber int
+	LineNumber int `json:"line_number"`
 }
 
 // ReferenceNode represents a node in the reference tree.