Merge branch 'main' into add-github-action-for-nix

Author: jessfraz

codex-rs/Cargo.lock

@@ -175,8 +175,9 @@ tracing = "0.1.41"
 tracing-appender = "0.2.3"
 tracing-subscriber = "0.3.20"
 tracing-test = "0.2.5"
-tree-sitter = "0.25.9"
-tree-sitter-bash = "0.25.0"
+tree-sitter = "0.25.10"
+tree-sitter-bash = "0.25"
+tree-sitter-highlight = "0.25.10"
 ts-rs = "11"
 unicode-segmentation = "1.12.0"
 unicode-width = "0.2"

codex-rs/Cargo.toml

@@ -68,6 +68,8 @@ strum_macros = { workspace = true }
 supports-color = { workspace = true }
 tempfile = { workspace = true }
 textwrap = { workspace = true }
+tree-sitter-highlight = { workspace = true }
+tree-sitter-bash = { workspace = true }
 tokio = { workspace = true, features = [
     "io-std",
     "macros",

codex-rs/tui/Cargo.toml

@@ -1,81 +1,146 @@
-use codex_core::bash::try_parse_bash;
+use ratatui::style::Style;
 use ratatui::style::Stylize;
 use ratatui::text::Line;
 use ratatui::text::Span;
+use std::sync::OnceLock;
+use tree_sitter_highlight::Highlight;
+use tree_sitter_highlight::HighlightConfiguration;
+use tree_sitter_highlight::HighlightEvent;
+use tree_sitter_highlight::Highlighter;
 
-/// Convert the full bash script into per-line styled content by first
-/// computing operator-dimmed spans across the entire script, then splitting
-/// by newlines and dimming heredoc body lines. Performs a single parse and
-/// reuses it for both highlighting and heredoc detection.
-pub(crate) fn highlight_bash_to_lines(script: &str) -> Vec<Line<'static>> {
-    // Parse once; use the tree for both highlighting and heredoc body detection.
-    let spans: Vec<Span<'static>> = if let Some(tree) = try_parse_bash(script) {
-        // Single walk: collect operator ranges and heredoc rows.
-        let root = tree.root_node();
-        let mut cursor = root.walk();
-        let mut stack = vec![root];
-        let mut ranges: Vec<(usize, usize)> = Vec::new();
-        while let Some(node) = stack.pop() {
-            if !node.is_named() && !node.is_extra() {
-                let kind = node.kind();
-                let is_quote = matches!(kind, "\"" | "'" | "`");
-                let is_whitespace = kind.trim().is_empty();
-                if !is_quote && !is_whitespace {
-                    ranges.push((node.start_byte(), node.end_byte()));
-                }
-            } else if node.kind() == "heredoc_body" {
-                ranges.push((node.start_byte(), node.end_byte()));
-            }
-            for child in node.children(&mut cursor) {
-                stack.push(child);
-            }
+// Ref: https://github.com/tree-sitter/tree-sitter-bash/blob/master/queries/highlights.scm
+#[derive(Copy, Clone)]
+enum BashHighlight {
+    Comment,
+    Constant,
+    Embedded,
+    Function,
+    Keyword,
+    Number,
+    Operator,
+    Property,
+    String,
+}
+
+impl BashHighlight {
+    const ALL: [Self; 9] = [
+        Self::Comment,
+        Self::Constant,
+        Self::Embedded,
+        Self::Function,
+        Self::Keyword,
+        Self::Number,
+        Self::Operator,
+        Self::Property,
+        Self::String,
+    ];
+
+    const fn as_str(self) -> &'static str {
+        match self {
+            Self::Comment => "comment",
+            Self::Constant => "constant",
+            Self::Embedded => "embedded",
+            Self::Function => "function",
+            Self::Keyword => "keyword",
+            Self::Number => "number",
+            Self::Operator => "operator",
+            Self::Property => "property",
+            Self::String => "string",
         }
-        if ranges.is_empty() {
-            ranges.push((script.len(), script.len()));
+    }
+
+    fn style(self) -> Style {
+        match self {
+            Self::Comment | Self::Operator | Self::String => Style::default().dim(),
+            _ => Style::default(),
         }
-        ranges.sort_by_key(|(st, _)| *st);
-        let mut spans: Vec<Span<'static>> = Vec::new();
-        let mut i = 0usize;
-        for (start, end) in ranges.into_iter() {
-            let dim_start = start.max(i);
-            let dim_end = end;
-            if dim_start < dim_end {
-                if dim_start > i {
-                    spans.push(script[i..dim_start].to_string().into());
-                }
-                spans.push(script[dim_start..dim_end].to_string().dim());
-                i = dim_end;
-            }
+    }
+}
+
+static HIGHLIGHT_CONFIG: OnceLock<HighlightConfiguration> = OnceLock::new();
+
+fn highlight_names() -> &'static [&'static str] {
+    static NAMES: OnceLock<[&'static str; BashHighlight::ALL.len()]> = OnceLock::new();
+    NAMES
+        .get_or_init(|| BashHighlight::ALL.map(BashHighlight::as_str))
+        .as_slice()
+}
+
+fn highlight_config() -> &'static HighlightConfiguration {
+    HIGHLIGHT_CONFIG.get_or_init(|| {
+        let language = tree_sitter_bash::LANGUAGE.into();
+        #[expect(clippy::expect_used)]
+        let mut config = HighlightConfiguration::new(
+            language,
+            "bash",
+            tree_sitter_bash::HIGHLIGHT_QUERY,
+            "",
+            "",
+        )
+        .expect("load bash highlight query");
+        config.configure(highlight_names());
+        config
+    })
+}
+
+fn highlight_for(highlight: Highlight) -> BashHighlight {
+    BashHighlight::ALL[highlight.0]
+}
+
+fn push_segment(lines: &mut Vec<Line<'static>>, segment: &str, style: Option<Style>) {
+    for (i, part) in segment.split('\n').enumerate() {
+        if i > 0 {
+            lines.push(Line::from(""));
         }
-        if i < script.len() {
-            spans.push(script[i..].to_string().into());
+        if part.is_empty() {
+            continue;
         }
-        spans
-    } else {
-        vec![script.to_string().into()]
-    };
-    // Split spans into lines preserving style boundaries and highlights across newlines.
+        let span = match style {
+            Some(style) => Span::styled(part.to_string(), style),
+            None => part.to_string().into(),
+        };
+        if let Some(last) = lines.last_mut() {
+            last.spans.push(span);
+        }
+    }
+}
+
+/// Convert a bash script into per-line styled content using tree-sitter's
+/// bash highlight query. The highlighter is streamed so multi-line content is
+/// split into `Line`s while preserving style boundaries.
+pub(crate) fn highlight_bash_to_lines(script: &str) -> Vec<Line<'static>> {
+    let mut highlighter = Highlighter::new();
+    let iterator =
+        match highlighter.highlight(highlight_config(), script.as_bytes(), None, |_| None) {
+            Ok(iter) => iter,
+            Err(_) => return vec![script.to_string().into()],
+        };
+
     let mut lines: Vec<Line<'static>> = vec![Line::from("")];
-    for sp in spans {
-        let style = sp.style;
-        let text = sp.content.into_owned();
-        for (i, part) in text.split('\n').enumerate() {
-            if i > 0 {
-                lines.push(Line::from(""));
-            }
-            if part.is_empty() {
-                continue;
+    let mut highlight_stack: Vec<Highlight> = Vec::new();
+
+    for event in iterator {
+        match event {
+            Ok(HighlightEvent::HighlightStart(highlight)) => highlight_stack.push(highlight),
+            Ok(HighlightEvent::HighlightEnd) => {
+                highlight_stack.pop();
             }
-            let span = Span {
-                style,
-                content: std::borrow::Cow::Owned(part.to_string()),
-            };
-            if let Some(last) = lines.last_mut() {
-                last.spans.push(span);
+            Ok(HighlightEvent::Source { start, end }) => {
+                if start == end {
+                    continue;
+                }
+                let style = highlight_stack.last().map(|h| highlight_for(*h).style());
+                push_segment(&mut lines, &script[start..end], style);
             }
+            Err(_) => return vec![script.to_string().into()],
         }
     }
-    lines
+
+    if lines.is_empty() {
+        vec![Line::from("")]
+    } else {
+        lines
+    }
 }
 
 #[cfg(test)]
@@ -84,11 +149,8 @@ mod tests {
     use pretty_assertions::assert_eq;
     use ratatui::style::Modifier;
 
-    #[test]
-    fn dims_expected_bash_operators() {
-        let s = "echo foo && bar || baz | qux & (echo hi)";
-        let lines = highlight_bash_to_lines(s);
-        let reconstructed: String = lines
+    fn reconstructed(lines: &[Line<'static>]) -> String {
+        lines
             .iter()
             .map(|l| {
                 l.spans
@@ -97,49 +159,78 @@ mod tests {
                     .collect::<String>()
             })
             .collect::<Vec<_>>()
-            .join("\n");
-        assert_eq!(reconstructed, s);
+            .join("\n")
+    }
 
-        fn is_dim(span: &Span<'_>) -> bool {
-            span.style.add_modifier.contains(Modifier::DIM)
-        }
-        let dimmed: Vec<String> = lines
+    fn dimmed_tokens(lines: &[Line<'static>]) -> Vec<String> {
+        lines
             .iter()
             .flat_map(|l| l.spans.iter())
-            .filter(|sp| is_dim(sp))
+            .filter(|sp| sp.style.add_modifier.contains(Modifier::DIM))
             .map(|sp| sp.content.clone().into_owned())
-            .collect();
-        assert_eq!(dimmed, vec!["&&", "||", "|", "&", "(", ")"]);
+            .map(|token| token.trim().to_string())
+            .filter(|token| !token.is_empty())
+            .collect()
+    }
+
+    #[test]
+    fn dims_expected_bash_operators() {
+        let s = "echo foo && bar || baz | qux & (echo hi)";
+        let lines = highlight_bash_to_lines(s);
+        assert_eq!(reconstructed(&lines), s);
+
+        let dimmed = dimmed_tokens(&lines);
+        assert!(dimmed.contains(&"&&".to_string()));
+        assert!(dimmed.contains(&"|".to_string()));
+        assert!(!dimmed.contains(&"echo".to_string()));
     }
 
     #[test]
-    fn does_not_dim_quotes_but_dims_other_punct() {
+    fn dims_redirects_and_strings() {
         let s = "echo \"hi\" > out.txt; echo 'ok'";
         let lines = highlight_bash_to_lines(s);
-        let reconstructed: String = lines
-            .iter()
-            .map(|l| {
-                l.spans
-                    .iter()
-                    .map(|sp| sp.content.clone())
-                    .collect::<String>()
-            })
-            .collect::<Vec<_>>()
-            .join("\n");
-        assert_eq!(reconstructed, s);
+        assert_eq!(reconstructed(&lines), s);
 
-        fn is_dim(span: &Span<'_>) -> bool {
-            span.style.add_modifier.contains(Modifier::DIM)
-        }
-        let dimmed: Vec<String> = lines
-            .iter()
-            .flat_map(|l| l.spans.iter())
-            .filter(|sp| is_dim(sp))
-            .map(|sp| sp.content.clone().into_owned())
-            .collect();
+        let dimmed = dimmed_tokens(&lines);
         assert!(dimmed.contains(&">".to_string()));
-        assert!(dimmed.contains(&";".to_string()));
-        assert!(!dimmed.contains(&"\"".to_string()));
-        assert!(!dimmed.contains(&"'".to_string()));
+        assert!(dimmed.contains(&"\"hi\"".to_string()));
+        assert!(dimmed.contains(&"'ok'".to_string()));
+    }
+
+    #[test]
+    fn highlights_command_and_strings() {
+        let s = "echo \"hi\"";
+        let lines = highlight_bash_to_lines(s);
+        let mut echo_style = None;
+        let mut string_style = None;
+        for span in &lines[0].spans {
+            let text = span.content.as_ref();
+            if text == "echo" {
+                echo_style = Some(span.style);
+            }
+            if text == "\"hi\"" {
+                string_style = Some(span.style);
+            }
+        }
+        let echo_style = echo_style.expect("echo span missing");
+        let string_style = string_style.expect("string span missing");
+        assert!(echo_style.fg.is_none());
+        assert!(!echo_style.add_modifier.contains(Modifier::DIM));
+        assert!(string_style.add_modifier.contains(Modifier::DIM));
+    }
+
+    #[test]
+    fn highlights_heredoc_body_as_string() {
+        let s = "cat <<EOF\nheredoc body\nEOF";
+        let lines = highlight_bash_to_lines(s);
+        let body_line = &lines[1];
+        let mut body_style = None;
+        for span in &body_line.spans {
+            if span.content.as_ref() == "heredoc body" {
+                body_style = Some(span.style);
+            }
+        }
+        let body_style = body_style.expect("missing heredoc span");
+        assert!(body_style.add_modifier.contains(Modifier::DIM));
     }
 }