parse eval properties

Adds the ability to parse "eval" properties given a specially crafted name.

Author: giovanni-guidini

benches/binary.rs

@@ -6,7 +6,7 @@ use rand::rngs::SmallRng;
 use rand::seq::SliceRandom as _;
 use rand::{Rng, SeedableRng};
 use test_results_parser::binary::*;
-use test_results_parser::{Outcome, Testrun, ValidatedString};
+use test_results_parser::{Outcome, PropertiesValue, Testrun, ValidatedString};
 
 criterion_group!(benches, binary);
 criterion_main!(benches);
@@ -184,6 +184,7 @@ fn create_random_testcases(
                         filename: None,
                         build_url: None,
                         computed_name: ValidatedString::default(),
+                        properties: PropertiesValue(None),
                     }
                 })
                 .collect();

src/binary/mod.rs

@@ -19,7 +19,7 @@ mod tests {
     use timestamps::DAY;
 
     use crate::{
-        testrun::{Outcome, Testrun},
+        testrun::{Outcome, PropertiesValue, Testrun},
         validated_string::ValidatedString,
     };
 
@@ -36,6 +36,7 @@ mod tests {
             filename: None,
             build_url: None,
             computed_name: ValidatedString::default(),
+            properties: PropertiesValue(None),
         }
     }
 

src/junit.rs

@@ -1,13 +1,15 @@
 use anyhow::{Context, Result};
 use pyo3::prelude::*;
+use serde_json::Value;
 use std::collections::HashSet;
+use std::fmt;
 
 use quick_xml::events::attributes::{Attribute, Attributes};
 use quick_xml::events::{BytesStart, Event};
 use quick_xml::reader::Reader;
 
 use crate::compute_name::{compute_name, unescape_str};
-use crate::testrun::{check_testsuites_name, Framework, Outcome, Testrun};
+use crate::testrun::{check_testsuites_name, Framework, Outcome, PropertiesValue, Testrun};
 use crate::validated_string::ValidatedString;
 use crate::warning::WarningInfo;
 use thiserror::Error;
@@ -126,6 +128,7 @@ fn populate(
         filename: file,
         build_url: None,
         computed_name: ValidatedString::default(),
+        properties: PropertiesValue(None),
     };
 
     let framework = framework.or_else(|| t.framework());
@@ -158,11 +161,131 @@ pub fn get_position_info(input: &[u8], byte_offset: usize) -> (usize, usize) {
     (line, column)
 }
 
+#[derive(Error, Debug)]
+struct NotEvalsPropertyError;
+
+impl fmt::Display for NotEvalsPropertyError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "not evals property")
+    }
+}
+
+/// Parses the `property` element found in the `testcase` element.
+///
+/// This function is used to parse the `evals` attribute of the `testcase` element.
+/// It will update the `properties` field of the `testrun` object with the new value.
+///
+/// The `name` attribute in `property` encodes the hierarchy of the `value` attribute
+/// inside `Testrun.properties` (which is a JSON object).
+/// For example
+/// &lt;property name="evals.scores.isUseful.type" value="boolean" /&gt;
+/// &lt;property name="evals.scores.isUseful.value" value="true" /&gt;
+/// &lt;property name="evals.scores.isUseful.sum" value="1" /&gt;
+/// &lt;property name="evals.scores.isUseful.llm_judge" value="gemini_2.5pro" /&gt;
+///
+/// will be parsed as:
+/// {
+///     "scores": {
+///         "isUseful": {
+///             "type": "boolean",
+///             "value": "true",
+///             "sum": "1",
+///             "llm_judge": "gemini_2.5pro"
+///         }
+///     }
+/// }
+fn parse_property_element(e: &BytesStart, existing_properties: &mut PropertiesValue) -> Result<()> {
+    // Early return if not an evals property
+    let name = get_attribute(e, "name")?
+        .filter(|n| n.starts_with("evals"))
+        .ok_or(NotEvalsPropertyError)?;
+
+    let value = get_attribute(e, "value")?
+        .ok_or_else(|| anyhow::anyhow!("Property must have value attribute"))?;
+
+    let name_parts: Vec<&str> = name.split(".").collect();
+    if name_parts.len() < 2 {
+        anyhow::bail!("Property name must have at least 2 parts");
+    }
+
+    // Initialize properties if needed
+    if existing_properties.0.is_none() {
+        *existing_properties = PropertiesValue(Some(serde_json::json!({})));
+    }
+
+    let mut current = existing_properties.0.as_mut().unwrap();
+
+    // Navigate through intermediate parts (skip first "evals" and last key)
+    for part in &name_parts[1..name_parts.len() - 1] {
+        current = match current {
+            Value::Object(map) => {
+                map.entry(part.to_string()).or_insert_with(|| {
+                    if *part == "evaluations" {
+                        serde_json::json!([])
+                    } else {
+                        serde_json::json!({})
+                    }
+                });
+                map.get_mut(*part).unwrap()
+            }
+            Value::Array(array) => {
+                let idx = part
+                    .parse::<usize>()
+                    .map_err(|_| anyhow::anyhow!("Invalid array index: {}", part))?;
+                if idx >= array.len() {
+                    array.resize(idx + 1, serde_json::json!({}));
+                }
+                array.get_mut(idx).unwrap()
+            }
+            _ => anyhow::bail!(
+                "Cannot drill down into non-object/non-array value at part: {}",
+                part
+            ),
+        };
+    }
+
+    // Set the final value
+    match current {
+        Value::Object(map) => {
+            map.insert(name_parts.last().unwrap().to_string(), Value::String(value));
+        }
+        _ => anyhow::bail!("Cannot set value in non-object at final key"),
+    }
+
+    Ok(())
+}
+
 enum TestrunOrSkipped {
     Testrun(Testrun),
     Skipped,
 }
 
+fn handle_property_element(
+    e: &BytesStart,
+    saved_testrun: &mut Option<TestrunOrSkipped>,
+    buffer_position: u64,
+    warnings: &mut Vec<WarningInfo>,
+) -> Result<()> {
+    // Check if there is a testrun currently being processed
+    if saved_testrun.is_none() {
+        return Ok(());
+    }
+    let saved = saved_testrun
+        .as_mut()
+        .context("Error accessing saved testrun")?;
+    if let TestrunOrSkipped::Testrun(testrun) = saved {
+        if let Err(e) = parse_property_element(e, &mut testrun.properties) {
+            if !e.is::<NotEvalsPropertyError>() {
+                warnings.push(WarningInfo::new(
+                    format!("Error parsing `property` element: {}", e),
+                    buffer_position,
+                ));
+            }
+        }
+    };
+    Ok(())
+}
+
 pub fn use_reader(
     reader: &mut Reader<&[u8]>,
     network: Option<&HashSet<String>>,
@@ -286,6 +409,12 @@ pub fn use_reader(
                     let testsuites_name = get_attribute(&e, "name")?;
                     framework = testsuites_name.and_then(|name| check_testsuites_name(&name))
                 }
+                b"property" => handle_property_element(
+                    &e,
+                    &mut saved_testrun,
+                    reader.buffer_position(),
+                    &mut warnings,
+                )?,
                 _ => {}
             },
             Event::End(e) => match e.name().as_ref() {
@@ -378,6 +507,12 @@ pub fn use_reader(
                         TestrunOrSkipped::Skipped => {}
                     }
                 }
+                b"property" => handle_property_element(
+                    &e,
+                    &mut saved_testrun,
+                    reader.buffer_position(),
+                    &mut warnings,
+                )?,
                 _ => {}
             },
             Event::Text(mut xml_failure_message) => {

src/lib.rs

@@ -10,7 +10,7 @@ mod testrun;
 mod validated_string;
 mod warning;
 
-pub use testrun::{Outcome, Testrun};
+pub use testrun::{Outcome, PropertiesValue, Testrun};
 pub use validated_string::ValidatedString;
 pyo3::create_exception!(test_results_parser, ComputeNameError, PyException);
 

src/snapshots/test_results_parser__raw_upload__tests__parse_raw_upload_success@ctest.xml.snap

@@ -1,5 +1,6 @@
 ---
 source: src/raw_upload.rs
+assertion_line: 173
 expression: results
 input_file: tests/ctest.xml
 ---
@@ -14,4 +15,5 @@ input_file: tests/ctest.xml
       filename: ~
       build_url: ~
       computed_name: "a_unit_test::a_unit_test"
+      properties: ~
   warnings: []

src/snapshots/test_results_parser__raw_upload__tests__parse_raw_upload_success@empty_failure.junit.xml.snap

@@ -1,5 +1,6 @@
 ---
 source: src/raw_upload.rs
+assertion_line: 173
 expression: results
 input_file: tests/empty_failure.junit.xml
 ---
@@ -14,6 +15,7 @@ input_file: tests/empty_failure.junit.xml
       filename: "./test.rb"
       build_url: ~
       computed_name: "test.test::test.test works"
+      properties: ~
     - name: test.test fails
       classname: test.test
       duration: 1
@@ -23,4 +25,5 @@ input_file: tests/empty_failure.junit.xml
       filename: "./test.rb"
       build_url: ~
       computed_name: "test.test::test.test fails"
+      properties: ~
   warnings: []

src/snapshots/test_results_parser__raw_upload__tests__parse_raw_upload_success@jest-junit.xml.snap

@@ -1,5 +1,6 @@
 ---
 source: src/raw_upload.rs
+assertion_line: 173
 expression: results
 input_file: tests/jest-junit.xml
 ---
@@ -14,6 +15,7 @@ input_file: tests/jest-junit.xml
       filename: ~
       build_url: ~
       computed_name: Title when rendered renders pull title
+      properties: ~
     - name: Title when rendered renders pull author
       classname: Title when rendered renders pull author
       duration: 0.005
@@ -23,6 +25,7 @@ input_file: tests/jest-junit.xml
       filename: ~
       build_url: ~
       computed_name: Title when rendered renders pull author
+      properties: ~
     - name: Title when rendered renders pull updatestamp
       classname: Title when rendered renders pull updatestamp
       duration: 0.002
@@ -32,6 +35,7 @@ input_file: tests/jest-junit.xml
       filename: ~
       build_url: ~
       computed_name: Title when rendered renders pull updatestamp
+      properties: ~
     - name: Title when rendered for first pull request renders pull title
       classname: Title when rendered for first pull request renders pull title
       duration: 0.006
@@ -41,4 +45,5 @@ input_file: tests/jest-junit.xml
       filename: ~
       build_url: ~
       computed_name: Title when rendered for first pull request renders pull title
+      properties: ~
   warnings: []

src/snapshots/test_results_parser__raw_upload__tests__parse_raw_upload_success@junit-nested-testsuite.xml.snap

@@ -1,5 +1,6 @@
 ---
 source: src/raw_upload.rs
+assertion_line: 173
 expression: results
 input_file: tests/junit-nested-testsuite.xml
 ---
@@ -14,6 +15,7 @@ input_file: tests/junit-nested-testsuite.xml
       filename: ~
       build_url: ~
       computed_name: "tests.test_parsers.TestParsers::test_junit[junit.xml--True]"
+      properties: ~
     - name: "test_junit[jest-junit.xml--False]"
       classname: tests.test_parsers.TestParsers
       duration: 0.186
@@ -23,4 +25,5 @@ input_file: tests/junit-nested-testsuite.xml
       filename: ~
       build_url: ~
       computed_name: "tests.test_parsers.TestParsers::test_junit[jest-junit.xml--False]"
+      properties: ~
   warnings: []

src/snapshots/test_results_parser__raw_upload__tests__parse_raw_upload_success@junit-no-testcase-timestamp.xml.snap

@@ -1,5 +1,6 @@
 ---
 source: src/raw_upload.rs
+assertion_line: 173
 expression: results
 input_file: tests/junit-no-testcase-timestamp.xml
 ---
@@ -14,6 +15,7 @@ input_file: tests/junit-no-testcase-timestamp.xml
       filename: ~
       build_url: ~
       computed_name: "tests.test_parsers.TestParsers::test_junit[junit.xml--True]"
+      properties: ~
     - name: "test_junit[jest-junit.xml--False]"
       classname: tests.test_parsers.TestParsers
       duration: 0.186
@@ -23,4 +25,5 @@ input_file: tests/junit-no-testcase-timestamp.xml
       filename: ~
       build_url: ~
       computed_name: "tests.test_parsers.TestParsers::test_junit[jest-junit.xml--False]"
+      properties: ~
   warnings: []

src/snapshots/test_results_parser__raw_upload__tests__parse_raw_upload_success@junit.xml.snap

@@ -1,5 +1,6 @@
 ---
 source: src/raw_upload.rs
+assertion_line: 173
 expression: results
 input_file: tests/junit.xml
 ---
@@ -14,6 +15,7 @@ input_file: tests/junit.xml
       filename: ~
       build_url: ~
       computed_name: "tests.test_parsers.TestParsers::test_junit[junit.xml--True]"
+      properties: ~
     - name: "test_junit[jest-junit.xml--False]"
       classname: tests.test_parsers.TestParsers
       duration: 0.064
@@ -23,4 +25,5 @@ input_file: tests/junit.xml
       filename: ~
       build_url: ~
       computed_name: "tests.test_parsers.TestParsers::test_junit[jest-junit.xml--False]"
+      properties: ~
   warnings: []