renderdiff: enable webgpu and improvements (#9219)

 - Allow for presets to override selected models when presented
   in order.
 - Add 'gltf' for model search path
 - Add a few simple gltf models to the 'base' preset
 - Improve UI so that missing tests do not generate any html
   bits.
 - Add documentation on using the viewer
 - Add renderdiff documentation to the project webpage.

RDIFF_BRANCH=pf/renderdiff-enable-webgpu

Author: poweifeng

docs_src/build/duplicates.json

@@ -78,5 +78,8 @@
     },
     "docs_src/README.md": {
         "dest": "dup/docs.md"
+    },
+    "docs_src/test/renderdiff/README.md": {
+        "dest": "dup/renderdiff.md"
     }
 }

docs_src/src_mdbook/src/SUMMARY.md

@@ -27,6 +27,8 @@
     - [Code coverage analysis](./notes/coverage.md)
     - [Performance analysis](./notes/performance_analysis.md)
   - [Framegraph](./notes/framegraph.md)
+  - [Tests](./notes/tests.md)
+    - [renderdiff](./dup/renderdiff.md)
   - [Libraries](./notes/libs.md)
     - [bluegl](./dup/bluegl.md)
     - [bluevk](./dup/bluevk.md)

docs_src/src_mdbook/src/images/renderdiff_example.png

@@ -0,0 +1,4 @@
+# Tests
+
+Filament has a collection of tests that can be run locally. Many of them are used
+for in our Continuation Integration flow [on github](https://https://github.com/google/filament/tree/main/.github/workflows).

docs_src/src_mdbook/src/notes/tests.md

@@ -13,11 +13,33 @@ description file) and then running gltf_viewer to produce the renderings.
 In the `test` directory is a list of test descriptions that are specified in json.  Please see
 `sample.json` to parse the structure.
 
+## Setting up python
+The `renderdiff` project uses `python` extensively. To install the dependencies for producing
+renderings, do the following step
+ - Set up a virtual environment (from the root directory)
+   ```
+     python3 -m venv venv
+     . ./venv/bin/activate
+   ```
+ - Install the rendering dependencies
+   ```
+     pip install -r test/renderdiff/src/rendering_requirements.txt
+   ```
+ - Install the viewer depdencies
+   ```
+     pip install -r test/renderdiff/src/viewer_requirements.txt
+   ```
+ - For the commands in the following section, do not exit the virtual environment. Once you've
+   completed all your work, you can exit with
+   ```
+     deactivate
+   ```
+
 ## Running the test locally
  - To run the same presbumit as [`test-renderdiff`](presubmit-renderdiff), you can do
 
    ```
-   bash test/renderdiff/test.sh
+   bash test/renderdiff/local_test.sh
    ```
 
  - This script will generate the renderings based on the current state of your repo.
@@ -67,7 +89,7 @@ in the following fashion
 
  - Copy the new images to their appropriate place in `filament-assets`
  - Push the `filament-assets` working branch to remote
-   
+
    ```
    git push origin my-pr-branch-golden
    ```
@@ -78,14 +100,39 @@ in the following fashion
    RDIFF_BBRANCH=my-pr-branch-golden
    ```
 
-### Manually updating the golden repo
-
 Doing the above has multiple effects:
  - The presubmit test [`test-renderdiff`][presubmit-renderdiff] will test against the provided
    branch of the golden repo (i.e. `my-pr-branch-golden`).
  - If the PR is merged, then there is another workflow that will merge `my-pr-branch-golden` to
    the `main` branch of the golden repo.
 
+## Viewing test results
+We provide a viewer for looking at the result of a test run. The viewer is a webapp that can be used by
+pointing your browser to a localhost port. If you input the viewer with a PR or a directory, it will
+parse the test result and show the results and the rendered and/or golden images.
+
+![Viewer](docs/images/renderdiff_example.png)
+
+To run the viewer of a test output directory that has been generated locally, you would run the
+following
+
+```
+python3 test/renderdiff/src/viewer.py --diff=[test output]
+```
+where `[test output]` is a directory containing the `compare_results.json` of the test run.
+For example, it could be `out/renderdiff/diffs/presubmit` for the standard path to the
+`presubmit` test output.
+
+To see the results of a Pull Request initiated test run, you would do the following
+
+```
+python3 test/renderdiff/src/viewer.py --pr_number=[PR #] --github_token=[github token]
+```
+
+where `[PR #]` is the numeric ID of your pull request, and the `[github token]` is an acess token
+that you (as a github user) needs to generate ([reference][github_token_ref]).
+
+[github_token_ref]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
 [Mesa]: https://docs.mesa3d.org
 [SwiftShader]: https://github.com/google/swiftshader
 [presubmit-renderdiff]: https://github.com/google/filament/blob/e85dfe75c86106a05019e13ccdbef67e030af675/.github/workflows/presubmit.yml#L118

test/renderdiff/README.md

@@ -22,7 +22,6 @@
 from utils import execute, ArgParseImpl, mkdir_p, mv_f, important_print
 
 import test_config
-from golden_manager import GoldenManager
 from image_diff import same_image
 from results import RESULT_OK, RESULT_FAILED
 

test/renderdiff/src/render.py

@@ -0,0 +1,6 @@
+Mako==1.3.10
+MarkupSafe==3.0.2
+numpy==2.3.3
+PyYAML==6.0.2
+setuptools==80.9.0
+tifffile==2025.9.9

test/renderdiff/src/rendering_requirements.txt

@@ -76,9 +76,12 @@ def __init__(self, data, existing_models, presets):
       given_presets = {p.name: p for p in presets}
       assert all((name in given_presets) for name in apply_presets),\
         f'used preset {name} which is not in {given_presets}'
+
+      # Note that this needs to applied in order.  Models will be overwritten.
+      # Properties will be "added" in order.
       for preset in apply_presets:
         rendering.update(given_presets[preset].rendering)
-        preset_models += given_presets[preset].models
+        preset_models = given_presets[preset].models
 
     assert 'rendering' in data
     rendering.update(data['rendering'])

test/renderdiff/src/test_config.py

@@ -113,7 +113,7 @@ class ExpandedComparisonResult extends LitElement {
     #diffCanvas {
       width: 100%;
       height: 100%;
-      margin-top: 35px;
+      margin-top: 22px;
       margin-bottom: 5px;
     }
     .selector {
@@ -151,33 +151,9 @@ class ExpandedComparisonResult extends LitElement {
 
   _viewer(name, choices, current, viewType) {
     const url = viewType == 'rendered' ? getCompUrl(current) : getGoldenUrl(current);
-    const onSelect = (ev) => {
-      const testName = ev.target.value;
-      const test = this.tests.find((t) => t.name == testName);
-      if (name == 'left') {
-        this.left = test;
-        this.leftImageLoaded = false;
-      } else {
-        this.right = test;
-        this.rightImageLoaded = false;
-      }
-      this.showDiff = false;
-    };
-
-    const dropdown = () => {
-      if (this.disableDropdowns) {
-        return html`<div class="selector">${current.name} (${viewType})</div>`;
-      }
-      return html`
-        <select class="selector" @change=${onSelect}>
-          ${choices.map((c) => html`<option value=${c.name} ?selected=${c.name == current.name}>${c.name}</option>`)}
-        </select>
-      `;
-    }
-
     return html`
       <div style="flex: 1; margin: 0 5px;">
-        ${dropdown()}
+        <div>${current.name}</div>
         <tiff-viewer id="viewer-${name}" class="viewer"
                      name="${current.name}"
                      fileurl="${url}"></tiff-viewer>
@@ -405,6 +381,9 @@ class App extends LitElement {
     selectedTests: {type: Array},
     comparisonContent: {type: Object},
     compareMode: {type: Boolean},
+
+    // This is used to cache urls that are not found
+    missingFile: {type: Object},
   };
 
   async _init() {
@@ -420,12 +399,27 @@ class App extends LitElement {
     this.selectedTests = [];
     this.comparisonContent = null;
     this.compareMode = false;
+    this.missingFile = {
+      [getDiffUrl('undefined')]: true,
+    };
     this._init();
 
     this.addEventListener('dialog-closed', () => {
       this.dialogContent = null;
       this.comparisonContent = null;
     });
+
+    this.addEventListener('url-hit', (ev) => {
+      delete this.missingFile[ev.detail.value];
+      this.missingFile = this.missingFile;
+      this.requestUpdate();
+    });
+
+    this.addEventListener('url-miss', (ev) => {
+      this.missingFile[ev.detail.value] = true;
+      this.missingFile = this.missingFile;
+      this.requestUpdate();
+    });
   }
 
   updated(props) {
@@ -470,8 +464,24 @@ class App extends LitElement {
   }
 
   render() {
-    let passed = this.tests.filter((t) => t.result == 'ok');
-    let failed = this.tests.filter((t) => t.result != 'ok');
+    const sortFn = (a, b) => {
+      const aparts = a.name.split('.');
+      const bparts = b.name.split('.');
+      // 0 = test names
+      // 1 = backend
+      // 2 = model
+      for (let i of [0, 2, 1]) {
+        if (aparts[i] < bparts[i]) {
+          return -1;
+        }
+        if (aparts[i] > bparts[i]) {
+          return 1;
+        }
+      }
+      return 0;
+    };
+    let passed = this.tests.filter((t) => t.result == 'ok').sort(sortFn);
+    let failed = this.tests.filter((t) => t.result != 'ok').sort(sortFn);
     const singleTiff = (url) => {
       return html`<tiff-viewer style="max-width:100px" fileurl="${url}"></tiff-viewer>`;
     };
@@ -502,7 +512,8 @@ class App extends LitElement {
         [goldenUrl, 'golden'],
         [compUrl, 'rendered'],
         [diffUrl, 'diff']
-      ].map((a) => [singleTiff(a[0]), a[1]])
+      ].filter((a) => !this.missingFile[a[0]])
+       .map((a) => [singleTiff(a[0]), a[1]])
        .map((a) => wrap(...a));
       return html`
         <div class="test-item" @click="${(e)=>this._onClick(t, e)}" >

test/renderdiff/src/viewer_html/app.js

@@ -30,15 +30,20 @@ export class TiffViewer extends LitElement {
   static properties = {
     fileurl: {type: String, attribute: 'fileurl'},
     name: {type: String, attribute: 'name'},
+    failedToFetch: {type: Boolean },
   };
 
   constructor() {
     super();
     this.fileurl = null;
     this.name = null;
+    this.failedToFetch = false;
   }
 
   render() {
+    if (this.failedToFetch) {
+      return html``;
+    }
     return html`<canvas id="tiffCanvas"></canvas>`;
   }
 
@@ -49,7 +54,33 @@ export class TiffViewer extends LitElement {
   }
 
   async _updateImage(fileurl) {
-    const fileblob = await ((await fetch(this.fileurl)).arrayBuffer());
+    this.failedToFetch = false;
+    let fileblob = null;
+    try {
+      let res = await fetch(this.fileurl);
+      if (!res.ok) {
+        throw new Error(`Could not find ${this.fileurl}`);
+      }
+      fileblob = await res.arrayBuffer();
+    } catch (error) {
+      this.failedToFetch = true;
+    }
+    if (!fileblob) {
+      const event = new CustomEvent('url-miss', {
+        detail: { value: fileurl },
+        bubbles: true,
+        composed: true,
+      });
+      this.dispatchEvent(event);
+      return;
+    } else {
+      const event = new CustomEvent('url-hit', {
+        detail: { value: fileurl },
+        bubbles: true,
+        composed: true,
+      });
+      this.dispatchEvent(event);
+    }
     const canvas = this.shadowRoot.getElementById('tiffCanvas');
     const ctx = canvas.getContext('2d');