feat(ci): collect flaky results

Get one single table with every
results from flaky tests.

Author: BuonOmo

.github/workflows/ci.yml

@@ -26,13 +26,51 @@ jobs:
     name: Test Results
     needs: [test]
     steps:
-      - run: |
+      - name: Check Success
+        run: |
           result="${{ needs.test.result }}"
+
           if [[ $result == "success" || $result == "skipped" ]]; then
+            echo "All tests passed :taco:" >>$GITHUB_STEP_SUMMARY
             exit 0
           else
             exit 1
           fi
+      - name: Download Reports
+        if: failure()
+        uses: actions/download-artifact@v4
+        with:
+          path: reports
+      - name: Aggregate Reports
+        if: failure()
+        run: |
+          cat <<EOF >>$GITHUB_STEP_SUMMARY
+          # Failing Tests
+
+          <table>
+          <thead>
+            <tr>
+              <th></th>
+              <th>Test</th>
+            </tr>
+          </thead>
+          <tbody>
+          EOF
+
+          jq --slurp --raw-output '
+            map(.failed_tests|map("\(.klass)#\(.NAME)")) | flatten | unique | sort | to_entries[]
+            | "<tr><td><strong>\(.key)</strong></td><td>\(.value)</td></tr>"
+          ' reports/*/report.json >>$GITHUB_STEP_SUMMARY
+
+          cat <<EOF >>$GITHUB_STEP_SUMMARY
+          </tbody>
+          </table>
+          EOF
+
+          echo >>$GITHUB_STEP_SUMMARY
+          echo '```json' >>$GITHUB_STEP_SUMMARY
+          jq --slurp --compact-output '.' reports/*/report.json >>$GITHUB_STEP_SUMMARY
+          echo '```' >>$GITHUB_STEP_SUMMARY
 
   test:
     runs-on: ubuntu-latest
@@ -42,11 +80,20 @@ jobs:
         # https://www.cockroachlabs.com/docs/releases/release-support-policy
         crdb: [v24.3, v25.1, v25.2, v25.3]
         ruby: ["3.4"]
-    name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }}
+    name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }})
     steps:
       - name: Set Up Actions
         uses: actions/checkout@v4
       - uses: ./.github/actions/test-runner
+        id: test
         with:
           crdb: ${{ matrix.crdb }}
           ruby: ${{ matrix.ruby }}
+        env:
+          JSON_REPORTER: "report.json"
+      - name: Upload Report
+        if: ${{ failure() && steps.test.conclusion == 'failure' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.crdb }}-${{ matrix.ruby }}
+          path: report.json

.github/workflows/flaky.yml

@@ -53,8 +53,6 @@ jobs:
         ruby: ${{ fromJSON(needs.prepare-matrix.outputs.ruby) }}
         seed: ${{ fromJSON(needs.prepare-matrix.outputs.seeds) }}
     name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }} seed=${{ matrix.seed }})
-    env:
-      SEED: ${{ matrix.seed }}
     steps:
       - name: Set Up Actions
         uses: actions/checkout@v4
@@ -64,6 +62,52 @@ jobs:
           crdb: ${{ matrix.crdb }}
           ruby: ${{ matrix.ruby }}
           TESTOPTS: --fail-fast
-      - name: Bisect failing test
+        env:
+          JSON_REPORTER: "report.json"
+          SEED: ${{ matrix.seed }}
+      - name: Upload Report
         if: ${{ failure() && steps.test.conclusion == 'failure' }}
-        run: bin/minitest_bisect ${{ matrix.seed }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.crdb }}-${{ matrix.ruby }}-${{ matrix.seed }}
+          path: report.json
+  collect:
+    if: failure()
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Reports
+        uses: actions/download-artifact@v4
+        with:
+          path: reports
+      - name: Aggregate Reports
+        run: |
+          cat <<EOF >> $GITHUB_STEP_SUMMARY
+          # Failing Tests
+
+          <table>
+          <thead>
+            <tr>
+              <th>Seed</th>
+              <th>Time</th>
+              <th>Failure</th>
+            </tr>
+          </thead>
+          <tbody>
+          EOF
+
+          jq --slurp --raw-output '
+            sort_by(.total_time)[]
+            | {seed, time: .total_time | strftime("%H:%M:%S"), failure: .failed_tests[0].NAME }
+            | "<tr><td>\(.seed)</td><td>\(.time)</td><td>\(.failure)</td></tr>"
+          ' reports/*/report.json >> $GITHUB_STEP_SUMMARY
+
+
+          cat <<EOF >> $GITHUB_STEP_SUMMARY
+          </tbody>
+          </table>
+          EOF
+          echo >> $GITHUB_STEP_SUMMARY
+          echo '```json' >> $GITHUB_STEP_SUMMARY
+          jq --slurp --compact-output '.' reports/*/report.json >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY

Gemfile

@@ -56,7 +56,6 @@ group :development, :test do
   gem "debug"
   gem "minitest-bisect", github: "BuonOmo/minitest-bisect", branch: "main"
   gem "minitest-excludes", "~> 2.0.1"
-  gem "minitest-github_action_reporter", github: "BuonOmo/minitest-github_action_reporter", require: "minitest/github_action_reporter_plugin"
   gem "ostruct", "~> 0.6"
 
   # Gems used for tests meta-programming.

test/cases/helper_cockroachdb.rb

@@ -169,24 +169,33 @@ def with_postgresql_datetime_type(type)
 
 ActiveRecord::TestCase.prepend(SetDatetimeInCockroachDBAdapter)
 
-if ENV["GITHUB_ACTIONS"]
-  require "minitest/github_action_reporter"
-
+if ENV["JSON_REPORTER"]
+  puts "Generating JSON report: #{ENV["JSON_REPORTER"]}"
   module Minitest
-    module GithubActionReporterExt
-      def gh_link(loc)
-        return super unless loc.include?("/gems/")
-
-        path, _, line = loc[%r(/(?:test|spec|lib)/.*)][1..].rpartition(":")
-
-        rails_version = "v#{ActiveRecord::VERSION::STRING}"
-        "#{ENV["GITHUB_SERVER_URL"]}/rails/rails/blob/#{rails_version}/activerecord/#{path}#L#{line}"
-      rescue
-        warn "Failed to generate link for #{loc}"
+    class JSONReporter < StatisticsReporter
+      def report
         super
+        io.write(
+          {
+            seed: Minitest.seed,
+            assertions: assertions,
+            count: count,
+            failed_tests: results,
+            total_time: total_time,
+            failures: failures,
+            errors: errors,
+            warnings: warnings,
+            skips: skips,
+          }.to_json
+        )
       end
     end
-    GithubActionReporter.prepend(GithubActionReporterExt)
+
+    def self.plugin_json_reporter_init(*)
+      reporter << JSONReporter.new(File.open(ENV["JSON_REPORTER"], "w"))
+    end
+
+    self.extensions << "json_reporter"
   end
 end