Find a Generic way to fix flaky tests

.github/workflows/ci.yml

@@ -26,13 +26,51 @@ jobs:
     name: Test Results
     needs: [test]
     steps:
-      - run: |
+      - name: Check Success
+        run: |
           result="${{ needs.test.result }}"
+
           if [[ $result == "success" || $result == "skipped" ]]; then
+            echo "All tests passed :taco:" >>$GITHUB_STEP_SUMMARY
             exit 0
           else
             exit 1
           fi
+      - name: Download Reports
+        if: failure()
+        uses: actions/download-artifact@v4
+        with:
+          path: reports
+      - name: Aggregate Reports
+        if: failure()
+        run: |
+          cat <<EOF >>$GITHUB_STEP_SUMMARY
+          # Failing Tests
+
+          <table>
+          <thead>
+            <tr>
+              <th></th>
+              <th>Test</th>
+            </tr>
+          </thead>
+          <tbody>
+          EOF
+
+          jq --slurp --raw-output '
+            map(.failed_tests|map("\(.klass)#\(.NAME)")) | flatten | unique | sort | to_entries[]
+            | "<tr><td><strong>\(.key)</strong></td><td>\(.value)</td></tr>"
+          ' reports/*/report.json >>$GITHUB_STEP_SUMMARY
+
+          cat <<EOF >>$GITHUB_STEP_SUMMARY
+          </tbody>
+          </table>
+          EOF
+
+          echo >>$GITHUB_STEP_SUMMARY
+          echo '```json' >>$GITHUB_STEP_SUMMARY
+          jq --slurp --compact-output '.' reports/*/report.json >>$GITHUB_STEP_SUMMARY
+          echo '```' >>$GITHUB_STEP_SUMMARY
 
   test:
     runs-on: ubuntu-latest
@@ -42,11 +80,20 @@ jobs:
         # https://www.cockroachlabs.com/docs/releases/release-support-policy
         crdb: [v24.3, v25.1, v25.2, v25.3]
         ruby: ["3.4"]
-    name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }}
+    name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }})
     steps:
       - name: Set Up Actions
         uses: actions/checkout@v4
       - uses: ./.github/actions/test-runner
+        id: test
         with:
           crdb: ${{ matrix.crdb }}
           ruby: ${{ matrix.ruby }}
+        env:
+          JSON_REPORTER: "report.json"
+      - name: Upload Report
+        if: ${{ failure() && steps.test.conclusion == 'failure' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.crdb }}-${{ matrix.ruby }}
+          path: report.json

.github/workflows/flaky.yml

@@ -33,15 +33,16 @@ jobs:
           ruby=$(jq --raw-input --compact-output 'split(" ")' <<<"${{ github.event.inputs.ruby }}")
           crdb_len=$(wc -w <<<"${{ github.event.inputs.crdb }}")
           ruby_len=$(wc -w <<<"${{ github.event.inputs.ruby }}")
-          (( range_count = ${{github.event.inputs.max}} / ( crdb_len * ruby_len ) ))
-          range=$(jq --compact-output "[range($range_count)]" <<<[])
+          (( seeds_count = ${{github.event.inputs.max}} / ( crdb_len * ruby_len ) ))
+          seeds=$(shuf --input-range=1-65535 --head-count=$seeds_count | jq --slurp --compact-output)
+          echo $seeds
           echo "crdb=$crdb" >> $GITHUB_OUTPUT
           echo "ruby=$ruby" >> $GITHUB_OUTPUT
-          echo "numbers=$range" >> $GITHUB_OUTPUT
+          echo "seeds=$seeds" >> $GITHUB_OUTPUT
     outputs:
       crdb: ${{ steps.generate-matrix.outputs.crdb }}
       ruby: ${{ steps.generate-matrix.outputs.ruby }}
-      numbers: ${{ steps.generate-matrix.outputs.numbers }}
+      seeds: ${{ steps.generate-matrix.outputs.seeds }}
   test:
     runs-on: ubuntu-latest
     needs: prepare-matrix
@@ -50,13 +51,63 @@ jobs:
       matrix:
         crdb: ${{ fromJSON(needs.prepare-matrix.outputs.crdb) }}
         ruby: ${{ fromJSON(needs.prepare-matrix.outputs.ruby) }}
-        number: ${{ fromJSON(needs.prepare-matrix.outputs.numbers) }}
-    name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }} number=${{ matrix.number }})
+        seed: ${{ fromJSON(needs.prepare-matrix.outputs.seeds) }}
+    name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }} seed=${{ matrix.seed }})
     steps:
       - name: Set Up Actions
         uses: actions/checkout@v4
       - uses: ./.github/actions/test-runner
+        id: test
         with:
           crdb: ${{ matrix.crdb }}
           ruby: ${{ matrix.ruby }}
           TESTOPTS: --fail-fast
+        env:
+          JSON_REPORTER: "report.json"
+          SEED: ${{ matrix.seed }}
+      - name: Upload Report
+        if: ${{ failure() && steps.test.conclusion == 'failure' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.crdb }}-${{ matrix.ruby }}-${{ matrix.seed }}
+          path: report.json
+  collect:
+    if: failure()
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Reports
+        uses: actions/download-artifact@v4
+        with:
+          path: reports
+      - name: Aggregate Reports
+        run: |
+          cat <<EOF >> $GITHUB_STEP_SUMMARY
+          # Failing Tests
+
+          <table>
+          <thead>
+            <tr>
+              <th>Seed</th>
+              <th>Time</th>
+              <th>Failure</th>
+            </tr>
+          </thead>
+          <tbody>
+          EOF
+
+          jq --slurp --raw-output '
+            sort_by(.total_time)[]
+            | {seed, time: .total_time | strftime("%H:%M:%S"), failure: .failed_tests[0].NAME }
+            | "<tr><td>\(.seed)</td><td>\(.time)</td><td>\(.failure)</td></tr>"
+          ' reports/*/report.json >> $GITHUB_STEP_SUMMARY
+
+
+          cat <<EOF >> $GITHUB_STEP_SUMMARY
+          </tbody>
+          </table>
+          EOF
+          echo >> $GITHUB_STEP_SUMMARY
+          echo '```json' >> $GITHUB_STEP_SUMMARY
+          jq --slurp --compact-output '.' reports/*/report.json >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY

CONTRIBUTING.md

@@ -61,23 +61,16 @@ Only do it if you know the schema was left in a correct state.
 
 ### Run Tests from a Backup
 
-Loading the full test schema every time a test runs can take a while, so for cases where loading the schema sequentially is unimportant, it is possible to use a backup to set up the database. This is significantly faster than the standard method and is provided to run individual tests faster, but should not be used to validate a build.
-
-First create the template database.
-
-```bash
-bundle exec rake db:create_test_template
-```
-
-This will create a template database for the current version (ex. `activerecord_test_template611` for version 6.1.1) and create a `BACKUP` in the `nodelocal://self/activerecord-crdb-adapter/#{activerecord_version}` directory.
-
-To load from the template, use the `COCKROACH_LOAD_FROM_TEMPLATE` flag.
-
-```bash
-COCKROACH_LOAD_FROM_TEMPLATE=1 TEST_FILES="test/cases/adapters/postgresql/ddl_test.rb" bundle exec rake test
-```
-
-And the `activerecord_unittest` database will use the `RESTORE` command to load the schema from the template database.
+Loading the full test schema every time a test runs can take
+a while, so for cases where loading the schema sequentially
+is unimportant, it is possible to use a backup to set up the
+database. This is significantly faster than the standard
+method and is provided to run individual tests faster, but
+should not be used to validate a build.
+
+To do so, just set the env variable `COCKROACH_LOAD_FROM_TEMPLATE`.
+First run will generate and cache a template, latter runs will use
+it.
 
 # Improvements
 

Gemfile

@@ -54,8 +54,8 @@ group :development, :test do
 
   gem "rake"
   gem "debug"
+  gem "minitest-bisect", github: "BuonOmo/minitest-bisect", branch: "main"
   gem "minitest-excludes", "~> 2.0.1"
-  gem "minitest-github_action_reporter", github: "BuonOmo/minitest-github_action_reporter", require: "minitest/github_action_reporter_plugin"
   gem "ostruct", "~> 0.6"
 
   # Gems used for tests meta-programming.

Rakefile

@@ -2,28 +2,9 @@ require "bundler/gem_tasks"
 require "rake/testtask"
 require_relative 'test/support/paths_cockroachdb'
 require_relative 'test/support/rake_helpers'
-require_relative 'test/support/template_creator'
 
 task default: [:test]
 
-namespace :db do
-  task "create_test_template" do
-    ENV['DEBUG_COCKROACHDB_ADAPTER'] = "1"
-    ENV['COCKROACH_SKIP_LOAD_SCHEMA'] = "1"
-
-    TemplateCreator.connect
-    require_relative 'test/cases/helper'
-
-    # TODO: look into this more, but for some reason the blob alias
-    # is not defined while running this task.
-    ActiveRecord::ConnectionAdapters::CockroachDB::TableDefinition.class_eval do
-      alias :blob :binary
-    end
-
-    TemplateCreator.create_test_template
-  end
-end
-
 Rake::TestTask.new do |t|
   t.libs = ARTest::CockroachDB.test_load_paths
   t.test_files = RakeHelpers.test_files

bin/minitest_bisect

@@ -0,0 +1,19 @@
+#!/usr/bin/env -S bundle exec ruby
+
+require 'rake/file_list'
+require_relative '../test/support/paths_cockroachdb'
+require_relative '../test/support/rake_helpers'
+
+libs = ARTest::CockroachDB.test_load_paths
+test_files = RakeHelpers.test_files
+
+Dir.chdir(File.dirname __dir__) do
+  system(
+    "bundle",
+    "exec",
+  	"minitest_bisect",
+  	"--seed=#{ARGV[0]}",
+  	"-I" + libs.join(":"),
+  	*test_files
+  )
+end

test/cases/adapter_test.rb

@@ -70,6 +70,7 @@ class AdapterForeignKeyTest < ActiveRecord::TestCase
     fixtures :fk_test_has_pk
 
     def before_setup
+      super
       conn = ActiveRecord::Base.lease_connection
 
       conn.drop_table :fk_test_has_fk, if_exists: true
@@ -87,6 +88,7 @@ def before_setup
     end
 
     def setup
+      super
       @connection = ActiveRecord::Base.lease_connection
     end