From 57ac7e15b8f92da31fe206f8e1baa54727a3f909 Mon Sep 17 00:00:00 2001 From: Ulysse Buonomo Date: Thu, 9 Oct 2025 13:56:02 +0200 Subject: [PATCH 1/2] feat(ci): add flaky test detector Add a manually triggered workflow that runs a lot of tests in parallel with the `--fail-fast` option set. There are two goals here: - Detect new flaky tests - Get test seeds for fast failing flaky tests for an easier local reproduction Signed-off-by: Ulysse Buonomo --- .github/actions/test-runner/action.yml | 70 ++++++++++++++++++++++++++ .github/workflows/ci.yml | 54 ++------------------ .github/workflows/flaky.yml | 62 +++++++++++++++++++++++ .gitignore | 1 + test/cases/helper_cockroachdb.rb | 2 +- test/support/sql_logger.rb | 4 +- 6 files changed, 141 insertions(+), 52 deletions(-) create mode 100644 .github/actions/test-runner/action.yml create mode 100644 .github/workflows/flaky.yml diff --git a/.github/actions/test-runner/action.yml b/.github/actions/test-runner/action.yml new file mode 100644 index 00000000..373cdefa --- /dev/null +++ b/.github/actions/test-runner/action.yml @@ -0,0 +1,70 @@ +# Schema: https://json.schemastore.org/github-action.json + +name: Test Runner +description: Prepare and run tests for this repository + +inputs: + crdb: + description: "CockroachDB version" + required: true + ruby: + description: "Ruby version" + required: true + TESTOPTS: + description: "Rails Minitest options" + default: "--profile=5" +runs: + using: composite + steps: + - name: Install GEOS + shell: bash + run: sudo apt-get install -yqq libgeos-dev + - name: Set Up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ inputs.ruby }} + bundler-cache: true + - name: Show Rails version + shell: bash + run: bundle info rails + - name: Install and Start Cockroachdb + shell: bash + run: | + # Download CockroachDB + readonly full_version=$(ruby -rnet/http -ruri -ryaml -e ' + link = "https://raw.githubusercontent.com/cockroachdb/docs/main/src/current/_data/releases.yml" + puts YAML.safe_load(Net::HTTP.get(URI(link))).reverse.find { + _1["major_version"] == "${{ inputs.crdb }}" && + _1["release_type"] == "Production" && + !_1["cloud_only"] && + !_1["withdrawn"] && + !_1["release_name"].include?("-") # Pre-release + }["release_name"] + ') + + echo "Downloading $full_version..." + wget -qO- "https://binaries.cockroachdb.com/cockroach-$full_version.linux-amd64.tgz" | tar xvz + + export PATH=./cockroach-$full_version.linux-amd64/:$PATH + readonly urlfile=cockroach-url + + # Start a CockroachDB server and wait for it to become ready. + rm -f "$urlfile" + rm -rf cockroach-data + # Start CockroachDB. + cockroach start-single-node --max-sql-memory=25% --cache=25% --insecure --host=localhost --spatial-libs=./cockroach-$full_version.linux-amd64/lib --listening-url-file="$urlfile" >/dev/null 2>&1 & + # Ensure CockroachDB is stopped on script exit. + # Wait until CockroachDB has started. + for i in {0..3}; do + [[ -f "$urlfile" ]] && break + backoff=$((2 ** i)) + echo "server not yet available; sleeping for $backoff seconds" + sleep $backoff + done + cat ${{ github.workspace }}/setup.sql | cockroach sql --insecure + - name: Test + shell: bash + run: bundle exec rake test + env: + TESTOPTS: "${{ inputs.TESTOPTS }}" # Fail fast allows us to find easy local reproduction by isolating quickly failing jobs. + RAILS_MINITEST_PLUGIN: "1" # Make sure that we use the minitest plugin for profiling. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fb6126e..d2797ec4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,57 +40,13 @@ jobs: fail-fast: false matrix: # https://www.cockroachlabs.com/docs/releases/release-support-policy - crdb: [v24.1, v24.3, v25.1, v25.2] + crdb: [v24.3, v25.1, v25.2, v25.3] ruby: ["3.4"] - name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }}) + name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }} steps: - name: Set Up Actions uses: actions/checkout@v4 - - name: Install GEOS - run: sudo apt-get install -yqq libgeos-dev - - name: Set Up Ruby - uses: ruby/setup-ruby@v1 + - uses: ./.github/actions/test-runner with: - ruby-version: ${{ matrix.ruby }} - bundler-cache: true - - name: Show Rails version - run: bundle info rails - - name: Install and Start Cockroachdb - run: | - # Download CockroachDB - readonly full_version=$(ruby -rnet/http -ruri -ryaml -e ' - link = "https://raw.githubusercontent.com/cockroachdb/docs/main/src/current/_data/releases.yml" - puts YAML.safe_load(Net::HTTP.get(URI(link))).reverse.find { - _1["major_version"] == "${{ matrix.crdb }}" && - _1["release_type"] == "Production" && - !_1["cloud_only"] && - !_1["withdrawn"] && - !_1["release_name"].include?("-") # Pre-release - }["release_name"] - ') - - echo "Downloading $full_version..." - wget -qO- "https://binaries.cockroachdb.com/cockroach-$full_version.linux-amd64.tgz" | tar xvz - - export PATH=./cockroach-$full_version.linux-amd64/:$PATH - readonly urlfile=cockroach-url - - # Start a CockroachDB server and wait for it to become ready. - rm -f "$urlfile" - rm -rf cockroach-data - # Start CockroachDB. - cockroach start-single-node --max-sql-memory=25% --cache=25% --insecure --host=localhost --spatial-libs=./cockroach-$full_version.linux-amd64/lib --listening-url-file="$urlfile" >/dev/null 2>&1 & - # Ensure CockroachDB is stopped on script exit. - # Wait until CockroachDB has started. - for i in {0..3}; do - [[ -f "$urlfile" ]] && break - backoff=$((2 ** i)) - echo "server not yet available; sleeping for $backoff seconds" - sleep $backoff - done - cat ${{ github.workspace }}/setup.sql | cockroach sql --insecure - - name: Test - run: bundle exec rake test - env: - TESTOPTS: "--profile=5" - RAILS_MINITEST_PLUGIN: "1" # Make sure that we use the minitest plugin for profiling. + crdb: ${{ matrix.crdb }} + ruby: ${{ matrix.ruby }} diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml new file mode 100644 index 00000000..71d48bc6 --- /dev/null +++ b/.github/workflows/flaky.yml @@ -0,0 +1,62 @@ +name: Flaky Test Detector + +# Only run this workflow manually from the Actions tab +on: + workflow_dispatch: + inputs: + ruby: + description: "Ruby version(s) (space separated)" + required: true + default: "3.4" + crdb: + description: "CockroachDB version(s) (space separated)" + required: true + default: "v25.2" + max: + description: "Maximum number of tests" + required: true + default: "256" # Maximum number of jobs on GitHub Actions + +# This allows a subsequently queued workflow run to interrupt previous runs. +concurrency: + group: "${{ github.workflow }} @ ${{ github.ref }}" + cancel-in-progress: true + +jobs: + prepare-matrix: + runs-on: ubuntu-latest + name: Prepare Matrix + steps: + - id: generate-matrix + run: | + crdb=$(jq --raw-input --compact-output 'split(" ")' <<<"${{ github.event.inputs.crdb }}") + ruby=$(jq --raw-input --compact-output 'split(" ")' <<<"${{ github.event.inputs.ruby }}") + crdb_len=$(wc -w <<<"${{ github.event.inputs.crdb }}") + ruby_len=$(wc -w <<<"${{ github.event.inputs.ruby }}") + (( range_count = ${{github.event.inputs.max}} / ( crdb_len * ruby_len ) )) + range=$(jq --compact-output "[range($range_count)]" <<<[]) + echo "crdb=$crdb" >> $GITHUB_OUTPUT + echo "ruby=$ruby" >> $GITHUB_OUTPUT + echo "numbers=$range" >> $GITHUB_OUTPUT + outputs: + crdb: ${{ steps.generate-matrix.outputs.crdb }} + ruby: ${{ steps.generate-matrix.outputs.ruby }} + numbers: ${{ steps.generate-matrix.outputs.numbers }} + test: + runs-on: ubuntu-latest + needs: prepare-matrix + strategy: + fail-fast: false + matrix: + crdb: ${{ fromJSON(needs.prepare-matrix.outputs.crdb) }} + ruby: ${{ fromJSON(needs.prepare-matrix.outputs.ruby) }} + number: ${{ fromJSON(needs.prepare-matrix.outputs.numbers) }} + name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }} number=${{ matrix.number }}) + steps: + - name: Set Up Actions + uses: actions/checkout@v4 + - uses: ./.github/actions/test-runner + with: + crdb: ${{ matrix.crdb }} + ruby: ${{ matrix.ruby }} + TESTOPTS: --fail-fast diff --git a/.gitignore b/.gitignore index c39c52e2..16cd5bd8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ /ruby-build/ /test/db/ /.ruby-version +*.sqlite3 diff --git a/test/cases/helper_cockroachdb.rb b/test/cases/helper_cockroachdb.rb index 04d7321e..1f453420 100644 --- a/test/cases/helper_cockroachdb.rb +++ b/test/cases/helper_cockroachdb.rb @@ -189,7 +189,7 @@ def before_teardown super end end - MiniTest::Test.include(TraceLibPlugin) + Minitest::Test.include(TraceLibPlugin) end # Log all SQL queries and print total time spent in SQL. diff --git a/test/support/sql_logger.rb b/test/support/sql_logger.rb index e548e05e..ec48b30f 100644 --- a/test/support/sql_logger.rb +++ b/test/support/sql_logger.rb @@ -20,12 +20,12 @@ def summary_log detail = ActiveRecord::TotalTimeSubscriber.hash.map { |k,v| [k, [v.sum, v.sum / v.size, v.size]]}.sort_by { |_, (_total, avg, _)| -avg }.to_h time = detail.values.sum { |(total, _, _)| total } / 1_000 count = detail.values.sum { |(_, _, count)| count } - puts "Total time spent in SQL: #{time}s (#{count} queries)" - puts "Detail per query kind available in tmp/query_time.json (total time in ms, avg time in ms, query count). Sorted by avg time." File.write( "tmp/query_time.json", JSON.pretty_generate(detail) ) + puts "Total time spent in SQL: #{time}s (#{count} queries)" + puts "Detail per query kind available in tmp/query_time.json (total time in ms, avg time in ms, query count). Sorted by avg time." } end From 341c20fc582f91634760d9c69ab814df60481985 Mon Sep 17 00:00:00 2001 From: Ulysse Buonomo Date: Mon, 13 Oct 2025 13:46:18 +0200 Subject: [PATCH 2/2] fix(tests): clear query cache on insert Since [rails pull request #52428][1], `#execute_batch` does not trigger a cache clear anymore. However, `#insert_fixtures_set` relies on that clear to ensure consistency. In the postgresql adapter, this is ensured by a call to `#execute` rather than `#execute_batch` in `#disable_referential_integrity`. Since we are not always calling `#disable_referential_integrity`, we need to ensure that the cache is cleared when running our statements by calling `#execute` instead of `#execute_batch`. [1]: https://github.com/rails/rails/pull/52428 Signed-off-by: Ulysse Buonomo --- .../cockroachdb/database_statements.rb | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/active_record/connection_adapters/cockroachdb/database_statements.rb b/lib/active_record/connection_adapters/cockroachdb/database_statements.rb index 7f1011d3..100b4ca0 100644 --- a/lib/active_record/connection_adapters/cockroachdb/database_statements.rb +++ b/lib/active_record/connection_adapters/cockroachdb/database_statements.rb @@ -22,15 +22,26 @@ module DatabaseStatements def insert_fixtures_set(fixture_set, tables_to_delete = []) fixture_inserts = build_fixture_statements(fixture_set) table_deletes = tables_to_delete.map { |table| "DELETE FROM #{quote_table_name(table)}" } - statements = table_deletes + fixture_inserts + statements = (table_deletes + fixture_inserts).join(";") + + # Since [rails pull request #52428][1], `#execute_batch` does not + # trigger a cache clear anymore. However, `#insert_fixtures_set` + # relies on that clear to ensure consistency. In the postgresql + # adapter, this is ensured by a call to `#execute` rather than + # `#execute_batch` in `#disable_referential_integrity`. Since + # we are not always calling `#disable_referential_integrity`, + # we need to ensure that the cache is cleared when running + # our statements by calling `#execute` instead of `#execute_batch`. + # + # [1]: https://github.com/rails/rails/pull/52428 begin # much faster without disabling referential integrity, worth trying. transaction(requires_new: true) do - execute_batch(statements, "Fixtures Load") + execute(statements, "Fixtures Load") end rescue disable_referential_integrity do - execute_batch(statements, "Fixtures Load") + execute(statements, "Fixtures Load") end end end