chore: extract integration test into separate sub-module (#21846)

Signed-off-by: Michael Heinrichs <[email protected]>

Author: netopyr

.github/workflows/node-flow-build-application.yaml

@@ -128,6 +128,24 @@ jobs:
       codacy-project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
       codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
+  mats-integration-tests:
+    name: MATS - Integration Tests
+    uses: ./.github/workflows/node-zxc-compile-application-code.yaml
+    needs:
+      - dependency-check
+      - spotless
+    with:
+      custom-job-label: "MATS Integration Tests"
+      java-version: ${{ github.event.inputs.java-version || '21.0.6' }}
+      java-distribution: ${{ github.event.inputs.java-distribution || 'temurin' }}
+      enable-unit-tests: false
+      enable-integration-tests: ${{ github.event_name == 'push' || github.event.inputs.enable-unit-tests == 'true' }}
+      enable-network-log-capture: true
+    secrets:
+      access-token: ${{ secrets.GITHUB_TOKEN }}
+      gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }}
+      gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }}
+
   mats-hapi-tests-misc:
     name: MATS - HAPI Tests (Misc)
     uses: ./.github/workflows/node-zxc-compile-application-code.yaml
@@ -364,6 +382,7 @@ jobs:
     runs-on: hiero-citr-linux-medium
     needs:
       - mats-unit-tests
+      - mats-integration-tests
       - mats-hapi-tests-misc
       - mats-hapi-tests-misc-records
       - mats-hapi-tests-crypto
@@ -379,6 +398,7 @@ jobs:
       - mats-docker-determinism
     if: ${{ (github.event_name == 'push') &&
       needs.mats-unit-tests.result == 'success' &&
+      needs.mats-integration-tests.result == 'success' &&
       needs.mats-hapi-tests-misc.result == 'success' &&
       needs.mats-hapi-tests-misc-records.result == 'success' &&
       needs.mats-hapi-tests-crypto.result == 'success' &&
@@ -419,6 +439,7 @@ jobs:
       - dependency-check
       - spotless
       - mats-unit-tests
+      - mats-integration-tests
       - mats-hapi-tests-misc
       - mats-hapi-tests-misc-records
       - mats-hapi-tests-crypto
@@ -437,6 +458,7 @@ jobs:
       needs.dependency-check.result != 'success' ||
       needs.spotless.result != 'success' ||
       needs.mats-unit-tests.result != 'success' ||
+      needs.mats-integration-tests.result != 'success' ||
       needs.mats-hapi-tests-misc.result != 'success' ||
       needs.mats-hapi-tests-misc-records.result != 'success' ||
       needs.mats-hapi-tests-crypto.result != 'success' ||
@@ -522,6 +544,7 @@ jobs:
       - name: Calculate MATS Unit Test Result
         id: mats-unit-test-result
         if: ${{ needs.mats-unit-tests.result != 'success' ||
+          needs.mats-integration-tests.result != 'success' ||
           needs.mats-hapi-tests-misc.result != 'success' ||
           needs.mats-hapi-tests-misc-records.result != 'success' ||
           needs.mats-hapi-tests-crypto.result != 'success' ||

.github/workflows/node-flow-pull-request-checks.yaml

@@ -78,6 +78,22 @@ jobs:
       codacy-project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
       codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
+  integration-tests:
+    name: Integration Tests
+    uses: ./.github/workflows/node-zxc-compile-application-code.yaml
+    needs:
+      - dependency-check
+      - spotless
+    with:
+      custom-job-label: Standard
+      enable-unit-tests: false
+      enable-integration-tests: true
+      enable-network-log-capture: true
+    secrets:
+      access-token: ${{ secrets.GITHUB_TOKEN }}
+      gradle-cache-username: ${{ secrets.GRADLE_CACHE_USERNAME }}
+      gradle-cache-password: ${{ secrets.GRADLE_CACHE_PASSWORD }}
+
   hapi-tests-misc:
     name: HAPI Tests (Misc)
     uses: ./.github/workflows/node-zxc-compile-application-code.yaml

.github/workflows/node-zxc-compile-application-code.yaml

@@ -23,6 +23,11 @@ on:
         type: boolean
         required: false
         default: false
+      enable-integration-tests:
+        description: "Integration Testing Enabled"
+        type: boolean
+        required: false
+        default: false
       enable-hapi-tests-misc:
         description: "HAPI Testing (misc) Enabled"
         type: boolean
@@ -316,6 +321,28 @@ jobs:
           path: "**/build/test-results/hammer/TEST-*.xml"
           retention-days: 7
 
+      - name: Integration Tests
+        id: gradle-integration-tests
+        if: ${{ inputs.enable-integration-tests && steps.gradle-build.conclusion == 'success' && !cancelled() }}
+        run: ${GRADLE_EXEC} testIntegration --continue
+
+      - name: Publish Integration Test Report
+        uses: step-security/publish-unit-test-result-action@5d195d4dec0b9fa7b51a3dbc4298362a021247c7 # v2.20.4
+        if: ${{ inputs.enable-integration-tests && steps.gradle-build.conclusion == 'success' && !cancelled() }}
+        with:
+          check_name: "Node: Integration Test Results"
+          json_thousands_separator: ","
+          junit_files: "**/build/test-results/testIntegration/TEST-*.xml"
+          comment_mode: errors # only comment if we could not find or parse the JUnit XML files
+
+      - name: Upload Integration Test Report Artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ inputs.enable-integration-tests && steps.gradle-build.conclusion == 'success' && !cancelled() }}
+        with:
+          name: Integration Test Report
+          path: "**/build/test-results/testIntegration/TEST-*.xml"
+          retention-days: 7
+
       - name: HAPI Testing (Misc)
         id: gradle-hapi-misc
         if: ${{ inputs.enable-hapi-tests-misc && steps.gradle-build.conclusion == 'success' && !cancelled() }}
@@ -675,32 +702,50 @@ jobs:
             hedera-node/test-clients/build/*-test/**/output/**
           retention-days: 7
 
+      - name: Upload Otter Integration Tests Turtle Logs
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ inputs.enable-integration-tests && inputs.enable-network-log-capture && steps.gradle-integration-tests.conclusion == 'failure' && !cancelled() }}
+        with:
+          name: Otter Integration Tests Turtle Logs
+          path: |
+            platform-sdk/consensus-otter-tests/build/turtle/**
+          retention-days: 7
+
+      - name: Upload Otter Integration Tests Container Logs
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ inputs.enable-integration-tests && inputs.enable-network-log-capture && steps.gradle-integration-tests.conclusion == 'failure' && !cancelled() }}
+        with:
+          name: Otter Integration Tests Container Logs
+          path: |
+            platform-sdk/consensus-otter-tests/build/container/**
+          retention-days: 7
+
       - name: Otter Testing
         id: gradle-otter-tests
         if: ${{ inputs.enable-otter-tests && steps.gradle-build.conclusion == 'success' && !cancelled() }}
         env:
           LC_ALL: en.UTF-8
           LANG: en_US.UTF-8
-        run: ${GRADLE_EXEC} :consensus-otter-tests:testIntegration
+        run: ${GRADLE_EXEC} :consensus-otter-tests:testOtter
 
       - name: Publish Otter Testing Report
         uses: step-security/publish-unit-test-result-action@5d195d4dec0b9fa7b51a3dbc4298362a021247c7 # v2.20.4
         if: ${{ inputs.enable-otter-tests && steps.gradle-build.conclusion == 'success' && !cancelled() }}
         with:
           check_name: "Node: Otter Tests Results"
           json_thousands_separator: ","
-          junit_files: "**/consensus-otter-tests/build/test-results/testIntegration/TEST-*.xml"
+          junit_files: "**/consensus-otter-tests/build/test-results/testOtter/TEST-*.xml"
           comment_mode: errors # only comment if we could not find or parse the JUnit XML files
 
       - name: Upload Otter Tests Report Artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         if: ${{ inputs.enable-otter-tests && steps.gradle-build.conclusion == 'success' && !cancelled() }}
         with:
           name: Otter Tests Report
-          path: "**/consensus-otter-tests/build/test-results/testIntegration/TEST-*.xml"
+          path: "**/consensus-otter-tests/build/test-results/testOtter/TEST-*.xml"
           retention-days: 7
 
-      - name: Upload Otter Turtle Logs
+      - name: Upload Otter Tests Turtle Logs
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         if: ${{ inputs.enable-otter-tests && inputs.enable-network-log-capture && steps.gradle-otter-tests.conclusion == 'failure' && !cancelled() }}
         with:
@@ -709,7 +754,7 @@ jobs:
             platform-sdk/consensus-otter-tests/build/turtle/**
           retention-days: 7
 
-      - name: Upload Otter Container Logs
+      - name: Upload Otter Tests Container Logs
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         if: ${{ inputs.enable-otter-tests && inputs.enable-network-log-capture && steps.gradle-otter-tests.conclusion == 'failure' && !cancelled() }}
         with:
@@ -864,6 +909,7 @@ jobs:
             "${{ steps.gradle-timing-sensitive.outcome || 'skipped' }}"
             "${{ steps.gradle-time-consuming.outcome || 'skipped' }}"
             "${{ steps.gradle-hammer-tests.outcome || 'skipped' }}"
+            "${{ steps.gradle-integration-tests.outcome || 'skipped' }}"
             "${{ steps.gradle-hapi-misc.outcome || 'skipped' }}"
             "${{ steps.gradle-hapi-misc-records.outcome || 'skipped' }}"
             "${{ steps.gradle-hapi-crypto.outcome || 'skipped' }}"

platform-sdk/consensus-otter-tests/build.gradle.kts

@@ -23,6 +23,7 @@ plugins {
 description = "Consensus Otter Test Framework"
 
 testModuleInfo {
+    requires("com.swirlds.base")
     requires("com.swirlds.base.test.fixtures")
     requires("com.swirlds.common.test.fixtures")
     requires("com.swirlds.platform.core.test.fixtures")
@@ -35,18 +36,52 @@ testModuleInfo {
     requires("com.swirlds.metrics.api")
     requires("org.hiero.consensus.utility")
     requires("org.apache.logging.log4j")
-    requires("awaitility")
-    runtimeOnly("io.grpc.netty.shaded")
+}
+
+testing.suites {
+    val testOtter by
+        registering(JvmTestSuite::class) {
+            useJUnitJupiter()
+
+            dependencies {
+                implementation(project())
+                implementation(project.dependencies.testFixtures(project()))
+                implementation(project(":swirlds-common"))
+                implementation(project(":swirlds-platform-core"))
+                implementation(project(":base-crypto"))
+                implementation("org.junit.jupiter:junit-jupiter-params")
+                implementation("com.github.spotbugs:spotbugs-annotations")
+                runtimeOnly("io.grpc:grpc-netty-shaded")
+            }
+
+            targets {
+                all {
+                    testTask.configure {
+                        // Disable all parallelism
+                        systemProperty("junit.jupiter.execution.parallel.enabled", false)
+                        systemProperty(
+                            "junit.jupiter.testclass.order.default",
+                            "org.junit.jupiter.api.ClassOrderer\$OrderAnnotation",
+                        )
+
+                        // Limit heap and number of processors
+                        maxHeapSize = "8g"
+                        jvmArgs("-XX:ActiveProcessorCount=6")
+                    }
+                }
+            }
+        }
 }
 
 testIntegrationModuleInfo {
     requires("com.swirlds.common.test.fixtures")
     requires("com.swirlds.logging")
-    requires("org.hiero.base.crypto")
     requires("org.hiero.otter.fixtures")
     requires("org.assertj.core")
     requires("org.junit.jupiter.params")
     requires("com.github.spotbugs.annotations")
+    requires("org.apache.logging.log4j")
+    requires("awaitility")
     runtimeOnly("io.grpc.netty.shaded")
 }
 
@@ -60,8 +95,8 @@ tasks.compileTestFixturesJava {
 // Runs tests against the Turtle environment
 tasks.register<Test>("testTurtle") {
     useJUnitPlatform()
-    testClassesDirs = sourceSets.testIntegration.get().output.classesDirs
-    classpath = sourceSets.testIntegration.get().runtimeClasspath
+    testClassesDirs = sourceSets.named("testOtter").get().output.classesDirs
+    classpath = sourceSets.named("testOtter").get().runtimeClasspath
 
     // Disable all parallelism
     systemProperty("junit.jupiter.execution.parallel.enabled", false)
@@ -82,8 +117,8 @@ tasks.register<Test>("testContainer") {
     dependsOn(":consensus-otter-docker-app:copyDockerizedApp")
 
     useJUnitPlatform()
-    testClassesDirs = sourceSets.testIntegration.get().output.classesDirs
-    classpath = sourceSets.testIntegration.get().runtimeClasspath
+    testClassesDirs = sourceSets.named("testOtter").get().output.classesDirs
+    classpath = sourceSets.named("testOtter").get().runtimeClasspath
 
     // Disable all parallelism
     systemProperty("junit.jupiter.execution.parallel.enabled", false)

platform-sdk/consensus-otter-tests/src/test/java/org/hiero/otter/fixtures/NetworkIsolationTest.java renamed to platform-sdk/consensus-otter-tests/src/testIntegration/java/org/hiero/otter/fixtures/NetworkIsolationTest.java

@@ -14,7 +14,6 @@
 import org.hiero.otter.fixtures.network.Partition;
 import org.hiero.otter.fixtures.result.MultipleNodeLogResults;
 import org.hiero.otter.fixtures.turtle.TurtleTestEnvironment;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -40,7 +39,6 @@ public static Stream<TestEnvironment> environments() {
      */
     @ParameterizedTest
     @MethodSource("environments")
-    @Disabled("Disabled due to instability")
     void testIsolateAndRejoinSingleNode(@NonNull final TestEnvironment env) {
         try {
             final Network network = env.network();

platform-sdk/consensus-otter-tests/src/testIntegration/java/org/hiero/otter/zzz/NetworkPartitionTest.java renamed to platform-sdk/consensus-otter-tests/src/testIntegration/java/org/hiero/otter/fixtures/NetworkPartitionTest.java

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
-package org.hiero.otter.zzz;
+package org.hiero.otter.fixtures;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -13,10 +13,6 @@
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Stream;
-import org.hiero.otter.fixtures.Network;
-import org.hiero.otter.fixtures.Node;
-import org.hiero.otter.fixtures.TestEnvironment;
-import org.hiero.otter.fixtures.TimeManager;
 import org.hiero.otter.fixtures.container.ContainerTestEnvironment;
 import org.hiero.otter.fixtures.network.Partition;
 import org.hiero.otter.fixtures.result.MultipleNodeLogResults;