Skip to content

Commit 6e6b552

Browse files
authored
Merge pull request #1186 from lightninglabs/feat/itest-tranche-parallel
Add tranche-based parallel itests and enable in CI
2 parents af9f1d6 + 9a5e943 commit 6e6b552

File tree

7 files changed

+255
-12
lines changed

7 files changed

+255
-12
lines changed

.editorconfig

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# EditorConfig is awesome: https://EditorConfig.org
2+
3+
# Top-most EditorConfig file.
4+
root = true
5+
6+
# Unix-style newlines with a newline ending every file.
7+
[*.md]
8+
end_of_line = lf
9+
insert_final_newline = true
10+
max_line_length = 80
11+
12+
# 8 space indentation for Golang code.
13+
[*.go]
14+
indent_style = tab
15+
indent_size = 8
16+
max_line_length = 80

.github/workflows/main.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -279,13 +279,13 @@ jobs:
279279
matrix:
280280
include:
281281
- name: bbolt
282-
args: itest-no-backward-compat icase=terminal dbbackend=bbolt
282+
args: itest-parallel-no-backward-compat icase=terminal dbbackend=bbolt tranches=4
283283
- name: sqlite
284-
args: itest-no-backward-compat icase=terminal dbbackend=sqlite
284+
args: itest-parallel-no-backward-compat icase=terminal dbbackend=sqlite tranches=4
285285
- name: postgres
286-
args: itest-no-backward-compat icase=terminal dbbackend=postgres
286+
args: itest-parallel-no-backward-compat icase=terminal dbbackend=postgres tranches=4
287287
- name: custom-channels
288-
args: itest-only icase=custom_channels
288+
args: itest-parallel icase=custom_channels tranches=4
289289

290290
steps:
291291
- name: git checkout

Makefile

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,10 @@ build-itest:
239239
CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/btcd-itest -ldflags "$(ITEST_LDFLAGS)" $(BTCD_PKG)
240240
CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/lnd-itest -ldflags "$(ITEST_LDFLAGS)" $(LND_PKG)/cmd/lnd
241241

242+
build-itest-binary:
243+
@$(call print, "Building itest binary.")
244+
CGO_ENABLED=0 $(GOTEST) -v ./itest -tags="$(DEV_TAGS) $(ITEST_TAGS)" -c -o itest/itest.test
245+
242246
install-backward-compat-versions:
243247
@$(call print, "Installing old versions of litd for backward compatibility tests.")
244248
scripts/install-backward-compat-versions.sh '$(LITD_COMPAT_VERSIONS)'
@@ -254,9 +258,19 @@ run-itest-only:
254258

255259
itest-only: build-itest install-backward-compat-versions run-itest-only
256260

257-
itest: app-build build-itest itest-only
261+
itest: app-build itest-only
262+
263+
itest-no-backward-compat: app-build build-itest run-itest-only
258264

259-
itest-no-backward-compat: app-build build-itest build-itest run-itest-only
265+
itest-parallel: app-build build-itest install-backward-compat-versions build-itest-binary
266+
@$(call print, "Running integration tests in parallel.")
267+
rm -rf itest/*.log itest/.logs*; date
268+
scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
269+
270+
itest-parallel-no-backward-compat: app-build build-itest build-itest-binary
271+
@$(call print, "Running integration tests in parallel (no backward compat binaries).")
272+
rm -rf itest/*.log itest/.logs*; date
273+
scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
260274

261275
# =============
262276
# FLAKE HUNTING
@@ -349,5 +363,6 @@ flakehunter-unit:
349363
.PHONY: default all yarn-install build install go-build go-build-noui \
350364
go-install go-install-noui go-install-cli app-build release go-release \
351365
docker-release docker-tools scratch check unit unit-cover unit-race \
352-
clean-itest build-itest itest-only itest flake-unit fmt lint mod mod-check \
353-
list rpc protos protos-check rpc-js-compile clean
366+
clean-itest build-itest build-itest-binary itest-only itest \
367+
itest-parallel itest-parallel-no-backward-compat flake-unit fmt lint \
368+
mod mod-check list rpc protos protos-check rpc-js-compile clean

itest/litd_test.go

Lines changed: 125 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package itest
22

33
import (
4+
"flag"
5+
"fmt"
6+
"math/rand"
47
"os"
58
"strings"
69
"testing"
@@ -12,6 +15,44 @@ import (
1215
"github.com/stretchr/testify/require"
1316
)
1417

18+
const (
19+
// defaultSplitTranches is the default number of tranches to divide the
20+
// test suite into when no override is provided.
21+
defaultSplitTranches uint = 1
22+
23+
// defaultRunTranche is the default tranche index to execute when no
24+
// explicit tranche is selected.
25+
defaultRunTranche uint = 0
26+
)
27+
28+
var (
29+
// testCasesSplitTranches is the number of tranches the test cases
30+
// should be split into. By default this is set to 1, so no splitting
31+
// happens. If this value is increased, then the -runtranche flag must
32+
// be specified as well to indicate which part should be run in the
33+
// current invocation.
34+
testCasesSplitTranches = flag.Uint(
35+
"splittranches", defaultSplitTranches,
36+
"split the test cases in this many tranches and run the "+
37+
"tranche at 0-based index specified by the "+
38+
"-runtranche flag",
39+
)
40+
41+
// shuffleSeedFlag enables deterministic shuffling of test cases to
42+
// balance workload across tranches.
43+
shuffleSeedFlag = flag.Uint64(
44+
"shuffleseed", 0, "if set, shuffles the test cases using this "+
45+
"as the source of randomness",
46+
)
47+
48+
// testCasesRunTranche selects which tranche (0-based) to execute.
49+
testCasesRunTranche = flag.Uint(
50+
"runtranche", defaultRunTranche,
51+
"run the tranche of the split test cases with the given "+
52+
"(0-based) index",
53+
)
54+
)
55+
1556
// TestLightningTerminal performs a series of integration tests amongst a
1657
// programmatically driven network of lnd nodes.
1758
func TestLightningTerminal(t *testing.T) {
@@ -39,9 +80,18 @@ func TestLightningTerminal(t *testing.T) {
3980
"--rpcmiddleware.enable",
4081
}
4182

83+
testCases, trancheIndex, trancheOffset := selectTestTranche()
84+
totalTestCases := len(allTestCases)
85+
4286
// Run the subset of the test cases selected in this tranche.
43-
for _, testCase := range allTestCases {
44-
success := t.Run(testCase.name, func(t1 *testing.T) {
87+
for idx, testCase := range testCases {
88+
testOrdinal := int(trancheOffset) + idx + 1
89+
testName := fmt.Sprintf(
90+
"tranche%02d/%02d-of-%d/%s", int(trancheIndex),
91+
testOrdinal, totalTestCases, testCase.name,
92+
)
93+
94+
success := t.Run(testName, func(t1 *testing.T) {
4595
cleanTestCaseName := strings.ReplaceAll(
4696
testCase.name, " ", "_",
4797
)
@@ -107,6 +157,79 @@ func TestLightningTerminal(t *testing.T) {
107157
}
108158
}
109159

160+
// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is
161+
// set and not 0. This is used by parallel test runs to even out the work
162+
// across tranches.
163+
func maybeShuffleTestCases() {
164+
// Exit if not set or set to 0.
165+
if shuffleSeedFlag == nil || *shuffleSeedFlag == 0 {
166+
return
167+
}
168+
169+
// Init the seed and shuffle the test cases.
170+
// #nosec G404 -- This is not for cryptographic purposes.
171+
r := rand.New(rand.NewSource(int64(*shuffleSeedFlag)))
172+
r.Shuffle(len(allTestCases), func(i, j int) {
173+
allTestCases[i], allTestCases[j] =
174+
allTestCases[j], allTestCases[i]
175+
})
176+
}
177+
178+
// createIndices divides the number of test cases into pairs of indices that
179+
// specify the start and end of a tranche.
180+
func createIndices(numCases, numTranches uint) [][2]uint {
181+
base := numCases / numTranches
182+
remainder := numCases % numTranches
183+
184+
indices := make([][2]uint, numTranches)
185+
start := uint(0)
186+
187+
for i := uint(0); i < numTranches; i++ {
188+
end := start + base
189+
if i < remainder {
190+
end++
191+
}
192+
indices[i] = [2]uint{start, end}
193+
start = end
194+
}
195+
196+
return indices
197+
}
198+
199+
// selectTestTranche returns the sub slice of the test cases that should be run
200+
// as the current split tranche as well as the index and slice offset of the
201+
// tranche.
202+
func selectTestTranche() ([]*testCase, uint, uint) {
203+
numTranches := defaultSplitTranches
204+
if testCasesSplitTranches != nil {
205+
numTranches = *testCasesSplitTranches
206+
}
207+
runTranche := defaultRunTranche
208+
if testCasesRunTranche != nil {
209+
runTranche = *testCasesRunTranche
210+
}
211+
212+
// There's a special flake-hunt mode where we run the same test multiple
213+
// times in parallel. In that case the tranche index is equal to the
214+
// thread ID, but we need to actually run all tests for the regex
215+
// selection to work.
216+
threadID := runTranche
217+
if numTranches == 1 {
218+
runTranche = 0
219+
}
220+
221+
// Shuffle the test cases if the `shuffleseed` flag is set.
222+
maybeShuffleTestCases()
223+
224+
numCases := uint(len(allTestCases))
225+
indices := createIndices(numCases, numTranches)
226+
index := indices[runTranche]
227+
trancheOffset, trancheEnd := index[0], index[1]
228+
229+
return allTestCases[trancheOffset:trancheEnd], threadID,
230+
trancheOffset
231+
}
232+
110233
func init() {
111234
logger := btclog.NewSLogger(btclog.NewDefaultHandler(os.Stdout))
112235
UseLogger(logger.SubSystem(Subsystem))

make/testing_flags.mk

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,26 @@ include make/compile_flags.mk
33
TEST_FLAGS =
44
DEV_TAGS = dev
55

6+
NUM_ITEST_TRANCHES = 8
7+
ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
8+
SHUFFLE_SEED = 0
9+
10+
# Scale the number of parallel running itest tranches.
11+
ifneq ($(tranches),)
12+
NUM_ITEST_TRANCHES = $(tranches)
13+
ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
14+
endif
15+
16+
# Give the ability to run the same tranche multiple times at the same time.
17+
ifneq ($(parallel),)
18+
ITEST_PARALLELISM = $(parallel)
19+
endif
20+
21+
# Set the seed for shuffling the test cases.
22+
ifneq ($(shuffleseed),)
23+
SHUFFLE_SEED = $(shuffleseed)
24+
endif
25+
626
# Define the integration test.run filter if the icase argument was provided.
727
ifneq ($(icase),)
828
ITEST_FLAGS += -test.run="TestLightningTerminal/$(icase)"

scripts/itest_parallel.sh

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
# Get all the variables.
4+
PROCESSES=$1
5+
TRANCHES=$2
6+
SHUFFLE_SEED=$3
7+
8+
# Here we also shift 3 times and get the rest of our flags to pass on in $@.
9+
shift 3
10+
11+
# Create a variable to hold the final exit code.
12+
exit_code=0
13+
14+
# Run commands in parallel and track their PIDs.
15+
pids=()
16+
for ((i=0; i<PROCESSES; i++)); do
17+
scripts/itest_part.sh $i $TRANCHES $SHUFFLE_SEED "$@" &
18+
pids+=($!)
19+
done
20+
21+
# Wait for the processes created by xargs to finish.
22+
for pid in "${pids[@]}"; do
23+
wait $pid
24+
25+
# Once finished, grab its exit code.
26+
current_exit_code=$?
27+
28+
# Overwrite the exit code if current itest doesn't return 0.
29+
if [ $current_exit_code -ne 0 ]; then
30+
# Only write the exit code of the first failing itest.
31+
if [ $exit_code -eq 0 ]; then
32+
exit_code=$current_exit_code
33+
fi
34+
fi
35+
done
36+
37+
# Exit with the exit code of the first failing itest or 0.
38+
exit $exit_code

scripts/itest_part.sh

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,45 @@
33
# Let's work with absolute paths only, we run in the itest directory itself.
44
WORKDIR=$(pwd)/itest
55

6+
TRANCHE=0
7+
NUM_TRANCHES=1
8+
SHUFFLE_SEED=0
9+
10+
# If the first three arguments are integers, treat them as tranche settings.
11+
if [[ $# -ge 3 && "$1" =~ ^[0-9]+$ && "$2" =~ ^[0-9]+$ && "$3" =~ ^[0-9]+$ ]]; then
12+
TRANCHE=$1
13+
NUM_TRANCHES=$2
14+
SHUFFLE_SEED=$3
15+
shift 3
16+
fi
17+
618
# Windows insists on having the .exe suffix for an executable, we need to add
719
# that here if necessary.
820
EXEC="$WORKDIR"/itest.test
921
LITD_EXEC="$WORKDIR"/litd-itest
1022
BTCD_EXEC="$WORKDIR"/btcd-itest
11-
echo $EXEC -test.v "$@" -logoutput -logdir=.logs -litdexec=$LITD_EXEC -btcdexec=$BTCD_EXEC
23+
LOG_DIR="$WORKDIR/.logs"
24+
if [[ $NUM_TRANCHES -gt 1 ]]; then
25+
LOG_DIR="$WORKDIR/.logs/tranche$TRANCHE"
26+
fi
27+
28+
mkdir -p "$LOG_DIR"
29+
LOG_FILE="$LOG_DIR/output.log"
30+
31+
TRANCHE_FLAGS=(-splittranches="$NUM_TRANCHES" -runtranche="$TRANCHE" -shuffleseed="$SHUFFLE_SEED")
32+
33+
echo "$EXEC" -test.v "${TRANCHE_FLAGS[@]}" "$@" -logoutput -logdir="$LOG_DIR" -litdexec=$LITD_EXEC -btcdexec=$BTCD_EXEC
1234

1335
# Exit code 255 causes the parallel jobs to abort, so if one part fails the
1436
# other is aborted too.
1537
cd "$WORKDIR" || exit 255
16-
$EXEC -test.v "$@" -logoutput -logdir=.logs -litdexec=$LITD_EXEC -btcdexec=$BTCD_EXEC || exit 255
38+
$EXEC -test.v "${TRANCHE_FLAGS[@]}" "$@" -logoutput -logdir="$LOG_DIR" -litdexec=$LITD_EXEC -btcdexec=$BTCD_EXEC >"$LOG_FILE" 2>&1
39+
40+
exit_code=$?
41+
if [ $exit_code -ne 0 ]; then
42+
echo "Tranche $TRANCHE failed with exit code $exit_code"
43+
tail -n 100 "$LOG_FILE"
44+
exit 255
45+
else
46+
echo "Tranche $TRANCHE completed successfully"
47+
fi

0 commit comments

Comments
 (0)