Skip to content

Commit 4b3bb9b

Browse files
sjmonsonsjmonson
committed
Drop benchmark report delete in e2e
Signed-off-by: Samuel Monson <[email protected]>
1 parent f44cc79 commit 4b3bb9b

File tree

4 files changed

+63
-99
lines changed

4 files changed

+63
-99
lines changed

‎tests/e2e/test_max_error_benchmark.py‎

Lines changed: 22 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
GuidellmClient,
99
assert_constraint_triggered,
1010
assert_no_python_exceptions,
11-
cleanup_report_file,
1211
load_benchmark_report,
1312
)
1413
from tests.e2e.vllm_sim_server import VllmSimServer
@@ -51,33 +50,29 @@ def test_max_error_benchmark(server: VllmSimServer, tmp_path: Path):
5150
outputs=report_name,
5251
)
5352

54-
try:
55-
# Start the benchmark
56-
client.start_benchmark(
57-
rate=rate,
58-
max_seconds=25,
59-
max_error_rate=max_error_rate,
60-
)
61-
62-
# Wait for the benchmark to complete (server will be stopped after 15 seconds)
63-
client.wait_for_completion(timeout=30, stop_server_after=15, server=server)
53+
# Start the benchmark
54+
client.start_benchmark(
55+
rate=rate,
56+
max_seconds=25,
57+
max_error_rate=max_error_rate,
58+
)
6459

65-
# Assert no Python exceptions occurred
66-
assert_no_python_exceptions(client.stderr)
60+
# Wait for the benchmark to complete (server will be stopped after 15 seconds)
61+
client.wait_for_completion(timeout=30, stop_server_after=15, server=server)
6762

68-
# Load and validate the report
69-
report = load_benchmark_report(report_path)
70-
benchmark = report["benchmarks"][0]
63+
# Assert no Python exceptions occurred
64+
assert_no_python_exceptions(client.stderr)
7165

72-
# Check that the max error rate constraint was triggered
73-
assert_constraint_triggered(
74-
benchmark,
75-
"max_error_rate",
76-
{
77-
"exceeded_error_rate": True,
78-
"current_error_rate": lambda rate: rate >= max_error_rate,
79-
},
80-
)
66+
# Load and validate the report
67+
report = load_benchmark_report(report_path)
68+
benchmark = report["benchmarks"][0]
8169

82-
finally:
83-
cleanup_report_file(report_path)
70+
# Check that the max error rate constraint was triggered
71+
assert_constraint_triggered(
72+
benchmark,
73+
"max_error_rate",
74+
{
75+
"exceeded_error_rate": True,
76+
"current_error_rate": lambda rate: rate >= max_error_rate,
77+
},
78+
)

‎tests/e2e/test_over_saturated_benchmark.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
GuidellmClient,
77
assert_constraint_triggered,
88
assert_no_python_exceptions,
9-
cleanup_report_file,
109
load_benchmark_report,
1110
)
1211
from tests.e2e.vllm_sim_server import VllmSimServer
@@ -49,7 +48,6 @@ def test_over_saturated_benchmark(server: VllmSimServer, tmp_path: Path):
4948
outputs=report_name,
5049
)
5150

52-
cleanup_report_file(report_path)
5351
# Start the benchmark with --default-over-saturation flag
5452
client.start_benchmark(
5553
rate=rate,
@@ -75,8 +73,6 @@ def test_over_saturated_benchmark(server: VllmSimServer, tmp_path: Path):
7573
benchmark, "over_saturation", {"is_over_saturated": True}
7674
)
7775

78-
cleanup_report_file(report_path)
79-
8076

8177
@pytest.mark.timeout(60)
8278
def test_over_saturated_benchmark_with_dict_config(
@@ -96,7 +92,6 @@ def test_over_saturated_benchmark_with_dict_config(
9692
outputs=report_name,
9793
)
9894

99-
cleanup_report_file(report_path)
10095
# Start the benchmark with dictionary configuration for over-saturation
10196
client.start_benchmark(
10297
rate=rate,
@@ -127,5 +122,3 @@ def test_over_saturated_benchmark_with_dict_config(
127122
assert_constraint_triggered(
128123
benchmark, "over_saturation", {"is_over_saturated": True}
129124
)
130-
131-
cleanup_report_file(report_path)

‎tests/e2e/test_successful_benchmark.py‎

Lines changed: 41 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
assert_constraint_triggered,
1010
assert_no_python_exceptions,
1111
assert_successful_requests_fields,
12-
cleanup_report_file,
1312
load_benchmark_report,
1413
)
1514
from tests.e2e.vllm_sim_server import VllmSimServer
@@ -53,34 +52,28 @@ def test_max_seconds_benchmark(server: VllmSimServer, tmp_path: Path):
5352
outputs=report_name,
5453
)
5554

56-
try:
57-
# Start the benchmark
58-
client.start_benchmark(
59-
rate=rate,
60-
max_seconds=max_seconds,
61-
)
62-
63-
# Wait for the benchmark to complete
64-
client.wait_for_completion(timeout=30)
55+
# Start the benchmark
56+
client.start_benchmark(
57+
rate=rate,
58+
max_seconds=max_seconds,
59+
)
6560

66-
# Assert no Python exceptions occurred
67-
assert_no_python_exceptions(client.stderr)
61+
# Wait for the benchmark to complete
62+
client.wait_for_completion(timeout=30)
6863

69-
# Load and validate the report
70-
report = load_benchmark_report(report_path)
71-
benchmark = report["benchmarks"][0]
64+
# Assert no Python exceptions occurred
65+
assert_no_python_exceptions(client.stderr)
7266

73-
# Check that the max duration constraint was triggered
74-
assert_constraint_triggered(
75-
benchmark, "max_seconds", {"duration_exceeded": True}
76-
)
67+
# Load and validate the report
68+
report = load_benchmark_report(report_path)
69+
benchmark = report["benchmarks"][0]
7770

78-
# Validate successful requests have all expected fields
79-
successful_requests = benchmark["requests"]["successful"]
80-
assert_successful_requests_fields(successful_requests)
71+
# Check that the max duration constraint was triggered
72+
assert_constraint_triggered(benchmark, "max_seconds", {"duration_exceeded": True})
8173

82-
finally:
83-
cleanup_report_file(report_path)
74+
# Validate successful requests have all expected fields
75+
successful_requests = benchmark["requests"]["successful"]
76+
assert_successful_requests_fields(successful_requests)
8477

8578

8679
@pytest.mark.timeout(30)
@@ -102,35 +95,28 @@ def test_max_requests_benchmark(server: VllmSimServer, tmp_path: Path):
10295
outputs=report_name,
10396
)
10497

105-
try:
106-
# Start the benchmark
107-
client.start_benchmark(
108-
rate=rate,
109-
max_requests=max_requests,
110-
)
111-
112-
# Wait for the benchmark to complete
113-
client.wait_for_completion(timeout=30)
114-
115-
# Assert no Python exceptions occurred
116-
assert_no_python_exceptions(client.stderr)
117-
118-
# Load and validate the report
119-
report = load_benchmark_report(report_path)
120-
benchmark = report["benchmarks"][0]
121-
122-
# Check that the max requests constraint was triggered
123-
assert_constraint_triggered(
124-
benchmark, "max_requests", {"processed_exceeded": True}
125-
)
126-
127-
# Validate successful requests have all expected fields
128-
successful_requests = benchmark["requests"]["successful"]
129-
assert len(successful_requests) == max_requests, (
130-
f"Expected {max_requests} successful requests, "
131-
f"got {len(successful_requests)}"
132-
)
133-
assert_successful_requests_fields(successful_requests)
98+
# Start the benchmark
99+
client.start_benchmark(
100+
rate=rate,
101+
max_requests=max_requests,
102+
)
134103

135-
finally:
136-
cleanup_report_file(report_path)
104+
# Wait for the benchmark to complete
105+
client.wait_for_completion(timeout=30)
106+
107+
# Assert no Python exceptions occurred
108+
assert_no_python_exceptions(client.stderr)
109+
110+
# Load and validate the report
111+
report = load_benchmark_report(report_path)
112+
benchmark = report["benchmarks"][0]
113+
114+
# Check that the max requests constraint was triggered
115+
assert_constraint_triggered(benchmark, "max_requests", {"processed_exceeded": True})
116+
117+
# Validate successful requests have all expected fields
118+
successful_requests = benchmark["requests"]["successful"]
119+
assert len(successful_requests) == max_requests, (
120+
f"Expected {max_requests} successful requests, got {len(successful_requests)}"
121+
)
122+
assert_successful_requests_fields(successful_requests)

‎tests/e2e/utils.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -341,13 +341,3 @@ def assert_constraint_triggered(
341341
assert actual_value == expected_value, (
342342
f"Expected {key}={expected_value}, got {actual_value}"
343343
)
344-
345-
346-
def cleanup_report_file(report_path: Path) -> None:
347-
"""
348-
Clean up the report file if it exists.
349-
350-
:param report_path: Path to the report file to remove
351-
"""
352-
if report_path.exists():
353-
report_path.unlink()

0 commit comments

Comments
 (0)