Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions src/guidellm/benchmark/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,13 @@ async def finalize(self, report: GenerativeBenchmarksReport) -> Path:
benchmark_headers: list[str] = []
benchmark_values: list[str | float | list[float]] = []

# Add basic run description info
desc_headers, desc_values = (
self._get_benchmark_desc_headers_and_values(benchmark)
)
benchmark_headers.extend(desc_headers)
benchmark_values.extend(desc_values)

# Add status-based metrics
for status in StatusDistributionSummary.model_fields:
status_headers, status_values = (
Expand Down Expand Up @@ -672,6 +679,21 @@ def _get_benchmark_status_metrics_stats(
]
return headers, values

def _get_benchmark_extras_headers_and_values(
self, benchmark: GenerativeBenchmark,
) -> tuple[list[str], list[str]]:
headers = ["Profile", "Backend", "Generator Data"]
values: list[str] = [
benchmark.benchmarker.profile.model_dump_json(),
json.dumps(benchmark.benchmarker.backend),
json.dumps(benchmark.benchmarker.requests["attributes"]["data"]),
]

if len(headers) != len(values):
raise ValueError("Headers and values length mismatch.")

return headers, values


@GenerativeBenchmarkerOutput.register("html")
class GenerativeBenchmarkerHTML(GenerativeBenchmarkerOutput):
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/benchmark/test_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ def test_file_yaml():

mock_path.unlink()

@pytest.mark.skip(reason="CSV fix not merged yet")
@pytest.mark.asyncio
async def test_file_csv():
mock_benchmark = mock_generative_benchmark()
Expand All @@ -96,6 +95,7 @@ async def test_file_csv():
rows = list(reader)

assert "Type" in headers
assert "Profile" in headers
assert len(rows) == 1

mock_path.unlink()
Expand Down
6 changes: 5 additions & 1 deletion tests/unit/mock_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,11 @@ def mock_generative_benchmark() -> GenerativeBenchmark:
),
benchmarker=BenchmarkerDict(
profile=SynchronousProfile.create("synchronous", rate=None),
requests={},
requests={
"attributes": {
"data": "prompt_tokens=256,output_tokens=128",
},
},
backend={},
environment={},
aggregators={},
Expand Down
Loading