From 75d2e21bdb7c0e2c97a679e2d79c47be75b0f056 Mon Sep 17 00:00:00 2001 From: cascade812 Date: Fri, 23 May 2025 22:36:07 +0000 Subject: [PATCH 1/2] Disable prefix cache by default for benchmark Signed-off-by: cascade812 --- benchmarks/benchmark_latency.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/benchmark_latency.py b/benchmarks/benchmark_latency.py index d5aaceeb8c9c..84759c5c354d 100644 --- a/benchmarks/benchmark_latency.py +++ b/benchmarks/benchmark_latency.py @@ -189,5 +189,8 @@ def run_to_completion(profile_dir: Optional[str] = None): ) parser = EngineArgs.add_cli_args(parser) + # V1 enables prefix caching by default which skews the latency + # numbers. We need to disable prefix caching by default. + parser.set_defaults(enable_prefix_caching=False) args = parser.parse_args() main(args) From cfe28ba84179347d85a4e0f646aca3f86e8ae091 Mon Sep 17 00:00:00 2001 From: cascade812 Date: Mon, 26 May 2025 02:22:03 +0000 Subject: [PATCH 2/2] address comment Signed-off-by: cascade812 --- vllm/benchmarks/latency.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/benchmarks/latency.py b/vllm/benchmarks/latency.py index 06f6848f50cb..2c992727b139 100644 --- a/vllm/benchmarks/latency.py +++ b/vllm/benchmarks/latency.py @@ -80,6 +80,9 @@ def add_cli_args(parser: argparse.ArgumentParser): ) parser = EngineArgs.add_cli_args(parser) + # V1 enables prefix caching by default which skews the latency + # numbers. We need to disable prefix caching by default. + parser.set_defaults(enable_prefix_caching=True) def main(args: argparse.Namespace):