diff --git a/docs/assets/contributing/vllm_bench_serve_dataset_stats.png b/docs/assets/contributing/vllm_bench_serve_dataset_stats.png
new file mode 100644
index 0000000000..72c19d3d7c
Binary files /dev/null and b/docs/assets/contributing/vllm_bench_serve_dataset_stats.png differ
diff --git a/docs/assets/contributing/vllm_bench_serve_timeline.html b/docs/assets/contributing/vllm_bench_serve_timeline.html
new file mode 100644
index 0000000000..d463e202b6
--- /dev/null
+++ b/docs/assets/contributing/vllm_bench_serve_timeline.html
@@ -0,0 +1,3888 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md
index a7e50c907d..c9ceb67cce 100644
--- a/docs/benchmarking/cli.md
+++ b/docs/benchmarking/cli.md
@@ -108,6 +108,38 @@ P99 ITL (ms): 8.39
==================================================
```
+#### Results Visualization
+
+The `--plot-timeline` and `--plot-dataset-stats` can be used to generate respectively the requests completion timeline and dataset prompt and output tokens statistics, which can be useful for debugging purpose or for deeper analysis.
+
+```bash
+vllm bench serve \
+ --backend vllm \
+ --model meta-llama/Llama-3.1-8B-Instruct \
+ --endpoint /v1/completions \
+ --dataset-name sharegpt \
+ --dataset-path /ShareGPT_V3_unfiltered_cleaned_split.json \
+ --num-prompts 100 \
+ --plot-timeline \
+ --timeline-itl-thresholds 2,5 \
+ --plot-dataset-stats \
+ --save-result
+```
+
+##### Interactive Timeline
+
+The generated timeline is an interactive visualization in the form of an HTML file that can be rendered in most browsers. To customize the ITL color thresholds, one can use `--timeline-itl-thresholds` flag (default: 25ms, 50ms)
+
+Example output:
+
+
+
+##### Dataset statistics
+
+The generated figure shows the input prompt and output tokens distribution.
+
+Example output: 
+
#### Custom Dataset
If the dataset you want to benchmark is not supported yet in vLLM, even then you can benchmark on it using `CustomDataset`. Your data needs to be in `.jsonl` format and needs to have "prompt" field per entry, e.g., data.jsonl
diff --git a/pyproject.toml b/pyproject.toml
index 5c87de018c..c7b77f0e91 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -123,7 +123,8 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer
"benchmarks/sonnet.txt", "tests/lora/data/*", "build/*",
"examples/pooling/token_embed/*", "tests/models/language/pooling/*",
"vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", "tests/entrypoints/openai/speech_to_text/test_transcription_validation.py",
- "docs/governance/process.md", "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*"]
+ "docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
+ "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*"]
ignore-hidden = false
[tool.typos.default]
diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py
index a4133d17ff..dc8b293a42 100644
--- a/vllm/benchmarks/serve.py
+++ b/vllm/benchmarks/serve.py
@@ -1611,14 +1611,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
)
parser.add_argument(
"--timeline-itl-thresholds",
- type=float,
- nargs=2,
- default=[25.0, 50.0],
- metavar=("THRESHOLD1", "THRESHOLD2"),
+ type=str,
+ default="25,50",
help="ITL thresholds in milliseconds for timeline plot coloring. "
- "Specify two values to categorize inter-token latencies into three groups: "
- "below first threshold (green), between thresholds (orange), "
- "and above second threshold (red). Default: 25 50 (milliseconds).",
+ "Specify two comma-separated values to categorize inter-token "
+ "latencies into three groups: below first threshold (green), "
+ "between thresholds (orange), and above second threshold (red).",
)
parser.add_argument(
"--plot-dataset-stats",
@@ -1637,6 +1635,19 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
random.seed(args.seed)
np.random.seed(args.seed)
+ # Validate timeline ITL thresholds
+ if args.plot_timeline:
+ try:
+ itl_thresholds = [
+ float(t.strip()) for t in args.timeline_itl_thresholds.split(",")
+ ]
+ if len(itl_thresholds) != 2:
+ raise ValueError(
+ f"Expected 2 ITL threshold values, got {len(itl_thresholds)}"
+ )
+ except ValueError as e:
+ raise ValueError(f"Invalid --timeline-itl-thresholds format: {e}") from e
+
# Validate ramp-up arguments
if args.ramp_up_strategy is not None:
if args.request_rate != float("inf"):
@@ -1906,7 +1917,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
timeline_path = Path(file_name).with_suffix(".timeline.html")
# Convert thresholds from milliseconds to seconds
- itl_thresholds_sec = [t / 1000.0 for t in args.timeline_itl_thresholds]
+ itl_thresholds_sec = [
+ float(t) / 1000.0 for t in args.timeline_itl_thresholds.split(",")
+ ]
generate_timeline_plot(
per_request_data, timeline_path, itl_thresholds=itl_thresholds_sec
)