[CLI] Add flag to print TTFT and TPS in vllm chat (#46775)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Benjamin Chislett
2026-06-26 22:15:10 -07:00
committed by GitHub
parent d706dec904
commit 455f25aa13
2 changed files with 67 additions and 9 deletions
+6
View File
@@ -80,6 +80,9 @@ vllm chat --url http://{vllm-serve-host}:{vllm-serve-port}/v1
# Quick chat with a single prompt
vllm chat --quick "hi"
# Print TTFT and throughput statistics after each response
vllm chat --stats
```
See [vllm chat](./chat.md) for the full reference of all available arguments.
@@ -97,6 +100,9 @@ vllm complete --url http://{vllm-serve-host}:{vllm-serve-port}/v1
# Quick complete with a single prompt
vllm complete --quick "The future of AI is"
# Print TTFT and throughput statistics after each response
vllm complete --stats
```
See [vllm complete](./complete.md) for the full reference of all available arguments.
+61 -9
View File
@@ -5,6 +5,7 @@ import argparse
import os
import signal
import sys
import time
from typing import TYPE_CHECKING
from openai import OpenAI
@@ -44,25 +45,58 @@ def _interactive_cli(args: argparse.Namespace) -> tuple[str, OpenAI]:
return model_name, openai_client
def _print_chat_stream(stream) -> str:
def _print_chat_stream(stream, stats: bool = False) -> str:
output = ""
start = time.perf_counter()
ttft: float | None = None
completion_tokens = 0
for chunk in stream:
if chunk.usage is not None:
completion_tokens = chunk.usage.completion_tokens
if not chunk.choices:
continue
delta = chunk.choices[0].delta
if delta.content:
if ttft is None:
ttft = time.perf_counter() - start
output += delta.content
print(delta.content, end="", flush=True)
print()
if stats:
_print_metrics(start, ttft, completion_tokens)
return output
def _print_completion_stream(stream) -> str:
def _print_metrics(start: float, ttft: float | None, completion_tokens: int) -> None:
total_time = time.perf_counter() - start
if ttft is None or total_time <= 0:
return
print(f"{'TTFT:':<5} {ttft * 1000:.2f} ms")
print(
f"{'TPS:':<5} {completion_tokens / total_time:.2f} tokens/s "
f"({completion_tokens} tokens in {total_time:.2f}s)"
)
def _print_completion_stream(stream, stats: bool = False) -> str:
output = ""
start = time.perf_counter()
ttft: float | None = None
completion_tokens = 0
for chunk in stream:
if chunk.usage is not None:
completion_tokens = chunk.usage.completion_tokens
if not chunk.choices:
continue
text = chunk.choices[0].text
if text is not None:
if text:
if ttft is None:
ttft = time.perf_counter() - start
output += text
print(text, end="", flush=True)
print()
if stats:
_print_metrics(start, ttft, completion_tokens)
return output
@@ -127,18 +161,23 @@ class ChatCommand(CLISubcommand):
def cmd(args: argparse.Namespace) -> None:
model_name, client = _interactive_cli(args)
system_prompt = args.system_prompt
stats = args.stats
conversation: list[ChatCompletionMessageParam] = []
if system_prompt is not None:
conversation.append({"role": "system", "content": system_prompt})
create_kwargs = {"model": model_name, "stream": True}
if stats:
create_kwargs["stream_options"] = {"include_usage": True}
if args.quick:
conversation.append({"role": "user", "content": args.quick})
stream = client.chat.completions.create(
model=model_name, messages=conversation, stream=True
messages=conversation, **create_kwargs
)
output = _print_chat_stream(stream)
output = _print_chat_stream(stream, stats)
conversation.append({"role": "assistant", "content": output})
return
@@ -151,9 +190,9 @@ class ChatCommand(CLISubcommand):
conversation.append({"role": "user", "content": input_message})
stream = client.chat.completions.create(
model=model_name, messages=conversation, stream=True
messages=conversation, **create_kwargs
)
output = _print_chat_stream(stream)
output = _print_chat_stream(stream, stats)
conversation.append({"role": "assistant", "content": output})
@staticmethod
@@ -176,6 +215,11 @@ class ChatCommand(CLISubcommand):
metavar="MESSAGE",
help=("Send a single prompt as MESSAGE and print the response, then exit."),
)
parser.add_argument(
"--stats",
action="store_true",
help="Print TTFT and TPS statistics after each response.",
)
return parser
def subparser_init(
@@ -198,6 +242,7 @@ class CompleteCommand(CLISubcommand):
@staticmethod
def cmd(args: argparse.Namespace) -> None:
model_name, client = _interactive_cli(args)
stats = args.stats
kwargs = {
"model": model_name,
@@ -205,10 +250,12 @@ class CompleteCommand(CLISubcommand):
}
if args.max_tokens:
kwargs["max_tokens"] = args.max_tokens
if stats:
kwargs["stream_options"] = {"include_usage": True}
if args.quick:
stream = client.completions.create(prompt=args.quick, **kwargs)
_print_completion_stream(stream)
_print_completion_stream(stream, stats)
return
print("Please enter prompt to complete:")
@@ -218,7 +265,7 @@ class CompleteCommand(CLISubcommand):
except EOFError:
break
stream = client.completions.create(prompt=input_prompt, **kwargs)
_print_completion_stream(stream)
_print_completion_stream(stream, stats)
@staticmethod
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
@@ -236,6 +283,11 @@ class CompleteCommand(CLISubcommand):
metavar="PROMPT",
help="Send a single prompt and print the completion output, then exit.",
)
parser.add_argument(
"--stats",
action="store_true",
help="Print TTFT and TPS statistics after each response.",
)
return parser
def subparser_init(