diff --git a/docs/cli/README.md b/docs/cli/README.md index 08e986a746..4385770452 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -80,6 +80,9 @@ vllm chat --url http://{vllm-serve-host}:{vllm-serve-port}/v1 # Quick chat with a single prompt vllm chat --quick "hi" + +# Print TTFT and throughput statistics after each response +vllm chat --stats ``` See [vllm chat](./chat.md) for the full reference of all available arguments. @@ -97,6 +100,9 @@ vllm complete --url http://{vllm-serve-host}:{vllm-serve-port}/v1 # Quick complete with a single prompt vllm complete --quick "The future of AI is" + +# Print TTFT and throughput statistics after each response +vllm complete --stats ``` See [vllm complete](./complete.md) for the full reference of all available arguments. diff --git a/vllm/entrypoints/cli/openai.py b/vllm/entrypoints/cli/openai.py index 1c18b193d1..c9869077c0 100644 --- a/vllm/entrypoints/cli/openai.py +++ b/vllm/entrypoints/cli/openai.py @@ -5,6 +5,7 @@ import argparse import os import signal import sys +import time from typing import TYPE_CHECKING from openai import OpenAI @@ -44,25 +45,58 @@ def _interactive_cli(args: argparse.Namespace) -> tuple[str, OpenAI]: return model_name, openai_client -def _print_chat_stream(stream) -> str: +def _print_chat_stream(stream, stats: bool = False) -> str: output = "" + start = time.perf_counter() + ttft: float | None = None + completion_tokens = 0 for chunk in stream: + if chunk.usage is not None: + completion_tokens = chunk.usage.completion_tokens + if not chunk.choices: + continue delta = chunk.choices[0].delta if delta.content: + if ttft is None: + ttft = time.perf_counter() - start output += delta.content print(delta.content, end="", flush=True) print() + if stats: + _print_metrics(start, ttft, completion_tokens) return output -def _print_completion_stream(stream) -> str: +def _print_metrics(start: float, ttft: float | None, completion_tokens: int) -> None: + total_time = time.perf_counter() - start + if ttft is None or total_time <= 0: + return + print(f"{'TTFT:':<5} {ttft * 1000:.2f} ms") + print( + f"{'TPS:':<5} {completion_tokens / total_time:.2f} tokens/s " + f"({completion_tokens} tokens in {total_time:.2f}s)" + ) + + +def _print_completion_stream(stream, stats: bool = False) -> str: output = "" + start = time.perf_counter() + ttft: float | None = None + completion_tokens = 0 for chunk in stream: + if chunk.usage is not None: + completion_tokens = chunk.usage.completion_tokens + if not chunk.choices: + continue text = chunk.choices[0].text - if text is not None: + if text: + if ttft is None: + ttft = time.perf_counter() - start output += text print(text, end="", flush=True) print() + if stats: + _print_metrics(start, ttft, completion_tokens) return output @@ -127,18 +161,23 @@ class ChatCommand(CLISubcommand): def cmd(args: argparse.Namespace) -> None: model_name, client = _interactive_cli(args) system_prompt = args.system_prompt + stats = args.stats conversation: list[ChatCompletionMessageParam] = [] if system_prompt is not None: conversation.append({"role": "system", "content": system_prompt}) + create_kwargs = {"model": model_name, "stream": True} + if stats: + create_kwargs["stream_options"] = {"include_usage": True} + if args.quick: conversation.append({"role": "user", "content": args.quick}) stream = client.chat.completions.create( - model=model_name, messages=conversation, stream=True + messages=conversation, **create_kwargs ) - output = _print_chat_stream(stream) + output = _print_chat_stream(stream, stats) conversation.append({"role": "assistant", "content": output}) return @@ -151,9 +190,9 @@ class ChatCommand(CLISubcommand): conversation.append({"role": "user", "content": input_message}) stream = client.chat.completions.create( - model=model_name, messages=conversation, stream=True + messages=conversation, **create_kwargs ) - output = _print_chat_stream(stream) + output = _print_chat_stream(stream, stats) conversation.append({"role": "assistant", "content": output}) @staticmethod @@ -176,6 +215,11 @@ class ChatCommand(CLISubcommand): metavar="MESSAGE", help=("Send a single prompt as MESSAGE and print the response, then exit."), ) + parser.add_argument( + "--stats", + action="store_true", + help="Print TTFT and TPS statistics after each response.", + ) return parser def subparser_init( @@ -198,6 +242,7 @@ class CompleteCommand(CLISubcommand): @staticmethod def cmd(args: argparse.Namespace) -> None: model_name, client = _interactive_cli(args) + stats = args.stats kwargs = { "model": model_name, @@ -205,10 +250,12 @@ class CompleteCommand(CLISubcommand): } if args.max_tokens: kwargs["max_tokens"] = args.max_tokens + if stats: + kwargs["stream_options"] = {"include_usage": True} if args.quick: stream = client.completions.create(prompt=args.quick, **kwargs) - _print_completion_stream(stream) + _print_completion_stream(stream, stats) return print("Please enter prompt to complete:") @@ -218,7 +265,7 @@ class CompleteCommand(CLISubcommand): except EOFError: break stream = client.completions.create(prompt=input_prompt, **kwargs) - _print_completion_stream(stream) + _print_completion_stream(stream, stats) @staticmethod def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: @@ -236,6 +283,11 @@ class CompleteCommand(CLISubcommand): metavar="PROMPT", help="Send a single prompt and print the completion output, then exit.", ) + parser.add_argument( + "--stats", + action="store_true", + help="Print TTFT and TPS statistics after each response.", + ) return parser def subparser_init(