From bd3bb4eb2635dc93749ef22c2558b98fa0519823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Wed, 8 Jul 2026 11:43:34 +0200 Subject: [PATCH] [Misc][Docs] Add human-readable integer support for more cli-args (#47608) Signed-off-by: NickLucche --- docs/cli/README.md | 15 +++++++++++++++ tests/engine/test_arg_utils.py | 34 ++++++++++++++++++++++++++++++++++ vllm/engine/arg_utils.py | 22 +++++++++++++++++----- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/docs/cli/README.md b/docs/cli/README.md index 4385770452..123f3f109a 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -50,6 +50,21 @@ vllm serve --help=max-num-seqs vllm serve --help=max ``` +!!! tip "Human-readable integer arguments" + Many integer arguments accept human-readable suffixes for convenience. For example: + + - `1k` = 1,000 (decimal kilo) + - `1K` = 1,024 (binary kibibyte) + - `1m` = 1,000,000 (decimal mega) + - `1M` = 1,048,576 (binary mebibyte) + - `1g` / `1G` = 1 billion / 1 gibibyte + - `1t` / `1T` = 1 trillion / 1 tebibyte + + Decimal suffixes (`k`, `m`, `g`, `t`) also accept floating point: `25.6k` = 25,600. + Binary suffixes (`K`, `M`, `G`, `T`) require integers: `32K` = 32,768. + + Supported arguments include: `--max-model-len`, `--max-num-batched-tokens`, `--max-num-scheduled-tokens`, `--kv-cache-memory-bytes`, `--safetensors-prefetch-block-size`. + See [vllm serve](./serve.md) for the full reference of all available arguments. ## launch diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 7da3aa66c9..2feb9f7a03 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -565,6 +565,40 @@ def test_human_readable_model_len(): parser.parse_args(["--max-model-len", invalid]) +def test_human_readable_other_args(): + # Test human-readable parsing for other integer args + # that were added to use human_readable_int parser + parser = EngineArgs.add_cli_args(FlexibleArgumentParser(exit_on_error=False)) + + # Test max_num_scheduled_tokens + args = parser.parse_args(["--max-num-scheduled-tokens", "1024"]) + assert args.max_num_scheduled_tokens == 1024 + args = parser.parse_args(["--max-num-scheduled-tokens", "2k"]) + assert args.max_num_scheduled_tokens == 2_000 + args = parser.parse_args(["--max-num-scheduled-tokens", "4K"]) + assert args.max_num_scheduled_tokens == 2**10 * 4 + args = parser.parse_args(["--max-num-scheduled-tokens", "10.5k"]) + assert args.max_num_scheduled_tokens == 10500 + + # Test kv_cache_memory_bytes (existing human-readable arg) + args = parser.parse_args(["--kv-cache-memory-bytes", "100000"]) + assert args.kv_cache_memory_bytes == 100000 + args = parser.parse_args(["--kv-cache-memory-bytes", "100k"]) + assert args.kv_cache_memory_bytes == 100_000 + args = parser.parse_args(["--kv-cache-memory-bytes", "1M"]) + assert args.kv_cache_memory_bytes == 2**20 + args = parser.parse_args(["--kv-cache-memory-bytes", "1m"]) + assert args.kv_cache_memory_bytes == 1_000_000 + + # Test max_num_batched_tokens (existing human-readable arg) + args = parser.parse_args(["--max-num-batched-tokens", "1024"]) + assert args.max_num_batched_tokens == 1024 + args = parser.parse_args(["--max-num-batched-tokens", "2k"]) + assert args.max_num_batched_tokens == 2_000 + args = parser.parse_args(["--max-num-batched-tokens", "4K"]) + assert args.max_num_batched_tokens == 2**10 * 4 + + def test_numa_bind_args(): parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) args = parser.parse_args( diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index c7a9335bbe..d80ecb9c85 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -354,14 +354,17 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: elif contains_type(type_hints, set): kwargs[name].update(collection_to_kwargs(type_hints, set)) elif contains_type(type_hints, int): + # Arguments that accept human-readable integer strings (e.g., 1K, 2M, 1G) + human_readable_int_args = { + "max_num_batched_tokens", + "max_num_scheduled_tokens", + "kv_cache_memory_bytes", + "safetensors_prefetch_block_size", + } if name == "max_model_len": kwargs[name]["type"] = human_readable_int_or_auto kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}" - elif name in ( - "max_num_batched_tokens", - "kv_cache_memory_bytes", - "safetensors_prefetch_block_size", - ): + elif name in human_readable_int_args: kwargs[name]["type"] = human_readable_int kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}" else: @@ -523,6 +526,7 @@ class EngineArgs: gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes max_num_batched_tokens: int | None = None + max_num_scheduled_tokens: int | None = None max_num_partial_prefills: int = SchedulerConfig.max_num_partial_prefills max_long_partial_prefills: int = SchedulerConfig.max_long_partial_prefills long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold @@ -1412,6 +1416,13 @@ class EngineArgs: "default": None, }, ) + scheduler_group.add_argument( + "--max-num-scheduled-tokens", + **{ + **scheduler_kwargs["max_num_scheduled_tokens"], + "default": None, + }, + ) scheduler_group.add_argument( "--max-num-seqs", **{ @@ -2151,6 +2162,7 @@ class EngineArgs: scheduler_config = SchedulerConfig( runner_type=model_config.runner_type, max_num_batched_tokens=self.max_num_batched_tokens, + max_num_scheduled_tokens=self.max_num_scheduled_tokens, max_num_seqs=self.max_num_seqs, max_model_len=model_config.max_model_len, enable_chunked_prefill=self.enable_chunked_prefill,