[Misc][Docs] Add human-readable integer support for more cli-args (#47608)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -50,6 +50,21 @@ vllm serve --help=max-num-seqs
|
||||
vllm serve --help=max
|
||||
```
|
||||
|
||||
!!! tip "Human-readable integer arguments"
|
||||
Many integer arguments accept human-readable suffixes for convenience. For example:
|
||||
|
||||
- `1k` = 1,000 (decimal kilo)
|
||||
- `1K` = 1,024 (binary kibibyte)
|
||||
- `1m` = 1,000,000 (decimal mega)
|
||||
- `1M` = 1,048,576 (binary mebibyte)
|
||||
- `1g` / `1G` = 1 billion / 1 gibibyte
|
||||
- `1t` / `1T` = 1 trillion / 1 tebibyte
|
||||
|
||||
Decimal suffixes (`k`, `m`, `g`, `t`) also accept floating point: `25.6k` = 25,600.
|
||||
Binary suffixes (`K`, `M`, `G`, `T`) require integers: `32K` = 32,768.
|
||||
|
||||
Supported arguments include: `--max-model-len`, `--max-num-batched-tokens`, `--max-num-scheduled-tokens`, `--kv-cache-memory-bytes`, `--safetensors-prefetch-block-size`.
|
||||
|
||||
See [vllm serve](./serve.md) for the full reference of all available arguments.
|
||||
|
||||
## launch
|
||||
|
||||
@@ -565,6 +565,40 @@ def test_human_readable_model_len():
|
||||
parser.parse_args(["--max-model-len", invalid])
|
||||
|
||||
|
||||
def test_human_readable_other_args():
|
||||
# Test human-readable parsing for other integer args
|
||||
# that were added to use human_readable_int parser
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser(exit_on_error=False))
|
||||
|
||||
# Test max_num_scheduled_tokens
|
||||
args = parser.parse_args(["--max-num-scheduled-tokens", "1024"])
|
||||
assert args.max_num_scheduled_tokens == 1024
|
||||
args = parser.parse_args(["--max-num-scheduled-tokens", "2k"])
|
||||
assert args.max_num_scheduled_tokens == 2_000
|
||||
args = parser.parse_args(["--max-num-scheduled-tokens", "4K"])
|
||||
assert args.max_num_scheduled_tokens == 2**10 * 4
|
||||
args = parser.parse_args(["--max-num-scheduled-tokens", "10.5k"])
|
||||
assert args.max_num_scheduled_tokens == 10500
|
||||
|
||||
# Test kv_cache_memory_bytes (existing human-readable arg)
|
||||
args = parser.parse_args(["--kv-cache-memory-bytes", "100000"])
|
||||
assert args.kv_cache_memory_bytes == 100000
|
||||
args = parser.parse_args(["--kv-cache-memory-bytes", "100k"])
|
||||
assert args.kv_cache_memory_bytes == 100_000
|
||||
args = parser.parse_args(["--kv-cache-memory-bytes", "1M"])
|
||||
assert args.kv_cache_memory_bytes == 2**20
|
||||
args = parser.parse_args(["--kv-cache-memory-bytes", "1m"])
|
||||
assert args.kv_cache_memory_bytes == 1_000_000
|
||||
|
||||
# Test max_num_batched_tokens (existing human-readable arg)
|
||||
args = parser.parse_args(["--max-num-batched-tokens", "1024"])
|
||||
assert args.max_num_batched_tokens == 1024
|
||||
args = parser.parse_args(["--max-num-batched-tokens", "2k"])
|
||||
assert args.max_num_batched_tokens == 2_000
|
||||
args = parser.parse_args(["--max-num-batched-tokens", "4K"])
|
||||
assert args.max_num_batched_tokens == 2**10 * 4
|
||||
|
||||
|
||||
def test_numa_bind_args():
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = parser.parse_args(
|
||||
|
||||
@@ -354,14 +354,17 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
|
||||
elif contains_type(type_hints, set):
|
||||
kwargs[name].update(collection_to_kwargs(type_hints, set))
|
||||
elif contains_type(type_hints, int):
|
||||
# Arguments that accept human-readable integer strings (e.g., 1K, 2M, 1G)
|
||||
human_readable_int_args = {
|
||||
"max_num_batched_tokens",
|
||||
"max_num_scheduled_tokens",
|
||||
"kv_cache_memory_bytes",
|
||||
"safetensors_prefetch_block_size",
|
||||
}
|
||||
if name == "max_model_len":
|
||||
kwargs[name]["type"] = human_readable_int_or_auto
|
||||
kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
|
||||
elif name in (
|
||||
"max_num_batched_tokens",
|
||||
"kv_cache_memory_bytes",
|
||||
"safetensors_prefetch_block_size",
|
||||
):
|
||||
elif name in human_readable_int_args:
|
||||
kwargs[name]["type"] = human_readable_int
|
||||
kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
|
||||
else:
|
||||
@@ -523,6 +526,7 @@ class EngineArgs:
|
||||
gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization
|
||||
kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes
|
||||
max_num_batched_tokens: int | None = None
|
||||
max_num_scheduled_tokens: int | None = None
|
||||
max_num_partial_prefills: int = SchedulerConfig.max_num_partial_prefills
|
||||
max_long_partial_prefills: int = SchedulerConfig.max_long_partial_prefills
|
||||
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
|
||||
@@ -1412,6 +1416,13 @@ class EngineArgs:
|
||||
"default": None,
|
||||
},
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-scheduled-tokens",
|
||||
**{
|
||||
**scheduler_kwargs["max_num_scheduled_tokens"],
|
||||
"default": None,
|
||||
},
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-seqs",
|
||||
**{
|
||||
@@ -2151,6 +2162,7 @@ class EngineArgs:
|
||||
scheduler_config = SchedulerConfig(
|
||||
runner_type=model_config.runner_type,
|
||||
max_num_batched_tokens=self.max_num_batched_tokens,
|
||||
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
|
||||
max_num_seqs=self.max_num_seqs,
|
||||
max_model_len=model_config.max_model_len,
|
||||
enable_chunked_prefill=self.enable_chunked_prefill,
|
||||
|
||||
Reference in New Issue
Block a user