diff --git a/tests/v1/engine/test_engine_args.py b/tests/v1/engine/test_engine_args.py index c336f93c5f..c03c244bb1 100644 --- a/tests/v1/engine/test_engine_args.py +++ b/tests/v1/engine/test_engine_args.py @@ -5,7 +5,6 @@ from argparse import ArgumentError import pytest -from vllm.config import VllmConfig from vllm.engine.arg_utils import EngineArgs from vllm.usage.usage_lib import UsageContext from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -64,34 +63,6 @@ def test_prefix_caching_xxhash_from_cli(): assert vllm_config.cache_config.prefix_caching_hash_algo == "xxhash_cbor" -def test_defaults_with_usage_context(): - engine_args = EngineArgs(model="facebook/opt-125m") - vllm_config: VllmConfig = engine_args.create_engine_config(UsageContext.LLM_CLASS) - - from vllm.platforms import current_platform - from vllm.utils.mem_constants import GiB_bytes - - device_memory = current_platform.get_device_total_memory() - device_name = current_platform.get_device_name().lower() - if device_memory >= 70 * GiB_bytes and "a100" not in device_name: - # For GPUs like H100, H200, and MI300x with >= 70GB memory - default_llm_tokens = 16384 - default_server_tokens = 8192 - default_max_num_seqs = 1024 - else: - default_llm_tokens = 8192 - default_server_tokens = 2048 - default_max_num_seqs = 256 - - assert vllm_config.scheduler_config.max_num_seqs == default_max_num_seqs - assert vllm_config.scheduler_config.max_num_batched_tokens == default_llm_tokens # noqa: E501 - - engine_args = EngineArgs(model="facebook/opt-125m") - vllm_config = engine_args.create_engine_config(UsageContext.OPENAI_API_SERVER) - assert vllm_config.scheduler_config.max_num_seqs == default_max_num_seqs - assert vllm_config.scheduler_config.max_num_batched_tokens == default_server_tokens # noqa: E501 - - def test_mm_prefix_lm_raises_batched_tokens_floor(): """Verify that prefix-LM multimodal models auto-raise max_num_batched_tokens to fit at least one multimodal item. diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index f84e3c355b..3b9ead7f7e 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -2566,8 +2566,18 @@ class EngineArgs: # NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces # throughput, see PR #17885 for more details. # So here we do an extra device name check to prevent such regression. - if device_memory >= 70 * GiB_bytes and "a100" not in device_name: - # For GPUs like H100 and MI300x, use larger default values. + if device_memory >= 160 * GiB_bytes: + # for GPUs like B200/B300 with >= 160GB memory, use the largest defaults + default_max_num_batched_tokens = { + UsageContext.LLM_CLASS: 16384, + UsageContext.OPENAI_API_SERVER: 16384, + } + default_max_num_seqs = { + UsageContext.LLM_CLASS: 1024, + UsageContext.OPENAI_API_SERVER: 1024, + } + elif device_memory >= 70 * GiB_bytes and "a100" not in device_name: + # For GPUs like H100 and H200, use larger offline defaults. default_max_num_batched_tokens = { UsageContext.LLM_CLASS: 16384, UsageContext.OPENAI_API_SERVER: 8192,