[Config] Update default _max_num_batched_tokens from 8192 to 16384 (#51726)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -5,7 +5,6 @@ from argparse import ArgumentError
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
|
||||||
from vllm.engine.arg_utils import EngineArgs
|
from vllm.engine.arg_utils import EngineArgs
|
||||||
from vllm.usage.usage_lib import UsageContext
|
from vllm.usage.usage_lib import UsageContext
|
||||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
@@ -64,34 +63,6 @@ def test_prefix_caching_xxhash_from_cli():
|
|||||||
assert vllm_config.cache_config.prefix_caching_hash_algo == "xxhash_cbor"
|
assert vllm_config.cache_config.prefix_caching_hash_algo == "xxhash_cbor"
|
||||||
|
|
||||||
|
|
||||||
def test_defaults_with_usage_context():
|
|
||||||
engine_args = EngineArgs(model="facebook/opt-125m")
|
|
||||||
vllm_config: VllmConfig = engine_args.create_engine_config(UsageContext.LLM_CLASS)
|
|
||||||
|
|
||||||
from vllm.platforms import current_platform
|
|
||||||
from vllm.utils.mem_constants import GiB_bytes
|
|
||||||
|
|
||||||
device_memory = current_platform.get_device_total_memory()
|
|
||||||
device_name = current_platform.get_device_name().lower()
|
|
||||||
if device_memory >= 70 * GiB_bytes and "a100" not in device_name:
|
|
||||||
# For GPUs like H100, H200, and MI300x with >= 70GB memory
|
|
||||||
default_llm_tokens = 16384
|
|
||||||
default_server_tokens = 8192
|
|
||||||
default_max_num_seqs = 1024
|
|
||||||
else:
|
|
||||||
default_llm_tokens = 8192
|
|
||||||
default_server_tokens = 2048
|
|
||||||
default_max_num_seqs = 256
|
|
||||||
|
|
||||||
assert vllm_config.scheduler_config.max_num_seqs == default_max_num_seqs
|
|
||||||
assert vllm_config.scheduler_config.max_num_batched_tokens == default_llm_tokens # noqa: E501
|
|
||||||
|
|
||||||
engine_args = EngineArgs(model="facebook/opt-125m")
|
|
||||||
vllm_config = engine_args.create_engine_config(UsageContext.OPENAI_API_SERVER)
|
|
||||||
assert vllm_config.scheduler_config.max_num_seqs == default_max_num_seqs
|
|
||||||
assert vllm_config.scheduler_config.max_num_batched_tokens == default_server_tokens # noqa: E501
|
|
||||||
|
|
||||||
|
|
||||||
def test_mm_prefix_lm_raises_batched_tokens_floor():
|
def test_mm_prefix_lm_raises_batched_tokens_floor():
|
||||||
"""Verify that prefix-LM multimodal models auto-raise
|
"""Verify that prefix-LM multimodal models auto-raise
|
||||||
max_num_batched_tokens to fit at least one multimodal item.
|
max_num_batched_tokens to fit at least one multimodal item.
|
||||||
|
|||||||
@@ -2566,8 +2566,18 @@ class EngineArgs:
|
|||||||
# NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
|
# NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
|
||||||
# throughput, see PR #17885 for more details.
|
# throughput, see PR #17885 for more details.
|
||||||
# So here we do an extra device name check to prevent such regression.
|
# So here we do an extra device name check to prevent such regression.
|
||||||
if device_memory >= 70 * GiB_bytes and "a100" not in device_name:
|
if device_memory >= 160 * GiB_bytes:
|
||||||
# For GPUs like H100 and MI300x, use larger default values.
|
# for GPUs like B200/B300 with >= 160GB memory, use the largest defaults
|
||||||
|
default_max_num_batched_tokens = {
|
||||||
|
UsageContext.LLM_CLASS: 16384,
|
||||||
|
UsageContext.OPENAI_API_SERVER: 16384,
|
||||||
|
}
|
||||||
|
default_max_num_seqs = {
|
||||||
|
UsageContext.LLM_CLASS: 1024,
|
||||||
|
UsageContext.OPENAI_API_SERVER: 1024,
|
||||||
|
}
|
||||||
|
elif device_memory >= 70 * GiB_bytes and "a100" not in device_name:
|
||||||
|
# For GPUs like H100 and H200, use larger offline defaults.
|
||||||
default_max_num_batched_tokens = {
|
default_max_num_batched_tokens = {
|
||||||
UsageContext.LLM_CLASS: 16384,
|
UsageContext.LLM_CLASS: 16384,
|
||||||
UsageContext.OPENAI_API_SERVER: 8192,
|
UsageContext.OPENAI_API_SERVER: 8192,
|
||||||
|
|||||||
Reference in New Issue
Block a user