[Bugfix][Platform] Fall back to torch when device-level NVML memory is unsupported (GB10) (#57378)
Signed-off-by: hclsys <chenglunhu@gmail.com> Signed-off-by: Chenglun Hu <chenglunhu@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1054,3 +1054,34 @@ class TestDpDeviceIdSharding:
|
||||
get_physical_gpu_ids_for_local_dp_rank(
|
||||
evar, local_dp_rank=2, world_size=2, user_assigned_gpu_ids=[4, 5, 6, 7]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "mem_gib", "capability", "expected_api_tokens", "expected_api_seqs"),
|
||||
[
|
||||
("nvidia gb10", 121, (12, 1), 2048, 256),
|
||||
("nvidia h100", 80, (9, 0), 8192, 1024),
|
||||
("nvidia rtx pro 6000 blackwell", 96, (12, 0), 8192, 1024),
|
||||
],
|
||||
)
|
||||
def test_get_batch_defaults_gb10_stays_on_small_tier(
|
||||
monkeypatch, name, mem_gib, capability, expected_api_tokens, expected_api_seqs
|
||||
):
|
||||
"""GB10 reports >= 70 GiB (unified memory) but not the compute of a
|
||||
large discrete GPU, so it keeps the small-tier defaults."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
stub = SimpleNamespace(
|
||||
get_device_total_memory=lambda: mem_gib * GiB_bytes,
|
||||
get_device_name=lambda: name,
|
||||
is_device_capability=lambda cap: cap == capability,
|
||||
is_tpu=lambda: False,
|
||||
is_cpu=lambda: False,
|
||||
)
|
||||
monkeypatch.setattr("vllm.engine.arg_utils.current_platform", stub)
|
||||
tokens, seqs = EngineArgs.get_batch_defaults(1)
|
||||
assert tokens[UsageContext.OPENAI_API_SERVER] == expected_api_tokens
|
||||
assert seqs[UsageContext.OPENAI_API_SERVER] == expected_api_seqs
|
||||
|
||||
Reference in New Issue
Block a user