[Feature] Support manually enabling the cumem allocator (#33648)
Signed-off-by: Kebe <mail@kebe7jun.com>
This commit is contained in:
@@ -66,7 +66,10 @@ def test_kv_connector(
|
||||
|
||||
|
||||
def _build_config(
|
||||
*, kv_connector: str | None, enable_sleep_mode: bool = False
|
||||
*,
|
||||
kv_connector: str | None,
|
||||
enable_sleep_mode: bool = False,
|
||||
enable_cumem_allocator: bool = False,
|
||||
) -> VllmConfig:
|
||||
"""Build a VllmConfig that exercises _verify_kv_transfer_compat without
|
||||
requiring a real model (avoids HF downloads in CI)."""
|
||||
@@ -79,7 +82,10 @@ def _build_config(
|
||||
)
|
||||
cfg = VllmConfig.__new__(VllmConfig)
|
||||
cfg.kv_transfer_config = kv_transfer_config
|
||||
cfg.model_config = SimpleNamespace(enable_sleep_mode=enable_sleep_mode)
|
||||
cfg.model_config = SimpleNamespace(
|
||||
enable_sleep_mode=enable_sleep_mode,
|
||||
enable_cumem_allocator=(enable_cumem_allocator or enable_sleep_mode),
|
||||
)
|
||||
cfg._verify_kv_transfer_compat()
|
||||
return cfg
|
||||
|
||||
@@ -105,6 +111,14 @@ def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
|
||||
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
|
||||
|
||||
|
||||
def test_kv_connector_allows_expandable_segments_with_cumem_allocator(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Manual CuMem allocation must also bypass expandable_segments."""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
_build_config(kv_connector="NixlConnector", enable_cumem_allocator=True)
|
||||
|
||||
|
||||
def test_kv_connector_allows_other_alloc_conf(monkeypatch):
|
||||
"""Other PYTORCH_CUDA_ALLOC_CONF values must not be rejected."""
|
||||
monkeypatch.setenv(
|
||||
|
||||
Reference in New Issue
Block a user