[ROCm] Run init test engine in-process to avoid KV-cache OOM (#48527)
Signed-off-by: Djordje Ramic <djoramic@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
3b39fd284a
commit
b6770d7b54
@@ -14,7 +14,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
)
|
||||
from vllm.v1.engine.core import EngineCore as V1EngineCore
|
||||
|
||||
from ..utils import create_new_process_for_each_test
|
||||
from ..utils import create_new_process_for_each_test, requires_spawn_multiprocessing
|
||||
from .registry import (
|
||||
_TRANSFORMERS_BACKEND_MODELS,
|
||||
AUTO_EXAMPLE_MODELS,
|
||||
@@ -129,6 +129,12 @@ def can_initialize(
|
||||
patch.object(V1EngineCore, "_initialize_kv_caches", _initialize_kv_caches_v1),
|
||||
monkeypatch.context() as m,
|
||||
):
|
||||
if requires_spawn_multiprocessing():
|
||||
# The EngineCore subprocess re-imports the class and does not
|
||||
# inherit the KV-cache patch above, so it OOMs. Run in-process
|
||||
# so the patch applies.
|
||||
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
# FIXME: A hack to bypass FA3 assertion because our CI's L4 GPU
|
||||
# has cc==8.9 which hasn't supported FA3 yet. Remove this hack when
|
||||
# L4 supports FA3.
|
||||
|
||||
Reference in New Issue
Block a user