[ROCm][CI] Stabilize memory-release in the Hybrid model generation tests (#44046)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-04 22:34:24 +08:00
committed by GitHub
parent 0c96dd64fb
commit 6f68ca3e91
+76 -41
View File
@@ -2,11 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable from collections.abc import Callable
from contextlib import contextmanager, nullcontext
import pytest import pytest
from tests.models.registry import HF_EXAMPLE_MODELS from tests.models.registry import HF_EXAMPLE_MODELS
from tests.utils import multi_gpu_test from tests.utils import multi_gpu_test, wait_for_gpu_memory_to_clear
from vllm import LLM from vllm import LLM
from vllm.engine.arg_utils import EngineArgs from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform from vllm.platforms import current_platform
@@ -404,6 +405,30 @@ def _get_vllm_runner_params(
} }
def _wait_for_rocm_memory_to_settle() -> None:
if not current_platform.is_rocm():
return
num_gpus = current_platform.device_count()
if num_gpus == 0:
return
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=0.01,
timeout_s=120,
)
@contextmanager
def _owned_vLLM_runner(vllm_runner, kwargs):
try:
with vllm_runner(**kwargs) as runner:
yield runner
finally:
_wait_for_rocm_memory_to_settle()
def _get_vLLM_output( def _get_vLLM_output(
vllm_runner, vllm_runner,
kwargs, kwargs,
@@ -413,17 +438,21 @@ def _get_vLLM_output(
num_repetitions=1, num_repetitions=1,
vllm_model=None, vllm_model=None,
): ):
outs = [] runner_context = (
if vllm_model is None: _owned_vLLM_runner(vllm_runner, kwargs)
vllm_model = vllm_runner(**kwargs) if vllm_model is None
for _ in range(num_repetitions): else nullcontext(vllm_model)
if num_logprobs < 0: )
vllm_output = vllm_model.generate_greedy(prompts, max_tokens) with runner_context as runner:
else: outs = []
vllm_output = vllm_model.generate_greedy_logprobs( for _ in range(num_repetitions):
prompts, max_tokens, num_logprobs if num_logprobs < 0:
) vllm_output = runner.generate_greedy(prompts, max_tokens)
outs.append(vllm_output) else:
vllm_output = runner.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
outs.append(vllm_output)
return outs, vllm_model return outs, vllm_model
@@ -772,38 +801,44 @@ def test_apc_multiple_prompts_partial_cached_outputs(
# Cache only part of all the prompts # Cache only part of all the prompts
vllm_runner_kwargs["enable_prefix_caching"] = True vllm_runner_kwargs["enable_prefix_caching"] = True
vllm_outputs_partial_cache, vllm_model = _get_vLLM_output( with _owned_vLLM_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
vllm_runner, vllm_runner_kwargs, generated_prompts[:3], max_tokens, num_logprobs vllm_outputs_partial_cache, _ = _get_vLLM_output(
) vllm_runner,
vllm_runner_kwargs,
compare_operator( generated_prompts[:3],
outputs_0_lst=vllm_outputs_no_cache[0][:3], max_tokens,
outputs_1_lst=vllm_outputs_partial_cache[0], num_logprobs,
name_0="vllm_no_cache", vllm_model=vllm_model,
name_1="vllm_partial_cache", )
)
vllm_outputs_cache_rep, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
generated_prompts,
max_tokens,
num_logprobs,
n_repetitions,
vllm_model=vllm_model,
)
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
# In the first repetition, the caches are filled
# In the second repetition, these caches are reused
compare_operator( compare_operator(
outputs_0_lst=vllm_outputs_no_cache[0], outputs_0_lst=vllm_outputs_no_cache[0][:3],
outputs_1_lst=vllm_outputs_cache_itn, outputs_1_lst=vllm_outputs_partial_cache[0],
name_0="vllm_no_cache", name_0="vllm_no_cache",
name_1=f"vllm_cache_it_{r_idx + 1}", name_1="vllm_partial_cache",
) )
vllm_outputs_cache_rep, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
generated_prompts,
max_tokens,
num_logprobs,
n_repetitions,
vllm_model=vllm_model,
)
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
# In the first repetition, the caches are filled
# In the second repetition, these caches are reused
compare_operator(
outputs_0_lst=vllm_outputs_no_cache[0],
outputs_1_lst=vllm_outputs_cache_itn,
name_0="vllm_no_cache",
name_1=f"vllm_cache_it_{r_idx + 1}",
)
# Test that outputs match whether prefix caching is enabled or not for mamba. # Test that outputs match whether prefix caching is enabled or not for mamba.
@pytest.mark.parametrize("model", ["tiiuae/falcon-mamba-7b"]) @pytest.mark.parametrize("model", ["tiiuae/falcon-mamba-7b"])
@@ -826,7 +861,7 @@ def test_same_mamba_output_apc_on_vs_off(
# No prefix caching # No prefix caching
kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False} kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False}
with vllm_runner(**kwargs_no_apc) as vllm_model: with _owned_vLLM_runner(vllm_runner, kwargs_no_apc) as vllm_model:
outputs_no_apc, _ = _get_vLLM_output( outputs_no_apc, _ = _get_vLLM_output(
vllm_runner, vllm_runner,
kwargs_no_apc, kwargs_no_apc,
@@ -841,7 +876,7 @@ def test_same_mamba_output_apc_on_vs_off(
"enable_prefix_caching": True, "enable_prefix_caching": True,
"mamba_block_size": 16, "mamba_block_size": 16,
} }
with vllm_runner(**kwargs_with_apc) as vllm_model: with _owned_vLLM_runner(vllm_runner, kwargs_with_apc) as vllm_model:
outputs_with_apc, _ = _get_vLLM_output( outputs_with_apc, _ = _get_vLLM_output(
vllm_runner, vllm_runner,
kwargs_with_apc, kwargs_with_apc,