[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-13 12:24:02 -05:00
committed by GitHub
parent 69d4c3a06b
commit b96bcd0b47
24 changed files with 1002 additions and 1103 deletions
-73
View File
@@ -1,11 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import pytest
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
TEST_IMAGE_ASSETS = [
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
@@ -13,70 +7,3 @@ TEST_IMAGE_ASSETS = [
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
]
def _shutdown_llm(llm: Any, gpu_memory_utilization: float) -> None:
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.platforms import current_platform
try:
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
except Exception:
pass
del llm
try:
import torch
torch._dynamo.reset()
except Exception:
pass
cleanup_dist_env_and_memory()
if current_platform.is_rocm():
from tests.utils import wait_for_rocm_memory_to_settle
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
@contextmanager
def managed_llm(*args: Any, **kwargs: Any) -> Iterator[Any]:
from vllm import LLM
llm = LLM(*args, **kwargs)
gpu_memory_utilization = (
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
try:
yield llm
finally:
_shutdown_llm(llm, gpu_memory_utilization)
def _make_managed_llm_factory() -> Iterator[Callable[..., Any]]:
from vllm import LLM
llms: list[tuple[Any, float]] = []
def make_llm(*args: Any, **kwargs: Any) -> Any:
llm = LLM(*args, **kwargs)
gpu_memory_utilization = (
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
llms.append((llm, gpu_memory_utilization))
return llm
try:
yield make_llm
finally:
while llms:
llm, gpu_memory_utilization = llms.pop()
_shutdown_llm(llm, gpu_memory_utilization)
@pytest.fixture
def multimodal_llm_factory() -> Iterator[Callable[..., Any]]:
yield from _make_managed_llm_factory()
@@ -1,21 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
@pytest.fixture(scope="function")
def vision_llm(multimodal_llm_factory):
return multimodal_llm_factory(
model="microsoft/Phi-3.5-vision-instruct",
def vision_llm(vllm_runner):
with vllm_runner(
"microsoft/Phi-3.5-vision-instruct",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
trust_remote_code=True,
limit_mm_per_prompt={"image": 2},
seed=0,
)
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.parametrize(
@@ -69,7 +69,7 @@ def test_inject_into_mm_cache(
image_urls,
mm_processor_cache_type,
caplog_vllm,
multimodal_llm_factory,
vllm_runner,
):
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
the processor cache and MM cache hit metrics are updated correctly.
@@ -79,117 +79,115 @@ def test_inject_into_mm_cache(
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
then generate with a pre-rendered input -> verifies injection works
"""
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_type=mm_processor_cache_type,
)
) as runner:
# Step 1: Normal requests to populate the cache
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
# Step 1: Normal requests to populate the cache
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(50.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0)
# Step 2: Use a second image to get valid expanded tokens and
# placeholder positions via the renderer.
runner.llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(runner.llm.get_metrics())[0] # 3
# Step 2: Use a second image to get valid expanded tokens and
# placeholder positions via the renderer.
llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
renderer = runner.llm.llm_engine.renderer
cache = renderer.mm_processor_cache
assert cache is not None, "Processor cache should be enabled"
renderer = llm.llm_engine.renderer
cache = renderer.mm_processor_cache
assert cache is not None, "Processor cache should be enabled"
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[1])],
ChatParams(),
)
eng_input = eng_prompts[0]
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[1])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Inject pre-processed mm_kwargs with a NEW hash via public API
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
# Inject pre-processed mm_kwargs with a NEW hash via public API
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
# Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
runner.llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
# Verify cache was queried and injection happened
queries_after = _get_mm_cache_stats(llm.get_metrics())[0]
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
# Verify cache was queried and injection happened
queries_after = _get_mm_cache_stats(runner.llm.get_metrics())[0]
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(runner.llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
def test_inject_into_mm_cache_without_cache(
num_gpus_available,
image_urls,
multimodal_llm_factory,
vllm_runner,
):
"""Test that inject_into_mm_cache works gracefully when processor cache
is disabled (mm_processor_cache_gb=0). Should not crash.
"""
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_gb=0,
)
) as runner:
# Run a normal chat request first to warm up the model.
runner.llm.chat(_make_messages(image_urls[0]))
# Run a normal chat request first to warm up the model.
llm.chat(_make_messages(image_urls[0]))
# Use the renderer to get a proper EngineInput with expanded tokens
renderer = runner.llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Use the renderer to get a proper EngineInput with expanded tokens
renderer = llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
mm_hashes = {"image": ["abcd1234" * 8]}
mm_kwargs = eng_input["mm_kwargs"]
mm_hashes = {"image": ["abcd1234" * 8]}
mm_kwargs = eng_input["mm_kwargs"]
# inject_into_mm_cache should not crash even without cache
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# inject_into_mm_cache should not crash even without cache
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build and generate with pre-rendered input
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
# Build and generate with pre-rendered input
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
result = llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
result = runner.llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
@@ -61,38 +61,37 @@ def test_mm_cache_stats(
image_urls,
mm_processor_cache_type,
caplog_vllm,
multimodal_llm_factory,
vllm_runner,
):
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
mm_processor_cache_type=mm_processor_cache_type,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
)
) as runner:
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(33.3)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3)
# NOTE: This only resets hit rate stats in CachingMetrics
# The raw queries and hits counts remain unaffected
runner.llm.reset_mm_cache()
# NOTE: This only resets hit rate stats in CachingMetrics
# The raw queries and hits counts remain unaffected
llm.reset_mm_cache()
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (4, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from tests.entrypoints.multimodal.conftest import managed_llm
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.exceptions import VLLMValidationError
@@ -14,17 +15,19 @@ TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
@pytest.fixture(scope="module")
def llm():
def llm(vllm_runner):
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
with managed_llm(
model=MODEL,
with vllm_runner(
MODEL,
max_model_len=2048,
enforce_eager=True,
gpu_memory_utilization=0.8,
enable_mm_embeds=True,
limit_mm_per_prompt={"image": 0},
) as llm:
yield llm
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup