[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-13 12:24:02 -05:00
committed by GitHub
parent 69d4c3a06b
commit b96bcd0b47
24 changed files with 1002 additions and 1103 deletions
@@ -1,21 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
@pytest.fixture(scope="function")
def vision_llm(multimodal_llm_factory):
return multimodal_llm_factory(
model="microsoft/Phi-3.5-vision-instruct",
def vision_llm(vllm_runner):
with vllm_runner(
"microsoft/Phi-3.5-vision-instruct",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
trust_remote_code=True,
limit_mm_per_prompt={"image": 2},
seed=0,
)
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.parametrize(
@@ -69,7 +69,7 @@ def test_inject_into_mm_cache(
image_urls,
mm_processor_cache_type,
caplog_vllm,
multimodal_llm_factory,
vllm_runner,
):
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
the processor cache and MM cache hit metrics are updated correctly.
@@ -79,117 +79,115 @@ def test_inject_into_mm_cache(
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
then generate with a pre-rendered input -> verifies injection works
"""
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_type=mm_processor_cache_type,
)
) as runner:
# Step 1: Normal requests to populate the cache
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
# Step 1: Normal requests to populate the cache
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(50.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0)
# Step 2: Use a second image to get valid expanded tokens and
# placeholder positions via the renderer.
runner.llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(runner.llm.get_metrics())[0] # 3
# Step 2: Use a second image to get valid expanded tokens and
# placeholder positions via the renderer.
llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
renderer = runner.llm.llm_engine.renderer
cache = renderer.mm_processor_cache
assert cache is not None, "Processor cache should be enabled"
renderer = llm.llm_engine.renderer
cache = renderer.mm_processor_cache
assert cache is not None, "Processor cache should be enabled"
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[1])],
ChatParams(),
)
eng_input = eng_prompts[0]
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[1])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Inject pre-processed mm_kwargs with a NEW hash via public API
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
# Inject pre-processed mm_kwargs with a NEW hash via public API
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
# Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
runner.llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
# Verify cache was queried and injection happened
queries_after = _get_mm_cache_stats(llm.get_metrics())[0]
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
# Verify cache was queried and injection happened
queries_after = _get_mm_cache_stats(runner.llm.get_metrics())[0]
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(runner.llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
def test_inject_into_mm_cache_without_cache(
num_gpus_available,
image_urls,
multimodal_llm_factory,
vllm_runner,
):
"""Test that inject_into_mm_cache works gracefully when processor cache
is disabled (mm_processor_cache_gb=0). Should not crash.
"""
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_gb=0,
)
) as runner:
# Run a normal chat request first to warm up the model.
runner.llm.chat(_make_messages(image_urls[0]))
# Run a normal chat request first to warm up the model.
llm.chat(_make_messages(image_urls[0]))
# Use the renderer to get a proper EngineInput with expanded tokens
renderer = runner.llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Use the renderer to get a proper EngineInput with expanded tokens
renderer = llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
mm_hashes = {"image": ["abcd1234" * 8]}
mm_kwargs = eng_input["mm_kwargs"]
mm_hashes = {"image": ["abcd1234" * 8]}
mm_kwargs = eng_input["mm_kwargs"]
# inject_into_mm_cache should not crash even without cache
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# inject_into_mm_cache should not crash even without cache
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build and generate with pre-rendered input
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
# Build and generate with pre-rendered input
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
result = llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
result = runner.llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
@@ -61,38 +61,37 @@ def test_mm_cache_stats(
image_urls,
mm_processor_cache_type,
caplog_vllm,
multimodal_llm_factory,
vllm_runner,
):
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
mm_processor_cache_type=mm_processor_cache_type,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
)
) as runner:
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(33.3)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3)
# NOTE: This only resets hit rate stats in CachingMetrics
# The raw queries and hits counts remain unaffected
runner.llm.reset_mm_cache()
# NOTE: This only resets hit rate stats in CachingMetrics
# The raw queries and hits counts remain unaffected
llm.reset_mm_cache()
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (4, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from tests.entrypoints.multimodal.conftest import managed_llm
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.exceptions import VLLMValidationError
@@ -14,17 +15,19 @@ TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
@pytest.fixture(scope="module")
def llm():
def llm(vllm_runner):
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
with managed_llm(
model=MODEL,
with vllm_runner(
MODEL,
max_model_len=2048,
enforce_eager=True,
gpu_memory_utilization=0.8,
enable_mm_embeds=True,
limit_mm_per_prompt={"image": 0},
) as llm:
yield llm
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup