[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -1,11 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
|
||||
TEST_IMAGE_ASSETS = [
|
||||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||
@@ -13,70 +7,3 @@ TEST_IMAGE_ASSETS = [
|
||||
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
|
||||
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
|
||||
]
|
||||
|
||||
|
||||
def _shutdown_llm(llm: Any, gpu_memory_utilization: float) -> None:
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
try:
|
||||
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
|
||||
llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
del llm
|
||||
|
||||
try:
|
||||
import torch
|
||||
|
||||
torch._dynamo.reset()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from tests.utils import wait_for_rocm_memory_to_settle
|
||||
|
||||
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def managed_llm(*args: Any, **kwargs: Any) -> Iterator[Any]:
|
||||
from vllm import LLM
|
||||
|
||||
llm = LLM(*args, **kwargs)
|
||||
gpu_memory_utilization = (
|
||||
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
|
||||
)
|
||||
try:
|
||||
yield llm
|
||||
finally:
|
||||
_shutdown_llm(llm, gpu_memory_utilization)
|
||||
|
||||
|
||||
def _make_managed_llm_factory() -> Iterator[Callable[..., Any]]:
|
||||
from vllm import LLM
|
||||
|
||||
llms: list[tuple[Any, float]] = []
|
||||
|
||||
def make_llm(*args: Any, **kwargs: Any) -> Any:
|
||||
llm = LLM(*args, **kwargs)
|
||||
gpu_memory_utilization = (
|
||||
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
|
||||
)
|
||||
llms.append((llm, gpu_memory_utilization))
|
||||
return llm
|
||||
|
||||
try:
|
||||
yield make_llm
|
||||
finally:
|
||||
while llms:
|
||||
llm, gpu_memory_utilization = llms.pop()
|
||||
_shutdown_llm(llm, gpu_memory_utilization)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def multimodal_llm_factory() -> Iterator[Callable[..., Any]]:
|
||||
yield from _make_managed_llm_factory()
|
||||
|
||||
@@ -1,21 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def vision_llm(multimodal_llm_factory):
|
||||
return multimodal_llm_factory(
|
||||
model="microsoft/Phi-3.5-vision-instruct",
|
||||
def vision_llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
"microsoft/Phi-3.5-vision-instruct",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
seed=0,
|
||||
)
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -69,7 +69,7 @@ def test_inject_into_mm_cache(
|
||||
image_urls,
|
||||
mm_processor_cache_type,
|
||||
caplog_vllm,
|
||||
multimodal_llm_factory,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
|
||||
the processor cache and MM cache hit metrics are updated correctly.
|
||||
@@ -79,117 +79,115 @@ def test_inject_into_mm_cache(
|
||||
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
|
||||
then generate with a pre-rendered input -> verifies injection works
|
||||
"""
|
||||
llm = multimodal_llm_factory(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
with vllm_runner(
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
)
|
||||
) as runner:
|
||||
# Step 1: Normal requests to populate the cache
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
|
||||
|
||||
# Step 1: Normal requests to populate the cache
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 1)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(50.0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0)
|
||||
# Step 2: Use a second image to get valid expanded tokens and
|
||||
# placeholder positions via the renderer.
|
||||
runner.llm.chat(_make_messages(image_urls[1]))
|
||||
queries_before = _get_mm_cache_stats(runner.llm.get_metrics())[0] # 3
|
||||
|
||||
# Step 2: Use a second image to get valid expanded tokens and
|
||||
# placeholder positions via the renderer.
|
||||
llm.chat(_make_messages(image_urls[1]))
|
||||
queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
|
||||
renderer = runner.llm.llm_engine.renderer
|
||||
cache = renderer.mm_processor_cache
|
||||
assert cache is not None, "Processor cache should be enabled"
|
||||
|
||||
renderer = llm.llm_engine.renderer
|
||||
cache = renderer.mm_processor_cache
|
||||
assert cache is not None, "Processor cache should be enabled"
|
||||
_, eng_prompts = renderer.render_chat(
|
||||
[_make_messages(image_urls[1])],
|
||||
ChatParams(),
|
||||
)
|
||||
eng_input = eng_prompts[0]
|
||||
|
||||
_, eng_prompts = renderer.render_chat(
|
||||
[_make_messages(image_urls[1])],
|
||||
ChatParams(),
|
||||
)
|
||||
eng_input = eng_prompts[0]
|
||||
# Inject pre-processed mm_kwargs with a NEW hash via public API
|
||||
new_mm_hash = "deadbeef" * 8
|
||||
mm_hashes = {"image": [new_mm_hash]}
|
||||
mm_kwargs = eng_input["mm_kwargs"]
|
||||
|
||||
# Inject pre-processed mm_kwargs with a NEW hash via public API
|
||||
new_mm_hash = "deadbeef" * 8
|
||||
mm_hashes = {"image": [new_mm_hash]}
|
||||
mm_kwargs = eng_input["mm_kwargs"]
|
||||
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||
|
||||
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||
# Build pre-rendered input (no externally_processed flag needed)
|
||||
pre_rendered_input = {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||
"mm_kwargs": mm_kwargs,
|
||||
"mm_hashes": mm_hashes,
|
||||
"mm_placeholders": eng_input["mm_placeholders"],
|
||||
}
|
||||
|
||||
# Build pre-rendered input (no externally_processed flag needed)
|
||||
pre_rendered_input = {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||
"mm_kwargs": mm_kwargs,
|
||||
"mm_hashes": mm_hashes,
|
||||
"mm_placeholders": eng_input["mm_placeholders"],
|
||||
}
|
||||
runner.llm.generate(
|
||||
pre_rendered_input,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
)
|
||||
|
||||
llm.generate(
|
||||
pre_rendered_input,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
)
|
||||
|
||||
# Verify cache was queried and injection happened
|
||||
queries_after = _get_mm_cache_stats(llm.get_metrics())[0]
|
||||
assert queries_after > queries_before, (
|
||||
"Cache should have been queried for the injected item"
|
||||
)
|
||||
mm_rate = _get_mm_cache_log(llm, caplog_vllm)
|
||||
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
|
||||
# Verify cache was queried and injection happened
|
||||
queries_after = _get_mm_cache_stats(runner.llm.get_metrics())[0]
|
||||
assert queries_after > queries_before, (
|
||||
"Cache should have been queried for the injected item"
|
||||
)
|
||||
mm_rate = _get_mm_cache_log(runner.llm, caplog_vllm)
|
||||
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
|
||||
def test_inject_into_mm_cache_without_cache(
|
||||
num_gpus_available,
|
||||
image_urls,
|
||||
multimodal_llm_factory,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Test that inject_into_mm_cache works gracefully when processor cache
|
||||
is disabled (mm_processor_cache_gb=0). Should not crash.
|
||||
"""
|
||||
llm = multimodal_llm_factory(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
with vllm_runner(
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
mm_processor_cache_gb=0,
|
||||
)
|
||||
) as runner:
|
||||
# Run a normal chat request first to warm up the model.
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
|
||||
# Run a normal chat request first to warm up the model.
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
# Use the renderer to get a proper EngineInput with expanded tokens
|
||||
renderer = runner.llm.llm_engine.renderer
|
||||
_, eng_prompts = renderer.render_chat(
|
||||
[_make_messages(image_urls[0])],
|
||||
ChatParams(),
|
||||
)
|
||||
eng_input = eng_prompts[0]
|
||||
|
||||
# Use the renderer to get a proper EngineInput with expanded tokens
|
||||
renderer = llm.llm_engine.renderer
|
||||
_, eng_prompts = renderer.render_chat(
|
||||
[_make_messages(image_urls[0])],
|
||||
ChatParams(),
|
||||
)
|
||||
eng_input = eng_prompts[0]
|
||||
mm_hashes = {"image": ["abcd1234" * 8]}
|
||||
mm_kwargs = eng_input["mm_kwargs"]
|
||||
|
||||
mm_hashes = {"image": ["abcd1234" * 8]}
|
||||
mm_kwargs = eng_input["mm_kwargs"]
|
||||
# inject_into_mm_cache should not crash even without cache
|
||||
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||
|
||||
# inject_into_mm_cache should not crash even without cache
|
||||
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||
# Build and generate with pre-rendered input
|
||||
pre_rendered_input = {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||
"mm_kwargs": mm_kwargs,
|
||||
"mm_hashes": mm_hashes,
|
||||
"mm_placeholders": eng_input["mm_placeholders"],
|
||||
}
|
||||
|
||||
# Build and generate with pre-rendered input
|
||||
pre_rendered_input = {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||
"mm_kwargs": mm_kwargs,
|
||||
"mm_hashes": mm_hashes,
|
||||
"mm_placeholders": eng_input["mm_placeholders"],
|
||||
}
|
||||
|
||||
result = llm.generate(
|
||||
pre_rendered_input,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
)
|
||||
assert len(result) == 1, "Should produce one output"
|
||||
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
|
||||
result = runner.llm.generate(
|
||||
pre_rendered_input,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
)
|
||||
assert len(result) == 1, "Should produce one output"
|
||||
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
|
||||
|
||||
@@ -61,38 +61,37 @@ def test_mm_cache_stats(
|
||||
image_urls,
|
||||
mm_processor_cache_type,
|
||||
caplog_vllm,
|
||||
multimodal_llm_factory,
|
||||
vllm_runner,
|
||||
):
|
||||
llm = multimodal_llm_factory(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
with vllm_runner(
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
)
|
||||
) as runner:
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
||||
runner.llm.chat(_make_messages(image_urls[1]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 0)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[1]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (3, 1)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(33.3)
|
||||
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3)
|
||||
# NOTE: This only resets hit rate stats in CachingMetrics
|
||||
# The raw queries and hits counts remain unaffected
|
||||
runner.llm.reset_mm_cache()
|
||||
|
||||
# NOTE: This only resets hit rate stats in CachingMetrics
|
||||
# The raw queries and hits counts remain unaffected
|
||||
llm.reset_mm_cache()
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (4, 1)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[1]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
||||
runner.llm.chat(_make_messages(image_urls[1]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (5, 1)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.multimodal.conftest import managed_llm
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
@@ -14,17 +15,19 @@ TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
def llm(vllm_runner):
|
||||
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
|
||||
with managed_llm(
|
||||
model=MODEL,
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.8,
|
||||
enable_mm_embeds=True,
|
||||
limit_mm_per_prompt={"image": 0},
|
||||
) as llm:
|
||||
yield llm
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
Reference in New Issue
Block a user