[Bugfix] Clear P0 mm sender cache on sleep/pause to fix mm_hash desync (#43001)
Signed-off-by: Tobias Wasner <wasnertobias@gmail.com>
This commit is contained in:
@@ -549,3 +549,42 @@ def test_processor_cache_shared_across_loras():
|
||||
|
||||
receiver_cache.get_and_update_features([feature_lora_b])
|
||||
assert feature_lora_b.data == item_data
|
||||
|
||||
|
||||
_SLEEP_VISION_PROMPT = (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
|
||||
"\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
|
||||
"What is in the image?<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="sleep mode regression requires a CUDA GPU",
|
||||
)
|
||||
def test_sleep_wake_preserves_mm_cache_consistency():
|
||||
"""Regression for vllm-project/vllm#42995."""
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
image = ImageAsset("stop_sign").pil_image
|
||||
prompt = {
|
||||
"prompt": _SLEEP_VISION_PROMPT,
|
||||
"multi_modal_data": {"image": image},
|
||||
}
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=8)
|
||||
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen2-VL-2B-Instruct",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.5,
|
||||
max_model_len=2048,
|
||||
)
|
||||
|
||||
llm.generate([prompt], sampling_params)
|
||||
llm.sleep(level=1)
|
||||
llm.wake_up()
|
||||
output2 = llm.generate([prompt], sampling_params)
|
||||
assert output2[0].outputs[0].text
|
||||
|
||||
@@ -781,6 +781,8 @@ class AsyncLLM(EngineClient):
|
||||
stacklevel=2,
|
||||
)
|
||||
mode = "wait"
|
||||
if clear_cache:
|
||||
await self.renderer.clear_mm_cache_async()
|
||||
await self.engine_core.pause_scheduler_async(mode=mode, clear_cache=clear_cache)
|
||||
# Small sleep to help ensure that final outputs from any in-flight requests are
|
||||
# returned prior to this method returning. These outputs come out of the engine
|
||||
@@ -927,6 +929,8 @@ class AsyncLLM(EngineClient):
|
||||
await self.engine_core.reset_encoder_cache_async()
|
||||
|
||||
async def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
|
||||
if level >= 1:
|
||||
await self.renderer.clear_mm_cache_async()
|
||||
await self.engine_core.sleep_async(level, mode)
|
||||
|
||||
if self.logger_manager is not None:
|
||||
|
||||
@@ -350,6 +350,8 @@ class LLMEngine:
|
||||
self.engine_core.reset_encoder_cache()
|
||||
|
||||
def sleep(self, level: int = 1, mode: PauseMode = "abort"):
|
||||
if level >= 1:
|
||||
self.renderer.clear_mm_cache()
|
||||
self.engine_core.sleep(level, mode)
|
||||
|
||||
if self.logger_manager is not None:
|
||||
|
||||
Reference in New Issue
Block a user