[MM][V2] Enable encoder-only ViT CUDA graph capture (#56922)
Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -9,12 +9,15 @@ and tolerated (token-embedding fallback) when it is not, while a miss within
|
||||
the processed range still fails loudly.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
@@ -31,6 +34,56 @@ pytestmark = pytest.mark.cpu_test
|
||||
HIDDEN = 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,enforce_eager,supported,expected",
|
||||
[
|
||||
(True, False, True, True),
|
||||
(False, False, True, False),
|
||||
(True, True, True, False),
|
||||
(True, False, False, False),
|
||||
],
|
||||
)
|
||||
def test_encoder_graph_configuration(enabled, enforce_eager, supported, expected):
|
||||
"""Encoder graph eligibility is independent of decoder graph mode."""
|
||||
config = MagicMock()
|
||||
config.model_config.enforce_eager = enforce_eager
|
||||
config.model_config.dtype = torch.float32
|
||||
config.model_config.get_inputs_embeds_size.return_value = HIDDEN
|
||||
config.scheduler_config.max_num_batched_tokens = 8
|
||||
config.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
config.compilation_config.cudagraph_mm_encoder = enabled
|
||||
model = MagicMock(spec=SupportsEncoderCudaGraph) if supported else torch.nn.Module()
|
||||
state = _model_state(EncoderCache())
|
||||
with patch("vllm.v1.worker.gpu.model_states.interface.EncoderCudaGraphManager"):
|
||||
ModelState.__init__(state, config, model, state.encoder_cache, state.device)
|
||||
assert state.encoder_runner.has_cudagraph() is expected
|
||||
|
||||
|
||||
def test_encoder_graph_unsupported_modality_uses_eager_output():
|
||||
"""An image graph must not intercept another modality's encoder output."""
|
||||
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
|
||||
|
||||
runner = _make_runner([], [])
|
||||
output = torch.ones(2, HIDDEN)
|
||||
runner.model = MagicMock()
|
||||
runner.model.embed_multimodal.return_value = [output]
|
||||
manager = MagicMock(spec=EncoderCudaGraphManager)
|
||||
manager.config = SimpleNamespace(modalities=["image"])
|
||||
manager.is_captured.return_value = True
|
||||
manager.supports_modality.side_effect = lambda modality: (
|
||||
EncoderCudaGraphManager.supports_modality(manager, modality)
|
||||
)
|
||||
runner.cudagraph_manager = manager
|
||||
with patch(
|
||||
"vllm.v1.worker.gpu.mm.encoder_runner.group_and_batch_mm_kwargs",
|
||||
return_value=[("audio", 1, {"audio_values": torch.zeros(2)})],
|
||||
):
|
||||
result = runner.execute_mm_encoder([])
|
||||
assert len(result) == 1
|
||||
assert result[0] is output
|
||||
manager.execute.assert_not_called()
|
||||
|
||||
|
||||
def _model_state(cache: EncoderCache) -> MagicMock:
|
||||
"""A mock ModelState backed by a real EncoderCache."""
|
||||
state = MagicMock()
|
||||
|
||||
Reference in New Issue
Block a user