[MM][V2] Enable encoder-only ViT CUDA graph capture (#56922)

Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
jiangkuaixue123
2026-09-17 16:12:17 +08:00
committed by GitHub
co-authored by Codex
parent 4e5ffda19d
commit de24e51908
7 changed files with 362 additions and 5 deletions
@@ -13,7 +13,9 @@ Test organization:
"""
from collections.abc import Hashable
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
import torch
@@ -575,6 +577,99 @@ class TestEncoderCudaGraphCaptureReplay:
assert out.shape == (4, _HIDDEN)
# ---------------------------------------------------------------------------
# E-only capture and output lifecycle
# ---------------------------------------------------------------------------
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
)
@pytest.mark.usefixtures("dist_init", "workspace_init")
@pytest.mark.parametrize("profile_only", [False, True])
@torch.inference_mode()
def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only):
"""The E-only entry captures only the encoder and preserves cached outputs."""
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorMetadata,
)
from vllm.v1.worker.gpu.ec_connector import ActiveECConnector
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
from vllm.v1.worker.workspace import lock_workspace
device = torch.device("cuda:0")
dtype = torch.float16
model = SimpleMockViTModel().to(device).half()
manager = _make_manager_for_gpu(model, _BUDGETS, _MAX_BATCH, device, dtype)
encoder = object.__new__(EncoderRunner)
encoder.device = device
encoder.cudagraph_manager = manager
runner = object.__new__(MMEncoderModelRunner)
runner.model_state = SimpleNamespace(encoder_runner=encoder)
# No decoder manager is installed: capture must be encoder-only.
with patch(
"vllm.v1.worker.mm_encoder_model_runner.lock_workspace", wraps=lock_workspace
) as lock:
runner.capture_model(profile_only=profile_only)
assert lock.call_count == int(not profile_only)
assert len(manager.budget_graphs["default"]) == len(_BUDGETS)
cache: dict[str, torch.Tensor] = {}
pending_sends: dict[str, torch.Tensor] = {}
connector = MagicMock(spec=ECConnectorBase)
connector.is_producer = True
connector.is_consumer = False
connector.get_finished.return_value = (None, None)
connector.save_caches.side_effect = lambda *, encoder_cache, mm_hash: (
pending_sends.update({mm_hash: encoder_cache[mm_hash]})
)
with patch(
"vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector
):
ec = ActiveECConnector(SimpleNamespace(), cache)
scheduled = SimpleNamespace(
ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()
)
saved_outputs: list[tuple[torch.Tensor, torch.Tensor]] = []
# Mixed sizes exercise packing order; 64 is the boundary, 81 falls back.
for grids in ([[1, 8, 8], [1, 4, 4]], [[1, 16, 16]], [[1, 18, 18]], [[1, 4, 4]]):
inputs = _make_mm_kwargs(grids, device, dtype)
expected = model.encoder_eager_forward(inputs).split(
[t * (h // 2) * (w // 2) for t, h, w in grids]
)
with ec.maybe_get_output(scheduled) as ec_output:
outputs = manager.execute(inputs)
assert outputs is not None
cache[str(len(cache))] = outputs[0]
assert ec_output.finished_sending is None
assert outputs is not None
for actual, eager in zip(outputs, expected):
torch.testing.assert_close(actual, eager)
for previous, snapshot in saved_outputs:
torch.testing.assert_close(previous, snapshot, rtol=0, atol=0)
saved_outputs.extend((output, output.clone()) for output in outputs)
assert manager.graph_hits == 4
assert manager.graph_misses == 1
assert pending_sends.keys() == cache.keys()
connector.get_finished.return_value = (set(cache), None)
with ec.maybe_get_output(scheduled) as ec_output:
pass
assert ec_output.finished_sending == set(cache)
def test_eonly_without_encoder_graph_skips_capture():
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
encoder = object.__new__(EncoderRunner)
encoder.cudagraph_manager = None
runner = object.__new__(MMEncoderModelRunner)
runner.model_state = SimpleNamespace(encoder_runner=encoder)
assert runner.capture_model() == 0
# ---------------------------------------------------------------------------
# SimpleMockViTVideoModel — extends SimpleMockViTModel with video support
# ---------------------------------------------------------------------------