[MM][V2] Enable encoder-only ViT CUDA graph capture (#56922)
Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -13,7 +13,9 @@ Test organization:
|
||||
"""
|
||||
|
||||
from collections.abc import Hashable
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -575,6 +577,99 @@ class TestEncoderCudaGraphCaptureReplay:
|
||||
assert out.shape == (4, _HIDDEN)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# E-only capture and output lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
|
||||
)
|
||||
@pytest.mark.usefixtures("dist_init", "workspace_init")
|
||||
@pytest.mark.parametrize("profile_only", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only):
|
||||
"""The E-only entry captures only the encoder and preserves cached outputs."""
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.worker.gpu.ec_connector import ActiveECConnector
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
|
||||
from vllm.v1.worker.workspace import lock_workspace
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
dtype = torch.float16
|
||||
model = SimpleMockViTModel().to(device).half()
|
||||
manager = _make_manager_for_gpu(model, _BUDGETS, _MAX_BATCH, device, dtype)
|
||||
encoder = object.__new__(EncoderRunner)
|
||||
encoder.device = device
|
||||
encoder.cudagraph_manager = manager
|
||||
runner = object.__new__(MMEncoderModelRunner)
|
||||
runner.model_state = SimpleNamespace(encoder_runner=encoder)
|
||||
# No decoder manager is installed: capture must be encoder-only.
|
||||
with patch(
|
||||
"vllm.v1.worker.mm_encoder_model_runner.lock_workspace", wraps=lock_workspace
|
||||
) as lock:
|
||||
runner.capture_model(profile_only=profile_only)
|
||||
assert lock.call_count == int(not profile_only)
|
||||
assert len(manager.budget_graphs["default"]) == len(_BUDGETS)
|
||||
|
||||
cache: dict[str, torch.Tensor] = {}
|
||||
pending_sends: dict[str, torch.Tensor] = {}
|
||||
connector = MagicMock(spec=ECConnectorBase)
|
||||
connector.is_producer = True
|
||||
connector.is_consumer = False
|
||||
connector.get_finished.return_value = (None, None)
|
||||
connector.save_caches.side_effect = lambda *, encoder_cache, mm_hash: (
|
||||
pending_sends.update({mm_hash: encoder_cache[mm_hash]})
|
||||
)
|
||||
with patch(
|
||||
"vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector
|
||||
):
|
||||
ec = ActiveECConnector(SimpleNamespace(), cache)
|
||||
scheduled = SimpleNamespace(
|
||||
ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()
|
||||
)
|
||||
saved_outputs: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
# Mixed sizes exercise packing order; 64 is the boundary, 81 falls back.
|
||||
for grids in ([[1, 8, 8], [1, 4, 4]], [[1, 16, 16]], [[1, 18, 18]], [[1, 4, 4]]):
|
||||
inputs = _make_mm_kwargs(grids, device, dtype)
|
||||
expected = model.encoder_eager_forward(inputs).split(
|
||||
[t * (h // 2) * (w // 2) for t, h, w in grids]
|
||||
)
|
||||
with ec.maybe_get_output(scheduled) as ec_output:
|
||||
outputs = manager.execute(inputs)
|
||||
assert outputs is not None
|
||||
cache[str(len(cache))] = outputs[0]
|
||||
assert ec_output.finished_sending is None
|
||||
assert outputs is not None
|
||||
for actual, eager in zip(outputs, expected):
|
||||
torch.testing.assert_close(actual, eager)
|
||||
for previous, snapshot in saved_outputs:
|
||||
torch.testing.assert_close(previous, snapshot, rtol=0, atol=0)
|
||||
saved_outputs.extend((output, output.clone()) for output in outputs)
|
||||
assert manager.graph_hits == 4
|
||||
assert manager.graph_misses == 1
|
||||
assert pending_sends.keys() == cache.keys()
|
||||
connector.get_finished.return_value = (set(cache), None)
|
||||
with ec.maybe_get_output(scheduled) as ec_output:
|
||||
pass
|
||||
assert ec_output.finished_sending == set(cache)
|
||||
|
||||
|
||||
def test_eonly_without_encoder_graph_skips_capture():
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
|
||||
|
||||
encoder = object.__new__(EncoderRunner)
|
||||
encoder.cudagraph_manager = None
|
||||
runner = object.__new__(MMEncoderModelRunner)
|
||||
runner.model_state = SimpleNamespace(encoder_runner=encoder)
|
||||
assert runner.capture_model() == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SimpleMockViTVideoModel — extends SimpleMockViTModel with video support
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user