[MRV1] Scope breakable cudagraphs to the piecewise path only (#56312)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Nick Hill
2026-09-11 19:55:34 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent dffb863c97
commit cc5dd0a857
4 changed files with 165 additions and 17 deletions
+131 -1
View File
@@ -8,7 +8,7 @@ from __future__ import annotations
import threading
from contextlib import nullcontext
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import pytest
import torch
@@ -536,3 +536,133 @@ def test_nested_decorated_op_runs_inline(cuda_capture_stream):
# 0 -> +2 -> +1 (inner) -> +10 (outer) -> +100 = 113
assert torch.equal(x, torch.full((4,), 113.0, device="cuda"))
assert inner_calls == 2 # replay invokes the outer's lambda again
# ---------------------------------------------------------------------------
# BreakableCUDAGraphWrapper: runtime_mode dispatch gating
# ---------------------------------------------------------------------------
def _mock_vllm_config():
config = MagicMock()
config.parallel_config.data_parallel_size = 1
config.parallel_config.use_sequence_parallel_moe = False
config.parallel_config.is_moe_model = False
return config
def test_wrapper_falls_through_on_runtime_mode_mismatch():
"""A PIECEWISE-scoped wrapper runs other modes eagerly, capturing
nothing, so an outer CUDAGraphWrapper can own them."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
from vllm.config import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
calls = []
def model(x):
calls.append(x)
return x + 1
config = _mock_vllm_config()
wrapper = BreakableCUDAGraphWrapper(
model, config, runtime_mode=CUDAGraphMode.PIECEWISE
)
desc = BatchDescriptor(num_tokens=4)
with set_forward_context(
None,
config,
cudagraph_runtime_mode=CUDAGraphMode.FULL,
batch_descriptor=desc,
):
assert wrapper(1) == 2
assert calls == [1]
assert not wrapper.entries
def test_wrapper_captures_on_runtime_mode_match():
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
from vllm.config import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
calls = []
def model(x):
calls.append(x)
return x + 1
config = _mock_vllm_config()
wrapper = BreakableCUDAGraphWrapper(
model, config, runtime_mode=CUDAGraphMode.PIECEWISE
)
desc = BatchDescriptor(num_tokens=4)
sentinel = object()
with (
patch.object(
BreakableCUDAGraphWrapper, "_capture", return_value=sentinel
) as mock_capture,
set_forward_context(
None,
config,
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
batch_descriptor=desc,
),
):
assert wrapper(1) is sentinel
mock_capture.assert_called_once()
assert calls == []
assert desc in wrapper.entries
def test_wrapper_default_matches_any_non_none_mode():
"""runtime_mode=None matches any non-NONE mode (MRV2 relies on this)."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
from vllm.config import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
def model(x):
return x + 1
config = _mock_vllm_config()
wrapper = BreakableCUDAGraphWrapper(model, config)
desc = BatchDescriptor(num_tokens=4)
sentinel = object()
with (
patch.object(
BreakableCUDAGraphWrapper, "_capture", return_value=sentinel
) as mock_capture,
set_forward_context(
None,
config,
cudagraph_runtime_mode=CUDAGraphMode.FULL,
batch_descriptor=desc,
),
):
assert wrapper(1) is sentinel
mock_capture.assert_called_once()
assert desc in wrapper.entries
def test_unwrap_recurses_through_nested_wrappers():
"""unwrap() reaches the raw model through nested wrappers."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
from vllm.compilation.cuda_graph import CUDAGraphWrapper
from vllm.config import CUDAGraphMode
def model(x):
return x + 1
config = _mock_vllm_config()
inner = BreakableCUDAGraphWrapper(
model, config, runtime_mode=CUDAGraphMode.PIECEWISE
)
outer = CUDAGraphWrapper(inner, config, runtime_mode=CUDAGraphMode.FULL)
assert inner.unwrap() is model
assert outer.unwrap() is model