[MRV1] Scope breakable cudagraphs to the piecewise path only (#56312)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
dffb863c97
commit
cc5dd0a857
@@ -8,7 +8,7 @@ from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -536,3 +536,133 @@ def test_nested_decorated_op_runs_inline(cuda_capture_stream):
|
||||
# 0 -> +2 -> +1 (inner) -> +10 (outer) -> +100 = 113
|
||||
assert torch.equal(x, torch.full((4,), 113.0, device="cuda"))
|
||||
assert inner_calls == 2 # replay invokes the outer's lambda again
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BreakableCUDAGraphWrapper: runtime_mode dispatch gating
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mock_vllm_config():
|
||||
config = MagicMock()
|
||||
config.parallel_config.data_parallel_size = 1
|
||||
config.parallel_config.use_sequence_parallel_moe = False
|
||||
config.parallel_config.is_moe_model = False
|
||||
return config
|
||||
|
||||
|
||||
def test_wrapper_falls_through_on_runtime_mode_mismatch():
|
||||
"""A PIECEWISE-scoped wrapper runs other modes eagerly, capturing
|
||||
nothing, so an outer CUDAGraphWrapper can own them."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
|
||||
calls = []
|
||||
|
||||
def model(x):
|
||||
calls.append(x)
|
||||
return x + 1
|
||||
|
||||
config = _mock_vllm_config()
|
||||
wrapper = BreakableCUDAGraphWrapper(
|
||||
model, config, runtime_mode=CUDAGraphMode.PIECEWISE
|
||||
)
|
||||
desc = BatchDescriptor(num_tokens=4)
|
||||
|
||||
with set_forward_context(
|
||||
None,
|
||||
config,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||
batch_descriptor=desc,
|
||||
):
|
||||
assert wrapper(1) == 2
|
||||
|
||||
assert calls == [1]
|
||||
assert not wrapper.entries
|
||||
|
||||
|
||||
def test_wrapper_captures_on_runtime_mode_match():
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
|
||||
calls = []
|
||||
|
||||
def model(x):
|
||||
calls.append(x)
|
||||
return x + 1
|
||||
|
||||
config = _mock_vllm_config()
|
||||
wrapper = BreakableCUDAGraphWrapper(
|
||||
model, config, runtime_mode=CUDAGraphMode.PIECEWISE
|
||||
)
|
||||
desc = BatchDescriptor(num_tokens=4)
|
||||
|
||||
sentinel = object()
|
||||
with (
|
||||
patch.object(
|
||||
BreakableCUDAGraphWrapper, "_capture", return_value=sentinel
|
||||
) as mock_capture,
|
||||
set_forward_context(
|
||||
None,
|
||||
config,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
|
||||
batch_descriptor=desc,
|
||||
),
|
||||
):
|
||||
assert wrapper(1) is sentinel
|
||||
|
||||
mock_capture.assert_called_once()
|
||||
assert calls == []
|
||||
assert desc in wrapper.entries
|
||||
|
||||
|
||||
def test_wrapper_default_matches_any_non_none_mode():
|
||||
"""runtime_mode=None matches any non-NONE mode (MRV2 relies on this)."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
|
||||
def model(x):
|
||||
return x + 1
|
||||
|
||||
config = _mock_vllm_config()
|
||||
wrapper = BreakableCUDAGraphWrapper(model, config)
|
||||
desc = BatchDescriptor(num_tokens=4)
|
||||
|
||||
sentinel = object()
|
||||
with (
|
||||
patch.object(
|
||||
BreakableCUDAGraphWrapper, "_capture", return_value=sentinel
|
||||
) as mock_capture,
|
||||
set_forward_context(
|
||||
None,
|
||||
config,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||
batch_descriptor=desc,
|
||||
),
|
||||
):
|
||||
assert wrapper(1) is sentinel
|
||||
|
||||
mock_capture.assert_called_once()
|
||||
assert desc in wrapper.entries
|
||||
|
||||
|
||||
def test_unwrap_recurses_through_nested_wrappers():
|
||||
"""unwrap() reaches the raw model through nested wrappers."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode
|
||||
|
||||
def model(x):
|
||||
return x + 1
|
||||
|
||||
config = _mock_vllm_config()
|
||||
inner = BreakableCUDAGraphWrapper(
|
||||
model, config, runtime_mode=CUDAGraphMode.PIECEWISE
|
||||
)
|
||||
outer = CUDAGraphWrapper(inner, config, runtime_mode=CUDAGraphMode.FULL)
|
||||
|
||||
assert inner.unwrap() is model
|
||||
assert outer.unwrap() is model
|
||||
|
||||
Reference in New Issue
Block a user