[Model] Add PW CUDA graph support for Inkling [2/N] (#48822)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-16 10:28:56 -07:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent ce65385618
commit 251f7e478e
10 changed files with 98 additions and 32 deletions
@@ -34,17 +34,17 @@ def test_inkling_raw_2d_audio_is_rejected_as_ambiguous():
parser._parse_audio_data(np.zeros((2, 100), dtype=np.float32))
def test_inkling_supports_full_decode_only_cudagraphs():
def test_inkling_supports_piecewise_cudagraphs():
support = InklingSconvMetadataBuilder.get_cudagraph_support
assert support(None, None) == AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
assert support(None, None) == AttentionCGSupport.UNIFORM_BATCH
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL,
cudagraph_mode=CUDAGraphMode.PIECEWISE,
splitting_ops=[],
)
resolved_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE,
AttentionCGSupport.UNIFORM_BATCH,
"InklingSconvBackend",
)
assert resolved_mode == CUDAGraphMode.FULL_DECODE_ONLY
assert resolved_mode == CUDAGraphMode.PIECEWISE
@@ -8,6 +8,8 @@ from __future__ import annotations
import os
import threading
from contextlib import nullcontext
from unittest.mock import patch
import pytest
import torch
@@ -15,6 +17,54 @@ import torch
os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
from vllm.config import CUDAGraphMode
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
CudaGraphManager,
)
manager = CudaGraphManager.__new__(CudaGraphManager)
desc = BatchExecutionDescriptor(CUDAGraphMode.PIECEWISE, 8, None)
manager.device = torch.device("cpu")
manager._capture_descs = {CUDAGraphMode.PIECEWISE: [desc]}
manager._graphs_captured = False
manager.use_breakable_cg = True
create_calls = []
forward_calls = []
def create_forward_fn(desc_arg, warmup):
assert desc_arg == desc
metadata = {"layer": object()}
create_calls.append((warmup, metadata))
def forward_fn(cg_mode):
assert metadata
forward_calls.append((warmup, cg_mode, metadata))
return forward_fn
with (
patch(
"vllm.v1.worker.gpu.cudagraph_utils.graph_capture",
return_value=nullcontext(),
),
patch(
"vllm.v1.worker.gpu.cudagraph_utils.is_global_first_rank",
return_value=False,
),
):
manager.capture(create_forward_fn)
assert [warmup for warmup, _ in create_calls] == [True, False]
assert [mode for _, mode, _ in forward_calls] == [
CUDAGraphMode.NONE,
CUDAGraphMode.PIECEWISE,
]
assert create_calls[0][1] is not create_calls[1][1]
@pytest.fixture(autouse=True)
def _reset_breakable_tls():
"""Defensively clear thread-local capture state between tests so a