[Model] Add PW CUDA graph support for Inkling [2/N] (#48822)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
ce65385618
commit
251f7e478e
@@ -34,17 +34,17 @@ def test_inkling_raw_2d_audio_is_rejected_as_ambiguous():
|
|||||||
parser._parse_audio_data(np.zeros((2, 100), dtype=np.float32))
|
parser._parse_audio_data(np.zeros((2, 100), dtype=np.float32))
|
||||||
|
|
||||||
|
|
||||||
def test_inkling_supports_full_decode_only_cudagraphs():
|
def test_inkling_supports_piecewise_cudagraphs():
|
||||||
support = InklingSconvMetadataBuilder.get_cudagraph_support
|
support = InklingSconvMetadataBuilder.get_cudagraph_support
|
||||||
assert support(None, None) == AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
assert support(None, None) == AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
compilation_config = CompilationConfig(
|
compilation_config = CompilationConfig(
|
||||||
cudagraph_mode=CUDAGraphMode.FULL,
|
cudagraph_mode=CUDAGraphMode.PIECEWISE,
|
||||||
splitting_ops=[],
|
splitting_ops=[],
|
||||||
)
|
)
|
||||||
resolved_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
|
resolved_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||||
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE,
|
AttentionCGSupport.UNIFORM_BATCH,
|
||||||
"InklingSconvBackend",
|
"InklingSconvBackend",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert resolved_mode == CUDAGraphMode.FULL_DECODE_ONLY
|
assert resolved_mode == CUDAGraphMode.PIECEWISE
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import threading
|
import threading
|
||||||
|
from contextlib import nullcontext
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
@@ -15,6 +17,54 @@ import torch
|
|||||||
os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
|
os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
|
||||||
|
from vllm.config import CUDAGraphMode
|
||||||
|
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||||
|
BatchExecutionDescriptor,
|
||||||
|
CudaGraphManager,
|
||||||
|
)
|
||||||
|
|
||||||
|
manager = CudaGraphManager.__new__(CudaGraphManager)
|
||||||
|
desc = BatchExecutionDescriptor(CUDAGraphMode.PIECEWISE, 8, None)
|
||||||
|
manager.device = torch.device("cpu")
|
||||||
|
manager._capture_descs = {CUDAGraphMode.PIECEWISE: [desc]}
|
||||||
|
manager._graphs_captured = False
|
||||||
|
manager.use_breakable_cg = True
|
||||||
|
|
||||||
|
create_calls = []
|
||||||
|
forward_calls = []
|
||||||
|
|
||||||
|
def create_forward_fn(desc_arg, warmup):
|
||||||
|
assert desc_arg == desc
|
||||||
|
metadata = {"layer": object()}
|
||||||
|
create_calls.append((warmup, metadata))
|
||||||
|
|
||||||
|
def forward_fn(cg_mode):
|
||||||
|
assert metadata
|
||||||
|
forward_calls.append((warmup, cg_mode, metadata))
|
||||||
|
|
||||||
|
return forward_fn
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"vllm.v1.worker.gpu.cudagraph_utils.graph_capture",
|
||||||
|
return_value=nullcontext(),
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"vllm.v1.worker.gpu.cudagraph_utils.is_global_first_rank",
|
||||||
|
return_value=False,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
manager.capture(create_forward_fn)
|
||||||
|
|
||||||
|
assert [warmup for warmup, _ in create_calls] == [True, False]
|
||||||
|
assert [mode for _, mode, _ in forward_calls] == [
|
||||||
|
CUDAGraphMode.NONE,
|
||||||
|
CUDAGraphMode.PIECEWISE,
|
||||||
|
]
|
||||||
|
assert create_calls[0][1] is not create_calls[1][1]
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def _reset_breakable_tls():
|
def _reset_breakable_tls():
|
||||||
"""Defensively clear thread-local capture state between tests so a
|
"""Defensively clear thread-local capture state between tests so a
|
||||||
|
|||||||
@@ -1168,6 +1168,8 @@ class VllmConfig:
|
|||||||
in (
|
in (
|
||||||
"DeepseekV4ForCausalLM",
|
"DeepseekV4ForCausalLM",
|
||||||
"DeepSeekV4MTPModel",
|
"DeepSeekV4MTPModel",
|
||||||
|
"InklingForCausalLM",
|
||||||
|
"InklingForConditionalGeneration",
|
||||||
"MiniMaxM3SparseForCausalLM",
|
"MiniMaxM3SparseForCausalLM",
|
||||||
"MiniMaxM3SparseForConditionalGeneration",
|
"MiniMaxM3SparseForConditionalGeneration",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from typing import cast
|
|||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
|
|
||||||
|
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||||
from vllm.config import VllmConfig, get_current_vllm_config
|
from vllm.config import VllmConfig, get_current_vllm_config
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.forward_context import get_forward_context
|
from vllm.forward_context import get_forward_context
|
||||||
@@ -286,6 +287,7 @@ class InklingAttention(nn.Module, AttentionLayerBase):
|
|||||||
output, _ = self.wo_ud(flat)
|
output, _ = self.wo_ud(flat)
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
@eager_break_during_capture
|
||||||
def _attention(
|
def _attention(
|
||||||
self,
|
self,
|
||||||
q: torch.Tensor,
|
q: torch.Tensor,
|
||||||
|
|||||||
@@ -21,8 +21,8 @@ spec alike.
|
|||||||
All kernels address the cache purely by ``(slot, absolute_position)`` and
|
All kernels address the cache purely by ``(slot, absolute_position)`` and
|
||||||
allocate nothing inside the captured region; their grids depend only on the
|
allocate nothing inside the captured region; their grids depend only on the
|
||||||
token count (``fused_sconv`` on a fixed token/channel tiling), so the same
|
token count (``fused_sconv`` on a fixed token/channel tiling), so the same
|
||||||
decode path replays correctly under a full CUDA graph without any
|
path replays correctly under eager, breakable PIECEWISE, and FULL cudagraphs
|
||||||
data-dependent shape or branch.
|
without any data-dependent shape or branch.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -167,8 +167,8 @@ def fused_sconv(
|
|||||||
"""Single-launch insert + depthwise causal conv1d over the paged cache.
|
"""Single-launch insert + depthwise causal conv1d over the paged cache.
|
||||||
|
|
||||||
Reads same-forward taps from ``x`` and pre-forward taps from the cache, so
|
Reads same-forward taps from ``x`` and pre-forward taps from the cache, so
|
||||||
it is race-free in one launch for prefill / decode / spec and supports full
|
it is race-free in one launch for prefill / decode / spec and cudagraph-safe
|
||||||
CUDA-graph capture for decode.
|
under eager / piecewise / full capture.
|
||||||
"""
|
"""
|
||||||
T = x.shape[0]
|
T = x.shape[0]
|
||||||
out = torch.empty_like(x)
|
out = torch.empty_like(x)
|
||||||
|
|||||||
@@ -49,9 +49,7 @@ class InklingSconvMetadata(AttentionMetadata):
|
|||||||
|
|
||||||
|
|
||||||
class InklingSconvMetadataBuilder(AttentionMetadataBuilder[InklingSconvMetadata]):
|
class InklingSconvMetadataBuilder(AttentionMetadataBuilder[InklingSconvMetadata]):
|
||||||
_cudagraph_support: ClassVar[AttentionCGSupport] = (
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||||
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
|
||||||
)
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ Per-forward metadata (``block_table`` / ``slot_mapping`` / ``seq_idx`` /
|
|||||||
the owner's prefix in the forward context; the absolute ``positions`` are
|
the owner's prefix in the forward context; the absolute ``positions`` are
|
||||||
threaded in from the model. The insert + conv run in a single ``fused_sconv``
|
threaded in from the model. The insert + conv run in a single ``fused_sconv``
|
||||||
launch (same path for prefill / decode / mixed / spec). All inputs are
|
launch (same path for prefill / decode / mixed / spec). All inputs are
|
||||||
fixed-address persistent buffers and the grid is fixed, so decode can replay
|
fixed-address persistent buffers and the grid is fixed, so the conv replays
|
||||||
under a full CUDA graph.
|
correctly under eager, PIECEWISE, and FULL cudagraphs.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|||||||
@@ -138,8 +138,6 @@ class CudaGraphManager:
|
|||||||
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
|
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
|
||||||
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
||||||
|
|
||||||
self._init_candidates()
|
|
||||||
|
|
||||||
# Breakable CUDA graph (PW CUDA graph without torch.compile)
|
# Breakable CUDA graph (PW CUDA graph without torch.compile)
|
||||||
self.use_breakable_cg = (
|
self.use_breakable_cg = (
|
||||||
is_breakable_cudagraph_enabled()
|
is_breakable_cudagraph_enabled()
|
||||||
@@ -147,6 +145,8 @@ class CudaGraphManager:
|
|||||||
)
|
)
|
||||||
self.breakable_cg_runner: BreakableCUDAGraphWrapper | None = None
|
self.breakable_cg_runner: BreakableCUDAGraphWrapper | None = None
|
||||||
|
|
||||||
|
self._init_candidates()
|
||||||
|
|
||||||
def _build_lora_dispatch_map(self) -> tuple[dict[int, int], int]:
|
def _build_lora_dispatch_map(self) -> tuple[dict[int, int], int]:
|
||||||
"""Precompute actual num_active_loras -> effective captured case.
|
"""Precompute actual num_active_loras -> effective captured case.
|
||||||
|
|
||||||
@@ -255,11 +255,11 @@ class CudaGraphManager:
|
|||||||
# for PIECEWISE graphs there is no limit on requests when replaying
|
# for PIECEWISE graphs there is no limit on requests when replaying
|
||||||
# i.e. no request padding is needed
|
# i.e. no request padding is needed
|
||||||
# so we leave it as None
|
# so we leave it as None
|
||||||
num_reqs = (
|
num_reqs = None
|
||||||
min(num_tokens, self.max_num_reqs)
|
if mixed_mode == CUDAGraphMode.FULL or (
|
||||||
if mixed_mode == CUDAGraphMode.FULL
|
mixed_mode == CUDAGraphMode.PIECEWISE and self.use_breakable_cg
|
||||||
else None
|
):
|
||||||
)
|
num_reqs = min(num_tokens, self.max_num_reqs)
|
||||||
desc = BatchExecutionDescriptor(
|
desc = BatchExecutionDescriptor(
|
||||||
cg_mode=mixed_mode,
|
cg_mode=mixed_mode,
|
||||||
num_tokens=num_tokens,
|
num_tokens=num_tokens,
|
||||||
@@ -301,12 +301,10 @@ class CudaGraphManager:
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
|
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
|
||||||
returns a forward_fn. For FULL cudagraph mode, it is invoked
|
returns a forward_fn. For FULL and breakable PIECEWISE modes,
|
||||||
once with warmup=True for the warmup pass, and again with
|
it is invoked once with warmup=True and again with warmup=False
|
||||||
warmup=False for the captured pass. For attention backends
|
because attention backends may mutate or lazily initialize
|
||||||
that perform lazy metadata initialization (e.g. FlashMLA),
|
metadata during warmup.
|
||||||
FULL cudagraph capture requires distinct metadatas for warmup
|
|
||||||
and capture.
|
|
||||||
"""
|
"""
|
||||||
with graph_capture(device=self.device):
|
with graph_capture(device=self.device):
|
||||||
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
|
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
|
||||||
@@ -330,11 +328,17 @@ class CudaGraphManager:
|
|||||||
logger.debug(
|
logger.debug(
|
||||||
"CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc
|
"CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc
|
||||||
)
|
)
|
||||||
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
if (
|
||||||
|
desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||||
|
and not self.use_breakable_cg
|
||||||
|
):
|
||||||
forward_fn(CUDAGraphMode.PIECEWISE)
|
forward_fn(CUDAGraphMode.PIECEWISE)
|
||||||
else:
|
else:
|
||||||
# Capture with fresh attention state.
|
# Capture with fresh attention state.
|
||||||
forward_fn = create_forward_fn(desc, warmup=False)
|
forward_fn = create_forward_fn(desc, warmup=False)
|
||||||
|
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
||||||
|
forward_fn(CUDAGraphMode.PIECEWISE)
|
||||||
|
continue
|
||||||
assert desc not in self.graphs, (
|
assert desc not in self.graphs, (
|
||||||
f"Graph already captured for {desc}"
|
f"Graph already captured for {desc}"
|
||||||
)
|
)
|
||||||
@@ -489,7 +493,10 @@ class ModelCudaGraphManager(CudaGraphManager):
|
|||||||
block_tables,
|
block_tables,
|
||||||
attn_groups,
|
attn_groups,
|
||||||
kv_cache_config,
|
kv_cache_config,
|
||||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
skip_attn=(
|
||||||
|
desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||||
|
and not self.use_breakable_cg
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Capture with dummy rows marked as padding.
|
# Capture with dummy rows marked as padding.
|
||||||
@@ -498,7 +505,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
|||||||
def forward_fn(cg_mode: CUDAGraphMode) -> None:
|
def forward_fn(cg_mode: CUDAGraphMode) -> None:
|
||||||
batch_descriptor = None
|
batch_descriptor = None
|
||||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||||
assert attn_metadata is None
|
assert (attn_metadata is not None) == self.use_breakable_cg
|
||||||
batch_descriptor = BatchDescriptor(
|
batch_descriptor = BatchDescriptor(
|
||||||
num_tokens=num_tokens,
|
num_tokens=num_tokens,
|
||||||
has_lora=has_lora,
|
has_lora=has_lora,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from typing import Any
|
|||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
|
|
||||||
|
from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.compilation import CUDAGraphMode
|
from vllm.config.compilation import CUDAGraphMode
|
||||||
from vllm.v1.core.sched.output import NewRequestData
|
from vllm.v1.core.sched.output import NewRequestData
|
||||||
@@ -139,7 +140,10 @@ class DefaultModelState(ModelState):
|
|||||||
kv_cache_config: KVCacheConfig,
|
kv_cache_config: KVCacheConfig,
|
||||||
for_capture: bool = False,
|
for_capture: bool = False,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
if cudagraph_mode == CUDAGraphMode.FULL or (
|
||||||
|
cudagraph_mode == CUDAGraphMode.PIECEWISE
|
||||||
|
and is_breakable_cudagraph_enabled()
|
||||||
|
):
|
||||||
# Use padded sizes - padding is handled by model_runner.prepare_attn.
|
# Use padded sizes - padding is handled by model_runner.prepare_attn.
|
||||||
num_reqs = input_batch.num_reqs_after_padding
|
num_reqs = input_batch.num_reqs_after_padding
|
||||||
num_tokens = input_batch.num_tokens_after_padding
|
num_tokens = input_batch.num_tokens_after_padding
|
||||||
|
|||||||
@@ -56,7 +56,10 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
|
|||||||
block_tables,
|
block_tables,
|
||||||
attn_groups,
|
attn_groups,
|
||||||
kv_cache_config,
|
kv_cache_config,
|
||||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
skip_attn=(
|
||||||
|
desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||||
|
and not self.use_breakable_cg
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
return lambda cg_mode: forward_fn(
|
return lambda cg_mode: forward_fn(
|
||||||
|
|||||||
Reference in New Issue
Block a user