[Model] Add PW CUDA graph support for Inkling [2/N] (#48822)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
ce65385618
commit
251f7e478e
@@ -7,6 +7,7 @@ from typing import cast
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -286,6 +287,7 @@ class InklingAttention(nn.Module, AttentionLayerBase):
|
||||
output, _ = self.wo_ud(flat)
|
||||
return output
|
||||
|
||||
@eager_break_during_capture
|
||||
def _attention(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
|
||||
@@ -21,8 +21,8 @@ spec alike.
|
||||
All kernels address the cache purely by ``(slot, absolute_position)`` and
|
||||
allocate nothing inside the captured region; their grids depend only on the
|
||||
token count (``fused_sconv`` on a fixed token/channel tiling), so the same
|
||||
decode path replays correctly under a full CUDA graph without any
|
||||
data-dependent shape or branch.
|
||||
path replays correctly under eager, breakable PIECEWISE, and FULL cudagraphs
|
||||
without any data-dependent shape or branch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -167,8 +167,8 @@ def fused_sconv(
|
||||
"""Single-launch insert + depthwise causal conv1d over the paged cache.
|
||||
|
||||
Reads same-forward taps from ``x`` and pre-forward taps from the cache, so
|
||||
it is race-free in one launch for prefill / decode / spec and supports full
|
||||
CUDA-graph capture for decode.
|
||||
it is race-free in one launch for prefill / decode / spec and cudagraph-safe
|
||||
under eager / piecewise / full capture.
|
||||
"""
|
||||
T = x.shape[0]
|
||||
out = torch.empty_like(x)
|
||||
|
||||
@@ -49,9 +49,7 @@ class InklingSconvMetadata(AttentionMetadata):
|
||||
|
||||
|
||||
class InklingSconvMetadataBuilder(AttentionMetadataBuilder[InklingSconvMetadata]):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = (
|
||||
AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -17,8 +17,8 @@ Per-forward metadata (``block_table`` / ``slot_mapping`` / ``seq_idx`` /
|
||||
the owner's prefix in the forward context; the absolute ``positions`` are
|
||||
threaded in from the model. The insert + conv run in a single ``fused_sconv``
|
||||
launch (same path for prefill / decode / mixed / spec). All inputs are
|
||||
fixed-address persistent buffers and the grid is fixed, so decode can replay
|
||||
under a full CUDA graph.
|
||||
fixed-address persistent buffers and the grid is fixed, so the conv replays
|
||||
correctly under eager, PIECEWISE, and FULL cudagraphs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
Reference in New Issue
Block a user