[Model] Add PW CUDA graph support for Inkling [2/N] (#48822)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-16 10:28:56 -07:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent ce65385618
commit 251f7e478e
10 changed files with 98 additions and 32 deletions
+23 -16
View File
@@ -138,8 +138,6 @@ class CudaGraphManager:
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
self._init_candidates()
# Breakable CUDA graph (PW CUDA graph without torch.compile)
self.use_breakable_cg = (
is_breakable_cudagraph_enabled()
@@ -147,6 +145,8 @@ class CudaGraphManager:
)
self.breakable_cg_runner: BreakableCUDAGraphWrapper | None = None
self._init_candidates()
def _build_lora_dispatch_map(self) -> tuple[dict[int, int], int]:
"""Precompute actual num_active_loras -> effective captured case.
@@ -255,11 +255,11 @@ class CudaGraphManager:
# for PIECEWISE graphs there is no limit on requests when replaying
# i.e. no request padding is needed
# so we leave it as None
num_reqs = (
min(num_tokens, self.max_num_reqs)
if mixed_mode == CUDAGraphMode.FULL
else None
)
num_reqs = None
if mixed_mode == CUDAGraphMode.FULL or (
mixed_mode == CUDAGraphMode.PIECEWISE and self.use_breakable_cg
):
num_reqs = min(num_tokens, self.max_num_reqs)
desc = BatchExecutionDescriptor(
cg_mode=mixed_mode,
num_tokens=num_tokens,
@@ -301,12 +301,10 @@ class CudaGraphManager:
Args:
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
returns a forward_fn. For FULL cudagraph mode, it is invoked
once with warmup=True for the warmup pass, and again with
warmup=False for the captured pass. For attention backends
that perform lazy metadata initialization (e.g. FlashMLA),
FULL cudagraph capture requires distinct metadatas for warmup
and capture.
returns a forward_fn. For FULL and breakable PIECEWISE modes,
it is invoked once with warmup=True and again with warmup=False
because attention backends may mutate or lazily initialize
metadata during warmup.
"""
with graph_capture(device=self.device):
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
@@ -330,11 +328,17 @@ class CudaGraphManager:
logger.debug(
"CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc
)
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
if (
desc.cg_mode == CUDAGraphMode.PIECEWISE
and not self.use_breakable_cg
):
forward_fn(CUDAGraphMode.PIECEWISE)
else:
# Capture with fresh attention state.
forward_fn = create_forward_fn(desc, warmup=False)
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
forward_fn(CUDAGraphMode.PIECEWISE)
continue
assert desc not in self.graphs, (
f"Graph already captured for {desc}"
)
@@ -489,7 +493,10 @@ class ModelCudaGraphManager(CudaGraphManager):
block_tables,
attn_groups,
kv_cache_config,
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
skip_attn=(
desc.cg_mode == CUDAGraphMode.PIECEWISE
and not self.use_breakable_cg
),
)
# Capture with dummy rows marked as padding.
@@ -498,7 +505,7 @@ class ModelCudaGraphManager(CudaGraphManager):
def forward_fn(cg_mode: CUDAGraphMode) -> None:
batch_descriptor = None
if cg_mode == CUDAGraphMode.PIECEWISE:
assert attn_metadata is None
assert (attn_metadata is not None) == self.use_breakable_cg
batch_descriptor = BatchDescriptor(
num_tokens=num_tokens,
has_lora=has_lora,