[Model] Add PW CUDA graph support for Inkling [2/N] (#48822)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
ce65385618
commit
251f7e478e
@@ -138,8 +138,6 @@ class CudaGraphManager:
|
||||
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
|
||||
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
||||
|
||||
self._init_candidates()
|
||||
|
||||
# Breakable CUDA graph (PW CUDA graph without torch.compile)
|
||||
self.use_breakable_cg = (
|
||||
is_breakable_cudagraph_enabled()
|
||||
@@ -147,6 +145,8 @@ class CudaGraphManager:
|
||||
)
|
||||
self.breakable_cg_runner: BreakableCUDAGraphWrapper | None = None
|
||||
|
||||
self._init_candidates()
|
||||
|
||||
def _build_lora_dispatch_map(self) -> tuple[dict[int, int], int]:
|
||||
"""Precompute actual num_active_loras -> effective captured case.
|
||||
|
||||
@@ -255,11 +255,11 @@ class CudaGraphManager:
|
||||
# for PIECEWISE graphs there is no limit on requests when replaying
|
||||
# i.e. no request padding is needed
|
||||
# so we leave it as None
|
||||
num_reqs = (
|
||||
min(num_tokens, self.max_num_reqs)
|
||||
if mixed_mode == CUDAGraphMode.FULL
|
||||
else None
|
||||
)
|
||||
num_reqs = None
|
||||
if mixed_mode == CUDAGraphMode.FULL or (
|
||||
mixed_mode == CUDAGraphMode.PIECEWISE and self.use_breakable_cg
|
||||
):
|
||||
num_reqs = min(num_tokens, self.max_num_reqs)
|
||||
desc = BatchExecutionDescriptor(
|
||||
cg_mode=mixed_mode,
|
||||
num_tokens=num_tokens,
|
||||
@@ -301,12 +301,10 @@ class CudaGraphManager:
|
||||
|
||||
Args:
|
||||
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
|
||||
returns a forward_fn. For FULL cudagraph mode, it is invoked
|
||||
once with warmup=True for the warmup pass, and again with
|
||||
warmup=False for the captured pass. For attention backends
|
||||
that perform lazy metadata initialization (e.g. FlashMLA),
|
||||
FULL cudagraph capture requires distinct metadatas for warmup
|
||||
and capture.
|
||||
returns a forward_fn. For FULL and breakable PIECEWISE modes,
|
||||
it is invoked once with warmup=True and again with warmup=False
|
||||
because attention backends may mutate or lazily initialize
|
||||
metadata during warmup.
|
||||
"""
|
||||
with graph_capture(device=self.device):
|
||||
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
|
||||
@@ -330,11 +328,17 @@ class CudaGraphManager:
|
||||
logger.debug(
|
||||
"CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc
|
||||
)
|
||||
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
if (
|
||||
desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
and not self.use_breakable_cg
|
||||
):
|
||||
forward_fn(CUDAGraphMode.PIECEWISE)
|
||||
else:
|
||||
# Capture with fresh attention state.
|
||||
forward_fn = create_forward_fn(desc, warmup=False)
|
||||
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
forward_fn(CUDAGraphMode.PIECEWISE)
|
||||
continue
|
||||
assert desc not in self.graphs, (
|
||||
f"Graph already captured for {desc}"
|
||||
)
|
||||
@@ -489,7 +493,10 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
block_tables,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||
skip_attn=(
|
||||
desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
and not self.use_breakable_cg
|
||||
),
|
||||
)
|
||||
|
||||
# Capture with dummy rows marked as padding.
|
||||
@@ -498,7 +505,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
def forward_fn(cg_mode: CUDAGraphMode) -> None:
|
||||
batch_descriptor = None
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
assert attn_metadata is None
|
||||
assert (attn_metadata is not None) == self.use_breakable_cg
|
||||
batch_descriptor = BatchDescriptor(
|
||||
num_tokens=num_tokens,
|
||||
has_lora=has_lora,
|
||||
|
||||
Reference in New Issue
Block a user