[Bugfix][SparseMLA] Fix piecewise cudagraph capture crash in index group (#56825)

Signed-off-by: Jared Wen <jaredwen@inferact.ai>
Co-authored-by: Kimi Code <noreply@moonshot.ai>
This commit is contained in:
Jared Wen
2026-09-15 06:51:35 +00:00
committed by GitHub
co-authored by Kimi Code
parent 79e205e8cd
commit 51918e252a
4 changed files with 100 additions and 6 deletions
+9
View File
@@ -209,6 +209,15 @@ def is_forward_context_available() -> bool:
return _forward_context is not None
def in_piecewise_cudagraph() -> bool:
"""Whether the current forward runs in piecewise cudagraph mode (graph
segments separated by eager breaks), at capture or replay time."""
return (
is_forward_context_available()
and get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE
)
def create_forward_context(
attn_metadata: Any,
vllm_config: VllmConfig,
+12 -3
View File
@@ -10,6 +10,7 @@ import torch
from vllm import _custom_ops as ops
from vllm.config import VllmConfig
from vllm.forward_context import in_piecewise_cudagraph
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backends.mla.sparse_utils import (
triton_convert_req_index_to_global_index,
@@ -56,8 +57,16 @@ class SparseMLAIndexGroup:
return layer_index
def set_logical_topk_ready(self, layer_index: int) -> None:
if layer_index == 0 and self.has_indexer:
self.logical_topk_ready.record(current_stream())
if layer_index != 0 or not self.has_indexer:
return
# In piecewise cudagraph mode the consumer converts on the side stream
# from an eager break; an event recorded inside a captured segment is
# graph-local and any eager wait on it raises cudaErrorInvalidValue.
# The convert side falls back to wait_stream there instead (and never
# touches this event), so skip the record as well.
if in_piecewise_cudagraph():
return
self.logical_topk_ready.record(current_stream())
def prepare_for_batch(self, layer_index: int, attn_metadata: Any | None) -> None:
pass
@@ -129,7 +138,7 @@ class SparseMLAIndexGroup:
valid_topk_counts = self.valid_topk_counts[:num_tokens]
compute_stream = current_stream()
if layer_index == 0:
if self.has_indexer:
if self.has_indexer and not in_piecewise_cudagraph():
self.side_stream.wait_event(self.logical_topk_ready)
else:
self.side_stream.wait_stream(compute_stream)
+2 -2
View File
@@ -16,7 +16,7 @@ import torch
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.forward_context import get_forward_context, in_piecewise_cudagraph
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
@@ -975,7 +975,7 @@ class HiSparseRuntime:
) -> None:
group = self.index_group
compute_stream = current_stream()
if group.logical_topk_ready is not None:
if group.logical_topk_ready is not None and not in_piecewise_cudagraph():
group.copy_stream.wait_event(group.logical_topk_ready)
else:
group.copy_stream.wait_stream(compute_stream)