From 51918e252af4dfff381001f42b7ebe6a0454501b Mon Sep 17 00:00:00 2001 From: Jared Wen Date: Tue, 15 Sep 2026 14:51:35 +0800 Subject: [PATCH] [Bugfix][SparseMLA] Fix piecewise cudagraph capture crash in index group (#56825) Signed-off-by: Jared Wen Co-authored-by: Kimi Code --- .../v1/attention/test_sparse_mla_backends.py | 78 ++++++++++++++++++- vllm/forward_context.py | 9 +++ vllm/v1/attention/backends/mla/index_group.py | 15 +++- vllm/v1/hisparse/runtime.py | 4 +- 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 2ca839d59c..6d2c4fbd58 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -22,10 +22,17 @@ from tests.v1.attention.utils import ( create_vllm_config, ) from vllm import _custom_ops as ops -from vllm.config import HiSparseConfig, SpeculativeConfig, set_current_vllm_config +from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture +from vllm.config import ( + CUDAGraphMode, + HiSparseConfig, + SpeculativeConfig, + set_current_vllm_config, +) from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import ( HiSparseConnectorWorker, ) +from vllm.forward_context import set_forward_context from vllm.model_executor.layers.attention import mla_attention from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha from vllm.model_executor.layers.attention.sparse_mla_attention import ( @@ -897,6 +904,75 @@ def test_triton_convert_req_index_to_global_index_decode_only( torch.testing.assert_close(result, reference_result, rtol=0, atol=0) +def test_index_group_convert_during_piecewise_capture(): + """In piecewise cudagraph mode the indexer is captured while the convert + runs in the following eager break. The convert's side stream must not + wait on ``logical_topk_ready`` there: an event recorded inside a captured + segment is graph-local, and an eager wait on it raises + cudaErrorInvalidValue.""" + device = torch.device(DEVICE_TYPE) + num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16 + + logical_topk_indices = torch.randint( + 0, + block_size * blocks_per_req, + (num_tokens, num_topk), + dtype=torch.int32, + device=device, + ) + builder = SparseMLAIndexGroupBuilder(logical_topk_indices) + group, layer_index = builder.register_layer(is_index_producing_layer=True) + assert layer_index == 0 and group.has_indexer + + req_id = torch.arange(num_tokens, dtype=torch.int32, device=device) % num_requests + block_table = torch.arange( + num_requests * blocks_per_req, dtype=torch.int32, device=device + ).view(num_requests, blocks_per_req) + attn_metadata = SimpleNamespace( + req_id_per_token=req_id, block_table=block_table, block_size=block_size + ) + expected = triton_convert_req_index_to_global_index( + req_id, + block_table, + logical_topk_indices, + BLOCK_SIZE=block_size, + NUM_TOPK_TOKENS=num_topk, + ) + + vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B") + marker = torch.zeros(1, device=device) + capture_stream = torch.cuda.Stream(device=device) + # torch.cuda.stream routes through vLLM's patched torch.cuda.set_stream, + # so current_stream() tracks the capture stream as in production. + with ( + torch.cuda.stream(capture_stream), + set_forward_context( + None, vllm_config, cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE + ), + ): + capture = BreakableCUDAGraphCapture() + with capture: + marker.add_(1) + # Record inside a captured segment, convert in the eager + # break — mirrors mla.py -> unified_mla_attention_with_output. + group.set_logical_topk_ready(layer_index) + capture.add_eager( + lambda: group.convert_logical_to_physical_topk( + layer_index, + logical_topk_indices, + attn_metadata, + block_stride_rows=None, + return_valid_counts=False, + ) + ) + marker.add_(1) + capture.replay() + capture_stream.synchronize() + + result = group.physical_topk_indices[:num_tokens] + torch.testing.assert_close(result, expected, rtol=0, atol=0) + + @pytest.mark.parametrize("block_size", [16]) @pytest.mark.skipif( torch.cuda.get_device_capability() < (9, 0), diff --git a/vllm/forward_context.py b/vllm/forward_context.py index f57fc2c950..8d9eb5d7db 100644 --- a/vllm/forward_context.py +++ b/vllm/forward_context.py @@ -209,6 +209,15 @@ def is_forward_context_available() -> bool: return _forward_context is not None +def in_piecewise_cudagraph() -> bool: + """Whether the current forward runs in piecewise cudagraph mode (graph + segments separated by eager breaks), at capture or replay time.""" + return ( + is_forward_context_available() + and get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE + ) + + def create_forward_context( attn_metadata: Any, vllm_config: VllmConfig, diff --git a/vllm/v1/attention/backends/mla/index_group.py b/vllm/v1/attention/backends/mla/index_group.py index 8ba0171835..1b8455a232 100644 --- a/vllm/v1/attention/backends/mla/index_group.py +++ b/vllm/v1/attention/backends/mla/index_group.py @@ -10,6 +10,7 @@ import torch from vllm import _custom_ops as ops from vllm.config import VllmConfig +from vllm.forward_context import in_piecewise_cudagraph from vllm.utils.torch_utils import current_stream from vllm.v1.attention.backends.mla.sparse_utils import ( triton_convert_req_index_to_global_index, @@ -56,8 +57,16 @@ class SparseMLAIndexGroup: return layer_index def set_logical_topk_ready(self, layer_index: int) -> None: - if layer_index == 0 and self.has_indexer: - self.logical_topk_ready.record(current_stream()) + if layer_index != 0 or not self.has_indexer: + return + # In piecewise cudagraph mode the consumer converts on the side stream + # from an eager break; an event recorded inside a captured segment is + # graph-local and any eager wait on it raises cudaErrorInvalidValue. + # The convert side falls back to wait_stream there instead (and never + # touches this event), so skip the record as well. + if in_piecewise_cudagraph(): + return + self.logical_topk_ready.record(current_stream()) def prepare_for_batch(self, layer_index: int, attn_metadata: Any | None) -> None: pass @@ -129,7 +138,7 @@ class SparseMLAIndexGroup: valid_topk_counts = self.valid_topk_counts[:num_tokens] compute_stream = current_stream() if layer_index == 0: - if self.has_indexer: + if self.has_indexer and not in_piecewise_cudagraph(): self.side_stream.wait_event(self.logical_topk_ready) else: self.side_stream.wait_stream(compute_stream) diff --git a/vllm/v1/hisparse/runtime.py b/vllm/v1/hisparse/runtime.py index 314d3ba86b..5100379083 100644 --- a/vllm/v1/hisparse/runtime.py +++ b/vllm/v1/hisparse/runtime.py @@ -16,7 +16,7 @@ import torch from vllm.config import CUDAGraphMode, VllmConfig from vllm.distributed import get_tp_group -from vllm.forward_context import get_forward_context +from vllm.forward_context import get_forward_context, in_piecewise_cudagraph from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.math_utils import round_up @@ -975,7 +975,7 @@ class HiSparseRuntime: ) -> None: group = self.index_group compute_stream = current_stream() - if group.logical_topk_ready is not None: + if group.logical_topk_ready is not None and not in_piecewise_cudagraph(): group.copy_stream.wait_event(group.logical_topk_ready) else: group.copy_stream.wait_stream(compute_stream)