diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index 83fceef018..9c92ae554e 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -7,6 +7,9 @@ import pytest import torch from tests.v1.attention.utils import create_vllm_config +from vllm.model_executor.layers.attention.sparse_mla_attention import ( + SparseMLACommonMetadataBuilder, +) from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend, @@ -95,8 +98,11 @@ def test_fused_indexer_decode_metadata(query_lens, padding): @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") -@pytest.mark.parametrize("query_lens", [[1], [0, 257, 1, 0, 3], [6] * 64, [2, 6, 4, 0]]) -def test_device_token_request_mapping(query_lens): +@pytest.mark.parametrize( + "query_lens", [[1], [0, 257, 1, 0, 3], [6] * 64, [2, 6, 4, 0], [0, 0, 0]] +) +@pytest.mark.parametrize("use_sparse_mla_builder", [False, True]) +def test_device_token_request_mapping(query_lens, use_sparse_mla_builder): """Graph replay follows device boundaries even when CPU lengths are stale.""" lengths = torch.tensor(query_lens, device="cuda", dtype=torch.int32) qsl = torch.cat( @@ -117,7 +123,16 @@ def test_device_token_request_mapping(query_lens): ), slot_mapping=torch.full((n + 7,), -1, device="cuda", dtype=torch.int64), ) - result = common.token_to_req_indices(output) + + def build_mapping(): + if use_sparse_mla_builder: + builder = SimpleNamespace(req_id_per_token_buffer=output) + return SparseMLACommonMetadataBuilder._build_req_id_per_token( + builder, common + ) + return common.token_to_req_indices(output) + + result = build_mapping() assert result.data_ptr() == output.data_ptr() expected = torch.repeat_interleave( torch.arange(len(query_lens), device="cuda", dtype=torch.int32), lengths @@ -127,7 +142,7 @@ def test_device_token_request_mapping(query_lens): graph = torch.cuda.CUDAGraph() common._token_to_req_indices_cache = None with torch.cuda.graph(graph): - common.token_to_req_indices(output) + build_mapping() reversed_lens = lengths.flip(0) qsl[1:].copy_(reversed_lens.cumsum(0)) graph.replay() diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index aed5afdcf9..ee7409b190 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -291,17 +291,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): self, common_attn_metadata: "CommonAttentionMetadata", ) -> torch.Tensor: - num_tokens = common_attn_metadata.num_actual_tokens - starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32) - seg_lengths = np.diff(starts) - req_id_per_token = np.repeat( - np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths - ) - self.req_id_per_token_buffer.fill_(0) - self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_( - np_to_pinned_tensor(req_id_per_token), non_blocking=True - ) - return self.req_id_per_token_buffer[:num_tokens] + return common_attn_metadata.token_to_req_indices(self.req_id_per_token_buffer) def _build_chunked_context_fields( self, diff --git a/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py b/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py index 6048f6b5fd..0c3f0f4288 100644 --- a/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py @@ -178,6 +178,11 @@ class FlashAttnMLASparseImpl(SparseMLACommonImpl[FlashAttnMLASparseMetadata]): assert self.topk_indices_buffer is not None, ( "Indexer or topk_indices_buffer required for sparse MLA" ) + self.cu_seqlens_q_buffer = torch.arange( + self.topk_indices_buffer.shape[0] + 1, + dtype=torch.int32, + device=self.topk_indices_buffer.device, + ) self.supports_quant_query_input = False def forward_mqa( @@ -306,9 +311,7 @@ class FlashAttnMLASparseImpl(SparseMLACommonImpl[FlashAttnMLASparseMetadata]): kv_cache if cache_is_flat else flat_kv_row_view(kv_cache, block_size)[0] ) - cu_seqlens_q = torch.arange( - 0, q_rope.shape[0] + 1, dtype=torch.int32, device=q_rope.device - ) + cu_seqlens_q = self.cu_seqlens_q_buffer[: q_rope.shape[0] + 1] v_cache = kv_rows[:, : self.kv_lora_rank].unsqueeze(1).unsqueeze(1) if self.qk_rope_head_dim == 0: # FA3's QV path requires the 64-wide Q/K specialization. For NoPE diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index 32f67e7e93..5af50ef1bd 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -38,7 +38,6 @@ from vllm.v1.kv_cache_interface import AttentionSpec if TYPE_CHECKING: from vllm.model_executor.models.deepseek_v2 import Indexer - from vllm.v1.attention.backend import CommonAttentionMetadata logger = init_logger(__name__) @@ -301,12 +300,6 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde ), ) - def _build_req_id_per_token( - self, - common_attn_metadata: "CommonAttentionMetadata", - ) -> torch.Tensor: - return common_attn_metadata.token_to_req_indices(self.req_id_per_token_buffer) - # Global workspace buffer (lazily initialized) _fi_sparse_workspace: torch.Tensor | None = None