diff --git a/tests/v1/attention/test_flashinfer_mla_dcp.py b/tests/v1/attention/test_flashinfer_mla_dcp.py index 039d30c9aa..a72dbc21d1 100644 --- a/tests/v1/attention/test_flashinfer_mla_dcp.py +++ b/tests/v1/attention/test_flashinfer_mla_dcp.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import MethodType from unittest.mock import MagicMock import pytest @@ -61,6 +62,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch): attn_metadata.max_seq_len = 1 attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32) attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32) + attn_metadata.decode.max_query_len = 1 query = torch.ones(2, 24, 576, dtype=torch.bfloat16) kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16) @@ -100,6 +102,9 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal): impl.dcp_world_size = 8 impl.dcp_rank = 3 impl.cp_kv_cache_interleave_size = 1 + impl._flattened_decode_metadata = MethodType( + flashinfer_mla.FlashInferMLAImpl._flattened_decode_metadata, impl + ) num_reqs, query_len = 2, 3 num_tokens = num_reqs * query_len @@ -114,6 +119,11 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal): attn_metadata.decode.block_table = block_table attn_metadata.decode.seq_lens = seq_lens attn_metadata.decode.dcp_tot_seq_lens = global_causal_seq_lens + attn_metadata.decode.max_query_len = query_len + attn_metadata.decode.query_start_loc = torch.arange( + 0, num_tokens + 1, query_len, dtype=torch.int32 + ) + attn_metadata.decode.query_len = 0 query = torch.ones(num_tokens, 24, 576, dtype=torch.bfloat16) kv_cache = torch.ones(2, 128, 576, dtype=torch.bfloat16) diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index 597ffb03fc..2a9cd7736b 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -1267,6 +1267,7 @@ def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads( query_start_loc_cpu=query_start_loc, query_start_loc_device=query_start_loc, num_decode_tokens=2, + max_query_len=1, dcp_tot_seq_lens_device=None, ) diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index b8ba46d8de..2d8a5c2063 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -278,6 +278,7 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch): query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32), query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32), num_decode_tokens=2 * qlen, + max_query_len=qlen, dcp_tot_seq_lens_device=torch.tensor([10, 12], dtype=torch.int32), ) @@ -633,6 +634,7 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch): query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32), query_start_loc_device=torch.tensor([0, 4, 8], dtype=torch.int32), num_decode_tokens=8, + max_query_len=4, dcp_tot_seq_lens_device=None, ) @@ -683,6 +685,7 @@ def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch): query_start_loc_cpu=query_start_loc, query_start_loc_device=query_start_loc, num_decode_tokens=num_reqs * mtp_qlen, + max_query_len=mtp_qlen, dcp_tot_seq_lens_device=None, ) @@ -733,6 +736,7 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr( query_start_loc_cpu=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32), query_start_loc_device=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32), num_decode_tokens=seq_lens.numel() * mtp_qlen, + max_query_len=mtp_qlen, dcp_tot_seq_lens_device=None, ) @@ -801,6 +805,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch): query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.int32), query_start_loc_device=torch.tensor([0, 1, 2], dtype=torch.int32), num_decode_tokens=seq_lens.numel(), + max_query_len=1, dcp_tot_seq_lens_device=None, ) @@ -892,6 +897,7 @@ def test_persistent_metadata_gate( query_start_loc_cpu=query_start_loc, query_start_loc_device=query_start_loc, num_decode_tokens=num_reqs * qo_len, + max_query_len=qo_len, dcp_tot_seq_lens_device=None, ) @@ -961,6 +967,7 @@ def test_persistent_metadata_gate_without_gluon_build( query_start_loc_cpu=query_start_loc, query_start_loc_device=query_start_loc, num_decode_tokens=num_reqs * qo_len, + max_query_len=qo_len, dcp_tot_seq_lens_device=None, ) @@ -993,6 +1000,7 @@ def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen) query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32), query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32), num_decode_tokens=2 * qlen, + max_query_len=qlen, dcp_tot_seq_lens_device=None, ) return metadata, get_mla_metadata_v1 diff --git a/tests/v1/worker/test_mamba_hybrid_model_state.py b/tests/v1/worker/test_mamba_hybrid_model_state.py index f32c72dd6c..c65679059a 100644 --- a/tests/v1/worker/test_mamba_hybrid_model_state.py +++ b/tests/v1/worker/test_mamba_hybrid_model_state.py @@ -34,6 +34,7 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(), query_start_loc=torch.tensor([0, 1], dtype=torch.int32), num_scheduled_tokens=torch.tensor([1], dtype=torch.int32), + max_query_len=None, seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32), seq_lens=torch.tensor([1537], dtype=torch.int32), is_prefilling_np=torch.tensor([False]).numpy(), diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 1b8fa13a51..7defc96b36 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -2471,6 +2471,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> MLACommonDecodeMetadata: return MLACommonDecodeMetadata( @@ -2635,6 +2636,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): query_start_loc_cpu=query_start_loc_cpu[: num_decodes + 1], query_start_loc_device=query_start_loc[: num_decodes + 1], num_decode_tokens=num_decode_tokens, + max_query_len=min( + common_attn_metadata.max_query_len, self.reorder_batch_threshold + ), dcp_tot_seq_lens_device=dcp_tot_seq_lens_device, ) diff --git a/vllm/models/dots3_note/nvidia/attention.py b/vllm/models/dots3_note/nvidia/attention.py index 27928b6e6f..561a3bfd5c 100644 --- a/vllm/models/dots3_note/nvidia/attention.py +++ b/vllm/models/dots3_note/nvidia/attention.py @@ -353,9 +353,10 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder): query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> Dots3NoteDecodeMetadata: - del max_seq_len, query_start_loc_device, num_decode_tokens + del max_seq_len, query_start_loc_device, num_decode_tokens, max_query_len query_len = int(query_start_loc_cpu[1] - query_start_loc_cpu[0]) return Dots3NoteDecodeMetadata( block_table=block_table_tensor, diff --git a/vllm/models/kimi_k3/nvidia/dspark_mla.py b/vllm/models/kimi_k3/nvidia/dspark_mla.py index 87f9e94d9b..0baff884cc 100644 --- a/vllm/models/kimi_k3/nvidia/dspark_mla.py +++ b/vllm/models/kimi_k3/nvidia/dspark_mla.py @@ -15,7 +15,10 @@ from vllm.model_executor.layers.linear import ( ReplicatedLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead +from vllm.model_executor.models.qwen3_dspark import ( + DSparkConfidenceHead, + DSparkMarkovHead, +) from vllm.model_executor.models.utils import ( AutoWeightsLoader, WeightsMapper, @@ -188,6 +191,18 @@ class K3DSparkModel(nn.Module): self.config.markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) + self.confidence_head: DSparkConfidenceHead | None = None + if getattr(self.config, "enable_confidence_head", False): + with_markov = getattr(self.config, "confidence_head_with_markov", False) + input_dim = self.config.hidden_size + ( + self.config.markov_rank if with_markov else 0 + ) + self.confidence_head = DSparkConfidenceHead( + input_dim, + prefix=maybe_prefix(prefix, "confidence_head"), + bias=True, + with_markov=with_markov, + ) self._max_num_context_tokens = ( vllm_config.scheduler_config.max_num_batched_tokens ) @@ -436,6 +451,12 @@ class K3DSparkForCausalLM(nn.Module): assert vllm_config.speculative_config is not None self.draft_model_config = vllm_config.speculative_config.draft_model_config self.config = self.draft_model_config.hf_config + if getattr(self.config, "enable_confidence_head", False): + # The class mapper drops the head as training-only; keep it for drafts + # that actually built one, since adaptive verification reads it. + self.hf_to_vllm_mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={"confidence_head": "confidence_head"} + ) target_layer_num = vllm_config.model_config.get_total_num_hidden_layers() self.model = K3DSparkModel( vllm_config=vllm_config, @@ -494,6 +515,13 @@ class K3DSparkForCausalLM(nn.Module): def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor: return self.model.markov_head.bias(markov_embed, self.logits_processor) + def compute_confidence( + self, head_hidden: torch.Tensor, markov_embed: torch.Tensor + ) -> torch.Tensor: + """Per-position acceptance probability for each drafted token.""" + assert self.model.confidence_head is not None + return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed)) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) # read: 1. all weights. 2. context kv weights diff --git a/vllm/models/kimi_k3/nvidia/kda_metadata.py b/vllm/models/kimi_k3/nvidia/kda_metadata.py index d1c2b31921..2562171109 100644 --- a/vllm/models/kimi_k3/nvidia/kda_metadata.py +++ b/vllm/models/kimi_k3/nvidia/kda_metadata.py @@ -25,7 +25,7 @@ from vllm.model_executor.layers.mamba.checkpoint import ( from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import async_tensor_h2d -from vllm.v1.attention.backend import CommonAttentionMetadata +from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata from vllm.v1.attention.backends.gdn_attn import ( GDNAttentionBackend, GDNAttentionMetadata, @@ -305,6 +305,10 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata): class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder): + # Overrides GDN's UNIFORM_BATCH: adaptive verification requires ALWAYS from every + # builder, and KDA reads per-request offsets off device within a fixed k+1 window, + # so one k+1 graph replays any 1..k+1 mix. + _cudagraph_support = AttentionCGSupport.ALWAYS mamba_aligned_state_indices: torch.Tensor | None = None def __init__( @@ -407,6 +411,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder): query_start_loc_cpu.diff() > 0 ) spec_sequence_masks_cpu |= active_decode_mask_cpu + # Adaptive's cost-table profiler probes eager dummy rows longer than + # RecoverSSM's num_spec+1 activation capacity; route them out instead + # of tripping the capacity check below. + spec_sequence_masks_cpu &= ( + query_start_loc_cpu.diff() <= self.num_spec + 1 + ) # Native KDA can use its regular decode path when no draft token # was scheduled. RecoverSSM must preserve its extended conv window. if ( @@ -750,3 +760,9 @@ class KimiK3KDAAttentionBackend(GDNAttentionBackend): @staticmethod def get_builder_cls() -> type[KimiK3KDAMetadataBuilder]: return KimiK3KDAMetadataBuilder + + @classmethod + def supports_device_cpu_query_lens_mismatch(cls) -> bool: + # Pure spec-decode reads its plan from DEVICE offsets, so CPU/device + # mismatch is fine; the mixed branch still assumes CPU totals match. + return True diff --git a/vllm/v1/attention/backends/mla/flashattn_mla.py b/vllm/v1/attention/backends/mla/flashattn_mla.py index 084a1f4ce6..c3729fd06d 100644 --- a/vllm/v1/attention/backends/mla/flashattn_mla.py +++ b/vllm/v1/attention/backends/mla/flashattn_mla.py @@ -194,8 +194,11 @@ class FlashAttnMLAMetadataBuilder(MLACommonMetadataBuilder[FlashAttnMLAMetadata] query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> FlashAttnMLADecodeMetadata: + # Measured, not the promised bound: _schedule_decode asserts metadata size + # against a persistent buffer. query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] max_query_len = query_lens_cpu.max().item() diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla.py b/vllm/v1/attention/backends/mla/flashinfer_mla.py index 61414bba49..fea49ca154 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar import torch @@ -15,6 +16,7 @@ from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import ( MLACommonBackend, + MLACommonDecodeMetadata, MLACommonImpl, MLACommonMetadata, MLACommonMetadataBuilder, @@ -115,9 +117,28 @@ def _get_multi_ctas_kv_counter_buffer( return _fi_multi_ctas_kv_counter -class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): - _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH - query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM +@dataclass +class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata): + flattened_block_table: torch.Tensor | None = None + flattened_seq_lens: torch.Tensor | None = None + flattened_row_req: torch.Tensor | None = None + # Row count the flattened tensors were built for (0 = not built), not a query len. + query_len: int = 0 + query_start_loc: torch.Tensor | None = None + max_query_len: int = 1 + + +@dataclass +class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]): + pass + + +class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]): + # Adaptive verification requires ALWAYS from every builder, matching upstream's + # DeepseekV4FlashMLAMetadataBuilder. The kernels tile ragged queries from the device + # query offsets (flashinfer #3238), so one k+1 graph replays any 1..k+1 mix. + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS + query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.VARLEN # Non-causal DSpark blocks are flattened to single-token rows in forward_mqa. supports_non_causal_multi_token_decode: ClassVar[bool] = True supports_non_causal_multi_token_dcp: ClassVar[bool] = True @@ -143,10 +164,31 @@ class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): layer_names, vllm_config, device, - MLACommonMetadata, + FlashInferMLAMetadata, supports_dcp_with_varlen=True, ) + def _build_decode( + self, + block_table_tensor: torch.Tensor, + seq_lens_device: torch.Tensor, + max_seq_len: int, + query_start_loc_cpu: torch.Tensor, + query_start_loc_device: torch.Tensor, + num_decode_tokens: int, + max_query_len: int, + dcp_tot_seq_lens_device: torch.Tensor | None, + ) -> FlashInferMLADecodeMetadata: + # Promised bound, not measured: a capture dummy puts one token/req, so + # measuring would bake an undersized graph. Ragged only when max_query_len>1. + return FlashInferMLADecodeMetadata( + block_table=block_table_tensor, + seq_lens=seq_lens_device, + dcp_tot_seq_lens=dcp_tot_seq_lens_device, + query_start_loc=query_start_loc_device, + max_query_len=max_query_len, + ) + class FlashInferMLABackend(MLACommonBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] @@ -208,7 +250,7 @@ class FlashInferMLABackend(MLACommonBackend): return None -class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): +class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]): can_return_lse_for_decode: bool = True supports_dcp: bool = True # DCP is the only path that consumes LSE. It uses monolithic CuTeDSL, @@ -276,7 +318,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): self, q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], kv_c_and_k_pe_cache: torch.Tensor, - attn_metadata: MLACommonMetadata, + attn_metadata: FlashInferMLAMetadata, layer: AttentionLayer, ) -> tuple[torch.Tensor, torch.Tensor | None]: assert kv_c_and_k_pe_cache.numel() > 0 @@ -289,14 +331,32 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): block_table = attn_metadata.decode.block_table seq_lens = attn_metadata.decode.seq_lens + # Led by the promised bound (baked into the graph), not + # num_decode_tokens // num_decodes -- that average is a per-request length + # only for uniform batches. + multi_token_decode = ( + attn_metadata.decode.max_query_len > 1 + or attn_metadata.num_decode_tokens > attn_metadata.num_decodes + ) + cum_seq_lens_q: torch.Tensor | None = None + max_q_len: int | None = None + row_req: torch.Tensor | None = None + if not attn_metadata.causal: # FlashInfer decode has no causal flag. Flatten each non-causal # query block into independent single-token rows. - query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes q = q.unsqueeze(1) - if query_len > 1: - block_table = block_table.repeat_interleave(query_len, dim=0) - seq_lens = seq_lens.repeat_interleave(query_len) + if multi_token_decode: + block_table, seq_lens, row_req = self._flattened_decode_metadata( + attn_metadata, q.shape[0] + ) + elif attn_metadata.decode.max_query_len > 1 and self.dcp_world_size == 1: + # Causal spec decode: keep q compact and let the kernel tile each + # request's length (uniform 1+k and adaptive ragged). Mirrors vllm #52157. + # DCP keeps the uniform reshape below: it needs LSE, which the kernels + # do not return on the ragged path (flashinfer #3238). + cum_seq_lens_q = attn_metadata.decode.query_start_loc + max_q_len = attn_metadata.decode.max_query_len # trtllm API requires extra dimension q_len_per_request for MTP elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0: logger.warning_once( @@ -327,10 +387,8 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): if self.dcp_world_size > 1: causal_seqlens_kv_global = attn_metadata.decode.dcp_tot_seq_lens assert causal_seqlens_kv_global is not None - if not attn_metadata.causal and query_len > 1: - causal_seqlens_kv_global = causal_seqlens_kv_global.repeat_interleave( - query_len - ) + if row_req is not None: + causal_seqlens_kv_global = causal_seqlens_kv_global[row_req] extra_kwargs.update( enable_dcp=True, cp_world=self.dcp_world_size, @@ -342,6 +400,16 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): # trtllm-gen rejects MLA head counts it can't tile (e.g. 96); # fall back to cute-dsl for those. decode_backend = _select_mla_decode_backend(runtime_num_heads) + if cum_seq_lens_q is not None: + # Neither decode backend returns LSE on the ragged path + # (flashinfer #3238); DCP, the only LSE consumer, took the uniform + # branch above, so this only guards a future caller wiring the two. + assert not return_lse, ( + "FlashInferMLA ragged decode cannot return LSE; DCP and adaptive " + "variable-length decode are mutually exclusive." + ) + extra_kwargs["cum_seq_lens_q"] = cum_seq_lens_q + extra_kwargs["max_q_len"] = max_q_len if decode_backend: extra_kwargs["backend"] = decode_backend elif kv_c_and_k_pe_cache.shape[-2] in (32, 64): @@ -386,3 +454,37 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): o = o.view(-1, o.shape[-2], o.shape[-1]) return o, lse + + def _flattened_decode_metadata( + self, + attn_metadata: FlashInferMLAMetadata, + num_rows: int, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Expand per-request decode tensors to one row per query token. + + Cached across the layers of a group, which all see the same batch. + """ + decode = attn_metadata.decode + assert decode is not None + if decode.query_len != num_rows: + cu = decode.query_start_loc + assert cu is not None + # searchsorted on the device offsets, not repeat_interleave(uniform_len): + # the latter only lines up for uniform batches and hands ragged rows + # another request's KV (silent garbage). + rows = torch.arange(num_rows, device=cu.device, dtype=cu.dtype) + row_req = torch.searchsorted(cu[1:], rows, right=True).clamp_( + max=decode.block_table.shape[0] - 1 + ) + decode.flattened_row_req = row_req + decode.flattened_block_table = decode.block_table[row_req] + decode.flattened_seq_lens = decode.seq_lens[row_req] + decode.query_len = num_rows + assert decode.flattened_block_table is not None + assert decode.flattened_seq_lens is not None + assert decode.flattened_row_req is not None + return ( + decode.flattened_block_table, + decode.flattened_seq_lens, + decode.flattened_row_req, + ) diff --git a/vllm/v1/attention/backends/mla/flashmla.py b/vllm/v1/attention/backends/mla/flashmla.py index 8ebfbbba79..d17ae05c1a 100644 --- a/vllm/v1/attention/backends/mla/flashmla.py +++ b/vllm/v1/attention/backends/mla/flashmla.py @@ -158,6 +158,7 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]): query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> FlashMLADecodeMetadata: query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 75173e59a3..bad6ed201b 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -1007,6 +1007,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): query_start_loc_cpu: torch.Tensor, query_start_loc_device: torch.Tensor, num_decode_tokens: int, + max_query_len: int, dcp_tot_seq_lens_device: torch.Tensor | None, ) -> AiterMLADecodeMetadata: causal = self._decode_causal diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index bfad8a1b73..d079991907 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -250,7 +250,11 @@ class MambaHybridModelState(DefaultModelState): num_reqs = input_batch.num_reqs num_tokens = input_batch.num_tokens query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) - max_query_len = input_batch.num_scheduled_tokens.max().item() + # Prefer the promised bound: a capture dummy's measured max is its even + # split, not the length the graph must replay. + max_query_len = input_batch.max_query_len + if max_query_len is None: + max_query_len = input_batch.num_scheduled_tokens.max().item() seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound if for_capture: # Capture with worst-case max_seq_len so the graph is valid at any replay. @@ -278,10 +282,11 @@ class MambaHybridModelState(DefaultModelState): num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32) num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req if num_draft_tokens_per_req is not None: - # A row is a spec-decode row only when its whole prompt is already - # computed, i.e. exactly one non-draft (decode) token is scheduled. - is_decode = ( - input_batch.num_scheduled_tokens == num_draft_tokens_per_req + 1 + # Test request state, not num_scheduled_tokens == draft_count+1: + # adaptive rewrites num_scheduled_tokens to an even split, so that + # equality rarely holds and would demote every verify row to decode. + is_decode = (~input_batch.is_prefilling_np) & ( + input_batch.num_scheduled_tokens > 0 ) spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(