[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)

Signed-off-by: Albert Cheng <albecheng@nvidia.com>
Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Albert Cheng
2026-09-22 20:43:32 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Benjamin Chislett
parent 355d789f0d
commit 88aa0d287d
13 changed files with 203 additions and 22 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import MethodType
from unittest.mock import MagicMock
import pytest
@@ -61,6 +62,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
attn_metadata.max_seq_len = 1
attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32)
attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32)
attn_metadata.decode.max_query_len = 1
query = torch.ones(2, 24, 576, dtype=torch.bfloat16)
kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16)
@@ -100,6 +102,9 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal):
impl.dcp_world_size = 8
impl.dcp_rank = 3
impl.cp_kv_cache_interleave_size = 1
impl._flattened_decode_metadata = MethodType(
flashinfer_mla.FlashInferMLAImpl._flattened_decode_metadata, impl
)
num_reqs, query_len = 2, 3
num_tokens = num_reqs * query_len
@@ -114,6 +119,11 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal):
attn_metadata.decode.block_table = block_table
attn_metadata.decode.seq_lens = seq_lens
attn_metadata.decode.dcp_tot_seq_lens = global_causal_seq_lens
attn_metadata.decode.max_query_len = query_len
attn_metadata.decode.query_start_loc = torch.arange(
0, num_tokens + 1, query_len, dtype=torch.int32
)
attn_metadata.decode.query_len = 0
query = torch.ones(num_tokens, 24, 576, dtype=torch.bfloat16)
kv_cache = torch.ones(2, 128, 576, dtype=torch.bfloat16)
+1
View File
@@ -1267,6 +1267,7 @@ def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads(
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=2,
max_query_len=1,
dcp_tot_seq_lens_device=None,
)
@@ -278,6 +278,7 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
num_decode_tokens=2 * qlen,
max_query_len=qlen,
dcp_tot_seq_lens_device=torch.tensor([10, 12], dtype=torch.int32),
)
@@ -633,6 +634,7 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, 4, 8], dtype=torch.int32),
num_decode_tokens=8,
max_query_len=4,
dcp_tot_seq_lens_device=None,
)
@@ -683,6 +685,7 @@ def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch):
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * mtp_qlen,
max_query_len=mtp_qlen,
dcp_tot_seq_lens_device=None,
)
@@ -733,6 +736,7 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
query_start_loc_cpu=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32),
num_decode_tokens=seq_lens.numel() * mtp_qlen,
max_query_len=mtp_qlen,
dcp_tot_seq_lens_device=None,
)
@@ -801,6 +805,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch):
query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, 1, 2], dtype=torch.int32),
num_decode_tokens=seq_lens.numel(),
max_query_len=1,
dcp_tot_seq_lens_device=None,
)
@@ -892,6 +897,7 @@ def test_persistent_metadata_gate(
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * qo_len,
max_query_len=qo_len,
dcp_tot_seq_lens_device=None,
)
@@ -961,6 +967,7 @@ def test_persistent_metadata_gate_without_gluon_build(
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * qo_len,
max_query_len=qo_len,
dcp_tot_seq_lens_device=None,
)
@@ -993,6 +1000,7 @@ def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen)
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
num_decode_tokens=2 * qlen,
max_query_len=qlen,
dcp_tot_seq_lens_device=None,
)
return metadata, get_mla_metadata_v1
@@ -34,6 +34,7 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
max_query_len=None,
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
seq_lens=torch.tensor([1537], dtype=torch.int32),
is_prefilling_np=torch.tensor([False]).numpy(),
@@ -2471,6 +2471,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
max_query_len: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> MLACommonDecodeMetadata:
return MLACommonDecodeMetadata(
@@ -2635,6 +2636,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu=query_start_loc_cpu[: num_decodes + 1],
query_start_loc_device=query_start_loc[: num_decodes + 1],
num_decode_tokens=num_decode_tokens,
max_query_len=min(
common_attn_metadata.max_query_len, self.reorder_batch_threshold
),
dcp_tot_seq_lens_device=dcp_tot_seq_lens_device,
)
+2 -1
View File
@@ -353,9 +353,10 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder):
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
max_query_len: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> Dots3NoteDecodeMetadata:
del max_seq_len, query_start_loc_device, num_decode_tokens
del max_seq_len, query_start_loc_device, num_decode_tokens, max_query_len
query_len = int(query_start_loc_cpu[1] - query_start_loc_cpu[0])
return Dots3NoteDecodeMetadata(
block_table=block_table_tensor,
+29 -1
View File
@@ -15,7 +15,10 @@ from vllm.model_executor.layers.linear import (
ReplicatedLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -188,6 +191,18 @@ class K3DSparkModel(nn.Module):
self.config.markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(self.config, "enable_confidence_head", False):
with_markov = getattr(self.config, "confidence_head_with_markov", False)
input_dim = self.config.hidden_size + (
self.config.markov_rank if with_markov else 0
)
self.confidence_head = DSparkConfidenceHead(
input_dim,
prefix=maybe_prefix(prefix, "confidence_head"),
bias=True,
with_markov=with_markov,
)
self._max_num_context_tokens = (
vllm_config.scheduler_config.max_num_batched_tokens
)
@@ -436,6 +451,12 @@ class K3DSparkForCausalLM(nn.Module):
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
if getattr(self.config, "enable_confidence_head", False):
# The class mapper drops the head as training-only; keep it for drafts
# that actually built one, since adaptive verification reads it.
self.hf_to_vllm_mapper = self.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_substr={"confidence_head": "confidence_head"}
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = K3DSparkModel(
vllm_config=vllm_config,
@@ -494,6 +515,13 @@ class K3DSparkForCausalLM(nn.Module):
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
def compute_confidence(
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
) -> torch.Tensor:
"""Per-position acceptance probability for each drafted token."""
assert self.model.confidence_head is not None
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
# read: 1. all weights. 2. context kv weights
+17 -1
View File
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.mamba.checkpoint import (
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
@@ -305,6 +305,10 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
# Overrides GDN's UNIFORM_BATCH: adaptive verification requires ALWAYS from every
# builder, and KDA reads per-request offsets off device within a fixed k+1 window,
# so one k+1 graph replays any 1..k+1 mix.
_cudagraph_support = AttentionCGSupport.ALWAYS
mamba_aligned_state_indices: torch.Tensor | None = None
def __init__(
@@ -407,6 +411,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
query_start_loc_cpu.diff() > 0
)
spec_sequence_masks_cpu |= active_decode_mask_cpu
# Adaptive's cost-table profiler probes eager dummy rows longer than
# RecoverSSM's num_spec+1 activation capacity; route them out instead
# of tripping the capacity check below.
spec_sequence_masks_cpu &= (
query_start_loc_cpu.diff() <= self.num_spec + 1
)
# Native KDA can use its regular decode path when no draft token
# was scheduled. RecoverSSM must preserve its extended conv window.
if (
@@ -750,3 +760,9 @@ class KimiK3KDAAttentionBackend(GDNAttentionBackend):
@staticmethod
def get_builder_cls() -> type[KimiK3KDAMetadataBuilder]:
return KimiK3KDAMetadataBuilder
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
# Pure spec-decode reads its plan from DEVICE offsets, so CPU/device
# mismatch is fine; the mixed branch still assumes CPU totals match.
return True
@@ -194,8 +194,11 @@ class FlashAttnMLAMetadataBuilder(MLACommonMetadataBuilder[FlashAttnMLAMetadata]
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
max_query_len: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> FlashAttnMLADecodeMetadata:
# Measured, not the promised bound: _schedule_decode asserts metadata size
# against a persistent buffer.
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
max_query_len = query_lens_cpu.max().item()
+116 -14
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, ClassVar
import torch
@@ -15,6 +16,7 @@ from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBackend,
MLACommonDecodeMetadata,
MLACommonImpl,
MLACommonMetadata,
MLACommonMetadataBuilder,
@@ -115,9 +117,28 @@ def _get_multi_ctas_kv_counter_buffer(
return _fi_multi_ctas_kv_counter
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
@dataclass
class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata):
flattened_block_table: torch.Tensor | None = None
flattened_seq_lens: torch.Tensor | None = None
flattened_row_req: torch.Tensor | None = None
# Row count the flattened tensors were built for (0 = not built), not a query len.
query_len: int = 0
query_start_loc: torch.Tensor | None = None
max_query_len: int = 1
@dataclass
class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]):
pass
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]):
# Adaptive verification requires ALWAYS from every builder, matching upstream's
# DeepseekV4FlashMLAMetadataBuilder. The kernels tile ragged queries from the device
# query offsets (flashinfer #3238), so one k+1 graph replays any 1..k+1 mix.
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.VARLEN
# Non-causal DSpark blocks are flattened to single-token rows in forward_mqa.
supports_non_causal_multi_token_decode: ClassVar[bool] = True
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
@@ -143,10 +164,31 @@ class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
layer_names,
vllm_config,
device,
MLACommonMetadata,
FlashInferMLAMetadata,
supports_dcp_with_varlen=True,
)
def _build_decode(
self,
block_table_tensor: torch.Tensor,
seq_lens_device: torch.Tensor,
max_seq_len: int,
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
max_query_len: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> FlashInferMLADecodeMetadata:
# Promised bound, not measured: a capture dummy puts one token/req, so
# measuring would bake an undersized graph. Ragged only when max_query_len>1.
return FlashInferMLADecodeMetadata(
block_table=block_table_tensor,
seq_lens=seq_lens_device,
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
query_start_loc=query_start_loc_device,
max_query_len=max_query_len,
)
class FlashInferMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
@@ -208,7 +250,7 @@ class FlashInferMLABackend(MLACommonBackend):
return None
class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
can_return_lse_for_decode: bool = True
supports_dcp: bool = True
# DCP is the only path that consumes LSE. It uses monolithic CuTeDSL,
@@ -276,7 +318,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: MLACommonMetadata,
attn_metadata: FlashInferMLAMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert kv_c_and_k_pe_cache.numel() > 0
@@ -289,14 +331,32 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
block_table = attn_metadata.decode.block_table
seq_lens = attn_metadata.decode.seq_lens
# Led by the promised bound (baked into the graph), not
# num_decode_tokens // num_decodes -- that average is a per-request length
# only for uniform batches.
multi_token_decode = (
attn_metadata.decode.max_query_len > 1
or attn_metadata.num_decode_tokens > attn_metadata.num_decodes
)
cum_seq_lens_q: torch.Tensor | None = None
max_q_len: int | None = None
row_req: torch.Tensor | None = None
if not attn_metadata.causal:
# FlashInfer decode has no causal flag. Flatten each non-causal
# query block into independent single-token rows.
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
q = q.unsqueeze(1)
if query_len > 1:
block_table = block_table.repeat_interleave(query_len, dim=0)
seq_lens = seq_lens.repeat_interleave(query_len)
if multi_token_decode:
block_table, seq_lens, row_req = self._flattened_decode_metadata(
attn_metadata, q.shape[0]
)
elif attn_metadata.decode.max_query_len > 1 and self.dcp_world_size == 1:
# Causal spec decode: keep q compact and let the kernel tile each
# request's length (uniform 1+k and adaptive ragged). Mirrors vllm #52157.
# DCP keeps the uniform reshape below: it needs LSE, which the kernels
# do not return on the ragged path (flashinfer #3238).
cum_seq_lens_q = attn_metadata.decode.query_start_loc
max_q_len = attn_metadata.decode.max_query_len
# trtllm API requires extra dimension q_len_per_request for MTP
elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0:
logger.warning_once(
@@ -327,10 +387,8 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
if self.dcp_world_size > 1:
causal_seqlens_kv_global = attn_metadata.decode.dcp_tot_seq_lens
assert causal_seqlens_kv_global is not None
if not attn_metadata.causal and query_len > 1:
causal_seqlens_kv_global = causal_seqlens_kv_global.repeat_interleave(
query_len
)
if row_req is not None:
causal_seqlens_kv_global = causal_seqlens_kv_global[row_req]
extra_kwargs.update(
enable_dcp=True,
cp_world=self.dcp_world_size,
@@ -342,6 +400,16 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
# trtllm-gen rejects MLA head counts it can't tile (e.g. 96);
# fall back to cute-dsl for those.
decode_backend = _select_mla_decode_backend(runtime_num_heads)
if cum_seq_lens_q is not None:
# Neither decode backend returns LSE on the ragged path
# (flashinfer #3238); DCP, the only LSE consumer, took the uniform
# branch above, so this only guards a future caller wiring the two.
assert not return_lse, (
"FlashInferMLA ragged decode cannot return LSE; DCP and adaptive "
"variable-length decode are mutually exclusive."
)
extra_kwargs["cum_seq_lens_q"] = cum_seq_lens_q
extra_kwargs["max_q_len"] = max_q_len
if decode_backend:
extra_kwargs["backend"] = decode_backend
elif kv_c_and_k_pe_cache.shape[-2] in (32, 64):
@@ -386,3 +454,37 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
o = o.view(-1, o.shape[-2], o.shape[-1])
return o, lse
def _flattened_decode_metadata(
self,
attn_metadata: FlashInferMLAMetadata,
num_rows: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Expand per-request decode tensors to one row per query token.
Cached across the layers of a group, which all see the same batch.
"""
decode = attn_metadata.decode
assert decode is not None
if decode.query_len != num_rows:
cu = decode.query_start_loc
assert cu is not None
# searchsorted on the device offsets, not repeat_interleave(uniform_len):
# the latter only lines up for uniform batches and hands ragged rows
# another request's KV (silent garbage).
rows = torch.arange(num_rows, device=cu.device, dtype=cu.dtype)
row_req = torch.searchsorted(cu[1:], rows, right=True).clamp_(
max=decode.block_table.shape[0] - 1
)
decode.flattened_row_req = row_req
decode.flattened_block_table = decode.block_table[row_req]
decode.flattened_seq_lens = decode.seq_lens[row_req]
decode.query_len = num_rows
assert decode.flattened_block_table is not None
assert decode.flattened_seq_lens is not None
assert decode.flattened_row_req is not None
return (
decode.flattened_block_table,
decode.flattened_seq_lens,
decode.flattened_row_req,
)
@@ -158,6 +158,7 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]):
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
max_query_len: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> FlashMLADecodeMetadata:
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
@@ -1007,6 +1007,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
max_query_len: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> AiterMLADecodeMetadata:
causal = self._decode_causal
@@ -250,7 +250,11 @@ class MambaHybridModelState(DefaultModelState):
num_reqs = input_batch.num_reqs
num_tokens = input_batch.num_tokens
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
max_query_len = input_batch.num_scheduled_tokens.max().item()
# Prefer the promised bound: a capture dummy's measured max is its even
# split, not the length the graph must replay.
max_query_len = input_batch.max_query_len
if max_query_len is None:
max_query_len = input_batch.num_scheduled_tokens.max().item()
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound
if for_capture:
# Capture with worst-case max_seq_len so the graph is valid at any replay.
@@ -278,10 +282,11 @@ class MambaHybridModelState(DefaultModelState):
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req
if num_draft_tokens_per_req is not None:
# A row is a spec-decode row only when its whole prompt is already
# computed, i.e. exactly one non-draft (decode) token is scheduled.
is_decode = (
input_batch.num_scheduled_tokens == num_draft_tokens_per_req + 1
# Test request state, not num_scheduled_tokens == draft_count+1:
# adaptive rewrites num_scheduled_tokens to an even split, so that
# equality rarely holds and would demote every verify row to decode.
is_decode = (~input_batch.is_prefilling_np) & (
input_batch.num_scheduled_tokens > 0
)
spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(