[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)
Signed-off-by: Albert Cheng <albecheng@nvidia.com> Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Benjamin Chislett
parent
355d789f0d
commit
88aa0d287d
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import MethodType
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -61,6 +62,7 @@ def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
|
||||
attn_metadata.max_seq_len = 1
|
||||
attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32)
|
||||
attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32)
|
||||
attn_metadata.decode.max_query_len = 1
|
||||
|
||||
query = torch.ones(2, 24, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16)
|
||||
@@ -100,6 +102,9 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal):
|
||||
impl.dcp_world_size = 8
|
||||
impl.dcp_rank = 3
|
||||
impl.cp_kv_cache_interleave_size = 1
|
||||
impl._flattened_decode_metadata = MethodType(
|
||||
flashinfer_mla.FlashInferMLAImpl._flattened_decode_metadata, impl
|
||||
)
|
||||
|
||||
num_reqs, query_len = 2, 3
|
||||
num_tokens = num_reqs * query_len
|
||||
@@ -114,6 +119,11 @@ def test_flashinfer_mla_forward_uses_native_dcp_api(monkeypatch, causal):
|
||||
attn_metadata.decode.block_table = block_table
|
||||
attn_metadata.decode.seq_lens = seq_lens
|
||||
attn_metadata.decode.dcp_tot_seq_lens = global_causal_seq_lens
|
||||
attn_metadata.decode.max_query_len = query_len
|
||||
attn_metadata.decode.query_start_loc = torch.arange(
|
||||
0, num_tokens + 1, query_len, dtype=torch.int32
|
||||
)
|
||||
attn_metadata.decode.query_len = 0
|
||||
|
||||
query = torch.ones(num_tokens, 24, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.ones(2, 128, 576, dtype=torch.bfloat16)
|
||||
|
||||
@@ -1267,6 +1267,7 @@ def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads(
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=2,
|
||||
max_query_len=1,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -278,6 +278,7 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
|
||||
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
num_decode_tokens=2 * qlen,
|
||||
max_query_len=qlen,
|
||||
dcp_tot_seq_lens_device=torch.tensor([10, 12], dtype=torch.int32),
|
||||
)
|
||||
|
||||
@@ -633,6 +634,7 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
|
||||
query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, 4, 8], dtype=torch.int32),
|
||||
num_decode_tokens=8,
|
||||
max_query_len=4,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -683,6 +685,7 @@ def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch):
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * mtp_qlen,
|
||||
max_query_len=mtp_qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -733,6 +736,7 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
|
||||
query_start_loc_cpu=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, mtp_qlen, mtp_qlen], dtype=torch.int32),
|
||||
num_decode_tokens=seq_lens.numel() * mtp_qlen,
|
||||
max_query_len=mtp_qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -801,6 +805,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch):
|
||||
query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
num_decode_tokens=seq_lens.numel(),
|
||||
max_query_len=1,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -892,6 +897,7 @@ def test_persistent_metadata_gate(
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * qo_len,
|
||||
max_query_len=qo_len,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -961,6 +967,7 @@ def test_persistent_metadata_gate_without_gluon_build(
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * qo_len,
|
||||
max_query_len=qo_len,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
@@ -993,6 +1000,7 @@ def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen)
|
||||
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
num_decode_tokens=2 * qlen,
|
||||
max_query_len=qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
return metadata, get_mla_metadata_v1
|
||||
|
||||
@@ -34,6 +34,7 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non
|
||||
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
|
||||
max_query_len=None,
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1537], dtype=torch.int32),
|
||||
is_prefilling_np=torch.tensor([False]).numpy(),
|
||||
|
||||
@@ -2471,6 +2471,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> MLACommonDecodeMetadata:
|
||||
return MLACommonDecodeMetadata(
|
||||
@@ -2635,6 +2636,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu=query_start_loc_cpu[: num_decodes + 1],
|
||||
query_start_loc_device=query_start_loc[: num_decodes + 1],
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
max_query_len=min(
|
||||
common_attn_metadata.max_query_len, self.reorder_batch_threshold
|
||||
),
|
||||
dcp_tot_seq_lens_device=dcp_tot_seq_lens_device,
|
||||
)
|
||||
|
||||
|
||||
@@ -353,9 +353,10 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder):
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> Dots3NoteDecodeMetadata:
|
||||
del max_seq_len, query_start_loc_device, num_decode_tokens
|
||||
del max_seq_len, query_start_loc_device, num_decode_tokens, max_query_len
|
||||
query_len = int(query_start_loc_cpu[1] - query_start_loc_cpu[0])
|
||||
return Dots3NoteDecodeMetadata(
|
||||
block_table=block_table_tensor,
|
||||
|
||||
@@ -15,7 +15,10 @@ from vllm.model_executor.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -188,6 +191,18 @@ class K3DSparkModel(nn.Module):
|
||||
self.config.markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(self.config, "enable_confidence_head", False):
|
||||
with_markov = getattr(self.config, "confidence_head_with_markov", False)
|
||||
input_dim = self.config.hidden_size + (
|
||||
self.config.markov_rank if with_markov else 0
|
||||
)
|
||||
self.confidence_head = DSparkConfidenceHead(
|
||||
input_dim,
|
||||
prefix=maybe_prefix(prefix, "confidence_head"),
|
||||
bias=True,
|
||||
with_markov=with_markov,
|
||||
)
|
||||
self._max_num_context_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
@@ -436,6 +451,12 @@ class K3DSparkForCausalLM(nn.Module):
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
if getattr(self.config, "enable_confidence_head", False):
|
||||
# The class mapper drops the head as training-only; keep it for drafts
|
||||
# that actually built one, since adaptive verification reads it.
|
||||
self.hf_to_vllm_mapper = self.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_substr={"confidence_head": "confidence_head"}
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
self.model = K3DSparkModel(
|
||||
vllm_config=vllm_config,
|
||||
@@ -494,6 +515,13 @@ class K3DSparkForCausalLM(nn.Module):
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def compute_confidence(
|
||||
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Per-position acceptance probability for each drafted token."""
|
||||
assert self.model.confidence_head is not None
|
||||
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
# read: 1. all weights. 2. context kv weights
|
||||
|
||||
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionBackend,
|
||||
GDNAttentionMetadata,
|
||||
@@ -305,6 +305,10 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||
|
||||
|
||||
class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
# Overrides GDN's UNIFORM_BATCH: adaptive verification requires ALWAYS from every
|
||||
# builder, and KDA reads per-request offsets off device within a fixed k+1 window,
|
||||
# so one k+1 graph replays any 1..k+1 mix.
|
||||
_cudagraph_support = AttentionCGSupport.ALWAYS
|
||||
mamba_aligned_state_indices: torch.Tensor | None = None
|
||||
|
||||
def __init__(
|
||||
@@ -407,6 +411,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
query_start_loc_cpu.diff() > 0
|
||||
)
|
||||
spec_sequence_masks_cpu |= active_decode_mask_cpu
|
||||
# Adaptive's cost-table profiler probes eager dummy rows longer than
|
||||
# RecoverSSM's num_spec+1 activation capacity; route them out instead
|
||||
# of tripping the capacity check below.
|
||||
spec_sequence_masks_cpu &= (
|
||||
query_start_loc_cpu.diff() <= self.num_spec + 1
|
||||
)
|
||||
# Native KDA can use its regular decode path when no draft token
|
||||
# was scheduled. RecoverSSM must preserve its extended conv window.
|
||||
if (
|
||||
@@ -750,3 +760,9 @@ class KimiK3KDAAttentionBackend(GDNAttentionBackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[KimiK3KDAMetadataBuilder]:
|
||||
return KimiK3KDAMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||
# Pure spec-decode reads its plan from DEVICE offsets, so CPU/device
|
||||
# mismatch is fine; the mixed branch still assumes CPU totals match.
|
||||
return True
|
||||
|
||||
@@ -194,8 +194,11 @@ class FlashAttnMLAMetadataBuilder(MLACommonMetadataBuilder[FlashAttnMLAMetadata]
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> FlashAttnMLADecodeMetadata:
|
||||
# Measured, not the promised bound: _schedule_decode asserts metadata size
|
||||
# against a persistent buffer.
|
||||
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
max_query_len = query_lens_cpu.max().item()
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import torch
|
||||
@@ -15,6 +16,7 @@ from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBackend,
|
||||
MLACommonDecodeMetadata,
|
||||
MLACommonImpl,
|
||||
MLACommonMetadata,
|
||||
MLACommonMetadataBuilder,
|
||||
@@ -115,9 +117,28 @@ def _get_multi_ctas_kv_counter_buffer(
|
||||
return _fi_multi_ctas_kv_counter
|
||||
|
||||
|
||||
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
|
||||
@dataclass
|
||||
class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
flattened_block_table: torch.Tensor | None = None
|
||||
flattened_seq_lens: torch.Tensor | None = None
|
||||
flattened_row_req: torch.Tensor | None = None
|
||||
# Row count the flattened tensors were built for (0 = not built), not a query len.
|
||||
query_len: int = 0
|
||||
query_start_loc: torch.Tensor | None = None
|
||||
max_query_len: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]):
|
||||
pass
|
||||
|
||||
|
||||
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]):
|
||||
# Adaptive verification requires ALWAYS from every builder, matching upstream's
|
||||
# DeepseekV4FlashMLAMetadataBuilder. The kernels tile ragged queries from the device
|
||||
# query offsets (flashinfer #3238), so one k+1 graph replays any 1..k+1 mix.
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.VARLEN
|
||||
# Non-causal DSpark blocks are flattened to single-token rows in forward_mqa.
|
||||
supports_non_causal_multi_token_decode: ClassVar[bool] = True
|
||||
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
|
||||
@@ -143,10 +164,31 @@ class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
layer_names,
|
||||
vllm_config,
|
||||
device,
|
||||
MLACommonMetadata,
|
||||
FlashInferMLAMetadata,
|
||||
supports_dcp_with_varlen=True,
|
||||
)
|
||||
|
||||
def _build_decode(
|
||||
self,
|
||||
block_table_tensor: torch.Tensor,
|
||||
seq_lens_device: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> FlashInferMLADecodeMetadata:
|
||||
# Promised bound, not measured: a capture dummy puts one token/req, so
|
||||
# measuring would bake an undersized graph. Ragged only when max_query_len>1.
|
||||
return FlashInferMLADecodeMetadata(
|
||||
block_table=block_table_tensor,
|
||||
seq_lens=seq_lens_device,
|
||||
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
|
||||
query_start_loc=query_start_loc_device,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
|
||||
|
||||
class FlashInferMLABackend(MLACommonBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
@@ -208,7 +250,7 @@ class FlashInferMLABackend(MLACommonBackend):
|
||||
return None
|
||||
|
||||
|
||||
class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
# DCP is the only path that consumes LSE. It uses monolithic CuTeDSL,
|
||||
@@ -276,7 +318,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: MLACommonMetadata,
|
||||
attn_metadata: FlashInferMLAMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert kv_c_and_k_pe_cache.numel() > 0
|
||||
@@ -289,14 +331,32 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
block_table = attn_metadata.decode.block_table
|
||||
seq_lens = attn_metadata.decode.seq_lens
|
||||
|
||||
# Led by the promised bound (baked into the graph), not
|
||||
# num_decode_tokens // num_decodes -- that average is a per-request length
|
||||
# only for uniform batches.
|
||||
multi_token_decode = (
|
||||
attn_metadata.decode.max_query_len > 1
|
||||
or attn_metadata.num_decode_tokens > attn_metadata.num_decodes
|
||||
)
|
||||
cum_seq_lens_q: torch.Tensor | None = None
|
||||
max_q_len: int | None = None
|
||||
row_req: torch.Tensor | None = None
|
||||
|
||||
if not attn_metadata.causal:
|
||||
# FlashInfer decode has no causal flag. Flatten each non-causal
|
||||
# query block into independent single-token rows.
|
||||
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
|
||||
q = q.unsqueeze(1)
|
||||
if query_len > 1:
|
||||
block_table = block_table.repeat_interleave(query_len, dim=0)
|
||||
seq_lens = seq_lens.repeat_interleave(query_len)
|
||||
if multi_token_decode:
|
||||
block_table, seq_lens, row_req = self._flattened_decode_metadata(
|
||||
attn_metadata, q.shape[0]
|
||||
)
|
||||
elif attn_metadata.decode.max_query_len > 1 and self.dcp_world_size == 1:
|
||||
# Causal spec decode: keep q compact and let the kernel tile each
|
||||
# request's length (uniform 1+k and adaptive ragged). Mirrors vllm #52157.
|
||||
# DCP keeps the uniform reshape below: it needs LSE, which the kernels
|
||||
# do not return on the ragged path (flashinfer #3238).
|
||||
cum_seq_lens_q = attn_metadata.decode.query_start_loc
|
||||
max_q_len = attn_metadata.decode.max_query_len
|
||||
# trtllm API requires extra dimension q_len_per_request for MTP
|
||||
elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0:
|
||||
logger.warning_once(
|
||||
@@ -327,10 +387,8 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
if self.dcp_world_size > 1:
|
||||
causal_seqlens_kv_global = attn_metadata.decode.dcp_tot_seq_lens
|
||||
assert causal_seqlens_kv_global is not None
|
||||
if not attn_metadata.causal and query_len > 1:
|
||||
causal_seqlens_kv_global = causal_seqlens_kv_global.repeat_interleave(
|
||||
query_len
|
||||
)
|
||||
if row_req is not None:
|
||||
causal_seqlens_kv_global = causal_seqlens_kv_global[row_req]
|
||||
extra_kwargs.update(
|
||||
enable_dcp=True,
|
||||
cp_world=self.dcp_world_size,
|
||||
@@ -342,6 +400,16 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
# trtllm-gen rejects MLA head counts it can't tile (e.g. 96);
|
||||
# fall back to cute-dsl for those.
|
||||
decode_backend = _select_mla_decode_backend(runtime_num_heads)
|
||||
if cum_seq_lens_q is not None:
|
||||
# Neither decode backend returns LSE on the ragged path
|
||||
# (flashinfer #3238); DCP, the only LSE consumer, took the uniform
|
||||
# branch above, so this only guards a future caller wiring the two.
|
||||
assert not return_lse, (
|
||||
"FlashInferMLA ragged decode cannot return LSE; DCP and adaptive "
|
||||
"variable-length decode are mutually exclusive."
|
||||
)
|
||||
extra_kwargs["cum_seq_lens_q"] = cum_seq_lens_q
|
||||
extra_kwargs["max_q_len"] = max_q_len
|
||||
if decode_backend:
|
||||
extra_kwargs["backend"] = decode_backend
|
||||
elif kv_c_and_k_pe_cache.shape[-2] in (32, 64):
|
||||
@@ -386,3 +454,37 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
o = o.view(-1, o.shape[-2], o.shape[-1])
|
||||
|
||||
return o, lse
|
||||
|
||||
def _flattened_decode_metadata(
|
||||
self,
|
||||
attn_metadata: FlashInferMLAMetadata,
|
||||
num_rows: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Expand per-request decode tensors to one row per query token.
|
||||
|
||||
Cached across the layers of a group, which all see the same batch.
|
||||
"""
|
||||
decode = attn_metadata.decode
|
||||
assert decode is not None
|
||||
if decode.query_len != num_rows:
|
||||
cu = decode.query_start_loc
|
||||
assert cu is not None
|
||||
# searchsorted on the device offsets, not repeat_interleave(uniform_len):
|
||||
# the latter only lines up for uniform batches and hands ragged rows
|
||||
# another request's KV (silent garbage).
|
||||
rows = torch.arange(num_rows, device=cu.device, dtype=cu.dtype)
|
||||
row_req = torch.searchsorted(cu[1:], rows, right=True).clamp_(
|
||||
max=decode.block_table.shape[0] - 1
|
||||
)
|
||||
decode.flattened_row_req = row_req
|
||||
decode.flattened_block_table = decode.block_table[row_req]
|
||||
decode.flattened_seq_lens = decode.seq_lens[row_req]
|
||||
decode.query_len = num_rows
|
||||
assert decode.flattened_block_table is not None
|
||||
assert decode.flattened_seq_lens is not None
|
||||
assert decode.flattened_row_req is not None
|
||||
return (
|
||||
decode.flattened_block_table,
|
||||
decode.flattened_seq_lens,
|
||||
decode.flattened_row_req,
|
||||
)
|
||||
|
||||
@@ -158,6 +158,7 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]):
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> FlashMLADecodeMetadata:
|
||||
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
|
||||
@@ -1007,6 +1007,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> AiterMLADecodeMetadata:
|
||||
causal = self._decode_causal
|
||||
|
||||
@@ -250,7 +250,11 @@ class MambaHybridModelState(DefaultModelState):
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_tokens = input_batch.num_tokens
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
max_query_len = input_batch.num_scheduled_tokens.max().item()
|
||||
# Prefer the promised bound: a capture dummy's measured max is its even
|
||||
# split, not the length the graph must replay.
|
||||
max_query_len = input_batch.max_query_len
|
||||
if max_query_len is None:
|
||||
max_query_len = input_batch.num_scheduled_tokens.max().item()
|
||||
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound
|
||||
if for_capture:
|
||||
# Capture with worst-case max_seq_len so the graph is valid at any replay.
|
||||
@@ -278,10 +282,11 @@ class MambaHybridModelState(DefaultModelState):
|
||||
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
|
||||
num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req
|
||||
if num_draft_tokens_per_req is not None:
|
||||
# A row is a spec-decode row only when its whole prompt is already
|
||||
# computed, i.e. exactly one non-draft (decode) token is scheduled.
|
||||
is_decode = (
|
||||
input_batch.num_scheduled_tokens == num_draft_tokens_per_req + 1
|
||||
# Test request state, not num_scheduled_tokens == draft_count+1:
|
||||
# adaptive rewrites num_scheduled_tokens to an even split, so that
|
||||
# equality rarely holds and would demote every verify row to decode.
|
||||
is_decode = (~input_batch.is_prefilling_np) & (
|
||||
input_batch.num_scheduled_tokens > 0
|
||||
)
|
||||
spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode
|
||||
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
|
||||
|
||||
Reference in New Issue
Block a user