[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)

Signed-off-by: Albert Cheng <albecheng@nvidia.com>
Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Albert Cheng
2026-09-22 20:43:32 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Benjamin Chislett
parent 355d789f0d
commit 88aa0d287d
13 changed files with 203 additions and 22 deletions
@@ -2471,6 +2471,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
max_query_len: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> MLACommonDecodeMetadata:
return MLACommonDecodeMetadata(
@@ -2635,6 +2636,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu=query_start_loc_cpu[: num_decodes + 1],
query_start_loc_device=query_start_loc[: num_decodes + 1],
num_decode_tokens=num_decode_tokens,
max_query_len=min(
common_attn_metadata.max_query_len, self.reorder_batch_threshold
),
dcp_tot_seq_lens_device=dcp_tot_seq_lens_device,
)