[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)
Signed-off-by: Albert Cheng <albecheng@nvidia.com> Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Benjamin Chislett
parent
355d789f0d
commit
88aa0d287d
@@ -2471,6 +2471,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> MLACommonDecodeMetadata:
|
||||
return MLACommonDecodeMetadata(
|
||||
@@ -2635,6 +2636,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu=query_start_loc_cpu[: num_decodes + 1],
|
||||
query_start_loc_device=query_start_loc[: num_decodes + 1],
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
max_query_len=min(
|
||||
common_attn_metadata.max_query_len, self.reorder_batch_threshold
|
||||
),
|
||||
dcp_tot_seq_lens_device=dcp_tot_seq_lens_device,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user