[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)
Signed-off-by: Albert Cheng <albecheng@nvidia.com> Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Benjamin Chislett
parent
355d789f0d
commit
88aa0d287d
@@ -353,9 +353,10 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder):
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> Dots3NoteDecodeMetadata:
|
||||
del max_seq_len, query_start_loc_device, num_decode_tokens
|
||||
del max_seq_len, query_start_loc_device, num_decode_tokens, max_query_len
|
||||
query_len = int(query_start_loc_cpu[1] - query_start_loc_cpu[0])
|
||||
return Dots3NoteDecodeMetadata(
|
||||
block_table=block_table_tensor,
|
||||
|
||||
Reference in New Issue
Block a user