[Bugfix][Async] Fix async spec decoding with hybrid models (#38556)

Signed-off-by: SandishKumarHN <sandishkumarhn@gmail.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: SandishKumarHN <sandishkumarhn@gmail.com>
This commit is contained in:
Matthew Bonanni
2026-03-31 11:08:54 -04:00
committed by GitHub
co-authored by SandishKumarHN
parent 7337ff7f03
commit 757068dc65
6 changed files with 177 additions and 36 deletions
+1 -2
View File
@@ -859,7 +859,6 @@ class SpecDecodeBaseProposer:
def prepare_next_token_ids_padded(
self,
seq_lens_cpu: torch.Tensor,
sampled_token_ids: torch.Tensor,
requests: dict[str, CachedRequestState],
gpu_input_batch: InputBatch,
@@ -874,7 +873,7 @@ class SpecDecodeBaseProposer:
"""
# Precompute get_token_id for when there is no valid next token
num_reqs = gpu_input_batch.num_reqs
seq_lens_list = seq_lens_cpu[:num_reqs].tolist()
seq_lens_list = (gpu_input_batch.num_tokens_no_spec[:num_reqs] - 1).tolist()
self.backup_next_token_ids.np[:num_reqs] = np.array(
[
requests[gpu_input_batch.req_ids[i]].get_token_id(seq_lens_list[i])
+1 -2
View File
@@ -286,7 +286,6 @@ class ExtractHiddenStatesProposer:
def prepare_next_token_ids_padded(
self,
seq_lens: torch.Tensor,
sampled_token_ids: torch.Tensor,
requests: dict[str, CachedRequestState],
gpu_input_batch: InputBatch,
@@ -303,7 +302,7 @@ class ExtractHiddenStatesProposer:
device = sampled_token_ids.device
# Compute backup tokens for discarded / invalid requests
seq_lens_list = seq_lens[:num_reqs].tolist()
seq_lens_list = (gpu_input_batch.num_tokens_no_spec[:num_reqs] - 1).tolist()
backup_tokens_gpu = torch.tensor(
[
requests[gpu_input_batch.req_ids[i]].get_token_id(seq_lens_list[i])