[Bugfix][Async] Fix async spec decoding with hybrid models (#38556)
Signed-off-by: SandishKumarHN <sandishkumarhn@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: SandishKumarHN <sandishkumarhn@gmail.com>
This commit is contained in:
co-authored by
SandishKumarHN
parent
7337ff7f03
commit
757068dc65
@@ -859,7 +859,6 @@ class SpecDecodeBaseProposer:
|
||||
|
||||
def prepare_next_token_ids_padded(
|
||||
self,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
sampled_token_ids: torch.Tensor,
|
||||
requests: dict[str, CachedRequestState],
|
||||
gpu_input_batch: InputBatch,
|
||||
@@ -874,7 +873,7 @@ class SpecDecodeBaseProposer:
|
||||
"""
|
||||
# Precompute get_token_id for when there is no valid next token
|
||||
num_reqs = gpu_input_batch.num_reqs
|
||||
seq_lens_list = seq_lens_cpu[:num_reqs].tolist()
|
||||
seq_lens_list = (gpu_input_batch.num_tokens_no_spec[:num_reqs] - 1).tolist()
|
||||
self.backup_next_token_ids.np[:num_reqs] = np.array(
|
||||
[
|
||||
requests[gpu_input_batch.req_ids[i]].get_token_id(seq_lens_list[i])
|
||||
|
||||
@@ -286,7 +286,6 @@ class ExtractHiddenStatesProposer:
|
||||
|
||||
def prepare_next_token_ids_padded(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
sampled_token_ids: torch.Tensor,
|
||||
requests: dict[str, CachedRequestState],
|
||||
gpu_input_batch: InputBatch,
|
||||
@@ -303,7 +302,7 @@ class ExtractHiddenStatesProposer:
|
||||
device = sampled_token_ids.device
|
||||
|
||||
# Compute backup tokens for discarded / invalid requests
|
||||
seq_lens_list = seq_lens[:num_reqs].tolist()
|
||||
seq_lens_list = (gpu_input_batch.num_tokens_no_spec[:num_reqs] - 1).tolist()
|
||||
backup_tokens_gpu = torch.tensor(
|
||||
[
|
||||
requests[gpu_input_batch.req_ids[i]].get_token_id(seq_lens_list[i])
|
||||
|
||||
Reference in New Issue
Block a user