[Speculative Decoding] Fixing hidden states handling in batch expansion (#7508)

This commit is contained in:
Abhinav Goyal
2024-08-19 17:58:14 -07:00
committed by GitHub
parent e54ebc2f8f
commit 312f761232
6 changed files with 139 additions and 41 deletions
+2 -3
View File
@@ -646,9 +646,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
hidden_states = proposal_scores.hidden_states
if hidden_states is not None:
# Contract hidden states based on accepted tokens
hs_size = hidden_states.shape[1]
hidden_states = hidden_states.reshape(-1, max_proposal_len + 1,
hs_size)
hs_size = hidden_states.shape[-1]
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
index = accepted_index[:, None, None].expand(-1, 1, hs_size)