[Speculative Decoding] Fixing hidden states handling in batch expansion (#7508)
This commit is contained in:
@@ -646,9 +646,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
hidden_states = proposal_scores.hidden_states
|
||||
if hidden_states is not None:
|
||||
# Contract hidden states based on accepted tokens
|
||||
hs_size = hidden_states.shape[1]
|
||||
hidden_states = hidden_states.reshape(-1, max_proposal_len + 1,
|
||||
hs_size)
|
||||
hs_size = hidden_states.shape[-1]
|
||||
|
||||
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
|
||||
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
|
||||
index = accepted_index[:, None, None].expand(-1, 1, hs_size)
|
||||
|
||||
Reference in New Issue
Block a user