[Spec Decode] Add support for EAGLE3 heads that do not use_aux_hidden_states (#27688)
Signed-off-by: hjjq <hanjieq@nvidia.com> Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
699bca76c0
commit
5f9679a43b
@@ -375,7 +375,9 @@ class GPUModelRunner(
|
||||
elif self.speculative_config.use_eagle():
|
||||
self.drafter = EagleProposer(self.vllm_config, self.device, self)
|
||||
if self.speculative_config.method == "eagle3":
|
||||
self.use_aux_hidden_state_outputs = True
|
||||
self.use_aux_hidden_state_outputs = (
|
||||
self.drafter.eagle3_use_aux_hidden_state
|
||||
)
|
||||
elif self.speculative_config.method == "medusa":
|
||||
self.drafter = MedusaProposer(
|
||||
vllm_config=self.vllm_config, device=self.device
|
||||
|
||||
Reference in New Issue
Block a user