[Feature][Model Runner V2] Support extract_hidden_states speculation (#49811)
Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: OpenAI <support@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI
Misha Goin
OpenAI Codex
parent
cb09dd7488
commit
01af92e175
@@ -247,7 +247,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.is_last_pp_rank:
|
||||
self.speculator = init_speculator(self.vllm_config, self.device)
|
||||
|
||||
if self.speculative_config.method in ("eagle3", "dflash", "dspark"):
|
||||
if self.speculative_config.method in (
|
||||
"eagle3",
|
||||
"dflash",
|
||||
"dspark",
|
||||
"extract_hidden_states",
|
||||
):
|
||||
# Drafting may require auxiliary hidden states from target model outputs
|
||||
self.use_aux_hidden_state_outputs = True
|
||||
if self.use_pp:
|
||||
|
||||
Reference in New Issue
Block a user