[Feature][Model Runner V2] Support extract_hidden_states speculation (#49811)

Signed-off-by: Zupeng Wang <71580390+zupengwang@users.noreply.github.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: OpenAI <support@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Zupeng Wang
2026-08-20 11:13:43 -04:00
committed by GitHub
co-authored by OpenAI Misha Goin OpenAI Codex
parent cb09dd7488
commit 01af92e175
6 changed files with 314 additions and 2 deletions
+6 -1
View File
@@ -247,7 +247,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.is_last_pp_rank:
self.speculator = init_speculator(self.vllm_config, self.device)
if self.speculative_config.method in ("eagle3", "dflash", "dspark"):
if self.speculative_config.method in (
"eagle3",
"dflash",
"dspark",
"extract_hidden_states",
):
# Drafting may require auxiliary hidden states from target model outputs
self.use_aux_hidden_state_outputs = True
if self.use_pp: