[BugFix][CPU][Spec Decode] Fix Eagle implementation on CPU backend (#42468)

Signed-off-by: Ofir Zafrir <ofir.zafrir@intel.com>
This commit is contained in:
Ofir Zafrir
2026-05-19 03:16:07 +00:00
committed by GitHub
parent afd7b1dce9
commit 8f16c4a5c0
3 changed files with 28 additions and 21 deletions
+4
View File
@@ -313,6 +313,10 @@ def apply_top_k_top_p(
if p is None and k is None:
return logits
# Keep CPU logits on the PyTorch path to avoid invoking Triton kernels.
if current_platform.is_cpu():
return apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=True)
if HAS_TRITON and logits.shape[0] >= 8:
return apply_top_k_top_p_triton(logits, k, p)
+2
View File
@@ -111,6 +111,8 @@ class CPUModelRunner(GPUModelRunner):
logger.info_once("Loading drafter model...")
self.drafter.load_model(self.model)
self._setup_eagle3_aux_hidden_state_outputs()
def get_model(self) -> nn.Module:
return self.model
+22 -21
View File
@@ -5055,27 +5055,7 @@ class GPUModelRunner(
)
eplb_models += 1
if self.use_aux_hidden_state_outputs:
if not supports_eagle3(self.get_model()):
raise RuntimeError(
"Model does not support EAGLE3 interface but "
"aux_hidden_state_outputs was requested"
)
# Try to get auxiliary layers from speculative config,
# otherwise use model's default layers
aux_layers = self._get_eagle3_aux_layers_from_config()
if aux_layers:
logger.info(
"Using auxiliary layers from speculative config: %s",
aux_layers,
)
else:
aux_layers = (
self.model.get_eagle3_default_aux_hidden_state_layers()
)
self.model.set_aux_hidden_state_layers(aux_layers)
self._setup_eagle3_aux_hidden_state_outputs()
# Resolve the MoE model, unwrapping VLM wrappers if needed.
# VLM models (e.g. KimiK25ForConditionalGeneration) wrap the
@@ -5190,6 +5170,27 @@ class GPUModelRunner(
get_offloader().post_init()
def _setup_eagle3_aux_hidden_state_outputs(self) -> None:
if not self.use_aux_hidden_state_outputs:
return
if not supports_eagle3(self.get_model()):
raise RuntimeError(
"Model does not support EAGLE3 interface but "
"aux_hidden_state_outputs was requested"
)
# Try to get auxiliary layers from speculative config,
# otherwise use model's default layers
aux_layers = self._get_eagle3_aux_layers_from_config()
if aux_layers:
logger.info(
"Using auxiliary layers from speculative config: %s", aux_layers
)
else:
aux_layers = self.model.get_eagle3_default_aux_hidden_state_layers()
self.model.set_aux_hidden_state_layers(aux_layers)
def _get_eagle3_aux_layers_from_config(self) -> tuple[int, ...] | None:
"""Extract Eagle3 auxiliary layer indices from speculative config.