[BugFix][CPU][Spec Decode] Fix Eagle implementation on CPU backend (#42468)
Signed-off-by: Ofir Zafrir <ofir.zafrir@intel.com>
This commit is contained in:
@@ -313,6 +313,10 @@ def apply_top_k_top_p(
|
||||
if p is None and k is None:
|
||||
return logits
|
||||
|
||||
# Keep CPU logits on the PyTorch path to avoid invoking Triton kernels.
|
||||
if current_platform.is_cpu():
|
||||
return apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=True)
|
||||
|
||||
if HAS_TRITON and logits.shape[0] >= 8:
|
||||
return apply_top_k_top_p_triton(logits, k, p)
|
||||
|
||||
|
||||
@@ -111,6 +111,8 @@ class CPUModelRunner(GPUModelRunner):
|
||||
logger.info_once("Loading drafter model...")
|
||||
self.drafter.load_model(self.model)
|
||||
|
||||
self._setup_eagle3_aux_hidden_state_outputs()
|
||||
|
||||
def get_model(self) -> nn.Module:
|
||||
return self.model
|
||||
|
||||
|
||||
@@ -5055,27 +5055,7 @@ class GPUModelRunner(
|
||||
)
|
||||
eplb_models += 1
|
||||
|
||||
if self.use_aux_hidden_state_outputs:
|
||||
if not supports_eagle3(self.get_model()):
|
||||
raise RuntimeError(
|
||||
"Model does not support EAGLE3 interface but "
|
||||
"aux_hidden_state_outputs was requested"
|
||||
)
|
||||
|
||||
# Try to get auxiliary layers from speculative config,
|
||||
# otherwise use model's default layers
|
||||
aux_layers = self._get_eagle3_aux_layers_from_config()
|
||||
if aux_layers:
|
||||
logger.info(
|
||||
"Using auxiliary layers from speculative config: %s",
|
||||
aux_layers,
|
||||
)
|
||||
else:
|
||||
aux_layers = (
|
||||
self.model.get_eagle3_default_aux_hidden_state_layers()
|
||||
)
|
||||
|
||||
self.model.set_aux_hidden_state_layers(aux_layers)
|
||||
self._setup_eagle3_aux_hidden_state_outputs()
|
||||
|
||||
# Resolve the MoE model, unwrapping VLM wrappers if needed.
|
||||
# VLM models (e.g. KimiK25ForConditionalGeneration) wrap the
|
||||
@@ -5190,6 +5170,27 @@ class GPUModelRunner(
|
||||
|
||||
get_offloader().post_init()
|
||||
|
||||
def _setup_eagle3_aux_hidden_state_outputs(self) -> None:
|
||||
if not self.use_aux_hidden_state_outputs:
|
||||
return
|
||||
|
||||
if not supports_eagle3(self.get_model()):
|
||||
raise RuntimeError(
|
||||
"Model does not support EAGLE3 interface but "
|
||||
"aux_hidden_state_outputs was requested"
|
||||
)
|
||||
# Try to get auxiliary layers from speculative config,
|
||||
# otherwise use model's default layers
|
||||
aux_layers = self._get_eagle3_aux_layers_from_config()
|
||||
if aux_layers:
|
||||
logger.info(
|
||||
"Using auxiliary layers from speculative config: %s", aux_layers
|
||||
)
|
||||
else:
|
||||
aux_layers = self.model.get_eagle3_default_aux_hidden_state_layers()
|
||||
|
||||
self.model.set_aux_hidden_state_layers(aux_layers)
|
||||
|
||||
def _get_eagle3_aux_layers_from_config(self) -> tuple[int, ...] | None:
|
||||
"""Extract Eagle3 auxiliary layer indices from speculative config.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user