diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index cc31202338..1fc5f6e8cc 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -313,6 +313,10 @@ def apply_top_k_top_p( if p is None and k is None: return logits + # Keep CPU logits on the PyTorch path to avoid invoking Triton kernels. + if current_platform.is_cpu(): + return apply_top_k_top_p_pytorch(logits, k, p, allow_cpu_sync=True) + if HAS_TRITON and logits.shape[0] >= 8: return apply_top_k_top_p_triton(logits, k, p) diff --git a/vllm/v1/worker/cpu_model_runner.py b/vllm/v1/worker/cpu_model_runner.py index 22c1b5bd09..6afffa424d 100644 --- a/vllm/v1/worker/cpu_model_runner.py +++ b/vllm/v1/worker/cpu_model_runner.py @@ -111,6 +111,8 @@ class CPUModelRunner(GPUModelRunner): logger.info_once("Loading drafter model...") self.drafter.load_model(self.model) + self._setup_eagle3_aux_hidden_state_outputs() + def get_model(self) -> nn.Module: return self.model diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index b7aa8779fd..529bbc2d4a 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -5055,27 +5055,7 @@ class GPUModelRunner( ) eplb_models += 1 - if self.use_aux_hidden_state_outputs: - if not supports_eagle3(self.get_model()): - raise RuntimeError( - "Model does not support EAGLE3 interface but " - "aux_hidden_state_outputs was requested" - ) - - # Try to get auxiliary layers from speculative config, - # otherwise use model's default layers - aux_layers = self._get_eagle3_aux_layers_from_config() - if aux_layers: - logger.info( - "Using auxiliary layers from speculative config: %s", - aux_layers, - ) - else: - aux_layers = ( - self.model.get_eagle3_default_aux_hidden_state_layers() - ) - - self.model.set_aux_hidden_state_layers(aux_layers) + self._setup_eagle3_aux_hidden_state_outputs() # Resolve the MoE model, unwrapping VLM wrappers if needed. # VLM models (e.g. KimiK25ForConditionalGeneration) wrap the @@ -5190,6 +5170,27 @@ class GPUModelRunner( get_offloader().post_init() + def _setup_eagle3_aux_hidden_state_outputs(self) -> None: + if not self.use_aux_hidden_state_outputs: + return + + if not supports_eagle3(self.get_model()): + raise RuntimeError( + "Model does not support EAGLE3 interface but " + "aux_hidden_state_outputs was requested" + ) + # Try to get auxiliary layers from speculative config, + # otherwise use model's default layers + aux_layers = self._get_eagle3_aux_layers_from_config() + if aux_layers: + logger.info( + "Using auxiliary layers from speculative config: %s", aux_layers + ) + else: + aux_layers = self.model.get_eagle3_default_aux_hidden_state_layers() + + self.model.set_aux_hidden_state_layers(aux_layers) + def _get_eagle3_aux_layers_from_config(self) -> tuple[int, ...] | None: """Extract Eagle3 auxiliary layer indices from speculative config.