From 40b8363b45a9c59984907603b00b736e41d25065 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Thu, 19 Mar 2026 08:41:21 -0700 Subject: [PATCH] [MRV2] Use fp32 for draft logits (#37526) Signed-off-by: Woosuk Kwon --- vllm/v1/worker/gpu/model_runner.py | 1 - vllm/v1/worker/gpu/states.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 1f13de50bf..06e91d3802 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -195,7 +195,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_speculative_steps=self.num_speculative_steps, vocab_size=self.vocab_size, device=self.device, - model_dtype=self.dtype, cache_draft_logits=not use_strict_rejection_sampling, ) self.input_buffers = InputBuffers( diff --git a/vllm/v1/worker/gpu/states.py b/vllm/v1/worker/gpu/states.py index fcdb1fe0bd..3fb02c12d9 100644 --- a/vllm/v1/worker/gpu/states.py +++ b/vllm/v1/worker/gpu/states.py @@ -15,7 +15,6 @@ class RequestState: num_speculative_steps: int, vocab_size: int, device: torch.device, - model_dtype: torch.dtype, cache_draft_logits: bool, ): self.max_num_reqs = max_num_reqs @@ -81,7 +80,7 @@ class RequestState: self.max_num_reqs, self.num_speculative_steps, self.vocab_size, - dtype=model_dtype, + dtype=torch.float32, device=device, )