diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index e6e0e054c0..03716cb3c1 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -200,7 +200,7 @@ class TopKTopPSampler(nn.Module): elif self.logprobs_mode == "processed_logprobs": logits_to_return = logits.log_softmax(dim=-1, dtype=torch.float32) - if len(generators) != logits.shape[0] and not self.use_fp64_gumbel: + if not generators and not self.use_fp64_gumbel: return compiled_random_sample(logits), logits_to_return probs = logits.softmax(dim=-1, dtype=torch.float32)