[MRV2][XPU] use xpu sample kernel in mrv2 sampler (#57277)

Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
liuzhenwei
2026-09-21 13:24:41 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent 76ee20c1ec
commit 6654bcbfca
3 changed files with 89 additions and 43 deletions
+22 -8
View File
@@ -14,6 +14,8 @@ from vllm.v1.sample.ops.topk_topp_sampler import (
apply_top_k_top_p,
flashinfer_sample,
flashinfer_sampler_supported,
xpu_sample,
xpu_sampler_supported,
)
from vllm.v1.worker.gpu.input_batch import InputBatch, get_num_sampled_and_rejected
from vllm.v1.worker.gpu.metrics.logits import get_num_nans
@@ -86,6 +88,10 @@ class Sampler:
self.use_flashinfer = (
not return_sampling_mask and flashinfer_sampler_supported()
)
# The XPU kernel draws fp32 exponential noise, so it can't honor fp64.
self.use_xpu_sampler = (
not return_sampling_mask and not use_fp64_gumbel and xpu_sampler_supported()
)
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> None:
needs_processing = self.sampling_states.add_request(req_idx, sampling_params)
@@ -292,15 +298,18 @@ class Sampler:
top_k, top_p = self.sampling_states.get_top_k_top_p(
expanded_idx_mapping, idx_mapping_np
)
use_flashinfer = self.use_flashinfer and not (
# Don't use FI sampler if no requests use top_k/top_p, if there are
# any greedy requests or per-request seeds, or if post-processed
# logprobs need to be returned for any requests.
# Don't use a fused sampler if no requests use top_k/top_p, if there are
# any greedy requests or per-request seeds, or if post-processed
# logprobs need to be returned for any requests.
fused_sampler_eligible = not (
(top_k is None and top_p is None)
or (return_logprobs and self.logprobs_mode in PROCESSED_LOGPROBS_MODES)
or self.sampling_states.any_greedy(idx_mapping_np)
or self.sampling_states.any_explicit_seed(idx_mapping_np)
)
use_fused_sampler = (
self.use_flashinfer or self.use_xpu_sampler
) and fused_sampler_eligible
return self._sample_random(
processed_logits,
@@ -309,7 +318,7 @@ class Sampler:
pos,
top_k,
top_p,
use_flashinfer,
use_fused_sampler,
)
def _sample_random(
@@ -320,10 +329,15 @@ class Sampler:
pos: torch.Tensor,
top_k: torch.Tensor | None,
top_p: torch.Tensor | None,
use_flashinfer: bool,
use_fused_sampler: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
if use_flashinfer:
sampled = flashinfer_sample(processed_logits, top_k, top_p).to(torch.int64)
if use_fused_sampler:
if self.use_flashinfer:
sampled = flashinfer_sample(processed_logits, top_k, top_p).to(
torch.int64
)
else: # Use XPU sampler
sampled, _ = xpu_sample(processed_logits, top_k, top_p)
else:
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
sampled = gumbel_sample(