[MRV2][XPU] use xpu sample kernel in mrv2 sampler (#57277)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
76ee20c1ec
commit
6654bcbfca
@@ -14,6 +14,8 @@ from vllm.v1.sample.ops.topk_topp_sampler import (
|
||||
apply_top_k_top_p,
|
||||
flashinfer_sample,
|
||||
flashinfer_sampler_supported,
|
||||
xpu_sample,
|
||||
xpu_sampler_supported,
|
||||
)
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, get_num_sampled_and_rejected
|
||||
from vllm.v1.worker.gpu.metrics.logits import get_num_nans
|
||||
@@ -86,6 +88,10 @@ class Sampler:
|
||||
self.use_flashinfer = (
|
||||
not return_sampling_mask and flashinfer_sampler_supported()
|
||||
)
|
||||
# The XPU kernel draws fp32 exponential noise, so it can't honor fp64.
|
||||
self.use_xpu_sampler = (
|
||||
not return_sampling_mask and not use_fp64_gumbel and xpu_sampler_supported()
|
||||
)
|
||||
|
||||
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> None:
|
||||
needs_processing = self.sampling_states.add_request(req_idx, sampling_params)
|
||||
@@ -292,15 +298,18 @@ class Sampler:
|
||||
top_k, top_p = self.sampling_states.get_top_k_top_p(
|
||||
expanded_idx_mapping, idx_mapping_np
|
||||
)
|
||||
use_flashinfer = self.use_flashinfer and not (
|
||||
# Don't use FI sampler if no requests use top_k/top_p, if there are
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
# logprobs need to be returned for any requests.
|
||||
# Don't use a fused sampler if no requests use top_k/top_p, if there are
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
# logprobs need to be returned for any requests.
|
||||
fused_sampler_eligible = not (
|
||||
(top_k is None and top_p is None)
|
||||
or (return_logprobs and self.logprobs_mode in PROCESSED_LOGPROBS_MODES)
|
||||
or self.sampling_states.any_greedy(idx_mapping_np)
|
||||
or self.sampling_states.any_explicit_seed(idx_mapping_np)
|
||||
)
|
||||
use_fused_sampler = (
|
||||
self.use_flashinfer or self.use_xpu_sampler
|
||||
) and fused_sampler_eligible
|
||||
|
||||
return self._sample_random(
|
||||
processed_logits,
|
||||
@@ -309,7 +318,7 @@ class Sampler:
|
||||
pos,
|
||||
top_k,
|
||||
top_p,
|
||||
use_flashinfer,
|
||||
use_fused_sampler,
|
||||
)
|
||||
|
||||
def _sample_random(
|
||||
@@ -320,10 +329,15 @@ class Sampler:
|
||||
pos: torch.Tensor,
|
||||
top_k: torch.Tensor | None,
|
||||
top_p: torch.Tensor | None,
|
||||
use_flashinfer: bool,
|
||||
use_fused_sampler: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if use_flashinfer:
|
||||
sampled = flashinfer_sample(processed_logits, top_k, top_p).to(torch.int64)
|
||||
if use_fused_sampler:
|
||||
if self.use_flashinfer:
|
||||
sampled = flashinfer_sample(processed_logits, top_k, top_p).to(
|
||||
torch.int64
|
||||
)
|
||||
else: # Use XPU sampler
|
||||
sampled, _ = xpu_sample(processed_logits, top_k, top_p)
|
||||
else:
|
||||
processed_logits = apply_top_k_top_p(processed_logits, top_k, top_p)
|
||||
sampled = gumbel_sample(
|
||||
|
||||
Reference in New Issue
Block a user