[Perf] Enable FlashInfer top-k/top-p sampler by default (#40376)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
Artem Perevedentsev
2026-04-29 19:10:34 +04:00
committed by GitHub
parent 5560cac7e2
commit b92ef9ec5a
5 changed files with 336 additions and 23 deletions
+26 -18
View File
@@ -41,23 +41,35 @@ class TopKTopPSampler(nn.Module):
capability = current_platform.get_device_capability()
assert capability is not None
if not FlashInferBackend.supports_compute_capability(capability):
if FlashInferBackend.supports_compute_capability(capability):
logger.info_once(
"Using FlashInfer for top-p & top-k sampling.",
scope="global",
)
self.forward = self.forward_cuda
elif envs.is_set("VLLM_USE_FLASHINFER_SAMPLER"):
# User explicitly opted in but the GPU can't run FlashInfer.
capability_str = capability.as_version_str()
raise RuntimeError(
"FlashInfer does not support compute capability "
f"{capability_str}, unset VLLM_USE_FLASHINFER_SAMPLER=1."
)
# Users must opt in explicitly via VLLM_USE_FLASHINFER_SAMPLER=1.
logger.info_once(
"Using FlashInfer for top-p & top-k sampling.",
scope="global",
)
self.forward = self.forward_cuda
else:
# Default-on path; hardware can't run FlashInfer →
# quietly fall back to the PyTorch-native sampler
# instead of failing server startup.
logger.warning_once(
"FlashInfer top-p/top-k sampling not supported on "
"compute capability %s; falling back to PyTorch-native "
"sampler. Set VLLM_USE_FLASHINFER_SAMPLER=0 to silence.",
capability.as_version_str(),
)
self.forward = self.forward_native
else:
logger.debug_once(
"FlashInfer top-p/top-k sampling is available but disabled "
"by default. Set VLLM_USE_FLASHINFER_SAMPLER=1 to opt in "
"after verifying accuracy for your workloads."
# User explicitly set VLLM_USE_FLASHINFER_SAMPLER=0.
logger.info_once(
"FlashInfer top-p/top-k sampling disabled via "
"VLLM_USE_FLASHINFER_SAMPLER=0; using PyTorch-native sampler."
)
self.forward = self.forward_native
@@ -120,9 +132,9 @@ class TopKTopPSampler(nn.Module):
p: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""More optimized implementation for top-k and top-p sampling."""
# We prefer `random_sample` over `flashinfer_sample` when sorting is
# not needed. This is because `random_sample` does not require
# CPU-GPU synchronization while `flashinfer_sample` does.
# Fall back to the PyTorch-native path when FlashInfer has nothing
# to do (no top-k / top-p filter) or when per-request generators
# are present (unsupported by FlashInfer 0.2.3+).
if (k is None and p is None) or generators:
if generators:
logger.debug_once(
@@ -361,10 +373,6 @@ def flashinfer_sample(
NOTE: The outputs of this function do not necessarily match the outputs of
the `random_sample` function. It only guarantees that the outputs are
statistically equivalent.
NOTE: This function includes CPU-GPU synchronization, while `random_sample`
does not. Call this function at the end of the forward pass to minimize
the synchronization overhead.
"""
import flashinfer