[Perf] Enable FlashInfer top-k/top-p sampler by default (#40376)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
@@ -41,23 +41,35 @@ class TopKTopPSampler(nn.Module):
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
assert capability is not None
|
||||
if not FlashInferBackend.supports_compute_capability(capability):
|
||||
if FlashInferBackend.supports_compute_capability(capability):
|
||||
logger.info_once(
|
||||
"Using FlashInfer for top-p & top-k sampling.",
|
||||
scope="global",
|
||||
)
|
||||
self.forward = self.forward_cuda
|
||||
elif envs.is_set("VLLM_USE_FLASHINFER_SAMPLER"):
|
||||
# User explicitly opted in but the GPU can't run FlashInfer.
|
||||
capability_str = capability.as_version_str()
|
||||
raise RuntimeError(
|
||||
"FlashInfer does not support compute capability "
|
||||
f"{capability_str}, unset VLLM_USE_FLASHINFER_SAMPLER=1."
|
||||
)
|
||||
# Users must opt in explicitly via VLLM_USE_FLASHINFER_SAMPLER=1.
|
||||
logger.info_once(
|
||||
"Using FlashInfer for top-p & top-k sampling.",
|
||||
scope="global",
|
||||
)
|
||||
self.forward = self.forward_cuda
|
||||
else:
|
||||
# Default-on path; hardware can't run FlashInfer →
|
||||
# quietly fall back to the PyTorch-native sampler
|
||||
# instead of failing server startup.
|
||||
logger.warning_once(
|
||||
"FlashInfer top-p/top-k sampling not supported on "
|
||||
"compute capability %s; falling back to PyTorch-native "
|
||||
"sampler. Set VLLM_USE_FLASHINFER_SAMPLER=0 to silence.",
|
||||
capability.as_version_str(),
|
||||
)
|
||||
self.forward = self.forward_native
|
||||
else:
|
||||
logger.debug_once(
|
||||
"FlashInfer top-p/top-k sampling is available but disabled "
|
||||
"by default. Set VLLM_USE_FLASHINFER_SAMPLER=1 to opt in "
|
||||
"after verifying accuracy for your workloads."
|
||||
# User explicitly set VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
logger.info_once(
|
||||
"FlashInfer top-p/top-k sampling disabled via "
|
||||
"VLLM_USE_FLASHINFER_SAMPLER=0; using PyTorch-native sampler."
|
||||
)
|
||||
self.forward = self.forward_native
|
||||
|
||||
@@ -120,9 +132,9 @@ class TopKTopPSampler(nn.Module):
|
||||
p: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""More optimized implementation for top-k and top-p sampling."""
|
||||
# We prefer `random_sample` over `flashinfer_sample` when sorting is
|
||||
# not needed. This is because `random_sample` does not require
|
||||
# CPU-GPU synchronization while `flashinfer_sample` does.
|
||||
# Fall back to the PyTorch-native path when FlashInfer has nothing
|
||||
# to do (no top-k / top-p filter) or when per-request generators
|
||||
# are present (unsupported by FlashInfer 0.2.3+).
|
||||
if (k is None and p is None) or generators:
|
||||
if generators:
|
||||
logger.debug_once(
|
||||
@@ -361,10 +373,6 @@ def flashinfer_sample(
|
||||
NOTE: The outputs of this function do not necessarily match the outputs of
|
||||
the `random_sample` function. It only guarantees that the outputs are
|
||||
statistically equivalent.
|
||||
|
||||
NOTE: This function includes CPU-GPU synchronization, while `random_sample`
|
||||
does not. Call this function at the end of the forward pass to minimize
|
||||
the synchronization overhead.
|
||||
"""
|
||||
import flashinfer
|
||||
|
||||
|
||||
Reference in New Issue
Block a user