[Perf] Enable FlashInfer top-k/top-p sampler by default (#40376)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
Artem Perevedentsev
2026-04-29 19:10:34 +04:00
committed by GitHub
parent 5560cac7e2
commit b92ef9ec5a
5 changed files with 336 additions and 23 deletions
+5 -3
View File
@@ -45,7 +45,7 @@ if TYPE_CHECKING:
NO_COLOR: bool = False
VLLM_LOG_STATS_INTERVAL: float = 10.0
VLLM_TRACE_FUNCTION: int = 0
VLLM_USE_FLASHINFER_SAMPLER: bool | None = None
VLLM_USE_FLASHINFER_SAMPLER: bool = True
VLLM_PP_LAYER_PARTITION: str | None = None
VLLM_CPU_KVCACHE_SPACE: int | None = 0
VLLM_CPU_OMP_THREADS_BIND: str = "auto"
@@ -712,11 +712,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
# If set to 1, vllm will trace function calls
# Useful for debugging
"VLLM_TRACE_FUNCTION": lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")),
# If set, vllm will use flashinfer sampler
# Whether to use the FlashInfer top-k / top-p sampler on CUDA. Enabled
# by default when the hardware supports it — set to 0 to opt out
# explicitly, which forces the PyTorch-native (Triton for bs>=8) path.
"VLLM_USE_FLASHINFER_SAMPLER": lambda: (
bool(int(os.environ["VLLM_USE_FLASHINFER_SAMPLER"]))
if "VLLM_USE_FLASHINFER_SAMPLER" in os.environ
else None
else True
),
# Pipeline stage partition strategy
"VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),