[Perf] Enable FlashInfer top-k/top-p sampler by default (#40376)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
+5
-3
@@ -45,7 +45,7 @@ if TYPE_CHECKING:
|
||||
NO_COLOR: bool = False
|
||||
VLLM_LOG_STATS_INTERVAL: float = 10.0
|
||||
VLLM_TRACE_FUNCTION: int = 0
|
||||
VLLM_USE_FLASHINFER_SAMPLER: bool | None = None
|
||||
VLLM_USE_FLASHINFER_SAMPLER: bool = True
|
||||
VLLM_PP_LAYER_PARTITION: str | None = None
|
||||
VLLM_CPU_KVCACHE_SPACE: int | None = 0
|
||||
VLLM_CPU_OMP_THREADS_BIND: str = "auto"
|
||||
@@ -712,11 +712,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# If set to 1, vllm will trace function calls
|
||||
# Useful for debugging
|
||||
"VLLM_TRACE_FUNCTION": lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")),
|
||||
# If set, vllm will use flashinfer sampler
|
||||
# Whether to use the FlashInfer top-k / top-p sampler on CUDA. Enabled
|
||||
# by default when the hardware supports it — set to 0 to opt out
|
||||
# explicitly, which forces the PyTorch-native (Triton for bs>=8) path.
|
||||
"VLLM_USE_FLASHINFER_SAMPLER": lambda: (
|
||||
bool(int(os.environ["VLLM_USE_FLASHINFER_SAMPLER"]))
|
||||
if "VLLM_USE_FLASHINFER_SAMPLER" in os.environ
|
||||
else None
|
||||
else True
|
||||
),
|
||||
# Pipeline stage partition strategy
|
||||
"VLLM_PP_LAYER_PARTITION": lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
|
||||
|
||||
Reference in New Issue
Block a user