From 6a962071bdad41c2d739bb2e879e72bbe9df9e63 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 18:17:43 -0700 Subject: [PATCH] [Distributed] Enable FlashInfer all-reduce by default (#52998) Signed-off-by: Woosuk Kwon Co-authored-by: OpenAI Codex --- vllm/distributed/device_communicators/cuda_communicator.py | 5 ++++- vllm/envs.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index 06b441c5a4..f8d4e9fb28 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -58,7 +58,10 @@ class CudaCommunicator(DeviceCommunicatorBase): use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE use_torch_symm_mem = envs.VLLM_ALLREDUCE_USE_SYMM_MEM - use_flashinfer_allreduce = envs.VLLM_ALLREDUCE_USE_FLASHINFER + # FlashInfer all-reduce does not provide a fixed reduction order. + use_flashinfer_allreduce = ( + envs.VLLM_ALLREDUCE_USE_FLASHINFER and not envs.VLLM_BATCH_INVARIANT + ) use_aiter_allreduce = use_custom_allreduce and bool( rocm_aiter_ops.is_custom_all_reduce_enabled() ) diff --git a/vllm/envs.py b/vllm/envs.py index b49841d9ff..a928461da2 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -260,7 +260,7 @@ if TYPE_CHECKING: VLLM_HAS_FLASHINFER_CUBIN: bool = False VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True - VLLM_ALLREDUCE_USE_FLASHINFER: bool = False + VLLM_ALLREDUCE_USE_FLASHINFER: bool = True VLLM_TUNED_CONFIG_FOLDER: str | None = None VLLM_ENABLE_STARTUP_PLAN: bool = False VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set() @@ -1857,7 +1857,7 @@ environment_variables: dict[str, Callable[[], Any]] = { ), # Whether to use FlashInfer allreduce "VLLM_ALLREDUCE_USE_FLASHINFER": lambda: bool( - int(os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER", "0")) + int(os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER", "1")) ), # Experimental: use this to enable MCP tool calling for non harmony models "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": lambda: bool(