[Distributed] Enable FlashInfer all-reduce by default (#52998)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
58e5ee0158
commit
6a962071bd
@@ -58,7 +58,10 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
|||||||
|
|
||||||
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
|
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
|
||||||
use_torch_symm_mem = envs.VLLM_ALLREDUCE_USE_SYMM_MEM
|
use_torch_symm_mem = envs.VLLM_ALLREDUCE_USE_SYMM_MEM
|
||||||
use_flashinfer_allreduce = envs.VLLM_ALLREDUCE_USE_FLASHINFER
|
# FlashInfer all-reduce does not provide a fixed reduction order.
|
||||||
|
use_flashinfer_allreduce = (
|
||||||
|
envs.VLLM_ALLREDUCE_USE_FLASHINFER and not envs.VLLM_BATCH_INVARIANT
|
||||||
|
)
|
||||||
use_aiter_allreduce = use_custom_allreduce and bool(
|
use_aiter_allreduce = use_custom_allreduce and bool(
|
||||||
rocm_aiter_ops.is_custom_all_reduce_enabled()
|
rocm_aiter_ops.is_custom_all_reduce_enabled()
|
||||||
)
|
)
|
||||||
|
|||||||
+2
-2
@@ -260,7 +260,7 @@ if TYPE_CHECKING:
|
|||||||
VLLM_HAS_FLASHINFER_CUBIN: bool = False
|
VLLM_HAS_FLASHINFER_CUBIN: bool = False
|
||||||
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
|
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
|
||||||
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
||||||
VLLM_ALLREDUCE_USE_FLASHINFER: bool = False
|
VLLM_ALLREDUCE_USE_FLASHINFER: bool = True
|
||||||
VLLM_TUNED_CONFIG_FOLDER: str | None = None
|
VLLM_TUNED_CONFIG_FOLDER: str | None = None
|
||||||
VLLM_ENABLE_STARTUP_PLAN: bool = False
|
VLLM_ENABLE_STARTUP_PLAN: bool = False
|
||||||
VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
|
VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
|
||||||
@@ -1857,7 +1857,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
),
|
),
|
||||||
# Whether to use FlashInfer allreduce
|
# Whether to use FlashInfer allreduce
|
||||||
"VLLM_ALLREDUCE_USE_FLASHINFER": lambda: bool(
|
"VLLM_ALLREDUCE_USE_FLASHINFER": lambda: bool(
|
||||||
int(os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER", "0"))
|
int(os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER", "1"))
|
||||||
),
|
),
|
||||||
# Experimental: use this to enable MCP tool calling for non harmony models
|
# Experimental: use this to enable MCP tool calling for non harmony models
|
||||||
"VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": lambda: bool(
|
"VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": lambda: bool(
|
||||||
|
|||||||
Reference in New Issue
Block a user