[WideEP] Change DeepEPv2 to auto select hybrid mode by default (#57991)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
parent
797505747f
commit
50239dfc4d
@@ -1065,7 +1065,9 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
|
|||||||
if os.environ.get("EP_DISABLE_GIN", "0") != "0":
|
if os.environ.get("EP_DISABLE_GIN", "0") != "0":
|
||||||
return
|
return
|
||||||
|
|
||||||
gin_type = query_nccl_gin_type(group)
|
gin_type = query_nccl_gin_type(
|
||||||
|
group, railed=envs.VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE
|
||||||
|
)
|
||||||
if gin_type is None:
|
if gin_type is None:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"DeepEPv2 communicator properties query failed; "
|
"DeepEPv2 communicator properties query failed; "
|
||||||
|
|||||||
+6
-2
@@ -2009,9 +2009,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool(
|
"VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool(
|
||||||
int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0"))
|
int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0"))
|
||||||
),
|
),
|
||||||
# DeepEP v2: enable two-tier NVLink+RDMA hybrid mode
|
# DeepEP v2: use NVLink+RDMA hybrid communication across NVLink domains.
|
||||||
|
# * hybrid mode = multi-node RDMA
|
||||||
|
# * non hybrid mode = nvlink
|
||||||
|
# Set to 1 (the default) to allow DeepEP to autoselect hybrid vs non-hybrid.
|
||||||
|
# Set to 0 to disable hybrid mode.
|
||||||
"VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE": lambda: bool(
|
"VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE": lambda: bool(
|
||||||
int(os.getenv("VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE", "0"))
|
int(os.getenv("VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE", "1"))
|
||||||
),
|
),
|
||||||
# DeepEP v2: use fewer SMs at slight throughput cost
|
# DeepEP v2: use fewer SMs at slight throughput cost
|
||||||
"VLLM_DEEPEP_V2_PREFER_OVERLAP": lambda: bool(
|
"VLLM_DEEPEP_V2_PREFER_OVERLAP": lambda: bool(
|
||||||
|
|||||||
+5
-3
@@ -83,8 +83,10 @@ def find_nccl_library_paths() -> list[str] | None:
|
|||||||
return paths or None
|
return paths or None
|
||||||
|
|
||||||
|
|
||||||
def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None:
|
def query_nccl_gin_type(
|
||||||
"""Return the GIN type for an initialized group, or ``None`` on failure."""
|
group: torch.distributed.ProcessGroup, *, railed: bool = False
|
||||||
|
) -> int | None:
|
||||||
|
"""Return the full or railed GIN type, or ``None`` on query failure."""
|
||||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||||
NCCL_COMM_PROPERTIES_LAYOUT_VERSION,
|
NCCL_COMM_PROPERTIES_LAYOUT_VERSION,
|
||||||
NCCLLibrary,
|
NCCLLibrary,
|
||||||
@@ -126,4 +128,4 @@ def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None:
|
|||||||
if result != 0:
|
if result != 0:
|
||||||
logger.warning("ncclCommQueryProperties returned error %d", result)
|
logger.warning("ncclCommQueryProperties returned error %d", result)
|
||||||
return None
|
return None
|
||||||
return props.ginType
|
return props.railedGinType if railed else props.ginType
|
||||||
|
|||||||
Reference in New Issue
Block a user