[WideEP] Update NCCL to 2.30.7 to enable DeepEPv2 in the vllm/vllm-openai image (#45321)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Tyler Michael Smith <tyler@vllm.ai>
Signed-off-by: Tyler Michael Smith <tyler@tylermsmith.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Ilya Markov <ilmarkov@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Tyler Michael Smith
2026-07-24 13:00:02 -07:00
committed by GitHub
co-authored by Claude Ilya Markov OpenAI Codex Codex
parent e222c33f2f
commit 5d8e90a966
13 changed files with 338 additions and 78 deletions
+9 -3
View File
@@ -73,7 +73,10 @@ def _spawn_workers(worker_fn, world_size, *, dp_size=None):
err_queue.close()
err_queue.join_thread()
if errors:
pytest.fail("Worker(s) failed:\n" + "\n---\n".join(errors))
combined = "\n---\n".join(errors)
if "NCCL GIN" in combined:
pytest.skip("NCCL GIN not available on this system")
pytest.fail("Worker(s) failed:\n" + combined)
def _run_worker(rank, world_size, port, worker_fn, dp_size, dp_port, err_queue):
@@ -916,8 +919,11 @@ def _deepep_v2_lifecycle_worker(rank, world_size):
DeepEPV2All2AllManager,
)
cpu_group = get_ep_group().cpu_group
manager = DeepEPV2All2AllManager(cpu_group)
ep_group = get_ep_group()
manager = DeepEPV2All2AllManager(
ep_group.cpu_group,
device_group=ep_group.device_group,
)
assert manager.rank == rank
assert manager.world_size == world_size
+36 -12
View File
@@ -36,6 +36,10 @@ if has_deep_ep_v2():
P = ParamSpec("P")
class GINNotAvailableError(RuntimeError):
pass
@dataclasses.dataclass
class ProcessGroupInfo:
world_size: int
@@ -96,19 +100,27 @@ def parallel_launch(
**kwargs: P.kwargs,
) -> None:
assert not kwargs
spawn(
_worker_parallel_launch,
args=(
world_size,
world_size,
0,
f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{get_open_port()}",
worker,
try:
spawn(
_worker_parallel_launch,
args=(
world_size,
world_size,
0,
f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{get_open_port()}",
worker,
)
+ args,
nprocs=world_size,
join=True,
)
+ args,
nprocs=world_size,
join=True,
)
except Exception as exc:
# pytest.skip cannot propagate directly through torch.multiprocessing.
if "GINNotAvailableError" in str(exc):
import pytest
pytest.skip("NCCL GIN not available (no IBGDA-capable hardware)")
raise
## DeepEP specific utils
@@ -225,6 +237,18 @@ def make_deepep_v2_a2a(
):
import deep_ep
from vllm.utils.nccl import query_nccl_gin_type
# ElasticBuffer can segfault when GIN is unavailable. Initialize the
# lazy communicator and reject unsupported systems before entering DeepEP.
probe = torch.zeros(1, device=pgi.device)
torch.distributed.all_reduce(probe, group=pg)
gin_type = query_nccl_gin_type(pg)
if gin_type is None:
raise RuntimeError("Failed to determine NCCL GIN support")
if gin_type == 0:
raise GINNotAvailableError("NCCL GIN not available")
buffer = deep_ep.ElasticBuffer(
group=pg,
num_max_tokens_per_rank=v2_args.max_tokens_per_rank,
+101 -50
View File
@@ -14,6 +14,7 @@ from torch.distributed import ProcessGroup
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe import TritonExperts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
@@ -36,6 +37,14 @@ requires_deep_ep_v2 = pytest.mark.skipif(
)
def assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
close = torch.isclose(actual, expected, atol=2e-1, rtol=2e-1)
close_fraction = close.float().mean().item()
assert close_fraction > 0.99, (
f"Only {close_fraction:.1%} of FP8 outputs are within tolerance"
)
@dataclasses.dataclass
class TestConfig:
dtype: torch.dtype
@@ -50,6 +59,7 @@ class TestConfig:
class TestTensors:
rank_tokens: torch.Tensor
rank_token_scales: torch.Tensor | None
intermediate_scales: torch.Tensor | None
topk: torch.Tensor
topk_weights: torch.Tensor
config: TestConfig
@@ -63,6 +73,12 @@ class TestTensors:
rank_tokens = (
torch.randn((config.m, config.k), device="cuda", dtype=token_dtype) / 10
)
if config.dtype == torch.float8_e4m3fn:
rank_token_scales = torch.tensor(1 / 448, device="cuda")
intermediate_scales = torch.tensor(8 / 448, device="cuda")
else:
rank_token_scales = None
intermediate_scales = None
topk = torch.stack(
[
@@ -73,7 +89,8 @@ class TestTensors:
topk_weights = torch.randn(topk.shape, dtype=torch.float32, device="cuda")
return TestTensors(
rank_tokens=rank_tokens,
rank_token_scales=None,
rank_token_scales=rank_token_scales,
intermediate_scales=intermediate_scales,
topk=topk,
topk_weights=topk_weights,
config=config,
@@ -124,7 +141,6 @@ def make_modular_kernel(
mk = FusedMoEKernel(
prepare_finalize=a2a,
fused_experts=fused_experts,
inplace=False,
)
return mk
@@ -162,6 +178,7 @@ def deepep_v2_moe_impl(
w2_scale=w2_scale,
per_act_token_quant=per_act_token_quant,
a1_scale=test_tensors.rank_token_scales,
a2_scale=test_tensors.intermediate_scales,
)
hidden_size = test_tensors.rank_tokens.size(1)
@@ -231,6 +248,8 @@ def _deep_ep_v2_moe(
test_tensors.topk,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=test_tensors.rank_token_scales,
a2_scale=test_tensors.intermediate_scales,
quant_dtype=q_dtype,
per_act_token_quant=per_act_token_quant,
)
@@ -262,12 +281,15 @@ def _deep_ep_v2_moe(
per_act_token_quant,
)
torch.testing.assert_close(
torch_combined,
deepep_combined,
atol=6e-2,
rtol=6e-2,
)
if is_quantized:
assert_fp8_close(torch_combined, deepep_combined)
else:
torch.testing.assert_close(
torch_combined,
deepep_combined,
atol=6e-2,
rtol=6e-2,
)
MNKs = [
@@ -355,17 +377,29 @@ def _deep_ep_v2_moe_cudagraph(
num_local_experts = config.num_experts // pgi.world_size
hidden_size = config.k
# Create FP8 weights directly, then dequantize for bf16 reference.
w1_fp8 = torch.randn(
(config.num_experts, 2 * config.n, config.k),
device="cuda",
dtype=torch.bfloat16,
).to(torch.float8_e4m3fn)
w2_fp8 = torch.randn(
(config.num_experts, config.k, config.n),
device="cuda",
dtype=torch.bfloat16,
).to(torch.float8_e4m3fn)
# All ranks must use the same global weights before taking their EP slice.
w1_bf16 = (
torch.randn(
(config.num_experts, 2 * config.n, config.k),
device="cuda",
dtype=torch.bfloat16,
)
/ 15
)
w2_bf16 = (
torch.randn(
(config.num_experts, config.k, config.n),
device="cuda",
dtype=torch.bfloat16,
)
/ 15
)
torch.distributed.broadcast(w1_bf16, src=0, group=pg)
torch.distributed.broadcast(w2_bf16, src=0, group=pg)
# Round-trip through FP8 before constructing the reference and kernel weights.
w1_fp8 = w1_bf16.to(torch.float8_e4m3fn)
w2_fp8 = w2_bf16.to(torch.float8_e4m3fn)
w1_ref = w1_fp8.to(torch.bfloat16)
w2_ref = w2_fp8.to(torch.bfloat16)
@@ -385,21 +419,8 @@ def _deep_ep_v2_moe_cudagraph(
backend="nccl",
)
initialize_model_parallel(tensor_model_parallel_size=1)
# Reference MoE using dequantized bf16 weights
torch_combined = torch_experts(
test_tensors.rank_tokens,
w1_ref,
w2_ref,
test_tensors.topk_weights,
test_tensors.topk,
)
# Use the production pipeline: make_fused_moe_layer creates
# a FusedMoE layer, quantizes weights, runs
# process_weights_after_loading (TrtLLM W31 swap + BlockMajorK
# shuffle), and selects the kernel.
# Quantize weights using production helper, EP-slice, then
# convert to TrtLLM format.
# Mirror production weight processing: quantize, EP-slice, then
# convert to the TrtLLM BlockMajorK format.
from tests.kernels.moe.test_moe_layer import _quantize_fp8_halves
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsModular,
@@ -411,14 +432,32 @@ def _deep_ep_v2_moe_cudagraph(
block_shape = [128, 128]
qw = _quantize_fp8_halves(w1_ref, w2_ref, block_shape)
assert qw.w13_weight_scale is not None
assert qw.w2_weight_scale is not None
# Reference MoE using the same blockwise FP8 quantization scheme as
# the production kernel. torch_experts quantizes activations before
# both GEMMs and dequantizes the operands for the reference matmuls.
reference_topk_weights = test_tensors.topk_weights.to(torch.bfloat16).to(
torch.float32
)
torch_combined = torch_experts(
test_tensors.rank_tokens,
qw.w13_weight,
qw.w2_weight,
reference_topk_weights,
test_tensors.topk,
w1_scale=qw.w13_weight_scale,
w2_scale=qw.w2_weight_scale,
quant_dtype=torch.float8_e4m3fn,
block_shape=block_shape,
)
# EP-slice before format conversion
e_start = num_local_experts * pgi.rank
e_end = e_start + num_local_experts
w1_ep = qw.w13_weight[e_start:e_end]
w2_ep = qw.w2_weight[e_start:e_end]
assert qw.w13_weight_scale is not None
assert qw.w2_weight_scale is not None
w1_scale_ep = qw.w13_weight_scale[e_start:e_end]
w2_scale_ep = qw.w2_weight_scale[e_start:e_end]
@@ -452,11 +491,23 @@ def _deep_ep_v2_moe_cudagraph(
w2_scale=w2_scale_ep,
)
moe_config = make_dummy_moe_config(
num_experts=num_local_experts,
num_experts=config.num_experts,
num_local_experts=num_local_experts,
experts_per_token=config.topk,
hidden_dim=hidden_size,
intermediate_size=config.n,
)
moe_parallel_config = dataclasses.replace(
moe_config.moe_parallel_config,
ep_size=pgi.world_size,
ep_rank=pgi.rank,
use_ep=True,
all2all_backend="deepep_v2",
)
moe_config = dataclasses.replace(
moe_config,
moe_parallel_config=moe_parallel_config,
)
fused_experts = TrtLlmFp8ExpertsModular(
moe_config=moe_config,
quant_config=quant_config,
@@ -480,21 +531,21 @@ def _deep_ep_v2_moe_cudagraph(
mk_kernel = FusedMoEKernel(
prepare_finalize=a2a,
fused_experts=fused_experts,
inplace=False,
)
for _ in range(3):
out = mk_kernel.apply(
hidden_states=test_tensors.rank_tokens,
w1=w1_ep,
w2=w2_ep,
topk_weights=test_tensors.topk_weights,
topk_ids=test_tensors.topk,
activation=MoEActivation.SILU,
global_num_experts=config.num_experts,
expert_map=None,
apply_router_weight_on_input=False,
)
with set_forward_context(None, vllm_cfg):
for _ in range(3):
out = mk_kernel.apply(
hidden_states=test_tensors.rank_tokens,
w1=w1_ep,
w2=w2_ep,
topk_weights=test_tensors.topk_weights,
topk_ids=test_tensors.topk,
activation=MoEActivation.SILU,
global_num_experts=config.num_experts,
expert_map=None,
apply_router_weight_on_input=False,
)
torch.testing.assert_close(
torch_combined,