[WideEP] Integrate DeepEP v2 (#41183)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
05cb606cad
commit
e2f993dc41
@@ -15,7 +15,7 @@ from torch.distributed import ProcessGroup
|
||||
from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUsage]
|
||||
from typing_extensions import ParamSpec
|
||||
|
||||
from vllm.utils.import_utils import has_deep_ep
|
||||
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
if has_deep_ep():
|
||||
@@ -26,6 +26,11 @@ if has_deep_ep():
|
||||
DeepEPLLPrepareAndFinalize,
|
||||
)
|
||||
|
||||
if has_deep_ep_v2():
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
|
||||
DeepEPV2PrepareAndFinalize,
|
||||
)
|
||||
|
||||
## Parallel Processes Utils
|
||||
|
||||
P = ParamSpec("P")
|
||||
@@ -55,11 +60,10 @@ def _worker_parallel_launch(
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
device = torch.device("cuda", local_rank)
|
||||
torch.distributed.init_process_group(
|
||||
backend="cpu:gloo,cuda:nccl",
|
||||
backend="nccl",
|
||||
init_method=init_method,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device_id=device,
|
||||
)
|
||||
barrier = torch.tensor([rank], device=device)
|
||||
torch.distributed.all_reduce(barrier)
|
||||
@@ -200,3 +204,42 @@ def make_deepep_a2a(
|
||||
|
||||
assert deepep_ll_args is not None
|
||||
return make_deepep_ll_a2a(pg, pgi, deepep_ll_args, q_dtype, block_shape)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DeepEPV2Args:
|
||||
num_local_experts: int
|
||||
num_experts: int
|
||||
num_topk: int
|
||||
hidden_size: int
|
||||
max_tokens_per_rank: int
|
||||
use_fp8_dispatch: bool
|
||||
|
||||
|
||||
def make_deepep_v2_a2a(
|
||||
pg: ProcessGroup,
|
||||
pgi: ProcessGroupInfo,
|
||||
dp_size: int,
|
||||
v2_args: DeepEPV2Args,
|
||||
use_cudagraph: bool = False,
|
||||
):
|
||||
import deep_ep
|
||||
|
||||
buffer = deep_ep.ElasticBuffer(
|
||||
group=pg,
|
||||
num_max_tokens_per_rank=v2_args.max_tokens_per_rank,
|
||||
hidden=v2_args.hidden_size,
|
||||
num_topk=v2_args.num_topk,
|
||||
use_fp8_dispatch=v2_args.use_fp8_dispatch,
|
||||
explicitly_destroy=True,
|
||||
)
|
||||
return DeepEPV2PrepareAndFinalize(
|
||||
buffer=buffer,
|
||||
num_dispatchers=pgi.world_size,
|
||||
dp_size=dp_size,
|
||||
rank_expert_offset=pgi.rank * v2_args.num_local_experts,
|
||||
num_experts=v2_args.num_experts,
|
||||
num_topk=v2_args.num_topk,
|
||||
use_fp8_dispatch=v2_args.use_fp8_dispatch,
|
||||
use_cudagraph=use_cudagraph,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user