[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -623,7 +623,7 @@ def make_modular_kernel(
|
||||
num_experts=config.E,
|
||||
experts_per_token=config.topk,
|
||||
hidden_dim=config.K,
|
||||
intermediate_size_per_partition=config.N,
|
||||
intermediate_size=config.N,
|
||||
num_local_experts=config.num_local_experts,
|
||||
num_logical_experts=config.E,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
|
||||
@@ -88,6 +88,7 @@ def _worker_parallel_launch(
|
||||
rank = node_rank * world_local_size + local_rank
|
||||
device = torch.device("cuda", local_rank)
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.distributed.init_process_group(
|
||||
backend="cpu:gloo,cuda:nccl",
|
||||
init_method=init_method,
|
||||
@@ -125,6 +126,7 @@ def _worker_parallel_launch(
|
||||
traceback.print_exc()
|
||||
raise
|
||||
finally:
|
||||
torch.accelerator.synchronize()
|
||||
if vllm_config is not None:
|
||||
cleanup_dist_env_and_memory()
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user