Modularize fused experts and integrate PPLX kernels (#15956)
This commit is contained in:
@@ -27,6 +27,7 @@ batchsize_forward_time: defaultdict = defaultdict(list)
|
||||
|
||||
@dataclass
|
||||
class DPMetadata:
|
||||
max_tokens_across_dp_cpu: torch.Tensor
|
||||
cu_tokens_across_dp_cpu: torch.Tensor
|
||||
|
||||
|
||||
@@ -90,8 +91,10 @@ def set_forward_context(attn_metadata: Any,
|
||||
dtype=torch.int32)
|
||||
from vllm.distributed.parallel_state import get_dp_group
|
||||
dist.all_reduce(num_tokens_tensor, group=get_dp_group().cpu_group)
|
||||
max_tokens_across_dp_cpu = torch.max(num_tokens_tensor)
|
||||
cu_tokens_across_dp_cpu = torch.cumsum(num_tokens_tensor, dim=0)
|
||||
dp_metadata = DPMetadata(cu_tokens_across_dp_cpu)
|
||||
dp_metadata = DPMetadata(max_tokens_across_dp_cpu,
|
||||
cu_tokens_across_dp_cpu)
|
||||
|
||||
global _forward_context
|
||||
prev_context = _forward_context
|
||||
|
||||
Reference in New Issue
Block a user