DCP supports hybrid attention (#40996)

Signed-off-by: YanXu <yancey.yx@alibaba-inc.com>
Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
Yan Xu
2026-07-09 21:34:45 -07:00
committed by GitHub
co-authored by Jingyi Yang
parent 2d814a0082
commit 95ed0feaa5
26 changed files with 783 additions and 125 deletions
@@ -341,13 +341,30 @@ class CudaCommunicator(DeviceCommunicatorBase):
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
# Route uniform dim-0 all-gathers through NVLS symmetric memory when
# enabled (mirrors reduce_scatter); otherwise fall back to the
# base-class ring all-gather. Sequence parallelism's gather-before-GEMM
# uses dim=0 with tp-aligned (uniform) shards.
# PyNccl/base-class all-gather. Sequence parallelism's
# gather-before-GEMM uses dim=0 with tp-aligned (uniform) shards.
if dim < 0:
dim += input_.dim()
if dim == 0 and should_nccl_symm_mem_ag_rs():
return self._all_gather_symm_mem(input_.contiguous())
return super().all_gather(input_, dim)
pynccl_comm = self.pynccl_comm
if pynccl_comm is None or pynccl_comm.disabled:
return super().all_gather(input_, dim)
input_size = input_.size()
output_size = (input_size[0] * self.world_size,) + input_size[1:]
output_tensor = torch.empty(
output_size, dtype=input_.dtype, device=input_.device
)
pynccl_comm.all_gather(output_tensor, input_.contiguous())
output_tensor = output_tensor.reshape((self.world_size,) + input_size)
output_tensor = output_tensor.movedim(0, dim)
return output_tensor.reshape(
input_size[:dim]
+ (self.world_size * input_size[dim],)
+ input_size[dim + 1 :]
)
def reduce_scatter(self, input_: torch.Tensor, dim: int = -1):
world_size = self.world_size