DCP supports hybrid attention (#40996)

Signed-off-by: YanXu <yancey.yx@alibaba-inc.com>
Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
Yan Xu
2026-07-09 21:34:45 -07:00
committed by GitHub
co-authored by Jingyi Yang
parent 2d814a0082
commit 95ed0feaa5
26 changed files with 783 additions and 125 deletions
+47
View File
@@ -16,6 +16,7 @@ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
from vllm.distributed.parallel_state import (
ensure_model_parallel_initialized,
get_tp_group,
get_world_group,
graph_capture,
init_distributed_environment,
@@ -199,6 +200,52 @@ def test_pynccl_all_gather():
distributed_run(all_gather_worker_fn, 2)
@worker_fn_wrapper
def cuda_communicator_all_gather_dim_worker_fn():
with ensure_current_vllm_config():
ensure_model_parallel_initialized(2, 1)
tp_group = get_tp_group()
comm = tp_group.device_communicator
assert comm is not None
rank = tp_group.rank_in_group
world_size = tp_group.world_size
device = tp_group.device
shape = (2, 3, 4)
num_elems = 1
for size in shape:
num_elems *= size
for dim in (1, -1):
tensor = (
torch.arange(num_elems, dtype=torch.float32, device=device).reshape(shape)
+ rank * num_elems
)
expected = torch.cat(
[
torch.arange(num_elems, dtype=torch.float32, device=device).reshape(
shape
)
+ r * num_elems
for r in range(world_size)
],
dim=dim,
)
result = comm.all_gather(tensor, dim=dim)
torch.accelerator.synchronize()
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
@pytest.mark.skipif(
torch.accelerator.device_count() < 2, reason="Need at least 2 GPUs to run the test."
)
def test_cuda_communicator_all_gather_dim_not_zero():
distributed_run(cuda_communicator_all_gather_dim_worker_fn, 2)
@worker_fn_wrapper
def all_gatherv_worker_fn():
pynccl_comm = PyNcclCommunicator(