[Kernel] Pack output and LSE in DCP A2A (#41160)

This commit is contained in:
sungsoo ha
2026-05-01 09:01:17 -04:00
committed by GitHub
parent 2fa1f8ec00
commit 4f7bde572a
2 changed files with 555 additions and 157 deletions
+306 -3
View File
@@ -10,10 +10,95 @@ Tests cover:
import math
import multiprocess as mp
import pytest
import torch
import torch.distributed as dist
from vllm.config.parallel import ParallelConfig
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
mp.set_start_method("spawn", force=True)
class _FakeCPGroup:
def __init__(self, world_size: int, device_group: dist.ProcessGroup):
self.world_size = world_size
self.device_group = device_group
def _dtype_from_name(dtype_name: str) -> torch.dtype:
return {
"float16": torch.float16,
"bfloat16": torch.bfloat16,
"float32": torch.float32,
}[dtype_name]
def _packed_a2a_reference(
cp_attn_out: torch.Tensor,
cp_attn_lse: torch.Tensor,
world_size: int,
h_per_rank: int,
is_lse_base_on_e: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
B, _H, D = cp_attn_out.shape
outputs = (
cp_attn_out.view(B, world_size, h_per_rank, D)
.permute(1, 0, 2, 3)
.contiguous()
.float()
)
lses = cp_attn_lse.view(B, world_size, h_per_rank).permute(1, 0, 2).contiguous()
return _lse_weighted_combine(
outputs,
lses,
return_lse=True,
is_lse_base_on_e=is_lse_base_on_e,
)
def _assert_packed_a2a_close(
actual: torch.Tensor,
expected: torch.Tensor,
dtype: torch.dtype,
) -> None:
if dtype == torch.float32:
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
else:
torch.testing.assert_close(
actual.float(), expected.float(), rtol=3e-2, atol=3e-2
)
def _distributed_run(fn, world_size: int, extra_env: dict[str, str]) -> None:
port = str(get_open_port())
processes: list[mp.Process] = []
for rank in range(world_size):
env = {
"RANK": str(rank),
"LOCAL_RANK": str(rank),
"WORLD_SIZE": str(world_size),
"LOCAL_WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": port,
**extra_env,
}
process = mp.Process(target=fn, args=(env,))
processes.append(process)
process.start()
for process in processes:
process.join(timeout=120)
for process in processes:
if process.is_alive():
process.kill()
process.join()
assert process.exitcode == 0
class TestDCPCommBackendConfig:
@@ -38,14 +123,14 @@ class TestDCPCommBackendConfig:
"""A2A backend is valid when DCP > 1."""
config = ParallelConfig(
dcp_comm_backend="a2a",
tensor_parallel_size=8,
tensor_parallel_size=4,
decode_context_parallel_size=4,
)
assert config.dcp_comm_backend == "a2a"
def test_invalid_backend_rejected(self):
"""Invalid backend values are rejected."""
with pytest.raises(ValueError, match="must be one of"):
with pytest.raises(ValueError, match="must be one of|Input should be"):
ParallelConfig(
dcp_comm_backend="invalid",
)
@@ -134,7 +219,7 @@ class TestLSEWeightedCombine:
result = _lse_weighted_combine(outputs, lses)
assert result.shape == (B, H, D)
torch.testing.assert_close(result, outputs[1].squeeze(0), atol=1e-5, rtol=1e-5)
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
def test_mathematically_correct(self):
"""Verify mathematical correctness of LSE combination."""
@@ -187,6 +272,224 @@ class TestLSEWeightedCombine:
assert global_lse.shape == (B, H)
assert abs(global_lse.item() - expected_global_lse) < 1e-5
def test_base2_return_lse(self):
"""Base-2 LSE mode returns log2-sum-exp2 global LSE."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
outputs = torch.tensor(
[
[[[1.0, 2.0]]],
[[[3.0, 4.0]]],
]
)
lses = torch.tensor(
[
[[1.0]],
[[2.0]],
]
)
result, global_lse = _lse_weighted_combine(
outputs,
lses,
return_lse=True,
is_lse_base_on_e=False,
)
expected_global_lse = math.log2(2**1 + 2**2)
w0 = 2**1 / (2**1 + 2**2)
w1 = 2**2 / (2**1 + 2**2)
expected = torch.tensor([[[w0 * 1.0 + w1 * 3.0, w0 * 2.0 + w1 * 4.0]]])
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-5)
torch.testing.assert_close(
global_lse,
torch.tensor([[expected_global_lse]]),
rtol=1e-5,
atol=1e-5,
)
def test_lse_pack_dim(self):
"""Packed A2A stores one fp32 LSE in output-dtype lanes."""
from vllm.v1.attention.ops.dcp_alltoall import _dcp_a2a_lse_pack_dim
assert _dcp_a2a_lse_pack_dim(torch.bfloat16) == 2
assert _dcp_a2a_lse_pack_dim(torch.float16) == 2
assert _dcp_a2a_lse_pack_dim(torch.float32) == 1
class TestPackedA2AKernels:
@pytest.mark.skipif(
torch.accelerator.device_count() < 1, reason="CUDA is required."
)
@pytest.mark.parametrize("dtype_name", ["float16", "bfloat16", "float32"])
@pytest.mark.parametrize("return_lse", [False, True])
@pytest.mark.parametrize("is_lse_base_on_e", [False, True])
def test_pack_unpack_combine_matches_reference(
self,
dtype_name: str,
return_lse: bool,
is_lse_base_on_e: bool,
):
from vllm.v1.attention.ops.dcp_alltoall import (
_dcp_a2a_lse_pack_dim,
_dcp_a2a_pack_send,
_dcp_a2a_unpack_combine,
)
torch.manual_seed(0)
dtype = _dtype_from_name(dtype_name)
device = torch.device("cuda")
world_size, B, h_per_rank, D = 4, 7, 2, 32
H = world_size * h_per_rank
cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype)
cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32)
lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype)
send_buffer = torch.empty(
(world_size, B, h_per_rank, D + lse_pack_dim),
device=device,
dtype=dtype,
)
_dcp_a2a_pack_send(
cp_attn_out,
cp_attn_lse,
send_buffer,
world_size,
h_per_rank,
D,
lse_pack_dim,
)
actual = _dcp_a2a_unpack_combine(
send_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e
)
expected_out, expected_lse = _packed_a2a_reference(
cp_attn_out, cp_attn_lse, world_size, h_per_rank, is_lse_base_on_e
)
if return_lse:
actual_out, actual_lse = actual
_assert_packed_a2a_close(actual_out, expected_out, dtype)
torch.testing.assert_close(actual_lse, expected_lse, rtol=1e-4, atol=1e-4)
else:
_assert_packed_a2a_close(actual, expected_out, dtype)
def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
update_environment_variables(env)
local_rank = int(env["LOCAL_RANK"])
torch.accelerator.set_device_index(local_rank)
dist.init_process_group(backend="nccl")
use_workspace = env.get("USE_WORKSPACE") == "1"
if use_workspace:
from vllm.v1.worker.workspace import init_workspace_manager
init_workspace_manager(torch.device(f"cuda:{local_rank}"))
try:
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
dtype = _dtype_from_name(env["TEST_DTYPE"])
return_lse = env["RETURN_LSE"] == "1"
is_lse_base_on_e = env["LSE_BASE_E"] == "1"
rank = dist.get_rank()
world_size = dist.get_world_size()
B, h_per_rank, D = 5, 2, 32
H = world_size * h_per_rank
generator = torch.Generator(device=f"cuda:{local_rank}")
generator.manual_seed(1234 + rank)
cp_attn_out = torch.randn(
B,
H,
D,
device=f"cuda:{local_rank}",
dtype=dtype,
generator=generator,
)
cp_attn_lse = torch.randn(
B,
H,
device=f"cuda:{local_rank}",
dtype=torch.float32,
generator=generator,
)
actual = dcp_a2a_lse_reduce(
cp_attn_out,
cp_attn_lse,
_FakeCPGroup(world_size, dist.group.WORLD),
return_lse=return_lse,
is_lse_base_on_e=is_lse_base_on_e,
)
gathered_out = [torch.empty_like(cp_attn_out) for _ in range(world_size)]
gathered_lse = [torch.empty_like(cp_attn_lse) for _ in range(world_size)]
dist.all_gather(gathered_out, cp_attn_out)
dist.all_gather(gathered_lse, cp_attn_lse)
outputs = torch.stack(
[
t[:, rank * h_per_rank : (rank + 1) * h_per_rank, :]
for t in gathered_out
],
dim=0,
).float()
lses = torch.stack(
[t[:, rank * h_per_rank : (rank + 1) * h_per_rank] for t in gathered_lse],
dim=0,
)
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
expected_out, expected_lse = _lse_weighted_combine(
outputs,
lses,
return_lse=True,
is_lse_base_on_e=is_lse_base_on_e,
)
if return_lse:
actual_out, actual_lse = actual
_assert_packed_a2a_close(actual_out, expected_out, dtype)
torch.testing.assert_close(actual_lse, expected_lse, rtol=1e-4, atol=1e-4)
else:
_assert_packed_a2a_close(actual, expected_out, dtype)
finally:
if use_workspace:
from vllm.v1.worker.workspace import reset_workspace_manager
reset_workspace_manager()
dist.destroy_process_group()
@pytest.mark.skipif(
torch.accelerator.device_count() < 4, reason="Need at least 4 GPUs."
)
@pytest.mark.parametrize("dtype_name", ["float16", "bfloat16", "float32"])
def test_distributed_packed_a2a_matches_reference(dtype_name: str):
_distributed_run(
_distributed_packed_a2a_worker,
world_size=4,
extra_env={
"TEST_DTYPE": dtype_name,
"RETURN_LSE": "1",
"LSE_BASE_E": "1",
},
)
@pytest.mark.skipif(
torch.accelerator.device_count() < 4, reason="Need at least 4 GPUs."
)
def test_distributed_packed_a2a_with_workspace_matches_reference():
_distributed_run(
_distributed_packed_a2a_worker,
world_size=4,
extra_env={
"TEST_DTYPE": "bfloat16",
"RETURN_LSE": "1",
"LSE_BASE_E": "1",
"USE_WORKSPACE": "1",
},
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+249 -154
View File
@@ -9,10 +9,8 @@ Instead of gathering the full Q tensor and scattering partial outputs,
A2A exchanges partial attention outputs and their LSE values across
ranks, then combines them with exact LSE-weighted reduction.
This reduces the number of NCCL calls per attention layer from 3
(AG for Q, AG for K metadata, RS for output) to 2 (A2A for output,
A2A for LSE), lowering per-step communication overhead for long-context
decode where NCCL latency is a significant fraction of step time.
This reduces the number of NCCL calls per attention layer by exchanging
the partial output and LSE in a single packed All-to-All payload.
Usage:
vllm serve model --tp 16 --dcp 16 --dcp-comm-backend a2a
@@ -28,6 +26,10 @@ import torch
import torch.distributed as dist
from vllm.triton_utils import tl, triton
from vllm.v1.worker.workspace import (
current_workspace_manager,
is_workspace_manager_initialized,
)
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
@@ -44,7 +46,6 @@ def _lse_weighted_combine(
CPU reference implementation for LSE-weighted combination.
This is a pure PyTorch implementation used for testing and validation.
For GPU execution, use dcp_lse_combine_triton instead.
Args:
outputs: Partial attention outputs [N, B, H, D]
@@ -102,57 +103,137 @@ def _lse_weighted_combine(
return result
def _dcp_a2a_lse_pack_dim(output_dtype: torch.dtype) -> int:
bits = torch.finfo(output_dtype).bits
if bits == 16:
return 2
if bits == 32:
return 1
raise ValueError(f"Cannot pack fp32 LSE into output dtype {output_dtype}.")
def _dcp_a2a_send_recv_buffers(
shape: tuple[int, ...],
device: torch.device,
dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
if is_workspace_manager_initialized():
send_buffer, recv_buffer = current_workspace_manager().get_simultaneous(
(shape, dtype),
(shape, dtype),
)
return send_buffer, recv_buffer
return (
torch.empty(shape, device=device, dtype=dtype),
torch.empty(shape, device=device, dtype=dtype),
)
@triton.jit
def _dcp_lse_combine_kernel(
# Input pointers
recv_output_ptr,
recv_lse_ptr,
# Output pointers
def _dcp_a2a_pack_send_kernel(
out_ptr,
lse_ptr,
send_ptr,
out_stride_B,
out_stride_H,
out_stride_D,
lse_stride_B,
lse_stride_H,
send_stride_N,
send_stride_B,
send_stride_H,
send_stride_D,
N: tl.constexpr,
HEAD_DIM: tl.constexpr,
H_PER_RANK: tl.constexpr,
LSE_PACK_DIM: tl.constexpr,
):
batch_idx = tl.program_id(0).to(tl.int64)
local_head_idx = tl.program_id(1).to(tl.int64)
d_offsets = tl.arange(0, HEAD_DIM)
for rank_idx in tl.static_range(N):
src_head_idx = rank_idx * H_PER_RANK + local_head_idx
send_base = (
rank_idx * send_stride_N
+ batch_idx * send_stride_B
+ local_head_idx * send_stride_H
)
out_offsets = (
batch_idx * out_stride_B
+ src_head_idx * out_stride_H
+ d_offsets * out_stride_D
)
tl.store(
send_ptr + send_base + d_offsets * send_stride_D,
tl.load(out_ptr + out_offsets),
)
lse_val = tl.load(
lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H
)
if LSE_PACK_DIM == 1:
tl.store(
send_ptr + send_base + HEAD_DIM * send_stride_D,
lse_val.to(send_ptr.dtype.element_ty),
)
else:
lse_bits = lse_val.to(tl.uint32, bitcast=True)
lo = (lse_bits & 0xFFFF).to(tl.uint16)
hi = ((lse_bits >> 16) & 0xFFFF).to(tl.uint16)
tl.store(
send_ptr + send_base + HEAD_DIM * send_stride_D,
lo.to(send_ptr.dtype.element_ty, bitcast=True),
)
tl.store(
send_ptr + send_base + (HEAD_DIM + 1) * send_stride_D,
hi.to(send_ptr.dtype.element_ty, bitcast=True),
)
@triton.jit
def _dcp_a2a_unpack_combine_kernel(
recv_ptr,
out_ptr,
out_lse_ptr,
# Strides for recv_output [N, B, H_local, D]
ro_stride_N,
ro_stride_B,
ro_stride_H,
ro_stride_D,
# Strides for recv_lse [N, B, H_local]
rl_stride_N,
rl_stride_B,
rl_stride_H,
# Strides for output [B, H_local, D]
o_stride_B,
o_stride_H,
o_stride_D,
# Constants
recv_stride_N,
recv_stride_B,
recv_stride_H,
recv_stride_D,
out_stride_B,
out_stride_H,
out_stride_D,
out_lse_stride_B,
out_lse_stride_H,
N: tl.constexpr,
HEAD_DIM: tl.constexpr,
IS_BASE_E: tl.constexpr,
RETURN_LSE: tl.constexpr,
LSE_PACK_DIM: tl.constexpr,
):
"""
Triton kernel for LSE-weighted combination of partial attention outputs.
After All-to-All, each rank has:
- recv_output [N, B, H_local, D]: partial outputs from all KV shards
- recv_lse [N, B, H_local]: partial LSEs from all KV shards
This kernel computes the weighted combination locally (no communication).
Grid: (B, H_local)
Each program handles one (batch, head) and processes all D elements.
"""
batch_idx = tl.program_id(0).to(tl.int64)
head_idx = tl.program_id(1).to(tl.int64)
d_offsets = tl.arange(0, HEAD_DIM)
# Base offset for this (batch, head)
base_lse_offset = batch_idx * rl_stride_B + head_idx * rl_stride_H
base_out_offset = batch_idx * ro_stride_B + head_idx * ro_stride_H
# First pass: find max LSE for numerical stability
lse_max = -float("inf")
for n in tl.static_range(N):
lse_offset = n * rl_stride_N + base_lse_offset
lse_val = tl.load(recv_lse_ptr + lse_offset)
for rank_idx in tl.static_range(N):
recv_base = (
rank_idx * recv_stride_N
+ batch_idx * recv_stride_B
+ head_idx * recv_stride_H
)
if LSE_PACK_DIM == 1:
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
tl.float32
)
else:
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
lse_val = tl.where(
(lse_val != lse_val) | (lse_val == float("inf")),
-float("inf"),
@@ -162,11 +243,23 @@ def _dcp_lse_combine_kernel(
lse_max = tl.where(lse_max == -float("inf"), 0.0, lse_max)
# Second pass: compute sum of exp(lse - max)
lse_sum = 0.0
for n in tl.static_range(N):
lse_offset = n * rl_stride_N + base_lse_offset
lse_val = tl.load(recv_lse_ptr + lse_offset)
for rank_idx in tl.static_range(N):
recv_base = (
rank_idx * recv_stride_N
+ batch_idx * recv_stride_B
+ head_idx * recv_stride_H
)
if LSE_PACK_DIM == 1:
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
tl.float32
)
else:
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
lse_val = tl.where(
(lse_val != lse_val) | (lse_val == float("inf")),
-float("inf"),
@@ -177,19 +270,28 @@ def _dcp_lse_combine_kernel(
else:
lse_sum += tl.exp2(lse_val - lse_max)
# Compute global LSE
if IS_BASE_E: # noqa: SIM108
global_lse = tl.log(lse_sum) + lse_max
else:
global_lse = tl.log2(lse_sum) + lse_max
# Third pass: weighted combination across D dimension
d_offsets = tl.arange(0, HEAD_DIM)
acc = tl.zeros([HEAD_DIM], dtype=tl.float32)
for n in tl.static_range(N):
lse_offset = n * rl_stride_N + base_lse_offset
lse_val = tl.load(recv_lse_ptr + lse_offset)
for rank_idx in tl.static_range(N):
recv_base = (
rank_idx * recv_stride_N
+ batch_idx * recv_stride_B
+ head_idx * recv_stride_H
)
if LSE_PACK_DIM == 1:
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
tl.float32
)
else:
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
lse_val = tl.where(
(lse_val != lse_val) | (lse_val == float("inf")),
-float("inf"),
@@ -200,80 +302,89 @@ def _dcp_lse_combine_kernel(
else:
weight = tl.exp2(lse_val - global_lse)
weight = tl.where(weight != weight, 0.0, weight)
acc += (
tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(tl.float32)
* weight
)
out_offsets = n * ro_stride_N + base_out_offset + d_offsets * ro_stride_D
out_vals = tl.load(recv_output_ptr + out_offsets)
acc += out_vals.to(tl.float32) * weight
# Store result
final_offsets = (
batch_idx * o_stride_B + head_idx * o_stride_H + d_offsets * o_stride_D
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
)
tl.store(out_ptr + final_offsets, acc)
if RETURN_LSE:
tl.store(out_lse_ptr + base_lse_offset, global_lse)
out_lse_offset = batch_idx * out_lse_stride_B + head_idx * out_lse_stride_H
tl.store(out_lse_ptr + out_lse_offset, global_lse)
def dcp_lse_combine_triton(
recv_output: torch.Tensor,
recv_lse: torch.Tensor,
return_lse: bool = False,
is_lse_base_on_e: bool = True,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""
Triton-accelerated LSE-weighted combination for DCP A2A.
Args:
recv_output: [N, B, H_local, D] - partial outputs from all KV shards
recv_lse: [N, B, H_local] - partial LSEs from all KV shards
return_lse: If True, also return the global LSE
is_lse_base_on_e: If True, LSE is base e; if False, base 2
Returns:
Combined output [B, H_local, D]
If return_lse=True, also returns global_lse [B, H_local]
"""
N, B, H_local, D = recv_output.shape
out = torch.empty(
(B, H_local, D), device=recv_output.device, dtype=recv_output.dtype
def _dcp_a2a_pack_send(
cp_attn_out: torch.Tensor,
cp_attn_lse: torch.Tensor,
send_buffer: torch.Tensor,
world_size: int,
h_per_rank: int,
head_dim: int,
lse_pack_dim: int,
) -> None:
grid = (cp_attn_out.shape[0], h_per_rank, 1)
_dcp_a2a_pack_send_kernel[grid](
cp_attn_out,
cp_attn_lse,
send_buffer,
cp_attn_out.stride(0),
cp_attn_out.stride(1),
cp_attn_out.stride(2),
cp_attn_lse.stride(0),
cp_attn_lse.stride(1),
send_buffer.stride(0),
send_buffer.stride(1),
send_buffer.stride(2),
send_buffer.stride(3),
N=world_size,
HEAD_DIM=head_dim,
H_PER_RANK=h_per_rank,
LSE_PACK_DIM=lse_pack_dim,
)
if return_lse:
out_lse = torch.empty(
(B, H_local), device=recv_lse.device, dtype=recv_lse.dtype
)
else:
out_lse = torch.empty(1, device=recv_lse.device, dtype=recv_lse.dtype)
ro_stride_N, ro_stride_B, ro_stride_H, ro_stride_D = recv_output.stride()
rl_stride_N, rl_stride_B, rl_stride_H = recv_lse.stride()
o_stride_B, o_stride_H, o_stride_D = out.stride()
grid = (B, H_local, 1)
_dcp_lse_combine_kernel[grid](
recv_output,
recv_lse,
def _dcp_a2a_unpack_combine(
recv_buffer: torch.Tensor,
head_dim: int,
lse_pack_dim: int,
return_lse: bool,
is_lse_base_on_e: bool,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
world_size, num_tokens, h_per_rank, _ = recv_buffer.shape
out = torch.empty(
(num_tokens, h_per_rank, head_dim),
device=recv_buffer.device,
dtype=recv_buffer.dtype,
)
out_lse = torch.empty(
(num_tokens, h_per_rank) if return_lse else (1, 1),
device=recv_buffer.device,
dtype=torch.float32 if return_lse else recv_buffer.dtype,
)
grid = (num_tokens, h_per_rank, 1)
_dcp_a2a_unpack_combine_kernel[grid](
recv_buffer,
out,
out_lse,
ro_stride_N,
ro_stride_B,
ro_stride_H,
ro_stride_D,
rl_stride_N,
rl_stride_B,
rl_stride_H,
o_stride_B,
o_stride_H,
o_stride_D,
N=N,
HEAD_DIM=D,
recv_buffer.stride(0),
recv_buffer.stride(1),
recv_buffer.stride(2),
recv_buffer.stride(3),
out.stride(0),
out.stride(1),
out.stride(2),
out_lse.stride(0),
out_lse.stride(1),
N=world_size,
HEAD_DIM=head_dim,
IS_BASE_E=is_lse_base_on_e,
RETURN_LSE=return_lse,
LSE_PACK_DIM=lse_pack_dim,
)
if return_lse:
return out, out_lse
return out
@@ -290,17 +401,8 @@ def dcp_a2a_lse_reduce(
"""
Combine partial attention outputs across DCP ranks using All-to-All.
Each rank holds attention output for all heads but only a local shard
of the KV cache. This function:
1. Exchanges partial outputs across ranks via All-to-All
2. Exchanges LSE values via All-to-All
3. Combines them with exact LSE-weighted reduction (Triton kernel)
Tensor flow:
Input: cp_attn_out [B, H, D] - all heads, local KV shard
Reshape: [N, B, H/N, D] - split heads across ranks
A2A: Two all_to_all_single calls (output and LSE)
Combine: recv [N, B, H/N, D] + lse [N, B, H/N] -> [B, H/N, D]
The output and fp32 LSE are packed into a single output-dtype buffer, sent
with one All-to-All, then unpacked and combined with exact LSE weighting.
Args:
cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim
@@ -321,43 +423,36 @@ def dcp_a2a_lse_reduce(
return cp_attn_out, cp_attn_lse
return cp_attn_out
local_output = cp_attn_out.contiguous()
local_lse = cp_attn_lse.contiguous()
B, H, D = local_output.shape
B, H, D = cp_attn_out.shape
if H % world_size != 0:
raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.")
H_per_rank = H // world_size
lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype)
# Reshape for All-to-All: [B, H, D] -> [N, B, H/N, D]
# Split heads into N chunks, each destined for a different rank
send_output = (
local_output.view(B, world_size, H_per_rank, D).permute(1, 0, 2, 3).contiguous()
send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers(
(world_size, B, H_per_rank, D + lse_pack_dim),
device=cp_attn_out.device,
dtype=cp_attn_out.dtype,
)
recv_output = torch.empty_like(send_output)
# Same for LSE: [B, H] -> [N, B, H/N]
send_lse = local_lse.view(B, world_size, H_per_rank).permute(1, 0, 2).contiguous()
recv_lse = torch.empty_like(send_lse)
_dcp_a2a_pack_send(
cp_attn_out,
cp_attn_lse,
send_buffer,
world_size,
H_per_rank,
D,
lse_pack_dim,
)
# All-to-All for partial attention outputs and LSE values (async overlap)
work_output = dist.all_to_all_single(
recv_output.view(-1),
send_output.view(-1),
work = dist.all_to_all_single(
recv_buffer.view(-1),
send_buffer.view(-1),
group=cp_group.device_group,
async_op=True,
)
work_lse = dist.all_to_all_single(
recv_lse.view(-1),
send_lse.view(-1),
group=cp_group.device_group,
async_op=True,
)
work_output.wait()
work_lse.wait()
work.wait()
# LSE-weighted combination via Triton kernel (local, no communication)
return dcp_lse_combine_triton(
recv_output,
recv_lse,
return_lse=return_lse,
is_lse_base_on_e=is_lse_base_on_e,
return _dcp_a2a_unpack_combine(
recv_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e
)