From 4f7bde572ad05a6c013dde5f8874898fff3c1253 Mon Sep 17 00:00:00 2001 From: sungsoo ha Date: Fri, 1 May 2026 06:01:17 -0700 Subject: [PATCH] [Kernel] Pack output and LSE in DCP A2A (#41160) --- tests/distributed/test_dcp_a2a.py | 309 +++++++++++++++++++- vllm/v1/attention/ops/dcp_alltoall.py | 403 ++++++++++++++++---------- 2 files changed, 555 insertions(+), 157 deletions(-) diff --git a/tests/distributed/test_dcp_a2a.py b/tests/distributed/test_dcp_a2a.py index 2f92413e58..d80ed36be6 100644 --- a/tests/distributed/test_dcp_a2a.py +++ b/tests/distributed/test_dcp_a2a.py @@ -10,10 +10,95 @@ Tests cover: import math +import multiprocess as mp import pytest import torch +import torch.distributed as dist from vllm.config.parallel import ParallelConfig +from vllm.utils.network_utils import get_open_port +from vllm.utils.system_utils import update_environment_variables + +mp.set_start_method("spawn", force=True) + + +class _FakeCPGroup: + def __init__(self, world_size: int, device_group: dist.ProcessGroup): + self.world_size = world_size + self.device_group = device_group + + +def _dtype_from_name(dtype_name: str) -> torch.dtype: + return { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + }[dtype_name] + + +def _packed_a2a_reference( + cp_attn_out: torch.Tensor, + cp_attn_lse: torch.Tensor, + world_size: int, + h_per_rank: int, + is_lse_base_on_e: bool, +) -> tuple[torch.Tensor, torch.Tensor]: + from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + + B, _H, D = cp_attn_out.shape + outputs = ( + cp_attn_out.view(B, world_size, h_per_rank, D) + .permute(1, 0, 2, 3) + .contiguous() + .float() + ) + lses = cp_attn_lse.view(B, world_size, h_per_rank).permute(1, 0, 2).contiguous() + return _lse_weighted_combine( + outputs, + lses, + return_lse=True, + is_lse_base_on_e=is_lse_base_on_e, + ) + + +def _assert_packed_a2a_close( + actual: torch.Tensor, + expected: torch.Tensor, + dtype: torch.dtype, +) -> None: + if dtype == torch.float32: + torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5) + else: + torch.testing.assert_close( + actual.float(), expected.float(), rtol=3e-2, atol=3e-2 + ) + + +def _distributed_run(fn, world_size: int, extra_env: dict[str, str]) -> None: + port = str(get_open_port()) + processes: list[mp.Process] = [] + for rank in range(world_size): + env = { + "RANK": str(rank), + "LOCAL_RANK": str(rank), + "WORLD_SIZE": str(world_size), + "LOCAL_WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": port, + **extra_env, + } + process = mp.Process(target=fn, args=(env,)) + processes.append(process) + process.start() + + for process in processes: + process.join(timeout=120) + + for process in processes: + if process.is_alive(): + process.kill() + process.join() + assert process.exitcode == 0 class TestDCPCommBackendConfig: @@ -38,14 +123,14 @@ class TestDCPCommBackendConfig: """A2A backend is valid when DCP > 1.""" config = ParallelConfig( dcp_comm_backend="a2a", - tensor_parallel_size=8, + tensor_parallel_size=4, decode_context_parallel_size=4, ) assert config.dcp_comm_backend == "a2a" def test_invalid_backend_rejected(self): """Invalid backend values are rejected.""" - with pytest.raises(ValueError, match="must be one of"): + with pytest.raises(ValueError, match="must be one of|Input should be"): ParallelConfig( dcp_comm_backend="invalid", ) @@ -134,7 +219,7 @@ class TestLSEWeightedCombine: result = _lse_weighted_combine(outputs, lses) assert result.shape == (B, H, D) - torch.testing.assert_close(result, outputs[1].squeeze(0), atol=1e-5, rtol=1e-5) + torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5) def test_mathematically_correct(self): """Verify mathematical correctness of LSE combination.""" @@ -187,6 +272,224 @@ class TestLSEWeightedCombine: assert global_lse.shape == (B, H) assert abs(global_lse.item() - expected_global_lse) < 1e-5 + def test_base2_return_lse(self): + """Base-2 LSE mode returns log2-sum-exp2 global LSE.""" + from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + + outputs = torch.tensor( + [ + [[[1.0, 2.0]]], + [[[3.0, 4.0]]], + ] + ) + lses = torch.tensor( + [ + [[1.0]], + [[2.0]], + ] + ) + + result, global_lse = _lse_weighted_combine( + outputs, + lses, + return_lse=True, + is_lse_base_on_e=False, + ) + + expected_global_lse = math.log2(2**1 + 2**2) + w0 = 2**1 / (2**1 + 2**2) + w1 = 2**2 / (2**1 + 2**2) + expected = torch.tensor([[[w0 * 1.0 + w1 * 3.0, w0 * 2.0 + w1 * 4.0]]]) + + torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-5) + torch.testing.assert_close( + global_lse, + torch.tensor([[expected_global_lse]]), + rtol=1e-5, + atol=1e-5, + ) + + def test_lse_pack_dim(self): + """Packed A2A stores one fp32 LSE in output-dtype lanes.""" + from vllm.v1.attention.ops.dcp_alltoall import _dcp_a2a_lse_pack_dim + + assert _dcp_a2a_lse_pack_dim(torch.bfloat16) == 2 + assert _dcp_a2a_lse_pack_dim(torch.float16) == 2 + assert _dcp_a2a_lse_pack_dim(torch.float32) == 1 + + +class TestPackedA2AKernels: + @pytest.mark.skipif( + torch.accelerator.device_count() < 1, reason="CUDA is required." + ) + @pytest.mark.parametrize("dtype_name", ["float16", "bfloat16", "float32"]) + @pytest.mark.parametrize("return_lse", [False, True]) + @pytest.mark.parametrize("is_lse_base_on_e", [False, True]) + def test_pack_unpack_combine_matches_reference( + self, + dtype_name: str, + return_lse: bool, + is_lse_base_on_e: bool, + ): + from vllm.v1.attention.ops.dcp_alltoall import ( + _dcp_a2a_lse_pack_dim, + _dcp_a2a_pack_send, + _dcp_a2a_unpack_combine, + ) + + torch.manual_seed(0) + dtype = _dtype_from_name(dtype_name) + device = torch.device("cuda") + world_size, B, h_per_rank, D = 4, 7, 2, 32 + H = world_size * h_per_rank + cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype) + cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32) + lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype) + send_buffer = torch.empty( + (world_size, B, h_per_rank, D + lse_pack_dim), + device=device, + dtype=dtype, + ) + + _dcp_a2a_pack_send( + cp_attn_out, + cp_attn_lse, + send_buffer, + world_size, + h_per_rank, + D, + lse_pack_dim, + ) + actual = _dcp_a2a_unpack_combine( + send_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e + ) + expected_out, expected_lse = _packed_a2a_reference( + cp_attn_out, cp_attn_lse, world_size, h_per_rank, is_lse_base_on_e + ) + + if return_lse: + actual_out, actual_lse = actual + _assert_packed_a2a_close(actual_out, expected_out, dtype) + torch.testing.assert_close(actual_lse, expected_lse, rtol=1e-4, atol=1e-4) + else: + _assert_packed_a2a_close(actual, expected_out, dtype) + + +def _distributed_packed_a2a_worker(env: dict[str, str]) -> None: + update_environment_variables(env) + local_rank = int(env["LOCAL_RANK"]) + torch.accelerator.set_device_index(local_rank) + dist.init_process_group(backend="nccl") + use_workspace = env.get("USE_WORKSPACE") == "1" + if use_workspace: + from vllm.v1.worker.workspace import init_workspace_manager + + init_workspace_manager(torch.device(f"cuda:{local_rank}")) + try: + from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce + + dtype = _dtype_from_name(env["TEST_DTYPE"]) + return_lse = env["RETURN_LSE"] == "1" + is_lse_base_on_e = env["LSE_BASE_E"] == "1" + rank = dist.get_rank() + world_size = dist.get_world_size() + B, h_per_rank, D = 5, 2, 32 + H = world_size * h_per_rank + + generator = torch.Generator(device=f"cuda:{local_rank}") + generator.manual_seed(1234 + rank) + cp_attn_out = torch.randn( + B, + H, + D, + device=f"cuda:{local_rank}", + dtype=dtype, + generator=generator, + ) + cp_attn_lse = torch.randn( + B, + H, + device=f"cuda:{local_rank}", + dtype=torch.float32, + generator=generator, + ) + actual = dcp_a2a_lse_reduce( + cp_attn_out, + cp_attn_lse, + _FakeCPGroup(world_size, dist.group.WORLD), + return_lse=return_lse, + is_lse_base_on_e=is_lse_base_on_e, + ) + + gathered_out = [torch.empty_like(cp_attn_out) for _ in range(world_size)] + gathered_lse = [torch.empty_like(cp_attn_lse) for _ in range(world_size)] + dist.all_gather(gathered_out, cp_attn_out) + dist.all_gather(gathered_lse, cp_attn_lse) + outputs = torch.stack( + [ + t[:, rank * h_per_rank : (rank + 1) * h_per_rank, :] + for t in gathered_out + ], + dim=0, + ).float() + lses = torch.stack( + [t[:, rank * h_per_rank : (rank + 1) * h_per_rank] for t in gathered_lse], + dim=0, + ) + from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + + expected_out, expected_lse = _lse_weighted_combine( + outputs, + lses, + return_lse=True, + is_lse_base_on_e=is_lse_base_on_e, + ) + + if return_lse: + actual_out, actual_lse = actual + _assert_packed_a2a_close(actual_out, expected_out, dtype) + torch.testing.assert_close(actual_lse, expected_lse, rtol=1e-4, atol=1e-4) + else: + _assert_packed_a2a_close(actual, expected_out, dtype) + finally: + if use_workspace: + from vllm.v1.worker.workspace import reset_workspace_manager + + reset_workspace_manager() + dist.destroy_process_group() + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 4, reason="Need at least 4 GPUs." +) +@pytest.mark.parametrize("dtype_name", ["float16", "bfloat16", "float32"]) +def test_distributed_packed_a2a_matches_reference(dtype_name: str): + _distributed_run( + _distributed_packed_a2a_worker, + world_size=4, + extra_env={ + "TEST_DTYPE": dtype_name, + "RETURN_LSE": "1", + "LSE_BASE_E": "1", + }, + ) + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 4, reason="Need at least 4 GPUs." +) +def test_distributed_packed_a2a_with_workspace_matches_reference(): + _distributed_run( + _distributed_packed_a2a_worker, + world_size=4, + extra_env={ + "TEST_DTYPE": "bfloat16", + "RETURN_LSE": "1", + "LSE_BASE_E": "1", + "USE_WORKSPACE": "1", + }, + ) + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/vllm/v1/attention/ops/dcp_alltoall.py b/vllm/v1/attention/ops/dcp_alltoall.py index 92f50f63e3..1469a5c754 100644 --- a/vllm/v1/attention/ops/dcp_alltoall.py +++ b/vllm/v1/attention/ops/dcp_alltoall.py @@ -9,10 +9,8 @@ Instead of gathering the full Q tensor and scattering partial outputs, A2A exchanges partial attention outputs and their LSE values across ranks, then combines them with exact LSE-weighted reduction. -This reduces the number of NCCL calls per attention layer from 3 -(AG for Q, AG for K metadata, RS for output) to 2 (A2A for output, -A2A for LSE), lowering per-step communication overhead for long-context -decode where NCCL latency is a significant fraction of step time. +This reduces the number of NCCL calls per attention layer by exchanging +the partial output and LSE in a single packed All-to-All payload. Usage: vllm serve model --tp 16 --dcp 16 --dcp-comm-backend a2a @@ -28,6 +26,10 @@ import torch import torch.distributed as dist from vllm.triton_utils import tl, triton +from vllm.v1.worker.workspace import ( + current_workspace_manager, + is_workspace_manager_initialized, +) if TYPE_CHECKING: from vllm.distributed.parallel_state import GroupCoordinator @@ -44,7 +46,6 @@ def _lse_weighted_combine( CPU reference implementation for LSE-weighted combination. This is a pure PyTorch implementation used for testing and validation. - For GPU execution, use dcp_lse_combine_triton instead. Args: outputs: Partial attention outputs [N, B, H, D] @@ -102,57 +103,137 @@ def _lse_weighted_combine( return result +def _dcp_a2a_lse_pack_dim(output_dtype: torch.dtype) -> int: + bits = torch.finfo(output_dtype).bits + if bits == 16: + return 2 + if bits == 32: + return 1 + raise ValueError(f"Cannot pack fp32 LSE into output dtype {output_dtype}.") + + +def _dcp_a2a_send_recv_buffers( + shape: tuple[int, ...], + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor]: + if is_workspace_manager_initialized(): + send_buffer, recv_buffer = current_workspace_manager().get_simultaneous( + (shape, dtype), + (shape, dtype), + ) + return send_buffer, recv_buffer + + return ( + torch.empty(shape, device=device, dtype=dtype), + torch.empty(shape, device=device, dtype=dtype), + ) + + @triton.jit -def _dcp_lse_combine_kernel( - # Input pointers - recv_output_ptr, - recv_lse_ptr, - # Output pointers +def _dcp_a2a_pack_send_kernel( + out_ptr, + lse_ptr, + send_ptr, + out_stride_B, + out_stride_H, + out_stride_D, + lse_stride_B, + lse_stride_H, + send_stride_N, + send_stride_B, + send_stride_H, + send_stride_D, + N: tl.constexpr, + HEAD_DIM: tl.constexpr, + H_PER_RANK: tl.constexpr, + LSE_PACK_DIM: tl.constexpr, +): + batch_idx = tl.program_id(0).to(tl.int64) + local_head_idx = tl.program_id(1).to(tl.int64) + d_offsets = tl.arange(0, HEAD_DIM) + + for rank_idx in tl.static_range(N): + src_head_idx = rank_idx * H_PER_RANK + local_head_idx + send_base = ( + rank_idx * send_stride_N + + batch_idx * send_stride_B + + local_head_idx * send_stride_H + ) + + out_offsets = ( + batch_idx * out_stride_B + + src_head_idx * out_stride_H + + d_offsets * out_stride_D + ) + tl.store( + send_ptr + send_base + d_offsets * send_stride_D, + tl.load(out_ptr + out_offsets), + ) + + lse_val = tl.load( + lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H + ) + if LSE_PACK_DIM == 1: + tl.store( + send_ptr + send_base + HEAD_DIM * send_stride_D, + lse_val.to(send_ptr.dtype.element_ty), + ) + else: + lse_bits = lse_val.to(tl.uint32, bitcast=True) + lo = (lse_bits & 0xFFFF).to(tl.uint16) + hi = ((lse_bits >> 16) & 0xFFFF).to(tl.uint16) + tl.store( + send_ptr + send_base + HEAD_DIM * send_stride_D, + lo.to(send_ptr.dtype.element_ty, bitcast=True), + ) + tl.store( + send_ptr + send_base + (HEAD_DIM + 1) * send_stride_D, + hi.to(send_ptr.dtype.element_ty, bitcast=True), + ) + + +@triton.jit +def _dcp_a2a_unpack_combine_kernel( + recv_ptr, out_ptr, out_lse_ptr, - # Strides for recv_output [N, B, H_local, D] - ro_stride_N, - ro_stride_B, - ro_stride_H, - ro_stride_D, - # Strides for recv_lse [N, B, H_local] - rl_stride_N, - rl_stride_B, - rl_stride_H, - # Strides for output [B, H_local, D] - o_stride_B, - o_stride_H, - o_stride_D, - # Constants + recv_stride_N, + recv_stride_B, + recv_stride_H, + recv_stride_D, + out_stride_B, + out_stride_H, + out_stride_D, + out_lse_stride_B, + out_lse_stride_H, N: tl.constexpr, HEAD_DIM: tl.constexpr, IS_BASE_E: tl.constexpr, RETURN_LSE: tl.constexpr, + LSE_PACK_DIM: tl.constexpr, ): - """ - Triton kernel for LSE-weighted combination of partial attention outputs. - - After All-to-All, each rank has: - - recv_output [N, B, H_local, D]: partial outputs from all KV shards - - recv_lse [N, B, H_local]: partial LSEs from all KV shards - - This kernel computes the weighted combination locally (no communication). - - Grid: (B, H_local) - Each program handles one (batch, head) and processes all D elements. - """ batch_idx = tl.program_id(0).to(tl.int64) head_idx = tl.program_id(1).to(tl.int64) + d_offsets = tl.arange(0, HEAD_DIM) - # Base offset for this (batch, head) - base_lse_offset = batch_idx * rl_stride_B + head_idx * rl_stride_H - base_out_offset = batch_idx * ro_stride_B + head_idx * ro_stride_H - - # First pass: find max LSE for numerical stability lse_max = -float("inf") - for n in tl.static_range(N): - lse_offset = n * rl_stride_N + base_lse_offset - lse_val = tl.load(recv_lse_ptr + lse_offset) + for rank_idx in tl.static_range(N): + recv_base = ( + rank_idx * recv_stride_N + + batch_idx * recv_stride_B + + head_idx * recv_stride_H + ) + if LSE_PACK_DIM == 1: + lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( + tl.float32 + ) + else: + lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) + hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) + lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) lse_val = tl.where( (lse_val != lse_val) | (lse_val == float("inf")), -float("inf"), @@ -162,11 +243,23 @@ def _dcp_lse_combine_kernel( lse_max = tl.where(lse_max == -float("inf"), 0.0, lse_max) - # Second pass: compute sum of exp(lse - max) lse_sum = 0.0 - for n in tl.static_range(N): - lse_offset = n * rl_stride_N + base_lse_offset - lse_val = tl.load(recv_lse_ptr + lse_offset) + for rank_idx in tl.static_range(N): + recv_base = ( + rank_idx * recv_stride_N + + batch_idx * recv_stride_B + + head_idx * recv_stride_H + ) + if LSE_PACK_DIM == 1: + lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( + tl.float32 + ) + else: + lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) + hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) + lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) lse_val = tl.where( (lse_val != lse_val) | (lse_val == float("inf")), -float("inf"), @@ -177,19 +270,28 @@ def _dcp_lse_combine_kernel( else: lse_sum += tl.exp2(lse_val - lse_max) - # Compute global LSE if IS_BASE_E: # noqa: SIM108 global_lse = tl.log(lse_sum) + lse_max else: global_lse = tl.log2(lse_sum) + lse_max - # Third pass: weighted combination across D dimension - d_offsets = tl.arange(0, HEAD_DIM) acc = tl.zeros([HEAD_DIM], dtype=tl.float32) - - for n in tl.static_range(N): - lse_offset = n * rl_stride_N + base_lse_offset - lse_val = tl.load(recv_lse_ptr + lse_offset) + for rank_idx in tl.static_range(N): + recv_base = ( + rank_idx * recv_stride_N + + batch_idx * recv_stride_B + + head_idx * recv_stride_H + ) + if LSE_PACK_DIM == 1: + lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( + tl.float32 + ) + else: + lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) + hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) + lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) lse_val = tl.where( (lse_val != lse_val) | (lse_val == float("inf")), -float("inf"), @@ -200,80 +302,89 @@ def _dcp_lse_combine_kernel( else: weight = tl.exp2(lse_val - global_lse) weight = tl.where(weight != weight, 0.0, weight) + acc += ( + tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(tl.float32) + * weight + ) - out_offsets = n * ro_stride_N + base_out_offset + d_offsets * ro_stride_D - out_vals = tl.load(recv_output_ptr + out_offsets) - acc += out_vals.to(tl.float32) * weight - - # Store result final_offsets = ( - batch_idx * o_stride_B + head_idx * o_stride_H + d_offsets * o_stride_D + batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D ) tl.store(out_ptr + final_offsets, acc) if RETURN_LSE: - tl.store(out_lse_ptr + base_lse_offset, global_lse) + out_lse_offset = batch_idx * out_lse_stride_B + head_idx * out_lse_stride_H + tl.store(out_lse_ptr + out_lse_offset, global_lse) -def dcp_lse_combine_triton( - recv_output: torch.Tensor, - recv_lse: torch.Tensor, - return_lse: bool = False, - is_lse_base_on_e: bool = True, -) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """ - Triton-accelerated LSE-weighted combination for DCP A2A. - - Args: - recv_output: [N, B, H_local, D] - partial outputs from all KV shards - recv_lse: [N, B, H_local] - partial LSEs from all KV shards - return_lse: If True, also return the global LSE - is_lse_base_on_e: If True, LSE is base e; if False, base 2 - - Returns: - Combined output [B, H_local, D] - If return_lse=True, also returns global_lse [B, H_local] - """ - N, B, H_local, D = recv_output.shape - - out = torch.empty( - (B, H_local, D), device=recv_output.device, dtype=recv_output.dtype +def _dcp_a2a_pack_send( + cp_attn_out: torch.Tensor, + cp_attn_lse: torch.Tensor, + send_buffer: torch.Tensor, + world_size: int, + h_per_rank: int, + head_dim: int, + lse_pack_dim: int, +) -> None: + grid = (cp_attn_out.shape[0], h_per_rank, 1) + _dcp_a2a_pack_send_kernel[grid]( + cp_attn_out, + cp_attn_lse, + send_buffer, + cp_attn_out.stride(0), + cp_attn_out.stride(1), + cp_attn_out.stride(2), + cp_attn_lse.stride(0), + cp_attn_lse.stride(1), + send_buffer.stride(0), + send_buffer.stride(1), + send_buffer.stride(2), + send_buffer.stride(3), + N=world_size, + HEAD_DIM=head_dim, + H_PER_RANK=h_per_rank, + LSE_PACK_DIM=lse_pack_dim, ) - if return_lse: - out_lse = torch.empty( - (B, H_local), device=recv_lse.device, dtype=recv_lse.dtype - ) - else: - out_lse = torch.empty(1, device=recv_lse.device, dtype=recv_lse.dtype) - ro_stride_N, ro_stride_B, ro_stride_H, ro_stride_D = recv_output.stride() - rl_stride_N, rl_stride_B, rl_stride_H = recv_lse.stride() - o_stride_B, o_stride_H, o_stride_D = out.stride() - - grid = (B, H_local, 1) - - _dcp_lse_combine_kernel[grid]( - recv_output, - recv_lse, +def _dcp_a2a_unpack_combine( + recv_buffer: torch.Tensor, + head_dim: int, + lse_pack_dim: int, + return_lse: bool, + is_lse_base_on_e: bool, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + world_size, num_tokens, h_per_rank, _ = recv_buffer.shape + out = torch.empty( + (num_tokens, h_per_rank, head_dim), + device=recv_buffer.device, + dtype=recv_buffer.dtype, + ) + out_lse = torch.empty( + (num_tokens, h_per_rank) if return_lse else (1, 1), + device=recv_buffer.device, + dtype=torch.float32 if return_lse else recv_buffer.dtype, + ) + grid = (num_tokens, h_per_rank, 1) + _dcp_a2a_unpack_combine_kernel[grid]( + recv_buffer, out, out_lse, - ro_stride_N, - ro_stride_B, - ro_stride_H, - ro_stride_D, - rl_stride_N, - rl_stride_B, - rl_stride_H, - o_stride_B, - o_stride_H, - o_stride_D, - N=N, - HEAD_DIM=D, + recv_buffer.stride(0), + recv_buffer.stride(1), + recv_buffer.stride(2), + recv_buffer.stride(3), + out.stride(0), + out.stride(1), + out.stride(2), + out_lse.stride(0), + out_lse.stride(1), + N=world_size, + HEAD_DIM=head_dim, IS_BASE_E=is_lse_base_on_e, RETURN_LSE=return_lse, + LSE_PACK_DIM=lse_pack_dim, ) - if return_lse: return out, out_lse return out @@ -290,17 +401,8 @@ def dcp_a2a_lse_reduce( """ Combine partial attention outputs across DCP ranks using All-to-All. - Each rank holds attention output for all heads but only a local shard - of the KV cache. This function: - 1. Exchanges partial outputs across ranks via All-to-All - 2. Exchanges LSE values via All-to-All - 3. Combines them with exact LSE-weighted reduction (Triton kernel) - - Tensor flow: - Input: cp_attn_out [B, H, D] - all heads, local KV shard - Reshape: [N, B, H/N, D] - split heads across ranks - A2A: Two all_to_all_single calls (output and LSE) - Combine: recv [N, B, H/N, D] + lse [N, B, H/N] -> [B, H/N, D] + The output and fp32 LSE are packed into a single output-dtype buffer, sent + with one All-to-All, then unpacked and combined with exact LSE weighting. Args: cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim @@ -321,43 +423,36 @@ def dcp_a2a_lse_reduce( return cp_attn_out, cp_attn_lse return cp_attn_out - local_output = cp_attn_out.contiguous() - local_lse = cp_attn_lse.contiguous() - - B, H, D = local_output.shape + B, H, D = cp_attn_out.shape + if H % world_size != 0: + raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.") H_per_rank = H // world_size + lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype) - # Reshape for All-to-All: [B, H, D] -> [N, B, H/N, D] - # Split heads into N chunks, each destined for a different rank - send_output = ( - local_output.view(B, world_size, H_per_rank, D).permute(1, 0, 2, 3).contiguous() + send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers( + (world_size, B, H_per_rank, D + lse_pack_dim), + device=cp_attn_out.device, + dtype=cp_attn_out.dtype, ) - recv_output = torch.empty_like(send_output) - # Same for LSE: [B, H] -> [N, B, H/N] - send_lse = local_lse.view(B, world_size, H_per_rank).permute(1, 0, 2).contiguous() - recv_lse = torch.empty_like(send_lse) + _dcp_a2a_pack_send( + cp_attn_out, + cp_attn_lse, + send_buffer, + world_size, + H_per_rank, + D, + lse_pack_dim, + ) - # All-to-All for partial attention outputs and LSE values (async overlap) - work_output = dist.all_to_all_single( - recv_output.view(-1), - send_output.view(-1), + work = dist.all_to_all_single( + recv_buffer.view(-1), + send_buffer.view(-1), group=cp_group.device_group, async_op=True, ) - work_lse = dist.all_to_all_single( - recv_lse.view(-1), - send_lse.view(-1), - group=cp_group.device_group, - async_op=True, - ) - work_output.wait() - work_lse.wait() + work.wait() - # LSE-weighted combination via Triton kernel (local, no communication) - return dcp_lse_combine_triton( - recv_output, - recv_lse, - return_lse=return_lse, - is_lse_base_on_e=is_lse_base_on_e, + return _dcp_a2a_unpack_combine( + recv_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e )