[Kernel] Pack output and LSE in DCP A2A (#41160)
This commit is contained in:
@@ -10,10 +10,95 @@ Tests cover:
|
||||
|
||||
import math
|
||||
|
||||
import multiprocess as mp
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
mp.set_start_method("spawn", force=True)
|
||||
|
||||
|
||||
class _FakeCPGroup:
|
||||
def __init__(self, world_size: int, device_group: dist.ProcessGroup):
|
||||
self.world_size = world_size
|
||||
self.device_group = device_group
|
||||
|
||||
|
||||
def _dtype_from_name(dtype_name: str) -> torch.dtype:
|
||||
return {
|
||||
"float16": torch.float16,
|
||||
"bfloat16": torch.bfloat16,
|
||||
"float32": torch.float32,
|
||||
}[dtype_name]
|
||||
|
||||
|
||||
def _packed_a2a_reference(
|
||||
cp_attn_out: torch.Tensor,
|
||||
cp_attn_lse: torch.Tensor,
|
||||
world_size: int,
|
||||
h_per_rank: int,
|
||||
is_lse_base_on_e: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
|
||||
B, _H, D = cp_attn_out.shape
|
||||
outputs = (
|
||||
cp_attn_out.view(B, world_size, h_per_rank, D)
|
||||
.permute(1, 0, 2, 3)
|
||||
.contiguous()
|
||||
.float()
|
||||
)
|
||||
lses = cp_attn_lse.view(B, world_size, h_per_rank).permute(1, 0, 2).contiguous()
|
||||
return _lse_weighted_combine(
|
||||
outputs,
|
||||
lses,
|
||||
return_lse=True,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
|
||||
|
||||
def _assert_packed_a2a_close(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
if dtype == torch.float32:
|
||||
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
actual.float(), expected.float(), rtol=3e-2, atol=3e-2
|
||||
)
|
||||
|
||||
|
||||
def _distributed_run(fn, world_size: int, extra_env: dict[str, str]) -> None:
|
||||
port = str(get_open_port())
|
||||
processes: list[mp.Process] = []
|
||||
for rank in range(world_size):
|
||||
env = {
|
||||
"RANK": str(rank),
|
||||
"LOCAL_RANK": str(rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"LOCAL_WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": port,
|
||||
**extra_env,
|
||||
}
|
||||
process = mp.Process(target=fn, args=(env,))
|
||||
processes.append(process)
|
||||
process.start()
|
||||
|
||||
for process in processes:
|
||||
process.join(timeout=120)
|
||||
|
||||
for process in processes:
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
assert process.exitcode == 0
|
||||
|
||||
|
||||
class TestDCPCommBackendConfig:
|
||||
@@ -38,14 +123,14 @@ class TestDCPCommBackendConfig:
|
||||
"""A2A backend is valid when DCP > 1."""
|
||||
config = ParallelConfig(
|
||||
dcp_comm_backend="a2a",
|
||||
tensor_parallel_size=8,
|
||||
tensor_parallel_size=4,
|
||||
decode_context_parallel_size=4,
|
||||
)
|
||||
assert config.dcp_comm_backend == "a2a"
|
||||
|
||||
def test_invalid_backend_rejected(self):
|
||||
"""Invalid backend values are rejected."""
|
||||
with pytest.raises(ValueError, match="must be one of"):
|
||||
with pytest.raises(ValueError, match="must be one of|Input should be"):
|
||||
ParallelConfig(
|
||||
dcp_comm_backend="invalid",
|
||||
)
|
||||
@@ -134,7 +219,7 @@ class TestLSEWeightedCombine:
|
||||
result = _lse_weighted_combine(outputs, lses)
|
||||
|
||||
assert result.shape == (B, H, D)
|
||||
torch.testing.assert_close(result, outputs[1].squeeze(0), atol=1e-5, rtol=1e-5)
|
||||
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
|
||||
|
||||
def test_mathematically_correct(self):
|
||||
"""Verify mathematical correctness of LSE combination."""
|
||||
@@ -187,6 +272,224 @@ class TestLSEWeightedCombine:
|
||||
assert global_lse.shape == (B, H)
|
||||
assert abs(global_lse.item() - expected_global_lse) < 1e-5
|
||||
|
||||
def test_base2_return_lse(self):
|
||||
"""Base-2 LSE mode returns log2-sum-exp2 global LSE."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
|
||||
outputs = torch.tensor(
|
||||
[
|
||||
[[[1.0, 2.0]]],
|
||||
[[[3.0, 4.0]]],
|
||||
]
|
||||
)
|
||||
lses = torch.tensor(
|
||||
[
|
||||
[[1.0]],
|
||||
[[2.0]],
|
||||
]
|
||||
)
|
||||
|
||||
result, global_lse = _lse_weighted_combine(
|
||||
outputs,
|
||||
lses,
|
||||
return_lse=True,
|
||||
is_lse_base_on_e=False,
|
||||
)
|
||||
|
||||
expected_global_lse = math.log2(2**1 + 2**2)
|
||||
w0 = 2**1 / (2**1 + 2**2)
|
||||
w1 = 2**2 / (2**1 + 2**2)
|
||||
expected = torch.tensor([[[w0 * 1.0 + w1 * 3.0, w0 * 2.0 + w1 * 4.0]]])
|
||||
|
||||
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-5)
|
||||
torch.testing.assert_close(
|
||||
global_lse,
|
||||
torch.tensor([[expected_global_lse]]),
|
||||
rtol=1e-5,
|
||||
atol=1e-5,
|
||||
)
|
||||
|
||||
def test_lse_pack_dim(self):
|
||||
"""Packed A2A stores one fp32 LSE in output-dtype lanes."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _dcp_a2a_lse_pack_dim
|
||||
|
||||
assert _dcp_a2a_lse_pack_dim(torch.bfloat16) == 2
|
||||
assert _dcp_a2a_lse_pack_dim(torch.float16) == 2
|
||||
assert _dcp_a2a_lse_pack_dim(torch.float32) == 1
|
||||
|
||||
|
||||
class TestPackedA2AKernels:
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 1, reason="CUDA is required."
|
||||
)
|
||||
@pytest.mark.parametrize("dtype_name", ["float16", "bfloat16", "float32"])
|
||||
@pytest.mark.parametrize("return_lse", [False, True])
|
||||
@pytest.mark.parametrize("is_lse_base_on_e", [False, True])
|
||||
def test_pack_unpack_combine_matches_reference(
|
||||
self,
|
||||
dtype_name: str,
|
||||
return_lse: bool,
|
||||
is_lse_base_on_e: bool,
|
||||
):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import (
|
||||
_dcp_a2a_lse_pack_dim,
|
||||
_dcp_a2a_pack_send,
|
||||
_dcp_a2a_unpack_combine,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
dtype = _dtype_from_name(dtype_name)
|
||||
device = torch.device("cuda")
|
||||
world_size, B, h_per_rank, D = 4, 7, 2, 32
|
||||
H = world_size * h_per_rank
|
||||
cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype)
|
||||
cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32)
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype)
|
||||
send_buffer = torch.empty(
|
||||
(world_size, B, h_per_rank, D + lse_pack_dim),
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
_dcp_a2a_pack_send(
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
send_buffer,
|
||||
world_size,
|
||||
h_per_rank,
|
||||
D,
|
||||
lse_pack_dim,
|
||||
)
|
||||
actual = _dcp_a2a_unpack_combine(
|
||||
send_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e
|
||||
)
|
||||
expected_out, expected_lse = _packed_a2a_reference(
|
||||
cp_attn_out, cp_attn_lse, world_size, h_per_rank, is_lse_base_on_e
|
||||
)
|
||||
|
||||
if return_lse:
|
||||
actual_out, actual_lse = actual
|
||||
_assert_packed_a2a_close(actual_out, expected_out, dtype)
|
||||
torch.testing.assert_close(actual_lse, expected_lse, rtol=1e-4, atol=1e-4)
|
||||
else:
|
||||
_assert_packed_a2a_close(actual, expected_out, dtype)
|
||||
|
||||
|
||||
def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
update_environment_variables(env)
|
||||
local_rank = int(env["LOCAL_RANK"])
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
dist.init_process_group(backend="nccl")
|
||||
use_workspace = env.get("USE_WORKSPACE") == "1"
|
||||
if use_workspace:
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
init_workspace_manager(torch.device(f"cuda:{local_rank}"))
|
||||
try:
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
|
||||
dtype = _dtype_from_name(env["TEST_DTYPE"])
|
||||
return_lse = env["RETURN_LSE"] == "1"
|
||||
is_lse_base_on_e = env["LSE_BASE_E"] == "1"
|
||||
rank = dist.get_rank()
|
||||
world_size = dist.get_world_size()
|
||||
B, h_per_rank, D = 5, 2, 32
|
||||
H = world_size * h_per_rank
|
||||
|
||||
generator = torch.Generator(device=f"cuda:{local_rank}")
|
||||
generator.manual_seed(1234 + rank)
|
||||
cp_attn_out = torch.randn(
|
||||
B,
|
||||
H,
|
||||
D,
|
||||
device=f"cuda:{local_rank}",
|
||||
dtype=dtype,
|
||||
generator=generator,
|
||||
)
|
||||
cp_attn_lse = torch.randn(
|
||||
B,
|
||||
H,
|
||||
device=f"cuda:{local_rank}",
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
actual = dcp_a2a_lse_reduce(
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
_FakeCPGroup(world_size, dist.group.WORLD),
|
||||
return_lse=return_lse,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
|
||||
gathered_out = [torch.empty_like(cp_attn_out) for _ in range(world_size)]
|
||||
gathered_lse = [torch.empty_like(cp_attn_lse) for _ in range(world_size)]
|
||||
dist.all_gather(gathered_out, cp_attn_out)
|
||||
dist.all_gather(gathered_lse, cp_attn_lse)
|
||||
outputs = torch.stack(
|
||||
[
|
||||
t[:, rank * h_per_rank : (rank + 1) * h_per_rank, :]
|
||||
for t in gathered_out
|
||||
],
|
||||
dim=0,
|
||||
).float()
|
||||
lses = torch.stack(
|
||||
[t[:, rank * h_per_rank : (rank + 1) * h_per_rank] for t in gathered_lse],
|
||||
dim=0,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
|
||||
expected_out, expected_lse = _lse_weighted_combine(
|
||||
outputs,
|
||||
lses,
|
||||
return_lse=True,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
|
||||
if return_lse:
|
||||
actual_out, actual_lse = actual
|
||||
_assert_packed_a2a_close(actual_out, expected_out, dtype)
|
||||
torch.testing.assert_close(actual_lse, expected_lse, rtol=1e-4, atol=1e-4)
|
||||
else:
|
||||
_assert_packed_a2a_close(actual, expected_out, dtype)
|
||||
finally:
|
||||
if use_workspace:
|
||||
from vllm.v1.worker.workspace import reset_workspace_manager
|
||||
|
||||
reset_workspace_manager()
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 4, reason="Need at least 4 GPUs."
|
||||
)
|
||||
@pytest.mark.parametrize("dtype_name", ["float16", "bfloat16", "float32"])
|
||||
def test_distributed_packed_a2a_matches_reference(dtype_name: str):
|
||||
_distributed_run(
|
||||
_distributed_packed_a2a_worker,
|
||||
world_size=4,
|
||||
extra_env={
|
||||
"TEST_DTYPE": dtype_name,
|
||||
"RETURN_LSE": "1",
|
||||
"LSE_BASE_E": "1",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 4, reason="Need at least 4 GPUs."
|
||||
)
|
||||
def test_distributed_packed_a2a_with_workspace_matches_reference():
|
||||
_distributed_run(
|
||||
_distributed_packed_a2a_worker,
|
||||
world_size=4,
|
||||
extra_env={
|
||||
"TEST_DTYPE": "bfloat16",
|
||||
"RETURN_LSE": "1",
|
||||
"LSE_BASE_E": "1",
|
||||
"USE_WORKSPACE": "1",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -9,10 +9,8 @@ Instead of gathering the full Q tensor and scattering partial outputs,
|
||||
A2A exchanges partial attention outputs and their LSE values across
|
||||
ranks, then combines them with exact LSE-weighted reduction.
|
||||
|
||||
This reduces the number of NCCL calls per attention layer from 3
|
||||
(AG for Q, AG for K metadata, RS for output) to 2 (A2A for output,
|
||||
A2A for LSE), lowering per-step communication overhead for long-context
|
||||
decode where NCCL latency is a significant fraction of step time.
|
||||
This reduces the number of NCCL calls per attention layer by exchanging
|
||||
the partial output and LSE in a single packed All-to-All payload.
|
||||
|
||||
Usage:
|
||||
vllm serve model --tp 16 --dcp 16 --dcp-comm-backend a2a
|
||||
@@ -28,6 +26,10 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
@@ -44,7 +46,6 @@ def _lse_weighted_combine(
|
||||
CPU reference implementation for LSE-weighted combination.
|
||||
|
||||
This is a pure PyTorch implementation used for testing and validation.
|
||||
For GPU execution, use dcp_lse_combine_triton instead.
|
||||
|
||||
Args:
|
||||
outputs: Partial attention outputs [N, B, H, D]
|
||||
@@ -102,57 +103,137 @@ def _lse_weighted_combine(
|
||||
return result
|
||||
|
||||
|
||||
def _dcp_a2a_lse_pack_dim(output_dtype: torch.dtype) -> int:
|
||||
bits = torch.finfo(output_dtype).bits
|
||||
if bits == 16:
|
||||
return 2
|
||||
if bits == 32:
|
||||
return 1
|
||||
raise ValueError(f"Cannot pack fp32 LSE into output dtype {output_dtype}.")
|
||||
|
||||
|
||||
def _dcp_a2a_send_recv_buffers(
|
||||
shape: tuple[int, ...],
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if is_workspace_manager_initialized():
|
||||
send_buffer, recv_buffer = current_workspace_manager().get_simultaneous(
|
||||
(shape, dtype),
|
||||
(shape, dtype),
|
||||
)
|
||||
return send_buffer, recv_buffer
|
||||
|
||||
return (
|
||||
torch.empty(shape, device=device, dtype=dtype),
|
||||
torch.empty(shape, device=device, dtype=dtype),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dcp_lse_combine_kernel(
|
||||
# Input pointers
|
||||
recv_output_ptr,
|
||||
recv_lse_ptr,
|
||||
# Output pointers
|
||||
def _dcp_a2a_pack_send_kernel(
|
||||
out_ptr,
|
||||
lse_ptr,
|
||||
send_ptr,
|
||||
out_stride_B,
|
||||
out_stride_H,
|
||||
out_stride_D,
|
||||
lse_stride_B,
|
||||
lse_stride_H,
|
||||
send_stride_N,
|
||||
send_stride_B,
|
||||
send_stride_H,
|
||||
send_stride_D,
|
||||
N: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
H_PER_RANK: tl.constexpr,
|
||||
LSE_PACK_DIM: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0).to(tl.int64)
|
||||
local_head_idx = tl.program_id(1).to(tl.int64)
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
|
||||
for rank_idx in tl.static_range(N):
|
||||
src_head_idx = rank_idx * H_PER_RANK + local_head_idx
|
||||
send_base = (
|
||||
rank_idx * send_stride_N
|
||||
+ batch_idx * send_stride_B
|
||||
+ local_head_idx * send_stride_H
|
||||
)
|
||||
|
||||
out_offsets = (
|
||||
batch_idx * out_stride_B
|
||||
+ src_head_idx * out_stride_H
|
||||
+ d_offsets * out_stride_D
|
||||
)
|
||||
tl.store(
|
||||
send_ptr + send_base + d_offsets * send_stride_D,
|
||||
tl.load(out_ptr + out_offsets),
|
||||
)
|
||||
|
||||
lse_val = tl.load(
|
||||
lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H
|
||||
)
|
||||
if LSE_PACK_DIM == 1:
|
||||
tl.store(
|
||||
send_ptr + send_base + HEAD_DIM * send_stride_D,
|
||||
lse_val.to(send_ptr.dtype.element_ty),
|
||||
)
|
||||
else:
|
||||
lse_bits = lse_val.to(tl.uint32, bitcast=True)
|
||||
lo = (lse_bits & 0xFFFF).to(tl.uint16)
|
||||
hi = ((lse_bits >> 16) & 0xFFFF).to(tl.uint16)
|
||||
tl.store(
|
||||
send_ptr + send_base + HEAD_DIM * send_stride_D,
|
||||
lo.to(send_ptr.dtype.element_ty, bitcast=True),
|
||||
)
|
||||
tl.store(
|
||||
send_ptr + send_base + (HEAD_DIM + 1) * send_stride_D,
|
||||
hi.to(send_ptr.dtype.element_ty, bitcast=True),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dcp_a2a_unpack_combine_kernel(
|
||||
recv_ptr,
|
||||
out_ptr,
|
||||
out_lse_ptr,
|
||||
# Strides for recv_output [N, B, H_local, D]
|
||||
ro_stride_N,
|
||||
ro_stride_B,
|
||||
ro_stride_H,
|
||||
ro_stride_D,
|
||||
# Strides for recv_lse [N, B, H_local]
|
||||
rl_stride_N,
|
||||
rl_stride_B,
|
||||
rl_stride_H,
|
||||
# Strides for output [B, H_local, D]
|
||||
o_stride_B,
|
||||
o_stride_H,
|
||||
o_stride_D,
|
||||
# Constants
|
||||
recv_stride_N,
|
||||
recv_stride_B,
|
||||
recv_stride_H,
|
||||
recv_stride_D,
|
||||
out_stride_B,
|
||||
out_stride_H,
|
||||
out_stride_D,
|
||||
out_lse_stride_B,
|
||||
out_lse_stride_H,
|
||||
N: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
IS_BASE_E: tl.constexpr,
|
||||
RETURN_LSE: tl.constexpr,
|
||||
LSE_PACK_DIM: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Triton kernel for LSE-weighted combination of partial attention outputs.
|
||||
|
||||
After All-to-All, each rank has:
|
||||
- recv_output [N, B, H_local, D]: partial outputs from all KV shards
|
||||
- recv_lse [N, B, H_local]: partial LSEs from all KV shards
|
||||
|
||||
This kernel computes the weighted combination locally (no communication).
|
||||
|
||||
Grid: (B, H_local)
|
||||
Each program handles one (batch, head) and processes all D elements.
|
||||
"""
|
||||
batch_idx = tl.program_id(0).to(tl.int64)
|
||||
head_idx = tl.program_id(1).to(tl.int64)
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
|
||||
# Base offset for this (batch, head)
|
||||
base_lse_offset = batch_idx * rl_stride_B + head_idx * rl_stride_H
|
||||
base_out_offset = batch_idx * ro_stride_B + head_idx * ro_stride_H
|
||||
|
||||
# First pass: find max LSE for numerical stability
|
||||
lse_max = -float("inf")
|
||||
for n in tl.static_range(N):
|
||||
lse_offset = n * rl_stride_N + base_lse_offset
|
||||
lse_val = tl.load(recv_lse_ptr + lse_offset)
|
||||
for rank_idx in tl.static_range(N):
|
||||
recv_base = (
|
||||
rank_idx * recv_stride_N
|
||||
+ batch_idx * recv_stride_B
|
||||
+ head_idx * recv_stride_H
|
||||
)
|
||||
if LSE_PACK_DIM == 1:
|
||||
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
|
||||
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
|
||||
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
|
||||
lse_val = tl.where(
|
||||
(lse_val != lse_val) | (lse_val == float("inf")),
|
||||
-float("inf"),
|
||||
@@ -162,11 +243,23 @@ def _dcp_lse_combine_kernel(
|
||||
|
||||
lse_max = tl.where(lse_max == -float("inf"), 0.0, lse_max)
|
||||
|
||||
# Second pass: compute sum of exp(lse - max)
|
||||
lse_sum = 0.0
|
||||
for n in tl.static_range(N):
|
||||
lse_offset = n * rl_stride_N + base_lse_offset
|
||||
lse_val = tl.load(recv_lse_ptr + lse_offset)
|
||||
for rank_idx in tl.static_range(N):
|
||||
recv_base = (
|
||||
rank_idx * recv_stride_N
|
||||
+ batch_idx * recv_stride_B
|
||||
+ head_idx * recv_stride_H
|
||||
)
|
||||
if LSE_PACK_DIM == 1:
|
||||
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
|
||||
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
|
||||
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
|
||||
lse_val = tl.where(
|
||||
(lse_val != lse_val) | (lse_val == float("inf")),
|
||||
-float("inf"),
|
||||
@@ -177,19 +270,28 @@ def _dcp_lse_combine_kernel(
|
||||
else:
|
||||
lse_sum += tl.exp2(lse_val - lse_max)
|
||||
|
||||
# Compute global LSE
|
||||
if IS_BASE_E: # noqa: SIM108
|
||||
global_lse = tl.log(lse_sum) + lse_max
|
||||
else:
|
||||
global_lse = tl.log2(lse_sum) + lse_max
|
||||
|
||||
# Third pass: weighted combination across D dimension
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
acc = tl.zeros([HEAD_DIM], dtype=tl.float32)
|
||||
|
||||
for n in tl.static_range(N):
|
||||
lse_offset = n * rl_stride_N + base_lse_offset
|
||||
lse_val = tl.load(recv_lse_ptr + lse_offset)
|
||||
for rank_idx in tl.static_range(N):
|
||||
recv_base = (
|
||||
rank_idx * recv_stride_N
|
||||
+ batch_idx * recv_stride_B
|
||||
+ head_idx * recv_stride_H
|
||||
)
|
||||
if LSE_PACK_DIM == 1:
|
||||
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
|
||||
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
|
||||
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
|
||||
lse_val = tl.where(
|
||||
(lse_val != lse_val) | (lse_val == float("inf")),
|
||||
-float("inf"),
|
||||
@@ -200,80 +302,89 @@ def _dcp_lse_combine_kernel(
|
||||
else:
|
||||
weight = tl.exp2(lse_val - global_lse)
|
||||
weight = tl.where(weight != weight, 0.0, weight)
|
||||
acc += (
|
||||
tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(tl.float32)
|
||||
* weight
|
||||
)
|
||||
|
||||
out_offsets = n * ro_stride_N + base_out_offset + d_offsets * ro_stride_D
|
||||
out_vals = tl.load(recv_output_ptr + out_offsets)
|
||||
acc += out_vals.to(tl.float32) * weight
|
||||
|
||||
# Store result
|
||||
final_offsets = (
|
||||
batch_idx * o_stride_B + head_idx * o_stride_H + d_offsets * o_stride_D
|
||||
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
|
||||
)
|
||||
tl.store(out_ptr + final_offsets, acc)
|
||||
|
||||
if RETURN_LSE:
|
||||
tl.store(out_lse_ptr + base_lse_offset, global_lse)
|
||||
out_lse_offset = batch_idx * out_lse_stride_B + head_idx * out_lse_stride_H
|
||||
tl.store(out_lse_ptr + out_lse_offset, global_lse)
|
||||
|
||||
|
||||
def dcp_lse_combine_triton(
|
||||
recv_output: torch.Tensor,
|
||||
recv_lse: torch.Tensor,
|
||||
return_lse: bool = False,
|
||||
is_lse_base_on_e: bool = True,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Triton-accelerated LSE-weighted combination for DCP A2A.
|
||||
|
||||
Args:
|
||||
recv_output: [N, B, H_local, D] - partial outputs from all KV shards
|
||||
recv_lse: [N, B, H_local] - partial LSEs from all KV shards
|
||||
return_lse: If True, also return the global LSE
|
||||
is_lse_base_on_e: If True, LSE is base e; if False, base 2
|
||||
|
||||
Returns:
|
||||
Combined output [B, H_local, D]
|
||||
If return_lse=True, also returns global_lse [B, H_local]
|
||||
"""
|
||||
N, B, H_local, D = recv_output.shape
|
||||
|
||||
out = torch.empty(
|
||||
(B, H_local, D), device=recv_output.device, dtype=recv_output.dtype
|
||||
def _dcp_a2a_pack_send(
|
||||
cp_attn_out: torch.Tensor,
|
||||
cp_attn_lse: torch.Tensor,
|
||||
send_buffer: torch.Tensor,
|
||||
world_size: int,
|
||||
h_per_rank: int,
|
||||
head_dim: int,
|
||||
lse_pack_dim: int,
|
||||
) -> None:
|
||||
grid = (cp_attn_out.shape[0], h_per_rank, 1)
|
||||
_dcp_a2a_pack_send_kernel[grid](
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
send_buffer,
|
||||
cp_attn_out.stride(0),
|
||||
cp_attn_out.stride(1),
|
||||
cp_attn_out.stride(2),
|
||||
cp_attn_lse.stride(0),
|
||||
cp_attn_lse.stride(1),
|
||||
send_buffer.stride(0),
|
||||
send_buffer.stride(1),
|
||||
send_buffer.stride(2),
|
||||
send_buffer.stride(3),
|
||||
N=world_size,
|
||||
HEAD_DIM=head_dim,
|
||||
H_PER_RANK=h_per_rank,
|
||||
LSE_PACK_DIM=lse_pack_dim,
|
||||
)
|
||||
|
||||
if return_lse:
|
||||
out_lse = torch.empty(
|
||||
(B, H_local), device=recv_lse.device, dtype=recv_lse.dtype
|
||||
)
|
||||
else:
|
||||
out_lse = torch.empty(1, device=recv_lse.device, dtype=recv_lse.dtype)
|
||||
|
||||
ro_stride_N, ro_stride_B, ro_stride_H, ro_stride_D = recv_output.stride()
|
||||
rl_stride_N, rl_stride_B, rl_stride_H = recv_lse.stride()
|
||||
o_stride_B, o_stride_H, o_stride_D = out.stride()
|
||||
|
||||
grid = (B, H_local, 1)
|
||||
|
||||
_dcp_lse_combine_kernel[grid](
|
||||
recv_output,
|
||||
recv_lse,
|
||||
def _dcp_a2a_unpack_combine(
|
||||
recv_buffer: torch.Tensor,
|
||||
head_dim: int,
|
||||
lse_pack_dim: int,
|
||||
return_lse: bool,
|
||||
is_lse_base_on_e: bool,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
world_size, num_tokens, h_per_rank, _ = recv_buffer.shape
|
||||
out = torch.empty(
|
||||
(num_tokens, h_per_rank, head_dim),
|
||||
device=recv_buffer.device,
|
||||
dtype=recv_buffer.dtype,
|
||||
)
|
||||
out_lse = torch.empty(
|
||||
(num_tokens, h_per_rank) if return_lse else (1, 1),
|
||||
device=recv_buffer.device,
|
||||
dtype=torch.float32 if return_lse else recv_buffer.dtype,
|
||||
)
|
||||
grid = (num_tokens, h_per_rank, 1)
|
||||
_dcp_a2a_unpack_combine_kernel[grid](
|
||||
recv_buffer,
|
||||
out,
|
||||
out_lse,
|
||||
ro_stride_N,
|
||||
ro_stride_B,
|
||||
ro_stride_H,
|
||||
ro_stride_D,
|
||||
rl_stride_N,
|
||||
rl_stride_B,
|
||||
rl_stride_H,
|
||||
o_stride_B,
|
||||
o_stride_H,
|
||||
o_stride_D,
|
||||
N=N,
|
||||
HEAD_DIM=D,
|
||||
recv_buffer.stride(0),
|
||||
recv_buffer.stride(1),
|
||||
recv_buffer.stride(2),
|
||||
recv_buffer.stride(3),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
out_lse.stride(0),
|
||||
out_lse.stride(1),
|
||||
N=world_size,
|
||||
HEAD_DIM=head_dim,
|
||||
IS_BASE_E=is_lse_base_on_e,
|
||||
RETURN_LSE=return_lse,
|
||||
LSE_PACK_DIM=lse_pack_dim,
|
||||
)
|
||||
|
||||
if return_lse:
|
||||
return out, out_lse
|
||||
return out
|
||||
@@ -290,17 +401,8 @@ def dcp_a2a_lse_reduce(
|
||||
"""
|
||||
Combine partial attention outputs across DCP ranks using All-to-All.
|
||||
|
||||
Each rank holds attention output for all heads but only a local shard
|
||||
of the KV cache. This function:
|
||||
1. Exchanges partial outputs across ranks via All-to-All
|
||||
2. Exchanges LSE values via All-to-All
|
||||
3. Combines them with exact LSE-weighted reduction (Triton kernel)
|
||||
|
||||
Tensor flow:
|
||||
Input: cp_attn_out [B, H, D] - all heads, local KV shard
|
||||
Reshape: [N, B, H/N, D] - split heads across ranks
|
||||
A2A: Two all_to_all_single calls (output and LSE)
|
||||
Combine: recv [N, B, H/N, D] + lse [N, B, H/N] -> [B, H/N, D]
|
||||
The output and fp32 LSE are packed into a single output-dtype buffer, sent
|
||||
with one All-to-All, then unpacked and combined with exact LSE weighting.
|
||||
|
||||
Args:
|
||||
cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim
|
||||
@@ -321,43 +423,36 @@ def dcp_a2a_lse_reduce(
|
||||
return cp_attn_out, cp_attn_lse
|
||||
return cp_attn_out
|
||||
|
||||
local_output = cp_attn_out.contiguous()
|
||||
local_lse = cp_attn_lse.contiguous()
|
||||
|
||||
B, H, D = local_output.shape
|
||||
B, H, D = cp_attn_out.shape
|
||||
if H % world_size != 0:
|
||||
raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.")
|
||||
H_per_rank = H // world_size
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype)
|
||||
|
||||
# Reshape for All-to-All: [B, H, D] -> [N, B, H/N, D]
|
||||
# Split heads into N chunks, each destined for a different rank
|
||||
send_output = (
|
||||
local_output.view(B, world_size, H_per_rank, D).permute(1, 0, 2, 3).contiguous()
|
||||
send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers(
|
||||
(world_size, B, H_per_rank, D + lse_pack_dim),
|
||||
device=cp_attn_out.device,
|
||||
dtype=cp_attn_out.dtype,
|
||||
)
|
||||
recv_output = torch.empty_like(send_output)
|
||||
|
||||
# Same for LSE: [B, H] -> [N, B, H/N]
|
||||
send_lse = local_lse.view(B, world_size, H_per_rank).permute(1, 0, 2).contiguous()
|
||||
recv_lse = torch.empty_like(send_lse)
|
||||
_dcp_a2a_pack_send(
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
send_buffer,
|
||||
world_size,
|
||||
H_per_rank,
|
||||
D,
|
||||
lse_pack_dim,
|
||||
)
|
||||
|
||||
# All-to-All for partial attention outputs and LSE values (async overlap)
|
||||
work_output = dist.all_to_all_single(
|
||||
recv_output.view(-1),
|
||||
send_output.view(-1),
|
||||
work = dist.all_to_all_single(
|
||||
recv_buffer.view(-1),
|
||||
send_buffer.view(-1),
|
||||
group=cp_group.device_group,
|
||||
async_op=True,
|
||||
)
|
||||
work_lse = dist.all_to_all_single(
|
||||
recv_lse.view(-1),
|
||||
send_lse.view(-1),
|
||||
group=cp_group.device_group,
|
||||
async_op=True,
|
||||
)
|
||||
work_output.wait()
|
||||
work_lse.wait()
|
||||
work.wait()
|
||||
|
||||
# LSE-weighted combination via Triton kernel (local, no communication)
|
||||
return dcp_lse_combine_triton(
|
||||
recv_output,
|
||||
recv_lse,
|
||||
return_lse=return_lse,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
return _dcp_a2a_unpack_combine(
|
||||
recv_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user