[refactor] consolidate cp attn ops (#52839)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-19 18:04:11 -07:00
committed by GitHub
co-authored by Woosuk Kwon
parent bf2866f8bf
commit 58e5ee0158
18 changed files with 1603 additions and 1585 deletions
+17 -17
View File
@@ -44,7 +44,7 @@ def _packed_a2a_reference(
h_per_rank: int,
is_lse_base_on_e: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
B, _H, D = cp_attn_out.shape
outputs = (
@@ -157,13 +157,13 @@ class TestLSEWeightedCombine:
def test_importable(self):
"""Verify _lse_weighted_combine is importable."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
assert callable(_lse_weighted_combine)
def test_single_rank(self):
"""Single rank: output unchanged."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
# N=1, B=2, H=4, D=8
outputs = torch.randn(1, 2, 4, 8)
@@ -176,7 +176,7 @@ class TestLSEWeightedCombine:
def test_equal_lse(self):
"""Equal LSE values: outputs averaged equally."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
_N, B, H, D = 2, 1, 1, 4
outputs = torch.tensor(
@@ -200,7 +200,7 @@ class TestLSEWeightedCombine:
def test_dominant_rank(self):
"""Different LSE values: larger LSE gets more weight."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
B, H, D = 1, 1, 2
outputs = torch.tensor(
@@ -222,7 +222,7 @@ class TestLSEWeightedCombine:
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
def test_empty_shard_ignores_undefined_output(self):
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]])
lses = torch.tensor([[[-float("inf")]], [[0.0]]])
@@ -232,7 +232,7 @@ class TestLSEWeightedCombine:
torch.testing.assert_close(result, outputs[1])
def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch):
import vllm.v1.attention.ops.common as common
import vllm.v1.attention.ops.dcp as dcp
class FakeGroup:
world_size = 2
@@ -243,7 +243,7 @@ class TestLSEWeightedCombine:
return torch.cat((tensor, tensor), dim=dim)
monkeypatch.setattr(
common,
dcp,
"correct_attn_out",
lambda output, lses, *args, **kwargs: (output, lses[0]),
)
@@ -252,7 +252,7 @@ class TestLSEWeightedCombine:
seq_lens = torch.tensor([0, 2], dtype=torch.int32)
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
_, masked_lse = common._cp_lse_common(
_, masked_lse = dcp._cp_lse_common(
output,
lse,
FakeGroup(),
@@ -266,7 +266,7 @@ class TestLSEWeightedCombine:
def test_mathematically_correct(self):
"""Verify mathematical correctness of LSE combination."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
outputs = torch.tensor(
[
@@ -291,7 +291,7 @@ class TestLSEWeightedCombine:
def test_return_lse(self):
"""return_lse=True returns global LSE (logsumexp of inputs)."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
B, H, D = 1, 1, 2
outputs = torch.tensor(
@@ -317,7 +317,7 @@ class TestLSEWeightedCombine:
def test_base2_return_lse(self):
"""Base-2 LSE mode returns log2-sum-exp2 global LSE."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
outputs = torch.tensor(
[
@@ -354,7 +354,7 @@ class TestLSEWeightedCombine:
def test_lse_pack_dim(self):
"""Packed A2A stores one fp32 LSE in output-dtype lanes."""
from vllm.v1.attention.ops.dcp_alltoall import _dcp_a2a_lse_pack_dim
from vllm.v1.attention.ops.dcp import _dcp_a2a_lse_pack_dim
assert _dcp_a2a_lse_pack_dim(torch.bfloat16) == 2
assert _dcp_a2a_lse_pack_dim(torch.float16) == 2
@@ -374,7 +374,7 @@ class TestPackedA2AKernels:
return_lse: bool,
is_lse_base_on_e: bool,
):
from vllm.v1.attention.ops.dcp_alltoall import (
from vllm.v1.attention.ops.dcp import (
_dcp_a2a_lse_pack_dim,
_dcp_a2a_pack_send,
_dcp_a2a_unpack_combine,
@@ -421,7 +421,7 @@ class TestPackedA2AKernels:
torch.accelerator.device_count() < 1, reason="CUDA is required."
)
def test_empty_seq_lens_ignore_undefined_output(self):
from vllm.v1.attention.ops.dcp_alltoall import (
from vllm.v1.attention.ops.dcp import (
_dcp_a2a_lse_pack_dim,
_dcp_a2a_pack_send,
_dcp_a2a_unpack_combine,
@@ -505,7 +505,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
init_workspace_manager(torch.device(f"cuda:{local_rank}"))
try:
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.dcp import dcp_a2a_lse_reduce
dtype = _dtype_from_name(env["TEST_DTYPE"])
return_lse = env["RETURN_LSE"] == "1"
@@ -555,7 +555,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
[t[:, rank * h_per_rank : (rank + 1) * h_per_rank] for t in gathered_lse],
dim=0,
)
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
expected_out, expected_lse = _lse_weighted_combine(
outputs,
@@ -10,7 +10,8 @@ import pytest
import torch
import torch.distributed as dist
import vllm.v1.attention.ops.dcp_utils as dcp_utils
import vllm.v1.attention.ops.cp_common as cp_common
import vllm.v1.attention.ops.dcp as dcp
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
@@ -142,17 +143,17 @@ class _FakeProcessGroup:
class TestDirectDCPGating:
def test_env_disabled_returns_none(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
workspace = dcp_utils.get_direct_dcp_a2a_workspace(
dcp.get_direct_dcp_a2a_workspace.cache_clear()
workspace = dcp.get_direct_dcp_a2a_workspace(
_FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.bfloat16, 1
)
assert workspace is None
def test_forced_with_unsupported_dtype_raises(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
dcp.get_direct_dcp_a2a_workspace.cache_clear()
with pytest.raises(ValueError, match="does not support"):
dcp_utils.get_direct_dcp_a2a_workspace(
dcp.get_direct_dcp_a2a_workspace(
_FakeGroupCoordinator(),
torch.device("cpu"),
16,
@@ -164,14 +165,14 @@ class TestDirectDCPGating:
def test_zero_ubatches_raises(self):
with pytest.raises(ValueError, match="ubatch"):
dcp_utils.DirectDCPA2AWorkspace(
dcp.DirectDCPA2AWorkspace(
None, torch.device("cpu"), 16, 2, 32, torch.bfloat16, num_ubatches=0
)
def test_auto_with_unsupported_dtype_returns_none(self, monkeypatch):
monkeypatch.delenv("VLLM_USE_DIRECT_DCP_A2A", raising=False)
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
workspace = dcp_utils.get_direct_dcp_a2a_workspace(
dcp.get_direct_dcp_a2a_workspace.cache_clear()
workspace = dcp.get_direct_dcp_a2a_workspace(
_FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.float32, 1
)
assert workspace is None
@@ -179,8 +180,8 @@ class TestDirectDCPGating:
def test_q_gather_env_disabled_returns_none(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "0")
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear()
workspace = dcp_utils.get_direct_dcp_q_gather_workspace(
dcp.get_direct_dcp_q_gather_workspace.cache_clear()
workspace = dcp.get_direct_dcp_q_gather_workspace(
_FakeGroupCoordinator(),
torch.device("cpu"),
16,
@@ -194,17 +195,17 @@ class TestDirectDCPGating:
def test_q_gather_flag_is_independent(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "1")
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True)
dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear()
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: True)
dcp.get_direct_dcp_q_gather_workspace.cache_clear()
workspace = object()
init_workspace = MagicMock(return_value=workspace)
monkeypatch.setattr(
dcp_utils,
dcp,
"DirectDCPQGatherWorkspace",
init_workspace,
)
result = dcp_utils.get_direct_dcp_q_gather_workspace(
result = dcp.get_direct_dcp_q_gather_workspace(
_FakeGroupCoordinator(),
torch.device("cpu"),
16,
@@ -220,8 +221,8 @@ class TestDirectDCPGating:
def test_kv_gather_env_disabled_returns_none(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "0")
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear()
workspace = dcp_utils.get_direct_dcp_kv_gather_workspace(
dcp.get_direct_dcp_kv_gather_workspace.cache_clear()
workspace = dcp.get_direct_dcp_kv_gather_workspace(
_FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1
)
assert workspace is None
@@ -229,17 +230,17 @@ class TestDirectDCPGating:
def test_kv_gather_flag_is_independent(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "1")
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True)
dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear()
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: True)
dcp.get_direct_dcp_kv_gather_workspace.cache_clear()
workspace = object()
init_workspace = MagicMock(return_value=workspace)
monkeypatch.setattr(
dcp_utils,
dcp,
"DirectDCPKVGatherWorkspace",
init_workspace,
)
result = dcp_utils.get_direct_dcp_kv_gather_workspace(
result = dcp.get_direct_dcp_kv_gather_workspace(
_FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1
)
@@ -267,9 +268,9 @@ class TestDirectDCPGating:
factory_name,
factory_args,
):
factory = getattr(dcp_utils, factory_name)
factory = getattr(dcp, factory_name)
monkeypatch.setenv(flag_name, "1")
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: False)
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: False)
factory.cache_clear()
assert (
@@ -283,25 +284,25 @@ class TestDirectDCPGating:
def test_kv_gather_rejects_invalid_workspace_geometry(self):
with pytest.raises(ValueError, match="ubatch"):
dcp_utils.DirectDCPKVGatherWorkspace(
dcp.DirectDCPKVGatherWorkspace(
None, torch.device("cpu"), 64, 576, num_ubatches=0
)
with pytest.raises(ValueError, match="divide evenly"):
dcp_utils.DirectDCPKVGatherWorkspace(
dcp.DirectDCPKVGatherWorkspace(
_FakeProcessGroup(), torch.device("cpu"), 63, 576
)
with pytest.raises(ValueError, match="16-byte"):
dcp_utils.DirectDCPKVGatherWorkspace(
dcp.DirectDCPKVGatherWorkspace(
_FakeProcessGroup(), torch.device("cpu"), 64, 3
)
def test_q_gather_rejects_invalid_workspace_geometry(self):
with pytest.raises(ValueError, match="ubatch"):
dcp_utils.DirectDCPQGatherWorkspace(
dcp.DirectDCPQGatherWorkspace(
None, torch.device("cpu"), 16, 2, 32, num_ubatches=0
)
with pytest.raises(ValueError, match="padded heads"):
dcp_utils.DirectDCPQGatherWorkspace(
dcp.DirectDCPQGatherWorkspace(
_FakeProcessGroup(),
torch.device("cpu"),
16,
@@ -310,7 +311,7 @@ class TestDirectDCPGating:
padded_num_heads=7,
)
with pytest.raises(ValueError, match="16-byte"):
dcp_utils.DirectDCPQGatherWorkspace(
dcp.DirectDCPQGatherWorkspace(
_FakeProcessGroup(),
torch.device("cpu"),
16,
@@ -332,7 +333,7 @@ def _manager_config(dcp_comm_backend: str = "a2a"):
def test_mla_dcp_manager_selects_direct_backends(monkeypatch):
import vllm.v1.attention.ops.dcp_utils as dcp_manager
import vllm.v1.attention.ops.dcp as dcp_manager
group = MagicMock(world_size=2)
monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: group)
@@ -391,7 +392,7 @@ def test_mla_dcp_manager_selects_direct_backends(monkeypatch):
def test_mla_dcp_manager_selects_fallback_backends(monkeypatch):
import vllm.v1.attention.ops.dcp_utils as dcp_manager
import vllm.v1.attention.ops.dcp as dcp_manager
group = MagicMock(world_size=2)
gathered_query = torch.empty(1, 4, 8)
@@ -460,11 +461,11 @@ def test_dcp_workspace_covers_parallel_drafting():
config.num_speculative_tokens = 3
config.speculative_config = MagicMock(parallel_drafting=True)
assert dcp_utils.get_dcp_workspace_max_num_tokens(config) == 28
assert dcp.get_dcp_workspace_max_num_tokens(config) == 28
def test_mla_dcp_manager_selects_pcp_combine(monkeypatch):
import vllm.v1.attention.ops.dcp_utils as dcp_manager
import vllm.v1.attention.ops.dcp as dcp_manager
monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: MagicMock(world_size=2))
manager = dcp_manager.MLADCPManager(
@@ -521,7 +522,7 @@ def test_sparse_mla_builder_initializes_dcp_manager(monkeypatch):
lambda _: MagicMock(kv_lora_rank=8, qk_rope_head_dim=4),
)
manager = object.__new__(dcp_utils.MLADCPManager)
manager = object.__new__(dcp.MLADCPManager)
manager.init_kv_gather = MagicMock()
layer = MagicMock(dcp_manager=manager)
config = MagicMock()
@@ -580,12 +581,12 @@ def _distributed_direct_q_gather_worker(env: dict[str, str]) -> None:
heads_per_rank, head_dim, max_num_tokens = 6, 576, 128
padded_num_heads = 128 if world_size == 4 else None
active_ubatch = [0]
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
for dtype_idx, dtype_name in enumerate(
("bfloat16", "float8_e4m3fn", "float32")
):
dtype = _dtype_from_name(dtype_name)
workspace = dcp_utils.DirectDCPQGatherWorkspace(
workspace = dcp.DirectDCPQGatherWorkspace(
dist.group.WORLD,
device,
max_num_tokens,
@@ -747,11 +748,11 @@ def _distributed_direct_kv_gather_worker(env: dict[str, str]) -> None:
token_dim = 576
max_gathered_tokens = 128 * world_size
active_ubatch = [0]
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
for dtype_idx, dtype_name in enumerate(("bfloat16", "float16")):
dtype = _dtype_from_name(dtype_name)
workspace = dcp_utils.DirectDCPKVGatherWorkspace(
workspace = dcp.DirectDCPKVGatherWorkspace(
dist.group.WORLD,
device,
max_gathered_tokens,
@@ -818,7 +819,7 @@ def _distributed_direct_a2a_worker(env: dict[str, str]) -> None:
torch.accelerator.set_device_index(local_rank)
dist.init_process_group(backend="nccl")
try:
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
rank = dist.get_rank()
world_size = dist.get_world_size()
@@ -829,8 +830,8 @@ def _distributed_direct_a2a_worker(env: dict[str, str]) -> None:
heads_per_rank, head_dim, max_num_tokens = 6, 512, 128
total_heads = world_size * heads_per_rank
active_ubatch = [0]
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
workspace = dcp_utils.DirectDCPA2AWorkspace(
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
workspace = dcp.DirectDCPA2AWorkspace(
dist.group.WORLD,
device,
max_num_tokens,