[refactor] consolidate cp attn ops (#52839)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Woosuk Kwon
parent
bf2866f8bf
commit
58e5ee0158
@@ -44,7 +44,7 @@ def _packed_a2a_reference(
|
||||
h_per_rank: int,
|
||||
is_lse_base_on_e: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
B, _H, D = cp_attn_out.shape
|
||||
outputs = (
|
||||
@@ -157,13 +157,13 @@ class TestLSEWeightedCombine:
|
||||
|
||||
def test_importable(self):
|
||||
"""Verify _lse_weighted_combine is importable."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
assert callable(_lse_weighted_combine)
|
||||
|
||||
def test_single_rank(self):
|
||||
"""Single rank: output unchanged."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
# N=1, B=2, H=4, D=8
|
||||
outputs = torch.randn(1, 2, 4, 8)
|
||||
@@ -176,7 +176,7 @@ class TestLSEWeightedCombine:
|
||||
|
||||
def test_equal_lse(self):
|
||||
"""Equal LSE values: outputs averaged equally."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
_N, B, H, D = 2, 1, 1, 4
|
||||
outputs = torch.tensor(
|
||||
@@ -200,7 +200,7 @@ class TestLSEWeightedCombine:
|
||||
|
||||
def test_dominant_rank(self):
|
||||
"""Different LSE values: larger LSE gets more weight."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
B, H, D = 1, 1, 2
|
||||
outputs = torch.tensor(
|
||||
@@ -222,7 +222,7 @@ class TestLSEWeightedCombine:
|
||||
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
|
||||
|
||||
def test_empty_shard_ignores_undefined_output(self):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]])
|
||||
lses = torch.tensor([[[-float("inf")]], [[0.0]]])
|
||||
@@ -232,7 +232,7 @@ class TestLSEWeightedCombine:
|
||||
torch.testing.assert_close(result, outputs[1])
|
||||
|
||||
def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch):
|
||||
import vllm.v1.attention.ops.common as common
|
||||
import vllm.v1.attention.ops.dcp as dcp
|
||||
|
||||
class FakeGroup:
|
||||
world_size = 2
|
||||
@@ -243,7 +243,7 @@ class TestLSEWeightedCombine:
|
||||
return torch.cat((tensor, tensor), dim=dim)
|
||||
|
||||
monkeypatch.setattr(
|
||||
common,
|
||||
dcp,
|
||||
"correct_attn_out",
|
||||
lambda output, lses, *args, **kwargs: (output, lses[0]),
|
||||
)
|
||||
@@ -252,7 +252,7 @@ class TestLSEWeightedCombine:
|
||||
seq_lens = torch.tensor([0, 2], dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
|
||||
|
||||
_, masked_lse = common._cp_lse_common(
|
||||
_, masked_lse = dcp._cp_lse_common(
|
||||
output,
|
||||
lse,
|
||||
FakeGroup(),
|
||||
@@ -266,7 +266,7 @@ class TestLSEWeightedCombine:
|
||||
|
||||
def test_mathematically_correct(self):
|
||||
"""Verify mathematical correctness of LSE combination."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
outputs = torch.tensor(
|
||||
[
|
||||
@@ -291,7 +291,7 @@ class TestLSEWeightedCombine:
|
||||
|
||||
def test_return_lse(self):
|
||||
"""return_lse=True returns global LSE (logsumexp of inputs)."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
B, H, D = 1, 1, 2
|
||||
outputs = torch.tensor(
|
||||
@@ -317,7 +317,7 @@ class TestLSEWeightedCombine:
|
||||
|
||||
def test_base2_return_lse(self):
|
||||
"""Base-2 LSE mode returns log2-sum-exp2 global LSE."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
outputs = torch.tensor(
|
||||
[
|
||||
@@ -354,7 +354,7 @@ class TestLSEWeightedCombine:
|
||||
|
||||
def test_lse_pack_dim(self):
|
||||
"""Packed A2A stores one fp32 LSE in output-dtype lanes."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _dcp_a2a_lse_pack_dim
|
||||
from vllm.v1.attention.ops.dcp import _dcp_a2a_lse_pack_dim
|
||||
|
||||
assert _dcp_a2a_lse_pack_dim(torch.bfloat16) == 2
|
||||
assert _dcp_a2a_lse_pack_dim(torch.float16) == 2
|
||||
@@ -374,7 +374,7 @@ class TestPackedA2AKernels:
|
||||
return_lse: bool,
|
||||
is_lse_base_on_e: bool,
|
||||
):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import (
|
||||
from vllm.v1.attention.ops.dcp import (
|
||||
_dcp_a2a_lse_pack_dim,
|
||||
_dcp_a2a_pack_send,
|
||||
_dcp_a2a_unpack_combine,
|
||||
@@ -421,7 +421,7 @@ class TestPackedA2AKernels:
|
||||
torch.accelerator.device_count() < 1, reason="CUDA is required."
|
||||
)
|
||||
def test_empty_seq_lens_ignore_undefined_output(self):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import (
|
||||
from vllm.v1.attention.ops.dcp import (
|
||||
_dcp_a2a_lse_pack_dim,
|
||||
_dcp_a2a_pack_send,
|
||||
_dcp_a2a_unpack_combine,
|
||||
@@ -505,7 +505,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
|
||||
init_workspace_manager(torch.device(f"cuda:{local_rank}"))
|
||||
try:
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.attention.ops.dcp import dcp_a2a_lse_reduce
|
||||
|
||||
dtype = _dtype_from_name(env["TEST_DTYPE"])
|
||||
return_lse = env["RETURN_LSE"] == "1"
|
||||
@@ -555,7 +555,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
[t[:, rank * h_per_rank : (rank + 1) * h_per_rank] for t in gathered_lse],
|
||||
dim=0,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
expected_out, expected_lse = _lse_weighted_combine(
|
||||
outputs,
|
||||
|
||||
@@ -10,7 +10,8 @@ import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.v1.attention.ops.dcp_utils as dcp_utils
|
||||
import vllm.v1.attention.ops.cp_common as cp_common
|
||||
import vllm.v1.attention.ops.dcp as dcp
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
@@ -142,17 +143,17 @@ class _FakeProcessGroup:
|
||||
class TestDirectDCPGating:
|
||||
def test_env_disabled_returns_none(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
|
||||
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
|
||||
workspace = dcp_utils.get_direct_dcp_a2a_workspace(
|
||||
dcp.get_direct_dcp_a2a_workspace.cache_clear()
|
||||
workspace = dcp.get_direct_dcp_a2a_workspace(
|
||||
_FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.bfloat16, 1
|
||||
)
|
||||
assert workspace is None
|
||||
|
||||
def test_forced_with_unsupported_dtype_raises(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
|
||||
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
|
||||
dcp.get_direct_dcp_a2a_workspace.cache_clear()
|
||||
with pytest.raises(ValueError, match="does not support"):
|
||||
dcp_utils.get_direct_dcp_a2a_workspace(
|
||||
dcp.get_direct_dcp_a2a_workspace(
|
||||
_FakeGroupCoordinator(),
|
||||
torch.device("cpu"),
|
||||
16,
|
||||
@@ -164,14 +165,14 @@ class TestDirectDCPGating:
|
||||
|
||||
def test_zero_ubatches_raises(self):
|
||||
with pytest.raises(ValueError, match="ubatch"):
|
||||
dcp_utils.DirectDCPA2AWorkspace(
|
||||
dcp.DirectDCPA2AWorkspace(
|
||||
None, torch.device("cpu"), 16, 2, 32, torch.bfloat16, num_ubatches=0
|
||||
)
|
||||
|
||||
def test_auto_with_unsupported_dtype_returns_none(self, monkeypatch):
|
||||
monkeypatch.delenv("VLLM_USE_DIRECT_DCP_A2A", raising=False)
|
||||
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
|
||||
workspace = dcp_utils.get_direct_dcp_a2a_workspace(
|
||||
dcp.get_direct_dcp_a2a_workspace.cache_clear()
|
||||
workspace = dcp.get_direct_dcp_a2a_workspace(
|
||||
_FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.float32, 1
|
||||
)
|
||||
assert workspace is None
|
||||
@@ -179,8 +180,8 @@ class TestDirectDCPGating:
|
||||
def test_q_gather_env_disabled_returns_none(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "0")
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
|
||||
dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear()
|
||||
workspace = dcp_utils.get_direct_dcp_q_gather_workspace(
|
||||
dcp.get_direct_dcp_q_gather_workspace.cache_clear()
|
||||
workspace = dcp.get_direct_dcp_q_gather_workspace(
|
||||
_FakeGroupCoordinator(),
|
||||
torch.device("cpu"),
|
||||
16,
|
||||
@@ -194,17 +195,17 @@ class TestDirectDCPGating:
|
||||
def test_q_gather_flag_is_independent(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "1")
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
|
||||
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True)
|
||||
dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear()
|
||||
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: True)
|
||||
dcp.get_direct_dcp_q_gather_workspace.cache_clear()
|
||||
workspace = object()
|
||||
init_workspace = MagicMock(return_value=workspace)
|
||||
monkeypatch.setattr(
|
||||
dcp_utils,
|
||||
dcp,
|
||||
"DirectDCPQGatherWorkspace",
|
||||
init_workspace,
|
||||
)
|
||||
|
||||
result = dcp_utils.get_direct_dcp_q_gather_workspace(
|
||||
result = dcp.get_direct_dcp_q_gather_workspace(
|
||||
_FakeGroupCoordinator(),
|
||||
torch.device("cpu"),
|
||||
16,
|
||||
@@ -220,8 +221,8 @@ class TestDirectDCPGating:
|
||||
def test_kv_gather_env_disabled_returns_none(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "0")
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
|
||||
dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear()
|
||||
workspace = dcp_utils.get_direct_dcp_kv_gather_workspace(
|
||||
dcp.get_direct_dcp_kv_gather_workspace.cache_clear()
|
||||
workspace = dcp.get_direct_dcp_kv_gather_workspace(
|
||||
_FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1
|
||||
)
|
||||
assert workspace is None
|
||||
@@ -229,17 +230,17 @@ class TestDirectDCPGating:
|
||||
def test_kv_gather_flag_is_independent(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "1")
|
||||
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
|
||||
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True)
|
||||
dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear()
|
||||
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: True)
|
||||
dcp.get_direct_dcp_kv_gather_workspace.cache_clear()
|
||||
workspace = object()
|
||||
init_workspace = MagicMock(return_value=workspace)
|
||||
monkeypatch.setattr(
|
||||
dcp_utils,
|
||||
dcp,
|
||||
"DirectDCPKVGatherWorkspace",
|
||||
init_workspace,
|
||||
)
|
||||
|
||||
result = dcp_utils.get_direct_dcp_kv_gather_workspace(
|
||||
result = dcp.get_direct_dcp_kv_gather_workspace(
|
||||
_FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1
|
||||
)
|
||||
|
||||
@@ -267,9 +268,9 @@ class TestDirectDCPGating:
|
||||
factory_name,
|
||||
factory_args,
|
||||
):
|
||||
factory = getattr(dcp_utils, factory_name)
|
||||
factory = getattr(dcp, factory_name)
|
||||
monkeypatch.setenv(flag_name, "1")
|
||||
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: False)
|
||||
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: False)
|
||||
factory.cache_clear()
|
||||
|
||||
assert (
|
||||
@@ -283,25 +284,25 @@ class TestDirectDCPGating:
|
||||
|
||||
def test_kv_gather_rejects_invalid_workspace_geometry(self):
|
||||
with pytest.raises(ValueError, match="ubatch"):
|
||||
dcp_utils.DirectDCPKVGatherWorkspace(
|
||||
dcp.DirectDCPKVGatherWorkspace(
|
||||
None, torch.device("cpu"), 64, 576, num_ubatches=0
|
||||
)
|
||||
with pytest.raises(ValueError, match="divide evenly"):
|
||||
dcp_utils.DirectDCPKVGatherWorkspace(
|
||||
dcp.DirectDCPKVGatherWorkspace(
|
||||
_FakeProcessGroup(), torch.device("cpu"), 63, 576
|
||||
)
|
||||
with pytest.raises(ValueError, match="16-byte"):
|
||||
dcp_utils.DirectDCPKVGatherWorkspace(
|
||||
dcp.DirectDCPKVGatherWorkspace(
|
||||
_FakeProcessGroup(), torch.device("cpu"), 64, 3
|
||||
)
|
||||
|
||||
def test_q_gather_rejects_invalid_workspace_geometry(self):
|
||||
with pytest.raises(ValueError, match="ubatch"):
|
||||
dcp_utils.DirectDCPQGatherWorkspace(
|
||||
dcp.DirectDCPQGatherWorkspace(
|
||||
None, torch.device("cpu"), 16, 2, 32, num_ubatches=0
|
||||
)
|
||||
with pytest.raises(ValueError, match="padded heads"):
|
||||
dcp_utils.DirectDCPQGatherWorkspace(
|
||||
dcp.DirectDCPQGatherWorkspace(
|
||||
_FakeProcessGroup(),
|
||||
torch.device("cpu"),
|
||||
16,
|
||||
@@ -310,7 +311,7 @@ class TestDirectDCPGating:
|
||||
padded_num_heads=7,
|
||||
)
|
||||
with pytest.raises(ValueError, match="16-byte"):
|
||||
dcp_utils.DirectDCPQGatherWorkspace(
|
||||
dcp.DirectDCPQGatherWorkspace(
|
||||
_FakeProcessGroup(),
|
||||
torch.device("cpu"),
|
||||
16,
|
||||
@@ -332,7 +333,7 @@ def _manager_config(dcp_comm_backend: str = "a2a"):
|
||||
|
||||
|
||||
def test_mla_dcp_manager_selects_direct_backends(monkeypatch):
|
||||
import vllm.v1.attention.ops.dcp_utils as dcp_manager
|
||||
import vllm.v1.attention.ops.dcp as dcp_manager
|
||||
|
||||
group = MagicMock(world_size=2)
|
||||
monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: group)
|
||||
@@ -391,7 +392,7 @@ def test_mla_dcp_manager_selects_direct_backends(monkeypatch):
|
||||
|
||||
|
||||
def test_mla_dcp_manager_selects_fallback_backends(monkeypatch):
|
||||
import vllm.v1.attention.ops.dcp_utils as dcp_manager
|
||||
import vllm.v1.attention.ops.dcp as dcp_manager
|
||||
|
||||
group = MagicMock(world_size=2)
|
||||
gathered_query = torch.empty(1, 4, 8)
|
||||
@@ -460,11 +461,11 @@ def test_dcp_workspace_covers_parallel_drafting():
|
||||
config.num_speculative_tokens = 3
|
||||
config.speculative_config = MagicMock(parallel_drafting=True)
|
||||
|
||||
assert dcp_utils.get_dcp_workspace_max_num_tokens(config) == 28
|
||||
assert dcp.get_dcp_workspace_max_num_tokens(config) == 28
|
||||
|
||||
|
||||
def test_mla_dcp_manager_selects_pcp_combine(monkeypatch):
|
||||
import vllm.v1.attention.ops.dcp_utils as dcp_manager
|
||||
import vllm.v1.attention.ops.dcp as dcp_manager
|
||||
|
||||
monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: MagicMock(world_size=2))
|
||||
manager = dcp_manager.MLADCPManager(
|
||||
@@ -521,7 +522,7 @@ def test_sparse_mla_builder_initializes_dcp_manager(monkeypatch):
|
||||
lambda _: MagicMock(kv_lora_rank=8, qk_rope_head_dim=4),
|
||||
)
|
||||
|
||||
manager = object.__new__(dcp_utils.MLADCPManager)
|
||||
manager = object.__new__(dcp.MLADCPManager)
|
||||
manager.init_kv_gather = MagicMock()
|
||||
layer = MagicMock(dcp_manager=manager)
|
||||
config = MagicMock()
|
||||
@@ -580,12 +581,12 @@ def _distributed_direct_q_gather_worker(env: dict[str, str]) -> None:
|
||||
heads_per_rank, head_dim, max_num_tokens = 6, 576, 128
|
||||
padded_num_heads = 128 if world_size == 4 else None
|
||||
active_ubatch = [0]
|
||||
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
|
||||
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
|
||||
for dtype_idx, dtype_name in enumerate(
|
||||
("bfloat16", "float8_e4m3fn", "float32")
|
||||
):
|
||||
dtype = _dtype_from_name(dtype_name)
|
||||
workspace = dcp_utils.DirectDCPQGatherWorkspace(
|
||||
workspace = dcp.DirectDCPQGatherWorkspace(
|
||||
dist.group.WORLD,
|
||||
device,
|
||||
max_num_tokens,
|
||||
@@ -747,11 +748,11 @@ def _distributed_direct_kv_gather_worker(env: dict[str, str]) -> None:
|
||||
token_dim = 576
|
||||
max_gathered_tokens = 128 * world_size
|
||||
active_ubatch = [0]
|
||||
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
|
||||
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
|
||||
|
||||
for dtype_idx, dtype_name in enumerate(("bfloat16", "float16")):
|
||||
dtype = _dtype_from_name(dtype_name)
|
||||
workspace = dcp_utils.DirectDCPKVGatherWorkspace(
|
||||
workspace = dcp.DirectDCPKVGatherWorkspace(
|
||||
dist.group.WORLD,
|
||||
device,
|
||||
max_gathered_tokens,
|
||||
@@ -818,7 +819,7 @@ def _distributed_direct_a2a_worker(env: dict[str, str]) -> None:
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
dist.init_process_group(backend="nccl")
|
||||
try:
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
|
||||
|
||||
rank = dist.get_rank()
|
||||
world_size = dist.get_world_size()
|
||||
@@ -829,8 +830,8 @@ def _distributed_direct_a2a_worker(env: dict[str, str]) -> None:
|
||||
heads_per_rank, head_dim, max_num_tokens = 6, 512, 128
|
||||
total_heads = world_size * heads_per_rank
|
||||
active_ubatch = [0]
|
||||
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
|
||||
workspace = dcp_utils.DirectDCPA2AWorkspace(
|
||||
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
|
||||
workspace = dcp.DirectDCPA2AWorkspace(
|
||||
dist.group.WORLD,
|
||||
device,
|
||||
max_num_tokens,
|
||||
|
||||
@@ -15,7 +15,7 @@ requires_flashinfer_mla = pytest.mark.skipif(
|
||||
|
||||
|
||||
def test_mla_dcp_gathered_query_reserves_backend_head_storage():
|
||||
from vllm.v1.attention.ops.dcp_utils import reserve_query_head_storage
|
||||
from vllm.v1.attention.ops.dcp import reserve_query_head_storage
|
||||
|
||||
query = torch.randn(3, 24, 576, dtype=torch.bfloat16)
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
triton_filter_and_convert_dcp_index,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.attention.ops.common import CPTritonContext, correct_attn_out
|
||||
from vllm.v1.attention.ops.dcp import CPTritonContext, correct_attn_out
|
||||
|
||||
|
||||
def _local_count(length: int, rank: int, world: int, interleave: int) -> int:
|
||||
|
||||
@@ -242,10 +242,6 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
from vllm.model_executor.layers.attention.kv_transfer_utils import (
|
||||
maybe_transfer_kv_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -293,8 +289,12 @@ from vllm.v1.attention.backends.utils import (
|
||||
get_num_attention_heads_from_layers,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.ops.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
|
||||
@@ -32,7 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -13,7 +13,6 @@ from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.attention.pcp import maybe_gather_indexer_k
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
@@ -35,6 +34,7 @@ from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadata,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
|
||||
from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -13,10 +13,6 @@ from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.attention.attention import get_attention_context
|
||||
from vllm.model_executor.layers.attention.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -42,6 +38,10 @@ from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.ops.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
|
||||
@@ -92,7 +92,7 @@ from vllm.v1.attention.backend import (
|
||||
MLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, get_kv_quant_mode
|
||||
|
||||
@@ -886,7 +886,7 @@ class AttentionImplBase(ABC, Generic[T]):
|
||||
# False => base 2 (lse = log2(sum(exp(qk))))
|
||||
# -- e.g. FlashInfer trtllm-gen MLA
|
||||
# The DCP combine kernel (cp_lse_ag_out_rs / dcp_a2a_lse_reduce in
|
||||
# vllm/v1/attention/ops/common.py) branches on this via its IS_BASE_E
|
||||
# vllm/v1/attention/ops/dcp.py) branches on this via its IS_BASE_E
|
||||
# constexpr; getting it wrong silently corrupts the cross-shard
|
||||
# softmax denominator.
|
||||
lse_base_on_e: bool = True
|
||||
|
||||
@@ -34,8 +34,10 @@ from vllm.v1.attention.backends.utils import (
|
||||
fill_mm_prefix_query_ranges,
|
||||
get_dcp_local_seq_lens,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.attention.ops.dcp import (
|
||||
cp_lse_ag_out_rs,
|
||||
dcp_a2a_lse_reduce,
|
||||
)
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
|
||||
@@ -75,8 +75,10 @@ from vllm.v1.attention.backends.utils import (
|
||||
infer_global_hyperparameters,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.attention.ops.dcp import (
|
||||
cp_lse_ag_out_rs,
|
||||
dcp_a2a_lse_reduce,
|
||||
)
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
|
||||
@@ -2,308 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def mask_dcp_empty_shards_(
|
||||
lse: torch.Tensor,
|
||||
seq_lens: torch.Tensor | None,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
) -> None:
|
||||
if seq_lens is None and query_start_loc is None:
|
||||
return
|
||||
if seq_lens is None or query_start_loc is None:
|
||||
raise ValueError("seq_lens and query_start_loc must be provided together")
|
||||
if (
|
||||
seq_lens.ndim != 1
|
||||
or query_start_loc.ndim != 1
|
||||
or query_start_loc.shape[0] != seq_lens.shape[0] + 1
|
||||
):
|
||||
raise ValueError("query_start_loc must contain one boundary per sequence")
|
||||
|
||||
row_indices = torch.arange(
|
||||
lse.shape[0], device=lse.device, dtype=query_start_loc.dtype
|
||||
)
|
||||
sequence_indices = torch.searchsorted(
|
||||
query_start_loc[1:], row_indices, right=True
|
||||
).clamp_max(seq_lens.shape[0] - 1)
|
||||
empty_rows = (row_indices >= query_start_loc[-1]) | (
|
||||
seq_lens[sequence_indices] == 0
|
||||
)
|
||||
lse.masked_fill_(empty_rows[:, None], float("-inf"))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _correct_attn_cp_out_kernel(
|
||||
outputs_ptr,
|
||||
new_output_ptr,
|
||||
lses_ptr,
|
||||
vlse_ptr,
|
||||
outputs_stride_B,
|
||||
outputs_stride_H,
|
||||
outputs_stride_D,
|
||||
lses_stride_N,
|
||||
lses_stride_B,
|
||||
lses_stride_H,
|
||||
lse_idx,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
N_ROUNDED: tl.constexpr,
|
||||
IS_BASE_E: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Apply the all-gathered lses to correct each local rank's attention
|
||||
output. we still need perform a cross-rank reduction to obtain the
|
||||
final attention output.
|
||||
|
||||
Args:
|
||||
outputs_ptr (triton.PointerType):
|
||||
Pointer to input tensor of shape [ B, H, D ]
|
||||
lses_ptr (triton.PointerType):
|
||||
Pointer to input tensor of shape [ N, B, H ]
|
||||
new_output_ptr (triton.PointerType):
|
||||
Pointer to output tensor of shape [ B, H, D ]
|
||||
vlse_ptr (triton.PointerType):
|
||||
Pointer to output tensor of shape [ B, H ]
|
||||
"""
|
||||
batch_idx = tl.program_id(axis=0).to(tl.int64)
|
||||
head_idx = tl.program_id(axis=1).to(tl.int64)
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
num_n_offsets = tl.arange(0, N_ROUNDED)
|
||||
|
||||
# shape = [N]
|
||||
lse_offsets = (
|
||||
num_n_offsets * lses_stride_N
|
||||
+ batch_idx * lses_stride_B
|
||||
+ head_idx * lses_stride_H
|
||||
)
|
||||
|
||||
# calc final lse
|
||||
lse = tl.load(lses_ptr + lse_offsets).to(tl.float32)
|
||||
lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse)
|
||||
lse_max = tl.max(lse, axis=0)
|
||||
lse_max = tl.where(lse_max == -float("inf"), 0, lse_max)
|
||||
lse -= lse_max
|
||||
if IS_BASE_E:
|
||||
lse_exp = tl.exp(lse)
|
||||
lse_acc = tl.sum(lse_exp, axis=0)
|
||||
lse = tl.log(lse_acc)
|
||||
else:
|
||||
lse_exp = tl.exp2(lse)
|
||||
lse_acc = tl.sum(lse_exp, axis=0)
|
||||
lse = tl.log2(lse_acc)
|
||||
lse += lse_max
|
||||
|
||||
lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H
|
||||
tl.store(vlse_ptr + lse_offsets, lse)
|
||||
|
||||
# shape = [D]
|
||||
output_offsets = (
|
||||
batch_idx * outputs_stride_B
|
||||
+ head_idx * outputs_stride_H
|
||||
+ d_offsets * outputs_stride_D
|
||||
)
|
||||
|
||||
# correct output
|
||||
lse_offset = (
|
||||
lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H
|
||||
)
|
||||
lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32)
|
||||
lse_finally = lse_tmp - lse
|
||||
lse_finally = tl.where(
|
||||
(lse_finally != lse_finally) | (lse_finally == float("inf")),
|
||||
-float("inf"),
|
||||
lse_finally,
|
||||
)
|
||||
factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally)
|
||||
output = tl.load(outputs_ptr + output_offsets)
|
||||
output = output * factor
|
||||
output = tl.where(factor == 0.0, 0.0, output)
|
||||
|
||||
tl.store(new_output_ptr + output_offsets, output)
|
||||
|
||||
|
||||
class CPTritonContext:
|
||||
"""The CPTritonContext is used to avoid recompilation of the Triton JIT."""
|
||||
|
||||
def __init__(self):
|
||||
self.inner_kernel = None
|
||||
|
||||
def call_kernel(self, kernel, grid, *regular_args, **const_args):
|
||||
if self.inner_kernel is None:
|
||||
self.inner_kernel = kernel[grid](*regular_args, **const_args)
|
||||
else:
|
||||
self.inner_kernel[grid](*regular_args)
|
||||
|
||||
|
||||
def correct_attn_out(
|
||||
out: torch.Tensor,
|
||||
lses: torch.Tensor,
|
||||
cp_rank: int,
|
||||
ctx: CPTritonContext,
|
||||
is_lse_base_on_e: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Correct the attention output using the all-gathered lses.
|
||||
|
||||
Args:
|
||||
out: Tensor of shape [ B, H, D ]
|
||||
lses: Tensor of shape [ N, B, H ]
|
||||
cp_rank: Current rank in the context-parallel group
|
||||
ctx: Triton context to avoid recompilation
|
||||
|
||||
Returns:
|
||||
Tuple of (out, lse) with corrected attention and final log-sum-exp.
|
||||
"""
|
||||
if ctx is None:
|
||||
ctx = CPTritonContext()
|
||||
|
||||
# --- Normalize to 3D views ---
|
||||
if out.ndim == 4 and out.shape[1] == 1:
|
||||
out = out.squeeze(1)
|
||||
assert out.ndim == 3, f"expected out [B,H,D] or [B,1,H,D], got {tuple(out.shape)}"
|
||||
|
||||
if lses.ndim == 4 and lses.shape[-1] == 1:
|
||||
lses = lses.squeeze(-1)
|
||||
if lses.ndim == 4 and lses.shape[1] == 1:
|
||||
lses = lses.squeeze(1)
|
||||
assert lses.ndim == 3, (
|
||||
f"expected lses [N,B,H] (optionally with a 1-sized extra dim), "
|
||||
f"got {tuple(lses.shape)}"
|
||||
)
|
||||
|
||||
B, H, D = out.shape
|
||||
N = lses.shape[0]
|
||||
|
||||
# Strides after we normalized shapes to 3-D views. The kernel computes
|
||||
# offsets for `vlse_ptr` using lses_stride_B/H, so the output buffer must
|
||||
# have the same B/H stride layout as a slice of `lses`.
|
||||
o_sB, o_sH, o_sD = out.stride()
|
||||
l_sN, l_sB, l_sH = lses.stride()
|
||||
|
||||
# Allocate LSE with the same B/H strides as `lses` so writes land correctly
|
||||
# even when `lses` is a non-contiguous view (e.g., 4-D to 3-D squeeze).
|
||||
lse = torch.empty_strided(
|
||||
(B, H), (l_sB, l_sH), device=lses.device, dtype=lses.dtype
|
||||
)
|
||||
|
||||
# Kernel launch config
|
||||
grid = (B, H, 1)
|
||||
|
||||
regular_args = (
|
||||
out,
|
||||
out,
|
||||
lses,
|
||||
lse,
|
||||
o_sB,
|
||||
o_sH,
|
||||
o_sD,
|
||||
l_sN,
|
||||
l_sB,
|
||||
l_sH,
|
||||
cp_rank,
|
||||
)
|
||||
const_args = {"HEAD_DIM": D, "N_ROUNDED": N, "IS_BASE_E": is_lse_base_on_e}
|
||||
ctx.call_kernel(_correct_attn_cp_out_kernel, grid, *regular_args, **const_args)
|
||||
return out, lse
|
||||
|
||||
|
||||
def _cp_lse_common(
|
||||
cp_attn_out: torch.Tensor,
|
||||
cp_attn_lse: torch.Tensor,
|
||||
cp_group: GroupCoordinator,
|
||||
ctx: CPTritonContext | None = None,
|
||||
is_lse_base_on_e=True,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
cp_attn_out: [ B, H, D ]
|
||||
cp_attn_lse: [ B, H ]
|
||||
"""
|
||||
if cp_group.world_size == 1:
|
||||
return cp_attn_out
|
||||
|
||||
if ctx is None:
|
||||
ctx = CPTritonContext()
|
||||
|
||||
cp_attn_lse = cp_attn_lse.contiguous()
|
||||
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
|
||||
lses = cp_group.all_gather(cp_attn_lse, dim=0).reshape(
|
||||
(cp_group.world_size,) + cp_attn_lse.shape
|
||||
)
|
||||
out, lse = correct_attn_out(
|
||||
cp_attn_out,
|
||||
lses,
|
||||
cp_group.rank_in_group,
|
||||
ctx,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
return out, lse
|
||||
|
||||
|
||||
def cp_lse_ag_out_rs(
|
||||
cp_attn_out: torch.Tensor,
|
||||
cp_attn_lse: torch.Tensor,
|
||||
cp_group: GroupCoordinator,
|
||||
ctx: CPTritonContext | None = None,
|
||||
return_lse: bool = False,
|
||||
is_lse_base_on_e=True,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
cp_attn_out: [ B, H, D ]
|
||||
cp_attn_lse: [ B, H ]
|
||||
"""
|
||||
out, lse = _cp_lse_common(
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
cp_group,
|
||||
ctx=ctx,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
out = cp_group.reduce_scatter(out, dim=1)
|
||||
|
||||
if return_lse:
|
||||
cp_num_heads = lse.shape[1] // cp_group.world_size
|
||||
cp_rank = cp_group.rank_in_group
|
||||
lse = lse[:, cp_num_heads * cp_rank : cp_num_heads * (cp_rank + 1)]
|
||||
return out, lse
|
||||
return out
|
||||
|
||||
|
||||
def cp_lse_ag_out_ar(
|
||||
cp_attn_out: torch.Tensor,
|
||||
cp_attn_lse: torch.Tensor,
|
||||
cp_group: GroupCoordinator,
|
||||
ctx: CPTritonContext | None = None,
|
||||
return_lse: bool = False,
|
||||
is_lse_base_on_e=True,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
cp_attn_out: [ B, H, D ]
|
||||
cp_attn_lse: [ B, H ]
|
||||
"""
|
||||
out, lse = _cp_lse_common(
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
cp_group,
|
||||
ctx=ctx,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
out = cp_group.all_reduce(out)
|
||||
|
||||
if return_lse:
|
||||
return out, lse
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_seq_kernel(
|
||||
x_ptr, # [N, D]
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared symmetric-memory infrastructure for context-parallel attention."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.parallel_state import in_the_same_node_as
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
try:
|
||||
import torch.distributed._symmetric_memory as symm_mem
|
||||
|
||||
symm_mem_available = True
|
||||
except ImportError:
|
||||
symm_mem = None # type: ignore[assignment]
|
||||
symm_mem_available = False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _symm_mem_spans_group(group: GroupCoordinator) -> bool:
|
||||
"""Probe whether the group has NVLS symmetric memory."""
|
||||
if not symm_mem_available:
|
||||
return False
|
||||
try:
|
||||
from torch._C._autograd import DeviceType
|
||||
from torch._C._distributed_c10d import _SymmetricMemory
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index):
|
||||
return False
|
||||
probe = symm_mem.empty(8, dtype=torch.uint8, device=device)
|
||||
probe.zero_()
|
||||
torch.accelerator.synchronize()
|
||||
handle = symm_mem.rendezvous(probe, group.device_group.group_name)
|
||||
spans = handle is not None and handle.multicast_ptr != 0
|
||||
except Exception as error:
|
||||
logger.debug("Direct CP symmetric-memory probe failed: %s", error)
|
||||
return False
|
||||
logger.debug_once(
|
||||
"Direct CP symmetric memory across %d ranks: %s",
|
||||
group.world_size,
|
||||
"available" if spans else "unavailable",
|
||||
)
|
||||
return spans
|
||||
|
||||
|
||||
def direct_cp_enabled(
|
||||
group: GroupCoordinator,
|
||||
dtype: torch.dtype,
|
||||
use_direct: bool | None,
|
||||
supported_dtypes: tuple[torch.dtype, ...] | None = None,
|
||||
) -> bool:
|
||||
if use_direct is not None:
|
||||
return use_direct
|
||||
return (
|
||||
symm_mem_available
|
||||
and current_platform.is_cuda()
|
||||
and (supported_dtypes is None or dtype in supported_dtypes)
|
||||
and (
|
||||
all(in_the_same_node_as(group.cpu_group, source_rank=0))
|
||||
or _symm_mem_spans_group(group)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def direct_cp_multicast_enabled(
|
||||
group: GroupCoordinator,
|
||||
dtype: torch.dtype,
|
||||
use_direct: bool | None,
|
||||
supported_dtypes: tuple[torch.dtype, ...] | None = None,
|
||||
) -> bool:
|
||||
return direct_cp_enabled(
|
||||
group, dtype, use_direct, supported_dtypes
|
||||
) and _symm_mem_spans_group(group)
|
||||
|
||||
|
||||
class DirectCPWorkspace:
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
num_ubatches: int,
|
||||
) -> None:
|
||||
self.group = group
|
||||
self.world_size = group.size()
|
||||
self.rank = group.rank()
|
||||
self.device = torch.device(device)
|
||||
self.num_ubatches = num_ubatches
|
||||
self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device)
|
||||
self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = []
|
||||
|
||||
def _allocate(
|
||||
self, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
storage = symm_mem.empty(shape, device=self.device, dtype=dtype)
|
||||
storage.zero_()
|
||||
torch.accelerator.synchronize()
|
||||
handle = symm_mem.rendezvous(storage, self.group.group_name)
|
||||
assert handle is not None, "CP symmetric memory rendezvous returned None"
|
||||
handle.barrier()
|
||||
views = [
|
||||
handle.get_buffer(peer, list(shape), dtype, 0)
|
||||
for peer in range(self.world_size)
|
||||
]
|
||||
self.device = storage.device
|
||||
peer_ptrs = torch.tensor(
|
||||
[
|
||||
[view[ubatch].data_ptr() for view in views]
|
||||
for ubatch in range(self.num_ubatches)
|
||||
],
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
self._allocations.append((storage, handle, views))
|
||||
return storage, peer_ptrs
|
||||
|
||||
def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]:
|
||||
disabled = [0] * self.num_ubatches
|
||||
for allocated, handle, _ in self._allocations:
|
||||
if allocated is storage:
|
||||
break
|
||||
else:
|
||||
return disabled
|
||||
try:
|
||||
from torch._C._autograd import DeviceType
|
||||
from torch._C._distributed_c10d import _SymmetricMemory
|
||||
|
||||
if not _SymmetricMemory.has_multicast_support(
|
||||
DeviceType.CUDA, storage.device.index
|
||||
):
|
||||
return disabled
|
||||
multicast_base = handle.multicast_ptr
|
||||
except Exception:
|
||||
return disabled
|
||||
if not multicast_base:
|
||||
return disabled
|
||||
storage_base = storage.data_ptr()
|
||||
return [
|
||||
multicast_base + (storage[ubatch].data_ptr() - storage_base)
|
||||
for ubatch in range(self.num_ubatches)
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,470 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
DCP All-to-All communication backend for attention.
|
||||
|
||||
Provides All-to-All (A2A) communication as an alternative to
|
||||
AllGather + ReduceScatter (AG+RS) for Decode Context Parallel (DCP).
|
||||
Instead of gathering the full Q tensor and scattering partial outputs,
|
||||
A2A exchanges partial attention outputs and their LSE values across
|
||||
ranks, then combines them with exact LSE-weighted reduction.
|
||||
|
||||
This reduces the number of NCCL calls per attention layer by exchanging
|
||||
the partial output and LSE in a single packed All-to-All payload.
|
||||
|
||||
Usage:
|
||||
vllm serve model --tp 16 --dcp 16 --dcp-comm-backend a2a
|
||||
|
||||
Reference: https://arxiv.org/abs/2507.07120
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.common import mask_dcp_empty_shards_
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.v1.attention.ops.common import CPTritonContext
|
||||
|
||||
|
||||
def _lse_weighted_combine(
|
||||
outputs: torch.Tensor,
|
||||
lses: torch.Tensor,
|
||||
return_lse: bool = False,
|
||||
is_lse_base_on_e: bool = True,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
CPU reference implementation for LSE-weighted combination.
|
||||
|
||||
This is a pure PyTorch implementation used for testing and validation.
|
||||
|
||||
Args:
|
||||
outputs: Partial attention outputs [N, B, H, D]
|
||||
N = number of KV shards (ranks)
|
||||
B = batch size (num_tokens)
|
||||
H = number of heads per rank
|
||||
D = head dimension
|
||||
lses: Log-sum-exp values [N, B, H]
|
||||
return_lse: If True, also return the global LSE
|
||||
is_lse_base_on_e: If True, LSE is base e; if False, base 2
|
||||
|
||||
Returns:
|
||||
Combined output [B, H, D], and optionally global LSE [B, H]
|
||||
"""
|
||||
N, B, H, D = outputs.shape
|
||||
|
||||
# Handle NaN and inf in LSEs
|
||||
lses = torch.where(
|
||||
torch.isnan(lses) | torch.isinf(lses),
|
||||
torch.tensor(float("-inf"), device=lses.device, dtype=lses.dtype),
|
||||
lses,
|
||||
)
|
||||
|
||||
# Compute max LSE for numerical stability
|
||||
lse_max, _ = lses.max(dim=0) # [B, H]
|
||||
lse_max = torch.where(
|
||||
lse_max == float("-inf"),
|
||||
torch.zeros_like(lse_max),
|
||||
lse_max,
|
||||
)
|
||||
|
||||
# Compute weights: softmax over the N dimension
|
||||
if is_lse_base_on_e:
|
||||
weights = torch.exp(lses - lse_max.unsqueeze(0)) # [N, B, H]
|
||||
else:
|
||||
weights = torch.pow(2.0, lses - lse_max.unsqueeze(0)) # [N, B, H]
|
||||
|
||||
# Handle NaN weights
|
||||
weights = torch.where(torch.isnan(weights), torch.zeros_like(weights), weights)
|
||||
|
||||
# Normalize weights
|
||||
weight_sum = weights.sum(dim=0, keepdim=True) # [1, B, H]
|
||||
weights = weights / weight_sum.clamp(min=1e-10) # [N, B, H]
|
||||
|
||||
# Weighted combination: sum over N dimension
|
||||
weights = weights.unsqueeze(-1)
|
||||
outputs = torch.where(weights == 0, torch.zeros_like(outputs), outputs)
|
||||
result = (outputs * weights).sum(dim=0) # [B, H, D]
|
||||
|
||||
if return_lse:
|
||||
if is_lse_base_on_e:
|
||||
global_lse = torch.log(weight_sum.squeeze(0)) + lse_max # [B, H]
|
||||
else:
|
||||
global_lse = torch.log2(weight_sum.squeeze(0)) + lse_max # [B, H]
|
||||
return result, global_lse
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _dcp_a2a_lse_pack_dim(output_dtype: torch.dtype) -> int:
|
||||
bits = torch.finfo(output_dtype).bits
|
||||
if bits == 16:
|
||||
return 2
|
||||
if bits == 32:
|
||||
return 1
|
||||
raise ValueError(f"Cannot pack fp32 LSE into output dtype {output_dtype}.")
|
||||
|
||||
|
||||
def _dcp_a2a_send_recv_buffers(
|
||||
shape: tuple[int, ...],
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Don't use the shared WorkspaceManager here. A FULL cudagraph bakes in the
|
||||
# buffer address at capture, but the workspace is growable and sized only to
|
||||
# the largest *captured* batch (the cudagraph capture cap). Any eager a2a
|
||||
# with a bigger batch regrows it, freeing that address and poisoning every
|
||||
# captured graph -> illegal memory access on replay. This bites the very
|
||||
# first request: the post-capture warmup runs an eager decode at
|
||||
# max_num_seqs (> the cap), so the graphs are already dangling before the
|
||||
# server is ready. torch.empty buffers instead live in the graph's private
|
||||
# pool and stay valid for its lifetime (as _dcp_a2a_unpack_combine and the
|
||||
# AG+RS combine path already rely on).
|
||||
return (
|
||||
torch.empty(shape, device=device, dtype=dtype),
|
||||
torch.empty(shape, device=device, dtype=dtype),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dcp_a2a_pack_send_kernel(
|
||||
out_ptr,
|
||||
lse_ptr,
|
||||
send_ptr,
|
||||
out_stride_B,
|
||||
out_stride_H,
|
||||
out_stride_D,
|
||||
lse_stride_B,
|
||||
lse_stride_H,
|
||||
send_stride_N,
|
||||
send_stride_B,
|
||||
send_stride_H,
|
||||
send_stride_D,
|
||||
N: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
H_PER_RANK: tl.constexpr,
|
||||
LSE_PACK_DIM: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0).to(tl.int64)
|
||||
local_head_idx = tl.program_id(1).to(tl.int64)
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
|
||||
for rank_idx in tl.static_range(N):
|
||||
src_head_idx = rank_idx * H_PER_RANK + local_head_idx
|
||||
send_base = (
|
||||
rank_idx * send_stride_N
|
||||
+ batch_idx * send_stride_B
|
||||
+ local_head_idx * send_stride_H
|
||||
)
|
||||
|
||||
out_offsets = (
|
||||
batch_idx * out_stride_B
|
||||
+ src_head_idx * out_stride_H
|
||||
+ d_offsets * out_stride_D
|
||||
)
|
||||
tl.store(
|
||||
send_ptr + send_base + d_offsets * send_stride_D,
|
||||
tl.load(out_ptr + out_offsets),
|
||||
)
|
||||
|
||||
lse_val = tl.load(
|
||||
lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H
|
||||
).to(tl.float32)
|
||||
if LSE_PACK_DIM == 1:
|
||||
tl.store(
|
||||
send_ptr + send_base + HEAD_DIM * send_stride_D,
|
||||
lse_val.to(send_ptr.dtype.element_ty),
|
||||
)
|
||||
else:
|
||||
lse_bits = lse_val.to(tl.uint32, bitcast=True)
|
||||
lo = (lse_bits & 0xFFFF).to(tl.uint16)
|
||||
hi = ((lse_bits >> 16) & 0xFFFF).to(tl.uint16)
|
||||
tl.store(
|
||||
send_ptr + send_base + HEAD_DIM * send_stride_D,
|
||||
lo.to(send_ptr.dtype.element_ty, bitcast=True),
|
||||
)
|
||||
tl.store(
|
||||
send_ptr + send_base + (HEAD_DIM + 1) * send_stride_D,
|
||||
hi.to(send_ptr.dtype.element_ty, bitcast=True),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dcp_a2a_unpack_combine_kernel(
|
||||
recv_ptr,
|
||||
out_ptr,
|
||||
out_lse_ptr,
|
||||
recv_stride_N,
|
||||
recv_stride_B,
|
||||
recv_stride_H,
|
||||
recv_stride_D,
|
||||
out_stride_B,
|
||||
out_stride_H,
|
||||
out_stride_D,
|
||||
out_lse_stride_B,
|
||||
out_lse_stride_H,
|
||||
N: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
IS_BASE_E: tl.constexpr,
|
||||
RETURN_LSE: tl.constexpr,
|
||||
LSE_PACK_DIM: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0).to(tl.int64)
|
||||
head_idx = tl.program_id(1).to(tl.int64)
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
|
||||
lse_max = -float("inf")
|
||||
for rank_idx in tl.static_range(N):
|
||||
recv_base = (
|
||||
rank_idx * recv_stride_N
|
||||
+ batch_idx * recv_stride_B
|
||||
+ head_idx * recv_stride_H
|
||||
)
|
||||
if LSE_PACK_DIM == 1:
|
||||
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
|
||||
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
|
||||
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
|
||||
lse_val = tl.where(
|
||||
(lse_val != lse_val) | (lse_val == float("inf")),
|
||||
-float("inf"),
|
||||
lse_val,
|
||||
)
|
||||
lse_max = tl.maximum(lse_max, lse_val)
|
||||
|
||||
lse_max = tl.where(lse_max == -float("inf"), 0.0, lse_max)
|
||||
|
||||
lse_sum = 0.0
|
||||
for rank_idx in tl.static_range(N):
|
||||
recv_base = (
|
||||
rank_idx * recv_stride_N
|
||||
+ batch_idx * recv_stride_B
|
||||
+ head_idx * recv_stride_H
|
||||
)
|
||||
if LSE_PACK_DIM == 1:
|
||||
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
|
||||
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
|
||||
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
|
||||
lse_val = tl.where(
|
||||
(lse_val != lse_val) | (lse_val == float("inf")),
|
||||
-float("inf"),
|
||||
lse_val,
|
||||
)
|
||||
if IS_BASE_E:
|
||||
lse_sum += tl.exp(lse_val - lse_max)
|
||||
else:
|
||||
lse_sum += tl.exp2(lse_val - lse_max)
|
||||
|
||||
if IS_BASE_E: # noqa: SIM108
|
||||
global_lse = tl.log(lse_sum) + lse_max
|
||||
else:
|
||||
global_lse = tl.log2(lse_sum) + lse_max
|
||||
|
||||
acc = tl.zeros([HEAD_DIM], dtype=tl.float32)
|
||||
for rank_idx in tl.static_range(N):
|
||||
recv_base = (
|
||||
rank_idx * recv_stride_N
|
||||
+ batch_idx * recv_stride_B
|
||||
+ head_idx * recv_stride_H
|
||||
)
|
||||
if LSE_PACK_DIM == 1:
|
||||
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
|
||||
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
|
||||
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
|
||||
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
|
||||
lse_val = tl.where(
|
||||
(lse_val != lse_val) | (lse_val == float("inf")),
|
||||
-float("inf"),
|
||||
lse_val,
|
||||
)
|
||||
if IS_BASE_E:
|
||||
weight = tl.exp(lse_val - global_lse)
|
||||
else:
|
||||
weight = tl.exp2(lse_val - global_lse)
|
||||
weight = tl.where(weight != weight, 0.0, weight)
|
||||
partial = tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(
|
||||
tl.float32
|
||||
)
|
||||
partial = tl.where(weight == 0.0, 0.0, partial)
|
||||
acc += partial * weight
|
||||
|
||||
final_offsets = (
|
||||
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
|
||||
)
|
||||
tl.store(out_ptr + final_offsets, acc)
|
||||
|
||||
if RETURN_LSE:
|
||||
out_lse_offset = batch_idx * out_lse_stride_B + head_idx * out_lse_stride_H
|
||||
tl.store(out_lse_ptr + out_lse_offset, global_lse)
|
||||
|
||||
|
||||
def _dcp_a2a_pack_send(
|
||||
cp_attn_out: torch.Tensor,
|
||||
cp_attn_lse: torch.Tensor,
|
||||
send_buffer: torch.Tensor,
|
||||
world_size: int,
|
||||
h_per_rank: int,
|
||||
head_dim: int,
|
||||
lse_pack_dim: int,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
|
||||
grid = (cp_attn_out.shape[0], h_per_rank, 1)
|
||||
_dcp_a2a_pack_send_kernel[grid](
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
send_buffer,
|
||||
cp_attn_out.stride(0),
|
||||
cp_attn_out.stride(1),
|
||||
cp_attn_out.stride(2),
|
||||
cp_attn_lse.stride(0),
|
||||
cp_attn_lse.stride(1),
|
||||
send_buffer.stride(0),
|
||||
send_buffer.stride(1),
|
||||
send_buffer.stride(2),
|
||||
send_buffer.stride(3),
|
||||
N=world_size,
|
||||
HEAD_DIM=head_dim,
|
||||
H_PER_RANK=h_per_rank,
|
||||
LSE_PACK_DIM=lse_pack_dim,
|
||||
)
|
||||
|
||||
|
||||
def _dcp_a2a_unpack_combine(
|
||||
recv_buffer: torch.Tensor,
|
||||
head_dim: int,
|
||||
lse_pack_dim: int,
|
||||
return_lse: bool,
|
||||
is_lse_base_on_e: bool,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
world_size, num_tokens, h_per_rank, _ = recv_buffer.shape
|
||||
out = torch.empty(
|
||||
(num_tokens, h_per_rank, head_dim),
|
||||
device=recv_buffer.device,
|
||||
dtype=recv_buffer.dtype,
|
||||
)
|
||||
out_lse = torch.empty(
|
||||
(num_tokens, h_per_rank) if return_lse else (1, 1),
|
||||
device=recv_buffer.device,
|
||||
dtype=torch.float32 if return_lse else recv_buffer.dtype,
|
||||
)
|
||||
grid = (num_tokens, h_per_rank, 1)
|
||||
_dcp_a2a_unpack_combine_kernel[grid](
|
||||
recv_buffer,
|
||||
out,
|
||||
out_lse,
|
||||
recv_buffer.stride(0),
|
||||
recv_buffer.stride(1),
|
||||
recv_buffer.stride(2),
|
||||
recv_buffer.stride(3),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
out_lse.stride(0),
|
||||
out_lse.stride(1),
|
||||
N=world_size,
|
||||
HEAD_DIM=head_dim,
|
||||
IS_BASE_E=is_lse_base_on_e,
|
||||
RETURN_LSE=return_lse,
|
||||
LSE_PACK_DIM=lse_pack_dim,
|
||||
)
|
||||
if return_lse:
|
||||
return out, out_lse
|
||||
return out
|
||||
|
||||
|
||||
def dcp_a2a_lse_reduce(
|
||||
cp_attn_out: torch.Tensor,
|
||||
cp_attn_lse: torch.Tensor,
|
||||
cp_group: GroupCoordinator,
|
||||
ctx: CPTritonContext | None = None,
|
||||
return_lse: bool = False,
|
||||
is_lse_base_on_e: bool = True,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Combine partial attention outputs across DCP ranks using All-to-All.
|
||||
|
||||
The output and LSE are packed into a single output-dtype buffer, sent
|
||||
with one All-to-All, then unpacked and combined with exact LSE weighting.
|
||||
|
||||
Args:
|
||||
cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim
|
||||
cp_attn_lse: [B, H] floating-point log-sum-exp values
|
||||
cp_group: GroupCoordinator for DCP communication
|
||||
ctx: CPTritonContext (unused, for signature compatibility)
|
||||
return_lse: If True, also return the combined global LSE
|
||||
is_lse_base_on_e: If True, LSE is base e; if False, base 2
|
||||
seq_lens: Local KV lengths. Empty shards contribute zero weight.
|
||||
query_start_loc: Cumulative query-token offsets for each request.
|
||||
|
||||
Returns:
|
||||
Combined output [B, H/N, D] (head-scattered)
|
||||
If return_lse=True, also returns global_lse [B, H/N]
|
||||
"""
|
||||
world_size = cp_group.world_size
|
||||
|
||||
if world_size == 1:
|
||||
if return_lse:
|
||||
return cp_attn_out, cp_attn_lse
|
||||
return cp_attn_out
|
||||
|
||||
B, H, D = cp_attn_out.shape
|
||||
if H % world_size != 0:
|
||||
raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.")
|
||||
H_per_rank = H // world_size
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype)
|
||||
|
||||
send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers(
|
||||
(world_size, B, H_per_rank, D + lse_pack_dim),
|
||||
device=cp_attn_out.device,
|
||||
dtype=cp_attn_out.dtype,
|
||||
)
|
||||
|
||||
_dcp_a2a_pack_send(
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
send_buffer,
|
||||
world_size,
|
||||
H_per_rank,
|
||||
D,
|
||||
lse_pack_dim,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
|
||||
work = dist.all_to_all_single(
|
||||
recv_buffer.view(-1),
|
||||
send_buffer.view(-1),
|
||||
group=cp_group.device_group,
|
||||
async_op=True,
|
||||
)
|
||||
work.wait()
|
||||
|
||||
return _dcp_a2a_unpack_combine(
|
||||
recv_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e
|
||||
)
|
||||
@@ -1,740 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MLA DCP collective selection and direct symmetric-memory implementations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.distributed.parallel_state import in_the_same_node_as
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
|
||||
try:
|
||||
import torch.distributed._symmetric_memory as symm_mem
|
||||
|
||||
symm_mem_available = True
|
||||
except ImportError:
|
||||
symm_mem = None # type: ignore[assignment]
|
||||
symm_mem_available = False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _symm_mem_spans_group(group: GroupCoordinator) -> bool:
|
||||
"""Probe whether the group has NVLS symmetric memory."""
|
||||
if not symm_mem_available:
|
||||
return False
|
||||
try:
|
||||
from torch._C._autograd import DeviceType
|
||||
from torch._C._distributed_c10d import _SymmetricMemory
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index):
|
||||
return False
|
||||
probe = symm_mem.empty(8, dtype=torch.uint8, device=device)
|
||||
probe.zero_()
|
||||
torch.accelerator.synchronize()
|
||||
handle = symm_mem.rendezvous(probe, group.device_group.group_name)
|
||||
spans = handle is not None and handle.multicast_ptr != 0
|
||||
except Exception as error:
|
||||
logger.debug("Direct DCP symmetric-memory probe failed: %s", error)
|
||||
return False
|
||||
logger.debug_once(
|
||||
"Direct DCP symmetric memory across %d ranks: %s",
|
||||
group.world_size,
|
||||
"available" if spans else "unavailable",
|
||||
)
|
||||
return spans
|
||||
|
||||
|
||||
def _direct_dcp_enabled(
|
||||
group: GroupCoordinator,
|
||||
dtype: torch.dtype,
|
||||
use_direct: bool | None,
|
||||
supported_dtypes: tuple[torch.dtype, ...] | None = None,
|
||||
) -> bool:
|
||||
if use_direct is not None:
|
||||
return use_direct
|
||||
return (
|
||||
symm_mem_available
|
||||
and current_platform.is_cuda()
|
||||
and (supported_dtypes is None or dtype in supported_dtypes)
|
||||
and (
|
||||
all(in_the_same_node_as(group.cpu_group, source_rank=0))
|
||||
or _symm_mem_spans_group(group)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _direct_dcp_multicast_enabled(
|
||||
group: GroupCoordinator,
|
||||
dtype: torch.dtype,
|
||||
use_direct: bool | None,
|
||||
supported_dtypes: tuple[torch.dtype, ...] | None = None,
|
||||
) -> bool:
|
||||
return _direct_dcp_enabled(
|
||||
group, dtype, use_direct, supported_dtypes
|
||||
) and _symm_mem_spans_group(group)
|
||||
|
||||
|
||||
def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int:
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
speculative_tokens = vllm_config.num_speculative_tokens
|
||||
tokens_per_seq = (
|
||||
1
|
||||
+ (
|
||||
2
|
||||
if speculative_config is not None and speculative_config.parallel_drafting
|
||||
else 1
|
||||
)
|
||||
* speculative_tokens
|
||||
)
|
||||
return min(
|
||||
scheduler_config.max_num_batched_tokens,
|
||||
max(
|
||||
scheduler_config.max_num_seqs * tokens_per_seq,
|
||||
vllm_config.compilation_config.max_cudagraph_capture_size or 0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _DirectDCPWorkspace:
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
num_ubatches: int,
|
||||
) -> None:
|
||||
self.group = group
|
||||
self.world_size = group.size()
|
||||
self.rank = group.rank()
|
||||
self.device = torch.device(device)
|
||||
self.num_ubatches = num_ubatches
|
||||
self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device)
|
||||
self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = []
|
||||
|
||||
def _allocate(
|
||||
self, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
storage = symm_mem.empty(shape, device=self.device, dtype=dtype)
|
||||
storage.zero_()
|
||||
torch.accelerator.synchronize()
|
||||
handle = symm_mem.rendezvous(storage, self.group.group_name)
|
||||
assert handle is not None, "DCP symmetric memory rendezvous returned None"
|
||||
handle.barrier()
|
||||
views = [
|
||||
handle.get_buffer(peer, list(shape), dtype, 0)
|
||||
for peer in range(self.world_size)
|
||||
]
|
||||
self.device = storage.device
|
||||
peer_ptrs = torch.tensor(
|
||||
[
|
||||
[view[ubatch].data_ptr() for view in views]
|
||||
for ubatch in range(self.num_ubatches)
|
||||
],
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
self._allocations.append((storage, handle, views))
|
||||
return storage, peer_ptrs
|
||||
|
||||
def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]:
|
||||
disabled = [0] * self.num_ubatches
|
||||
for allocated, handle, _ in self._allocations:
|
||||
if allocated is storage:
|
||||
break
|
||||
else:
|
||||
return disabled
|
||||
try:
|
||||
from torch._C._autograd import DeviceType
|
||||
from torch._C._distributed_c10d import _SymmetricMemory
|
||||
|
||||
if not _SymmetricMemory.has_multicast_support(
|
||||
DeviceType.CUDA, storage.device.index
|
||||
):
|
||||
return disabled
|
||||
multicast_base = handle.multicast_ptr
|
||||
except Exception:
|
||||
return disabled
|
||||
if not multicast_base:
|
||||
return disabled
|
||||
storage_base = storage.data_ptr()
|
||||
return [
|
||||
multicast_base + (storage[ubatch].data_ptr() - storage_base)
|
||||
for ubatch in range(self.num_ubatches)
|
||||
]
|
||||
|
||||
|
||||
def reserve_query_head_storage(
|
||||
query: torch.Tensor, padded_num_heads: int
|
||||
) -> torch.Tensor:
|
||||
"""Reserve backing storage for fixed-head decode kernels."""
|
||||
assert query.ndim == 3
|
||||
assert query.shape[1] <= padded_num_heads
|
||||
padded = query.new_empty((query.shape[0], padded_num_heads, query.shape[2]))
|
||||
padded.resize_(query.shape)
|
||||
padded.copy_(query)
|
||||
return padded
|
||||
|
||||
|
||||
_A2A_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
|
||||
|
||||
|
||||
class DirectDCPA2AWorkspace(_DirectDCPWorkspace):
|
||||
"""Persistent symmetric buffers for direct DCP output exchange."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
max_num_tokens: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
num_ubatches: int = 1,
|
||||
) -> None:
|
||||
if dtype not in _A2A_SUPPORTED_DTYPES:
|
||||
raise ValueError(f"Direct DCP A2A does not support {dtype}")
|
||||
if num_ubatches < 1:
|
||||
raise ValueError(
|
||||
f"Direct DCP A2A requires at least one ubatch slot, got {num_ubatches}"
|
||||
)
|
||||
super().__init__(group, device, num_ubatches)
|
||||
self.max_num_tokens = max_num_tokens
|
||||
self.heads_per_rank = heads_per_rank
|
||||
self.head_dim = head_dim
|
||||
|
||||
output_shape = (
|
||||
num_ubatches,
|
||||
2,
|
||||
self.world_size,
|
||||
max_num_tokens,
|
||||
heads_per_rank,
|
||||
head_dim,
|
||||
)
|
||||
lse_shape = (
|
||||
num_ubatches,
|
||||
2,
|
||||
self.world_size,
|
||||
max_num_tokens,
|
||||
heads_per_rank,
|
||||
)
|
||||
signal_shape = (num_ubatches, 2, self.world_size)
|
||||
self.received_output, self.peer_output_ptrs = self._allocate(
|
||||
output_shape, dtype
|
||||
)
|
||||
self.received_lse, self.peer_lse_ptrs = self._allocate(lse_shape, torch.float32)
|
||||
self.received_signal, self.peer_signal_ptrs = self._allocate(
|
||||
signal_shape, torch.int32
|
||||
)
|
||||
|
||||
def lse_reduce(
|
||||
self,
|
||||
partial_output: torch.Tensor,
|
||||
partial_lse: torch.Tensor,
|
||||
is_lse_base_on_e: bool,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
ubatch = dbo_current_ubatch_id()
|
||||
num_tokens = partial_output.shape[0]
|
||||
output = partial_output.new_empty(
|
||||
(num_tokens, self.heads_per_rank, self.head_dim)
|
||||
)
|
||||
torch.ops._C.direct_dcp_a2a_lse_reduce(
|
||||
partial_output,
|
||||
partial_lse,
|
||||
seq_lens,
|
||||
query_start_loc,
|
||||
self.peer_output_ptrs[ubatch],
|
||||
self.peer_lse_ptrs[ubatch],
|
||||
self.peer_signal_ptrs[ubatch],
|
||||
self.received_output[ubatch],
|
||||
self.received_lse[ubatch],
|
||||
self.received_signal[ubatch],
|
||||
self.epoch[ubatch : ubatch + 1],
|
||||
output,
|
||||
self.world_size,
|
||||
self.rank,
|
||||
self.max_num_tokens,
|
||||
is_lse_base_on_e,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_direct_dcp_a2a_workspace(
|
||||
group: GroupCoordinator,
|
||||
device: torch.device,
|
||||
max_num_tokens: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
num_ubatches: int,
|
||||
) -> DirectDCPA2AWorkspace | None:
|
||||
if not _direct_dcp_enabled(
|
||||
group, dtype, envs.VLLM_USE_DIRECT_DCP_A2A, _A2A_SUPPORTED_DTYPES
|
||||
):
|
||||
return None
|
||||
return DirectDCPA2AWorkspace(
|
||||
group.device_group,
|
||||
device,
|
||||
max_num_tokens,
|
||||
heads_per_rank,
|
||||
head_dim,
|
||||
dtype,
|
||||
num_ubatches,
|
||||
)
|
||||
|
||||
|
||||
def _q_gather_layout_supported(
|
||||
world_size: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
padded_num_heads: int | None,
|
||||
) -> bool:
|
||||
element_size = torch.empty((), dtype=dtype).element_size()
|
||||
gathered_num_heads = world_size * heads_per_rank
|
||||
storage_num_heads = (
|
||||
gathered_num_heads if padded_num_heads is None else padded_num_heads
|
||||
)
|
||||
return (
|
||||
heads_per_rank * head_dim * element_size % 16 == 0
|
||||
and storage_num_heads * head_dim * element_size % 16 == 0
|
||||
)
|
||||
|
||||
|
||||
class DirectDCPQGatherWorkspace(_DirectDCPWorkspace):
|
||||
"""Publish query shards directly into the consumer-final symmetric buffer.
|
||||
|
||||
The final buffer is reusable after the downstream DCP output combine. That
|
||||
combine orders all ranks after attention has consumed the gathered query.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
max_num_tokens: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
num_ubatches: int = 1,
|
||||
padded_num_heads: int | None = None,
|
||||
) -> None:
|
||||
if num_ubatches < 1:
|
||||
raise ValueError(
|
||||
"Direct DCP q-gather requires at least one ubatch slot, "
|
||||
f"got {num_ubatches}"
|
||||
)
|
||||
if max_num_tokens < 1 or heads_per_rank < 1 or head_dim < 1:
|
||||
raise ValueError(
|
||||
"Direct DCP q-gather dimensions must be positive, got "
|
||||
f"T={max_num_tokens}, H={heads_per_rank}, D={head_dim}"
|
||||
)
|
||||
gathered_num_heads = group.size() * heads_per_rank
|
||||
if not _q_gather_layout_supported(
|
||||
group.size(), heads_per_rank, head_dim, dtype, padded_num_heads
|
||||
):
|
||||
raise ValueError("Direct DCP q-gather requires 16-byte-aligned query rows.")
|
||||
super().__init__(group, device, num_ubatches)
|
||||
if self.world_size <= 1:
|
||||
raise ValueError("Direct DCP q-gather requires at least two ranks")
|
||||
self.max_num_tokens = max_num_tokens
|
||||
self.heads_per_rank = heads_per_rank
|
||||
self.gathered_num_heads = gathered_num_heads
|
||||
self.padded_num_heads = (
|
||||
self.gathered_num_heads if padded_num_heads is None else padded_num_heads
|
||||
)
|
||||
if self.padded_num_heads < self.gathered_num_heads:
|
||||
raise ValueError(
|
||||
"Direct DCP q-gather padded heads must cover gathered heads: "
|
||||
f"{self.padded_num_heads} < {self.gathered_num_heads}"
|
||||
)
|
||||
self.head_dim = head_dim
|
||||
|
||||
query_shape = (
|
||||
num_ubatches,
|
||||
max_num_tokens,
|
||||
self.padded_num_heads,
|
||||
head_dim,
|
||||
)
|
||||
signal_shape = (num_ubatches, 2, self.world_size)
|
||||
self.final_query, _ = self._allocate(query_shape, dtype)
|
||||
self.received_signal, _ = self._allocate(signal_shape, torch.int32)
|
||||
query_multicast_ptrs = self._multicast_ptrs(self.final_query)
|
||||
signal_multicast_ptrs = self._multicast_ptrs(self.received_signal)
|
||||
self.multicast_ptrs = list(
|
||||
zip(query_multicast_ptrs, signal_multicast_ptrs, strict=True)
|
||||
)
|
||||
if not all(
|
||||
query_ptr and signal_ptr for query_ptr, signal_ptr in self.multicast_ptrs
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Direct DCP q-gather requires NVLS symmetric-memory multicast."
|
||||
)
|
||||
self.completion = self.received_signal.new_zeros((num_ubatches, 1))
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def gather(self, local_query: torch.Tensor) -> torch.Tensor:
|
||||
ubatch = dbo_current_ubatch_id()
|
||||
if not 0 <= ubatch < self.num_ubatches:
|
||||
raise ValueError(
|
||||
f"DCP q-gather ubatch {ubatch} exceeds {self.num_ubatches} slots"
|
||||
)
|
||||
if local_query.ndim == 3 and local_query.shape[1] != self.heads_per_rank:
|
||||
raise ValueError(
|
||||
f"DCP q-gather expected {self.heads_per_rank} local query heads, "
|
||||
f"got {local_query.shape[1]}"
|
||||
)
|
||||
|
||||
num_tokens = local_query.shape[0]
|
||||
output = torch.as_strided(
|
||||
self.final_query[ubatch],
|
||||
size=(num_tokens, self.gathered_num_heads, self.head_dim),
|
||||
stride=(
|
||||
self.gathered_num_heads * self.head_dim,
|
||||
self.head_dim,
|
||||
1,
|
||||
),
|
||||
)
|
||||
query_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch]
|
||||
torch.ops._C.direct_dcp_q_gather(
|
||||
local_query,
|
||||
output,
|
||||
self.received_signal[ubatch],
|
||||
self.completion[ubatch],
|
||||
self.epoch[ubatch : ubatch + 1],
|
||||
self.world_size,
|
||||
self.rank,
|
||||
self.max_num_tokens,
|
||||
self.padded_num_heads,
|
||||
query_multicast_ptr,
|
||||
signal_multicast_ptr,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_direct_dcp_q_gather_workspace(
|
||||
group: GroupCoordinator,
|
||||
device: torch.device,
|
||||
max_num_tokens: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
num_ubatches: int,
|
||||
padded_num_heads: int | None = None,
|
||||
) -> DirectDCPQGatherWorkspace | None:
|
||||
if not _direct_dcp_multicast_enabled(
|
||||
group, dtype, envs.VLLM_USE_DIRECT_DCP_Q_GATHER
|
||||
):
|
||||
return None
|
||||
if not _q_gather_layout_supported(
|
||||
group.world_size, heads_per_rank, head_dim, dtype, padded_num_heads
|
||||
):
|
||||
return None
|
||||
return DirectDCPQGatherWorkspace(
|
||||
group.device_group,
|
||||
device,
|
||||
max_num_tokens,
|
||||
heads_per_rank,
|
||||
head_dim,
|
||||
dtype,
|
||||
num_ubatches,
|
||||
padded_num_heads,
|
||||
)
|
||||
|
||||
|
||||
_KV_GATHER_SUPPORTED_DTYPES = (
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
torch.float8_e4m3fn,
|
||||
)
|
||||
|
||||
|
||||
def _kv_gather_layout_supported(token_dim: int, dtype: torch.dtype) -> bool:
|
||||
return token_dim * torch.empty((), dtype=dtype).element_size() % 16 == 0
|
||||
|
||||
|
||||
class DirectDCPKVGatherWorkspace(_DirectDCPWorkspace):
|
||||
"""Persistent symmetric buffers for direct DCP KV gather."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
max_gathered_tokens: int,
|
||||
token_dim: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
num_ubatches: int = 1,
|
||||
) -> None:
|
||||
if dtype not in _KV_GATHER_SUPPORTED_DTYPES:
|
||||
raise ValueError(f"Direct DCP kv-gather does not support {dtype}")
|
||||
if num_ubatches < 1:
|
||||
raise ValueError(
|
||||
"Direct DCP kv-gather requires at least one ubatch slot, "
|
||||
f"got {num_ubatches}"
|
||||
)
|
||||
if max_gathered_tokens < 1 or token_dim < 1:
|
||||
raise ValueError(
|
||||
"Direct DCP kv-gather dimensions must be positive, got "
|
||||
f"T={max_gathered_tokens}, D={token_dim}"
|
||||
)
|
||||
if not _kv_gather_layout_supported(token_dim, dtype):
|
||||
raise ValueError("Direct DCP kv-gather requires 16-byte-aligned KV rows.")
|
||||
super().__init__(group, device, num_ubatches)
|
||||
if self.world_size <= 1:
|
||||
raise ValueError("Direct DCP kv-gather requires at least two ranks")
|
||||
if max_gathered_tokens % self.world_size != 0:
|
||||
raise ValueError(
|
||||
"Direct DCP kv-gather capacity must divide evenly across "
|
||||
f"ranks: {max_gathered_tokens} % {self.world_size} != 0"
|
||||
)
|
||||
self.max_gathered_tokens = max_gathered_tokens
|
||||
|
||||
kv_shape = (num_ubatches, 2, max_gathered_tokens, token_dim)
|
||||
signal_shape = (num_ubatches, 2, self.world_size)
|
||||
self.received_kv, _ = self._allocate(kv_shape, dtype)
|
||||
self.received_signal, _ = self._allocate(signal_shape, torch.int32)
|
||||
kv_multicast_ptrs = self._multicast_ptrs(self.received_kv)
|
||||
signal_multicast_ptrs = self._multicast_ptrs(self.received_signal)
|
||||
self.multicast_ptrs = list(
|
||||
zip(kv_multicast_ptrs, signal_multicast_ptrs, strict=True)
|
||||
)
|
||||
if not all(kv_ptr and signal_ptr for kv_ptr, signal_ptr in self.multicast_ptrs):
|
||||
raise RuntimeError(
|
||||
"Direct DCP kv-gather requires NVLS symmetric-memory multicast."
|
||||
)
|
||||
self.completion = self.received_signal.new_zeros((num_ubatches, 2))
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def gather(self, gathered_kv: torch.Tensor, local_kv: torch.Tensor) -> None:
|
||||
ubatch = dbo_current_ubatch_id()
|
||||
if not 0 <= ubatch < self.num_ubatches:
|
||||
raise ValueError(
|
||||
f"DCP kv-gather ubatch {ubatch} exceeds {self.num_ubatches} slots"
|
||||
)
|
||||
kv_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch]
|
||||
torch.ops._C.direct_dcp_kv_gather(
|
||||
local_kv,
|
||||
self.received_kv[ubatch],
|
||||
self.received_signal[ubatch],
|
||||
self.completion[ubatch],
|
||||
self.epoch[ubatch : ubatch + 1],
|
||||
gathered_kv,
|
||||
self.world_size,
|
||||
self.rank,
|
||||
self.max_gathered_tokens,
|
||||
kv_multicast_ptr,
|
||||
signal_multicast_ptr,
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_direct_dcp_kv_gather_workspace(
|
||||
group: GroupCoordinator,
|
||||
device: torch.device,
|
||||
max_gathered_tokens: int,
|
||||
token_dim: int,
|
||||
dtype: torch.dtype,
|
||||
num_ubatches: int,
|
||||
) -> DirectDCPKVGatherWorkspace | None:
|
||||
if not _direct_dcp_multicast_enabled(
|
||||
group,
|
||||
dtype,
|
||||
envs.VLLM_USE_DIRECT_DCP_KV_GATHER,
|
||||
_KV_GATHER_SUPPORTED_DTYPES,
|
||||
):
|
||||
return None
|
||||
if not _kv_gather_layout_supported(token_dim, dtype):
|
||||
return None
|
||||
return DirectDCPKVGatherWorkspace(
|
||||
group.device_group,
|
||||
device,
|
||||
max_gathered_tokens,
|
||||
token_dim,
|
||||
dtype,
|
||||
num_ubatches,
|
||||
)
|
||||
|
||||
|
||||
class DCPCombine(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
partial_output: torch.Tensor,
|
||||
partial_lse: torch.Tensor,
|
||||
*,
|
||||
seq_lens: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
) -> torch.Tensor: ...
|
||||
|
||||
|
||||
class MLADCPManager:
|
||||
"""Select and own layer-level collective implementations for MLA DCP."""
|
||||
|
||||
_kv_gather: Callable[[torch.Tensor, torch.Tensor], object]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
num_heads: int,
|
||||
query_head_dim: int,
|
||||
output_head_dim: int,
|
||||
query_dtype: torch.dtype,
|
||||
output_dtype: torch.dtype,
|
||||
padded_num_heads: int | None,
|
||||
is_lse_base_on_e: bool,
|
||||
use_pcp: bool,
|
||||
) -> None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.group = get_dcp_group()
|
||||
self.device = torch.device(device)
|
||||
self.num_ubatches = max(parallel_config.num_ubatches, 1)
|
||||
self.max_num_tokens = get_dcp_workspace_max_num_tokens(vllm_config)
|
||||
self.use_a2a = parallel_config.dcp_comm_backend == "a2a"
|
||||
self.padded_num_heads = padded_num_heads
|
||||
|
||||
self.combine = self._init_combine(
|
||||
num_heads,
|
||||
output_head_dim,
|
||||
output_dtype,
|
||||
is_lse_base_on_e,
|
||||
use_pcp,
|
||||
)
|
||||
self.query_gather = (
|
||||
None
|
||||
if use_pcp
|
||||
else self._init_query_gather(
|
||||
num_heads,
|
||||
query_head_dim,
|
||||
query_dtype,
|
||||
)
|
||||
)
|
||||
|
||||
def _init_combine(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
is_lse_base_on_e: bool,
|
||||
use_pcp: bool,
|
||||
) -> DCPCombine:
|
||||
direct_workspace = None
|
||||
if self.use_a2a:
|
||||
direct_workspace = get_direct_dcp_a2a_workspace(
|
||||
self.group,
|
||||
self.device,
|
||||
self.max_num_tokens,
|
||||
num_heads,
|
||||
head_dim,
|
||||
dtype,
|
||||
self.num_ubatches,
|
||||
)
|
||||
if direct_workspace is not None:
|
||||
logger.info_once("Using direct symmetric-memory DCP A2A for MLA.")
|
||||
return functools.partial(
|
||||
direct_workspace.lse_reduce,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
|
||||
combine_fn = (
|
||||
dcp_a2a_lse_reduce
|
||||
if self.use_a2a
|
||||
else cp_lse_ag_out_ar
|
||||
if use_pcp
|
||||
else cp_lse_ag_out_rs
|
||||
)
|
||||
return functools.partial(
|
||||
combine_fn,
|
||||
cp_group=self.group,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
|
||||
def _init_query_gather(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
) -> Callable[[torch.Tensor], torch.Tensor]:
|
||||
direct_workspace = get_direct_dcp_q_gather_workspace(
|
||||
self.group,
|
||||
self.device,
|
||||
self.max_num_tokens,
|
||||
num_heads,
|
||||
head_dim,
|
||||
dtype,
|
||||
self.num_ubatches,
|
||||
self.padded_num_heads,
|
||||
)
|
||||
if direct_workspace is not None:
|
||||
logger.info_once("Using direct symmetric-memory DCP query gather for MLA.")
|
||||
return direct_workspace.gather
|
||||
return self._gather_query
|
||||
|
||||
def _gather_query(self, query: torch.Tensor) -> torch.Tensor:
|
||||
query = self.group.all_gather(query, dim=1)
|
||||
if self.padded_num_heads is not None:
|
||||
query = reserve_query_head_storage(query, self.padded_num_heads)
|
||||
return query
|
||||
|
||||
def init_kv_gather(
|
||||
self,
|
||||
workspace: torch.Tensor,
|
||||
max_gathered_tokens: int,
|
||||
) -> None:
|
||||
world_size = self.group.world_size
|
||||
assert max_gathered_tokens > 0
|
||||
assert max_gathered_tokens % world_size == 0
|
||||
assert workspace.ndim == 2
|
||||
assert workspace.is_contiguous()
|
||||
assert workspace.shape[0] == (
|
||||
max_gathered_tokens + max_gathered_tokens // world_size
|
||||
)
|
||||
assert workspace.shape[1] > 0
|
||||
|
||||
direct_workspace = get_direct_dcp_kv_gather_workspace(
|
||||
self.group,
|
||||
workspace.device,
|
||||
max_gathered_tokens,
|
||||
workspace.shape[1],
|
||||
workspace.dtype,
|
||||
self.num_ubatches,
|
||||
)
|
||||
if direct_workspace is not None:
|
||||
logger.info_once(
|
||||
"Using direct symmetric-memory DCP chunked-context KV gather for MLA."
|
||||
)
|
||||
self._kv_gather = direct_workspace.gather
|
||||
else:
|
||||
self._kv_gather = functools.partial(
|
||||
torch.distributed.all_gather_into_tensor,
|
||||
group=self.group.device_group,
|
||||
)
|
||||
|
||||
def kv_gather(
|
||||
self,
|
||||
gathered_kv: torch.Tensor,
|
||||
local_kv: torch.Tensor,
|
||||
) -> object:
|
||||
return self._kv_gather(gathered_kv, local_kv)
|
||||
Reference in New Issue
Block a user