[refactor] consolidate cp attn ops (#52839)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-19 18:04:11 -07:00
committed by GitHub
co-authored by Woosuk Kwon
parent bf2866f8bf
commit 58e5ee0158
18 changed files with 1603 additions and 1585 deletions
+17 -17
View File
@@ -44,7 +44,7 @@ def _packed_a2a_reference(
h_per_rank: int,
is_lse_base_on_e: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
B, _H, D = cp_attn_out.shape
outputs = (
@@ -157,13 +157,13 @@ class TestLSEWeightedCombine:
def test_importable(self):
"""Verify _lse_weighted_combine is importable."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
assert callable(_lse_weighted_combine)
def test_single_rank(self):
"""Single rank: output unchanged."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
# N=1, B=2, H=4, D=8
outputs = torch.randn(1, 2, 4, 8)
@@ -176,7 +176,7 @@ class TestLSEWeightedCombine:
def test_equal_lse(self):
"""Equal LSE values: outputs averaged equally."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
_N, B, H, D = 2, 1, 1, 4
outputs = torch.tensor(
@@ -200,7 +200,7 @@ class TestLSEWeightedCombine:
def test_dominant_rank(self):
"""Different LSE values: larger LSE gets more weight."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
B, H, D = 1, 1, 2
outputs = torch.tensor(
@@ -222,7 +222,7 @@ class TestLSEWeightedCombine:
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
def test_empty_shard_ignores_undefined_output(self):
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]])
lses = torch.tensor([[[-float("inf")]], [[0.0]]])
@@ -232,7 +232,7 @@ class TestLSEWeightedCombine:
torch.testing.assert_close(result, outputs[1])
def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch):
import vllm.v1.attention.ops.common as common
import vllm.v1.attention.ops.dcp as dcp
class FakeGroup:
world_size = 2
@@ -243,7 +243,7 @@ class TestLSEWeightedCombine:
return torch.cat((tensor, tensor), dim=dim)
monkeypatch.setattr(
common,
dcp,
"correct_attn_out",
lambda output, lses, *args, **kwargs: (output, lses[0]),
)
@@ -252,7 +252,7 @@ class TestLSEWeightedCombine:
seq_lens = torch.tensor([0, 2], dtype=torch.int32)
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
_, masked_lse = common._cp_lse_common(
_, masked_lse = dcp._cp_lse_common(
output,
lse,
FakeGroup(),
@@ -266,7 +266,7 @@ class TestLSEWeightedCombine:
def test_mathematically_correct(self):
"""Verify mathematical correctness of LSE combination."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
outputs = torch.tensor(
[
@@ -291,7 +291,7 @@ class TestLSEWeightedCombine:
def test_return_lse(self):
"""return_lse=True returns global LSE (logsumexp of inputs)."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
B, H, D = 1, 1, 2
outputs = torch.tensor(
@@ -317,7 +317,7 @@ class TestLSEWeightedCombine:
def test_base2_return_lse(self):
"""Base-2 LSE mode returns log2-sum-exp2 global LSE."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
outputs = torch.tensor(
[
@@ -354,7 +354,7 @@ class TestLSEWeightedCombine:
def test_lse_pack_dim(self):
"""Packed A2A stores one fp32 LSE in output-dtype lanes."""
from vllm.v1.attention.ops.dcp_alltoall import _dcp_a2a_lse_pack_dim
from vllm.v1.attention.ops.dcp import _dcp_a2a_lse_pack_dim
assert _dcp_a2a_lse_pack_dim(torch.bfloat16) == 2
assert _dcp_a2a_lse_pack_dim(torch.float16) == 2
@@ -374,7 +374,7 @@ class TestPackedA2AKernels:
return_lse: bool,
is_lse_base_on_e: bool,
):
from vllm.v1.attention.ops.dcp_alltoall import (
from vllm.v1.attention.ops.dcp import (
_dcp_a2a_lse_pack_dim,
_dcp_a2a_pack_send,
_dcp_a2a_unpack_combine,
@@ -421,7 +421,7 @@ class TestPackedA2AKernels:
torch.accelerator.device_count() < 1, reason="CUDA is required."
)
def test_empty_seq_lens_ignore_undefined_output(self):
from vllm.v1.attention.ops.dcp_alltoall import (
from vllm.v1.attention.ops.dcp import (
_dcp_a2a_lse_pack_dim,
_dcp_a2a_pack_send,
_dcp_a2a_unpack_combine,
@@ -505,7 +505,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
init_workspace_manager(torch.device(f"cuda:{local_rank}"))
try:
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.dcp import dcp_a2a_lse_reduce
dtype = _dtype_from_name(env["TEST_DTYPE"])
return_lse = env["RETURN_LSE"] == "1"
@@ -555,7 +555,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
[t[:, rank * h_per_rank : (rank + 1) * h_per_rank] for t in gathered_lse],
dim=0,
)
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
expected_out, expected_lse = _lse_weighted_combine(
outputs,
@@ -10,7 +10,8 @@ import pytest
import torch
import torch.distributed as dist
import vllm.v1.attention.ops.dcp_utils as dcp_utils
import vllm.v1.attention.ops.cp_common as cp_common
import vllm.v1.attention.ops.dcp as dcp
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
@@ -142,17 +143,17 @@ class _FakeProcessGroup:
class TestDirectDCPGating:
def test_env_disabled_returns_none(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
workspace = dcp_utils.get_direct_dcp_a2a_workspace(
dcp.get_direct_dcp_a2a_workspace.cache_clear()
workspace = dcp.get_direct_dcp_a2a_workspace(
_FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.bfloat16, 1
)
assert workspace is None
def test_forced_with_unsupported_dtype_raises(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
dcp.get_direct_dcp_a2a_workspace.cache_clear()
with pytest.raises(ValueError, match="does not support"):
dcp_utils.get_direct_dcp_a2a_workspace(
dcp.get_direct_dcp_a2a_workspace(
_FakeGroupCoordinator(),
torch.device("cpu"),
16,
@@ -164,14 +165,14 @@ class TestDirectDCPGating:
def test_zero_ubatches_raises(self):
with pytest.raises(ValueError, match="ubatch"):
dcp_utils.DirectDCPA2AWorkspace(
dcp.DirectDCPA2AWorkspace(
None, torch.device("cpu"), 16, 2, 32, torch.bfloat16, num_ubatches=0
)
def test_auto_with_unsupported_dtype_returns_none(self, monkeypatch):
monkeypatch.delenv("VLLM_USE_DIRECT_DCP_A2A", raising=False)
dcp_utils.get_direct_dcp_a2a_workspace.cache_clear()
workspace = dcp_utils.get_direct_dcp_a2a_workspace(
dcp.get_direct_dcp_a2a_workspace.cache_clear()
workspace = dcp.get_direct_dcp_a2a_workspace(
_FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.float32, 1
)
assert workspace is None
@@ -179,8 +180,8 @@ class TestDirectDCPGating:
def test_q_gather_env_disabled_returns_none(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "0")
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear()
workspace = dcp_utils.get_direct_dcp_q_gather_workspace(
dcp.get_direct_dcp_q_gather_workspace.cache_clear()
workspace = dcp.get_direct_dcp_q_gather_workspace(
_FakeGroupCoordinator(),
torch.device("cpu"),
16,
@@ -194,17 +195,17 @@ class TestDirectDCPGating:
def test_q_gather_flag_is_independent(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "1")
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True)
dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear()
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: True)
dcp.get_direct_dcp_q_gather_workspace.cache_clear()
workspace = object()
init_workspace = MagicMock(return_value=workspace)
monkeypatch.setattr(
dcp_utils,
dcp,
"DirectDCPQGatherWorkspace",
init_workspace,
)
result = dcp_utils.get_direct_dcp_q_gather_workspace(
result = dcp.get_direct_dcp_q_gather_workspace(
_FakeGroupCoordinator(),
torch.device("cpu"),
16,
@@ -220,8 +221,8 @@ class TestDirectDCPGating:
def test_kv_gather_env_disabled_returns_none(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "0")
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1")
dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear()
workspace = dcp_utils.get_direct_dcp_kv_gather_workspace(
dcp.get_direct_dcp_kv_gather_workspace.cache_clear()
workspace = dcp.get_direct_dcp_kv_gather_workspace(
_FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1
)
assert workspace is None
@@ -229,17 +230,17 @@ class TestDirectDCPGating:
def test_kv_gather_flag_is_independent(self, monkeypatch):
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "1")
monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0")
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True)
dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear()
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: True)
dcp.get_direct_dcp_kv_gather_workspace.cache_clear()
workspace = object()
init_workspace = MagicMock(return_value=workspace)
monkeypatch.setattr(
dcp_utils,
dcp,
"DirectDCPKVGatherWorkspace",
init_workspace,
)
result = dcp_utils.get_direct_dcp_kv_gather_workspace(
result = dcp.get_direct_dcp_kv_gather_workspace(
_FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1
)
@@ -267,9 +268,9 @@ class TestDirectDCPGating:
factory_name,
factory_args,
):
factory = getattr(dcp_utils, factory_name)
factory = getattr(dcp, factory_name)
monkeypatch.setenv(flag_name, "1")
monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: False)
monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: False)
factory.cache_clear()
assert (
@@ -283,25 +284,25 @@ class TestDirectDCPGating:
def test_kv_gather_rejects_invalid_workspace_geometry(self):
with pytest.raises(ValueError, match="ubatch"):
dcp_utils.DirectDCPKVGatherWorkspace(
dcp.DirectDCPKVGatherWorkspace(
None, torch.device("cpu"), 64, 576, num_ubatches=0
)
with pytest.raises(ValueError, match="divide evenly"):
dcp_utils.DirectDCPKVGatherWorkspace(
dcp.DirectDCPKVGatherWorkspace(
_FakeProcessGroup(), torch.device("cpu"), 63, 576
)
with pytest.raises(ValueError, match="16-byte"):
dcp_utils.DirectDCPKVGatherWorkspace(
dcp.DirectDCPKVGatherWorkspace(
_FakeProcessGroup(), torch.device("cpu"), 64, 3
)
def test_q_gather_rejects_invalid_workspace_geometry(self):
with pytest.raises(ValueError, match="ubatch"):
dcp_utils.DirectDCPQGatherWorkspace(
dcp.DirectDCPQGatherWorkspace(
None, torch.device("cpu"), 16, 2, 32, num_ubatches=0
)
with pytest.raises(ValueError, match="padded heads"):
dcp_utils.DirectDCPQGatherWorkspace(
dcp.DirectDCPQGatherWorkspace(
_FakeProcessGroup(),
torch.device("cpu"),
16,
@@ -310,7 +311,7 @@ class TestDirectDCPGating:
padded_num_heads=7,
)
with pytest.raises(ValueError, match="16-byte"):
dcp_utils.DirectDCPQGatherWorkspace(
dcp.DirectDCPQGatherWorkspace(
_FakeProcessGroup(),
torch.device("cpu"),
16,
@@ -332,7 +333,7 @@ def _manager_config(dcp_comm_backend: str = "a2a"):
def test_mla_dcp_manager_selects_direct_backends(monkeypatch):
import vllm.v1.attention.ops.dcp_utils as dcp_manager
import vllm.v1.attention.ops.dcp as dcp_manager
group = MagicMock(world_size=2)
monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: group)
@@ -391,7 +392,7 @@ def test_mla_dcp_manager_selects_direct_backends(monkeypatch):
def test_mla_dcp_manager_selects_fallback_backends(monkeypatch):
import vllm.v1.attention.ops.dcp_utils as dcp_manager
import vllm.v1.attention.ops.dcp as dcp_manager
group = MagicMock(world_size=2)
gathered_query = torch.empty(1, 4, 8)
@@ -460,11 +461,11 @@ def test_dcp_workspace_covers_parallel_drafting():
config.num_speculative_tokens = 3
config.speculative_config = MagicMock(parallel_drafting=True)
assert dcp_utils.get_dcp_workspace_max_num_tokens(config) == 28
assert dcp.get_dcp_workspace_max_num_tokens(config) == 28
def test_mla_dcp_manager_selects_pcp_combine(monkeypatch):
import vllm.v1.attention.ops.dcp_utils as dcp_manager
import vllm.v1.attention.ops.dcp as dcp_manager
monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: MagicMock(world_size=2))
manager = dcp_manager.MLADCPManager(
@@ -521,7 +522,7 @@ def test_sparse_mla_builder_initializes_dcp_manager(monkeypatch):
lambda _: MagicMock(kv_lora_rank=8, qk_rope_head_dim=4),
)
manager = object.__new__(dcp_utils.MLADCPManager)
manager = object.__new__(dcp.MLADCPManager)
manager.init_kv_gather = MagicMock()
layer = MagicMock(dcp_manager=manager)
config = MagicMock()
@@ -580,12 +581,12 @@ def _distributed_direct_q_gather_worker(env: dict[str, str]) -> None:
heads_per_rank, head_dim, max_num_tokens = 6, 576, 128
padded_num_heads = 128 if world_size == 4 else None
active_ubatch = [0]
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
for dtype_idx, dtype_name in enumerate(
("bfloat16", "float8_e4m3fn", "float32")
):
dtype = _dtype_from_name(dtype_name)
workspace = dcp_utils.DirectDCPQGatherWorkspace(
workspace = dcp.DirectDCPQGatherWorkspace(
dist.group.WORLD,
device,
max_num_tokens,
@@ -747,11 +748,11 @@ def _distributed_direct_kv_gather_worker(env: dict[str, str]) -> None:
token_dim = 576
max_gathered_tokens = 128 * world_size
active_ubatch = [0]
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
for dtype_idx, dtype_name in enumerate(("bfloat16", "float16")):
dtype = _dtype_from_name(dtype_name)
workspace = dcp_utils.DirectDCPKVGatherWorkspace(
workspace = dcp.DirectDCPKVGatherWorkspace(
dist.group.WORLD,
device,
max_gathered_tokens,
@@ -818,7 +819,7 @@ def _distributed_direct_a2a_worker(env: dict[str, str]) -> None:
torch.accelerator.set_device_index(local_rank)
dist.init_process_group(backend="nccl")
try:
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
from vllm.v1.attention.ops.dcp import _lse_weighted_combine
rank = dist.get_rank()
world_size = dist.get_world_size()
@@ -829,8 +830,8 @@ def _distributed_direct_a2a_worker(env: dict[str, str]) -> None:
heads_per_rank, head_dim, max_num_tokens = 6, 512, 128
total_heads = world_size * heads_per_rank
active_ubatch = [0]
dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0]
workspace = dcp_utils.DirectDCPA2AWorkspace(
dcp.dbo_current_ubatch_id = lambda: active_ubatch[0]
workspace = dcp.DirectDCPA2AWorkspace(
dist.group.WORLD,
device,
max_num_tokens,
@@ -15,7 +15,7 @@ requires_flashinfer_mla = pytest.mark.skipif(
def test_mla_dcp_gathered_query_reserves_backend_head_storage():
from vllm.v1.attention.ops.dcp_utils import reserve_query_head_storage
from vllm.v1.attention.ops.dcp import reserve_query_head_storage
query = torch.randn(3, 24, 576, dtype=torch.bfloat16)
@@ -12,7 +12,7 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
triton_filter_and_convert_dcp_index,
)
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.attention.ops.common import CPTritonContext, correct_attn_out
from vllm.v1.attention.ops.dcp import CPTritonContext, correct_attn_out
def _local_count(length: int, rank: int, world: int, interleave: int) -> int:
@@ -242,10 +242,6 @@ from vllm.model_executor.layers.attention.attention import (
from vllm.model_executor.layers.attention.kv_transfer_utils import (
maybe_transfer_kv_layer,
)
from vllm.model_executor.layers.attention.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -293,8 +289,12 @@ from vllm.v1.attention.backends.utils import (
get_num_attention_heads_from_layers,
split_decodes_and_prefills,
)
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.attention.ops.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import (
AttentionSpec,
@@ -32,7 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
if TYPE_CHECKING:
@@ -13,7 +13,6 @@ from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.attention.pcp import maybe_gather_indexer_k
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
@@ -35,6 +34,7 @@ from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerMetadata,
)
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k
from vllm.v1.worker.workspace import current_workspace_manager
logger = init_logger(__name__)
+4 -4
View File
@@ -13,10 +13,6 @@ from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.attention.attention import get_attention_context
from vllm.model_executor.layers.attention.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -42,6 +38,10 @@ from vllm.model_executor.models.utils import extract_layer_index
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.ops.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
+1 -1
View File
@@ -92,7 +92,7 @@ from vllm.v1.attention.backend import (
MLAAttentionImpl,
)
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, get_kv_quant_mode
+1 -1
View File
@@ -886,7 +886,7 @@ class AttentionImplBase(ABC, Generic[T]):
# False => base 2 (lse = log2(sum(exp(qk))))
# -- e.g. FlashInfer trtllm-gen MLA
# The DCP combine kernel (cp_lse_ag_out_rs / dcp_a2a_lse_reduce in
# vllm/v1/attention/ops/common.py) branches on this via its IS_BASE_E
# vllm/v1/attention/ops/dcp.py) branches on this via its IS_BASE_E
# constexpr; getting it wrong silently corrupts the cross-shard
# softmax denominator.
lse_base_on_e: bool = True
+4 -2
View File
@@ -34,8 +34,10 @@ from vllm.v1.attention.backends.utils import (
fill_mm_prefix_query_ranges,
get_dcp_local_seq_lens,
)
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.dcp import (
cp_lse_ag_out_rs,
dcp_a2a_lse_reduce,
)
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.worker.workspace import current_workspace_manager
+4 -2
View File
@@ -75,8 +75,10 @@ from vllm.v1.attention.backends.utils import (
infer_global_hyperparameters,
split_decodes_and_prefills,
)
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.dcp import (
cp_lse_ag_out_rs,
dcp_a2a_lse_reduce,
)
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.kv_cache_interface import (
AttentionSpec,
-299
View File
@@ -2,308 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.triton_utils import tl, triton
def mask_dcp_empty_shards_(
lse: torch.Tensor,
seq_lens: torch.Tensor | None,
query_start_loc: torch.Tensor | None,
) -> None:
if seq_lens is None and query_start_loc is None:
return
if seq_lens is None or query_start_loc is None:
raise ValueError("seq_lens and query_start_loc must be provided together")
if (
seq_lens.ndim != 1
or query_start_loc.ndim != 1
or query_start_loc.shape[0] != seq_lens.shape[0] + 1
):
raise ValueError("query_start_loc must contain one boundary per sequence")
row_indices = torch.arange(
lse.shape[0], device=lse.device, dtype=query_start_loc.dtype
)
sequence_indices = torch.searchsorted(
query_start_loc[1:], row_indices, right=True
).clamp_max(seq_lens.shape[0] - 1)
empty_rows = (row_indices >= query_start_loc[-1]) | (
seq_lens[sequence_indices] == 0
)
lse.masked_fill_(empty_rows[:, None], float("-inf"))
@triton.jit
def _correct_attn_cp_out_kernel(
outputs_ptr,
new_output_ptr,
lses_ptr,
vlse_ptr,
outputs_stride_B,
outputs_stride_H,
outputs_stride_D,
lses_stride_N,
lses_stride_B,
lses_stride_H,
lse_idx,
HEAD_DIM: tl.constexpr,
N_ROUNDED: tl.constexpr,
IS_BASE_E: tl.constexpr,
):
"""
Apply the all-gathered lses to correct each local rank's attention
output. we still need perform a cross-rank reduction to obtain the
final attention output.
Args:
outputs_ptr (triton.PointerType):
Pointer to input tensor of shape [ B, H, D ]
lses_ptr (triton.PointerType):
Pointer to input tensor of shape [ N, B, H ]
new_output_ptr (triton.PointerType):
Pointer to output tensor of shape [ B, H, D ]
vlse_ptr (triton.PointerType):
Pointer to output tensor of shape [ B, H ]
"""
batch_idx = tl.program_id(axis=0).to(tl.int64)
head_idx = tl.program_id(axis=1).to(tl.int64)
d_offsets = tl.arange(0, HEAD_DIM)
num_n_offsets = tl.arange(0, N_ROUNDED)
# shape = [N]
lse_offsets = (
num_n_offsets * lses_stride_N
+ batch_idx * lses_stride_B
+ head_idx * lses_stride_H
)
# calc final lse
lse = tl.load(lses_ptr + lse_offsets).to(tl.float32)
lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse)
lse_max = tl.max(lse, axis=0)
lse_max = tl.where(lse_max == -float("inf"), 0, lse_max)
lse -= lse_max
if IS_BASE_E:
lse_exp = tl.exp(lse)
lse_acc = tl.sum(lse_exp, axis=0)
lse = tl.log(lse_acc)
else:
lse_exp = tl.exp2(lse)
lse_acc = tl.sum(lse_exp, axis=0)
lse = tl.log2(lse_acc)
lse += lse_max
lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H
tl.store(vlse_ptr + lse_offsets, lse)
# shape = [D]
output_offsets = (
batch_idx * outputs_stride_B
+ head_idx * outputs_stride_H
+ d_offsets * outputs_stride_D
)
# correct output
lse_offset = (
lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H
)
lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32)
lse_finally = lse_tmp - lse
lse_finally = tl.where(
(lse_finally != lse_finally) | (lse_finally == float("inf")),
-float("inf"),
lse_finally,
)
factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally)
output = tl.load(outputs_ptr + output_offsets)
output = output * factor
output = tl.where(factor == 0.0, 0.0, output)
tl.store(new_output_ptr + output_offsets, output)
class CPTritonContext:
"""The CPTritonContext is used to avoid recompilation of the Triton JIT."""
def __init__(self):
self.inner_kernel = None
def call_kernel(self, kernel, grid, *regular_args, **const_args):
if self.inner_kernel is None:
self.inner_kernel = kernel[grid](*regular_args, **const_args)
else:
self.inner_kernel[grid](*regular_args)
def correct_attn_out(
out: torch.Tensor,
lses: torch.Tensor,
cp_rank: int,
ctx: CPTritonContext,
is_lse_base_on_e: bool = True,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Correct the attention output using the all-gathered lses.
Args:
out: Tensor of shape [ B, H, D ]
lses: Tensor of shape [ N, B, H ]
cp_rank: Current rank in the context-parallel group
ctx: Triton context to avoid recompilation
Returns:
Tuple of (out, lse) with corrected attention and final log-sum-exp.
"""
if ctx is None:
ctx = CPTritonContext()
# --- Normalize to 3D views ---
if out.ndim == 4 and out.shape[1] == 1:
out = out.squeeze(1)
assert out.ndim == 3, f"expected out [B,H,D] or [B,1,H,D], got {tuple(out.shape)}"
if lses.ndim == 4 and lses.shape[-1] == 1:
lses = lses.squeeze(-1)
if lses.ndim == 4 and lses.shape[1] == 1:
lses = lses.squeeze(1)
assert lses.ndim == 3, (
f"expected lses [N,B,H] (optionally with a 1-sized extra dim), "
f"got {tuple(lses.shape)}"
)
B, H, D = out.shape
N = lses.shape[0]
# Strides after we normalized shapes to 3-D views. The kernel computes
# offsets for `vlse_ptr` using lses_stride_B/H, so the output buffer must
# have the same B/H stride layout as a slice of `lses`.
o_sB, o_sH, o_sD = out.stride()
l_sN, l_sB, l_sH = lses.stride()
# Allocate LSE with the same B/H strides as `lses` so writes land correctly
# even when `lses` is a non-contiguous view (e.g., 4-D to 3-D squeeze).
lse = torch.empty_strided(
(B, H), (l_sB, l_sH), device=lses.device, dtype=lses.dtype
)
# Kernel launch config
grid = (B, H, 1)
regular_args = (
out,
out,
lses,
lse,
o_sB,
o_sH,
o_sD,
l_sN,
l_sB,
l_sH,
cp_rank,
)
const_args = {"HEAD_DIM": D, "N_ROUNDED": N, "IS_BASE_E": is_lse_base_on_e}
ctx.call_kernel(_correct_attn_cp_out_kernel, grid, *regular_args, **const_args)
return out, lse
def _cp_lse_common(
cp_attn_out: torch.Tensor,
cp_attn_lse: torch.Tensor,
cp_group: GroupCoordinator,
ctx: CPTritonContext | None = None,
is_lse_base_on_e=True,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
):
"""
cp_attn_out: [ B, H, D ]
cp_attn_lse: [ B, H ]
"""
if cp_group.world_size == 1:
return cp_attn_out
if ctx is None:
ctx = CPTritonContext()
cp_attn_lse = cp_attn_lse.contiguous()
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
lses = cp_group.all_gather(cp_attn_lse, dim=0).reshape(
(cp_group.world_size,) + cp_attn_lse.shape
)
out, lse = correct_attn_out(
cp_attn_out,
lses,
cp_group.rank_in_group,
ctx,
is_lse_base_on_e=is_lse_base_on_e,
)
return out, lse
def cp_lse_ag_out_rs(
cp_attn_out: torch.Tensor,
cp_attn_lse: torch.Tensor,
cp_group: GroupCoordinator,
ctx: CPTritonContext | None = None,
return_lse: bool = False,
is_lse_base_on_e=True,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
):
"""
cp_attn_out: [ B, H, D ]
cp_attn_lse: [ B, H ]
"""
out, lse = _cp_lse_common(
cp_attn_out,
cp_attn_lse,
cp_group,
ctx=ctx,
is_lse_base_on_e=is_lse_base_on_e,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
out = cp_group.reduce_scatter(out, dim=1)
if return_lse:
cp_num_heads = lse.shape[1] // cp_group.world_size
cp_rank = cp_group.rank_in_group
lse = lse[:, cp_num_heads * cp_rank : cp_num_heads * (cp_rank + 1)]
return out, lse
return out
def cp_lse_ag_out_ar(
cp_attn_out: torch.Tensor,
cp_attn_lse: torch.Tensor,
cp_group: GroupCoordinator,
ctx: CPTritonContext | None = None,
return_lse: bool = False,
is_lse_base_on_e=True,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
):
"""
cp_attn_out: [ B, H, D ]
cp_attn_lse: [ B, H ]
"""
out, lse = _cp_lse_common(
cp_attn_out,
cp_attn_lse,
cp_group,
ctx=ctx,
is_lse_base_on_e=is_lse_base_on_e,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
out = cp_group.all_reduce(out)
if return_lse:
return out, lse
return out
@triton.jit
def _pack_seq_kernel(
x_ptr, # [N, D]
+154
View File
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared symmetric-memory infrastructure for context-parallel attention."""
from __future__ import annotations
import functools
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.parallel_state import in_the_same_node_as
from vllm.logger import init_logger
from vllm.platforms import current_platform
if TYPE_CHECKING:
from torch.distributed import ProcessGroup
from vllm.distributed.parallel_state import GroupCoordinator
logger = init_logger(__name__)
try:
import torch.distributed._symmetric_memory as symm_mem
symm_mem_available = True
except ImportError:
symm_mem = None # type: ignore[assignment]
symm_mem_available = False
@functools.cache
def _symm_mem_spans_group(group: GroupCoordinator) -> bool:
"""Probe whether the group has NVLS symmetric memory."""
if not symm_mem_available:
return False
try:
from torch._C._autograd import DeviceType
from torch._C._distributed_c10d import _SymmetricMemory
device = torch.device("cuda", torch.accelerator.current_device_index())
if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index):
return False
probe = symm_mem.empty(8, dtype=torch.uint8, device=device)
probe.zero_()
torch.accelerator.synchronize()
handle = symm_mem.rendezvous(probe, group.device_group.group_name)
spans = handle is not None and handle.multicast_ptr != 0
except Exception as error:
logger.debug("Direct CP symmetric-memory probe failed: %s", error)
return False
logger.debug_once(
"Direct CP symmetric memory across %d ranks: %s",
group.world_size,
"available" if spans else "unavailable",
)
return spans
def direct_cp_enabled(
group: GroupCoordinator,
dtype: torch.dtype,
use_direct: bool | None,
supported_dtypes: tuple[torch.dtype, ...] | None = None,
) -> bool:
if use_direct is not None:
return use_direct
return (
symm_mem_available
and current_platform.is_cuda()
and (supported_dtypes is None or dtype in supported_dtypes)
and (
all(in_the_same_node_as(group.cpu_group, source_rank=0))
or _symm_mem_spans_group(group)
)
)
def direct_cp_multicast_enabled(
group: GroupCoordinator,
dtype: torch.dtype,
use_direct: bool | None,
supported_dtypes: tuple[torch.dtype, ...] | None = None,
) -> bool:
return direct_cp_enabled(
group, dtype, use_direct, supported_dtypes
) and _symm_mem_spans_group(group)
class DirectCPWorkspace:
def __init__(
self,
group: ProcessGroup,
device: torch.device,
num_ubatches: int,
) -> None:
self.group = group
self.world_size = group.size()
self.rank = group.rank()
self.device = torch.device(device)
self.num_ubatches = num_ubatches
self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device)
self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = []
def _allocate(
self, shape: tuple[int, ...], dtype: torch.dtype
) -> tuple[torch.Tensor, torch.Tensor]:
storage = symm_mem.empty(shape, device=self.device, dtype=dtype)
storage.zero_()
torch.accelerator.synchronize()
handle = symm_mem.rendezvous(storage, self.group.group_name)
assert handle is not None, "CP symmetric memory rendezvous returned None"
handle.barrier()
views = [
handle.get_buffer(peer, list(shape), dtype, 0)
for peer in range(self.world_size)
]
self.device = storage.device
peer_ptrs = torch.tensor(
[
[view[ubatch].data_ptr() for view in views]
for ubatch in range(self.num_ubatches)
],
dtype=torch.int64,
device=self.device,
)
self._allocations.append((storage, handle, views))
return storage, peer_ptrs
def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]:
disabled = [0] * self.num_ubatches
for allocated, handle, _ in self._allocations:
if allocated is storage:
break
else:
return disabled
try:
from torch._C._autograd import DeviceType
from torch._C._distributed_c10d import _SymmetricMemory
if not _SymmetricMemory.has_multicast_support(
DeviceType.CUDA, storage.device.index
):
return disabled
multicast_base = handle.multicast_ptr
except Exception:
return disabled
if not multicast_base:
return disabled
storage_base = storage.data_ptr()
return [
multicast_base + (storage[ubatch].data_ptr() - storage_base)
for ubatch in range(self.num_ubatches)
]
File diff suppressed because it is too large Load Diff
-470
View File
@@ -1,470 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
DCP All-to-All communication backend for attention.
Provides All-to-All (A2A) communication as an alternative to
AllGather + ReduceScatter (AG+RS) for Decode Context Parallel (DCP).
Instead of gathering the full Q tensor and scattering partial outputs,
A2A exchanges partial attention outputs and their LSE values across
ranks, then combines them with exact LSE-weighted reduction.
This reduces the number of NCCL calls per attention layer by exchanging
the partial output and LSE in a single packed All-to-All payload.
Usage:
vllm serve model --tp 16 --dcp 16 --dcp-comm-backend a2a
Reference: https://arxiv.org/abs/2507.07120
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
import torch.distributed as dist
from vllm.triton_utils import tl, triton
from vllm.v1.attention.ops.common import mask_dcp_empty_shards_
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.v1.attention.ops.common import CPTritonContext
def _lse_weighted_combine(
outputs: torch.Tensor,
lses: torch.Tensor,
return_lse: bool = False,
is_lse_base_on_e: bool = True,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""
CPU reference implementation for LSE-weighted combination.
This is a pure PyTorch implementation used for testing and validation.
Args:
outputs: Partial attention outputs [N, B, H, D]
N = number of KV shards (ranks)
B = batch size (num_tokens)
H = number of heads per rank
D = head dimension
lses: Log-sum-exp values [N, B, H]
return_lse: If True, also return the global LSE
is_lse_base_on_e: If True, LSE is base e; if False, base 2
Returns:
Combined output [B, H, D], and optionally global LSE [B, H]
"""
N, B, H, D = outputs.shape
# Handle NaN and inf in LSEs
lses = torch.where(
torch.isnan(lses) | torch.isinf(lses),
torch.tensor(float("-inf"), device=lses.device, dtype=lses.dtype),
lses,
)
# Compute max LSE for numerical stability
lse_max, _ = lses.max(dim=0) # [B, H]
lse_max = torch.where(
lse_max == float("-inf"),
torch.zeros_like(lse_max),
lse_max,
)
# Compute weights: softmax over the N dimension
if is_lse_base_on_e:
weights = torch.exp(lses - lse_max.unsqueeze(0)) # [N, B, H]
else:
weights = torch.pow(2.0, lses - lse_max.unsqueeze(0)) # [N, B, H]
# Handle NaN weights
weights = torch.where(torch.isnan(weights), torch.zeros_like(weights), weights)
# Normalize weights
weight_sum = weights.sum(dim=0, keepdim=True) # [1, B, H]
weights = weights / weight_sum.clamp(min=1e-10) # [N, B, H]
# Weighted combination: sum over N dimension
weights = weights.unsqueeze(-1)
outputs = torch.where(weights == 0, torch.zeros_like(outputs), outputs)
result = (outputs * weights).sum(dim=0) # [B, H, D]
if return_lse:
if is_lse_base_on_e:
global_lse = torch.log(weight_sum.squeeze(0)) + lse_max # [B, H]
else:
global_lse = torch.log2(weight_sum.squeeze(0)) + lse_max # [B, H]
return result, global_lse
return result
def _dcp_a2a_lse_pack_dim(output_dtype: torch.dtype) -> int:
bits = torch.finfo(output_dtype).bits
if bits == 16:
return 2
if bits == 32:
return 1
raise ValueError(f"Cannot pack fp32 LSE into output dtype {output_dtype}.")
def _dcp_a2a_send_recv_buffers(
shape: tuple[int, ...],
device: torch.device,
dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
# Don't use the shared WorkspaceManager here. A FULL cudagraph bakes in the
# buffer address at capture, but the workspace is growable and sized only to
# the largest *captured* batch (the cudagraph capture cap). Any eager a2a
# with a bigger batch regrows it, freeing that address and poisoning every
# captured graph -> illegal memory access on replay. This bites the very
# first request: the post-capture warmup runs an eager decode at
# max_num_seqs (> the cap), so the graphs are already dangling before the
# server is ready. torch.empty buffers instead live in the graph's private
# pool and stay valid for its lifetime (as _dcp_a2a_unpack_combine and the
# AG+RS combine path already rely on).
return (
torch.empty(shape, device=device, dtype=dtype),
torch.empty(shape, device=device, dtype=dtype),
)
@triton.jit
def _dcp_a2a_pack_send_kernel(
out_ptr,
lse_ptr,
send_ptr,
out_stride_B,
out_stride_H,
out_stride_D,
lse_stride_B,
lse_stride_H,
send_stride_N,
send_stride_B,
send_stride_H,
send_stride_D,
N: tl.constexpr,
HEAD_DIM: tl.constexpr,
H_PER_RANK: tl.constexpr,
LSE_PACK_DIM: tl.constexpr,
):
batch_idx = tl.program_id(0).to(tl.int64)
local_head_idx = tl.program_id(1).to(tl.int64)
d_offsets = tl.arange(0, HEAD_DIM)
for rank_idx in tl.static_range(N):
src_head_idx = rank_idx * H_PER_RANK + local_head_idx
send_base = (
rank_idx * send_stride_N
+ batch_idx * send_stride_B
+ local_head_idx * send_stride_H
)
out_offsets = (
batch_idx * out_stride_B
+ src_head_idx * out_stride_H
+ d_offsets * out_stride_D
)
tl.store(
send_ptr + send_base + d_offsets * send_stride_D,
tl.load(out_ptr + out_offsets),
)
lse_val = tl.load(
lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H
).to(tl.float32)
if LSE_PACK_DIM == 1:
tl.store(
send_ptr + send_base + HEAD_DIM * send_stride_D,
lse_val.to(send_ptr.dtype.element_ty),
)
else:
lse_bits = lse_val.to(tl.uint32, bitcast=True)
lo = (lse_bits & 0xFFFF).to(tl.uint16)
hi = ((lse_bits >> 16) & 0xFFFF).to(tl.uint16)
tl.store(
send_ptr + send_base + HEAD_DIM * send_stride_D,
lo.to(send_ptr.dtype.element_ty, bitcast=True),
)
tl.store(
send_ptr + send_base + (HEAD_DIM + 1) * send_stride_D,
hi.to(send_ptr.dtype.element_ty, bitcast=True),
)
@triton.jit
def _dcp_a2a_unpack_combine_kernel(
recv_ptr,
out_ptr,
out_lse_ptr,
recv_stride_N,
recv_stride_B,
recv_stride_H,
recv_stride_D,
out_stride_B,
out_stride_H,
out_stride_D,
out_lse_stride_B,
out_lse_stride_H,
N: tl.constexpr,
HEAD_DIM: tl.constexpr,
IS_BASE_E: tl.constexpr,
RETURN_LSE: tl.constexpr,
LSE_PACK_DIM: tl.constexpr,
):
batch_idx = tl.program_id(0).to(tl.int64)
head_idx = tl.program_id(1).to(tl.int64)
d_offsets = tl.arange(0, HEAD_DIM)
lse_max = -float("inf")
for rank_idx in tl.static_range(N):
recv_base = (
rank_idx * recv_stride_N
+ batch_idx * recv_stride_B
+ head_idx * recv_stride_H
)
if LSE_PACK_DIM == 1:
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
tl.float32
)
else:
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
lse_val = tl.where(
(lse_val != lse_val) | (lse_val == float("inf")),
-float("inf"),
lse_val,
)
lse_max = tl.maximum(lse_max, lse_val)
lse_max = tl.where(lse_max == -float("inf"), 0.0, lse_max)
lse_sum = 0.0
for rank_idx in tl.static_range(N):
recv_base = (
rank_idx * recv_stride_N
+ batch_idx * recv_stride_B
+ head_idx * recv_stride_H
)
if LSE_PACK_DIM == 1:
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
tl.float32
)
else:
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
lse_val = tl.where(
(lse_val != lse_val) | (lse_val == float("inf")),
-float("inf"),
lse_val,
)
if IS_BASE_E:
lse_sum += tl.exp(lse_val - lse_max)
else:
lse_sum += tl.exp2(lse_val - lse_max)
if IS_BASE_E: # noqa: SIM108
global_lse = tl.log(lse_sum) + lse_max
else:
global_lse = tl.log2(lse_sum) + lse_max
acc = tl.zeros([HEAD_DIM], dtype=tl.float32)
for rank_idx in tl.static_range(N):
recv_base = (
rank_idx * recv_stride_N
+ batch_idx * recv_stride_B
+ head_idx * recv_stride_H
)
if LSE_PACK_DIM == 1:
lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to(
tl.float32
)
else:
lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D)
hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D)
lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32)
lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True)
lse_val = tl.where(
(lse_val != lse_val) | (lse_val == float("inf")),
-float("inf"),
lse_val,
)
if IS_BASE_E:
weight = tl.exp(lse_val - global_lse)
else:
weight = tl.exp2(lse_val - global_lse)
weight = tl.where(weight != weight, 0.0, weight)
partial = tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(
tl.float32
)
partial = tl.where(weight == 0.0, 0.0, partial)
acc += partial * weight
final_offsets = (
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
)
tl.store(out_ptr + final_offsets, acc)
if RETURN_LSE:
out_lse_offset = batch_idx * out_lse_stride_B + head_idx * out_lse_stride_H
tl.store(out_lse_ptr + out_lse_offset, global_lse)
def _dcp_a2a_pack_send(
cp_attn_out: torch.Tensor,
cp_attn_lse: torch.Tensor,
send_buffer: torch.Tensor,
world_size: int,
h_per_rank: int,
head_dim: int,
lse_pack_dim: int,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
) -> None:
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
grid = (cp_attn_out.shape[0], h_per_rank, 1)
_dcp_a2a_pack_send_kernel[grid](
cp_attn_out,
cp_attn_lse,
send_buffer,
cp_attn_out.stride(0),
cp_attn_out.stride(1),
cp_attn_out.stride(2),
cp_attn_lse.stride(0),
cp_attn_lse.stride(1),
send_buffer.stride(0),
send_buffer.stride(1),
send_buffer.stride(2),
send_buffer.stride(3),
N=world_size,
HEAD_DIM=head_dim,
H_PER_RANK=h_per_rank,
LSE_PACK_DIM=lse_pack_dim,
)
def _dcp_a2a_unpack_combine(
recv_buffer: torch.Tensor,
head_dim: int,
lse_pack_dim: int,
return_lse: bool,
is_lse_base_on_e: bool,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
world_size, num_tokens, h_per_rank, _ = recv_buffer.shape
out = torch.empty(
(num_tokens, h_per_rank, head_dim),
device=recv_buffer.device,
dtype=recv_buffer.dtype,
)
out_lse = torch.empty(
(num_tokens, h_per_rank) if return_lse else (1, 1),
device=recv_buffer.device,
dtype=torch.float32 if return_lse else recv_buffer.dtype,
)
grid = (num_tokens, h_per_rank, 1)
_dcp_a2a_unpack_combine_kernel[grid](
recv_buffer,
out,
out_lse,
recv_buffer.stride(0),
recv_buffer.stride(1),
recv_buffer.stride(2),
recv_buffer.stride(3),
out.stride(0),
out.stride(1),
out.stride(2),
out_lse.stride(0),
out_lse.stride(1),
N=world_size,
HEAD_DIM=head_dim,
IS_BASE_E=is_lse_base_on_e,
RETURN_LSE=return_lse,
LSE_PACK_DIM=lse_pack_dim,
)
if return_lse:
return out, out_lse
return out
def dcp_a2a_lse_reduce(
cp_attn_out: torch.Tensor,
cp_attn_lse: torch.Tensor,
cp_group: GroupCoordinator,
ctx: CPTritonContext | None = None,
return_lse: bool = False,
is_lse_base_on_e: bool = True,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""
Combine partial attention outputs across DCP ranks using All-to-All.
The output and LSE are packed into a single output-dtype buffer, sent
with one All-to-All, then unpacked and combined with exact LSE weighting.
Args:
cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim
cp_attn_lse: [B, H] floating-point log-sum-exp values
cp_group: GroupCoordinator for DCP communication
ctx: CPTritonContext (unused, for signature compatibility)
return_lse: If True, also return the combined global LSE
is_lse_base_on_e: If True, LSE is base e; if False, base 2
seq_lens: Local KV lengths. Empty shards contribute zero weight.
query_start_loc: Cumulative query-token offsets for each request.
Returns:
Combined output [B, H/N, D] (head-scattered)
If return_lse=True, also returns global_lse [B, H/N]
"""
world_size = cp_group.world_size
if world_size == 1:
if return_lse:
return cp_attn_out, cp_attn_lse
return cp_attn_out
B, H, D = cp_attn_out.shape
if H % world_size != 0:
raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.")
H_per_rank = H // world_size
lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype)
send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers(
(world_size, B, H_per_rank, D + lse_pack_dim),
device=cp_attn_out.device,
dtype=cp_attn_out.dtype,
)
_dcp_a2a_pack_send(
cp_attn_out,
cp_attn_lse,
send_buffer,
world_size,
H_per_rank,
D,
lse_pack_dim,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
work = dist.all_to_all_single(
recv_buffer.view(-1),
send_buffer.view(-1),
group=cp_group.device_group,
async_op=True,
)
work.wait()
return _dcp_a2a_unpack_combine(
recv_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e
)
-740
View File
@@ -1,740 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MLA DCP collective selection and direct symmetric-memory implementations."""
from __future__ import annotations
import functools
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Protocol
import torch
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import get_dcp_group
from vllm.distributed.parallel_state import in_the_same_node_as
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
logger = init_logger(__name__)
if TYPE_CHECKING:
from torch.distributed import ProcessGroup
from vllm.distributed.parallel_state import GroupCoordinator
try:
import torch.distributed._symmetric_memory as symm_mem
symm_mem_available = True
except ImportError:
symm_mem = None # type: ignore[assignment]
symm_mem_available = False
@functools.cache
def _symm_mem_spans_group(group: GroupCoordinator) -> bool:
"""Probe whether the group has NVLS symmetric memory."""
if not symm_mem_available:
return False
try:
from torch._C._autograd import DeviceType
from torch._C._distributed_c10d import _SymmetricMemory
device = torch.device("cuda", torch.accelerator.current_device_index())
if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index):
return False
probe = symm_mem.empty(8, dtype=torch.uint8, device=device)
probe.zero_()
torch.accelerator.synchronize()
handle = symm_mem.rendezvous(probe, group.device_group.group_name)
spans = handle is not None and handle.multicast_ptr != 0
except Exception as error:
logger.debug("Direct DCP symmetric-memory probe failed: %s", error)
return False
logger.debug_once(
"Direct DCP symmetric memory across %d ranks: %s",
group.world_size,
"available" if spans else "unavailable",
)
return spans
def _direct_dcp_enabled(
group: GroupCoordinator,
dtype: torch.dtype,
use_direct: bool | None,
supported_dtypes: tuple[torch.dtype, ...] | None = None,
) -> bool:
if use_direct is not None:
return use_direct
return (
symm_mem_available
and current_platform.is_cuda()
and (supported_dtypes is None or dtype in supported_dtypes)
and (
all(in_the_same_node_as(group.cpu_group, source_rank=0))
or _symm_mem_spans_group(group)
)
)
def _direct_dcp_multicast_enabled(
group: GroupCoordinator,
dtype: torch.dtype,
use_direct: bool | None,
supported_dtypes: tuple[torch.dtype, ...] | None = None,
) -> bool:
return _direct_dcp_enabled(
group, dtype, use_direct, supported_dtypes
) and _symm_mem_spans_group(group)
def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int:
scheduler_config = vllm_config.scheduler_config
speculative_config = vllm_config.speculative_config
speculative_tokens = vllm_config.num_speculative_tokens
tokens_per_seq = (
1
+ (
2
if speculative_config is not None and speculative_config.parallel_drafting
else 1
)
* speculative_tokens
)
return min(
scheduler_config.max_num_batched_tokens,
max(
scheduler_config.max_num_seqs * tokens_per_seq,
vllm_config.compilation_config.max_cudagraph_capture_size or 0,
),
)
class _DirectDCPWorkspace:
def __init__(
self,
group: ProcessGroup,
device: torch.device,
num_ubatches: int,
) -> None:
self.group = group
self.world_size = group.size()
self.rank = group.rank()
self.device = torch.device(device)
self.num_ubatches = num_ubatches
self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device)
self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = []
def _allocate(
self, shape: tuple[int, ...], dtype: torch.dtype
) -> tuple[torch.Tensor, torch.Tensor]:
storage = symm_mem.empty(shape, device=self.device, dtype=dtype)
storage.zero_()
torch.accelerator.synchronize()
handle = symm_mem.rendezvous(storage, self.group.group_name)
assert handle is not None, "DCP symmetric memory rendezvous returned None"
handle.barrier()
views = [
handle.get_buffer(peer, list(shape), dtype, 0)
for peer in range(self.world_size)
]
self.device = storage.device
peer_ptrs = torch.tensor(
[
[view[ubatch].data_ptr() for view in views]
for ubatch in range(self.num_ubatches)
],
dtype=torch.int64,
device=self.device,
)
self._allocations.append((storage, handle, views))
return storage, peer_ptrs
def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]:
disabled = [0] * self.num_ubatches
for allocated, handle, _ in self._allocations:
if allocated is storage:
break
else:
return disabled
try:
from torch._C._autograd import DeviceType
from torch._C._distributed_c10d import _SymmetricMemory
if not _SymmetricMemory.has_multicast_support(
DeviceType.CUDA, storage.device.index
):
return disabled
multicast_base = handle.multicast_ptr
except Exception:
return disabled
if not multicast_base:
return disabled
storage_base = storage.data_ptr()
return [
multicast_base + (storage[ubatch].data_ptr() - storage_base)
for ubatch in range(self.num_ubatches)
]
def reserve_query_head_storage(
query: torch.Tensor, padded_num_heads: int
) -> torch.Tensor:
"""Reserve backing storage for fixed-head decode kernels."""
assert query.ndim == 3
assert query.shape[1] <= padded_num_heads
padded = query.new_empty((query.shape[0], padded_num_heads, query.shape[2]))
padded.resize_(query.shape)
padded.copy_(query)
return padded
_A2A_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
class DirectDCPA2AWorkspace(_DirectDCPWorkspace):
"""Persistent symmetric buffers for direct DCP output exchange."""
def __init__(
self,
group: ProcessGroup,
device: torch.device,
max_num_tokens: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype = torch.bfloat16,
num_ubatches: int = 1,
) -> None:
if dtype not in _A2A_SUPPORTED_DTYPES:
raise ValueError(f"Direct DCP A2A does not support {dtype}")
if num_ubatches < 1:
raise ValueError(
f"Direct DCP A2A requires at least one ubatch slot, got {num_ubatches}"
)
super().__init__(group, device, num_ubatches)
self.max_num_tokens = max_num_tokens
self.heads_per_rank = heads_per_rank
self.head_dim = head_dim
output_shape = (
num_ubatches,
2,
self.world_size,
max_num_tokens,
heads_per_rank,
head_dim,
)
lse_shape = (
num_ubatches,
2,
self.world_size,
max_num_tokens,
heads_per_rank,
)
signal_shape = (num_ubatches, 2, self.world_size)
self.received_output, self.peer_output_ptrs = self._allocate(
output_shape, dtype
)
self.received_lse, self.peer_lse_ptrs = self._allocate(lse_shape, torch.float32)
self.received_signal, self.peer_signal_ptrs = self._allocate(
signal_shape, torch.int32
)
def lse_reduce(
self,
partial_output: torch.Tensor,
partial_lse: torch.Tensor,
is_lse_base_on_e: bool,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
) -> torch.Tensor:
ubatch = dbo_current_ubatch_id()
num_tokens = partial_output.shape[0]
output = partial_output.new_empty(
(num_tokens, self.heads_per_rank, self.head_dim)
)
torch.ops._C.direct_dcp_a2a_lse_reduce(
partial_output,
partial_lse,
seq_lens,
query_start_loc,
self.peer_output_ptrs[ubatch],
self.peer_lse_ptrs[ubatch],
self.peer_signal_ptrs[ubatch],
self.received_output[ubatch],
self.received_lse[ubatch],
self.received_signal[ubatch],
self.epoch[ubatch : ubatch + 1],
output,
self.world_size,
self.rank,
self.max_num_tokens,
is_lse_base_on_e,
)
return output
@functools.cache
def get_direct_dcp_a2a_workspace(
group: GroupCoordinator,
device: torch.device,
max_num_tokens: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype,
num_ubatches: int,
) -> DirectDCPA2AWorkspace | None:
if not _direct_dcp_enabled(
group, dtype, envs.VLLM_USE_DIRECT_DCP_A2A, _A2A_SUPPORTED_DTYPES
):
return None
return DirectDCPA2AWorkspace(
group.device_group,
device,
max_num_tokens,
heads_per_rank,
head_dim,
dtype,
num_ubatches,
)
def _q_gather_layout_supported(
world_size: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype,
padded_num_heads: int | None,
) -> bool:
element_size = torch.empty((), dtype=dtype).element_size()
gathered_num_heads = world_size * heads_per_rank
storage_num_heads = (
gathered_num_heads if padded_num_heads is None else padded_num_heads
)
return (
heads_per_rank * head_dim * element_size % 16 == 0
and storage_num_heads * head_dim * element_size % 16 == 0
)
class DirectDCPQGatherWorkspace(_DirectDCPWorkspace):
"""Publish query shards directly into the consumer-final symmetric buffer.
The final buffer is reusable after the downstream DCP output combine. That
combine orders all ranks after attention has consumed the gathered query.
"""
def __init__(
self,
group: ProcessGroup,
device: torch.device,
max_num_tokens: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype = torch.bfloat16,
num_ubatches: int = 1,
padded_num_heads: int | None = None,
) -> None:
if num_ubatches < 1:
raise ValueError(
"Direct DCP q-gather requires at least one ubatch slot, "
f"got {num_ubatches}"
)
if max_num_tokens < 1 or heads_per_rank < 1 or head_dim < 1:
raise ValueError(
"Direct DCP q-gather dimensions must be positive, got "
f"T={max_num_tokens}, H={heads_per_rank}, D={head_dim}"
)
gathered_num_heads = group.size() * heads_per_rank
if not _q_gather_layout_supported(
group.size(), heads_per_rank, head_dim, dtype, padded_num_heads
):
raise ValueError("Direct DCP q-gather requires 16-byte-aligned query rows.")
super().__init__(group, device, num_ubatches)
if self.world_size <= 1:
raise ValueError("Direct DCP q-gather requires at least two ranks")
self.max_num_tokens = max_num_tokens
self.heads_per_rank = heads_per_rank
self.gathered_num_heads = gathered_num_heads
self.padded_num_heads = (
self.gathered_num_heads if padded_num_heads is None else padded_num_heads
)
if self.padded_num_heads < self.gathered_num_heads:
raise ValueError(
"Direct DCP q-gather padded heads must cover gathered heads: "
f"{self.padded_num_heads} < {self.gathered_num_heads}"
)
self.head_dim = head_dim
query_shape = (
num_ubatches,
max_num_tokens,
self.padded_num_heads,
head_dim,
)
signal_shape = (num_ubatches, 2, self.world_size)
self.final_query, _ = self._allocate(query_shape, dtype)
self.received_signal, _ = self._allocate(signal_shape, torch.int32)
query_multicast_ptrs = self._multicast_ptrs(self.final_query)
signal_multicast_ptrs = self._multicast_ptrs(self.received_signal)
self.multicast_ptrs = list(
zip(query_multicast_ptrs, signal_multicast_ptrs, strict=True)
)
if not all(
query_ptr and signal_ptr for query_ptr, signal_ptr in self.multicast_ptrs
):
raise RuntimeError(
"Direct DCP q-gather requires NVLS symmetric-memory multicast."
)
self.completion = self.received_signal.new_zeros((num_ubatches, 1))
torch.accelerator.synchronize()
def gather(self, local_query: torch.Tensor) -> torch.Tensor:
ubatch = dbo_current_ubatch_id()
if not 0 <= ubatch < self.num_ubatches:
raise ValueError(
f"DCP q-gather ubatch {ubatch} exceeds {self.num_ubatches} slots"
)
if local_query.ndim == 3 and local_query.shape[1] != self.heads_per_rank:
raise ValueError(
f"DCP q-gather expected {self.heads_per_rank} local query heads, "
f"got {local_query.shape[1]}"
)
num_tokens = local_query.shape[0]
output = torch.as_strided(
self.final_query[ubatch],
size=(num_tokens, self.gathered_num_heads, self.head_dim),
stride=(
self.gathered_num_heads * self.head_dim,
self.head_dim,
1,
),
)
query_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch]
torch.ops._C.direct_dcp_q_gather(
local_query,
output,
self.received_signal[ubatch],
self.completion[ubatch],
self.epoch[ubatch : ubatch + 1],
self.world_size,
self.rank,
self.max_num_tokens,
self.padded_num_heads,
query_multicast_ptr,
signal_multicast_ptr,
)
return output
@functools.cache
def get_direct_dcp_q_gather_workspace(
group: GroupCoordinator,
device: torch.device,
max_num_tokens: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype,
num_ubatches: int,
padded_num_heads: int | None = None,
) -> DirectDCPQGatherWorkspace | None:
if not _direct_dcp_multicast_enabled(
group, dtype, envs.VLLM_USE_DIRECT_DCP_Q_GATHER
):
return None
if not _q_gather_layout_supported(
group.world_size, heads_per_rank, head_dim, dtype, padded_num_heads
):
return None
return DirectDCPQGatherWorkspace(
group.device_group,
device,
max_num_tokens,
heads_per_rank,
head_dim,
dtype,
num_ubatches,
padded_num_heads,
)
_KV_GATHER_SUPPORTED_DTYPES = (
torch.float16,
torch.bfloat16,
torch.float8_e4m3fn,
)
def _kv_gather_layout_supported(token_dim: int, dtype: torch.dtype) -> bool:
return token_dim * torch.empty((), dtype=dtype).element_size() % 16 == 0
class DirectDCPKVGatherWorkspace(_DirectDCPWorkspace):
"""Persistent symmetric buffers for direct DCP KV gather."""
def __init__(
self,
group: ProcessGroup,
device: torch.device,
max_gathered_tokens: int,
token_dim: int,
dtype: torch.dtype = torch.bfloat16,
num_ubatches: int = 1,
) -> None:
if dtype not in _KV_GATHER_SUPPORTED_DTYPES:
raise ValueError(f"Direct DCP kv-gather does not support {dtype}")
if num_ubatches < 1:
raise ValueError(
"Direct DCP kv-gather requires at least one ubatch slot, "
f"got {num_ubatches}"
)
if max_gathered_tokens < 1 or token_dim < 1:
raise ValueError(
"Direct DCP kv-gather dimensions must be positive, got "
f"T={max_gathered_tokens}, D={token_dim}"
)
if not _kv_gather_layout_supported(token_dim, dtype):
raise ValueError("Direct DCP kv-gather requires 16-byte-aligned KV rows.")
super().__init__(group, device, num_ubatches)
if self.world_size <= 1:
raise ValueError("Direct DCP kv-gather requires at least two ranks")
if max_gathered_tokens % self.world_size != 0:
raise ValueError(
"Direct DCP kv-gather capacity must divide evenly across "
f"ranks: {max_gathered_tokens} % {self.world_size} != 0"
)
self.max_gathered_tokens = max_gathered_tokens
kv_shape = (num_ubatches, 2, max_gathered_tokens, token_dim)
signal_shape = (num_ubatches, 2, self.world_size)
self.received_kv, _ = self._allocate(kv_shape, dtype)
self.received_signal, _ = self._allocate(signal_shape, torch.int32)
kv_multicast_ptrs = self._multicast_ptrs(self.received_kv)
signal_multicast_ptrs = self._multicast_ptrs(self.received_signal)
self.multicast_ptrs = list(
zip(kv_multicast_ptrs, signal_multicast_ptrs, strict=True)
)
if not all(kv_ptr and signal_ptr for kv_ptr, signal_ptr in self.multicast_ptrs):
raise RuntimeError(
"Direct DCP kv-gather requires NVLS symmetric-memory multicast."
)
self.completion = self.received_signal.new_zeros((num_ubatches, 2))
torch.accelerator.synchronize()
def gather(self, gathered_kv: torch.Tensor, local_kv: torch.Tensor) -> None:
ubatch = dbo_current_ubatch_id()
if not 0 <= ubatch < self.num_ubatches:
raise ValueError(
f"DCP kv-gather ubatch {ubatch} exceeds {self.num_ubatches} slots"
)
kv_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch]
torch.ops._C.direct_dcp_kv_gather(
local_kv,
self.received_kv[ubatch],
self.received_signal[ubatch],
self.completion[ubatch],
self.epoch[ubatch : ubatch + 1],
gathered_kv,
self.world_size,
self.rank,
self.max_gathered_tokens,
kv_multicast_ptr,
signal_multicast_ptr,
)
@functools.cache
def get_direct_dcp_kv_gather_workspace(
group: GroupCoordinator,
device: torch.device,
max_gathered_tokens: int,
token_dim: int,
dtype: torch.dtype,
num_ubatches: int,
) -> DirectDCPKVGatherWorkspace | None:
if not _direct_dcp_multicast_enabled(
group,
dtype,
envs.VLLM_USE_DIRECT_DCP_KV_GATHER,
_KV_GATHER_SUPPORTED_DTYPES,
):
return None
if not _kv_gather_layout_supported(token_dim, dtype):
return None
return DirectDCPKVGatherWorkspace(
group.device_group,
device,
max_gathered_tokens,
token_dim,
dtype,
num_ubatches,
)
class DCPCombine(Protocol):
def __call__(
self,
partial_output: torch.Tensor,
partial_lse: torch.Tensor,
*,
seq_lens: torch.Tensor,
query_start_loc: torch.Tensor,
) -> torch.Tensor: ...
class MLADCPManager:
"""Select and own layer-level collective implementations for MLA DCP."""
_kv_gather: Callable[[torch.Tensor, torch.Tensor], object]
def __init__(
self,
vllm_config: VllmConfig,
device: torch.device,
num_heads: int,
query_head_dim: int,
output_head_dim: int,
query_dtype: torch.dtype,
output_dtype: torch.dtype,
padded_num_heads: int | None,
is_lse_base_on_e: bool,
use_pcp: bool,
) -> None:
parallel_config = vllm_config.parallel_config
self.group = get_dcp_group()
self.device = torch.device(device)
self.num_ubatches = max(parallel_config.num_ubatches, 1)
self.max_num_tokens = get_dcp_workspace_max_num_tokens(vllm_config)
self.use_a2a = parallel_config.dcp_comm_backend == "a2a"
self.padded_num_heads = padded_num_heads
self.combine = self._init_combine(
num_heads,
output_head_dim,
output_dtype,
is_lse_base_on_e,
use_pcp,
)
self.query_gather = (
None
if use_pcp
else self._init_query_gather(
num_heads,
query_head_dim,
query_dtype,
)
)
def _init_combine(
self,
num_heads: int,
head_dim: int,
dtype: torch.dtype,
is_lse_base_on_e: bool,
use_pcp: bool,
) -> DCPCombine:
direct_workspace = None
if self.use_a2a:
direct_workspace = get_direct_dcp_a2a_workspace(
self.group,
self.device,
self.max_num_tokens,
num_heads,
head_dim,
dtype,
self.num_ubatches,
)
if direct_workspace is not None:
logger.info_once("Using direct symmetric-memory DCP A2A for MLA.")
return functools.partial(
direct_workspace.lse_reduce,
is_lse_base_on_e=is_lse_base_on_e,
)
combine_fn = (
dcp_a2a_lse_reduce
if self.use_a2a
else cp_lse_ag_out_ar
if use_pcp
else cp_lse_ag_out_rs
)
return functools.partial(
combine_fn,
cp_group=self.group,
is_lse_base_on_e=is_lse_base_on_e,
)
def _init_query_gather(
self,
num_heads: int,
head_dim: int,
dtype: torch.dtype,
) -> Callable[[torch.Tensor], torch.Tensor]:
direct_workspace = get_direct_dcp_q_gather_workspace(
self.group,
self.device,
self.max_num_tokens,
num_heads,
head_dim,
dtype,
self.num_ubatches,
self.padded_num_heads,
)
if direct_workspace is not None:
logger.info_once("Using direct symmetric-memory DCP query gather for MLA.")
return direct_workspace.gather
return self._gather_query
def _gather_query(self, query: torch.Tensor) -> torch.Tensor:
query = self.group.all_gather(query, dim=1)
if self.padded_num_heads is not None:
query = reserve_query_head_storage(query, self.padded_num_heads)
return query
def init_kv_gather(
self,
workspace: torch.Tensor,
max_gathered_tokens: int,
) -> None:
world_size = self.group.world_size
assert max_gathered_tokens > 0
assert max_gathered_tokens % world_size == 0
assert workspace.ndim == 2
assert workspace.is_contiguous()
assert workspace.shape[0] == (
max_gathered_tokens + max_gathered_tokens // world_size
)
assert workspace.shape[1] > 0
direct_workspace = get_direct_dcp_kv_gather_workspace(
self.group,
workspace.device,
max_gathered_tokens,
workspace.shape[1],
workspace.dtype,
self.num_ubatches,
)
if direct_workspace is not None:
logger.info_once(
"Using direct symmetric-memory DCP chunked-context KV gather for MLA."
)
self._kv_gather = direct_workspace.gather
else:
self._kv_gather = functools.partial(
torch.distributed.all_gather_into_tensor,
group=self.group.device_group,
)
def kv_gather(
self,
gathered_kv: torch.Tensor,
local_kv: torch.Tensor,
) -> object:
return self._kv_gather(gathered_kv, local_kv)