diff --git a/tests/distributed/test_dcp_a2a.py b/tests/distributed/test_dcp_a2a.py index dfd291742c..73b6e4a18b 100644 --- a/tests/distributed/test_dcp_a2a.py +++ b/tests/distributed/test_dcp_a2a.py @@ -44,7 +44,7 @@ def _packed_a2a_reference( h_per_rank: int, is_lse_base_on_e: bool, ) -> tuple[torch.Tensor, torch.Tensor]: - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine B, _H, D = cp_attn_out.shape outputs = ( @@ -157,13 +157,13 @@ class TestLSEWeightedCombine: def test_importable(self): """Verify _lse_weighted_combine is importable.""" - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine assert callable(_lse_weighted_combine) def test_single_rank(self): """Single rank: output unchanged.""" - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine # N=1, B=2, H=4, D=8 outputs = torch.randn(1, 2, 4, 8) @@ -176,7 +176,7 @@ class TestLSEWeightedCombine: def test_equal_lse(self): """Equal LSE values: outputs averaged equally.""" - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine _N, B, H, D = 2, 1, 1, 4 outputs = torch.tensor( @@ -200,7 +200,7 @@ class TestLSEWeightedCombine: def test_dominant_rank(self): """Different LSE values: larger LSE gets more weight.""" - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine B, H, D = 1, 1, 2 outputs = torch.tensor( @@ -222,7 +222,7 @@ class TestLSEWeightedCombine: torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5) def test_empty_shard_ignores_undefined_output(self): - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]]) lses = torch.tensor([[[-float("inf")]], [[0.0]]]) @@ -232,7 +232,7 @@ class TestLSEWeightedCombine: torch.testing.assert_close(result, outputs[1]) def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch): - import vllm.v1.attention.ops.common as common + import vllm.v1.attention.ops.dcp as dcp class FakeGroup: world_size = 2 @@ -243,7 +243,7 @@ class TestLSEWeightedCombine: return torch.cat((tensor, tensor), dim=dim) monkeypatch.setattr( - common, + dcp, "correct_attn_out", lambda output, lses, *args, **kwargs: (output, lses[0]), ) @@ -252,7 +252,7 @@ class TestLSEWeightedCombine: seq_lens = torch.tensor([0, 2], dtype=torch.int32) query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32) - _, masked_lse = common._cp_lse_common( + _, masked_lse = dcp._cp_lse_common( output, lse, FakeGroup(), @@ -266,7 +266,7 @@ class TestLSEWeightedCombine: def test_mathematically_correct(self): """Verify mathematical correctness of LSE combination.""" - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine outputs = torch.tensor( [ @@ -291,7 +291,7 @@ class TestLSEWeightedCombine: def test_return_lse(self): """return_lse=True returns global LSE (logsumexp of inputs).""" - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine B, H, D = 1, 1, 2 outputs = torch.tensor( @@ -317,7 +317,7 @@ class TestLSEWeightedCombine: def test_base2_return_lse(self): """Base-2 LSE mode returns log2-sum-exp2 global LSE.""" - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine outputs = torch.tensor( [ @@ -354,7 +354,7 @@ class TestLSEWeightedCombine: def test_lse_pack_dim(self): """Packed A2A stores one fp32 LSE in output-dtype lanes.""" - from vllm.v1.attention.ops.dcp_alltoall import _dcp_a2a_lse_pack_dim + from vllm.v1.attention.ops.dcp import _dcp_a2a_lse_pack_dim assert _dcp_a2a_lse_pack_dim(torch.bfloat16) == 2 assert _dcp_a2a_lse_pack_dim(torch.float16) == 2 @@ -374,7 +374,7 @@ class TestPackedA2AKernels: return_lse: bool, is_lse_base_on_e: bool, ): - from vllm.v1.attention.ops.dcp_alltoall import ( + from vllm.v1.attention.ops.dcp import ( _dcp_a2a_lse_pack_dim, _dcp_a2a_pack_send, _dcp_a2a_unpack_combine, @@ -421,7 +421,7 @@ class TestPackedA2AKernels: torch.accelerator.device_count() < 1, reason="CUDA is required." ) def test_empty_seq_lens_ignore_undefined_output(self): - from vllm.v1.attention.ops.dcp_alltoall import ( + from vllm.v1.attention.ops.dcp import ( _dcp_a2a_lse_pack_dim, _dcp_a2a_pack_send, _dcp_a2a_unpack_combine, @@ -505,7 +505,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None: init_workspace_manager(torch.device(f"cuda:{local_rank}")) try: - from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce + from vllm.v1.attention.ops.dcp import dcp_a2a_lse_reduce dtype = _dtype_from_name(env["TEST_DTYPE"]) return_lse = env["RETURN_LSE"] == "1" @@ -555,7 +555,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None: [t[:, rank * h_per_rank : (rank + 1) * h_per_rank] for t in gathered_lse], dim=0, ) - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine expected_out, expected_lse = _lse_weighted_combine( outputs, diff --git a/tests/distributed/test_dcp_direct_a2a_lse_reduce.py b/tests/distributed/test_dcp_direct_a2a_lse_reduce.py index 13561bac78..e5764954bd 100644 --- a/tests/distributed/test_dcp_direct_a2a_lse_reduce.py +++ b/tests/distributed/test_dcp_direct_a2a_lse_reduce.py @@ -10,7 +10,8 @@ import pytest import torch import torch.distributed as dist -import vllm.v1.attention.ops.dcp_utils as dcp_utils +import vllm.v1.attention.ops.cp_common as cp_common +import vllm.v1.attention.ops.dcp as dcp from vllm.utils.network_utils import get_open_port from vllm.utils.system_utils import update_environment_variables @@ -142,17 +143,17 @@ class _FakeProcessGroup: class TestDirectDCPGating: def test_env_disabled_returns_none(self, monkeypatch): monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0") - dcp_utils.get_direct_dcp_a2a_workspace.cache_clear() - workspace = dcp_utils.get_direct_dcp_a2a_workspace( + dcp.get_direct_dcp_a2a_workspace.cache_clear() + workspace = dcp.get_direct_dcp_a2a_workspace( _FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.bfloat16, 1 ) assert workspace is None def test_forced_with_unsupported_dtype_raises(self, monkeypatch): monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1") - dcp_utils.get_direct_dcp_a2a_workspace.cache_clear() + dcp.get_direct_dcp_a2a_workspace.cache_clear() with pytest.raises(ValueError, match="does not support"): - dcp_utils.get_direct_dcp_a2a_workspace( + dcp.get_direct_dcp_a2a_workspace( _FakeGroupCoordinator(), torch.device("cpu"), 16, @@ -164,14 +165,14 @@ class TestDirectDCPGating: def test_zero_ubatches_raises(self): with pytest.raises(ValueError, match="ubatch"): - dcp_utils.DirectDCPA2AWorkspace( + dcp.DirectDCPA2AWorkspace( None, torch.device("cpu"), 16, 2, 32, torch.bfloat16, num_ubatches=0 ) def test_auto_with_unsupported_dtype_returns_none(self, monkeypatch): monkeypatch.delenv("VLLM_USE_DIRECT_DCP_A2A", raising=False) - dcp_utils.get_direct_dcp_a2a_workspace.cache_clear() - workspace = dcp_utils.get_direct_dcp_a2a_workspace( + dcp.get_direct_dcp_a2a_workspace.cache_clear() + workspace = dcp.get_direct_dcp_a2a_workspace( _FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.float32, 1 ) assert workspace is None @@ -179,8 +180,8 @@ class TestDirectDCPGating: def test_q_gather_env_disabled_returns_none(self, monkeypatch): monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "0") monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1") - dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear() - workspace = dcp_utils.get_direct_dcp_q_gather_workspace( + dcp.get_direct_dcp_q_gather_workspace.cache_clear() + workspace = dcp.get_direct_dcp_q_gather_workspace( _FakeGroupCoordinator(), torch.device("cpu"), 16, @@ -194,17 +195,17 @@ class TestDirectDCPGating: def test_q_gather_flag_is_independent(self, monkeypatch): monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "1") monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0") - monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True) - dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear() + monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: True) + dcp.get_direct_dcp_q_gather_workspace.cache_clear() workspace = object() init_workspace = MagicMock(return_value=workspace) monkeypatch.setattr( - dcp_utils, + dcp, "DirectDCPQGatherWorkspace", init_workspace, ) - result = dcp_utils.get_direct_dcp_q_gather_workspace( + result = dcp.get_direct_dcp_q_gather_workspace( _FakeGroupCoordinator(), torch.device("cpu"), 16, @@ -220,8 +221,8 @@ class TestDirectDCPGating: def test_kv_gather_env_disabled_returns_none(self, monkeypatch): monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "0") monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1") - dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear() - workspace = dcp_utils.get_direct_dcp_kv_gather_workspace( + dcp.get_direct_dcp_kv_gather_workspace.cache_clear() + workspace = dcp.get_direct_dcp_kv_gather_workspace( _FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1 ) assert workspace is None @@ -229,17 +230,17 @@ class TestDirectDCPGating: def test_kv_gather_flag_is_independent(self, monkeypatch): monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "1") monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0") - monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True) - dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear() + monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: True) + dcp.get_direct_dcp_kv_gather_workspace.cache_clear() workspace = object() init_workspace = MagicMock(return_value=workspace) monkeypatch.setattr( - dcp_utils, + dcp, "DirectDCPKVGatherWorkspace", init_workspace, ) - result = dcp_utils.get_direct_dcp_kv_gather_workspace( + result = dcp.get_direct_dcp_kv_gather_workspace( _FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1 ) @@ -267,9 +268,9 @@ class TestDirectDCPGating: factory_name, factory_args, ): - factory = getattr(dcp_utils, factory_name) + factory = getattr(dcp, factory_name) monkeypatch.setenv(flag_name, "1") - monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: False) + monkeypatch.setattr(cp_common, "_symm_mem_spans_group", lambda group: False) factory.cache_clear() assert ( @@ -283,25 +284,25 @@ class TestDirectDCPGating: def test_kv_gather_rejects_invalid_workspace_geometry(self): with pytest.raises(ValueError, match="ubatch"): - dcp_utils.DirectDCPKVGatherWorkspace( + dcp.DirectDCPKVGatherWorkspace( None, torch.device("cpu"), 64, 576, num_ubatches=0 ) with pytest.raises(ValueError, match="divide evenly"): - dcp_utils.DirectDCPKVGatherWorkspace( + dcp.DirectDCPKVGatherWorkspace( _FakeProcessGroup(), torch.device("cpu"), 63, 576 ) with pytest.raises(ValueError, match="16-byte"): - dcp_utils.DirectDCPKVGatherWorkspace( + dcp.DirectDCPKVGatherWorkspace( _FakeProcessGroup(), torch.device("cpu"), 64, 3 ) def test_q_gather_rejects_invalid_workspace_geometry(self): with pytest.raises(ValueError, match="ubatch"): - dcp_utils.DirectDCPQGatherWorkspace( + dcp.DirectDCPQGatherWorkspace( None, torch.device("cpu"), 16, 2, 32, num_ubatches=0 ) with pytest.raises(ValueError, match="padded heads"): - dcp_utils.DirectDCPQGatherWorkspace( + dcp.DirectDCPQGatherWorkspace( _FakeProcessGroup(), torch.device("cpu"), 16, @@ -310,7 +311,7 @@ class TestDirectDCPGating: padded_num_heads=7, ) with pytest.raises(ValueError, match="16-byte"): - dcp_utils.DirectDCPQGatherWorkspace( + dcp.DirectDCPQGatherWorkspace( _FakeProcessGroup(), torch.device("cpu"), 16, @@ -332,7 +333,7 @@ def _manager_config(dcp_comm_backend: str = "a2a"): def test_mla_dcp_manager_selects_direct_backends(monkeypatch): - import vllm.v1.attention.ops.dcp_utils as dcp_manager + import vllm.v1.attention.ops.dcp as dcp_manager group = MagicMock(world_size=2) monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: group) @@ -391,7 +392,7 @@ def test_mla_dcp_manager_selects_direct_backends(monkeypatch): def test_mla_dcp_manager_selects_fallback_backends(monkeypatch): - import vllm.v1.attention.ops.dcp_utils as dcp_manager + import vllm.v1.attention.ops.dcp as dcp_manager group = MagicMock(world_size=2) gathered_query = torch.empty(1, 4, 8) @@ -460,11 +461,11 @@ def test_dcp_workspace_covers_parallel_drafting(): config.num_speculative_tokens = 3 config.speculative_config = MagicMock(parallel_drafting=True) - assert dcp_utils.get_dcp_workspace_max_num_tokens(config) == 28 + assert dcp.get_dcp_workspace_max_num_tokens(config) == 28 def test_mla_dcp_manager_selects_pcp_combine(monkeypatch): - import vllm.v1.attention.ops.dcp_utils as dcp_manager + import vllm.v1.attention.ops.dcp as dcp_manager monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: MagicMock(world_size=2)) manager = dcp_manager.MLADCPManager( @@ -521,7 +522,7 @@ def test_sparse_mla_builder_initializes_dcp_manager(monkeypatch): lambda _: MagicMock(kv_lora_rank=8, qk_rope_head_dim=4), ) - manager = object.__new__(dcp_utils.MLADCPManager) + manager = object.__new__(dcp.MLADCPManager) manager.init_kv_gather = MagicMock() layer = MagicMock(dcp_manager=manager) config = MagicMock() @@ -580,12 +581,12 @@ def _distributed_direct_q_gather_worker(env: dict[str, str]) -> None: heads_per_rank, head_dim, max_num_tokens = 6, 576, 128 padded_num_heads = 128 if world_size == 4 else None active_ubatch = [0] - dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0] + dcp.dbo_current_ubatch_id = lambda: active_ubatch[0] for dtype_idx, dtype_name in enumerate( ("bfloat16", "float8_e4m3fn", "float32") ): dtype = _dtype_from_name(dtype_name) - workspace = dcp_utils.DirectDCPQGatherWorkspace( + workspace = dcp.DirectDCPQGatherWorkspace( dist.group.WORLD, device, max_num_tokens, @@ -747,11 +748,11 @@ def _distributed_direct_kv_gather_worker(env: dict[str, str]) -> None: token_dim = 576 max_gathered_tokens = 128 * world_size active_ubatch = [0] - dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0] + dcp.dbo_current_ubatch_id = lambda: active_ubatch[0] for dtype_idx, dtype_name in enumerate(("bfloat16", "float16")): dtype = _dtype_from_name(dtype_name) - workspace = dcp_utils.DirectDCPKVGatherWorkspace( + workspace = dcp.DirectDCPKVGatherWorkspace( dist.group.WORLD, device, max_gathered_tokens, @@ -818,7 +819,7 @@ def _distributed_direct_a2a_worker(env: dict[str, str]) -> None: torch.accelerator.set_device_index(local_rank) dist.init_process_group(backend="nccl") try: - from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + from vllm.v1.attention.ops.dcp import _lse_weighted_combine rank = dist.get_rank() world_size = dist.get_world_size() @@ -829,8 +830,8 @@ def _distributed_direct_a2a_worker(env: dict[str, str]) -> None: heads_per_rank, head_dim, max_num_tokens = 6, 512, 128 total_heads = world_size * heads_per_rank active_ubatch = [0] - dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0] - workspace = dcp_utils.DirectDCPA2AWorkspace( + dcp.dbo_current_ubatch_id = lambda: active_ubatch[0] + workspace = dcp.DirectDCPA2AWorkspace( dist.group.WORLD, device, max_num_tokens, diff --git a/tests/v1/attention/test_flashinfer_mla_dcp.py b/tests/v1/attention/test_flashinfer_mla_dcp.py index f3a46bbb19..1841b25d49 100644 --- a/tests/v1/attention/test_flashinfer_mla_dcp.py +++ b/tests/v1/attention/test_flashinfer_mla_dcp.py @@ -15,7 +15,7 @@ requires_flashinfer_mla = pytest.mark.skipif( def test_mla_dcp_gathered_query_reserves_backend_head_storage(): - from vllm.v1.attention.ops.dcp_utils import reserve_query_head_storage + from vllm.v1.attention.ops.dcp import reserve_query_head_storage query = torch.randn(3, 24, 576, dtype=torch.bfloat16) diff --git a/tests/v1/attention/test_indexer_dcp_localize.py b/tests/v1/attention/test_indexer_dcp_localize.py index 60468cd337..48f3ffd5ac 100644 --- a/tests/v1/attention/test_indexer_dcp_localize.py +++ b/tests/v1/attention/test_indexer_dcp_localize.py @@ -12,7 +12,7 @@ from vllm.v1.attention.backends.mla.sparse_utils import ( triton_filter_and_convert_dcp_index, ) from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens -from vllm.v1.attention.ops.common import CPTritonContext, correct_attn_out +from vllm.v1.attention.ops.dcp import CPTritonContext, correct_attn_out def _local_count(length: int, rank: int, world: int, interleave: int) -> int: diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 37dc2c95ff..b41c810d76 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -242,10 +242,6 @@ from vllm.model_executor.layers.attention.attention import ( from vllm.model_executor.layers.attention.kv_transfer_utils import ( maybe_transfer_kv_layer, ) -from vllm.model_executor.layers.attention.pcp import ( - finalize_mla_pcp_decode, - maybe_gather_mla_latent_cache_inputs, -) from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -293,8 +289,12 @@ from vllm.v1.attention.backends.utils import ( get_num_attention_heads_from_layers, split_decodes_and_prefills, ) -from vllm.v1.attention.ops.dcp_utils import MLADCPManager +from vllm.v1.attention.ops.dcp import MLADCPManager from vllm.v1.attention.ops.merge_attn_states import merge_attn_states +from vllm.v1.attention.ops.pcp import ( + finalize_mla_pcp_decode, + maybe_gather_mla_latent_cache_inputs, +) from vllm.v1.attention.selector import get_attn_backend from vllm.v1.kv_cache_interface import ( AttentionSpec, diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index 85e09cd5d4..1fe267f3c8 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -32,7 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder from vllm.v1.attention.backends.fa_utils import get_flash_attn_version from vllm.v1.attention.backends.utils import split_decodes_and_prefills -from vllm.v1.attention.ops.dcp_utils import MLADCPManager +from vllm.v1.attention.ops.dcp import MLADCPManager from vllm.v1.attention.ops.merge_attn_states import merge_attn_states if TYPE_CHECKING: diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index 4f0e00cfab..8c7332aac7 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -13,7 +13,6 @@ from vllm.distributed import get_dcp_group, get_pcp_group from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp -from vllm.model_executor.layers.attention.pcp import maybe_gather_indexer_k from vllm.model_executor.layers.quantization.utils.quant_utils import ( get_fp8_min_max, ) @@ -35,6 +34,7 @@ from vllm.v1.attention.backends.mla.indexer import ( DeepseekV32IndexerMetadata, ) from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton +from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k from vllm.v1.worker.workspace import current_workspace_manager logger = init_logger(__name__) diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 02582810ca..4edcae4640 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -13,10 +13,6 @@ from vllm.distributed.parallel_state import get_tp_group from vllm.forward_context import get_forward_context from vllm.model_executor.layers.attention import MLAAttention from vllm.model_executor.layers.attention.attention import get_attention_context -from vllm.model_executor.layers.attention.pcp import ( - finalize_mla_pcp_decode, - maybe_gather_mla_latent_cache_inputs, -) from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -42,6 +38,10 @@ from vllm.model_executor.models.utils import extract_layer_index from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q from vllm.platforms import current_platform from vllm.utils.torch_utils import is_quantized_kv_cache +from vllm.v1.attention.ops.pcp import ( + finalize_mla_pcp_decode, + maybe_gather_mla_latent_cache_inputs, +) if TYPE_CHECKING: from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata diff --git a/vllm/models/kimi_k3/nvidia/mla.py b/vllm/models/kimi_k3/nvidia/mla.py index 2c67a8a4c4..07f51a9250 100644 --- a/vllm/models/kimi_k3/nvidia/mla.py +++ b/vllm/models/kimi_k3/nvidia/mla.py @@ -92,7 +92,7 @@ from vllm.v1.attention.backend import ( MLAAttentionImpl, ) from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend -from vllm.v1.attention.ops.dcp_utils import MLADCPManager +from vllm.v1.attention.ops.dcp import MLADCPManager from vllm.v1.attention.ops.merge_attn_states import merge_attn_states from vllm.v1.attention.selector import get_attn_backend from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, get_kv_quant_mode diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index d0da103fb3..04e91abb03 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -886,7 +886,7 @@ class AttentionImplBase(ABC, Generic[T]): # False => base 2 (lse = log2(sum(exp(qk)))) # -- e.g. FlashInfer trtllm-gen MLA # The DCP combine kernel (cp_lse_ag_out_rs / dcp_a2a_lse_reduce in - # vllm/v1/attention/ops/common.py) branches on this via its IS_BASE_E + # vllm/v1/attention/ops/dcp.py) branches on this via its IS_BASE_E # constexpr; getting it wrong silently corrupts the cross-shard # softmax denominator. lse_base_on_e: bool = True diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index aa89e3010d..7ccbddf124 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -34,8 +34,10 @@ from vllm.v1.attention.backends.utils import ( fill_mm_prefix_query_ranges, get_dcp_local_seq_lens, ) -from vllm.v1.attention.ops.common import cp_lse_ag_out_rs -from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce +from vllm.v1.attention.ops.dcp import ( + cp_lse_ag_out_rs, + dcp_a2a_lse_reduce, +) from vllm.v1.attention.ops.merge_attn_states import merge_attn_states from vllm.v1.worker.workspace import current_workspace_manager diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index de51e2198b..fa8faf2a58 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -75,8 +75,10 @@ from vllm.v1.attention.backends.utils import ( infer_global_hyperparameters, split_decodes_and_prefills, ) -from vllm.v1.attention.ops.common import cp_lse_ag_out_rs -from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce +from vllm.v1.attention.ops.dcp import ( + cp_lse_ag_out_rs, + dcp_a2a_lse_reduce, +) from vllm.v1.attention.ops.merge_attn_states import merge_attn_states from vllm.v1.kv_cache_interface import ( AttentionSpec, diff --git a/vllm/v1/attention/ops/common.py b/vllm/v1/attention/ops/common.py index cbe24cf87c..f3316ac557 100644 --- a/vllm/v1/attention/ops/common.py +++ b/vllm/v1/attention/ops/common.py @@ -2,308 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -from vllm.distributed.parallel_state import GroupCoordinator from vllm.triton_utils import tl, triton -def mask_dcp_empty_shards_( - lse: torch.Tensor, - seq_lens: torch.Tensor | None, - query_start_loc: torch.Tensor | None, -) -> None: - if seq_lens is None and query_start_loc is None: - return - if seq_lens is None or query_start_loc is None: - raise ValueError("seq_lens and query_start_loc must be provided together") - if ( - seq_lens.ndim != 1 - or query_start_loc.ndim != 1 - or query_start_loc.shape[0] != seq_lens.shape[0] + 1 - ): - raise ValueError("query_start_loc must contain one boundary per sequence") - - row_indices = torch.arange( - lse.shape[0], device=lse.device, dtype=query_start_loc.dtype - ) - sequence_indices = torch.searchsorted( - query_start_loc[1:], row_indices, right=True - ).clamp_max(seq_lens.shape[0] - 1) - empty_rows = (row_indices >= query_start_loc[-1]) | ( - seq_lens[sequence_indices] == 0 - ) - lse.masked_fill_(empty_rows[:, None], float("-inf")) - - -@triton.jit -def _correct_attn_cp_out_kernel( - outputs_ptr, - new_output_ptr, - lses_ptr, - vlse_ptr, - outputs_stride_B, - outputs_stride_H, - outputs_stride_D, - lses_stride_N, - lses_stride_B, - lses_stride_H, - lse_idx, - HEAD_DIM: tl.constexpr, - N_ROUNDED: tl.constexpr, - IS_BASE_E: tl.constexpr, -): - """ - Apply the all-gathered lses to correct each local rank's attention - output. we still need perform a cross-rank reduction to obtain the - final attention output. - - Args: - outputs_ptr (triton.PointerType): - Pointer to input tensor of shape [ B, H, D ] - lses_ptr (triton.PointerType): - Pointer to input tensor of shape [ N, B, H ] - new_output_ptr (triton.PointerType): - Pointer to output tensor of shape [ B, H, D ] - vlse_ptr (triton.PointerType): - Pointer to output tensor of shape [ B, H ] - """ - batch_idx = tl.program_id(axis=0).to(tl.int64) - head_idx = tl.program_id(axis=1).to(tl.int64) - d_offsets = tl.arange(0, HEAD_DIM) - num_n_offsets = tl.arange(0, N_ROUNDED) - - # shape = [N] - lse_offsets = ( - num_n_offsets * lses_stride_N - + batch_idx * lses_stride_B - + head_idx * lses_stride_H - ) - - # calc final lse - lse = tl.load(lses_ptr + lse_offsets).to(tl.float32) - lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse) - lse_max = tl.max(lse, axis=0) - lse_max = tl.where(lse_max == -float("inf"), 0, lse_max) - lse -= lse_max - if IS_BASE_E: - lse_exp = tl.exp(lse) - lse_acc = tl.sum(lse_exp, axis=0) - lse = tl.log(lse_acc) - else: - lse_exp = tl.exp2(lse) - lse_acc = tl.sum(lse_exp, axis=0) - lse = tl.log2(lse_acc) - lse += lse_max - - lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H - tl.store(vlse_ptr + lse_offsets, lse) - - # shape = [D] - output_offsets = ( - batch_idx * outputs_stride_B - + head_idx * outputs_stride_H - + d_offsets * outputs_stride_D - ) - - # correct output - lse_offset = ( - lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H - ) - lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32) - lse_finally = lse_tmp - lse - lse_finally = tl.where( - (lse_finally != lse_finally) | (lse_finally == float("inf")), - -float("inf"), - lse_finally, - ) - factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally) - output = tl.load(outputs_ptr + output_offsets) - output = output * factor - output = tl.where(factor == 0.0, 0.0, output) - - tl.store(new_output_ptr + output_offsets, output) - - -class CPTritonContext: - """The CPTritonContext is used to avoid recompilation of the Triton JIT.""" - - def __init__(self): - self.inner_kernel = None - - def call_kernel(self, kernel, grid, *regular_args, **const_args): - if self.inner_kernel is None: - self.inner_kernel = kernel[grid](*regular_args, **const_args) - else: - self.inner_kernel[grid](*regular_args) - - -def correct_attn_out( - out: torch.Tensor, - lses: torch.Tensor, - cp_rank: int, - ctx: CPTritonContext, - is_lse_base_on_e: bool = True, -) -> tuple[torch.Tensor, torch.Tensor]: - """Correct the attention output using the all-gathered lses. - - Args: - out: Tensor of shape [ B, H, D ] - lses: Tensor of shape [ N, B, H ] - cp_rank: Current rank in the context-parallel group - ctx: Triton context to avoid recompilation - - Returns: - Tuple of (out, lse) with corrected attention and final log-sum-exp. - """ - if ctx is None: - ctx = CPTritonContext() - - # --- Normalize to 3D views --- - if out.ndim == 4 and out.shape[1] == 1: - out = out.squeeze(1) - assert out.ndim == 3, f"expected out [B,H,D] or [B,1,H,D], got {tuple(out.shape)}" - - if lses.ndim == 4 and lses.shape[-1] == 1: - lses = lses.squeeze(-1) - if lses.ndim == 4 and lses.shape[1] == 1: - lses = lses.squeeze(1) - assert lses.ndim == 3, ( - f"expected lses [N,B,H] (optionally with a 1-sized extra dim), " - f"got {tuple(lses.shape)}" - ) - - B, H, D = out.shape - N = lses.shape[0] - - # Strides after we normalized shapes to 3-D views. The kernel computes - # offsets for `vlse_ptr` using lses_stride_B/H, so the output buffer must - # have the same B/H stride layout as a slice of `lses`. - o_sB, o_sH, o_sD = out.stride() - l_sN, l_sB, l_sH = lses.stride() - - # Allocate LSE with the same B/H strides as `lses` so writes land correctly - # even when `lses` is a non-contiguous view (e.g., 4-D to 3-D squeeze). - lse = torch.empty_strided( - (B, H), (l_sB, l_sH), device=lses.device, dtype=lses.dtype - ) - - # Kernel launch config - grid = (B, H, 1) - - regular_args = ( - out, - out, - lses, - lse, - o_sB, - o_sH, - o_sD, - l_sN, - l_sB, - l_sH, - cp_rank, - ) - const_args = {"HEAD_DIM": D, "N_ROUNDED": N, "IS_BASE_E": is_lse_base_on_e} - ctx.call_kernel(_correct_attn_cp_out_kernel, grid, *regular_args, **const_args) - return out, lse - - -def _cp_lse_common( - cp_attn_out: torch.Tensor, - cp_attn_lse: torch.Tensor, - cp_group: GroupCoordinator, - ctx: CPTritonContext | None = None, - is_lse_base_on_e=True, - seq_lens: torch.Tensor | None = None, - query_start_loc: torch.Tensor | None = None, -): - """ - cp_attn_out: [ B, H, D ] - cp_attn_lse: [ B, H ] - """ - if cp_group.world_size == 1: - return cp_attn_out - - if ctx is None: - ctx = CPTritonContext() - - cp_attn_lse = cp_attn_lse.contiguous() - mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc) - lses = cp_group.all_gather(cp_attn_lse, dim=0).reshape( - (cp_group.world_size,) + cp_attn_lse.shape - ) - out, lse = correct_attn_out( - cp_attn_out, - lses, - cp_group.rank_in_group, - ctx, - is_lse_base_on_e=is_lse_base_on_e, - ) - return out, lse - - -def cp_lse_ag_out_rs( - cp_attn_out: torch.Tensor, - cp_attn_lse: torch.Tensor, - cp_group: GroupCoordinator, - ctx: CPTritonContext | None = None, - return_lse: bool = False, - is_lse_base_on_e=True, - seq_lens: torch.Tensor | None = None, - query_start_loc: torch.Tensor | None = None, -): - """ - cp_attn_out: [ B, H, D ] - cp_attn_lse: [ B, H ] - """ - out, lse = _cp_lse_common( - cp_attn_out, - cp_attn_lse, - cp_group, - ctx=ctx, - is_lse_base_on_e=is_lse_base_on_e, - seq_lens=seq_lens, - query_start_loc=query_start_loc, - ) - out = cp_group.reduce_scatter(out, dim=1) - - if return_lse: - cp_num_heads = lse.shape[1] // cp_group.world_size - cp_rank = cp_group.rank_in_group - lse = lse[:, cp_num_heads * cp_rank : cp_num_heads * (cp_rank + 1)] - return out, lse - return out - - -def cp_lse_ag_out_ar( - cp_attn_out: torch.Tensor, - cp_attn_lse: torch.Tensor, - cp_group: GroupCoordinator, - ctx: CPTritonContext | None = None, - return_lse: bool = False, - is_lse_base_on_e=True, - seq_lens: torch.Tensor | None = None, - query_start_loc: torch.Tensor | None = None, -): - """ - cp_attn_out: [ B, H, D ] - cp_attn_lse: [ B, H ] - """ - out, lse = _cp_lse_common( - cp_attn_out, - cp_attn_lse, - cp_group, - ctx=ctx, - is_lse_base_on_e=is_lse_base_on_e, - seq_lens=seq_lens, - query_start_loc=query_start_loc, - ) - out = cp_group.all_reduce(out) - - if return_lse: - return out, lse - return out - - @triton.jit def _pack_seq_kernel( x_ptr, # [N, D] diff --git a/vllm/v1/attention/ops/cp_common.py b/vllm/v1/attention/ops/cp_common.py new file mode 100644 index 0000000000..3ee29b93e6 --- /dev/null +++ b/vllm/v1/attention/ops/cp_common.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared symmetric-memory infrastructure for context-parallel attention.""" + +from __future__ import annotations + +import functools +from typing import TYPE_CHECKING, Any + +import torch + +from vllm.distributed.parallel_state import in_the_same_node_as +from vllm.logger import init_logger +from vllm.platforms import current_platform + +if TYPE_CHECKING: + from torch.distributed import ProcessGroup + + from vllm.distributed.parallel_state import GroupCoordinator + +logger = init_logger(__name__) + +try: + import torch.distributed._symmetric_memory as symm_mem + + symm_mem_available = True +except ImportError: + symm_mem = None # type: ignore[assignment] + symm_mem_available = False + + +@functools.cache +def _symm_mem_spans_group(group: GroupCoordinator) -> bool: + """Probe whether the group has NVLS symmetric memory.""" + if not symm_mem_available: + return False + try: + from torch._C._autograd import DeviceType + from torch._C._distributed_c10d import _SymmetricMemory + + device = torch.device("cuda", torch.accelerator.current_device_index()) + if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index): + return False + probe = symm_mem.empty(8, dtype=torch.uint8, device=device) + probe.zero_() + torch.accelerator.synchronize() + handle = symm_mem.rendezvous(probe, group.device_group.group_name) + spans = handle is not None and handle.multicast_ptr != 0 + except Exception as error: + logger.debug("Direct CP symmetric-memory probe failed: %s", error) + return False + logger.debug_once( + "Direct CP symmetric memory across %d ranks: %s", + group.world_size, + "available" if spans else "unavailable", + ) + return spans + + +def direct_cp_enabled( + group: GroupCoordinator, + dtype: torch.dtype, + use_direct: bool | None, + supported_dtypes: tuple[torch.dtype, ...] | None = None, +) -> bool: + if use_direct is not None: + return use_direct + return ( + symm_mem_available + and current_platform.is_cuda() + and (supported_dtypes is None or dtype in supported_dtypes) + and ( + all(in_the_same_node_as(group.cpu_group, source_rank=0)) + or _symm_mem_spans_group(group) + ) + ) + + +def direct_cp_multicast_enabled( + group: GroupCoordinator, + dtype: torch.dtype, + use_direct: bool | None, + supported_dtypes: tuple[torch.dtype, ...] | None = None, +) -> bool: + return direct_cp_enabled( + group, dtype, use_direct, supported_dtypes + ) and _symm_mem_spans_group(group) + + +class DirectCPWorkspace: + def __init__( + self, + group: ProcessGroup, + device: torch.device, + num_ubatches: int, + ) -> None: + self.group = group + self.world_size = group.size() + self.rank = group.rank() + self.device = torch.device(device) + self.num_ubatches = num_ubatches + self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device) + self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = [] + + def _allocate( + self, shape: tuple[int, ...], dtype: torch.dtype + ) -> tuple[torch.Tensor, torch.Tensor]: + storage = symm_mem.empty(shape, device=self.device, dtype=dtype) + storage.zero_() + torch.accelerator.synchronize() + handle = symm_mem.rendezvous(storage, self.group.group_name) + assert handle is not None, "CP symmetric memory rendezvous returned None" + handle.barrier() + views = [ + handle.get_buffer(peer, list(shape), dtype, 0) + for peer in range(self.world_size) + ] + self.device = storage.device + peer_ptrs = torch.tensor( + [ + [view[ubatch].data_ptr() for view in views] + for ubatch in range(self.num_ubatches) + ], + dtype=torch.int64, + device=self.device, + ) + self._allocations.append((storage, handle, views)) + return storage, peer_ptrs + + def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]: + disabled = [0] * self.num_ubatches + for allocated, handle, _ in self._allocations: + if allocated is storage: + break + else: + return disabled + try: + from torch._C._autograd import DeviceType + from torch._C._distributed_c10d import _SymmetricMemory + + if not _SymmetricMemory.has_multicast_support( + DeviceType.CUDA, storage.device.index + ): + return disabled + multicast_base = handle.multicast_ptr + except Exception: + return disabled + if not multicast_base: + return disabled + storage_base = storage.data_ptr() + return [ + multicast_base + (storage[ubatch].data_ptr() - storage_base) + for ubatch in range(self.num_ubatches) + ] diff --git a/vllm/v1/attention/ops/dcp.py b/vllm/v1/attention/ops/dcp.py new file mode 100644 index 0000000000..6381bfa487 --- /dev/null +++ b/vllm/v1/attention/ops/dcp.py @@ -0,0 +1,1368 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MLA DCP collective selection and direct symmetric-memory implementations.""" + +from __future__ import annotations + +import functools +from collections.abc import Callable +from typing import TYPE_CHECKING, Protocol + +import torch +import torch.distributed as dist + +import vllm.envs as envs +from vllm.config import VllmConfig +from vllm.distributed import get_dcp_group +from vllm.logger import init_logger +from vllm.triton_utils import tl, triton +from vllm.v1.attention.ops.cp_common import ( + DirectCPWorkspace, + direct_cp_enabled, + direct_cp_multicast_enabled, +) +from vllm.v1.worker.ubatching import dbo_current_ubatch_id + +logger = init_logger(__name__) + +if TYPE_CHECKING: + from torch.distributed import ProcessGroup + + from vllm.distributed.parallel_state import GroupCoordinator + + +# LSE/output combine + + +def mask_dcp_empty_shards_( + lse: torch.Tensor, + seq_lens: torch.Tensor | None, + query_start_loc: torch.Tensor | None, +) -> None: + if seq_lens is None and query_start_loc is None: + return + if seq_lens is None or query_start_loc is None: + raise ValueError("seq_lens and query_start_loc must be provided together") + if ( + seq_lens.ndim != 1 + or query_start_loc.ndim != 1 + or query_start_loc.shape[0] != seq_lens.shape[0] + 1 + ): + raise ValueError("query_start_loc must contain one boundary per sequence") + + row_indices = torch.arange( + lse.shape[0], device=lse.device, dtype=query_start_loc.dtype + ) + sequence_indices = torch.searchsorted( + query_start_loc[1:], row_indices, right=True + ).clamp_max(seq_lens.shape[0] - 1) + empty_rows = (row_indices >= query_start_loc[-1]) | ( + seq_lens[sequence_indices] == 0 + ) + lse.masked_fill_(empty_rows[:, None], float("-inf")) + + +# AG + RS/AR implementation + + +@triton.jit +def _correct_attn_cp_out_kernel( + outputs_ptr, + new_output_ptr, + lses_ptr, + vlse_ptr, + outputs_stride_B, + outputs_stride_H, + outputs_stride_D, + lses_stride_N, + lses_stride_B, + lses_stride_H, + lse_idx, + HEAD_DIM: tl.constexpr, + N_ROUNDED: tl.constexpr, + IS_BASE_E: tl.constexpr, +): + """ + Apply the all-gathered lses to correct each local rank's attention + output. we still need perform a cross-rank reduction to obtain the + final attention output. + + Args: + outputs_ptr (triton.PointerType): + Pointer to input tensor of shape [ B, H, D ] + lses_ptr (triton.PointerType): + Pointer to input tensor of shape [ N, B, H ] + new_output_ptr (triton.PointerType): + Pointer to output tensor of shape [ B, H, D ] + vlse_ptr (triton.PointerType): + Pointer to output tensor of shape [ B, H ] + """ + batch_idx = tl.program_id(axis=0).to(tl.int64) + head_idx = tl.program_id(axis=1).to(tl.int64) + d_offsets = tl.arange(0, HEAD_DIM) + num_n_offsets = tl.arange(0, N_ROUNDED) + + # shape = [N] + lse_offsets = ( + num_n_offsets * lses_stride_N + + batch_idx * lses_stride_B + + head_idx * lses_stride_H + ) + + # calc final lse + lse = tl.load(lses_ptr + lse_offsets).to(tl.float32) + lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse) + lse_max = tl.max(lse, axis=0) + lse_max = tl.where(lse_max == -float("inf"), 0, lse_max) + lse -= lse_max + if IS_BASE_E: + lse_exp = tl.exp(lse) + lse_acc = tl.sum(lse_exp, axis=0) + lse = tl.log(lse_acc) + else: + lse_exp = tl.exp2(lse) + lse_acc = tl.sum(lse_exp, axis=0) + lse = tl.log2(lse_acc) + lse += lse_max + + lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H + tl.store(vlse_ptr + lse_offsets, lse) + + # shape = [D] + output_offsets = ( + batch_idx * outputs_stride_B + + head_idx * outputs_stride_H + + d_offsets * outputs_stride_D + ) + + # correct output + lse_offset = ( + lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H + ) + lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32) + lse_finally = lse_tmp - lse + lse_finally = tl.where( + (lse_finally != lse_finally) | (lse_finally == float("inf")), + -float("inf"), + lse_finally, + ) + factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally) + output = tl.load(outputs_ptr + output_offsets) + output = output * factor + output = tl.where(factor == 0.0, 0.0, output) + + tl.store(new_output_ptr + output_offsets, output) + + +class CPTritonContext: + """The CPTritonContext is used to avoid recompilation of the Triton JIT.""" + + def __init__(self): + self.inner_kernel = None + + def call_kernel(self, kernel, grid, *regular_args, **const_args): + if self.inner_kernel is None: + self.inner_kernel = kernel[grid](*regular_args, **const_args) + else: + self.inner_kernel[grid](*regular_args) + + +def correct_attn_out( + out: torch.Tensor, + lses: torch.Tensor, + cp_rank: int, + ctx: CPTritonContext, + is_lse_base_on_e: bool = True, +) -> tuple[torch.Tensor, torch.Tensor]: + """Correct the attention output using the all-gathered lses. + + Args: + out: Tensor of shape [ B, H, D ] + lses: Tensor of shape [ N, B, H ] + cp_rank: Current rank in the context-parallel group + ctx: Triton context to avoid recompilation + + Returns: + Tuple of (out, lse) with corrected attention and final log-sum-exp. + """ + if ctx is None: + ctx = CPTritonContext() + + # --- Normalize to 3D views --- + if out.ndim == 4 and out.shape[1] == 1: + out = out.squeeze(1) + assert out.ndim == 3, f"expected out [B,H,D] or [B,1,H,D], got {tuple(out.shape)}" + + if lses.ndim == 4 and lses.shape[-1] == 1: + lses = lses.squeeze(-1) + if lses.ndim == 4 and lses.shape[1] == 1: + lses = lses.squeeze(1) + assert lses.ndim == 3, ( + f"expected lses [N,B,H] (optionally with a 1-sized extra dim), " + f"got {tuple(lses.shape)}" + ) + + B, H, D = out.shape + N = lses.shape[0] + + # Strides after we normalized shapes to 3-D views. The kernel computes + # offsets for `vlse_ptr` using lses_stride_B/H, so the output buffer must + # have the same B/H stride layout as a slice of `lses`. + o_sB, o_sH, o_sD = out.stride() + l_sN, l_sB, l_sH = lses.stride() + + # Allocate LSE with the same B/H strides as `lses` so writes land correctly + # even when `lses` is a non-contiguous view (e.g., 4-D to 3-D squeeze). + lse = torch.empty_strided( + (B, H), (l_sB, l_sH), device=lses.device, dtype=lses.dtype + ) + + # Kernel launch config + grid = (B, H, 1) + + regular_args = ( + out, + out, + lses, + lse, + o_sB, + o_sH, + o_sD, + l_sN, + l_sB, + l_sH, + cp_rank, + ) + const_args = {"HEAD_DIM": D, "N_ROUNDED": N, "IS_BASE_E": is_lse_base_on_e} + ctx.call_kernel(_correct_attn_cp_out_kernel, grid, *regular_args, **const_args) + return out, lse + + +def _cp_lse_common( + cp_attn_out: torch.Tensor, + cp_attn_lse: torch.Tensor, + cp_group: GroupCoordinator, + ctx: CPTritonContext | None = None, + is_lse_base_on_e=True, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, +): + """ + cp_attn_out: [ B, H, D ] + cp_attn_lse: [ B, H ] + """ + if cp_group.world_size == 1: + return cp_attn_out + + if ctx is None: + ctx = CPTritonContext() + + cp_attn_lse = cp_attn_lse.contiguous() + mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc) + lses = cp_group.all_gather(cp_attn_lse, dim=0).reshape( + (cp_group.world_size,) + cp_attn_lse.shape + ) + out, lse = correct_attn_out( + cp_attn_out, + lses, + cp_group.rank_in_group, + ctx, + is_lse_base_on_e=is_lse_base_on_e, + ) + return out, lse + + +def cp_lse_ag_out_rs( + cp_attn_out: torch.Tensor, + cp_attn_lse: torch.Tensor, + cp_group: GroupCoordinator, + ctx: CPTritonContext | None = None, + return_lse: bool = False, + is_lse_base_on_e=True, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, +): + """ + cp_attn_out: [ B, H, D ] + cp_attn_lse: [ B, H ] + """ + out, lse = _cp_lse_common( + cp_attn_out, + cp_attn_lse, + cp_group, + ctx=ctx, + is_lse_base_on_e=is_lse_base_on_e, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + out = cp_group.reduce_scatter(out, dim=1) + + if return_lse: + cp_num_heads = lse.shape[1] // cp_group.world_size + cp_rank = cp_group.rank_in_group + lse = lse[:, cp_num_heads * cp_rank : cp_num_heads * (cp_rank + 1)] + return out, lse + return out + + +def cp_lse_ag_out_ar( + cp_attn_out: torch.Tensor, + cp_attn_lse: torch.Tensor, + cp_group: GroupCoordinator, + ctx: CPTritonContext | None = None, + return_lse: bool = False, + is_lse_base_on_e=True, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, +): + """ + cp_attn_out: [ B, H, D ] + cp_attn_lse: [ B, H ] + """ + out, lse = _cp_lse_common( + cp_attn_out, + cp_attn_lse, + cp_group, + ctx=ctx, + is_lse_base_on_e=is_lse_base_on_e, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + out = cp_group.all_reduce(out) + + if return_lse: + return out, lse + return out + + +# Standard A2A implementation + + +def _lse_weighted_combine( + outputs: torch.Tensor, + lses: torch.Tensor, + return_lse: bool = False, + is_lse_base_on_e: bool = True, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """ + CPU reference implementation for LSE-weighted combination. + + This is a pure PyTorch implementation used for testing and validation. + + Args: + outputs: Partial attention outputs [N, B, H, D] + N = number of KV shards (ranks) + B = batch size (num_tokens) + H = number of heads per rank + D = head dimension + lses: Log-sum-exp values [N, B, H] + return_lse: If True, also return the global LSE + is_lse_base_on_e: If True, LSE is base e; if False, base 2 + + Returns: + Combined output [B, H, D], and optionally global LSE [B, H] + """ + N, B, H, D = outputs.shape + + # Handle NaN and inf in LSEs + lses = torch.where( + torch.isnan(lses) | torch.isinf(lses), + torch.tensor(float("-inf"), device=lses.device, dtype=lses.dtype), + lses, + ) + + # Compute max LSE for numerical stability + lse_max, _ = lses.max(dim=0) # [B, H] + lse_max = torch.where( + lse_max == float("-inf"), + torch.zeros_like(lse_max), + lse_max, + ) + + # Compute weights: softmax over the N dimension + if is_lse_base_on_e: + weights = torch.exp(lses - lse_max.unsqueeze(0)) # [N, B, H] + else: + weights = torch.pow(2.0, lses - lse_max.unsqueeze(0)) # [N, B, H] + + # Handle NaN weights + weights = torch.where(torch.isnan(weights), torch.zeros_like(weights), weights) + + # Normalize weights + weight_sum = weights.sum(dim=0, keepdim=True) # [1, B, H] + weights = weights / weight_sum.clamp(min=1e-10) # [N, B, H] + + # Weighted combination: sum over N dimension + weights = weights.unsqueeze(-1) + outputs = torch.where(weights == 0, torch.zeros_like(outputs), outputs) + result = (outputs * weights).sum(dim=0) # [B, H, D] + + if return_lse: + if is_lse_base_on_e: + global_lse = torch.log(weight_sum.squeeze(0)) + lse_max # [B, H] + else: + global_lse = torch.log2(weight_sum.squeeze(0)) + lse_max # [B, H] + return result, global_lse + + return result + + +def _dcp_a2a_lse_pack_dim(output_dtype: torch.dtype) -> int: + bits = torch.finfo(output_dtype).bits + if bits == 16: + return 2 + if bits == 32: + return 1 + raise ValueError(f"Cannot pack fp32 LSE into output dtype {output_dtype}.") + + +def _dcp_a2a_send_recv_buffers( + shape: tuple[int, ...], + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor]: + # Don't use the shared WorkspaceManager here. A FULL cudagraph bakes in the + # buffer address at capture, but the workspace is growable and sized only to + # the largest *captured* batch (the cudagraph capture cap). Any eager a2a + # with a bigger batch regrows it, freeing that address and poisoning every + # captured graph -> illegal memory access on replay. This bites the very + # first request: the post-capture warmup runs an eager decode at + # max_num_seqs (> the cap), so the graphs are already dangling before the + # server is ready. torch.empty buffers instead live in the graph's private + # pool and stay valid for its lifetime (as _dcp_a2a_unpack_combine and the + # AG+RS combine path already rely on). + return ( + torch.empty(shape, device=device, dtype=dtype), + torch.empty(shape, device=device, dtype=dtype), + ) + + +@triton.jit +def _dcp_a2a_pack_send_kernel( + out_ptr, + lse_ptr, + send_ptr, + out_stride_B, + out_stride_H, + out_stride_D, + lse_stride_B, + lse_stride_H, + send_stride_N, + send_stride_B, + send_stride_H, + send_stride_D, + N: tl.constexpr, + HEAD_DIM: tl.constexpr, + H_PER_RANK: tl.constexpr, + LSE_PACK_DIM: tl.constexpr, +): + batch_idx = tl.program_id(0).to(tl.int64) + local_head_idx = tl.program_id(1).to(tl.int64) + d_offsets = tl.arange(0, HEAD_DIM) + + for rank_idx in tl.static_range(N): + src_head_idx = rank_idx * H_PER_RANK + local_head_idx + send_base = ( + rank_idx * send_stride_N + + batch_idx * send_stride_B + + local_head_idx * send_stride_H + ) + + out_offsets = ( + batch_idx * out_stride_B + + src_head_idx * out_stride_H + + d_offsets * out_stride_D + ) + tl.store( + send_ptr + send_base + d_offsets * send_stride_D, + tl.load(out_ptr + out_offsets), + ) + + lse_val = tl.load( + lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H + ).to(tl.float32) + if LSE_PACK_DIM == 1: + tl.store( + send_ptr + send_base + HEAD_DIM * send_stride_D, + lse_val.to(send_ptr.dtype.element_ty), + ) + else: + lse_bits = lse_val.to(tl.uint32, bitcast=True) + lo = (lse_bits & 0xFFFF).to(tl.uint16) + hi = ((lse_bits >> 16) & 0xFFFF).to(tl.uint16) + tl.store( + send_ptr + send_base + HEAD_DIM * send_stride_D, + lo.to(send_ptr.dtype.element_ty, bitcast=True), + ) + tl.store( + send_ptr + send_base + (HEAD_DIM + 1) * send_stride_D, + hi.to(send_ptr.dtype.element_ty, bitcast=True), + ) + + +@triton.jit +def _dcp_a2a_unpack_combine_kernel( + recv_ptr, + out_ptr, + out_lse_ptr, + recv_stride_N, + recv_stride_B, + recv_stride_H, + recv_stride_D, + out_stride_B, + out_stride_H, + out_stride_D, + out_lse_stride_B, + out_lse_stride_H, + N: tl.constexpr, + HEAD_DIM: tl.constexpr, + IS_BASE_E: tl.constexpr, + RETURN_LSE: tl.constexpr, + LSE_PACK_DIM: tl.constexpr, +): + batch_idx = tl.program_id(0).to(tl.int64) + head_idx = tl.program_id(1).to(tl.int64) + d_offsets = tl.arange(0, HEAD_DIM) + + lse_max = -float("inf") + for rank_idx in tl.static_range(N): + recv_base = ( + rank_idx * recv_stride_N + + batch_idx * recv_stride_B + + head_idx * recv_stride_H + ) + if LSE_PACK_DIM == 1: + lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( + tl.float32 + ) + else: + lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) + hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) + lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) + lse_val = tl.where( + (lse_val != lse_val) | (lse_val == float("inf")), + -float("inf"), + lse_val, + ) + lse_max = tl.maximum(lse_max, lse_val) + + lse_max = tl.where(lse_max == -float("inf"), 0.0, lse_max) + + lse_sum = 0.0 + for rank_idx in tl.static_range(N): + recv_base = ( + rank_idx * recv_stride_N + + batch_idx * recv_stride_B + + head_idx * recv_stride_H + ) + if LSE_PACK_DIM == 1: + lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( + tl.float32 + ) + else: + lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) + hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) + lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) + lse_val = tl.where( + (lse_val != lse_val) | (lse_val == float("inf")), + -float("inf"), + lse_val, + ) + if IS_BASE_E: + lse_sum += tl.exp(lse_val - lse_max) + else: + lse_sum += tl.exp2(lse_val - lse_max) + + if IS_BASE_E: # noqa: SIM108 + global_lse = tl.log(lse_sum) + lse_max + else: + global_lse = tl.log2(lse_sum) + lse_max + + acc = tl.zeros([HEAD_DIM], dtype=tl.float32) + for rank_idx in tl.static_range(N): + recv_base = ( + rank_idx * recv_stride_N + + batch_idx * recv_stride_B + + head_idx * recv_stride_H + ) + if LSE_PACK_DIM == 1: + lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( + tl.float32 + ) + else: + lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) + hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) + lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) + lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) + lse_val = tl.where( + (lse_val != lse_val) | (lse_val == float("inf")), + -float("inf"), + lse_val, + ) + if IS_BASE_E: + weight = tl.exp(lse_val - global_lse) + else: + weight = tl.exp2(lse_val - global_lse) + weight = tl.where(weight != weight, 0.0, weight) + partial = tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to( + tl.float32 + ) + partial = tl.where(weight == 0.0, 0.0, partial) + acc += partial * weight + + final_offsets = ( + batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D + ) + tl.store(out_ptr + final_offsets, acc) + + if RETURN_LSE: + out_lse_offset = batch_idx * out_lse_stride_B + head_idx * out_lse_stride_H + tl.store(out_lse_ptr + out_lse_offset, global_lse) + + +def _dcp_a2a_pack_send( + cp_attn_out: torch.Tensor, + cp_attn_lse: torch.Tensor, + send_buffer: torch.Tensor, + world_size: int, + h_per_rank: int, + head_dim: int, + lse_pack_dim: int, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, +) -> None: + mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc) + grid = (cp_attn_out.shape[0], h_per_rank, 1) + _dcp_a2a_pack_send_kernel[grid]( + cp_attn_out, + cp_attn_lse, + send_buffer, + cp_attn_out.stride(0), + cp_attn_out.stride(1), + cp_attn_out.stride(2), + cp_attn_lse.stride(0), + cp_attn_lse.stride(1), + send_buffer.stride(0), + send_buffer.stride(1), + send_buffer.stride(2), + send_buffer.stride(3), + N=world_size, + HEAD_DIM=head_dim, + H_PER_RANK=h_per_rank, + LSE_PACK_DIM=lse_pack_dim, + ) + + +def _dcp_a2a_unpack_combine( + recv_buffer: torch.Tensor, + head_dim: int, + lse_pack_dim: int, + return_lse: bool, + is_lse_base_on_e: bool, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + world_size, num_tokens, h_per_rank, _ = recv_buffer.shape + out = torch.empty( + (num_tokens, h_per_rank, head_dim), + device=recv_buffer.device, + dtype=recv_buffer.dtype, + ) + out_lse = torch.empty( + (num_tokens, h_per_rank) if return_lse else (1, 1), + device=recv_buffer.device, + dtype=torch.float32 if return_lse else recv_buffer.dtype, + ) + grid = (num_tokens, h_per_rank, 1) + _dcp_a2a_unpack_combine_kernel[grid]( + recv_buffer, + out, + out_lse, + recv_buffer.stride(0), + recv_buffer.stride(1), + recv_buffer.stride(2), + recv_buffer.stride(3), + out.stride(0), + out.stride(1), + out.stride(2), + out_lse.stride(0), + out_lse.stride(1), + N=world_size, + HEAD_DIM=head_dim, + IS_BASE_E=is_lse_base_on_e, + RETURN_LSE=return_lse, + LSE_PACK_DIM=lse_pack_dim, + ) + if return_lse: + return out, out_lse + return out + + +def dcp_a2a_lse_reduce( + cp_attn_out: torch.Tensor, + cp_attn_lse: torch.Tensor, + cp_group: GroupCoordinator, + ctx: CPTritonContext | None = None, + return_lse: bool = False, + is_lse_base_on_e: bool = True, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """ + Combine partial attention outputs across DCP ranks using All-to-All. + + The output and LSE are packed into a single output-dtype buffer, sent + with one All-to-All, then unpacked and combined with exact LSE weighting. + + Args: + cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim + cp_attn_lse: [B, H] floating-point log-sum-exp values + cp_group: GroupCoordinator for DCP communication + ctx: CPTritonContext (unused, for signature compatibility) + return_lse: If True, also return the combined global LSE + is_lse_base_on_e: If True, LSE is base e; if False, base 2 + seq_lens: Local KV lengths. Empty shards contribute zero weight. + query_start_loc: Cumulative query-token offsets for each request. + + Returns: + Combined output [B, H/N, D] (head-scattered) + If return_lse=True, also returns global_lse [B, H/N] + """ + world_size = cp_group.world_size + + if world_size == 1: + if return_lse: + return cp_attn_out, cp_attn_lse + return cp_attn_out + + B, H, D = cp_attn_out.shape + if H % world_size != 0: + raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.") + H_per_rank = H // world_size + lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype) + + send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers( + (world_size, B, H_per_rank, D + lse_pack_dim), + device=cp_attn_out.device, + dtype=cp_attn_out.dtype, + ) + + _dcp_a2a_pack_send( + cp_attn_out, + cp_attn_lse, + send_buffer, + world_size, + H_per_rank, + D, + lse_pack_dim, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + + work = dist.all_to_all_single( + recv_buffer.view(-1), + send_buffer.view(-1), + group=cp_group.device_group, + async_op=True, + ) + work.wait() + + return _dcp_a2a_unpack_combine( + recv_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e + ) + + +def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int: + scheduler_config = vllm_config.scheduler_config + speculative_config = vllm_config.speculative_config + speculative_tokens = vllm_config.num_speculative_tokens + tokens_per_seq = ( + 1 + + ( + 2 + if speculative_config is not None and speculative_config.parallel_drafting + else 1 + ) + * speculative_tokens + ) + return min( + scheduler_config.max_num_batched_tokens, + max( + scheduler_config.max_num_seqs * tokens_per_seq, + vllm_config.compilation_config.max_cudagraph_capture_size or 0, + ), + ) + + +def reserve_query_head_storage( + query: torch.Tensor, padded_num_heads: int +) -> torch.Tensor: + """Reserve backing storage for fixed-head decode kernels.""" + assert query.ndim == 3 + assert query.shape[1] <= padded_num_heads + padded = query.new_empty((query.shape[0], padded_num_heads, query.shape[2])) + padded.resize_(query.shape) + padded.copy_(query) + return padded + + +# Symmetric-memory A2A implementation + + +_A2A_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16) + + +class DirectDCPA2AWorkspace(DirectCPWorkspace): + """Persistent symmetric buffers for direct DCP output exchange.""" + + def __init__( + self, + group: ProcessGroup, + device: torch.device, + max_num_tokens: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype = torch.bfloat16, + num_ubatches: int = 1, + ) -> None: + if dtype not in _A2A_SUPPORTED_DTYPES: + raise ValueError(f"Direct DCP A2A does not support {dtype}") + if num_ubatches < 1: + raise ValueError( + f"Direct DCP A2A requires at least one ubatch slot, got {num_ubatches}" + ) + super().__init__(group, device, num_ubatches) + self.max_num_tokens = max_num_tokens + self.heads_per_rank = heads_per_rank + self.head_dim = head_dim + + output_shape = ( + num_ubatches, + 2, + self.world_size, + max_num_tokens, + heads_per_rank, + head_dim, + ) + lse_shape = ( + num_ubatches, + 2, + self.world_size, + max_num_tokens, + heads_per_rank, + ) + signal_shape = (num_ubatches, 2, self.world_size) + self.received_output, self.peer_output_ptrs = self._allocate( + output_shape, dtype + ) + self.received_lse, self.peer_lse_ptrs = self._allocate(lse_shape, torch.float32) + self.received_signal, self.peer_signal_ptrs = self._allocate( + signal_shape, torch.int32 + ) + + def lse_reduce( + self, + partial_output: torch.Tensor, + partial_lse: torch.Tensor, + is_lse_base_on_e: bool, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, + ) -> torch.Tensor: + ubatch = dbo_current_ubatch_id() + num_tokens = partial_output.shape[0] + output = partial_output.new_empty( + (num_tokens, self.heads_per_rank, self.head_dim) + ) + torch.ops._C.direct_dcp_a2a_lse_reduce( + partial_output, + partial_lse, + seq_lens, + query_start_loc, + self.peer_output_ptrs[ubatch], + self.peer_lse_ptrs[ubatch], + self.peer_signal_ptrs[ubatch], + self.received_output[ubatch], + self.received_lse[ubatch], + self.received_signal[ubatch], + self.epoch[ubatch : ubatch + 1], + output, + self.world_size, + self.rank, + self.max_num_tokens, + is_lse_base_on_e, + ) + return output + + +@functools.cache +def get_direct_dcp_a2a_workspace( + group: GroupCoordinator, + device: torch.device, + max_num_tokens: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype, + num_ubatches: int, +) -> DirectDCPA2AWorkspace | None: + if not direct_cp_enabled( + group, dtype, envs.VLLM_USE_DIRECT_DCP_A2A, _A2A_SUPPORTED_DTYPES + ): + return None + return DirectDCPA2AWorkspace( + group.device_group, + device, + max_num_tokens, + heads_per_rank, + head_dim, + dtype, + num_ubatches, + ) + + +# Q gather + +# Symmetric-memory implementation + + +def _q_gather_layout_supported( + world_size: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype, + padded_num_heads: int | None, +) -> bool: + element_size = torch.empty((), dtype=dtype).element_size() + gathered_num_heads = world_size * heads_per_rank + storage_num_heads = ( + gathered_num_heads if padded_num_heads is None else padded_num_heads + ) + return ( + heads_per_rank * head_dim * element_size % 16 == 0 + and storage_num_heads * head_dim * element_size % 16 == 0 + ) + + +class DirectDCPQGatherWorkspace(DirectCPWorkspace): + """Publish query shards directly into the consumer-final symmetric buffer. + + The final buffer is reusable after the downstream DCP output combine. That + combine orders all ranks after attention has consumed the gathered query. + """ + + def __init__( + self, + group: ProcessGroup, + device: torch.device, + max_num_tokens: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype = torch.bfloat16, + num_ubatches: int = 1, + padded_num_heads: int | None = None, + ) -> None: + if num_ubatches < 1: + raise ValueError( + "Direct DCP q-gather requires at least one ubatch slot, " + f"got {num_ubatches}" + ) + if max_num_tokens < 1 or heads_per_rank < 1 or head_dim < 1: + raise ValueError( + "Direct DCP q-gather dimensions must be positive, got " + f"T={max_num_tokens}, H={heads_per_rank}, D={head_dim}" + ) + gathered_num_heads = group.size() * heads_per_rank + if not _q_gather_layout_supported( + group.size(), heads_per_rank, head_dim, dtype, padded_num_heads + ): + raise ValueError("Direct DCP q-gather requires 16-byte-aligned query rows.") + super().__init__(group, device, num_ubatches) + if self.world_size <= 1: + raise ValueError("Direct DCP q-gather requires at least two ranks") + self.max_num_tokens = max_num_tokens + self.heads_per_rank = heads_per_rank + self.gathered_num_heads = gathered_num_heads + self.padded_num_heads = ( + self.gathered_num_heads if padded_num_heads is None else padded_num_heads + ) + if self.padded_num_heads < self.gathered_num_heads: + raise ValueError( + "Direct DCP q-gather padded heads must cover gathered heads: " + f"{self.padded_num_heads} < {self.gathered_num_heads}" + ) + self.head_dim = head_dim + + query_shape = ( + num_ubatches, + max_num_tokens, + self.padded_num_heads, + head_dim, + ) + signal_shape = (num_ubatches, 2, self.world_size) + self.final_query, _ = self._allocate(query_shape, dtype) + self.received_signal, _ = self._allocate(signal_shape, torch.int32) + query_multicast_ptrs = self._multicast_ptrs(self.final_query) + signal_multicast_ptrs = self._multicast_ptrs(self.received_signal) + self.multicast_ptrs = list( + zip(query_multicast_ptrs, signal_multicast_ptrs, strict=True) + ) + if not all( + query_ptr and signal_ptr for query_ptr, signal_ptr in self.multicast_ptrs + ): + raise RuntimeError( + "Direct DCP q-gather requires NVLS symmetric-memory multicast." + ) + self.completion = self.received_signal.new_zeros((num_ubatches, 1)) + torch.accelerator.synchronize() + + def gather(self, local_query: torch.Tensor) -> torch.Tensor: + ubatch = dbo_current_ubatch_id() + if not 0 <= ubatch < self.num_ubatches: + raise ValueError( + f"DCP q-gather ubatch {ubatch} exceeds {self.num_ubatches} slots" + ) + if local_query.ndim == 3 and local_query.shape[1] != self.heads_per_rank: + raise ValueError( + f"DCP q-gather expected {self.heads_per_rank} local query heads, " + f"got {local_query.shape[1]}" + ) + + num_tokens = local_query.shape[0] + output = torch.as_strided( + self.final_query[ubatch], + size=(num_tokens, self.gathered_num_heads, self.head_dim), + stride=( + self.gathered_num_heads * self.head_dim, + self.head_dim, + 1, + ), + ) + query_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch] + torch.ops._C.direct_dcp_q_gather( + local_query, + output, + self.received_signal[ubatch], + self.completion[ubatch], + self.epoch[ubatch : ubatch + 1], + self.world_size, + self.rank, + self.max_num_tokens, + self.padded_num_heads, + query_multicast_ptr, + signal_multicast_ptr, + ) + return output + + +@functools.cache +def get_direct_dcp_q_gather_workspace( + group: GroupCoordinator, + device: torch.device, + max_num_tokens: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype, + num_ubatches: int, + padded_num_heads: int | None = None, +) -> DirectDCPQGatherWorkspace | None: + if not direct_cp_multicast_enabled(group, dtype, envs.VLLM_USE_DIRECT_DCP_Q_GATHER): + return None + if not _q_gather_layout_supported( + group.world_size, heads_per_rank, head_dim, dtype, padded_num_heads + ): + return None + return DirectDCPQGatherWorkspace( + group.device_group, + device, + max_num_tokens, + heads_per_rank, + head_dim, + dtype, + num_ubatches, + padded_num_heads, + ) + + +# KV gather + +# Symmetric-memory implementation + + +_KV_GATHER_SUPPORTED_DTYPES = ( + torch.float16, + torch.bfloat16, + torch.float8_e4m3fn, +) + + +def _kv_gather_layout_supported(token_dim: int, dtype: torch.dtype) -> bool: + return token_dim * torch.empty((), dtype=dtype).element_size() % 16 == 0 + + +class DirectDCPKVGatherWorkspace(DirectCPWorkspace): + """Persistent symmetric buffers for direct DCP KV gather.""" + + def __init__( + self, + group: ProcessGroup, + device: torch.device, + max_gathered_tokens: int, + token_dim: int, + dtype: torch.dtype = torch.bfloat16, + num_ubatches: int = 1, + ) -> None: + if dtype not in _KV_GATHER_SUPPORTED_DTYPES: + raise ValueError(f"Direct DCP kv-gather does not support {dtype}") + if num_ubatches < 1: + raise ValueError( + "Direct DCP kv-gather requires at least one ubatch slot, " + f"got {num_ubatches}" + ) + if max_gathered_tokens < 1 or token_dim < 1: + raise ValueError( + "Direct DCP kv-gather dimensions must be positive, got " + f"T={max_gathered_tokens}, D={token_dim}" + ) + if not _kv_gather_layout_supported(token_dim, dtype): + raise ValueError("Direct DCP kv-gather requires 16-byte-aligned KV rows.") + super().__init__(group, device, num_ubatches) + if self.world_size <= 1: + raise ValueError("Direct DCP kv-gather requires at least two ranks") + if max_gathered_tokens % self.world_size != 0: + raise ValueError( + "Direct DCP kv-gather capacity must divide evenly across " + f"ranks: {max_gathered_tokens} % {self.world_size} != 0" + ) + self.max_gathered_tokens = max_gathered_tokens + + kv_shape = (num_ubatches, 2, max_gathered_tokens, token_dim) + signal_shape = (num_ubatches, 2, self.world_size) + self.received_kv, _ = self._allocate(kv_shape, dtype) + self.received_signal, _ = self._allocate(signal_shape, torch.int32) + kv_multicast_ptrs = self._multicast_ptrs(self.received_kv) + signal_multicast_ptrs = self._multicast_ptrs(self.received_signal) + self.multicast_ptrs = list( + zip(kv_multicast_ptrs, signal_multicast_ptrs, strict=True) + ) + if not all(kv_ptr and signal_ptr for kv_ptr, signal_ptr in self.multicast_ptrs): + raise RuntimeError( + "Direct DCP kv-gather requires NVLS symmetric-memory multicast." + ) + self.completion = self.received_signal.new_zeros((num_ubatches, 2)) + torch.accelerator.synchronize() + + def gather(self, gathered_kv: torch.Tensor, local_kv: torch.Tensor) -> None: + ubatch = dbo_current_ubatch_id() + if not 0 <= ubatch < self.num_ubatches: + raise ValueError( + f"DCP kv-gather ubatch {ubatch} exceeds {self.num_ubatches} slots" + ) + kv_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch] + torch.ops._C.direct_dcp_kv_gather( + local_kv, + self.received_kv[ubatch], + self.received_signal[ubatch], + self.completion[ubatch], + self.epoch[ubatch : ubatch + 1], + gathered_kv, + self.world_size, + self.rank, + self.max_gathered_tokens, + kv_multicast_ptr, + signal_multicast_ptr, + ) + + +@functools.cache +def get_direct_dcp_kv_gather_workspace( + group: GroupCoordinator, + device: torch.device, + max_gathered_tokens: int, + token_dim: int, + dtype: torch.dtype, + num_ubatches: int, +) -> DirectDCPKVGatherWorkspace | None: + if not direct_cp_multicast_enabled( + group, + dtype, + envs.VLLM_USE_DIRECT_DCP_KV_GATHER, + _KV_GATHER_SUPPORTED_DTYPES, + ): + return None + if not _kv_gather_layout_supported(token_dim, dtype): + return None + return DirectDCPKVGatherWorkspace( + group.device_group, + device, + max_gathered_tokens, + token_dim, + dtype, + num_ubatches, + ) + + +# MLA DCP backend selection + + +class DCPCombine(Protocol): + def __call__( + self, + partial_output: torch.Tensor, + partial_lse: torch.Tensor, + *, + seq_lens: torch.Tensor, + query_start_loc: torch.Tensor, + ) -> torch.Tensor: ... + + +class MLADCPManager: + """Select and own layer-level collective implementations for MLA DCP.""" + + _kv_gather: Callable[[torch.Tensor, torch.Tensor], object] + + def __init__( + self, + vllm_config: VllmConfig, + device: torch.device, + num_heads: int, + query_head_dim: int, + output_head_dim: int, + query_dtype: torch.dtype, + output_dtype: torch.dtype, + padded_num_heads: int | None, + is_lse_base_on_e: bool, + use_pcp: bool, + ) -> None: + parallel_config = vllm_config.parallel_config + self.group = get_dcp_group() + self.device = torch.device(device) + self.num_ubatches = max(parallel_config.num_ubatches, 1) + self.max_num_tokens = get_dcp_workspace_max_num_tokens(vllm_config) + self.use_a2a = parallel_config.dcp_comm_backend == "a2a" + self.padded_num_heads = padded_num_heads + + self.combine = self._init_combine( + num_heads, + output_head_dim, + output_dtype, + is_lse_base_on_e, + use_pcp, + ) + self.query_gather = ( + None + if use_pcp + else self._init_query_gather( + num_heads, + query_head_dim, + query_dtype, + ) + ) + + def _init_combine( + self, + num_heads: int, + head_dim: int, + dtype: torch.dtype, + is_lse_base_on_e: bool, + use_pcp: bool, + ) -> DCPCombine: + direct_workspace = None + if self.use_a2a: + direct_workspace = get_direct_dcp_a2a_workspace( + self.group, + self.device, + self.max_num_tokens, + num_heads, + head_dim, + dtype, + self.num_ubatches, + ) + if direct_workspace is not None: + logger.info_once("Using direct symmetric-memory DCP A2A for MLA.") + return functools.partial( + direct_workspace.lse_reduce, + is_lse_base_on_e=is_lse_base_on_e, + ) + + combine_fn = ( + dcp_a2a_lse_reduce + if self.use_a2a + else cp_lse_ag_out_ar + if use_pcp + else cp_lse_ag_out_rs + ) + return functools.partial( + combine_fn, + cp_group=self.group, + is_lse_base_on_e=is_lse_base_on_e, + ) + + def _init_query_gather( + self, + num_heads: int, + head_dim: int, + dtype: torch.dtype, + ) -> Callable[[torch.Tensor], torch.Tensor]: + direct_workspace = get_direct_dcp_q_gather_workspace( + self.group, + self.device, + self.max_num_tokens, + num_heads, + head_dim, + dtype, + self.num_ubatches, + self.padded_num_heads, + ) + if direct_workspace is not None: + logger.info_once("Using direct symmetric-memory DCP query gather for MLA.") + return direct_workspace.gather + return self._gather_query + + def _gather_query(self, query: torch.Tensor) -> torch.Tensor: + query = self.group.all_gather(query, dim=1) + if self.padded_num_heads is not None: + query = reserve_query_head_storage(query, self.padded_num_heads) + return query + + def init_kv_gather( + self, + workspace: torch.Tensor, + max_gathered_tokens: int, + ) -> None: + world_size = self.group.world_size + assert max_gathered_tokens > 0 + assert max_gathered_tokens % world_size == 0 + assert workspace.ndim == 2 + assert workspace.is_contiguous() + assert workspace.shape[0] == ( + max_gathered_tokens + max_gathered_tokens // world_size + ) + assert workspace.shape[1] > 0 + + direct_workspace = get_direct_dcp_kv_gather_workspace( + self.group, + workspace.device, + max_gathered_tokens, + workspace.shape[1], + workspace.dtype, + self.num_ubatches, + ) + if direct_workspace is not None: + logger.info_once( + "Using direct symmetric-memory DCP chunked-context KV gather for MLA." + ) + self._kv_gather = direct_workspace.gather + else: + self._kv_gather = functools.partial( + torch.distributed.all_gather_into_tensor, + group=self.group.device_group, + ) + + def kv_gather( + self, + gathered_kv: torch.Tensor, + local_kv: torch.Tensor, + ) -> object: + return self._kv_gather(gathered_kv, local_kv) diff --git a/vllm/v1/attention/ops/dcp_alltoall.py b/vllm/v1/attention/ops/dcp_alltoall.py deleted file mode 100644 index 5939b61a8d..0000000000 --- a/vllm/v1/attention/ops/dcp_alltoall.py +++ /dev/null @@ -1,470 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -DCP All-to-All communication backend for attention. - -Provides All-to-All (A2A) communication as an alternative to -AllGather + ReduceScatter (AG+RS) for Decode Context Parallel (DCP). -Instead of gathering the full Q tensor and scattering partial outputs, -A2A exchanges partial attention outputs and their LSE values across -ranks, then combines them with exact LSE-weighted reduction. - -This reduces the number of NCCL calls per attention layer by exchanging -the partial output and LSE in a single packed All-to-All payload. - -Usage: - vllm serve model --tp 16 --dcp 16 --dcp-comm-backend a2a - -Reference: https://arxiv.org/abs/2507.07120 -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import torch -import torch.distributed as dist - -from vllm.triton_utils import tl, triton -from vllm.v1.attention.ops.common import mask_dcp_empty_shards_ - -if TYPE_CHECKING: - from vllm.distributed.parallel_state import GroupCoordinator - from vllm.v1.attention.ops.common import CPTritonContext - - -def _lse_weighted_combine( - outputs: torch.Tensor, - lses: torch.Tensor, - return_lse: bool = False, - is_lse_base_on_e: bool = True, -) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """ - CPU reference implementation for LSE-weighted combination. - - This is a pure PyTorch implementation used for testing and validation. - - Args: - outputs: Partial attention outputs [N, B, H, D] - N = number of KV shards (ranks) - B = batch size (num_tokens) - H = number of heads per rank - D = head dimension - lses: Log-sum-exp values [N, B, H] - return_lse: If True, also return the global LSE - is_lse_base_on_e: If True, LSE is base e; if False, base 2 - - Returns: - Combined output [B, H, D], and optionally global LSE [B, H] - """ - N, B, H, D = outputs.shape - - # Handle NaN and inf in LSEs - lses = torch.where( - torch.isnan(lses) | torch.isinf(lses), - torch.tensor(float("-inf"), device=lses.device, dtype=lses.dtype), - lses, - ) - - # Compute max LSE for numerical stability - lse_max, _ = lses.max(dim=0) # [B, H] - lse_max = torch.where( - lse_max == float("-inf"), - torch.zeros_like(lse_max), - lse_max, - ) - - # Compute weights: softmax over the N dimension - if is_lse_base_on_e: - weights = torch.exp(lses - lse_max.unsqueeze(0)) # [N, B, H] - else: - weights = torch.pow(2.0, lses - lse_max.unsqueeze(0)) # [N, B, H] - - # Handle NaN weights - weights = torch.where(torch.isnan(weights), torch.zeros_like(weights), weights) - - # Normalize weights - weight_sum = weights.sum(dim=0, keepdim=True) # [1, B, H] - weights = weights / weight_sum.clamp(min=1e-10) # [N, B, H] - - # Weighted combination: sum over N dimension - weights = weights.unsqueeze(-1) - outputs = torch.where(weights == 0, torch.zeros_like(outputs), outputs) - result = (outputs * weights).sum(dim=0) # [B, H, D] - - if return_lse: - if is_lse_base_on_e: - global_lse = torch.log(weight_sum.squeeze(0)) + lse_max # [B, H] - else: - global_lse = torch.log2(weight_sum.squeeze(0)) + lse_max # [B, H] - return result, global_lse - - return result - - -def _dcp_a2a_lse_pack_dim(output_dtype: torch.dtype) -> int: - bits = torch.finfo(output_dtype).bits - if bits == 16: - return 2 - if bits == 32: - return 1 - raise ValueError(f"Cannot pack fp32 LSE into output dtype {output_dtype}.") - - -def _dcp_a2a_send_recv_buffers( - shape: tuple[int, ...], - device: torch.device, - dtype: torch.dtype, -) -> tuple[torch.Tensor, torch.Tensor]: - # Don't use the shared WorkspaceManager here. A FULL cudagraph bakes in the - # buffer address at capture, but the workspace is growable and sized only to - # the largest *captured* batch (the cudagraph capture cap). Any eager a2a - # with a bigger batch regrows it, freeing that address and poisoning every - # captured graph -> illegal memory access on replay. This bites the very - # first request: the post-capture warmup runs an eager decode at - # max_num_seqs (> the cap), so the graphs are already dangling before the - # server is ready. torch.empty buffers instead live in the graph's private - # pool and stay valid for its lifetime (as _dcp_a2a_unpack_combine and the - # AG+RS combine path already rely on). - return ( - torch.empty(shape, device=device, dtype=dtype), - torch.empty(shape, device=device, dtype=dtype), - ) - - -@triton.jit -def _dcp_a2a_pack_send_kernel( - out_ptr, - lse_ptr, - send_ptr, - out_stride_B, - out_stride_H, - out_stride_D, - lse_stride_B, - lse_stride_H, - send_stride_N, - send_stride_B, - send_stride_H, - send_stride_D, - N: tl.constexpr, - HEAD_DIM: tl.constexpr, - H_PER_RANK: tl.constexpr, - LSE_PACK_DIM: tl.constexpr, -): - batch_idx = tl.program_id(0).to(tl.int64) - local_head_idx = tl.program_id(1).to(tl.int64) - d_offsets = tl.arange(0, HEAD_DIM) - - for rank_idx in tl.static_range(N): - src_head_idx = rank_idx * H_PER_RANK + local_head_idx - send_base = ( - rank_idx * send_stride_N - + batch_idx * send_stride_B - + local_head_idx * send_stride_H - ) - - out_offsets = ( - batch_idx * out_stride_B - + src_head_idx * out_stride_H - + d_offsets * out_stride_D - ) - tl.store( - send_ptr + send_base + d_offsets * send_stride_D, - tl.load(out_ptr + out_offsets), - ) - - lse_val = tl.load( - lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H - ).to(tl.float32) - if LSE_PACK_DIM == 1: - tl.store( - send_ptr + send_base + HEAD_DIM * send_stride_D, - lse_val.to(send_ptr.dtype.element_ty), - ) - else: - lse_bits = lse_val.to(tl.uint32, bitcast=True) - lo = (lse_bits & 0xFFFF).to(tl.uint16) - hi = ((lse_bits >> 16) & 0xFFFF).to(tl.uint16) - tl.store( - send_ptr + send_base + HEAD_DIM * send_stride_D, - lo.to(send_ptr.dtype.element_ty, bitcast=True), - ) - tl.store( - send_ptr + send_base + (HEAD_DIM + 1) * send_stride_D, - hi.to(send_ptr.dtype.element_ty, bitcast=True), - ) - - -@triton.jit -def _dcp_a2a_unpack_combine_kernel( - recv_ptr, - out_ptr, - out_lse_ptr, - recv_stride_N, - recv_stride_B, - recv_stride_H, - recv_stride_D, - out_stride_B, - out_stride_H, - out_stride_D, - out_lse_stride_B, - out_lse_stride_H, - N: tl.constexpr, - HEAD_DIM: tl.constexpr, - IS_BASE_E: tl.constexpr, - RETURN_LSE: tl.constexpr, - LSE_PACK_DIM: tl.constexpr, -): - batch_idx = tl.program_id(0).to(tl.int64) - head_idx = tl.program_id(1).to(tl.int64) - d_offsets = tl.arange(0, HEAD_DIM) - - lse_max = -float("inf") - for rank_idx in tl.static_range(N): - recv_base = ( - rank_idx * recv_stride_N - + batch_idx * recv_stride_B - + head_idx * recv_stride_H - ) - if LSE_PACK_DIM == 1: - lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( - tl.float32 - ) - else: - lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) - hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) - lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) - hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) - lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) - lse_val = tl.where( - (lse_val != lse_val) | (lse_val == float("inf")), - -float("inf"), - lse_val, - ) - lse_max = tl.maximum(lse_max, lse_val) - - lse_max = tl.where(lse_max == -float("inf"), 0.0, lse_max) - - lse_sum = 0.0 - for rank_idx in tl.static_range(N): - recv_base = ( - rank_idx * recv_stride_N - + batch_idx * recv_stride_B - + head_idx * recv_stride_H - ) - if LSE_PACK_DIM == 1: - lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( - tl.float32 - ) - else: - lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) - hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) - lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) - hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) - lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) - lse_val = tl.where( - (lse_val != lse_val) | (lse_val == float("inf")), - -float("inf"), - lse_val, - ) - if IS_BASE_E: - lse_sum += tl.exp(lse_val - lse_max) - else: - lse_sum += tl.exp2(lse_val - lse_max) - - if IS_BASE_E: # noqa: SIM108 - global_lse = tl.log(lse_sum) + lse_max - else: - global_lse = tl.log2(lse_sum) + lse_max - - acc = tl.zeros([HEAD_DIM], dtype=tl.float32) - for rank_idx in tl.static_range(N): - recv_base = ( - rank_idx * recv_stride_N - + batch_idx * recv_stride_B - + head_idx * recv_stride_H - ) - if LSE_PACK_DIM == 1: - lse_val = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D).to( - tl.float32 - ) - else: - lo_raw = tl.load(recv_ptr + recv_base + HEAD_DIM * recv_stride_D) - hi_raw = tl.load(recv_ptr + recv_base + (HEAD_DIM + 1) * recv_stride_D) - lo = lo_raw.to(tl.uint16, bitcast=True).to(tl.uint32) - hi = hi_raw.to(tl.uint16, bitcast=True).to(tl.uint32) - lse_val = (lo | (hi << 16)).to(tl.float32, bitcast=True) - lse_val = tl.where( - (lse_val != lse_val) | (lse_val == float("inf")), - -float("inf"), - lse_val, - ) - if IS_BASE_E: - weight = tl.exp(lse_val - global_lse) - else: - weight = tl.exp2(lse_val - global_lse) - weight = tl.where(weight != weight, 0.0, weight) - partial = tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to( - tl.float32 - ) - partial = tl.where(weight == 0.0, 0.0, partial) - acc += partial * weight - - final_offsets = ( - batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D - ) - tl.store(out_ptr + final_offsets, acc) - - if RETURN_LSE: - out_lse_offset = batch_idx * out_lse_stride_B + head_idx * out_lse_stride_H - tl.store(out_lse_ptr + out_lse_offset, global_lse) - - -def _dcp_a2a_pack_send( - cp_attn_out: torch.Tensor, - cp_attn_lse: torch.Tensor, - send_buffer: torch.Tensor, - world_size: int, - h_per_rank: int, - head_dim: int, - lse_pack_dim: int, - seq_lens: torch.Tensor | None = None, - query_start_loc: torch.Tensor | None = None, -) -> None: - mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc) - grid = (cp_attn_out.shape[0], h_per_rank, 1) - _dcp_a2a_pack_send_kernel[grid]( - cp_attn_out, - cp_attn_lse, - send_buffer, - cp_attn_out.stride(0), - cp_attn_out.stride(1), - cp_attn_out.stride(2), - cp_attn_lse.stride(0), - cp_attn_lse.stride(1), - send_buffer.stride(0), - send_buffer.stride(1), - send_buffer.stride(2), - send_buffer.stride(3), - N=world_size, - HEAD_DIM=head_dim, - H_PER_RANK=h_per_rank, - LSE_PACK_DIM=lse_pack_dim, - ) - - -def _dcp_a2a_unpack_combine( - recv_buffer: torch.Tensor, - head_dim: int, - lse_pack_dim: int, - return_lse: bool, - is_lse_base_on_e: bool, -) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - world_size, num_tokens, h_per_rank, _ = recv_buffer.shape - out = torch.empty( - (num_tokens, h_per_rank, head_dim), - device=recv_buffer.device, - dtype=recv_buffer.dtype, - ) - out_lse = torch.empty( - (num_tokens, h_per_rank) if return_lse else (1, 1), - device=recv_buffer.device, - dtype=torch.float32 if return_lse else recv_buffer.dtype, - ) - grid = (num_tokens, h_per_rank, 1) - _dcp_a2a_unpack_combine_kernel[grid]( - recv_buffer, - out, - out_lse, - recv_buffer.stride(0), - recv_buffer.stride(1), - recv_buffer.stride(2), - recv_buffer.stride(3), - out.stride(0), - out.stride(1), - out.stride(2), - out_lse.stride(0), - out_lse.stride(1), - N=world_size, - HEAD_DIM=head_dim, - IS_BASE_E=is_lse_base_on_e, - RETURN_LSE=return_lse, - LSE_PACK_DIM=lse_pack_dim, - ) - if return_lse: - return out, out_lse - return out - - -def dcp_a2a_lse_reduce( - cp_attn_out: torch.Tensor, - cp_attn_lse: torch.Tensor, - cp_group: GroupCoordinator, - ctx: CPTritonContext | None = None, - return_lse: bool = False, - is_lse_base_on_e: bool = True, - seq_lens: torch.Tensor | None = None, - query_start_loc: torch.Tensor | None = None, -) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """ - Combine partial attention outputs across DCP ranks using All-to-All. - - The output and LSE are packed into a single output-dtype buffer, sent - with one All-to-All, then unpacked and combined with exact LSE weighting. - - Args: - cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim - cp_attn_lse: [B, H] floating-point log-sum-exp values - cp_group: GroupCoordinator for DCP communication - ctx: CPTritonContext (unused, for signature compatibility) - return_lse: If True, also return the combined global LSE - is_lse_base_on_e: If True, LSE is base e; if False, base 2 - seq_lens: Local KV lengths. Empty shards contribute zero weight. - query_start_loc: Cumulative query-token offsets for each request. - - Returns: - Combined output [B, H/N, D] (head-scattered) - If return_lse=True, also returns global_lse [B, H/N] - """ - world_size = cp_group.world_size - - if world_size == 1: - if return_lse: - return cp_attn_out, cp_attn_lse - return cp_attn_out - - B, H, D = cp_attn_out.shape - if H % world_size != 0: - raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.") - H_per_rank = H // world_size - lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype) - - send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers( - (world_size, B, H_per_rank, D + lse_pack_dim), - device=cp_attn_out.device, - dtype=cp_attn_out.dtype, - ) - - _dcp_a2a_pack_send( - cp_attn_out, - cp_attn_lse, - send_buffer, - world_size, - H_per_rank, - D, - lse_pack_dim, - seq_lens=seq_lens, - query_start_loc=query_start_loc, - ) - - work = dist.all_to_all_single( - recv_buffer.view(-1), - send_buffer.view(-1), - group=cp_group.device_group, - async_op=True, - ) - work.wait() - - return _dcp_a2a_unpack_combine( - recv_buffer, D, lse_pack_dim, return_lse, is_lse_base_on_e - ) diff --git a/vllm/v1/attention/ops/dcp_utils.py b/vllm/v1/attention/ops/dcp_utils.py deleted file mode 100644 index b28d337a10..0000000000 --- a/vllm/v1/attention/ops/dcp_utils.py +++ /dev/null @@ -1,740 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""MLA DCP collective selection and direct symmetric-memory implementations.""" - -from __future__ import annotations - -import functools -from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Protocol - -import torch - -import vllm.envs as envs -from vllm.config import VllmConfig -from vllm.distributed import get_dcp_group -from vllm.distributed.parallel_state import in_the_same_node_as -from vllm.logger import init_logger -from vllm.platforms import current_platform -from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs -from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce -from vllm.v1.worker.ubatching import dbo_current_ubatch_id - -logger = init_logger(__name__) - -if TYPE_CHECKING: - from torch.distributed import ProcessGroup - - from vllm.distributed.parallel_state import GroupCoordinator - -try: - import torch.distributed._symmetric_memory as symm_mem - - symm_mem_available = True -except ImportError: - symm_mem = None # type: ignore[assignment] - symm_mem_available = False - - -@functools.cache -def _symm_mem_spans_group(group: GroupCoordinator) -> bool: - """Probe whether the group has NVLS symmetric memory.""" - if not symm_mem_available: - return False - try: - from torch._C._autograd import DeviceType - from torch._C._distributed_c10d import _SymmetricMemory - - device = torch.device("cuda", torch.accelerator.current_device_index()) - if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index): - return False - probe = symm_mem.empty(8, dtype=torch.uint8, device=device) - probe.zero_() - torch.accelerator.synchronize() - handle = symm_mem.rendezvous(probe, group.device_group.group_name) - spans = handle is not None and handle.multicast_ptr != 0 - except Exception as error: - logger.debug("Direct DCP symmetric-memory probe failed: %s", error) - return False - logger.debug_once( - "Direct DCP symmetric memory across %d ranks: %s", - group.world_size, - "available" if spans else "unavailable", - ) - return spans - - -def _direct_dcp_enabled( - group: GroupCoordinator, - dtype: torch.dtype, - use_direct: bool | None, - supported_dtypes: tuple[torch.dtype, ...] | None = None, -) -> bool: - if use_direct is not None: - return use_direct - return ( - symm_mem_available - and current_platform.is_cuda() - and (supported_dtypes is None or dtype in supported_dtypes) - and ( - all(in_the_same_node_as(group.cpu_group, source_rank=0)) - or _symm_mem_spans_group(group) - ) - ) - - -def _direct_dcp_multicast_enabled( - group: GroupCoordinator, - dtype: torch.dtype, - use_direct: bool | None, - supported_dtypes: tuple[torch.dtype, ...] | None = None, -) -> bool: - return _direct_dcp_enabled( - group, dtype, use_direct, supported_dtypes - ) and _symm_mem_spans_group(group) - - -def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int: - scheduler_config = vllm_config.scheduler_config - speculative_config = vllm_config.speculative_config - speculative_tokens = vllm_config.num_speculative_tokens - tokens_per_seq = ( - 1 - + ( - 2 - if speculative_config is not None and speculative_config.parallel_drafting - else 1 - ) - * speculative_tokens - ) - return min( - scheduler_config.max_num_batched_tokens, - max( - scheduler_config.max_num_seqs * tokens_per_seq, - vllm_config.compilation_config.max_cudagraph_capture_size or 0, - ), - ) - - -class _DirectDCPWorkspace: - def __init__( - self, - group: ProcessGroup, - device: torch.device, - num_ubatches: int, - ) -> None: - self.group = group - self.world_size = group.size() - self.rank = group.rank() - self.device = torch.device(device) - self.num_ubatches = num_ubatches - self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device) - self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = [] - - def _allocate( - self, shape: tuple[int, ...], dtype: torch.dtype - ) -> tuple[torch.Tensor, torch.Tensor]: - storage = symm_mem.empty(shape, device=self.device, dtype=dtype) - storage.zero_() - torch.accelerator.synchronize() - handle = symm_mem.rendezvous(storage, self.group.group_name) - assert handle is not None, "DCP symmetric memory rendezvous returned None" - handle.barrier() - views = [ - handle.get_buffer(peer, list(shape), dtype, 0) - for peer in range(self.world_size) - ] - self.device = storage.device - peer_ptrs = torch.tensor( - [ - [view[ubatch].data_ptr() for view in views] - for ubatch in range(self.num_ubatches) - ], - dtype=torch.int64, - device=self.device, - ) - self._allocations.append((storage, handle, views)) - return storage, peer_ptrs - - def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]: - disabled = [0] * self.num_ubatches - for allocated, handle, _ in self._allocations: - if allocated is storage: - break - else: - return disabled - try: - from torch._C._autograd import DeviceType - from torch._C._distributed_c10d import _SymmetricMemory - - if not _SymmetricMemory.has_multicast_support( - DeviceType.CUDA, storage.device.index - ): - return disabled - multicast_base = handle.multicast_ptr - except Exception: - return disabled - if not multicast_base: - return disabled - storage_base = storage.data_ptr() - return [ - multicast_base + (storage[ubatch].data_ptr() - storage_base) - for ubatch in range(self.num_ubatches) - ] - - -def reserve_query_head_storage( - query: torch.Tensor, padded_num_heads: int -) -> torch.Tensor: - """Reserve backing storage for fixed-head decode kernels.""" - assert query.ndim == 3 - assert query.shape[1] <= padded_num_heads - padded = query.new_empty((query.shape[0], padded_num_heads, query.shape[2])) - padded.resize_(query.shape) - padded.copy_(query) - return padded - - -_A2A_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16) - - -class DirectDCPA2AWorkspace(_DirectDCPWorkspace): - """Persistent symmetric buffers for direct DCP output exchange.""" - - def __init__( - self, - group: ProcessGroup, - device: torch.device, - max_num_tokens: int, - heads_per_rank: int, - head_dim: int, - dtype: torch.dtype = torch.bfloat16, - num_ubatches: int = 1, - ) -> None: - if dtype not in _A2A_SUPPORTED_DTYPES: - raise ValueError(f"Direct DCP A2A does not support {dtype}") - if num_ubatches < 1: - raise ValueError( - f"Direct DCP A2A requires at least one ubatch slot, got {num_ubatches}" - ) - super().__init__(group, device, num_ubatches) - self.max_num_tokens = max_num_tokens - self.heads_per_rank = heads_per_rank - self.head_dim = head_dim - - output_shape = ( - num_ubatches, - 2, - self.world_size, - max_num_tokens, - heads_per_rank, - head_dim, - ) - lse_shape = ( - num_ubatches, - 2, - self.world_size, - max_num_tokens, - heads_per_rank, - ) - signal_shape = (num_ubatches, 2, self.world_size) - self.received_output, self.peer_output_ptrs = self._allocate( - output_shape, dtype - ) - self.received_lse, self.peer_lse_ptrs = self._allocate(lse_shape, torch.float32) - self.received_signal, self.peer_signal_ptrs = self._allocate( - signal_shape, torch.int32 - ) - - def lse_reduce( - self, - partial_output: torch.Tensor, - partial_lse: torch.Tensor, - is_lse_base_on_e: bool, - seq_lens: torch.Tensor | None = None, - query_start_loc: torch.Tensor | None = None, - ) -> torch.Tensor: - ubatch = dbo_current_ubatch_id() - num_tokens = partial_output.shape[0] - output = partial_output.new_empty( - (num_tokens, self.heads_per_rank, self.head_dim) - ) - torch.ops._C.direct_dcp_a2a_lse_reduce( - partial_output, - partial_lse, - seq_lens, - query_start_loc, - self.peer_output_ptrs[ubatch], - self.peer_lse_ptrs[ubatch], - self.peer_signal_ptrs[ubatch], - self.received_output[ubatch], - self.received_lse[ubatch], - self.received_signal[ubatch], - self.epoch[ubatch : ubatch + 1], - output, - self.world_size, - self.rank, - self.max_num_tokens, - is_lse_base_on_e, - ) - return output - - -@functools.cache -def get_direct_dcp_a2a_workspace( - group: GroupCoordinator, - device: torch.device, - max_num_tokens: int, - heads_per_rank: int, - head_dim: int, - dtype: torch.dtype, - num_ubatches: int, -) -> DirectDCPA2AWorkspace | None: - if not _direct_dcp_enabled( - group, dtype, envs.VLLM_USE_DIRECT_DCP_A2A, _A2A_SUPPORTED_DTYPES - ): - return None - return DirectDCPA2AWorkspace( - group.device_group, - device, - max_num_tokens, - heads_per_rank, - head_dim, - dtype, - num_ubatches, - ) - - -def _q_gather_layout_supported( - world_size: int, - heads_per_rank: int, - head_dim: int, - dtype: torch.dtype, - padded_num_heads: int | None, -) -> bool: - element_size = torch.empty((), dtype=dtype).element_size() - gathered_num_heads = world_size * heads_per_rank - storage_num_heads = ( - gathered_num_heads if padded_num_heads is None else padded_num_heads - ) - return ( - heads_per_rank * head_dim * element_size % 16 == 0 - and storage_num_heads * head_dim * element_size % 16 == 0 - ) - - -class DirectDCPQGatherWorkspace(_DirectDCPWorkspace): - """Publish query shards directly into the consumer-final symmetric buffer. - - The final buffer is reusable after the downstream DCP output combine. That - combine orders all ranks after attention has consumed the gathered query. - """ - - def __init__( - self, - group: ProcessGroup, - device: torch.device, - max_num_tokens: int, - heads_per_rank: int, - head_dim: int, - dtype: torch.dtype = torch.bfloat16, - num_ubatches: int = 1, - padded_num_heads: int | None = None, - ) -> None: - if num_ubatches < 1: - raise ValueError( - "Direct DCP q-gather requires at least one ubatch slot, " - f"got {num_ubatches}" - ) - if max_num_tokens < 1 or heads_per_rank < 1 or head_dim < 1: - raise ValueError( - "Direct DCP q-gather dimensions must be positive, got " - f"T={max_num_tokens}, H={heads_per_rank}, D={head_dim}" - ) - gathered_num_heads = group.size() * heads_per_rank - if not _q_gather_layout_supported( - group.size(), heads_per_rank, head_dim, dtype, padded_num_heads - ): - raise ValueError("Direct DCP q-gather requires 16-byte-aligned query rows.") - super().__init__(group, device, num_ubatches) - if self.world_size <= 1: - raise ValueError("Direct DCP q-gather requires at least two ranks") - self.max_num_tokens = max_num_tokens - self.heads_per_rank = heads_per_rank - self.gathered_num_heads = gathered_num_heads - self.padded_num_heads = ( - self.gathered_num_heads if padded_num_heads is None else padded_num_heads - ) - if self.padded_num_heads < self.gathered_num_heads: - raise ValueError( - "Direct DCP q-gather padded heads must cover gathered heads: " - f"{self.padded_num_heads} < {self.gathered_num_heads}" - ) - self.head_dim = head_dim - - query_shape = ( - num_ubatches, - max_num_tokens, - self.padded_num_heads, - head_dim, - ) - signal_shape = (num_ubatches, 2, self.world_size) - self.final_query, _ = self._allocate(query_shape, dtype) - self.received_signal, _ = self._allocate(signal_shape, torch.int32) - query_multicast_ptrs = self._multicast_ptrs(self.final_query) - signal_multicast_ptrs = self._multicast_ptrs(self.received_signal) - self.multicast_ptrs = list( - zip(query_multicast_ptrs, signal_multicast_ptrs, strict=True) - ) - if not all( - query_ptr and signal_ptr for query_ptr, signal_ptr in self.multicast_ptrs - ): - raise RuntimeError( - "Direct DCP q-gather requires NVLS symmetric-memory multicast." - ) - self.completion = self.received_signal.new_zeros((num_ubatches, 1)) - torch.accelerator.synchronize() - - def gather(self, local_query: torch.Tensor) -> torch.Tensor: - ubatch = dbo_current_ubatch_id() - if not 0 <= ubatch < self.num_ubatches: - raise ValueError( - f"DCP q-gather ubatch {ubatch} exceeds {self.num_ubatches} slots" - ) - if local_query.ndim == 3 and local_query.shape[1] != self.heads_per_rank: - raise ValueError( - f"DCP q-gather expected {self.heads_per_rank} local query heads, " - f"got {local_query.shape[1]}" - ) - - num_tokens = local_query.shape[0] - output = torch.as_strided( - self.final_query[ubatch], - size=(num_tokens, self.gathered_num_heads, self.head_dim), - stride=( - self.gathered_num_heads * self.head_dim, - self.head_dim, - 1, - ), - ) - query_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch] - torch.ops._C.direct_dcp_q_gather( - local_query, - output, - self.received_signal[ubatch], - self.completion[ubatch], - self.epoch[ubatch : ubatch + 1], - self.world_size, - self.rank, - self.max_num_tokens, - self.padded_num_heads, - query_multicast_ptr, - signal_multicast_ptr, - ) - return output - - -@functools.cache -def get_direct_dcp_q_gather_workspace( - group: GroupCoordinator, - device: torch.device, - max_num_tokens: int, - heads_per_rank: int, - head_dim: int, - dtype: torch.dtype, - num_ubatches: int, - padded_num_heads: int | None = None, -) -> DirectDCPQGatherWorkspace | None: - if not _direct_dcp_multicast_enabled( - group, dtype, envs.VLLM_USE_DIRECT_DCP_Q_GATHER - ): - return None - if not _q_gather_layout_supported( - group.world_size, heads_per_rank, head_dim, dtype, padded_num_heads - ): - return None - return DirectDCPQGatherWorkspace( - group.device_group, - device, - max_num_tokens, - heads_per_rank, - head_dim, - dtype, - num_ubatches, - padded_num_heads, - ) - - -_KV_GATHER_SUPPORTED_DTYPES = ( - torch.float16, - torch.bfloat16, - torch.float8_e4m3fn, -) - - -def _kv_gather_layout_supported(token_dim: int, dtype: torch.dtype) -> bool: - return token_dim * torch.empty((), dtype=dtype).element_size() % 16 == 0 - - -class DirectDCPKVGatherWorkspace(_DirectDCPWorkspace): - """Persistent symmetric buffers for direct DCP KV gather.""" - - def __init__( - self, - group: ProcessGroup, - device: torch.device, - max_gathered_tokens: int, - token_dim: int, - dtype: torch.dtype = torch.bfloat16, - num_ubatches: int = 1, - ) -> None: - if dtype not in _KV_GATHER_SUPPORTED_DTYPES: - raise ValueError(f"Direct DCP kv-gather does not support {dtype}") - if num_ubatches < 1: - raise ValueError( - "Direct DCP kv-gather requires at least one ubatch slot, " - f"got {num_ubatches}" - ) - if max_gathered_tokens < 1 or token_dim < 1: - raise ValueError( - "Direct DCP kv-gather dimensions must be positive, got " - f"T={max_gathered_tokens}, D={token_dim}" - ) - if not _kv_gather_layout_supported(token_dim, dtype): - raise ValueError("Direct DCP kv-gather requires 16-byte-aligned KV rows.") - super().__init__(group, device, num_ubatches) - if self.world_size <= 1: - raise ValueError("Direct DCP kv-gather requires at least two ranks") - if max_gathered_tokens % self.world_size != 0: - raise ValueError( - "Direct DCP kv-gather capacity must divide evenly across " - f"ranks: {max_gathered_tokens} % {self.world_size} != 0" - ) - self.max_gathered_tokens = max_gathered_tokens - - kv_shape = (num_ubatches, 2, max_gathered_tokens, token_dim) - signal_shape = (num_ubatches, 2, self.world_size) - self.received_kv, _ = self._allocate(kv_shape, dtype) - self.received_signal, _ = self._allocate(signal_shape, torch.int32) - kv_multicast_ptrs = self._multicast_ptrs(self.received_kv) - signal_multicast_ptrs = self._multicast_ptrs(self.received_signal) - self.multicast_ptrs = list( - zip(kv_multicast_ptrs, signal_multicast_ptrs, strict=True) - ) - if not all(kv_ptr and signal_ptr for kv_ptr, signal_ptr in self.multicast_ptrs): - raise RuntimeError( - "Direct DCP kv-gather requires NVLS symmetric-memory multicast." - ) - self.completion = self.received_signal.new_zeros((num_ubatches, 2)) - torch.accelerator.synchronize() - - def gather(self, gathered_kv: torch.Tensor, local_kv: torch.Tensor) -> None: - ubatch = dbo_current_ubatch_id() - if not 0 <= ubatch < self.num_ubatches: - raise ValueError( - f"DCP kv-gather ubatch {ubatch} exceeds {self.num_ubatches} slots" - ) - kv_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch] - torch.ops._C.direct_dcp_kv_gather( - local_kv, - self.received_kv[ubatch], - self.received_signal[ubatch], - self.completion[ubatch], - self.epoch[ubatch : ubatch + 1], - gathered_kv, - self.world_size, - self.rank, - self.max_gathered_tokens, - kv_multicast_ptr, - signal_multicast_ptr, - ) - - -@functools.cache -def get_direct_dcp_kv_gather_workspace( - group: GroupCoordinator, - device: torch.device, - max_gathered_tokens: int, - token_dim: int, - dtype: torch.dtype, - num_ubatches: int, -) -> DirectDCPKVGatherWorkspace | None: - if not _direct_dcp_multicast_enabled( - group, - dtype, - envs.VLLM_USE_DIRECT_DCP_KV_GATHER, - _KV_GATHER_SUPPORTED_DTYPES, - ): - return None - if not _kv_gather_layout_supported(token_dim, dtype): - return None - return DirectDCPKVGatherWorkspace( - group.device_group, - device, - max_gathered_tokens, - token_dim, - dtype, - num_ubatches, - ) - - -class DCPCombine(Protocol): - def __call__( - self, - partial_output: torch.Tensor, - partial_lse: torch.Tensor, - *, - seq_lens: torch.Tensor, - query_start_loc: torch.Tensor, - ) -> torch.Tensor: ... - - -class MLADCPManager: - """Select and own layer-level collective implementations for MLA DCP.""" - - _kv_gather: Callable[[torch.Tensor, torch.Tensor], object] - - def __init__( - self, - vllm_config: VllmConfig, - device: torch.device, - num_heads: int, - query_head_dim: int, - output_head_dim: int, - query_dtype: torch.dtype, - output_dtype: torch.dtype, - padded_num_heads: int | None, - is_lse_base_on_e: bool, - use_pcp: bool, - ) -> None: - parallel_config = vllm_config.parallel_config - self.group = get_dcp_group() - self.device = torch.device(device) - self.num_ubatches = max(parallel_config.num_ubatches, 1) - self.max_num_tokens = get_dcp_workspace_max_num_tokens(vllm_config) - self.use_a2a = parallel_config.dcp_comm_backend == "a2a" - self.padded_num_heads = padded_num_heads - - self.combine = self._init_combine( - num_heads, - output_head_dim, - output_dtype, - is_lse_base_on_e, - use_pcp, - ) - self.query_gather = ( - None - if use_pcp - else self._init_query_gather( - num_heads, - query_head_dim, - query_dtype, - ) - ) - - def _init_combine( - self, - num_heads: int, - head_dim: int, - dtype: torch.dtype, - is_lse_base_on_e: bool, - use_pcp: bool, - ) -> DCPCombine: - direct_workspace = None - if self.use_a2a: - direct_workspace = get_direct_dcp_a2a_workspace( - self.group, - self.device, - self.max_num_tokens, - num_heads, - head_dim, - dtype, - self.num_ubatches, - ) - if direct_workspace is not None: - logger.info_once("Using direct symmetric-memory DCP A2A for MLA.") - return functools.partial( - direct_workspace.lse_reduce, - is_lse_base_on_e=is_lse_base_on_e, - ) - - combine_fn = ( - dcp_a2a_lse_reduce - if self.use_a2a - else cp_lse_ag_out_ar - if use_pcp - else cp_lse_ag_out_rs - ) - return functools.partial( - combine_fn, - cp_group=self.group, - is_lse_base_on_e=is_lse_base_on_e, - ) - - def _init_query_gather( - self, - num_heads: int, - head_dim: int, - dtype: torch.dtype, - ) -> Callable[[torch.Tensor], torch.Tensor]: - direct_workspace = get_direct_dcp_q_gather_workspace( - self.group, - self.device, - self.max_num_tokens, - num_heads, - head_dim, - dtype, - self.num_ubatches, - self.padded_num_heads, - ) - if direct_workspace is not None: - logger.info_once("Using direct symmetric-memory DCP query gather for MLA.") - return direct_workspace.gather - return self._gather_query - - def _gather_query(self, query: torch.Tensor) -> torch.Tensor: - query = self.group.all_gather(query, dim=1) - if self.padded_num_heads is not None: - query = reserve_query_head_storage(query, self.padded_num_heads) - return query - - def init_kv_gather( - self, - workspace: torch.Tensor, - max_gathered_tokens: int, - ) -> None: - world_size = self.group.world_size - assert max_gathered_tokens > 0 - assert max_gathered_tokens % world_size == 0 - assert workspace.ndim == 2 - assert workspace.is_contiguous() - assert workspace.shape[0] == ( - max_gathered_tokens + max_gathered_tokens // world_size - ) - assert workspace.shape[1] > 0 - - direct_workspace = get_direct_dcp_kv_gather_workspace( - self.group, - workspace.device, - max_gathered_tokens, - workspace.shape[1], - workspace.dtype, - self.num_ubatches, - ) - if direct_workspace is not None: - logger.info_once( - "Using direct symmetric-memory DCP chunked-context KV gather for MLA." - ) - self._kv_gather = direct_workspace.gather - else: - self._kv_gather = functools.partial( - torch.distributed.all_gather_into_tensor, - group=self.group.device_group, - ) - - def kv_gather( - self, - gathered_kv: torch.Tensor, - local_kv: torch.Tensor, - ) -> object: - return self._kv_gather(gathered_kv, local_kv) diff --git a/vllm/model_executor/layers/attention/pcp.py b/vllm/v1/attention/ops/pcp.py similarity index 100% rename from vllm/model_executor/layers/attention/pcp.py rename to vllm/v1/attention/ops/pcp.py