[ROCm][MLA] Keep DCP prefill context in FP8 through AllGather (#59070)

Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Code <noreply@anthropic.com>
Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
This commit is contained in:
Hexiang Wang
2026-10-13 09:20:38 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Code Fangzhou Ai mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Simon Danielsson
parent 98f7211162
commit c6566bb9f6
9 changed files with 847 additions and 17 deletions
+3
View File
@@ -3539,8 +3539,11 @@ steps:
- examples/rl/rlhf_async_new_apis.py
- tests/distributed/test_weight_transfer.py
- tests/distributed/test_packed_tensor.py
- tests/v1/attention/test_rocm_mla_dcp_context.py
- vllm/v1/attention/ops/rocm_aiter_mla_prefill.py
commands:
- pytest -v -s tests/distributed/test_context_parallel.py
- pytest -v -s tests/v1/attention/test_rocm_mla_dcp_context.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
+3
View File
@@ -528,8 +528,11 @@ steps:
- examples/rl/rlhf_async_new_apis.py
- tests/distributed/test_weight_transfer.py
- tests/distributed/test_packed_tensor.py
- tests/v1/attention/test_rocm_mla_dcp_context.py
- vllm/v1/attention/ops/rocm_aiter_mla_prefill.py
commands:
- pytest -v -s tests/distributed/test_context_parallel.py
- pytest -v -s tests/v1/attention/test_rocm_mla_dcp_context.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
@@ -33,6 +33,26 @@ def _require_aiter():
pytest.skip("aiter is required on supported ROCm hardware for this test")
@torch.inference_mode()
def test_gather_kv_b_proj_op_schema() -> None:
"""Validate fake tensors and mutation of both expanded K/V outputs."""
_require_aiter()
from vllm._aiter_ops import rocm_aiter_ops # noqa: F401
rows, num_heads = 32, 12
cache = torch.randn(rows, 1, 576, device="cuda").to(current_platform.fp8_dtype())
scale = torch.tensor([0.07], device="cuda")
cu_seq_lens = torch.tensor([0, 13, rows], dtype=torch.int32, device="cuda")
indices = torch.arange(rows, dtype=torch.int32, device="cuda").flip(0)
weight = torch.randn(num_heads * 256, 512, dtype=torch.bfloat16, device="cuda")
k = torch.empty(rows, num_heads, 192, dtype=torch.bfloat16, device="cuda")
v = torch.empty(rows, num_heads, 128, dtype=torch.bfloat16, device="cuda")
opcheck(
torch.ops.vllm.rocm_aiter_gather_kv_b_proj,
(cache, scale, cu_seq_lens, indices, cu_seq_lens, weight, k, v),
)
@torch.inference_mode()
def test_mla_decode_fwd_op_schema() -> None:
"""Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
@@ -142,6 +142,13 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
def init_common_builder(self, *args, **kwargs):
self.num_heads = gathered_heads // DCP
self.q_data_type = torch.bfloat16
self.mla_dims = SimpleNamespace(
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
)
self.dcp_world_size = DCP
self.reorder_batch_threshold = 5
@@ -740,26 +740,34 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
torch.testing.assert_close(output, reference, rtol=3e-2, atol=3e-2)
@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128])
@pytest.mark.parametrize(
"spec_method, parallel_drafting",
"num_heads,spec_method,parallel_drafting,group_cache_dtype,needs_row_map",
[
("deepseek_mtp", False),
# A drafter that is not one of the historically recognized MTP methods,
# and a parallel one, so the threshold is 1 + 2 * num_spec rather than
# 1 + num_spec. Sizing the metadata off a method name instead leaves
# these at qlen=1 while the router still admits the full range.
("custom", True),
("eagle", False),
*[
(num_heads, method, parallel, None, True)
for num_heads in [8, 16, 24, 32, 64, 128]
for method, parallel in [
("deepseek_mtp", False),
# Parallel drafting needs qlen=1+2*num_spec, regardless of name.
("custom", True),
("eagle", False),
]
],
pytest.param(16, "deepseek_mtp", False, "auto", False, id="bf16-group"),
pytest.param(16, "deepseek_mtp", False, "fp8_e5m2", False, id="e5m2-group"),
],
)
def test_mtp_builder_init_sizes_native_fp8_metadata(
monkeypatch, num_heads, spec_method, parallel_drafting
def test_builder_init_sizes_decode_metadata_and_filters_dcp_maps(
monkeypatch,
num_heads,
spec_method,
parallel_drafting,
group_cache_dtype,
needs_row_map,
):
"""Aiter init sizes the metadata for every query length decode can be handed.
"""Size decode metadata for padded heads/qlen; skip unsupported DCP maps.
Sweeping num_heads asserts metadata is sized for the padded decode shape,
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
The group's raw cache format must override the global FP8 configuration.
"""
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
info_calls = []
@@ -789,6 +797,13 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
def init_common_builder(self, *args, **kwargs):
self.num_heads = num_heads
self.q_data_type = config.model_config.dtype
self.mla_dims = SimpleNamespace(
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
)
self.dcp_world_size = 1
self.cp_kv_cache_interleave_size = 1
# Mirror what _init_reorder_batch_threshold would have left behind: the
@@ -813,6 +828,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
init_common_builder,
)
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
monkeypatch.setattr(rocm_aiter_mla, "is_aiter_found_and_supported", lambda: True)
config = SimpleNamespace(
speculative_config=SimpleNamespace(
@@ -840,7 +856,9 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
),
)
builder = AiterMLAMetadataBuilder(
kv_cache_spec=SimpleNamespace(block_size=1, dtype=torch.bfloat16),
kv_cache_spec=SimpleNamespace(
block_size=1, dtype=torch.bfloat16, cache_dtype_str=group_cache_dtype
),
layer_names=["layer.0"],
vllm_config=config,
device=torch.device("cpu"),
@@ -860,6 +878,35 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
assert builder._mla_q_dtype == dtypes.fp8
assert builder._mla_kv_dtype == dtypes.fp8
builder.dcp_world_size = 2
chunk = object()
rows = torch.tensor([0], dtype=torch.int64)
build_rows = mock.Mock(return_value=rows)
monkeypatch.setattr(rocm_aiter_mla, "context_row_indices", build_rows)
monkeypatch.setattr(
rocm_aiter_mla,
"_supports_fused_dcp_prefill_config",
mock.Mock(side_effect=AssertionError("Support must be cached at init")),
)
monkeypatch.setattr(
rocm_aiter_mla.MLACommonMetadataBuilder,
"build",
lambda *args: SimpleNamespace(
prefill=SimpleNamespace(
chunked_context=SimpleNamespace(chunks=[chunk]),
block_table=torch.empty(0),
),
decode=None,
dcp_context_row_indices=None,
),
)
for _ in range(2):
metadata = builder.build(0, SimpleNamespace(causal=True))
assert metadata.dcp_context_row_indices == ([rows] if needs_row_map else None)
assert build_rows.call_args_list == (
[mock.call(chunk, torch.device("cpu"))] * 2 if needs_row_map else []
)
def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
get_mla_metadata_v1 = mock.MagicMock()
@@ -0,0 +1,445 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from tests.utils import multi_gpu_only
from vllm.platforms import current_platform
from vllm.v1.attention.ops.rocm_aiter_mla_prefill import context_row_indices
@pytest.mark.parametrize(
"padded,lengths,starts,total,expected",
[
([3, 2], [[7, 6], [2, 1]], [5, 0], 6, [0, 1, 5, 3, 4, 8]),
([3], [[2, 9]], [4], 3, [3, 4, 5]),
([3], [[1, 1]], [0], 3, None),
],
ids=["padding_and_chunk_starts", "empty_rank", "inconsistent_total"],
)
def test_dcp_context_indices(padded, lengths, starts, total, expected):
chunk = SimpleNamespace(
padded_local_seq_lens=padded,
local_context_lens_allranks=lengths,
local_starts=starts,
num_local_context_tokens=sum(padded),
num_context_tokens=total,
)
if expected is None:
with pytest.raises(AssertionError):
context_row_indices(chunk, torch.device("cpu"))
else:
rows = context_row_indices(chunk, torch.device("cpu"))
assert rows.tolist() == expected
assert rows.dtype == torch.int32
def test_compressed_gather_preserves_bytes_and_workspace_partition(monkeypatch):
from vllm.v1.attention.ops import rocm_aiter_mla_prefill as prefill
workspace = torch.full((12, 4), -1, dtype=torch.bfloat16)
cache = torch.arange(8, dtype=torch.uint8).reshape(2, 4)
chunk = SimpleNamespace(
num_local_context_tokens=2,
local_context_lens_allranks=[[2, 2]],
padded_local_cu_seq_lens=torch.tensor([0, 2]),
num_requests=1,
starts=torch.tensor([0]),
)
def local_gather(src, dst, *args):
assert src.dtype == dst.dtype == torch.uint8
dst.copy_(src)
def allgather(dst, src):
assert dst.dtype == src.dtype == torch.uint8
assert dst.shape == (4, 4)
assert src.shape == (2, 4)
dst[:2].copy_(src)
dst[2:].copy_(src + 16)
monkeypatch.setattr(prefill.ops, "cp_gather_cache", local_gather)
gathered = prefill.gather_compressed_context(
cache, workspace, torch.empty(1, 1), chunk, allgather, torch.float8_e4m3fn
)
torch.testing.assert_close(
gathered.view(torch.uint8), torch.cat((cache, cache + 16))
)
# Retain the original logical capacity even though the byte view is larger.
assert gathered.data_ptr() - workspace.data_ptr() == 4 * 4
torch.testing.assert_close(workspace.view(torch.uint8).reshape(-1, 4)[:2], cache)
@pytest.mark.parametrize(
"unsupported",
[
"packed_cache",
"e5m2",
"bf16_cache",
"quantized_weight",
"bias",
"no_indices",
"query_dtype",
"prefill_dtype",
"weight_dtype",
"weight_layout",
"cache_layout",
"dimensions",
"no_aiter",
],
)
def test_dcp_prefill_unsupported_formats_use_original_path(monkeypatch, unsupported):
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm MLA backend")
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLAImpl,
MLACommonImpl,
)
if unsupported == "no_aiter":
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.rocm_aiter_mla."
"is_aiter_found_and_supported",
lambda: False,
)
impl = object.__new__(AiterMLAImpl)
impl.kv_cache_dtype = {
"packed_cache": "fp8_ds_mla",
"e5m2": "fp8_e5m2",
"bf16_cache": "auto",
}.get(unsupported, "fp8")
impl.kv_lora_rank = 512
if unsupported == "dimensions":
impl.kv_lora_rank = 256
impl.qk_nope_head_dim = impl.v_head_dim = 128
impl.qk_rope_head_dim = 64
impl.num_heads = 1
impl.kv_b_proj = SimpleNamespace(
weight=torch.empty(256, 512, dtype=torch.bfloat16),
weight_scale=torch.ones(1) if unsupported == "quantized_weight" else None,
bias=torch.ones(256) if unsupported == "bias" else None,
)
if unsupported == "weight_dtype":
impl.kv_b_proj.weight = impl.kv_b_proj.weight.half()
elif unsupported == "weight_layout":
impl.kv_b_proj.weight = torch.empty(512, 256, dtype=torch.bfloat16).T
metadata = SimpleNamespace(
prefill=SimpleNamespace(
q_data_type=torch.float16
if unsupported == "prefill_dtype"
else torch.bfloat16
),
dcp_context_row_indices=None if unsupported == "no_indices" else [],
)
expected = (torch.ones(1), torch.zeros(1))
def fused_mla_kv_concat(kv_nope, k_pe, use_fp8_prefill):
raise AssertionError("The fallback should receive the callback")
def original_path(*args):
assert args[-1] is fused_mla_kv_concat
return expected
monkeypatch.setattr(
MLACommonImpl,
"_context_parallel_compute_prefill_context",
original_path,
)
q = torch.empty(1, 1, 192, dtype=torch.bfloat16)
cache = torch.empty(1, 1536, 576, dtype=torch.uint8)
if unsupported == "query_dtype":
q = q.half()
elif unsupported == "cache_layout":
cache = torch.empty(1, 1536, 1152, dtype=torch.uint8)[..., ::2]
assert (
impl._context_parallel_compute_prefill_context(
q,
cache,
metadata,
torch.ones(1),
8,
fused_mla_kv_concat_fn=fused_mla_kv_concat,
)
is expected
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm MLA backend")
@multi_gpu_only(num_gpus=2)
@pytest.mark.parametrize("interleave", [1, 64])
@pytest.mark.parametrize(
"context_lens,workspace_size",
[([145, 37], 1024), ([545, 0, 3, 0], 256)],
ids=["single_chunk", "continuations_and_empty_context"],
)
def test_dcp_prefill_matches_full_attention(
interleave, context_lens, workspace_size, tmp_path
):
"""Real DCP gather, fused expansion and suffix/chunk merges match dense SDPA."""
import torch.multiprocessing as mp
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip("AITER on CDNA 3 or newer is required")
mp.spawn(
_dcp_prefill_worker,
args=(
f"file://{tmp_path / 'dist_init'}",
interleave,
context_lens,
workspace_size,
),
nprocs=2,
)
@torch.inference_mode()
def _dcp_prefill_worker(rank, init_method, interleave, context_lens, workspace_size):
import torch.nn.functional as F
from tests.v1.attention.test_mla_backends import MockMLAAttentionLayer
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import set_current_vllm_config
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import (
AiterFlashAttnPrefillBackend,
)
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLAImpl,
AiterMLAMetadataBuilder,
)
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import init_workspace_manager
torch.accelerator.set_device_index(rank)
torch.set_num_threads(1)
device = torch.device(f"cuda:{rank}")
init_workspace_manager(device)
world_size, num_heads, block_size = 2, 12, 64
config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-V2-Lite",
dtype=torch.bfloat16,
tensor_parallel_size=world_size,
max_model_len=2048,
block_size=block_size,
max_num_seqs=8,
max_num_batched_tokens=2048,
)
config.cache_config.cache_dtype = "fp8"
config.parallel_config.decode_context_parallel_size = world_size
config.parallel_config.cp_kv_cache_interleave_size = interleave
config.attention_config.use_prefill_query_quantization = False
config.kernel_config.enable_jit_warmup = False
with set_current_vllm_config(config):
init_distributed_environment(
world_size=world_size,
rank=rank,
local_rank=rank,
distributed_init_method=init_method,
)
initialize_model_parallel(
tensor_model_parallel_size=world_size,
decode_context_model_parallel_size=world_size,
)
try:
projection = ColumnParallelLinear(
512,
num_heads * world_size * 256,
bias=False,
params_dtype=torch.bfloat16,
).to(device)
torch.manual_seed(101 + rank)
projection.weight.copy_(torch.randn_like(projection.weight) * 0.02)
dims = dict(
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
)
impl = AiterMLAImpl(
num_heads=num_heads,
head_size=576,
scale=192**-0.5,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="fp8",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
q_lora_rank=None,
qk_head_dim=192,
kv_b_proj=projection,
**dims,
)
layer = MockMLAAttentionLayer(
impl=impl,
num_heads=num_heads,
device=device,
kv_b_proj=projection,
q_scale=1.0,
k_scale=0.07,
**dims,
)
layer.prefill_backend = AiterFlashAttnPrefillBackend(
num_heads=num_heads,
scale=impl.scale,
vllm_config=config,
**dims,
)
layer.dcp_manager = MLADCPManager(
config,
device,
num_heads,
576,
512,
torch.bfloat16,
torch.bfloat16,
None,
True,
False,
)
config.compilation_config.static_forward_context["mla"] = layer
spec = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=576,
dtype=current_platform.fp8_dtype(),
cache_dtype_str="fp8",
)
builder = AiterMLAMetadataBuilder(spec, ["mla"], config, device)
builder.chunked_prefill_workspace_size = workspace_size
builder.chunked_prefill_workspace = torch.empty(
(workspace_size + workspace_size // world_size, 576),
dtype=torch.bfloat16,
device=device,
)
layer.dcp_manager.init_kv_gather(
builder.chunked_prefill_workspace, workspace_size
)
query_lens = [i + 2 for i in range(len(context_lens))]
batch = BatchSpec(
seq_lens=[c + q for c, q in zip(context_lens, query_lens)],
query_lens=query_lens,
)
common = create_common_attn_metadata(
batch, block_size, device, arange_block_indices=True
)
metadata = builder.build(0, common)
assert metadata.prefill is not None
chunked = metadata.prefill.chunked_context
assert chunked is not None and metadata.dcp_context_row_indices is not None
if 0 in context_lens:
assert chunked.empty_token_slices
assert any(c.is_continuation for c in chunked.chunks)
else:
assert len(chunked.chunks) == 1
blocks_per_req = common.block_table_tensor.shape[1]
cache = torch.zeros(
(len(context_lens) * blocks_per_req, block_size, 576),
dtype=torch.bfloat16,
device=device,
).to(current_platform.fp8_dtype())
queries, suffixes, references = [], [], []
# All ranks reconstruct the same global cache before sharding it.
torch.manual_seed(42)
for request, (context_len, query_len) in enumerate(
zip(context_lens, query_lens)
):
latent = torch.randn(
context_len, 576, device=device, dtype=torch.bfloat16
)
quantized = (latent.float() / layer._k_scale).to(cache.dtype)
positions = torch.arange(context_len, device=device)
shard = quantized.view(torch.uint8)[
(positions // interleave) % world_size == rank
]
cache.view(len(context_lens), -1, 576).view(torch.uint8)[
request, : len(shard)
].copy_(shard)
suffix = torch.randn(
query_len, 576, device=device, dtype=torch.bfloat16
)
q = (
torch.randn(
query_len, num_heads, 192, device=device, dtype=torch.bfloat16
)
* 0.5
)
full_latent = torch.cat(
(
(quantized.float() * layer._k_scale).to(torch.bfloat16),
suffix,
)
)
kv = F.linear(full_latent[:, :512], projection.weight).view(
-1, num_heads, 256
)
k_nope, v = kv.split(128, dim=-1)
k = torch.cat(
(k_nope, full_latent[:, None, 512:].expand(-1, num_heads, -1)),
dim=-1,
)
mask = torch.arange(context_len + query_len, device=device)[
None, :
] <= (context_len + torch.arange(query_len, device=device)[:, None])
references.append(
F.scaled_dot_product_attention(
q.transpose(0, 1).float(),
k.transpose(0, 1).float(),
v.transpose(0, 1).float(),
attn_mask=mask,
scale=impl.scale,
).transpose(0, 1)
)
queries.append(q)
suffixes.append(suffix)
q, suffix = torch.cat(queries), torch.cat(suffixes)
output = torch.empty(
q.shape[0], num_heads * 128, device=device, dtype=torch.bfloat16
)
# Ensure this test cannot silently succeed through the fallback.
with pytest.MonkeyPatch.context() as patch:
def unexpected_fallback(*args, **kwargs):
raise AssertionError("Expected the compressed DCP path")
patch.setattr(
"vllm.model_executor.layers.attention.mla_attention."
"MLACommonImpl._context_parallel_compute_prefill_context",
unexpected_fallback,
)
impl.forward_mha(
q,
suffix[:, :512],
suffix[:, None, 512:],
cache,
metadata,
layer._k_scale,
output,
)
torch.testing.assert_close(
output.view(-1, num_heads, 128).float(),
torch.cat(references),
rtol=0.03,
atol=0.003,
)
finally:
cleanup_dist_env_and_memory()
+52
View File
@@ -643,6 +643,36 @@ def _check_aiter_mla_fp8_support() -> bool:
return _AITER_MLA_SUPPORTS_FP8
def _rocm_aiter_gather_kv_b_proj_impl(
kv_cache: torch.Tensor,
kv_scale: torch.Tensor,
kv_indptr: torch.Tensor,
kv_indices: torch.Tensor,
cu_seq_lens: torch.Tensor,
weight: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
) -> None:
from aiter.ops.triton.gather_kv_b_proj import gather_kv_b_proj
gather_kv_b_proj(
kv_cache, kv_scale, kv_indptr, kv_indices, cu_seq_lens, weight, None, k, v
)
def _rocm_aiter_gather_kv_b_proj_fake(
kv_cache: torch.Tensor,
kv_scale: torch.Tensor,
kv_indptr: torch.Tensor,
kv_indices: torch.Tensor,
cu_seq_lens: torch.Tensor,
weight: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
) -> None:
pass
def _rocm_aiter_mla_decode_fwd_impl(
q: torch.Tensor,
kv_buffer: torch.Tensor,
@@ -2851,6 +2881,13 @@ class rocm_aiter_ops:
dispatch_key=current_platform.dispatch_key,
)
direct_register_custom_op(
op_name="rocm_aiter_gather_kv_b_proj",
op_func=_rocm_aiter_gather_kv_b_proj_impl,
mutates_args=["k", "v"],
fake_impl=_rocm_aiter_gather_kv_b_proj_fake,
dispatch_key=current_platform.dispatch_key,
)
direct_register_custom_op(
op_name="rocm_aiter_mla_decode_fwd",
op_func=_rocm_aiter_mla_decode_fwd_impl,
@@ -3465,6 +3502,21 @@ class rocm_aiter_ops:
extra_indices=extra_kv_indices,
)
@staticmethod
def gather_kv_b_proj(
kv_cache: torch.Tensor,
kv_scale: torch.Tensor,
kv_indptr: torch.Tensor,
kv_indices: torch.Tensor,
cu_seq_lens: torch.Tensor,
weight: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
) -> None:
torch.ops.vllm.rocm_aiter_gather_kv_b_proj(
kv_cache, kv_scale, kv_indptr, kv_indices, cu_seq_lens, weight, k, v
)
@staticmethod
def mla_decode_fwd(
q: torch.Tensor,
@@ -2,15 +2,16 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, ClassVar, Final
from typing import TYPE_CHECKING, ClassVar, Final, cast
import torch
from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.distributed import get_dcp_group
@@ -22,6 +23,8 @@ from vllm.model_executor.layers.attention.mla_attention import (
MLACommonMetadata,
MLACommonMetadataBuilder,
QueryLenSupport,
accumulate_mla_context_chunk,
init_mla_context_partial,
)
from vllm.triton_utils import tl, triton
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -38,6 +41,11 @@ from vllm.v1.attention.backends.utils import (
from vllm.v1.attention.ops.rocm_aiter_mla_merge import (
merge_mla_segments_triton,
)
from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
context_row_indices,
expand_context,
gather_compressed_context,
)
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
if TYPE_CHECKING:
@@ -46,6 +54,26 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _supports_fused_dcp_prefill_config(
kv_cache_dtype: str | None,
query_dtype: torch.dtype,
kv_lora_rank: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
) -> bool:
"""Whether the configuration supports fused DCP context expansion."""
return (
kv_cache_dtype in ("fp8", "fp8_e4m3")
and query_dtype == torch.bfloat16
and kv_lora_rank == 512
and qk_nope_head_dim == 128
and qk_rope_head_dim == 64
and v_head_dim == 128
and is_aiter_found_and_supported()
)
def _segmented_mla_page_size(block_size: int) -> int:
"""Largest supported power-of-two subpage dividing a physical KV block.
@@ -507,6 +535,7 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
@dataclass
class AiterMLAMetadata(MLACommonMetadata[AiterMLADecodeMetadata]):
dcp_context_row_indices: list[torch.Tensor] | None = None
work_meta_data: torch.Tensor | None = None
work_indptr: torch.Tensor | None = None
work_info_set: torch.Tensor | None = None
@@ -664,6 +693,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device
).multi_processor_count
# Use the group's original format: decode normalizes E5M2 to "fp8".
kv_cache_dtype = getattr(kv_cache_spec, "cache_dtype_str", None)
if kv_cache_dtype is None:
kv_cache_dtype = vllm_config.cache_config.cache_dtype
self._supports_fused_dcp_prefill = _supports_fused_dcp_prefill_config(
kv_cache_dtype,
self.q_data_type,
self.mla_dims.kv_lora_rank,
self.mla_dims.qk_nope_head_dim,
self.mla_dims.qk_rope_head_dim,
self.mla_dims.v_head_dim,
)
self.compilation_config = vllm_config.compilation_config
self.decode_attn_out_dtype = vllm_config.model_config.dtype
@@ -1577,6 +1619,17 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
attn_metadata = super().build(
common_prefix_len, common_attn_metadata, fast_build
)
prefill = attn_metadata.prefill
if (
prefill is not None
and prefill.chunked_context is not None
and self.dcp_world_size > 1
and self._supports_fused_dcp_prefill
):
attn_metadata.dcp_context_row_indices = [
context_row_indices(chunk, prefill.block_table.device)
for chunk in prefill.chunked_context.chunks
]
if (
attn_metadata.decode is not None
and attn_metadata.decode.has_persistent_metadata
@@ -1975,6 +2028,101 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
self._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
self._mla_reduce_v1 = mla_reduce_v1
def _context_parallel_compute_prefill_context(
self,
q: torch.Tensor,
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: MLACommonMetadata,
k_scale: torch.Tensor,
dcp_world_size: int,
fused_mla_kv_concat_fn: Callable[
[torch.Tensor, torch.Tensor, bool], tuple[torch.Tensor, torch.Tensor]
]
| None = None,
):
attn_metadata = cast(AiterMLAMetadata, attn_metadata)
assert attn_metadata.prefill is not None
prefill = attn_metadata.prefill
weight = getattr(self.kv_b_proj, "weight", None)
eligible = (
_supports_fused_dcp_prefill_config(
self.kv_cache_dtype,
prefill.q_data_type,
self.kv_lora_rank,
self.qk_nope_head_dim,
self.qk_rope_head_dim,
self.v_head_dim,
)
and q.dtype == torch.bfloat16
and weight is not None
and weight.dtype == torch.bfloat16
and weight.shape == (self.num_heads * 256, 512)
and weight.is_contiguous()
and getattr(self.kv_b_proj, "bias", None) is None
and getattr(self.kv_b_proj, "weight_scale", None) is None
and kv_c_and_k_pe_cache.is_contiguous()
and attn_metadata.dcp_context_row_indices is not None
)
if not eligible:
return super()._context_parallel_compute_prefill_context(
q,
kv_c_and_k_pe_cache,
attn_metadata,
k_scale,
dcp_world_size,
fused_mla_kv_concat_fn,
)
from vllm.platforms import current_platform
assert weight is not None
assert attn_metadata.dcp_context_row_indices is not None
chunked = prefill.chunked_context
assert chunked is not None and chunked.dcp_manager is not None
assert prefill.prefill_backend is not None
assert len(chunked.chunks) == len(attn_metadata.dcp_context_row_indices)
output = output_lse = None
for chunk, row_indices in zip(
chunked.chunks, attn_metadata.dcp_context_row_indices
):
gathered = gather_compressed_context(
kv_c_and_k_pe_cache,
chunked.workspace,
prefill.block_table[chunk.request_slice],
chunk,
chunked.dcp_manager.kv_gather,
current_platform.fp8_dtype(),
)
k, v = expand_context(
gathered,
k_scale,
row_indices,
chunk.cu_seq_lens,
weight,
self.num_heads,
self.qk_nope_head_dim,
self.qk_rope_head_dim,
self.v_head_dim,
)
attn_output, attn_lse = prefill.prefill_backend.run_prefill_context_chunk(
chunk=chunk,
q=q[chunk.token_slice],
k=k,
v=v,
)
if output is None:
if len(chunked.chunks) == 1 and not chunked.empty_token_slices:
return attn_output, attn_lse
output, output_lse = init_mla_context_partial(
chunked,
attn_output,
attn_lse,
num_tokens=q.shape[0],
)
accumulate_mla_context_chunk(
chunk, attn_output, attn_lse, output, output_lse
)
return output, output_lse
def _flash_attn_varlen_diff_headdims(
self, q, k, v, return_softmax_lse=False, softmax_scale=None, **kwargs
):
@@ -0,0 +1,105 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compressed DCP context preparation for ROCm MLA."""
from collections.abc import Callable
import torch
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.attention.mla_attention import MLACommonPrefillMetadata
def context_row_indices(
chunk: MLACommonPrefillMetadata.ContextChunk, device: torch.device
) -> torch.Tensor:
"""Map compact request-major rows to the padded rank-major AllGather."""
assert chunk.padded_local_seq_lens is not None
assert chunk.local_context_lens_allranks is not None
assert chunk.local_starts is not None
segments = []
offset = 0
for padded, lengths, start in zip(
chunk.padded_local_seq_lens,
chunk.local_context_lens_allranks,
chunk.local_starts,
):
for rank, length in enumerate(lengths):
count = min(max(0, length - start), padded)
if count:
first = rank * chunk.num_local_context_tokens + offset
segments.append(torch.arange(first, first + count, dtype=torch.int32))
offset += padded
rows = torch.cat(segments) if segments else torch.empty(0, dtype=torch.int32)
assert rows.numel() == chunk.num_context_tokens
return rows.to(device=device, non_blocking=True)
def gather_compressed_context(
cache: torch.Tensor,
workspace: torch.Tensor,
block_table: torch.Tensor,
chunk: MLACommonPrefillMetadata.ContextChunk,
gather: Callable[[torch.Tensor, torch.Tensor], object],
cache_dtype: torch.dtype,
) -> torch.Tensor:
"""Gather FP8 bytes, preserving the existing workspace row capacity."""
assert chunk.local_context_lens_allranks is not None
assert workspace.is_contiguous()
rows, width = workspace.shape
fp8_workspace = workspace.view(cache_dtype).view(-1, width)[:rows]
local_rows = chunk.num_local_context_tokens
world_size = len(chunk.local_context_lens_allranks[0])
offset = rows // (world_size + 1)
assert offset * (world_size + 1) == rows
assert local_rows <= offset
local = fp8_workspace[:local_rows]
gathered = fp8_workspace[offset : offset + world_size * local_rows]
assert chunk.padded_local_cu_seq_lens is not None
ops.cp_gather_cache(
cache.view(torch.uint8),
local.view(torch.uint8),
block_table,
chunk.padded_local_cu_seq_lens,
chunk.num_requests,
chunk.starts,
)
gather(gathered.view(torch.uint8), local.view(torch.uint8))
return gathered
def expand_context(
gathered: torch.Tensor,
scale: torch.Tensor,
row_indices: torch.Tensor,
cu_seq_lens: torch.Tensor,
weight: torch.Tensor,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fuse row reorganization, dequantization, projection and K/V packing."""
k = torch.empty(
(row_indices.numel(), num_heads, qk_nope_head_dim + qk_rope_head_dim),
device=gathered.device,
dtype=weight.dtype,
)
v = torch.empty(
(row_indices.numel(), num_heads, v_head_dim),
device=gathered.device,
dtype=weight.dtype,
)
if row_indices.numel():
rocm_aiter_ops.gather_kv_b_proj(
gathered.unsqueeze(1),
scale,
cu_seq_lens,
row_indices,
cu_seq_lens,
weight,
k,
v,
)
return k, v