[Attention] Require explicit DCP support from attention implementations (#55780)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
cb22234687
commit
db3814a4f2
@@ -10,6 +10,7 @@ import torch
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
CacheConfig,
|
||||
ParallelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
@@ -29,7 +30,11 @@ else:
|
||||
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import _cached_get_attn_backend, get_attn_backend
|
||||
from vllm.v1.attention.selector import (
|
||||
AttentionSelectorConfig,
|
||||
_cached_get_attn_backend,
|
||||
get_attn_backend,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -77,33 +82,54 @@ def generate_params():
|
||||
else DEVICE_REGULAR_ATTN_BACKENDS[device]
|
||||
)
|
||||
for name in backends:
|
||||
block_sizes = DEVICE_MLA_BLOCK_SIZES[device] if use_mla else [16]
|
||||
block_sizes = (
|
||||
[128]
|
||||
if name == "CUTLASS_MLA"
|
||||
else DEVICE_MLA_BLOCK_SIZES[device]
|
||||
if use_mla
|
||||
else [16]
|
||||
)
|
||||
for block_size in block_sizes:
|
||||
params.append(
|
||||
pytest.param(
|
||||
device,
|
||||
name,
|
||||
use_mla,
|
||||
block_size,
|
||||
id=f"{device}_{name}_mla_{str(use_mla)[0]}_blks{block_size}",
|
||||
for use_dcp in [False, True] if device != "cpu" else [False]:
|
||||
params.append(
|
||||
pytest.param(
|
||||
device,
|
||||
name,
|
||||
use_mla,
|
||||
block_size,
|
||||
use_dcp,
|
||||
id=(
|
||||
f"{device}_{name}_mla_{str(use_mla)[0]}"
|
||||
f"_blks{block_size}_dcp{use_dcp}"
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device, name, use_mla, block_size", generate_params())
|
||||
@pytest.mark.parametrize(
|
||||
"device, name, use_mla, block_size, use_dcp", generate_params()
|
||||
)
|
||||
def test_backend_selection(
|
||||
device: str,
|
||||
name: str,
|
||||
use_mla: bool,
|
||||
block_size: int,
|
||||
use_dcp: bool,
|
||||
):
|
||||
"""Test attention backend selection with valid device-backend pairs."""
|
||||
"""Supported GPU backends remain selectable when DCP is enabled."""
|
||||
# Create AttentionConfig with the specified backend
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum[name])
|
||||
cache_config = CacheConfig(block_size=block_size)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config, cache_config=cache_config
|
||||
attention_config=attention_config,
|
||||
cache_config=cache_config,
|
||||
parallel_config=ParallelConfig(
|
||||
# Skip GPU-count autodetection: selection does not launch workers.
|
||||
distributed_executor_backend="mp" if use_dcp else None,
|
||||
tensor_parallel_size=2 if use_dcp else 1,
|
||||
decode_context_parallel_size=2 if use_dcp else 1,
|
||||
),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
@@ -135,9 +161,15 @@ def test_backend_selection(
|
||||
expected = name
|
||||
assert backend.get_name() == expected
|
||||
else:
|
||||
backend = get_attn_backend(32, torch.float16, None, use_mla=use_mla)
|
||||
expected = "ROCM_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
if use_dcp:
|
||||
with pytest.raises(ValueError, match="DCP not supported"):
|
||||
get_attn_backend(32, torch.float16, None, use_mla=use_mla)
|
||||
else:
|
||||
backend = get_attn_backend(
|
||||
32, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "ROCM_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
elif device == "cuda":
|
||||
if CudaPlatform is None:
|
||||
@@ -230,6 +262,52 @@ def test_backend_selection(
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="GPU attention backends"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name",
|
||||
[
|
||||
"FLASH_ATTN",
|
||||
"FLASH_ATTN_DIFFKV",
|
||||
"FLASHINFER",
|
||||
"TRITON_MLA",
|
||||
"ROCM_AITER_MLA",
|
||||
"ROCM_AITER_TRITON_MLA",
|
||||
"CUTLASS_MLA",
|
||||
"FLASHMLA",
|
||||
"FLASHINFER_MLA",
|
||||
"TOKENSPEED_MLA",
|
||||
"FLASH_ATTN_MLA",
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
],
|
||||
)
|
||||
def test_supported_backend_preserves_dcp_eligibility(backend_name, default_vllm_config):
|
||||
"""DCP opt-ins preserve eligibility and unrelated configuration restrictions."""
|
||||
if backend_name.startswith("ROCM_") and not current_platform.is_rocm():
|
||||
pytest.skip("ROCm-specific backend")
|
||||
try:
|
||||
backend_cls = AttentionBackendEnum[backend_name].get_class()
|
||||
except ImportError as error:
|
||||
pytest.skip(f"Optional backend dependency unavailable: {error}")
|
||||
|
||||
config = AttentionSelectorConfig(
|
||||
head_size=576 if backend_cls.is_mla() else 128,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=backend_cls.get_preferred_block_size(64),
|
||||
use_mla=backend_cls.is_mla(),
|
||||
use_sparse=backend_cls.is_sparse(),
|
||||
)
|
||||
kwargs = dict(
|
||||
device_capability=current_platform.get_device_capability(), **config._asdict()
|
||||
)
|
||||
invalid_without_dcp = backend_cls.validate_configuration(**kwargs)
|
||||
kwargs["use_dcp"] = True
|
||||
assert backend_cls.validate_configuration(**kwargs) == invalid_without_dcp
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda", "hip"])
|
||||
def test_fp32_fallback(device: str):
|
||||
"""Test attention backend selection with fp32."""
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionImpl,
|
||||
@@ -9,6 +12,7 @@ from vllm.v1.attention.backends.registry import (
|
||||
MambaAttentionBackendEnum,
|
||||
register_backend,
|
||||
)
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
|
||||
|
||||
class CustomAttentionImpl(AttentionImpl):
|
||||
@@ -67,6 +71,23 @@ class CustomMambaAttentionBackend(AttentionBackend):
|
||||
return None
|
||||
|
||||
|
||||
def test_custom_attention_backend_requires_dcp_opt_in():
|
||||
"""A new implementation remains usable without implicitly opting into DCP."""
|
||||
config = AttentionSelectorConfig(
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=16,
|
||||
)
|
||||
kwargs = dict(device_capability=DeviceCapability(9, 0), **config._asdict())
|
||||
assert CustomAttentionBackend.validate_configuration(**kwargs) == []
|
||||
|
||||
kwargs["use_dcp"] = True
|
||||
assert CustomAttentionBackend.validate_configuration(**kwargs) == [
|
||||
"DCP not supported"
|
||||
]
|
||||
|
||||
|
||||
def test_custom_is_not_alias_of_any_backend():
|
||||
# Get all members of AttentionBackendEnum
|
||||
all_backends = list(AttentionBackendEnum)
|
||||
|
||||
@@ -74,6 +74,7 @@ def test_aiter_unified_attention_capture_preserves_query_start_locations():
|
||||
assert torch.equal(actual.query_start_loc, expected_query_start_loc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_dcp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"env_vars, selected_backend, expected_backend_path",
|
||||
[
|
||||
@@ -143,11 +144,12 @@ def test_standard_attention_backend_selection(
|
||||
env_vars,
|
||||
selected_backend,
|
||||
expected_backend_path,
|
||||
use_dcp,
|
||||
mock_vllm_config,
|
||||
mock_get_cdna_version,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Test standard attention backend selection with various configurations."""
|
||||
"""Standard ROCm backends remain selectable without DCP and reject DCP."""
|
||||
# Set environment variables
|
||||
for key, value in env_vars.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
@@ -176,8 +178,14 @@ def test_standard_attention_backend_selection(
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_dcp=use_dcp,
|
||||
)
|
||||
|
||||
if use_dcp:
|
||||
with pytest.raises(ValueError, match="DCP not supported"):
|
||||
RocmPlatform.get_attn_backend_cls(backend_enum, attn_selector_config)
|
||||
return
|
||||
|
||||
backend_path = RocmPlatform.get_attn_backend_cls(
|
||||
selected_backend=backend_enum, attn_selector_config=attn_selector_config
|
||||
)
|
||||
@@ -185,6 +193,7 @@ def test_standard_attention_backend_selection(
|
||||
assert backend_path == expected_backend_path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_dcp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"env_vars, selected_backend, block_size, expected_backend_path, should_raise",
|
||||
[
|
||||
@@ -261,10 +270,11 @@ def test_mla_backend_selection(
|
||||
block_size,
|
||||
expected_backend_path,
|
||||
should_raise,
|
||||
use_dcp,
|
||||
mock_vllm_config,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Test MLA backend selection with various configurations."""
|
||||
"""Dense MLA remains selectable with DCP for valid block sizes."""
|
||||
# Set environment variables
|
||||
for key, value in env_vars.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
@@ -303,6 +313,7 @@ def test_mla_backend_selection(
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_dcp=use_dcp,
|
||||
)
|
||||
attn_selector_config = AttentionSelectorConfig(
|
||||
head_size=128,
|
||||
@@ -312,6 +323,7 @@ def test_mla_backend_selection(
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_dcp=use_dcp,
|
||||
)
|
||||
backend_path = RocmPlatform.get_attn_backend_cls(
|
||||
selected_backend=backend_enum,
|
||||
@@ -327,6 +339,7 @@ def test_mla_backend_selection(
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_dcp=use_dcp,
|
||||
)
|
||||
|
||||
backend_path = RocmPlatform.get_attn_backend_cls(
|
||||
@@ -336,6 +349,63 @@ def test_mla_backend_selection(
|
||||
assert backend_path == expected_backend_path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_dcp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"selected_backend", [None, AttentionBackendEnum.ROCM_AITER_MLA_SPARSE]
|
||||
)
|
||||
def test_sparse_mla_backend_rejects_dcp(selected_backend, use_dcp):
|
||||
"""Sparse MLA remains selectable without DCP and fails early with DCP."""
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
selector_config = AttentionSelectorConfig(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=16,
|
||||
use_mla=True,
|
||||
use_sparse=True,
|
||||
use_dcp=use_dcp,
|
||||
)
|
||||
if use_dcp:
|
||||
with pytest.raises(ValueError, match="DCP not supported"):
|
||||
RocmPlatform.get_attn_backend_cls(selected_backend, selector_config)
|
||||
else:
|
||||
assert RocmPlatform.get_attn_backend_cls(selected_backend, selector_config) == (
|
||||
AttentionBackendEnum.ROCM_AITER_MLA_SPARSE.get_path()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_dcp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"selected_backend, head_size, kv_cache_dtype",
|
||||
[
|
||||
(AttentionBackendEnum.TRITON_ATTN_DIFFKV, 192, "bfloat16"),
|
||||
# A 128-dimensional head uses 118 packed bytes, or 59 fp16 elements.
|
||||
(AttentionBackendEnum.TURBOQUANT, 59, "turboquant_k3v4_nc"),
|
||||
],
|
||||
)
|
||||
def test_specialized_attention_backends_reject_dcp(
|
||||
selected_backend, head_size, kv_cache_dtype, use_dcp
|
||||
):
|
||||
"""Valid DiffKV and compressed-cache configurations must reject DCP."""
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
selector_config = AttentionSelectorConfig(
|
||||
head_size=head_size,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
block_size=16,
|
||||
use_dcp=use_dcp,
|
||||
)
|
||||
if use_dcp:
|
||||
with pytest.raises(ValueError, match="DCP not supported"):
|
||||
RocmPlatform.get_attn_backend_cls(selected_backend, selector_config)
|
||||
else:
|
||||
assert RocmPlatform.get_attn_backend_cls(selected_backend, selector_config) == (
|
||||
selected_backend.get_path()
|
||||
)
|
||||
|
||||
|
||||
def test_aiter_fa_requires_mi3xx(mock_vllm_config):
|
||||
"""Test that ROCM_AITER_FA requires CDNA3+ architecture."""
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
@@ -819,7 +819,7 @@ class AttentionImplBase(ABC, Generic[T]):
|
||||
# Whether the attention impl supports Prefill Context Parallelism.
|
||||
supports_pcp: bool = False
|
||||
# Whether the attention impl supports Decode Context Parallelism.
|
||||
supports_dcp: bool = True
|
||||
supports_dcp: bool = False
|
||||
# Whether the attention impl(or ops) supports MTP
|
||||
# when cp_kv_cache_interleave_size > 1
|
||||
supports_mtp_with_cp_non_trivial_interleave_size: bool = False
|
||||
|
||||
@@ -832,6 +832,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
|
||||
class FlashAttentionImpl(AttentionImpl):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1779,6 +1779,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
|
||||
class FlashInferImpl(AttentionImpl):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -103,6 +103,7 @@ MAX_HEADS = 128
|
||||
|
||||
class CutlassMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -255,6 +255,7 @@ class FlashAttnMLAMetadataBuilder(MLACommonMetadataBuilder[FlashAttnMLAMetadata]
|
||||
|
||||
class FlashAttnMLAImpl(MLACommonImpl[FlashAttnMLAMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -210,6 +210,7 @@ class FlashInferMLABackend(MLACommonBackend):
|
||||
|
||||
class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
# DCP is the only path that consumes LSE. It uses monolithic CuTeDSL,
|
||||
# whose public LSE contract is natural-log.
|
||||
lse_base_on_e: bool = True
|
||||
|
||||
@@ -319,6 +319,7 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||
"""
|
||||
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
lse_base_on_e: bool = False
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -207,6 +207,7 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]):
|
||||
|
||||
class FlashMLAImpl(MLACommonImpl[FlashMLAMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -582,6 +582,7 @@ class FlashMLASparseMetadataBuilder(
|
||||
|
||||
class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
|
||||
@staticmethod
|
||||
def _compute_fp8_decode_padded_heads(num_heads: int) -> int:
|
||||
|
||||
@@ -1421,6 +1421,7 @@ class AiterMLAHelper:
|
||||
class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
# DCP decode paths return natural-log softmax LSE for the cross-rank merge.
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
# Measured on gfx950: aiter.mla.mla_decode_fwd(return_lse=True) matches
|
||||
# logsumexp to fp32 exactly, and merge_mla_segments_triton converts AITER's
|
||||
# base-2 segment statistics with LOGE2. Stated rather than inherited because
|
||||
|
||||
@@ -154,6 +154,7 @@ class TokenspeedMLABackend(MLACommonBackend):
|
||||
|
||||
class TokenspeedMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
# tokenspeed_mla_decode returns LSE in log2 units; its own DCP test merges
|
||||
# partial outputs with exp2(lse).
|
||||
lse_base_on_e: bool = False
|
||||
|
||||
@@ -181,6 +181,7 @@ class TritonMLABackend(MLACommonBackend):
|
||||
|
||||
class TritonMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
supports_dcp: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user