[SM100] Hdim 256 optimized (#52980)
Signed-off-by: Simon Veitner <sveitner@redhat.com>
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import patch
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -14,6 +15,7 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
if current_platform.is_cuda():
|
||||
from vllm.platforms.cuda import CudaPlatform
|
||||
@@ -25,6 +27,7 @@ if current_platform.is_rocm():
|
||||
else:
|
||||
RocmPlatform = None
|
||||
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import _cached_get_attn_backend, get_attn_backend
|
||||
|
||||
@@ -555,3 +558,175 @@ def test_flash_attn_accepts_handled_fp8_variants(
|
||||
# import order across earlier tests that patch vllm.platforms.current_platform.
|
||||
monkeypatch.setattr(fa_utils_mod.current_platform, "is_xpu", lambda: True)
|
||||
assert FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
|
||||
|
||||
blackwell_only = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="FA4 is CUDA-only"
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _blackwell(vllm_config=None):
|
||||
platform = MagicMock()
|
||||
platform.is_xpu.return_value = False
|
||||
platform.is_rocm.return_value = False
|
||||
platform.get_device_capability.return_value = DeviceCapability(10, 0)
|
||||
with (
|
||||
patch("vllm.v1.attention.backends.fa_utils.current_platform", platform),
|
||||
patch(
|
||||
"vllm.vllm_flash_attn.flash_attn_interface.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
patch("vllm.config.get_current_vllm_config_or_none", return_value=vllm_config),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.flash_attn.get_current_vllm_config_or_none",
|
||||
return_value=vllm_config,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def _hd256_config(
|
||||
*,
|
||||
is_mm_prefix_lm=False,
|
||||
rswa_window=None,
|
||||
dcp_size=1,
|
||||
softcap=None,
|
||||
head_size=256,
|
||||
cache_dtype="auto",
|
||||
):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.attention_config.flash_attn_version = None
|
||||
vllm_config.model_config.is_mm_prefix_lm = is_mm_prefix_lm
|
||||
vllm_config.model_config.rswa_window = rswa_window
|
||||
vllm_config.model_config.hf_text_config.attn_logit_softcapping = softcap
|
||||
vllm_config.model_config.get_head_size.return_value = head_size
|
||||
vllm_config.cache_config.cache_dtype = cache_dtype
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
return vllm_config
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs,config_kwargs,expected",
|
||||
[
|
||||
({}, {}, 4),
|
||||
({"supports_fa4_hd256": False}, {}, 2),
|
||||
({"has_sinks": True}, {}, 2),
|
||||
({"requires_softcap": True}, {}, 2),
|
||||
# Larger block sizes normalize to 128.
|
||||
({"kv_cache_block_size": 16}, {}, 2),
|
||||
({"kv_cache_block_size": 64}, {}, 2),
|
||||
({"kv_cache_block_size": 256}, {}, 4),
|
||||
({}, {"is_mm_prefix_lm": True}, 2),
|
||||
({}, {"rswa_window": 512}, 2),
|
||||
({}, {"dcp_size": 2}, 2),
|
||||
({}, {"softcap": 50.0}, 2),
|
||||
({}, {"cache_dtype": "fp8"}, 2),
|
||||
({"head_size": 128, "kv_cache_block_size": 16}, {}, 4),
|
||||
({"head_size": 192, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 4),
|
||||
({"head_size": 256, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 2),
|
||||
({"head_size": 256, "head_size_v": 64}, {}, 2),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_fallback_matrix(kwargs, config_kwargs, expected):
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
|
||||
kwargs = {
|
||||
"head_size": 256,
|
||||
"kv_cache_block_size": 128,
|
||||
"supports_fa4_hd256": True,
|
||||
**kwargs,
|
||||
}
|
||||
with _blackwell(_hd256_config(head_size=kwargs["head_size"], **config_kwargs)):
|
||||
assert get_flash_attn_version(**kwargs) == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,config_kwargs,expected",
|
||||
[
|
||||
("FLASH_ATTN", {}, 128),
|
||||
("FLASH_ATTN", {"softcap": 50.0}, 16),
|
||||
("FLASH_ATTN", {"head_size": 128}, 16),
|
||||
("FLASH_ATTN", {"cache_dtype": "fp8"}, 16),
|
||||
("FLASH_ATTN_DIFFKV", {}, 16),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_block_size_advertisement(
|
||||
backend_name: str, config_kwargs: dict, expected: int
|
||||
):
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm.v1.attention.backends.flash_attn_diffkv import (
|
||||
FlashAttentionDiffKVBackend,
|
||||
)
|
||||
|
||||
backend = {
|
||||
"FLASH_ATTN": FlashAttentionBackend,
|
||||
"FLASH_ATTN_DIFFKV": FlashAttentionDiffKVBackend,
|
||||
}[backend_name]
|
||||
vllm_config = _hd256_config(**config_kwargs)
|
||||
with _blackwell(vllm_config):
|
||||
(size,) = backend.get_supported_kernel_block_sizes()
|
||||
preferred = backend.get_preferred_block_size(16)
|
||||
assert preferred == expected
|
||||
if expected == 128:
|
||||
assert size == 128
|
||||
else:
|
||||
assert isinstance(size, MultipleOf) and size.base == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
def test_fa4_hd256_mm_prefix_deselects_flash_attn():
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
with _blackwell(_hd256_config(is_mm_prefix_lm=True)):
|
||||
assert (
|
||||
FlashAttentionBackend.supports_combination(
|
||||
head_size=256,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=None,
|
||||
block_size=128,
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=True,
|
||||
device_capability=DeviceCapability(10, 0),
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda()
|
||||
or not current_platform.is_device_capability_family(100),
|
||||
reason="requires a Blackwell GPU",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_type,sliding_window,expected",
|
||||
[
|
||||
(AttentionType.DECODER, None, 4),
|
||||
(AttentionType.DECODER, 512, 4),
|
||||
# Local encoder attention lacks the required seqused tensors.
|
||||
(AttentionType.ENCODER_ONLY, None, 4),
|
||||
(AttentionType.ENCODER_ONLY, 512, 2),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionImpl
|
||||
|
||||
# Implementations are constructed before the platform updates the block size.
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
impl = FlashAttentionImpl(
|
||||
num_heads=8,
|
||||
head_size=256,
|
||||
scale=0.0625,
|
||||
num_kv_heads=8,
|
||||
alibi_slopes=None,
|
||||
sliding_window=sliding_window,
|
||||
kv_cache_dtype="auto",
|
||||
attn_type=attn_type,
|
||||
)
|
||||
assert impl.vllm_flash_attn_version == expected
|
||||
assert impl.fa4_hd256 == (expected == 4)
|
||||
|
||||
@@ -215,3 +215,78 @@ def test_varlen_with_paged_kv(
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="FA4 only has a dedicated head_size=256 kernel on Blackwell",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens", [[(1, 1328), (5, 18), (129, 463)], [(1, 523), (1, 37), (1, 2011)]]
|
||||
)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
|
||||
@torch.inference_mode()
|
||||
def test_fa4_hd256_paged_call_shape(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
sliding_window: int | None,
|
||||
) -> None:
|
||||
"""Handles excess block-table columns with page-aligned ``max_seqlen_k``."""
|
||||
from vllm.v1.attention.backends.fa_utils import FA4_HD256_PAGE_SIZE
|
||||
|
||||
if not is_fa_version_supported(4):
|
||||
pytest.skip(f'FA4 unsupported: "{fa_version_unsupported_reason(4)}"')
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
|
||||
head_size = 256
|
||||
block_size = FA4_HD256_PAGE_SIZE
|
||||
num_blocks = 2048
|
||||
dtype = torch.bfloat16
|
||||
num_query_heads, num_kv_heads = num_heads
|
||||
query_lens = [x[0] for x in seq_lens]
|
||||
kv_lens = [x[1] for x in seq_lens]
|
||||
max_kv_len = max(kv_lens)
|
||||
window_size = (sliding_window - 1, 0) if sliding_window is not None else (-1, -1)
|
||||
scale = head_size**-0.5
|
||||
|
||||
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
|
||||
key_cache = torch.randn(
|
||||
num_blocks, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
)
|
||||
num_pages = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, num_blocks, (len(seq_lens), num_pages + 3), dtype=torch.int32
|
||||
)
|
||||
|
||||
output = flash_attn_varlen_func(
|
||||
q=query,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
seqused_k=torch.tensor(kv_lens, dtype=torch.int32),
|
||||
max_seqlen_q=max(query_lens),
|
||||
max_seqlen_k=num_pages * block_size,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
block_table=block_tables[:, :num_pages],
|
||||
num_splits=1,
|
||||
fa_version=4,
|
||||
)
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
query_lens=query_lens,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
torch.testing.assert_close(output, ref_output, atol=1.5e-2, rtol=1e-2)
|
||||
|
||||
Reference in New Issue
Block a user