Xqa decode kernels (#43232)
Signed-off-by: Dan Blanaru <48605845+DanBlanaru@users.noreply.github.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
d715b3aa1e
commit
d29125c085
@@ -78,14 +78,29 @@ def test_supports_sm100_with_artifactory(_art, _cap):
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability", return_value=False
|
||||
)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
return_value=False,
|
||||
)
|
||||
def test_supports_non_sm100_platform(_cap):
|
||||
def test_supports_unsupported_platform(_family, _cap):
|
||||
assert supports_trtllm_attention() is False
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch("vllm.utils.flashinfer.current_platform.is_device_capability", return_value=True)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
return_value=False,
|
||||
)
|
||||
@patch("vllm.utils.flashinfer.has_nvidia_artifactory", return_value=True)
|
||||
def test_supports_sm90_decode_only(_art, _family, _cap):
|
||||
assert supports_trtllm_attention(is_prefill=False) is True
|
||||
assert supports_trtllm_attention(is_prefill=True) is False
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
|
||||
@@ -16,7 +16,7 @@ from tests.v1.attention.utils import (
|
||||
try_backend_includes_kv_cache_update,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config import ModelConfig, set_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -24,7 +24,11 @@ from vllm.utils.torch_utils import (
|
||||
is_torch_equal_or_newer,
|
||||
set_random_seed,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionType, CommonAttentionMetadata
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
AttentionType,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
set_kv_cache_layout,
|
||||
@@ -626,6 +630,118 @@ def test_causal_backend_correctness(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype():
|
||||
"""XQA decode should only apply q_scale when decode Q is FP8."""
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
class MockLayer:
|
||||
_q_scale_float = 2.0
|
||||
_k_scale_float = 3.0
|
||||
|
||||
impl = object.__new__(flashinfer_backend.FlashInferImpl)
|
||||
impl.scale = 0.5
|
||||
impl.kv_cache_dtype = "fp8"
|
||||
|
||||
assert impl.get_xqa_bmm1_scale(MockLayer, torch.bfloat16) == 1.5
|
||||
assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_sm90_xqa_decode_correctness(default_vllm_config):
|
||||
"""FlashInfer should route Hopper decode through XQA and match SDPA."""
|
||||
if not current_platform.is_cuda() or not current_platform.is_device_capability(90):
|
||||
pytest.skip("FlashInfer XQA decode requires SM90.")
|
||||
|
||||
import unittest.mock
|
||||
|
||||
from vllm.utils.flashinfer import can_use_trtllm_attention
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
from vllm.v1.attention.backends.utils import PerLayerParameters
|
||||
|
||||
def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
|
||||
return {
|
||||
"placeholder": PerLayerParameters(
|
||||
window_left=-1,
|
||||
logits_soft_cap=0.0,
|
||||
sm_scale=1.0,
|
||||
)
|
||||
}
|
||||
|
||||
def causal_mask_mod(
|
||||
b: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
q_idx: torch.Tensor,
|
||||
kv_idx: torch.Tensor,
|
||||
*,
|
||||
context_len: int,
|
||||
):
|
||||
return (q_idx + context_len) >= kv_idx
|
||||
|
||||
batch_spec = BATCH_SPECS["small_decode"]
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="meta-llama/Meta-Llama-3-8B",
|
||||
max_model_len=max(batch_spec.seq_lens),
|
||||
block_size=16,
|
||||
)
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=vllm_config.model_config.get_num_kv_heads(
|
||||
vllm_config.parallel_config
|
||||
),
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if not can_use_trtllm_attention(
|
||||
vllm_config.model_config.get_num_attention_heads(
|
||||
vllm_config.parallel_config
|
||||
),
|
||||
kv_cache_spec.num_kv_heads,
|
||||
is_prefill=False,
|
||||
):
|
||||
pytest.skip("FlashInfer XQA decode is not available in this setup.")
|
||||
|
||||
with unittest.mock.patch(
|
||||
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
|
||||
mock_get_per_layer_parameters,
|
||||
):
|
||||
builder = flashinfer_backend.FlashInferMetadataBuilder(
|
||||
kv_cache_spec, ["placeholder"], vllm_config, device
|
||||
)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, vllm_config.cache_config.block_size, device
|
||||
)
|
||||
attn_metadata = builder.build(0, common_attn_metadata)
|
||||
|
||||
assert (
|
||||
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
assert isinstance(
|
||||
attn_metadata.decode,
|
||||
flashinfer_backend.FlashInferTrtllmAPIDecode,
|
||||
)
|
||||
assert attn_metadata.decode.kernel == flashinfer_backend.FlashInferDecodeKernel.XQA
|
||||
|
||||
_test_backend_correctness(
|
||||
batch_spec,
|
||||
"meta-llama/Meta-Llama-3-8B",
|
||||
[AttentionBackendEnum.FLASHINFER],
|
||||
causal_mask_mod,
|
||||
)
|
||||
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# FLASH_ATTN is not supported on ROCm
|
||||
SLIDING_WINDOW_BACKENDS_TO_TEST = [
|
||||
@@ -656,7 +772,7 @@ else:
|
||||
@pytest.mark.parametrize("model", ["microsoft/Phi-tiny-MoE-instruct"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
||||
def test_sliding_window_backend_correctness(
|
||||
batch_spec_name: str, model: str, tensor_parallel_size: int
|
||||
default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int
|
||||
):
|
||||
"""Test backend's correctness with sliding window attention."""
|
||||
|
||||
@@ -718,7 +834,7 @@ def test_sliding_window_backend_correctness(
|
||||
@pytest.mark.parametrize("model", ["google/embeddinggemma-300m"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2])
|
||||
def test_sliding_window_encoder_backend_correctness(
|
||||
batch_spec_name: str, model: str, tensor_parallel_size: int
|
||||
default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int
|
||||
):
|
||||
"""Test backend's correctness with sliding window attention."""
|
||||
|
||||
|
||||
@@ -32,9 +32,10 @@ if not current_platform.is_device_capability_family(100):
|
||||
)
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import ( # noqa: E402
|
||||
FlashInferDecodeKernel,
|
||||
FlashInferImpl,
|
||||
FlashInferMetadataBuilder,
|
||||
TRTLLMDecode,
|
||||
FlashInferTrtllmAPIDecode,
|
||||
TRTLLMPrefill,
|
||||
)
|
||||
|
||||
@@ -435,9 +436,11 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
|
||||
f"Expected TRTLLMPrefill, got {type(attn_metadata.prefill)}"
|
||||
)
|
||||
if has_decodes:
|
||||
assert isinstance(attn_metadata.decode, TRTLLMDecode), (
|
||||
f"Expected TRTLLMDecode, got {type(attn_metadata.decode)}"
|
||||
assert isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode), (
|
||||
"Expected FlashInferTrtllmAPIDecode, got "
|
||||
f"{type(attn_metadata.decode)}"
|
||||
)
|
||||
assert attn_metadata.decode.kernel == FlashInferDecodeKernel.TRTLLM_GEN
|
||||
|
||||
impl = FlashInferImpl(
|
||||
num_heads=num_q_heads,
|
||||
|
||||
Reference in New Issue
Block a user