[Attention][UX][1/N] Add AttentionConfig and change attention env vars to CLI arguments (#26315)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni001@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Harry Mellor
Lucas Wilkinson
parent
dff0a2b394
commit
66e674cdd5
@@ -8,7 +8,6 @@ from typing import ClassVar
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
AttentionImpl,
|
||||
@@ -264,6 +263,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.attention_config = vllm_config.attention_config
|
||||
|
||||
self.num_heads_q = self.model_config.get_num_attention_heads(
|
||||
self.parallel_config
|
||||
@@ -304,7 +304,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
# When using cuda graph, we need to set the upper bound of the
|
||||
# number of splits so that large enough intermediate buffers are
|
||||
# pre-allocated during capture.
|
||||
self.max_num_splits = envs.VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH
|
||||
self.max_num_splits = (
|
||||
self.attention_config.flash_attn_max_num_splits_for_cuda_graph
|
||||
)
|
||||
|
||||
# Sliding window size to be used with the AOT scheduler will be
|
||||
# populated on first build() call.
|
||||
@@ -554,8 +556,7 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
"heads in the layer"
|
||||
)
|
||||
|
||||
def supports_quant_query_input(self) -> bool:
|
||||
return True
|
||||
self.supports_quant_query_input = True
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.attention.backends.abstract import (
|
||||
)
|
||||
from vllm.attention.ops.common import cp_lse_ag_out_rs
|
||||
from vllm.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
from vllm.logger import init_logger
|
||||
@@ -43,7 +43,6 @@ from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.flashinfer import (
|
||||
can_use_trtllm_attention,
|
||||
flashinfer_disable_q_quantization,
|
||||
use_trtllm_attention,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -362,7 +361,8 @@ class FlashInferBackend(AttentionBackend):
|
||||
supports_trtllm_attention,
|
||||
)
|
||||
|
||||
# Respect explicit disable flag (e.g., VLLM_USE_TRTLLM_ATTENTION=0)
|
||||
# Respect explicit disable flag (e.g.,
|
||||
# --attention-config.use_trtllm_attention=0)
|
||||
if force_use_trtllm_attention() is False:
|
||||
return False
|
||||
|
||||
@@ -500,11 +500,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
self.kv_cache_dtype = self.kv_cache_spec.dtype
|
||||
|
||||
# Use model dtype as q dtype when TRTLLM attn is not supported, or
|
||||
# VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION is set to 1. Otherwise, try to
|
||||
# use fp8 q if kv cache is fp8, and will fall back to model dtype
|
||||
# --attention-config.disable_flashinfer_q_quantization is set to 1. Otherwise,
|
||||
# try to use fp8 q if kv cache is fp8, and will fall back to model dtype
|
||||
# if TRTLLM attention kernel is not used when building attn metadata
|
||||
can_use_trtllm = can_use_trtllm_attention(self.num_qo_heads, self.num_kv_heads)
|
||||
if can_use_trtllm and not flashinfer_disable_q_quantization():
|
||||
if (
|
||||
can_use_trtllm
|
||||
and not vllm_config.attention_config.disable_flashinfer_q_quantization
|
||||
):
|
||||
self.q_data_type = self.kv_cache_dtype
|
||||
else:
|
||||
self.q_data_type = self.model_config.dtype
|
||||
@@ -1035,6 +1038,11 @@ class FlashInferImpl(AttentionImpl):
|
||||
self.sinks = sinks
|
||||
|
||||
self.support_trtllm_attn = can_use_trtllm_attention(num_heads, num_kv_heads)
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.supports_quant_query_input = (
|
||||
self.support_trtllm_attn
|
||||
and not vllm_config.attention_config.disable_flashinfer_q_quantization
|
||||
)
|
||||
self.bmm1_scale: float | None = None
|
||||
self.bmm2_scale: float | None = None
|
||||
self.o_sf_scale: float | None = None
|
||||
@@ -1046,12 +1054,6 @@ class FlashInferImpl(AttentionImpl):
|
||||
and quant_key in (kFp8StaticTensorSym, kNvfp4Quant)
|
||||
)
|
||||
|
||||
def supports_quant_query_input(self) -> bool:
|
||||
if flashinfer_disable_q_quantization():
|
||||
return False
|
||||
|
||||
return self.support_trtllm_attn
|
||||
|
||||
# FlashInfer requires attention sinks to be float32
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
if self.sinks is not None and self.sinks.dtype != torch.float32:
|
||||
|
||||
@@ -438,19 +438,25 @@ A = TypeVar("A")
|
||||
def use_flashinfer_prefill() -> bool:
|
||||
# For blackwell default to flashinfer prefill if it's available since
|
||||
# it is faster than FA2.
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
return (
|
||||
not envs.VLLM_DISABLE_FLASHINFER_PREFILL
|
||||
not vllm_config.attention_config.disable_flashinfer_prefill
|
||||
and flashinfer_available
|
||||
and not envs.VLLM_USE_CUDNN_PREFILL
|
||||
and not envs.VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL
|
||||
and not vllm_config.attention_config.use_cudnn_prefill
|
||||
and not vllm_config.attention_config.use_trtllm_ragged_deepseek_prefill
|
||||
and current_platform.is_device_capability(100)
|
||||
)
|
||||
|
||||
|
||||
def use_cudnn_prefill() -> bool:
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
return (
|
||||
flashinfer_available
|
||||
and envs.VLLM_USE_CUDNN_PREFILL
|
||||
and vllm_config.attention_config.use_cudnn_prefill
|
||||
and current_platform.is_device_capability(100)
|
||||
and has_nvidia_artifactory()
|
||||
)
|
||||
@@ -458,9 +464,12 @@ def use_cudnn_prefill() -> bool:
|
||||
|
||||
def use_trtllm_ragged_deepseek_prefill() -> bool:
|
||||
"""Check if TRT-LLM ragged DeepSeek prefill should be used."""
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
return (
|
||||
flashinfer_available
|
||||
and envs.VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL
|
||||
and vllm_config.attention_config.use_trtllm_ragged_deepseek_prefill
|
||||
and current_platform.is_device_capability(100)
|
||||
)
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.attention.backends.abstract import (
|
||||
AttentionLayer,
|
||||
AttentionType,
|
||||
@@ -131,7 +130,9 @@ class FlashAttnMLAMetadataBuilder(MLACommonMetadataBuilder[FlashAttnMLAMetadata]
|
||||
# When using cuda graph, we need to set the upper bound of the
|
||||
# number of splits so that large enough intermediate buffers are
|
||||
# pre-allocated during capture.
|
||||
self.max_num_splits = envs.VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH
|
||||
self.max_num_splits = (
|
||||
vllm_config.attention_config.flash_attn_max_num_splits_for_cuda_graph
|
||||
)
|
||||
|
||||
if vllm_is_batch_invariant():
|
||||
self.max_num_splits = 1
|
||||
|
||||
@@ -165,7 +165,7 @@ class RocmAttentionBackend(AttentionBackend):
|
||||
raise ValueError(
|
||||
f"Head size {head_size} is not supported by {attn_type}. "
|
||||
f"Supported head sizes are: {cls.get_supported_head_sizes()}. "
|
||||
"Set VLLM_ATTENTION_BACKEND=FLEX_ATTENTION to use "
|
||||
"Set --attention-config.backend=FLEX_ATTENTION to use "
|
||||
"FlexAttention backend which supports all head sizes."
|
||||
)
|
||||
|
||||
|
||||
@@ -210,9 +210,6 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
def fused_output_quant_supported(self, quant_key: QuantKey):
|
||||
return quant_key == kFp8StaticTensorSym
|
||||
|
||||
def supports_quant_query_input(self) -> bool:
|
||||
return current_platform.is_cuda()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
@@ -262,6 +259,8 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
f"num_heads: {num_heads}."
|
||||
)
|
||||
|
||||
self.supports_quant_query_input = current_platform.is_cuda()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
Reference in New Issue
Block a user