Move query quantization to attention layer for Flashinfer & Triton. (#26534)
Signed-off-by: adabeyta <aabeyta@redhat.com> Signed-off-by: Adrian Abeyta <aabeyta@redhat.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
e5b438a247
commit
0a9ef0cfce
@@ -41,14 +41,6 @@ class AttentionBackend(ABC):
|
||||
# makes sure the output tensor is allocated inside the cudagraph.
|
||||
accept_output_buffer: bool = False
|
||||
|
||||
# Whether this backend supports receiving pre-quantized query input.
|
||||
# If True, the attention layer will handle query quantization instead
|
||||
# of the backend, allowing torch.compile to fuse quantization with
|
||||
# previous operations.
|
||||
# Needs to be worked through for all backends
|
||||
# https://github.com/vllm-project/vllm/issues/25584
|
||||
supports_quant_query_input: bool = False
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_name() -> str:
|
||||
@@ -199,6 +191,22 @@ class AttentionImpl(ABC, Generic[T]):
|
||||
"""
|
||||
return False
|
||||
|
||||
def supports_quant_query_input(self) -> bool:
|
||||
"""
|
||||
Check if this attention implementation supports pre-quantized query input.
|
||||
|
||||
When True, the attention layer will quantize queries before passing them
|
||||
to this backend, allowing torch.compile to fuse the quantization with
|
||||
previous operations. This is typically supported when using FP8 KV cache
|
||||
with compatible attention kernels (e.g., TRT-LLM).
|
||||
TODO add support to more backends:
|
||||
https://github.com/vllm-project/vllm/issues/25584
|
||||
|
||||
Returns:
|
||||
bool: True if the implementation can accept pre-quantized queries.
|
||||
"""
|
||||
return False
|
||||
|
||||
|
||||
class MLAAttentionImpl(AttentionImpl[T], Generic[T]):
|
||||
@abstractmethod
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.model_executor.models.vision import get_vit_attn_backend
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import GiB_bytes, direct_register_custom_op
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
logger = init_logger(__name__)
|
||||
USE_XFORMERS_OPS = None
|
||||
|
||||
@@ -304,7 +305,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
self.query_quant = None
|
||||
if (
|
||||
self.kv_cache_dtype.startswith("fp8")
|
||||
and self.attn_backend.supports_quant_query_input
|
||||
and self.impl.supports_quant_query_input()
|
||||
):
|
||||
self.query_quant = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
|
||||
@@ -329,7 +330,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
"""
|
||||
if self.calculate_kv_scales:
|
||||
torch.ops.vllm.maybe_calc_kv_scales(query, key, value, self.layer_name)
|
||||
|
||||
output_dtype = query.dtype
|
||||
if self.query_quant is not None:
|
||||
# quantizing with a simple torch operation enables
|
||||
@@ -338,7 +338,10 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# Otherwise queries are quantized using custom ops
|
||||
# which causes decoding overheads
|
||||
assert self.kv_cache_dtype in {"fp8", "fp8_e4m3"}
|
||||
query, _ = self.query_quant(query, self._q_scale)
|
||||
|
||||
# check if query quantization is supported
|
||||
if self.impl.supports_quant_query_input():
|
||||
query, _ = self.query_quant(query, self._q_scale)
|
||||
|
||||
if self.use_output:
|
||||
output_shape = output_shape if output_shape is not None else query.shape
|
||||
|
||||
Reference in New Issue
Block a user