[ROCm] Enable fp8 query for Triton attention (#60296)
Signed-off-by: xawang <xawang@amd.com> Co-authored-by: JartX <sagformas@epdcenter.es>
This commit is contained in:
@@ -1191,7 +1191,7 @@ class RocmPlatform(Platform):
|
||||
|
||||
@classmethod
|
||||
def supports_fp8(cls) -> bool:
|
||||
return on_cdna() or on_rdna4()
|
||||
return on_cdna() or on_rdna()
|
||||
|
||||
@classmethod
|
||||
def is_fp8_fnuz(cls) -> bool:
|
||||
|
||||
@@ -549,6 +549,11 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
self.use_alibi_sqrt = use_alibi_sqrt
|
||||
self.chunk_lookback = chunk_lookback
|
||||
self.supports_quant_query_input = current_platform.is_cuda()
|
||||
if current_platform.is_rocm():
|
||||
# Enable GPUs with fp8 MMA support
|
||||
from vllm.platforms.rocm import on_gfx11, on_gfx90a
|
||||
|
||||
self.supports_quant_query_input = not (on_gfx90a() or on_gfx11())
|
||||
|
||||
self._kv_quant_mode = get_kv_quant_mode(kv_cache_dtype)
|
||||
self._is_per_token_head_quant = self._kv_quant_mode.is_per_token_head
|
||||
|
||||
Reference in New Issue
Block a user