[Bugfix][ROCm] Fix MiniMax-M3 FP8 KV cache dtype (#45720)
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cameron Quilici <cjquilici@gmail.com> Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Hongxia Yang
TJian
parent
556b063e45
commit
efd15e192a
@@ -15,11 +15,13 @@ from vllm.models.minimax_m3.common.ops.index_topk import (
|
||||
minimax_m3_index_topk,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
_FP8_DTYPES,
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseBackend,
|
||||
MiniMaxM3SparseTritonImpl,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import set_kv_cache_layout
|
||||
@@ -79,6 +81,48 @@ SM_SCALE = HEAD_DIM**-0.5
|
||||
TOPK = 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kv_cache_dtype", "expected_dtype"),
|
||||
[
|
||||
("fp8", current_platform.fp8_dtype()),
|
||||
("fp8_e4m3", current_platform.fp8_dtype()),
|
||||
(
|
||||
"fp8_e5m2",
|
||||
torch.float8_e5m2fnuz
|
||||
if current_platform.is_fp8_fnuz()
|
||||
else torch.float8_e5m2,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sparse_impl_uses_platform_fp8_dtype(
|
||||
kv_cache_dtype: str,
|
||||
expected_dtype: torch.dtype,
|
||||
):
|
||||
impl = MiniMaxM3SparseTritonImpl(
|
||||
num_heads=NUM_Q_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
scale=SM_SCALE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
)
|
||||
assert impl.kv_cache_fp8_dtype == expected_dtype
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dtype",
|
||||
[
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e4m3fnuz,
|
||||
torch.float8_e5m2,
|
||||
torch.float8_e5m2fnuz,
|
||||
],
|
||||
)
|
||||
def test_sparse_kernels_recognize_fp8_dtypes(dtype: torch.dtype):
|
||||
assert dtype in _FP8_DTYPES
|
||||
|
||||
|
||||
# Index top-k kernels.
|
||||
def _reference_index_topk(
|
||||
idx_q: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user