[AMD][Minimax-M3][perf] Enable packed LBHNC AITER QK-norm fusion for MiniMax-M3 on ROCm (#54535)
Signed-off-by: Wei-Ting Liao <wei-ting.liao@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Cursor
Shanshan Shen
parent
f05b88751e
commit
b8cf275382
@@ -19,7 +19,10 @@ from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.sparse_attention import MiniMaxM3SparseTritonImpl
|
||||
from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseTritonImpl,
|
||||
minimax_m3_rebase_slots_to_page16,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.utils import record_kv_cache_layout
|
||||
@@ -28,6 +31,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheLayout,
|
||||
MLAAttentionSpec,
|
||||
compute_layer_kv_cache_shape_bytes,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
||||
@@ -1672,8 +1676,9 @@ def test_main_cache_layout_contract():
|
||||
assert set(order) == set(range(len(order)))
|
||||
|
||||
|
||||
def test_aiter_sparse_pa_layout_contract(monkeypatch):
|
||||
"""The shuffle-only AITER path retains separately contiguous K/V storage."""
|
||||
@pytest.mark.parametrize("layout", [KVCacheLayout.LHBNC, KVCacheLayout.LBHNC])
|
||||
def test_aiter_sparse_pa_layout_contract(monkeypatch, layout):
|
||||
"""Build the exact separate and packed page-16 views used by the model."""
|
||||
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
|
||||
|
||||
monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True)
|
||||
@@ -1685,21 +1690,43 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch):
|
||||
|
||||
backend = sparse_attn_mod.MiniMaxM3SparseBackend
|
||||
assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts()
|
||||
assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts()
|
||||
|
||||
nb, h = 7, 1
|
||||
nb, h = 3, 1
|
||||
cache_dtype = current_platform.fp8_dtype()
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=h,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
dtype=cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode("fp8"),
|
||||
num_head_slots=2,
|
||||
state_content_bytes=h * HEAD_DIM * DTYPE.itemsize,
|
||||
state_content_bytes=h * HEAD_DIM * cache_dtype.itemsize,
|
||||
)
|
||||
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
||||
view = dense_kv_cache_views(raw, spec, nb, 1, KVCacheLayout.LHBNC)[0]
|
||||
key_cache, value_cache = view.unbind(1)
|
||||
assert key_cache.is_contiguous()
|
||||
assert value_cache.is_contiguous()
|
||||
kv_cache = dense_kv_cache_views(raw, spec, nb, 1, layout)[0].view(cache_dtype)
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
x = 16 // cache_dtype.itemsize
|
||||
pages_per_side = BLOCK_SIZE // 16
|
||||
if key_cache.is_contiguous() and value_cache.is_contiguous():
|
||||
k_src, v_src = key_cache, value_cache
|
||||
block_pages, v_page_offset = pages_per_side, 0
|
||||
else:
|
||||
assert kv_cache.is_contiguous()
|
||||
k_src = v_src = kv_cache
|
||||
block_pages, v_page_offset = 2 * pages_per_side, pages_per_side
|
||||
|
||||
num_phys16 = nb * block_pages
|
||||
k_view = k_src.view(num_phys16, h, HEAD_DIM // x, 16, x)
|
||||
v_view = v_src.view(num_phys16, h, 16 // x, HEAD_DIM, x)[v_page_offset:]
|
||||
assert k_view.is_contiguous()
|
||||
assert v_view.is_contiguous()
|
||||
if layout is KVCacheLayout.LHBNC:
|
||||
assert k_view.shape == (24, 1, 8, 16, 16)
|
||||
assert v_view.shape == (24, 1, 1, 128, 16)
|
||||
else:
|
||||
assert k_view.shape == (48, 1, 8, 16, 16)
|
||||
assert v_view.shape == (40, 1, 1, 128, 16)
|
||||
|
||||
|
||||
def test_aiter_sparse_pa_rejects_multiple_kv_heads(monkeypatch):
|
||||
@@ -1769,6 +1796,330 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
|
||||
assert reason is None or "AITER sparse PA attend" not in reason
|
||||
|
||||
|
||||
def test_aiter_consolidated_qknorm_enabled(monkeypatch):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
|
||||
|
||||
scale = torch.tensor([0.25])
|
||||
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
|
||||
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8", scale, scale)
|
||||
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e4m3", scale, scale)
|
||||
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8")
|
||||
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8", scale, None)
|
||||
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e5m2", scale, scale)
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", False)
|
||||
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
|
||||
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: False)
|
||||
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
|
||||
|
||||
|
||||
def _fake_aiter_qknorm_args(kv_cache_dtype: str = "auto") -> dict:
|
||||
tensor = torch.empty(1)
|
||||
return {
|
||||
"qkv": tensor,
|
||||
"q_norm_weight": tensor,
|
||||
"k_norm_weight": tensor,
|
||||
"cos_sin_cache": tensor,
|
||||
"positions": tensor,
|
||||
"num_heads": 16,
|
||||
"num_kv_heads": 1,
|
||||
"rotary_dim": 64,
|
||||
"eps": 1e-6,
|
||||
"slot_mapping": tensor,
|
||||
"kv_cache_k": tensor,
|
||||
"kv_cache_v": tensor,
|
||||
"q_out": tensor,
|
||||
"kv_cache_dtype": kv_cache_dtype,
|
||||
}
|
||||
|
||||
|
||||
def _install_fake_aiter_qknorm(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
|
||||
"""Stub ``aiter.fused_qknorm_idxrqknorm`` so mapping tests run without AITER."""
|
||||
import sys
|
||||
import types
|
||||
from typing import Any
|
||||
|
||||
calls: list[dict] = []
|
||||
|
||||
def fake_op(*args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
|
||||
existing = sys.modules.get("aiter")
|
||||
if existing is not None:
|
||||
monkeypatch.setattr(existing, "fused_qknorm_idxrqknorm", fake_op, raising=False)
|
||||
else:
|
||||
fake_mod: Any = types.ModuleType("aiter")
|
||||
fake_mod.fused_qknorm_idxrqknorm = fake_op
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_mod)
|
||||
return calls
|
||||
|
||||
|
||||
def test_aiter_consolidated_qknorm_packed_shuffle_unequal_spans(monkeypatch):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
calls = _install_fake_aiter_qknorm(monkeypatch)
|
||||
call_args = _fake_aiter_qknorm_args()
|
||||
call_args["kv_cache_k"] = torch.empty(2, 1)
|
||||
call_args["kv_cache_v"] = torch.empty(1, 1)
|
||||
|
||||
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kv_cache_dtype", "aiter_dtype", "passes_scales"),
|
||||
[
|
||||
("auto", "auto", False),
|
||||
("fp8", "fp8_e4m3_static", True),
|
||||
("fp8_e4m3", "fp8_e4m3_static", True),
|
||||
],
|
||||
)
|
||||
def test_aiter_consolidated_qknorm_dtype_and_layout_mapping(
|
||||
monkeypatch, kv_cache_dtype, aiter_dtype, passes_scales
|
||||
):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
calls = _install_fake_aiter_qknorm(monkeypatch)
|
||||
call_args = _fake_aiter_qknorm_args(kv_cache_dtype)
|
||||
k_scale = torch.tensor([0.25])
|
||||
v_scale = torch.tensor([0.5])
|
||||
call_args.update(k_scale=k_scale, v_scale=v_scale)
|
||||
|
||||
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
|
||||
assert len(calls) == 1
|
||||
kwargs = calls[0]
|
||||
assert kwargs["kv_cache_dtype"] == aiter_dtype
|
||||
assert kwargs["index_cache_dtype"] == "auto"
|
||||
assert kwargs["block_size"] == 16
|
||||
assert kwargs["asm_layout"] is True
|
||||
if passes_scales:
|
||||
assert kwargs["k_scale"] is k_scale
|
||||
assert kwargs["v_scale"] is v_scale
|
||||
else:
|
||||
assert kwargs["k_scale"] is None
|
||||
assert kwargs["v_scale"] is None
|
||||
|
||||
|
||||
def test_aiter_consolidated_qknorm_fp8_index_dtype_mapping(monkeypatch):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
calls = _install_fake_aiter_qknorm(monkeypatch)
|
||||
fp8 = current_platform.fp8_dtype()
|
||||
call_args = _fake_aiter_qknorm_args("fp8")
|
||||
call_args.update(
|
||||
k_scale=torch.tensor([0.25]),
|
||||
v_scale=torch.tensor([0.5]),
|
||||
index_cache=torch.empty(1, dtype=fp8),
|
||||
index_q_out=torch.empty(1, dtype=fp8),
|
||||
num_index_heads=1,
|
||||
)
|
||||
|
||||
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
|
||||
assert calls[0]["index_cache_dtype"] == "fp8"
|
||||
assert calls[0]["kv_cache_dtype"] == "fp8_e4m3_static"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("skip_index_branch", [False, True])
|
||||
def test_aiter_consolidated_qknorm_full_and_skip_index_args(
|
||||
monkeypatch, skip_index_branch
|
||||
):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
calls = _install_fake_aiter_qknorm(monkeypatch)
|
||||
call_args = _fake_aiter_qknorm_args()
|
||||
if skip_index_branch:
|
||||
call_args.update(skip_index_branch=True)
|
||||
else:
|
||||
tensor = torch.empty(1)
|
||||
call_args.update(
|
||||
index_q_norm_weight=tensor,
|
||||
index_k_norm_weight=tensor,
|
||||
num_index_heads=1,
|
||||
index_cache=tensor,
|
||||
index_q_out=tensor,
|
||||
index_slot_mapping=tensor,
|
||||
)
|
||||
|
||||
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
|
||||
kwargs = calls[0]
|
||||
assert kwargs["skip_index_branch"] is skip_index_branch
|
||||
if skip_index_branch:
|
||||
assert kwargs["index_q_norm_weight"] is None
|
||||
assert kwargs["index_k_norm_weight"] is None
|
||||
assert kwargs["index_cache"] is None
|
||||
assert kwargs["index_q_out"] is None
|
||||
assert kwargs["index_slot_mapping"] is None
|
||||
else:
|
||||
assert kwargs["index_q_norm_weight"] is not None
|
||||
assert kwargs["index_k_norm_weight"] is not None
|
||||
assert kwargs["index_cache"] is not None
|
||||
assert kwargs["index_q_out"] is not None
|
||||
assert kwargs["index_slot_mapping"] is not None
|
||||
|
||||
|
||||
def test_aiter_consolidated_qknorm_rejects_fp8_e5m2(monkeypatch):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
|
||||
scale = torch.tensor([0.25])
|
||||
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e5m2", scale, scale)
|
||||
|
||||
|
||||
def test_aiter_sparse_pa_slot_rebase_preserves_page_offsets():
|
||||
slots = torch.tensor([0, 15, 16, 127, 128, 129, -1])
|
||||
actual = minimax_m3_rebase_slots_to_page16(slots, block_size=128)
|
||||
expected = torch.tensor([0, 15, 16, 127, 256, 257, -1])
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="The consolidated packed-SHUFFLE AITER op is ROCm-only",
|
||||
)
|
||||
@pytest.mark.parametrize("skip_index_branch", [False, True])
|
||||
@pytest.mark.parametrize("fp8_index", [False, True])
|
||||
def test_aiter_consolidated_qknorm_real_packed_shuffle(skip_index_branch, fp8_index):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
torch.manual_seed(20260831)
|
||||
num_tokens = 4
|
||||
num_heads = 16
|
||||
num_kv_heads = 1
|
||||
num_index_heads = 1
|
||||
rotary_dim = 64
|
||||
head_dim = 128
|
||||
logical_block_size = 128
|
||||
num_blocks = 3
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
qkv = torch.randn(
|
||||
num_tokens,
|
||||
(num_heads + 2 * num_kv_heads + num_index_heads + 1) * head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
weights = [
|
||||
torch.randn(head_dim, dtype=torch.bfloat16, device="cuda") for _ in range(4)
|
||||
]
|
||||
cos_sin_cache = torch.randn(16, rotary_dim, dtype=torch.bfloat16, device="cuda")
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device="cuda")
|
||||
logical_slots = torch.tensor([0, 127, 128, -1], dtype=torch.int64, device="cuda")
|
||||
slot_mapping = minimax_m3_rebase_slots_to_page16(logical_slots, logical_block_size)
|
||||
index_slot_mapping = logical_slots.clone()
|
||||
|
||||
packed = torch.zeros(
|
||||
num_blocks,
|
||||
2,
|
||||
logical_block_size,
|
||||
head_dim,
|
||||
dtype=fp8_dtype,
|
||||
device="cuda",
|
||||
)
|
||||
x = 16 // packed.element_size()
|
||||
pages_per_side = logical_block_size // 16
|
||||
num_phys16 = num_blocks * 2 * pages_per_side
|
||||
kv_cache_k = packed.view(num_phys16, num_kv_heads, head_dim // x, 16, x)
|
||||
kv_cache_v = packed.view(num_phys16, num_kv_heads, 16 // x, head_dim, x)[
|
||||
pages_per_side:
|
||||
]
|
||||
assert kv_cache_k.shape == (48, 1, 8, 16, 16)
|
||||
assert kv_cache_v.shape == (40, 1, 1, 128, 16)
|
||||
|
||||
q_out = torch.zeros(
|
||||
num_tokens, num_heads * head_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
index_dtype = fp8_dtype if fp8_index else torch.bfloat16
|
||||
if fp8_index:
|
||||
index_q_out = torch.empty(
|
||||
(num_tokens, num_index_heads * head_dim),
|
||||
dtype=index_dtype,
|
||||
device="cuda",
|
||||
)
|
||||
index_cache = torch.empty(
|
||||
(num_blocks, logical_block_size, head_dim),
|
||||
dtype=index_dtype,
|
||||
device="cuda",
|
||||
)
|
||||
index_q_out.view(torch.uint8).fill_(7)
|
||||
index_cache.view(torch.uint8).fill_(7)
|
||||
else:
|
||||
index_q_out = torch.full(
|
||||
(num_tokens, num_index_heads * head_dim),
|
||||
7,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
index_cache = torch.full(
|
||||
(num_blocks, logical_block_size, head_dim),
|
||||
7,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
scale = torch.tensor([0.02], dtype=torch.float32, device="cuda")
|
||||
|
||||
kwargs = {}
|
||||
if not skip_index_branch:
|
||||
kwargs = {
|
||||
"index_q_norm_weight": weights[2],
|
||||
"index_k_norm_weight": weights[3],
|
||||
"index_cache": index_cache,
|
||||
"index_q_out": index_q_out,
|
||||
"index_slot_mapping": index_slot_mapping,
|
||||
}
|
||||
rocm_aiter_ops.fused_qknorm_idxrqknorm(
|
||||
qkv,
|
||||
weights[0],
|
||||
weights[1],
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim,
|
||||
1e-6,
|
||||
slot_mapping,
|
||||
kv_cache_k,
|
||||
kv_cache_v,
|
||||
q_out,
|
||||
"fp8",
|
||||
scale,
|
||||
scale,
|
||||
num_index_heads=num_index_heads,
|
||||
skip_index_branch=skip_index_branch,
|
||||
**kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.isfinite(q_out).all()
|
||||
assert torch.count_nonzero(q_out) > 0
|
||||
assert torch.count_nonzero(packed) > 0
|
||||
if fp8_index:
|
||||
index_q_untouched = torch.all(index_q_out.view(torch.uint8) == 7)
|
||||
index_cache_untouched = torch.all(index_cache.view(torch.uint8) == 7)
|
||||
index_cache_t0 = torch.all(index_cache[0, 0].view(torch.uint8) == 7)
|
||||
index_cache_t127 = torch.all(index_cache[0, 127].view(torch.uint8) == 7)
|
||||
index_cache_t128 = torch.all(index_cache[1, 0].view(torch.uint8) == 7)
|
||||
else:
|
||||
index_q_untouched = torch.all(index_q_out == 7)
|
||||
index_cache_untouched = torch.all(index_cache == 7)
|
||||
index_cache_t0 = torch.all(index_cache[0, 0] == 7)
|
||||
index_cache_t127 = torch.all(index_cache[0, 127] == 7)
|
||||
index_cache_t128 = torch.all(index_cache[1, 0] == 7)
|
||||
if skip_index_branch:
|
||||
assert index_q_untouched
|
||||
assert index_cache_untouched
|
||||
else:
|
||||
assert not index_q_untouched
|
||||
assert not index_cache_t0
|
||||
assert not index_cache_t127
|
||||
assert not index_cache_t128
|
||||
|
||||
|
||||
def test_indexer_cache_squeezes_to_contiguous_3d():
|
||||
"""The indexer side cache is standardized 4D with H=1: under both layouts
|
||||
the allocator's logical view stays contiguous and squeezes (as
|
||||
|
||||
@@ -35,10 +35,22 @@ except ImportError:
|
||||
# which is a host op, so we cache it once here.
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
_HIPB_MM_INITIALIZED_DEVICES: set[int] = set()
|
||||
_FP8_E4M3_DTYPES = {
|
||||
dtype
|
||||
for dtype in (
|
||||
getattr(torch, "float8_e4m3fn", None),
|
||||
getattr(torch, "float8_e4m3fnuz", None),
|
||||
)
|
||||
if dtype is not None
|
||||
}
|
||||
KB = 1024
|
||||
MB = 1024 * KB
|
||||
|
||||
|
||||
def _is_fp8_e4m3_tensor(t: torch.Tensor | None) -> bool:
|
||||
return t is not None and t.dtype in _FP8_E4M3_DTYPES
|
||||
|
||||
|
||||
def _get_or_create_aiter_qr_rmsnorm_comm(
|
||||
device_comm: Any,
|
||||
) -> tuple[int, int, bool] | None:
|
||||
@@ -2200,6 +2212,106 @@ class rocm_aiter_ops:
|
||||
def is_shuffle_kv_cache_enabled(cls) -> bool:
|
||||
return cls._SHUFFLE_KV_CACHE_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def fused_qknorm_idxrqknorm_enabled(
|
||||
cls,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
) -> bool:
|
||||
"""Whether MiniMax-M3 can use AITER's consolidated fused QK-norm.
|
||||
|
||||
Requires AITER to be installed and enabled. bf16 (``auto``) always
|
||||
qualifies; fp8 e4m3 also needs K/V scales. Other cache dtypes fall
|
||||
back to vLLM's fused kernel.
|
||||
"""
|
||||
if not cls._AITER_ENABLED:
|
||||
return False
|
||||
if kv_cache_dtype == "auto":
|
||||
return True
|
||||
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
return k_scale is not None and v_scale is not None
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def fused_qknorm_idxrqknorm(
|
||||
cls,
|
||||
qkv: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
rotary_dim: int,
|
||||
eps: float,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache_k: torch.Tensor,
|
||||
kv_cache_v: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
index_q_norm_weight: torch.Tensor | None = None,
|
||||
index_k_norm_weight: torch.Tensor | None = None,
|
||||
num_index_heads: int = 0,
|
||||
index_cache: torch.Tensor | None = None,
|
||||
index_q_out: torch.Tensor | None = None,
|
||||
index_slot_mapping: torch.Tensor | None = None,
|
||||
skip_index_branch: bool = False,
|
||||
) -> None:
|
||||
"""Run consolidated MiniMax-M3 QK-norm fusion.
|
||||
|
||||
Callers must check ``fused_qknorm_idxrqknorm_enabled`` first. Runtime
|
||||
failures from the AITER op are deliberately propagated.
|
||||
"""
|
||||
if kv_cache_dtype == "auto":
|
||||
aiter_kv_cache_dtype = "auto"
|
||||
aiter_k_scale = None
|
||||
aiter_v_scale = None
|
||||
elif kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
aiter_kv_cache_dtype = "fp8_e4m3_static"
|
||||
aiter_k_scale = k_scale
|
||||
aiter_v_scale = v_scale
|
||||
else:
|
||||
raise ValueError(
|
||||
"AITER fused QK-norm requires kv_cache_dtype 'auto', 'fp8', "
|
||||
f"or 'fp8_e4m3', got {kv_cache_dtype!r}"
|
||||
)
|
||||
aiter_index_cache_dtype = "fp8" if _is_fp8_e4m3_tensor(index_cache) else "auto"
|
||||
|
||||
from aiter import fused_qknorm_idxrqknorm
|
||||
|
||||
fused_qknorm_idxrqknorm(
|
||||
qkv,
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
index_q_norm_weight=index_q_norm_weight,
|
||||
index_k_norm_weight=index_k_norm_weight,
|
||||
num_index_heads=num_index_heads,
|
||||
slot_mapping=slot_mapping,
|
||||
kv_cache_k=kv_cache_k,
|
||||
kv_cache_v=kv_cache_v,
|
||||
index_cache=index_cache,
|
||||
block_size=16,
|
||||
q_out=q_out,
|
||||
index_q_out=index_q_out,
|
||||
index_slot_mapping=index_slot_mapping,
|
||||
kv_cache_dtype=aiter_kv_cache_dtype,
|
||||
index_cache_dtype=aiter_index_cache_dtype,
|
||||
k_scale=aiter_k_scale,
|
||||
v_scale=aiter_v_scale,
|
||||
asm_layout=True,
|
||||
skip_index_branch=skip_index_branch,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_triton_unified_attn_enabled(cls) -> bool:
|
||||
|
||||
@@ -890,6 +890,30 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
self._ensure_aiter_sparse_pa_kv_cache()
|
||||
return self.kv_cache_k, self.kv_cache_v
|
||||
|
||||
def _get_aiter_sparse_pa_slot_mapping(
|
||||
self,
|
||||
slot_mapping: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if value_cache.shape[0] == key_cache.shape[0]:
|
||||
return slot_mapping
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
page16_slot_mapping = None
|
||||
if isinstance(attn_metadata, dict):
|
||||
main_md = attn_metadata[self.layer_name]
|
||||
assert isinstance(main_md, MiniMaxM3SparseMetadata)
|
||||
page16_slot_mapping = main_md.page16_slot_mapping
|
||||
if (
|
||||
page16_slot_mapping is None
|
||||
or page16_slot_mapping.shape != slot_mapping.shape
|
||||
):
|
||||
page16_slot_mapping = minimax_m3_rebase_slots_to_page16(
|
||||
slot_mapping, self.kv_cache.shape[2]
|
||||
)
|
||||
return page16_slot_mapping
|
||||
|
||||
def _insert_aiter_sparse_pa_kv(
|
||||
self,
|
||||
k: torch.Tensor,
|
||||
@@ -897,6 +921,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
index_k: torch.Tensor | None,
|
||||
slot_mapping: torch.Tensor,
|
||||
index_slot_mapping: torch.Tensor | None,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
) -> None:
|
||||
if self.kv_cache.numel() == 0:
|
||||
return
|
||||
@@ -906,22 +932,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
minimax_m3_insert_index_cache,
|
||||
)
|
||||
|
||||
key_cache, value_cache = self.get_aiter_sparse_pa_kv_cache()
|
||||
if value_cache.shape[0] != key_cache.shape[0]:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
page16_slot_mapping = None
|
||||
if isinstance(attn_metadata, dict):
|
||||
main_md = attn_metadata[self.layer_name]
|
||||
assert isinstance(main_md, MiniMaxM3SparseMetadata)
|
||||
page16_slot_mapping = main_md.page16_slot_mapping
|
||||
if (
|
||||
page16_slot_mapping is None
|
||||
or page16_slot_mapping.shape != slot_mapping.shape
|
||||
):
|
||||
page16_slot_mapping = minimax_m3_rebase_slots_to_page16(
|
||||
slot_mapping, self.kv_cache.shape[2]
|
||||
)
|
||||
slot_mapping = page16_slot_mapping
|
||||
kv_cache_dtype = (
|
||||
self.kv_cache_dtype
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
@@ -976,38 +986,78 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
main_slot_mapping = fwd_slot_mapping[self.layer_name]
|
||||
q = qkv.new_empty((num_tokens, self.q_size))
|
||||
key_cache = value_cache = page16_slot_mapping = None
|
||||
k_scale = v_scale = None
|
||||
use_fused_qknorm = False
|
||||
if self.use_aiter_sparse_pa:
|
||||
key_cache, value_cache = self.get_aiter_sparse_pa_kv_cache()
|
||||
page16_slot_mapping = self._get_aiter_sparse_pa_slot_mapping(
|
||||
main_slot_mapping, key_cache, value_cache
|
||||
)
|
||||
k_scale = getattr(self, "_k_scale", None)
|
||||
v_scale = getattr(self, "_v_scale", None)
|
||||
use_fused_qknorm = rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled(
|
||||
self.kv_cache_dtype, k_scale, v_scale
|
||||
)
|
||||
if self.skip_index_topk:
|
||||
index_q = None
|
||||
if self.use_aiter_sparse_pa:
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
num_index_heads=self.num_idx_heads,
|
||||
q_out=q,
|
||||
skip_index_branch=True,
|
||||
)
|
||||
k_start = self.q_size
|
||||
v_start = k_start + self.kv_size
|
||||
k = qkv[:, k_start:v_start].view(
|
||||
num_tokens, self.num_kv_heads, self.head_dim
|
||||
)
|
||||
v = qkv[:, v_start : v_start + self.kv_size].view(
|
||||
num_tokens, self.num_kv_heads, self.head_dim
|
||||
)
|
||||
self._insert_aiter_sparse_pa_kv(
|
||||
k,
|
||||
v,
|
||||
None,
|
||||
main_slot_mapping,
|
||||
None,
|
||||
)
|
||||
assert key_cache is not None
|
||||
assert value_cache is not None
|
||||
assert page16_slot_mapping is not None
|
||||
if use_fused_qknorm:
|
||||
rocm_aiter_ops.fused_qknorm_idxrqknorm(
|
||||
qkv,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
page16_slot_mapping,
|
||||
key_cache,
|
||||
value_cache,
|
||||
q,
|
||||
self.kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
num_index_heads=self.num_idx_heads,
|
||||
skip_index_branch=True,
|
||||
)
|
||||
else:
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
num_index_heads=self.num_idx_heads,
|
||||
q_out=q,
|
||||
skip_index_branch=True,
|
||||
)
|
||||
k_start = self.q_size
|
||||
v_start = k_start + self.kv_size
|
||||
k = qkv[:, k_start:v_start].view(
|
||||
num_tokens, self.num_kv_heads, self.head_dim
|
||||
)
|
||||
v = qkv[:, v_start : v_start + self.kv_size].view(
|
||||
num_tokens, self.num_kv_heads, self.head_dim
|
||||
)
|
||||
self._insert_aiter_sparse_pa_kv(
|
||||
k,
|
||||
v,
|
||||
None,
|
||||
page16_slot_mapping,
|
||||
None,
|
||||
key_cache,
|
||||
value_cache,
|
||||
)
|
||||
else:
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
@@ -1037,42 +1087,74 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
dtype=self.indexer.index_cache.dtype,
|
||||
)
|
||||
if self.use_aiter_sparse_pa:
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
self.index_q_norm.weight,
|
||||
self.index_k_norm.weight,
|
||||
self.num_idx_heads,
|
||||
q_out=q,
|
||||
index_q_out=index_q,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
k_start = self.q_size
|
||||
v_start = k_start + self.kv_size
|
||||
index_k_start = v_start + self.kv_size + self.index_q_size
|
||||
k = qkv[:, k_start:v_start].view(
|
||||
num_tokens, self.num_kv_heads, self.head_dim
|
||||
)
|
||||
v = qkv[:, v_start : v_start + self.kv_size].view(
|
||||
num_tokens, self.num_kv_heads, self.head_dim
|
||||
)
|
||||
index_k = qkv[
|
||||
:, index_k_start : index_k_start + self.idx_head_dim
|
||||
].view(num_tokens, self.idx_head_dim)
|
||||
self._insert_aiter_sparse_pa_kv(
|
||||
k,
|
||||
v,
|
||||
index_k,
|
||||
main_slot_mapping,
|
||||
index_slot_mapping,
|
||||
)
|
||||
assert key_cache is not None
|
||||
assert value_cache is not None
|
||||
assert page16_slot_mapping is not None
|
||||
index_cache = self.indexer.index_cache.kv_cache
|
||||
if use_fused_qknorm:
|
||||
rocm_aiter_ops.fused_qknorm_idxrqknorm(
|
||||
qkv,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
page16_slot_mapping,
|
||||
key_cache,
|
||||
value_cache,
|
||||
q,
|
||||
self.kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
self.index_q_norm.weight,
|
||||
self.index_k_norm.weight,
|
||||
self.num_idx_heads,
|
||||
index_cache,
|
||||
index_q,
|
||||
index_slot_mapping,
|
||||
)
|
||||
else:
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
self.index_q_norm.weight,
|
||||
self.index_k_norm.weight,
|
||||
self.num_idx_heads,
|
||||
q_out=q,
|
||||
index_q_out=index_q,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
k_start = self.q_size
|
||||
v_start = k_start + self.kv_size
|
||||
index_k_start = v_start + self.kv_size + self.index_q_size
|
||||
k = qkv[:, k_start:v_start].view(
|
||||
num_tokens, self.num_kv_heads, self.head_dim
|
||||
)
|
||||
v = qkv[:, v_start : v_start + self.kv_size].view(
|
||||
num_tokens, self.num_kv_heads, self.head_dim
|
||||
)
|
||||
index_k = qkv[
|
||||
:, index_k_start : index_k_start + self.idx_head_dim
|
||||
].view(num_tokens, self.idx_head_dim)
|
||||
self._insert_aiter_sparse_pa_kv(
|
||||
k,
|
||||
v,
|
||||
index_k,
|
||||
page16_slot_mapping,
|
||||
index_slot_mapping,
|
||||
key_cache,
|
||||
value_cache,
|
||||
)
|
||||
else:
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
|
||||
Reference in New Issue
Block a user