[Perf] [Feat] [ROCm] Add densemha support to ROCm AITER Sparse MLA (#49263)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -33,6 +33,101 @@ class _FakeAiter(ModuleType):
|
||||
get_mla_metadata_v1: Mock
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [16, 64, 128])
|
||||
@pytest.mark.parametrize("topk_tokens", [128, 256])
|
||||
def test_rocm_index_conversion_matrix(block_size, topk_tokens):
|
||||
"""Convert valid, masked, and out-of-range sparse indices exactly."""
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 4
|
||||
max_blocks = 4
|
||||
req_ids = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor(
|
||||
[[3, 5, 7, 9], [2, 4, 6, 8]], dtype=torch.int32, device=device
|
||||
)
|
||||
token_indices = torch.arange(
|
||||
num_tokens * topk_tokens, dtype=torch.int32, device=device
|
||||
).view(num_tokens, topk_tokens)
|
||||
token_indices %= max_blocks * block_size
|
||||
token_indices[0, 0] = -1
|
||||
token_indices[1, 1] = max_blocks * block_size
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
(num_tokens + 1) * topk_tokens,
|
||||
topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
output = torch.full(
|
||||
(num_tokens * topk_tokens,), -99, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
token_indices,
|
||||
cu_seqlens,
|
||||
output,
|
||||
BLOCK_STRIDE_ROWS=block_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=topk_tokens,
|
||||
)
|
||||
|
||||
block_ids = token_indices // block_size
|
||||
valid = (token_indices >= 0) & (block_ids < max_blocks)
|
||||
safe_blocks = block_ids.clamp(0, max_blocks - 1)
|
||||
bases = block_table[req_ids[:, None].long(), safe_blocks.long()]
|
||||
expected = torch.where(valid, bases * block_size + token_indices % block_size, 0)
|
||||
torch.testing.assert_close(output.view_as(expected), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_rocm_index_conversion_rejects_mismatched_token_extents():
|
||||
"""Reject full-batch request IDs paired with decode-subset indices."""
|
||||
with pytest.raises(AssertionError, match="must cover the same tokens"):
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
torch.zeros(3, dtype=torch.int32),
|
||||
torch.zeros(1, 4, dtype=torch.int32),
|
||||
torch.zeros(2, 128, dtype=torch.int32),
|
||||
torch.arange(3, dtype=torch.int32),
|
||||
torch.zeros(256, dtype=torch.int32),
|
||||
BLOCK_STRIDE_ROWS=64,
|
||||
NUM_TOPK_TOKENS=128,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_index_conversion_respects_packed_valid_counts():
|
||||
"""ROCm's indptr contract packs only each token's valid top-k prefix."""
|
||||
device = torch.device("cuda")
|
||||
topk_tokens = 128
|
||||
valid_counts = torch.tensor([1, 17, 128], dtype=torch.int32, device=device)
|
||||
cu_seqlens = torch.zeros(4, dtype=torch.int32, device=device)
|
||||
torch.cumsum(valid_counts, dim=0, out=cu_seqlens[1:])
|
||||
req_ids = torch.tensor([0, 0, 0], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor([[2, 3, 4]], dtype=torch.int32, device=device)
|
||||
token_indices = torch.arange(
|
||||
3 * topk_tokens, dtype=torch.int32, device=device
|
||||
).view(3, topk_tokens)
|
||||
token_indices %= 3 * 64
|
||||
output = torch.full((int(cu_seqlens[-1]),), -99, dtype=torch.int32, device=device)
|
||||
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
token_indices,
|
||||
cu_seqlens,
|
||||
output,
|
||||
BLOCK_STRIDE_ROWS=64,
|
||||
BLOCK_SIZE=64,
|
||||
NUM_TOPK_TOKENS=topk_tokens,
|
||||
)
|
||||
|
||||
expected_parts = []
|
||||
for token_id, count in enumerate(valid_counts.tolist()):
|
||||
local = token_indices[token_id, :count]
|
||||
expected_parts.append(block_table[0, (local // 64).long()] * 64 + local % 64)
|
||||
expected = torch.cat(expected_parts)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _make_builder():
|
||||
builder = object.__new__(sparse_mod.ROCMAiterMLASparseMetadataBuilder)
|
||||
max_num_batched_tokens = 8
|
||||
@@ -44,6 +139,18 @@ def _make_builder():
|
||||
builder.kv_cache_dtype = "fp8"
|
||||
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
|
||||
builder.topk_tokens = topk_tokens
|
||||
# SparseMLACommonMetadataBuilder state consumed by its build() method.
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.use_pcp = False
|
||||
builder._prefill_backend = None
|
||||
|
||||
def cpu_token_to_req(metadata):
|
||||
return torch.repeat_interleave(
|
||||
torch.arange(metadata.num_reqs, dtype=torch.int32),
|
||||
metadata.query_start_loc_cpu.diff(),
|
||||
)
|
||||
|
||||
builder._build_req_id_per_token = cpu_token_to_req # type: ignore[method-assign]
|
||||
builder.req_id_per_token_buffer = torch.zeros(
|
||||
max_num_batched_tokens, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
@@ -62,6 +169,8 @@ def _make_builder():
|
||||
builder._use_persistent_metadata = True
|
||||
builder._num_attention_heads = 16
|
||||
builder._num_compute_units = current_platform.num_compute_units()
|
||||
builder._mla_q_dtype = torch.bfloat16
|
||||
builder._mla_kv_dtype = torch.bfloat16
|
||||
builder._mla_work_meta_data = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_work_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_work_info_set = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
@@ -216,6 +325,8 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
|
||||
assert events == ["metadata", "sync"]
|
||||
assert fake_get_mla_metadata_v1_mock.call_count == 1
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["max_split_per_batch"] == 1
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_q"] == torch.bfloat16
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_kv"] == torch.bfloat16
|
||||
|
||||
events.clear()
|
||||
|
||||
@@ -223,3 +334,395 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
|
||||
|
||||
assert events == []
|
||||
assert fake_get_mla_metadata_v1_mock.call_count == 1
|
||||
|
||||
|
||||
def test_dense_prefill_builds_sparse_metadata_for_decode_subset(monkeypatch):
|
||||
"""Dense prefill must exclude prefill tokens from AITER sparse metadata.
|
||||
|
||||
A mixed batch is routed as sparse MQA for its leading decode requests and
|
||||
dense MHA for its trailing short prefill requests. The AITER metadata
|
||||
buffers therefore must describe only the decode subset; including prefill
|
||||
tokens would mismatch the query passed to ``forward_mqa``.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16, 4], dtype=torch.int32)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
num_actual_tokens=5,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(5),
|
||||
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
req_id_per_token=torch.tensor([0, 1, 1, 1, 1], dtype=torch.int32),
|
||||
topk_tokens=4,
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=4,
|
||||
prefill=SimpleNamespace(use_dense_mha=True),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
sparse_call: dict[str, torch.Tensor] = {}
|
||||
|
||||
def fake_generate_sparse_seqlen(
|
||||
query_lens, sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
|
||||
):
|
||||
sparse_call.update(
|
||||
query_lens=query_lens.clone(),
|
||||
seq_lens=sparse_seq_lens.clone(),
|
||||
query_start_loc=sparse_query_start_loc.clone(),
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
return torch.ones(num_tokens, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
|
||||
)
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = Mock()
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
assert result.prefill is metadata.prefill
|
||||
assert result.num_prefills == 1
|
||||
assert result.req_id_per_token.tolist() == [0]
|
||||
assert result.qo_indptr.numel() == 2
|
||||
assert result.paged_kv_indptr.numel() == 2
|
||||
assert sparse_call["num_tokens"] == 1
|
||||
assert sparse_call["query_lens"].tolist() == [1]
|
||||
assert sparse_call["seq_lens"].tolist() == [16]
|
||||
assert sparse_call["query_start_loc"].tolist() == [0, 1]
|
||||
|
||||
|
||||
def test_long_prefill_shares_full_sparse_metadata_with_decode(monkeypatch):
|
||||
"""A prefill longer than top-k must remain in shared sparse metadata.
|
||||
|
||||
Dense MHA is only selected when the prefill sequence fits within the
|
||||
sparse top-k window. Once it exceeds that window, both decode and prefill
|
||||
are handled by sparse MQA, so every query token must be represented by the
|
||||
same sparse paging metadata.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
|
||||
# One decode token followed by a five-token prefill. topk_tokens is four,
|
||||
# so the prefill must not be split onto the dense-MHA path.
|
||||
query_start_loc = torch.tensor([0, 1, 6], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16, 5], dtype=torch.int32)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=5,
|
||||
max_seq_len=16,
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=5,
|
||||
max_seq_len=16,
|
||||
num_actual_tokens=6,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(6),
|
||||
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
req_id_per_token=torch.tensor([0, 1, 1, 1, 1, 1], dtype=torch.int32),
|
||||
topk_tokens=4,
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=5,
|
||||
prefill=SimpleNamespace(use_dense_mha=False),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
sparse_call: dict[str, torch.Tensor] = {}
|
||||
|
||||
def fake_generate_sparse_seqlen(
|
||||
query_lens, sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
|
||||
):
|
||||
sparse_call.update(
|
||||
query_lens=query_lens.clone(),
|
||||
seq_lens=sparse_seq_lens.clone(),
|
||||
query_start_loc=sparse_query_start_loc.clone(),
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
return torch.ones(num_tokens, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
|
||||
)
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
get_persistent_metadata = Mock()
|
||||
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
assert result.num_decodes == 1
|
||||
assert result.num_prefills == 1
|
||||
assert result.req_id_per_token.tolist() == [0, 1, 1, 1, 1, 1]
|
||||
assert result.qo_indptr.numel() == 7
|
||||
assert result.paged_kv_indptr.numel() == 7
|
||||
assert result.paged_kv_indices.numel() == 6 * result.topk_tokens
|
||||
assert sparse_call["num_tokens"] == 6
|
||||
assert sparse_call["query_lens"].tolist() == [1, 5]
|
||||
assert sparse_call["seq_lens"].tolist() == [16, 5]
|
||||
assert sparse_call["query_start_loc"].tolist() == [0, 1, 6]
|
||||
get_persistent_metadata.assert_called_once()
|
||||
assert result.work_meta_data is builder._mla_work_meta_data
|
||||
assert result.reduce_indptr is builder._mla_reduce_indptr
|
||||
|
||||
|
||||
def test_forward_mqa_slices_full_batch_metadata_to_query_tokens(monkeypatch):
|
||||
"""The sparse kernel inputs must have the same token extent as its query.
|
||||
|
||||
In a mixed dense-prefill batch, ``forward_mqa`` receives only decode query
|
||||
tokens while some metadata buffers may represent the full batch. This
|
||||
guards the slicing that prevents the index conversion from writing beyond
|
||||
the decode-sized output.
|
||||
"""
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.num_heads = 4
|
||||
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
|
||||
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
|
||||
forward_mla = Mock(return_value=(expected_output, None))
|
||||
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
|
||||
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
|
||||
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
|
||||
block_size=1,
|
||||
topk_tokens=4,
|
||||
)
|
||||
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
|
||||
captured = {}
|
||||
|
||||
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
|
||||
captured["req_ids"] = req_ids.clone()
|
||||
captured["token_indices"] = token_indices.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
|
||||
)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
torch.zeros(1, 1, 576, dtype=torch.bfloat16),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert output is expected_output
|
||||
assert lse is None
|
||||
assert captured["req_ids"].tolist() == [0, 1]
|
||||
assert captured["token_indices"].shape == (2, 4)
|
||||
forward_mla.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_forward_mqa_slices_decode_subset_for_bf16_and_fp8(monkeypatch, kv_cache_dtype):
|
||||
"""Both cache dtypes must slice mixed-batch metadata to decode queries."""
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.kv_cache_dtype = kv_cache_dtype
|
||||
impl.num_heads = 4
|
||||
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
|
||||
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
|
||||
forward_mla = Mock(return_value=(expected_output, None))
|
||||
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
|
||||
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
|
||||
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
|
||||
block_size=1,
|
||||
topk_tokens=4,
|
||||
)
|
||||
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.zeros(1, 1, 576, dtype=torch.uint8)
|
||||
captured = {}
|
||||
|
||||
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
|
||||
captured["req_ids"] = req_ids.clone()
|
||||
captured["token_indices"] = token_indices.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
|
||||
)
|
||||
if kv_cache_dtype == "fp8":
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.ops,
|
||||
"scaled_fp8_quant",
|
||||
lambda q, scale: (q.to(fp8_dtype), None),
|
||||
)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
kv_cache,
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale=torch.tensor(1.0)),
|
||||
)
|
||||
|
||||
assert output is expected_output
|
||||
assert lse is None
|
||||
assert captured["req_ids"].tolist() == [0, 1]
|
||||
assert captured["token_indices"].shape == (2, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("length_relation", ["short", "long"])
|
||||
@pytest.mark.parametrize("workload", ["prefill", "mixed", "decode"])
|
||||
def test_sparse_metadata_workload_length_dtype_matrix(
|
||||
monkeypatch, workload, length_relation, kv_cache_dtype
|
||||
):
|
||||
"""Cover routing and persistent metadata across the ROCm support matrix.
|
||||
|
||||
The matrix crosses pure prefill, mixed prefill/decode, and decode-only
|
||||
workloads with sequence lengths below/above top-k and BF16/FP8 KV cache.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
if kv_cache_dtype == "fp8":
|
||||
builder._mla_q_dtype = current_platform.fp8_dtype()
|
||||
builder._mla_kv_dtype = current_platform.fp8_dtype()
|
||||
|
||||
is_long = length_relation == "long"
|
||||
prefill_len = 5 if is_long else 3
|
||||
decode_seq_len = 8 if is_long else 3
|
||||
if workload == "prefill":
|
||||
query_lens = [prefill_len]
|
||||
seq_lens_list = [prefill_len]
|
||||
num_decodes, num_prefills = 0, 1
|
||||
elif workload == "mixed":
|
||||
query_lens = [1, prefill_len]
|
||||
seq_lens_list = [decode_seq_len, prefill_len]
|
||||
num_decodes, num_prefills = 1, 1
|
||||
else:
|
||||
query_lens = [1, 1]
|
||||
seq_lens_list = [decode_seq_len, decode_seq_len + 1]
|
||||
num_decodes, num_prefills = 2, 0
|
||||
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32)
|
||||
num_tokens = sum(query_lens)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=len(query_lens),
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(num_tokens),
|
||||
block_table=torch.arange(len(query_lens) * 8, dtype=torch.int32).view(
|
||||
len(query_lens), 8
|
||||
),
|
||||
req_id_per_token=torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32),
|
||||
torch.tensor(query_lens),
|
||||
),
|
||||
topk_tokens=4,
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decodes,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=prefill_len if num_prefills else 0,
|
||||
prefill=SimpleNamespace(use_dense_mha=not is_long) if num_prefills else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"generate_sparse_seqlen_triton",
|
||||
lambda *args, **kwargs: torch.ones(args[4], dtype=torch.int32),
|
||||
)
|
||||
get_persistent_metadata = Mock()
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
uses_dense_prefill = workload in ("prefill", "mixed") and not is_long
|
||||
expected_sparse_tokens = num_decodes if uses_dense_prefill else num_tokens
|
||||
assert result.req_id_per_token.numel() == expected_sparse_tokens
|
||||
assert result.qo_indptr.numel() == expected_sparse_tokens + 1
|
||||
assert result.paged_kv_indptr.numel() == expected_sparse_tokens + 1
|
||||
|
||||
uses_fallback = workload == "prefill" and is_long and kv_cache_dtype == "auto"
|
||||
if expected_sparse_tokens == 0 or uses_fallback:
|
||||
get_persistent_metadata.assert_not_called()
|
||||
else:
|
||||
get_persistent_metadata.assert_called_once()
|
||||
if uses_fallback or expected_sparse_tokens == 0:
|
||||
assert result.work_meta_data is None
|
||||
assert result.reduce_indptr is None
|
||||
else:
|
||||
assert result.work_meta_data is builder._mla_work_meta_data
|
||||
assert result.reduce_indptr is builder._mla_reduce_indptr
|
||||
|
||||
Reference in New Issue
Block a user