[Bugfix] Sparse MLA: enable fp8_ds_mla dense prefill (#48642)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f17be06fbe
commit
67f9046e4a
@@ -27,8 +27,8 @@ def _build_test_case(seq_lens, block_size, seed=42):
|
||||
seed: Random seed for reproducibility.
|
||||
|
||||
Returns:
|
||||
Tuple of (cache, block_table, seq_lens_t, workspace_starts_t,
|
||||
num_reqs, total_tokens, expected_output).
|
||||
Tuple of (cache, block_table, workspace_starts_t, num_reqs,
|
||||
total_tokens, expected_output).
|
||||
"""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
@@ -112,7 +112,6 @@ def _build_test_case(seq_lens, block_size, seed=42):
|
||||
# Expected output: exact copy
|
||||
expected[out_idx, NOPE_DIM:] = rope
|
||||
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
|
||||
workspace_starts_t = torch.tensor(
|
||||
workspace_starts, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
@@ -120,7 +119,6 @@ def _build_test_case(seq_lens, block_size, seed=42):
|
||||
return (
|
||||
cache,
|
||||
block_table,
|
||||
seq_lens_t,
|
||||
workspace_starts_t,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
@@ -197,7 +195,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42):
|
||||
flat_cache[:sl] = token_data[ws : ws + sl]
|
||||
block_start += nb
|
||||
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
|
||||
workspace_starts_t = torch.tensor(
|
||||
workspace_starts, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
@@ -205,7 +202,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42):
|
||||
return (
|
||||
cache,
|
||||
block_table,
|
||||
seq_lens_t,
|
||||
workspace_starts_t,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
@@ -232,7 +228,6 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size):
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
seq_lens_t,
|
||||
workspace_starts_t,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
@@ -244,7 +239,7 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size):
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs
|
||||
cache, dst, block_table, workspace_starts_t, num_reqs
|
||||
)
|
||||
|
||||
# NoPE: fp8 dequant has rounding error, so we allow small tolerance.
|
||||
@@ -277,8 +272,6 @@ def test_cp_gather_fp8_shuffled_blocks():
|
||||
)
|
||||
block_table = torch.tensor([[3, 1]], dtype=torch.int32, device="cuda")
|
||||
workspace_starts = torch.tensor([0], dtype=torch.int32, device="cuda")
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
|
||||
|
||||
expected = torch.zeros(
|
||||
total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
@@ -314,7 +307,7 @@ def test_cp_gather_fp8_shuffled_blocks():
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache, dst, block_table, seq_lens_t, workspace_starts, len(seq_lens)
|
||||
cache, dst, block_table, workspace_starts, len(seq_lens)
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
@@ -323,6 +316,64 @@ def test_cp_gather_fp8_shuffled_blocks():
|
||||
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"gather_seq_lens,seq_starts",
|
||||
[
|
||||
([6, 5, 3], [3, 4, 5]),
|
||||
([0, 5, 3], [12, 4, 5]),
|
||||
],
|
||||
)
|
||||
def test_cp_gather_fp8_with_sequence_starts(gather_seq_lens, seq_starts):
|
||||
"""Gather request slices beginning at arbitrary cache positions."""
|
||||
full_seq_lens = [12, 11, 9]
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
_workspace_starts_t,
|
||||
num_reqs,
|
||||
_total_tokens,
|
||||
full_expected,
|
||||
) = _build_test_case(full_seq_lens, block_size=4)
|
||||
|
||||
workspace_starts = torch.tensor(
|
||||
[0, gather_seq_lens[0], sum(gather_seq_lens[:2])],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
seq_starts_t = torch.tensor(seq_starts, dtype=torch.int32, device="cuda")
|
||||
dst = torch.empty(
|
||||
sum(gather_seq_lens),
|
||||
NOPE_DIM + ROPE_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache,
|
||||
dst,
|
||||
block_table,
|
||||
workspace_starts,
|
||||
num_reqs,
|
||||
seq_starts_t,
|
||||
)
|
||||
|
||||
full_workspace_starts = [0, full_seq_lens[0], sum(full_seq_lens[:2])]
|
||||
expected = torch.cat(
|
||||
[
|
||||
full_expected[
|
||||
full_workspace_starts[i] + seq_starts[i] : full_workspace_starts[i]
|
||||
+ seq_starts[i]
|
||||
+ gather_seq_lens[i]
|
||||
]
|
||||
for i in range(num_reqs)
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2
|
||||
)
|
||||
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,block_size",
|
||||
[
|
||||
@@ -342,7 +393,6 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
seq_lens_t,
|
||||
workspace_starts_t,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
@@ -354,7 +404,7 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs
|
||||
cache, dst, block_table, workspace_starts_t, num_reqs
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
|
||||
@@ -42,7 +42,10 @@ from vllm.v1.attention.backends.mla.prefill import (
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
BACKENDS_TO_TEST = [
|
||||
AttentionBackendEnum.CUTLASS_MLA,
|
||||
@@ -55,6 +58,31 @@ BACKENDS_TO_TEST = [
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cache_dtype", "expected_quant_mode"),
|
||||
[
|
||||
("auto", KVQuantMode.NONE),
|
||||
("fp8_ds_mla", KVQuantMode.FP8_PER_TENSOR),
|
||||
],
|
||||
)
|
||||
def test_mla_kv_cache_spec_uses_layer_cache_dtype(
|
||||
cache_dtype: str, expected_quant_mode: KVQuantMode
|
||||
):
|
||||
layer = SimpleNamespace(kv_cache_dtype=cache_dtype, head_size=576)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=64), model_config=None
|
||||
)
|
||||
|
||||
spec = MLAAttention.get_kv_cache_spec(layer, vllm_config)
|
||||
|
||||
assert isinstance(spec, MLAAttentionSpec)
|
||||
assert spec.cache_dtype_str == cache_dtype
|
||||
assert spec.kv_quant_mode == expected_quant_mode
|
||||
if cache_dtype == "fp8_ds_mla":
|
||||
assert spec.page_size_bytes == 64 * 656
|
||||
|
||||
|
||||
# Remove sm100 backends from the list if not using sm100
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_properties(0).major < 10:
|
||||
BACKENDS_TO_TEST.remove(AttentionBackendEnum.CUTLASS_MLA)
|
||||
|
||||
@@ -40,10 +40,16 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
FlashMLASparseBackend,
|
||||
FlashMLASparseImpl,
|
||||
FlashMLASparseMetadata,
|
||||
FlashMLASparseMetadataBuilder,
|
||||
triton_convert_req_index_to_global_index,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import split_indexer_prefill_chunks
|
||||
from vllm.v1.attention.backends.utils import split_prefill_chunks
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
split_decodes_and_prefills,
|
||||
split_prefill_chunks,
|
||||
)
|
||||
from vllm.v1.attention.ops import flashmla
|
||||
|
||||
SPARSE_BACKEND_BATCH_SPECS = {
|
||||
@@ -721,6 +727,120 @@ def test_triton_convert_req_index_to_global_index_with_prefill_workspace(block_s
|
||||
torch.testing.assert_close(result, reference_result, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability() < (9, 0),
|
||||
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
|
||||
)
|
||||
def test_triton_convert_rejects_req_id_longer_than_token_indices():
|
||||
"""Guard against the #47327 regression: the kernel grid is sized by
|
||||
req_id but the output is allocated like token_indices, so a full-batch
|
||||
req_id combined with an MQA-subset token_indices wrote past the end of
|
||||
the output buffer. The wrapper must reject the length mismatch instead
|
||||
of corrupting memory."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_topk_tokens = 128
|
||||
block_size = 64
|
||||
block_table = torch.arange(40, dtype=torch.int32, device=device).view(4, 10)
|
||||
|
||||
# Full batch: 2 decode tokens + 10 prefill tokens
|
||||
req_id_full = torch.tensor(
|
||||
[0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3], dtype=torch.int32, device=device
|
||||
)
|
||||
num_mqa_tokens = 2
|
||||
token_indices = torch.randint(
|
||||
0,
|
||||
block_size * 10,
|
||||
(num_mqa_tokens, num_topk_tokens),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
with pytest.raises(AssertionError, match="must cover the same tokens"):
|
||||
triton_convert_req_index_to_global_index(
|
||||
req_id_full,
|
||||
block_table,
|
||||
token_indices,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk_tokens,
|
||||
)
|
||||
|
||||
# The sliced call is the intended usage and must match the reference.
|
||||
result = triton_convert_req_index_to_global_index(
|
||||
req_id_full[:num_mqa_tokens],
|
||||
block_table,
|
||||
token_indices,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk_tokens,
|
||||
)
|
||||
reference = _triton_convert_reference_impl(
|
||||
req_id_full[:num_mqa_tokens],
|
||||
block_table,
|
||||
token_indices,
|
||||
block_size,
|
||||
num_topk_tokens,
|
||||
)
|
||||
torch.testing.assert_close(result, reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability() < (9, 0),
|
||||
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
|
||||
)
|
||||
def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
|
||||
"""Guard against the #47327 regression: when the dense-MHA prefill split
|
||||
is active, forward_mqa only receives the leading decode tokens, but
|
||||
_forward_bf16_kv passed the full-batch req_id_per_token to the index
|
||||
conversion, making it write past the end of its output buffer. The call
|
||||
site must slice req_id_per_token to the MQA tokens."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_topk_tokens = 128
|
||||
block_size = 64
|
||||
num_batch_tokens = 12
|
||||
num_mqa_tokens = 2
|
||||
|
||||
attn_metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor(
|
||||
[0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32, device=device
|
||||
),
|
||||
block_table=torch.arange(40, dtype=torch.int32, device=device).view(4, 10),
|
||||
block_size=block_size,
|
||||
)
|
||||
assert attn_metadata.req_id_per_token.shape[0] == num_batch_tokens
|
||||
|
||||
q = torch.zeros(num_mqa_tokens, 4, 576, dtype=torch.bfloat16, device=device)
|
||||
kv_cache = torch.zeros(40 * block_size, 576, dtype=torch.bfloat16, device=device)
|
||||
topk_indices = torch.randint(
|
||||
0,
|
||||
block_size * 10,
|
||||
(num_mqa_tokens, num_topk_tokens),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
def _stub_kernel(q, kv, indices, lengths):
|
||||
captured["indices"] = indices
|
||||
return torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device)
|
||||
|
||||
stub_impl = SimpleNamespace(_bf16_flash_mla_kernel=_stub_kernel)
|
||||
|
||||
out = FlashMLASparseImpl._forward_bf16_kv(
|
||||
stub_impl, q, kv_cache, topk_indices, attn_metadata
|
||||
)
|
||||
|
||||
assert out.shape[0] == num_mqa_tokens
|
||||
assert captured["indices"].shape[0] == num_mqa_tokens
|
||||
reference = _triton_convert_reference_impl(
|
||||
attn_metadata.req_id_per_token[:num_mqa_tokens],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
block_size,
|
||||
num_topk_tokens,
|
||||
)
|
||||
torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,max_buf,expected",
|
||||
[
|
||||
@@ -1124,3 +1244,203 @@ def test_triton_convert_returns_valid_counts():
|
||||
)
|
||||
assert isinstance(result_only, torch.Tensor)
|
||||
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_flashmla_cache_dtype_aliases_use_ds_layout():
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
|
||||
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
|
||||
# stores kv_cache_dtype, so they cannot bypass the gate.
|
||||
for alias in ("fp8", "fp8_e4m3"):
|
||||
assert (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype(FlashMLASparseBackend, alias)
|
||||
== "fp8_ds_mla"
|
||||
)
|
||||
|
||||
|
||||
def test_flashmla_fp8_metadata_reuses_common_batch_split():
|
||||
builder = SimpleNamespace(
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=8)),
|
||||
)
|
||||
common_metadata = SimpleNamespace(
|
||||
num_actual_tokens=1,
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1]),
|
||||
query_start_loc_cpu=torch.tensor([0, 1]),
|
||||
block_table_tensor=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
)
|
||||
metadata = FlashMLASparseMetadata(
|
||||
num_reqs=1,
|
||||
max_query_len=1,
|
||||
max_seq_len=1,
|
||||
num_actual_tokens=1,
|
||||
query_start_loc=torch.tensor([0, 1], device=DEVICE_TYPE),
|
||||
slot_mapping=torch.tensor([0], device=DEVICE_TYPE),
|
||||
block_table=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
req_id_per_token=torch.zeros(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
num_decodes=0,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=0,
|
||||
)
|
||||
|
||||
fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
|
||||
builder, common_metadata, metadata
|
||||
)
|
||||
|
||||
assert fp8_metadata.num_decodes == 0
|
||||
assert fp8_metadata.num_prefills == 1
|
||||
assert fp8_metadata.num_decode_tokens == 0
|
||||
assert fp8_metadata.num_prefill_tokens == 1
|
||||
|
||||
|
||||
def test_flashmla_common_metadata_requires_uniform_decodes():
|
||||
common_metadata = SimpleNamespace(
|
||||
max_query_len=3,
|
||||
num_reqs=3,
|
||||
num_actual_tokens=6,
|
||||
query_start_loc_cpu=torch.tensor([0, 1, 3, 6]),
|
||||
is_prefilling=None,
|
||||
)
|
||||
|
||||
split = split_decodes_and_prefills(
|
||||
common_metadata,
|
||||
decode_threshold=128,
|
||||
require_uniform=FlashMLASparseMetadataBuilder.require_uniform_decodes,
|
||||
)
|
||||
|
||||
assert split == (1, 2, 1, 5)
|
||||
|
||||
|
||||
def test_flashmla_fp8_metadata_excludes_zero_token_decode_padding(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse.get_mla_metadata",
|
||||
lambda: (object(), None),
|
||||
)
|
||||
builder = SimpleNamespace(
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
dummy_block_table=torch.zeros(7, 1, device=DEVICE_TYPE),
|
||||
max_model_len_tensor=torch.zeros(7, device=DEVICE_TYPE),
|
||||
)
|
||||
query_start_loc_cpu = torch.tensor([0, 110, 220, 330, 440, 550, 660, 660])
|
||||
common_metadata = SimpleNamespace(
|
||||
num_actual_tokens=660,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
seq_lens=torch.arange(7, device=DEVICE_TYPE),
|
||||
)
|
||||
metadata = FlashMLASparseMetadata(
|
||||
num_reqs=7,
|
||||
max_query_len=110,
|
||||
max_seq_len=110,
|
||||
num_actual_tokens=660,
|
||||
query_start_loc=query_start_loc_cpu.to(DEVICE_TYPE),
|
||||
slot_mapping=torch.arange(660, device=DEVICE_TYPE),
|
||||
block_table=torch.zeros(7, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
req_id_per_token=torch.zeros(660, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
num_decodes=7,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=660,
|
||||
)
|
||||
|
||||
fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
|
||||
builder, common_metadata, metadata
|
||||
)
|
||||
|
||||
assert fp8_metadata.num_decodes == 6
|
||||
assert fp8_metadata.num_decode_tokens == 660
|
||||
assert fp8_metadata.decode is not None
|
||||
assert fp8_metadata.decode.decode_query_len == 110
|
||||
torch.testing.assert_close(
|
||||
fp8_metadata.decode.seq_lens, torch.arange(6, device=DEVICE_TYPE)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mixed_batch", [False, True])
|
||||
def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: bool):
|
||||
num_decode_tokens = 2
|
||||
num_batch_tokens = 5
|
||||
q = torch.empty(num_decode_tokens, 2, 3, device=DEVICE_TYPE)
|
||||
topk_indices = torch.empty(num_decode_tokens, 4, device=DEVICE_TYPE)
|
||||
kernel_q_shapes = []
|
||||
|
||||
def convert_indices(*args, **kwargs): # noqa: ARG001
|
||||
assert not kwargs.get("HAS_PREFILL_WORKSPACE", False)
|
||||
if not kwargs.get("return_valid_counts", False):
|
||||
return topk_indices
|
||||
valid_counts = torch.full(
|
||||
(num_decode_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE
|
||||
)
|
||||
return topk_indices, valid_counts
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse."
|
||||
"triton_convert_req_index_to_global_index",
|
||||
convert_indices,
|
||||
)
|
||||
|
||||
def run_kernel(**kwargs):
|
||||
kernel_q_shapes.append(kwargs["q"].shape)
|
||||
return kwargs["q"][..., :1], None
|
||||
|
||||
if use_mixed_batch:
|
||||
fp8_metadata = FlashMLASparseMetadata.FP8KernelMetadata(
|
||||
scheduler_metadata=object(), # type: ignore[arg-type]
|
||||
dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
)
|
||||
else:
|
||||
FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
|
||||
fp8_metadata = FP8Meta(
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefill_tokens=num_batch_tokens - num_decode_tokens,
|
||||
decode=FP8Meta.Decode(
|
||||
seq_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
kernel_metadata=object(), # type: ignore[arg-type]
|
||||
decode_query_len=num_decode_tokens,
|
||||
),
|
||||
prefill=FP8Meta.Prefill(
|
||||
request_ids=torch.empty(
|
||||
num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
|
||||
),
|
||||
workspace_starts=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
chunks=[],
|
||||
),
|
||||
)
|
||||
metadata = SimpleNamespace(
|
||||
fp8_extra_metadata=fp8_metadata,
|
||||
fp8_use_mixed_batch=use_mixed_batch,
|
||||
num_actual_tokens=num_batch_tokens,
|
||||
req_id_per_token=torch.empty(
|
||||
num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
|
||||
),
|
||||
block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
block_size=64,
|
||||
)
|
||||
impl = SimpleNamespace(
|
||||
kv_cache_dtype="fp8_ds_mla",
|
||||
topk_indices_buffer=topk_indices,
|
||||
num_heads=2,
|
||||
kv_lora_rank=1,
|
||||
_fp8_flash_mla_kernel=run_kernel,
|
||||
)
|
||||
impl._forward_fp8_kv_mixed_batch = MethodType(
|
||||
FlashMLASparseImpl._forward_fp8_kv_mixed_batch, impl
|
||||
)
|
||||
impl._forward_fp8_kv_separate_prefill_decode = MethodType(
|
||||
FlashMLASparseImpl._forward_fp8_kv_separate_prefill_decode, impl
|
||||
)
|
||||
|
||||
output, lse = FlashMLASparseImpl.forward_mqa(
|
||||
impl,
|
||||
q,
|
||||
torch.empty(0, device=DEVICE_TYPE),
|
||||
metadata,
|
||||
None,
|
||||
)
|
||||
|
||||
assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
|
||||
assert output.shape == (num_decode_tokens, 2, 1)
|
||||
assert lse is None
|
||||
|
||||
Reference in New Issue
Block a user