[Kimi-K3] DCP support (#50484)
Signed-off-by: girasoley <girasoley@inferact.ai> Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: 云挚 <ningyunxiao.nyx@antgroup.com> Co-authored-by: foraxe <73625538+foraxe@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5
云挚
foraxe
Woosuk Kwon
parent
3f142bd85e
commit
63ac04a61e
@@ -222,6 +222,49 @@ class TestLSEWeightedCombine:
|
||||
assert result.shape == (B, H, D)
|
||||
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
|
||||
|
||||
def test_empty_shard_ignores_undefined_output(self):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
|
||||
outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]])
|
||||
lses = torch.tensor([[[-float("inf")]], [[0.0]]])
|
||||
|
||||
result = _lse_weighted_combine(outputs, lses)
|
||||
|
||||
torch.testing.assert_close(result, outputs[1])
|
||||
|
||||
def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch):
|
||||
import vllm.v1.attention.ops.common as common
|
||||
|
||||
class FakeGroup:
|
||||
world_size = 2
|
||||
rank_in_group = 0
|
||||
|
||||
def all_gather(self, tensor, dim):
|
||||
assert dim == 0
|
||||
return torch.cat((tensor, tensor), dim=dim)
|
||||
|
||||
monkeypatch.setattr(
|
||||
common,
|
||||
"correct_attn_out",
|
||||
lambda output, lses, *args, **kwargs: (output, lses[0]),
|
||||
)
|
||||
output = torch.ones(7, 1, 1)
|
||||
lse = torch.ones(7, 1)
|
||||
seq_lens = torch.tensor([0, 2], dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
|
||||
|
||||
_, masked_lse = common._cp_lse_common(
|
||||
output,
|
||||
lse,
|
||||
FakeGroup(),
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
|
||||
assert torch.isneginf(masked_lse[:1]).all()
|
||||
torch.testing.assert_close(masked_lse[1:5], torch.ones_like(masked_lse[1:5]))
|
||||
assert torch.isneginf(masked_lse[5:]).all()
|
||||
|
||||
def test_mathematically_correct(self):
|
||||
"""Verify mathematical correctness of LSE combination."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
@@ -344,7 +387,7 @@ class TestPackedA2AKernels:
|
||||
world_size, B, h_per_rank, D = 4, 7, 2, 32
|
||||
H = world_size * h_per_rank
|
||||
cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype)
|
||||
cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32)
|
||||
cp_attn_lse = torch.randn(B, H, device=device, dtype=dtype)
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype)
|
||||
send_buffer = torch.empty(
|
||||
(world_size, B, h_per_rank, D + lse_pack_dim),
|
||||
@@ -375,6 +418,67 @@ class TestPackedA2AKernels:
|
||||
else:
|
||||
_assert_packed_a2a_close(actual, expected_out, dtype)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 1, reason="CUDA is required."
|
||||
)
|
||||
def test_empty_seq_lens_ignore_undefined_output(self):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import (
|
||||
_dcp_a2a_lse_pack_dim,
|
||||
_dcp_a2a_pack_send,
|
||||
_dcp_a2a_unpack_combine,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
world_size, num_tokens, h_per_rank, head_dim = 2, 5, 1, 32
|
||||
num_heads = world_size * h_per_rank
|
||||
output = torch.randn(
|
||||
num_tokens,
|
||||
num_heads,
|
||||
head_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
output[:1] = float("nan")
|
||||
lse = torch.randn(num_tokens, num_heads, device=device, dtype=output.dtype)
|
||||
seq_lens = torch.tensor([0, 2], device=device, dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0, 1, 5], device=device, dtype=torch.int32)
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(output.dtype)
|
||||
send_buffer = torch.empty(
|
||||
(
|
||||
world_size,
|
||||
num_tokens,
|
||||
h_per_rank,
|
||||
head_dim + lse_pack_dim,
|
||||
),
|
||||
device=device,
|
||||
dtype=output.dtype,
|
||||
)
|
||||
|
||||
_dcp_a2a_pack_send(
|
||||
output,
|
||||
lse,
|
||||
send_buffer,
|
||||
world_size,
|
||||
h_per_rank,
|
||||
head_dim,
|
||||
lse_pack_dim,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
actual_output, actual_lse = _dcp_a2a_unpack_combine(
|
||||
send_buffer,
|
||||
head_dim,
|
||||
lse_pack_dim,
|
||||
return_lse=True,
|
||||
is_lse_base_on_e=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_output[:1], torch.zeros_like(actual_output[:1])
|
||||
)
|
||||
assert torch.isneginf(actual_lse[:1]).all()
|
||||
assert torch.isfinite(actual_output[1:]).all()
|
||||
|
||||
|
||||
def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
update_environment_variables(env)
|
||||
@@ -417,7 +521,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
B,
|
||||
H,
|
||||
device=f"cuda:{local_rank}",
|
||||
dtype=torch.float32,
|
||||
dtype=dtype,
|
||||
generator=generator,
|
||||
)
|
||||
actual = dcp_a2a_lse_reduce(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,128 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import VllmRunner
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import SamplingParams, TokensPrompt
|
||||
|
||||
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
|
||||
|
||||
|
||||
def _make_tiny_overrides() -> dict:
|
||||
linear_attn_config = {
|
||||
"full_attn_layers": [4],
|
||||
"head_dim": 128,
|
||||
"kda_layers": [1, 2, 3],
|
||||
"num_heads": 8,
|
||||
"short_conv_kernel_size": 4,
|
||||
"use_full_rank_gate": True,
|
||||
}
|
||||
|
||||
return {
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"vocab_size": 256,
|
||||
"hidden_size": 256,
|
||||
"head_dim": 32,
|
||||
"intermediate_size": 512,
|
||||
"num_hidden_layers": 4,
|
||||
"num_attention_heads": 8,
|
||||
"num_key_value_heads": 8,
|
||||
"num_experts": None,
|
||||
"num_experts_per_token": None,
|
||||
"num_shared_experts": 0,
|
||||
"q_lora_rank": 128,
|
||||
"kv_lora_rank": 512,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"mla_use_nope": True,
|
||||
"mla_use_output_gate": True,
|
||||
"linear_attn_config": linear_attn_config,
|
||||
"max_position_embeddings": 512,
|
||||
"model_max_length": 512,
|
||||
}
|
||||
|
||||
|
||||
def _run_tiny_model(
|
||||
vllm_runner: type[VllmRunner], dcp_size: int
|
||||
) -> list[tuple[list[int], list[float]]]:
|
||||
with vllm_runner(
|
||||
model_name=MODEL,
|
||||
skip_tokenizer_init=True,
|
||||
load_format="dummy",
|
||||
hf_overrides=_make_tiny_overrides(),
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
distributed_executor_backend="mp",
|
||||
dtype="bfloat16",
|
||||
seed=0,
|
||||
enforce_eager=True,
|
||||
max_model_len=512,
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=1024,
|
||||
gpu_memory_utilization=0.25,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
) as runner:
|
||||
lengths = [1, 32, 33, 129]
|
||||
prompts = [
|
||||
TokensPrompt(
|
||||
prompt_token_ids=[
|
||||
3 + ((request_idx + token_idx) % 251) for token_idx in range(length)
|
||||
]
|
||||
)
|
||||
for request_idx, length in enumerate(lengths)
|
||||
]
|
||||
outputs = runner.llm.generate(
|
||||
prompts,
|
||||
SamplingParams(
|
||||
temperature=0,
|
||||
max_tokens=8,
|
||||
seed=0,
|
||||
ignore_eos=True,
|
||||
logprobs=20,
|
||||
),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
results = []
|
||||
for request_output in outputs:
|
||||
completion = request_output.outputs[0]
|
||||
token_ids = list(completion.token_ids)
|
||||
assert completion.logprobs is not None
|
||||
selected_logprobs = [
|
||||
step_logprobs[token_id].logprob
|
||||
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
|
||||
]
|
||||
results.append((token_ids, selected_logprobs))
|
||||
return results
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
def test_kimi_linear_dcp_tiny(
|
||||
vllm_runner: type[VllmRunner],
|
||||
num_gpus_available: int,
|
||||
) -> None:
|
||||
if num_gpus_available < 2:
|
||||
pytest.skip("Need at least 2 GPUs")
|
||||
|
||||
baseline = _run_tiny_model(vllm_runner, dcp_size=1)
|
||||
dcp = _run_tiny_model(vllm_runner, dcp_size=2)
|
||||
|
||||
assert [tokens for tokens, _ in dcp] == [tokens for tokens, _ in baseline]
|
||||
logprob_drifts = []
|
||||
for (_, baseline_logprobs), (_, dcp_logprobs) in zip(baseline, dcp):
|
||||
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
|
||||
assert math.isfinite(baseline_logprob)
|
||||
assert math.isfinite(dcp_logprob)
|
||||
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
||||
|
||||
assert max(logprob_drifts) <= 1e-2
|
||||
@@ -0,0 +1,246 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
NUM_TOKENS = 4
|
||||
NUM_HEADS = 2
|
||||
KV_LORA_RANK = 512
|
||||
NOPE_HEAD_DIM = 128
|
||||
ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
|
||||
V_HEAD_DIM = 128
|
||||
CACHE_ENTRY = KV_LORA_RANK + ROPE_HEAD_DIM
|
||||
DS_MLA_CACHE_ENTRY = 656
|
||||
BLOCK_SIZE = 8
|
||||
OWNED_TOKEN_INDICES = [1, 3]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA ops require CUDA"
|
||||
)
|
||||
|
||||
|
||||
def _inputs() -> dict[str, torch.Tensor]:
|
||||
torch.manual_seed(0)
|
||||
|
||||
def make(*shape: int) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
return {
|
||||
"q": make(NUM_TOKENS, NUM_HEADS, QK_HEAD_DIM),
|
||||
"k_nope": make(NUM_TOKENS, NUM_HEADS, NOPE_HEAD_DIM),
|
||||
"k_pe": make(NUM_TOKENS, ROPE_HEAD_DIM),
|
||||
"kv_c": make(NUM_TOKENS, KV_LORA_RANK),
|
||||
"v": make(NUM_TOKENS, NUM_HEADS, V_HEAD_DIM),
|
||||
"ql_nope": make(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK),
|
||||
"q_pe": make(NUM_TOKENS, NUM_HEADS, ROPE_HEAD_DIM),
|
||||
}
|
||||
|
||||
|
||||
def _slot_mapping() -> torch.Tensor:
|
||||
return torch.tensor([-1, 0, -1, 5], device="cuda", dtype=torch.int64)
|
||||
|
||||
|
||||
def _owned(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return tensor[OWNED_TOKEN_INDICES]
|
||||
|
||||
|
||||
def _full_key(k_nope: torch.Tensor, k_pe: torch.Tensor) -> torch.Tensor:
|
||||
shared_pe = k_pe[:, None, :].expand(-1, k_nope.shape[1], -1)
|
||||
return torch.cat((k_nope, shared_pe), dim=-1)
|
||||
|
||||
|
||||
def _latent_query(ql_nope: torch.Tensor, q_pe: torch.Tensor) -> torch.Tensor:
|
||||
return torch.cat((ql_nope, q_pe), dim=-1)
|
||||
|
||||
|
||||
def _make_cache(cache_format: str) -> torch.Tensor:
|
||||
if cache_format == "bf16":
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, CACHE_ENTRY),
|
||||
-7,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
if cache_format == "fp8":
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, CACHE_ENTRY),
|
||||
1.0,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
).to(torch.float8_e4m3fn)
|
||||
assert cache_format == "fp8_ds_mla"
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
|
||||
165,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
|
||||
|
||||
def _reference_cache(
|
||||
cache_format: str,
|
||||
inputs: dict[str, torch.Tensor],
|
||||
initial_cache: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
reference_cache = initial_cache.clone()
|
||||
valid_slots = _owned(slots)
|
||||
if cache_format == "fp8_ds_mla":
|
||||
ops.concat_and_cache_mla(
|
||||
_owned(inputs["kv_c"]),
|
||||
_owned(inputs["k_pe"]),
|
||||
reference_cache,
|
||||
valid_slots,
|
||||
cache_format,
|
||||
torch.ones(1, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
return reference_cache
|
||||
|
||||
latent = torch.cat(
|
||||
(_owned(inputs["kv_c"]), _owned(inputs["k_pe"])),
|
||||
dim=-1,
|
||||
)
|
||||
flat_reference_cache = reference_cache.view(-1, CACHE_ENTRY)
|
||||
for row, slot in zip(latent, valid_slots.tolist()):
|
||||
flat_reference_cache[slot].copy_(row.to(reference_cache.dtype))
|
||||
return reference_cache
|
||||
|
||||
|
||||
def _assert_cache_matches_reference(
|
||||
cache: torch.Tensor,
|
||||
reference_cache: torch.Tensor,
|
||||
initial_cache: torch.Tensor,
|
||||
) -> None:
|
||||
assert torch.equal(cache, reference_cache)
|
||||
for slot in (0, 5):
|
||||
assert not torch.equal(cache[0, slot], initial_cache[0, slot])
|
||||
for slot in (1, 2, 3, 4, 6, 7):
|
||||
assert torch.equal(cache[0, slot], initial_cache[0, slot])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_format", ["bf16", "fp8_ds_mla"])
|
||||
def test_prefill_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache(cache_format)
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
|
||||
op = (
|
||||
fused_mla_key_concat_kv_cache_insert
|
||||
if cache_format == "bf16"
|
||||
else fused_mla_key_concat_ds_mla_insert
|
||||
)
|
||||
output = op(
|
||||
inputs["q"],
|
||||
inputs["k_nope"],
|
||||
inputs["k_pe"],
|
||||
inputs["kv_c"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
output, _full_key(inputs["k_nope"], inputs["k_pe"]), rtol=0, atol=0
|
||||
)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
def test_fp8_prefill_outputs_ignore_negative_slots_for_cache() -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache("fp8")
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache("fp8", inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
|
||||
q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
inputs["q"],
|
||||
inputs["k_nope"],
|
||||
inputs["k_pe"],
|
||||
inputs["kv_c"],
|
||||
inputs["v"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
)
|
||||
|
||||
expected = (
|
||||
inputs["q"].to(torch.float8_e4m3fn),
|
||||
_full_key(inputs["k_nope"], inputs["k_pe"]).to(torch.float8_e4m3fn),
|
||||
inputs["v"].to(torch.float8_e4m3fn),
|
||||
)
|
||||
for actual, reference in zip((q, k, v), expected):
|
||||
torch.testing.assert_close(actual.float(), reference.float(), rtol=0, atol=0)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_format", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache(cache_format)
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
kwargs = {
|
||||
"ds_mla": cache_format == "fp8_ds_mla",
|
||||
"q_scale_inv": scale_inv if cache_format == "fp8" else None,
|
||||
"cache_scale_inv": scale_inv if cache_format == "fp8" else None,
|
||||
}
|
||||
|
||||
output = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
inputs["ql_nope"],
|
||||
inputs["q_pe"],
|
||||
inputs["kv_c"],
|
||||
inputs["k_pe"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
|
||||
if cache_format == "fp8":
|
||||
expected = expected.to(torch.float8_e4m3fn)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
else:
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
def test_ds_mla_cache_insert_bit_compatible_with_reference() -> None:
|
||||
"""Fused ds_mla insertion must match the reference bit-for-bit."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_blocks = 33, 16
|
||||
dt = torch.bfloat16
|
||||
kv_c = torch.randn(num_tokens, KV_LORA_RANK, device="cuda", dtype=dt)
|
||||
k_pe = torch.randn(num_tokens, 1, ROPE_HEAD_DIM, device="cuda", dtype=dt)
|
||||
k_nope = torch.randn(num_tokens, NUM_HEADS, NOPE_HEAD_DIM, device="cuda", dtype=dt)
|
||||
q = torch.randn(num_tokens, NUM_HEADS, QK_HEAD_DIM, device="cuda", dtype=dt)
|
||||
slots = torch.randperm(num_blocks * BLOCK_SIZE, device="cuda")[:num_tokens].long()
|
||||
slots[torch.rand(num_tokens, device="cuda") < 0.3] = -1
|
||||
ref = torch.full(
|
||||
(num_blocks, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
|
||||
165,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
got = ref.clone()
|
||||
scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale)
|
||||
fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots)
|
||||
assert torch.equal(ref, got)
|
||||
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
requires_flashinfer_mla = pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(100),
|
||||
reason="FlashInfer MLA requires compute capability 10 or above.",
|
||||
)
|
||||
|
||||
|
||||
def test_mla_dcp_gathered_query_reserves_backend_head_storage():
|
||||
from vllm.v1.attention.ops.dcp_utils import reserve_query_head_storage
|
||||
|
||||
query = torch.randn(3, 24, 576, dtype=torch.bfloat16)
|
||||
|
||||
padded = reserve_query_head_storage(query, 128)
|
||||
|
||||
assert padded.shape == query.shape
|
||||
assert padded.stride() == query.stride()
|
||||
assert padded.untyped_storage().nbytes() >= 3 * 128 * 576 * 2
|
||||
assert torch.equal(padded, query)
|
||||
|
||||
|
||||
@requires_flashinfer_mla
|
||||
def test_flashinfer_mla_selects_backend_for_gathered_heads():
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla import (
|
||||
_select_mla_decode_backend,
|
||||
)
|
||||
|
||||
assert _select_mla_decode_backend(6) is None
|
||||
assert _select_mla_decode_backend(24) == "cute-dsl"
|
||||
|
||||
|
||||
@requires_flashinfer_mla
|
||||
def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla as flashinfer_mla
|
||||
|
||||
impl = MagicMock()
|
||||
impl.bmm1_scale = 1.0
|
||||
impl.bmm2_scale = 1.0
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.num_heads = 6
|
||||
impl.qk_nope_head_dim = 128
|
||||
impl.kv_lora_rank = 512
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.kv_cache_dtype = "auto"
|
||||
|
||||
attn_metadata = MagicMock()
|
||||
attn_metadata.causal = True
|
||||
attn_metadata.num_decode_tokens = 2
|
||||
attn_metadata.num_decodes = 2
|
||||
attn_metadata.max_seq_len = 1
|
||||
attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32)
|
||||
attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32)
|
||||
|
||||
query = torch.ones(2, 24, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16)
|
||||
kernel = MagicMock(
|
||||
return_value=(
|
||||
torch.ones(2, 1, 24, 512, dtype=torch.bfloat16),
|
||||
torch.ones(2, 24, dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_mla, "_get_workspace_buffer", MagicMock())
|
||||
monkeypatch.setattr(flashinfer_mla, "trtllm_batch_decode_with_kv_cache_mla", kernel)
|
||||
layer = MagicMock()
|
||||
|
||||
output, lse = flashinfer_mla.FlashInferMLAImpl.forward_mqa(
|
||||
impl, query, kv_cache, attn_metadata, layer
|
||||
)
|
||||
|
||||
assert kernel.call_args.kwargs["backend"] == "cute-dsl"
|
||||
assert output.shape == (2, 24, 512)
|
||||
assert lse is not None and lse.shape == (2, 24)
|
||||
Reference in New Issue
Block a user