[New model] Kimi K3 (#50000)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
Wei Zhao
parent
0c64be8873
commit
aeeb36b1f1
@@ -25,6 +25,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantDesc,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
|
||||
deepgemm_moe_permute,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
)
|
||||
@@ -41,6 +44,35 @@ from vllm.utils.deep_gemm import (
|
||||
BLOCK_SIZE = [128, 128]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
|
||||
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
|
||||
activations, scales = per_token_group_quant_fp8(
|
||||
hidden_states,
|
||||
group_size=128,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64)
|
||||
|
||||
_, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute(
|
||||
aq=activations,
|
||||
aq_scale=scales,
|
||||
topk_ids=topk_ids,
|
||||
local_num_experts=2,
|
||||
expert_map=None,
|
||||
expert_tokens_meta=None,
|
||||
)
|
||||
|
||||
padding = expert_ids < 0
|
||||
assert padding.any()
|
||||
torch.testing.assert_close(
|
||||
permuted_scales[padding],
|
||||
torch.zeros_like(permuted_scales[padding]),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
def make_block_quant_fp8_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
|
||||
@@ -1026,6 +1026,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"moonshotai/Kimi-K2.5",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"KimiK3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-K3",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"KimiVLForConditionalGeneration": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-VL-A3B-Instruct",
|
||||
extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
|
||||
@@ -1466,6 +1470,14 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True, # DSpark has >1 draft block
|
||||
),
|
||||
"K3DSparkModel": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-K3",
|
||||
speculative_model="Inferact/Kimi-K3-DSpark",
|
||||
use_original_num_layers=True, # DSpark has >1 draft block
|
||||
trust_remote_code=True,
|
||||
# FIXME: Investigate the NVML failure in CI.
|
||||
is_available_online=False,
|
||||
),
|
||||
"Qwen3DSparkModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
speculative_model="deepseek-ai/dspark_qwen3_8b_block7",
|
||||
@@ -1664,6 +1676,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
max_model_len=4096,
|
||||
),
|
||||
"KimiK3MTPModel": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-K3",
|
||||
speculative_model="moonshotai/Kimi-K3",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"LongCatFlashMTPModel": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Chat",
|
||||
trust_remote_code=True,
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.model_executor.models.registry import ModelRegistry
|
||||
from vllm.models.kimi_k3.nvidia import dspark_mla
|
||||
from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel
|
||||
|
||||
|
||||
def test_dspark_mla_uses_compile_free_model_entrypoint():
|
||||
assert ModelRegistry._try_load_model_cls("K3DSparkModel") is K3DSparkForCausalLM
|
||||
assert not issubclass(K3DSparkModel, TorchCompileWithNoGuardsWrapper)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("checkpoint_name", "runtime_name", "shard_id"),
|
||||
[
|
||||
(
|
||||
"layers.0.self_attn.q_a_proj.weight",
|
||||
"model.layers.0.self_attn.fused_qkv_a_proj.weight",
|
||||
0,
|
||||
),
|
||||
(
|
||||
"layers.0.self_attn.kv_a_proj_with_mqa.weight",
|
||||
"model.layers.0.self_attn.fused_qkv_a_proj.weight",
|
||||
1,
|
||||
),
|
||||
(
|
||||
"layers.0.mlp.gate_proj.weight",
|
||||
"model.layers.0.mlp.gate_up_proj.weight",
|
||||
0,
|
||||
),
|
||||
(
|
||||
"layers.0.mlp.up_proj.weight",
|
||||
"model.layers.0.mlp.gate_up_proj.weight",
|
||||
1,
|
||||
),
|
||||
("context_proj.weight", "model.context_proj.weight", None),
|
||||
],
|
||||
)
|
||||
def test_dspark_mla_checkpoint_weight_mapping(checkpoint_name, runtime_name, shard_id):
|
||||
assert K3DSparkForCausalLM.hf_to_vllm_mapper._map_name_with_shard(
|
||||
checkpoint_name
|
||||
) == (runtime_name, shard_id)
|
||||
|
||||
|
||||
def test_dspark_mla_shares_frozen_target_weights_and_skips_training_head():
|
||||
assert not K3DSparkForCausalLM.has_own_embed_tokens
|
||||
assert not K3DSparkForCausalLM.has_own_lm_head
|
||||
assert set(K3DSparkForCausalLM.checkpoint_skip_substrs) == {
|
||||
"confidence_head",
|
||||
"embed_tokens",
|
||||
"lm_head",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_dspark_markov_head_is_replicated(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from vllm.model_executor.layers import logits_processor, vocab_parallel_embedding
|
||||
|
||||
monkeypatch.setattr(
|
||||
vocab_parallel_embedding, "get_tensor_model_parallel_rank", lambda: 3
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
vocab_parallel_embedding,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 8,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
logits_processor,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=None),
|
||||
)
|
||||
|
||||
head = DSparkMarkovHead(128, 128, 8, prefix="markov_head")
|
||||
assert head.markov_w2.tp_size == 1
|
||||
assert head.markov_w1.weight.shape == (128, 8)
|
||||
assert head.markov_w2.weight.shape == (128, 8)
|
||||
|
||||
def fail_collective(*args, **kwargs):
|
||||
raise AssertionError("replicated Markov head must not invoke TP collectives")
|
||||
|
||||
monkeypatch.setattr(
|
||||
vocab_parallel_embedding,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
fail_collective,
|
||||
)
|
||||
logits_processor = LogitsProcessor(128)
|
||||
monkeypatch.setattr(logits_processor, "_gather_logits", fail_collective)
|
||||
|
||||
markov_embed = head.embed(torch.tensor([1, 2]))
|
||||
bias = head.bias(markov_embed, logits_processor)
|
||||
assert markov_embed.shape == (2, 8)
|
||||
assert bias.shape == (2, 128)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_k3_dspark_uses_replicated_markov_head(monkeypatch: pytest.MonkeyPatch):
|
||||
markov_head_calls = []
|
||||
|
||||
class DummyModule(nn.Module):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
def make_markov_head(*args, **kwargs):
|
||||
markov_head_calls.append((args, kwargs))
|
||||
return DummyModule()
|
||||
|
||||
monkeypatch.setattr(dspark_mla, "get_draft_quant_config", lambda _: None)
|
||||
monkeypatch.setattr(dspark_mla, "ReplicatedLinear", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "RMSNorm", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "K3DSparkDecoderLayer", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "DSparkMarkovHead", make_markov_head)
|
||||
|
||||
config = SimpleNamespace(
|
||||
target_hidden_size=16,
|
||||
num_target_layers=2,
|
||||
hidden_size=8,
|
||||
rms_norm_eps=1e-6,
|
||||
num_hidden_layers=1,
|
||||
vocab_size=128,
|
||||
draft_vocab_size=128,
|
||||
markov_rank=4,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(hf_config=config)
|
||||
)
|
||||
)
|
||||
|
||||
K3DSparkModel(vllm_config=vllm_config, start_layer_id=0, prefix="model")
|
||||
|
||||
assert len(markov_head_calls) == 1
|
||||
@@ -1235,6 +1235,26 @@ def test_vllm_config_defaults_are_none():
|
||||
assert getattr(config.compilation_config, k) is None
|
||||
|
||||
|
||||
def test_validate_mamba_align_subblock_prefill():
|
||||
"""Align mode permits configured prefill chunks smaller than a block."""
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=11392,
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
),
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_batched_tokens=8192,
|
||||
long_prefill_token_threshold=4096,
|
||||
disable_chunked_mm_input=False,
|
||||
),
|
||||
)
|
||||
|
||||
VllmConfig.validate_block_size(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "compilation_config", "optimization_level"),
|
||||
[
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
|
||||
from vllm.transformers_utils.config import get_config
|
||||
from vllm.transformers_utils.configs.k3_dspark import K3DSparkConfig
|
||||
|
||||
|
||||
def _write_dspark_config(path, **overrides):
|
||||
path.mkdir()
|
||||
config = {
|
||||
"architectures": ["K3DSparkModel"],
|
||||
"model_type": "k3_dspark",
|
||||
"hidden_size": 7168,
|
||||
"intermediate_size": 14336,
|
||||
"num_hidden_layers": 5,
|
||||
"num_attention_heads": 64,
|
||||
"num_key_value_heads": 64,
|
||||
"q_lora_rank": 1536,
|
||||
"kv_lora_rank": 512,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"vocab_size": 163840,
|
||||
"rms_norm_eps": 1e-5,
|
||||
"max_position_embeddings": 32768,
|
||||
"rope_theta": 50000.0,
|
||||
"num_target_layers": 5,
|
||||
"target_hidden_size": 7168,
|
||||
"target_num_hidden_layers": 93,
|
||||
"target_layer_ids": [2, 23, 47, 71, 89],
|
||||
"markov_rank": 256,
|
||||
"draft_vocab_size": 163840,
|
||||
"torch_dtype": "bfloat16",
|
||||
}
|
||||
config.update(overrides)
|
||||
(path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
def _write_target_config(path):
|
||||
path.mkdir()
|
||||
config = {
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"model_type": "llama",
|
||||
"hidden_size": 7168,
|
||||
"intermediate_size": 14336,
|
||||
"num_hidden_layers": 93,
|
||||
"num_attention_heads": 56,
|
||||
"num_key_value_heads": 8,
|
||||
"vocab_size": 163840,
|
||||
"max_position_embeddings": 32768,
|
||||
"torch_dtype": "bfloat16",
|
||||
}
|
||||
(path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
def test_dspark_mla_config_loads_from_local_json(tmp_path):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(draft_path)
|
||||
|
||||
config = get_config(draft_path, trust_remote_code=False)
|
||||
|
||||
assert isinstance(config, K3DSparkConfig)
|
||||
assert config.model_type == "k3_dspark"
|
||||
assert config.architectures == ["K3DSparkModel"]
|
||||
assert config.hidden_act == "silu"
|
||||
assert config.rope_parameters == {
|
||||
"rope_type": "default",
|
||||
"rope_theta": 50000.0,
|
||||
}
|
||||
assert config.n_routed_experts == 0
|
||||
assert config.draft_vocab_size == config.vocab_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"mla_use_nope": True},
|
||||
{"mla_use_output_gate": True},
|
||||
{"mla_use_qk_norm": True},
|
||||
{"dspark_bonus_anchor": True},
|
||||
{"q_lora_rank": None},
|
||||
{"draft_vocab_size": 8192},
|
||||
{"target_layer_ids": []},
|
||||
{"num_target_layers": 4},
|
||||
],
|
||||
)
|
||||
def test_dspark_mla_rejects_unsupported_checkpoint_options(tmp_path, overrides):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(draft_path, **overrides)
|
||||
|
||||
with pytest.raises(ValueError, match="MLA DSpark"):
|
||||
get_config(draft_path, trust_remote_code=False)
|
||||
|
||||
|
||||
def test_dspark_mla_uses_latent_kv_geometry(tmp_path):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(draft_path)
|
||||
model_config = ModelConfig(
|
||||
model=str(draft_path),
|
||||
tokenizer_mode="skip",
|
||||
runner="draft",
|
||||
max_model_len=32768,
|
||||
)
|
||||
|
||||
assert model_config.is_deepseek_mla
|
||||
assert model_config.use_mla
|
||||
assert model_config.get_head_size() == 576
|
||||
# external_launcher skips ParallelConfig's local-GPU-count check so the
|
||||
# config logic can be exercised at TP8 on a single-GPU test node.
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=8, distributed_executor_backend="external_launcher"
|
||||
)
|
||||
assert model_config.get_num_kv_heads(parallel_config) == 1
|
||||
assert model_config.get_num_attention_heads(parallel_config) == 8
|
||||
assert model_config.get_num_experts() == 0
|
||||
|
||||
|
||||
def test_dspark_mla_speculative_config_preserves_architecture(tmp_path):
|
||||
target_path = tmp_path / "target"
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_target_config(target_path)
|
||||
_write_dspark_config(draft_path)
|
||||
target_config = ModelConfig(
|
||||
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
|
||||
)
|
||||
speculative_config = SpeculativeConfig(
|
||||
model=str(draft_path),
|
||||
method="dspark",
|
||||
num_speculative_tokens=8,
|
||||
target_model_config=target_config,
|
||||
target_parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
assert speculative_config.parallel_drafting
|
||||
assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"]
|
||||
assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark"
|
||||
assert speculative_config.draft_model_config.use_mla
|
||||
|
||||
|
||||
def test_dspark_mla_rejects_decode_context_parallelism(tmp_path):
|
||||
target_path = tmp_path / "target"
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_target_config(target_path)
|
||||
_write_dspark_config(draft_path)
|
||||
target_config = ModelConfig(
|
||||
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not currently support decode context"):
|
||||
SpeculativeConfig(
|
||||
model=str(draft_path),
|
||||
method="dspark",
|
||||
num_speculative_tokens=8,
|
||||
target_model_config=target_config,
|
||||
target_parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=2,
|
||||
distributed_executor_backend="external_launcher",
|
||||
),
|
||||
)
|
||||
@@ -73,7 +73,11 @@ DEVICE_TYPE = current_platform.device_type
|
||||
def test_mla_kv_cache_spec_uses_layer_cache_dtype(
|
||||
cache_dtype: str, expected_quant_mode: KVQuantMode
|
||||
):
|
||||
layer = SimpleNamespace(kv_cache_dtype=cache_dtype, head_size=576)
|
||||
layer = SimpleNamespace(
|
||||
kv_cache_dtype=cache_dtype,
|
||||
head_size=576,
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=64), model_config=None
|
||||
)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonMetadata,
|
||||
MLACommonMetadataBuilder,
|
||||
QueryLenSupport,
|
||||
)
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
|
||||
|
||||
class _NonCausalMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
supports_non_causal_multi_token_decode = True
|
||||
|
||||
|
||||
def _metadata(
|
||||
query_start_loc: list[int], num_tokens: int | None = None
|
||||
) -> CommonAttentionMetadata:
|
||||
num_reqs = len(query_start_loc) - 1
|
||||
num_tokens = query_start_loc[-1] if num_tokens is None else num_tokens
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=torch.tensor(query_start_loc, dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor(query_start_loc, dtype=torch.int32),
|
||||
seq_lens=torch.arange(1, num_reqs + 1, dtype=torch.int32) * 100 + 8,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=num_tokens,
|
||||
max_query_len=max(
|
||||
end - start for start, end in zip(query_start_loc, query_start_loc[1:])
|
||||
),
|
||||
max_seq_len=num_reqs * 100 + 8,
|
||||
block_table_tensor=torch.arange(num_reqs * 3, dtype=torch.int32).view(
|
||||
num_reqs, 3
|
||||
),
|
||||
slot_mapping=torch.arange(num_tokens),
|
||||
causal=False,
|
||||
seq_lens_cpu_upper_bound=None,
|
||||
)
|
||||
|
||||
|
||||
def _builder(marked: bool = True) -> _NonCausalMLAMetadataBuilder:
|
||||
builder = object.__new__(_NonCausalMLAMetadataBuilder)
|
||||
builder.device = torch.device("cpu")
|
||||
builder.reorder_batch_threshold = 1
|
||||
builder.query_len_support = QueryLenSupport.SINGLE_ONLY
|
||||
builder.non_causal_multi_token_decode = marked
|
||||
builder.dcp_world_size = 1
|
||||
builder.metadata_cls = MLACommonMetadata
|
||||
builder.model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16, get_head_size=lambda: 576
|
||||
)
|
||||
return builder
|
||||
|
||||
|
||||
def test_noncausal_block_uses_decode_without_cpu_lengths():
|
||||
common_metadata = _metadata([0, 8, 16])
|
||||
metadata = _builder().build(0, common_metadata)
|
||||
|
||||
assert metadata.num_decodes == 2
|
||||
assert metadata.num_decode_tokens == 16
|
||||
assert metadata.num_prefills == 0
|
||||
assert metadata.prefill is None
|
||||
assert metadata.decode is not None
|
||||
assert metadata.decode.block_table.shape == (2, 3)
|
||||
assert metadata.decode.seq_lens.shape == (2,)
|
||||
assert torch.equal(metadata.decode.block_table, common_metadata.block_table_tensor)
|
||||
assert torch.equal(metadata.decode.seq_lens, common_metadata.seq_lens)
|
||||
assert not metadata.causal
|
||||
|
||||
|
||||
def test_noncausal_support_is_explicit_and_uniform():
|
||||
with pytest.raises(ValueError, match="explicitly supported"):
|
||||
_builder(marked=False).build(0, _metadata([0, 8, 16]))
|
||||
with pytest.raises(ValueError, match="uniform query block"):
|
||||
_builder().build(0, _metadata([0, 3, 8]))
|
||||
|
||||
|
||||
def test_noncausal_block_allows_trailing_cudagraph_padding():
|
||||
common_metadata = _metadata([0, 8, 16, 16], num_tokens=24)
|
||||
common_metadata.seq_lens[-1] = 0
|
||||
|
||||
metadata = _builder().build(0, common_metadata)
|
||||
|
||||
assert metadata.num_decodes == 3
|
||||
assert metadata.num_decode_tokens == 24
|
||||
assert metadata.decode is not None
|
||||
assert metadata.decode.seq_lens.tolist() == [108, 208, 0]
|
||||
|
||||
|
||||
def test_noncausal_block_rejects_non_trailing_padding():
|
||||
with pytest.raises(ValueError, match="uniform query block"):
|
||||
_builder().build(0, _metadata([0, 8, 8, 16], num_tokens=24))
|
||||
|
||||
|
||||
def test_noncausal_decode_metadata_keeps_live_request_buffers():
|
||||
common_metadata = _metadata([0, 8, 16])
|
||||
metadata = _builder().build(0, common_metadata)
|
||||
|
||||
assert metadata.decode is not None
|
||||
assert metadata.decode.seq_lens.data_ptr() == common_metadata.seq_lens.data_ptr()
|
||||
assert (
|
||||
metadata.decode.block_table.data_ptr()
|
||||
== common_metadata.block_table_tensor.data_ptr()
|
||||
)
|
||||
|
||||
|
||||
def test_mla_cache_marker_is_promoted_to_group_capability():
|
||||
kwargs = {
|
||||
"block_size": 64,
|
||||
"num_kv_heads": 1,
|
||||
"head_size": 576,
|
||||
"dtype": torch.bfloat16,
|
||||
}
|
||||
marked = MLAAttentionSpec(**kwargs, non_causal_multi_token_decode=True)
|
||||
unmarked = MLAAttentionSpec(**kwargs)
|
||||
|
||||
assert MLAAttentionSpec.merge([marked, marked]).non_causal_multi_token_decode
|
||||
assert MLAAttentionSpec.merge([marked, unmarked]).non_causal_multi_token_decode
|
||||
assert not MLAAttentionSpec.merge(
|
||||
[unmarked, unmarked]
|
||||
).non_causal_multi_token_decode
|
||||
@@ -40,6 +40,8 @@ def test_mamba_align_split_partial_tail_schedule():
|
||||
hash_block_size = 32
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=8192,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
@@ -75,6 +77,78 @@ def test_mamba_align_split_partial_tail_schedule():
|
||||
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
|
||||
|
||||
|
||||
def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
||||
"""Sub-block chunks make progress only when no step can fit a full block."""
|
||||
block_size = 11392
|
||||
token_budget = 8192
|
||||
prompt_length = 30000
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
|
||||
mock.max_num_scheduled_tokens = block_size
|
||||
assert split(self=mock, request=req, num_new_tokens=token_budget) == 0
|
||||
mock.max_num_scheduled_tokens = token_budget
|
||||
|
||||
scheduled_chunks = []
|
||||
while req.num_computed_tokens < prompt_length:
|
||||
num_new_tokens = min(token_budget, prompt_length - req.num_computed_tokens)
|
||||
num_scheduled_tokens = split(
|
||||
self=mock,
|
||||
request=req,
|
||||
num_new_tokens=num_new_tokens,
|
||||
)
|
||||
assert 0 < num_scheduled_tokens <= token_budget
|
||||
scheduled_chunks.append(num_scheduled_tokens)
|
||||
req.num_computed_tokens += num_scheduled_tokens
|
||||
|
||||
assert scheduled_chunks == [8192, 3200, 8192, 3200, 7216]
|
||||
|
||||
|
||||
def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
||||
"""A long-prefill cap below the block size permits sub-block progress."""
|
||||
block_size = 512
|
||||
token_budget = 8192
|
||||
long_prefill_threshold = 384
|
||||
prompt_length = 1300
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(
|
||||
long_prefill_token_threshold=long_prefill_threshold
|
||||
),
|
||||
use_eagle=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
|
||||
scheduled_chunks = []
|
||||
while req.num_computed_tokens < prompt_length:
|
||||
num_new_tokens = min(
|
||||
long_prefill_threshold,
|
||||
prompt_length - req.num_computed_tokens,
|
||||
)
|
||||
num_scheduled_tokens = split(
|
||||
self=mock,
|
||||
request=req,
|
||||
num_new_tokens=num_new_tokens,
|
||||
)
|
||||
assert 0 < num_scheduled_tokens <= long_prefill_threshold
|
||||
scheduled_chunks.append(num_scheduled_tokens)
|
||||
req.num_computed_tokens += num_scheduled_tokens
|
||||
|
||||
assert scheduled_chunks == [384, 128, 384, 128, 276]
|
||||
|
||||
|
||||
def test_hybrid_mamba_align_partial_hash_hit():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 2 * hash_block_size
|
||||
|
||||
@@ -1078,6 +1078,8 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
# Create minimal mock with just the needed attributes
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=3 * block_size,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
hash_block_size=block_size,
|
||||
mamba_partial_cache_hit=False,
|
||||
|
||||
@@ -333,6 +333,54 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
|
||||
assert torch.equal(x, torch.full((4,), 15.0, device="cuda"))
|
||||
|
||||
|
||||
def test_eager_attention_inside_multistream_overlap(cuda_capture_stream):
|
||||
"""Handle an eager attention break inside a multi-stream overlap region."""
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
BreakableCUDAGraphCapture,
|
||||
eager_break_during_capture,
|
||||
)
|
||||
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
||||
|
||||
x = torch.zeros(1024, device="cuda")
|
||||
output = torch.empty_like(x)
|
||||
aux_stream = torch.cuda.Stream()
|
||||
main_event = torch.cuda.Event()
|
||||
aux_event = torch.cuda.Event()
|
||||
|
||||
@eager_break_during_capture
|
||||
def attention(query: torch.Tensor, out: torch.Tensor) -> None:
|
||||
torch.add(query, 3.0, out=out)
|
||||
|
||||
def attention_frontend() -> torch.Tensor:
|
||||
query = x * 2.0
|
||||
attention_output = torch.empty_like(x)
|
||||
attention(query, attention_output)
|
||||
return attention_output
|
||||
|
||||
cap = BreakableCUDAGraphCapture()
|
||||
with cap:
|
||||
attention_output, gate = maybe_execute_in_parallel(
|
||||
attention_frontend,
|
||||
lambda: x * 5.0,
|
||||
main_event,
|
||||
aux_event,
|
||||
aux_stream,
|
||||
)
|
||||
torch.add(attention_output, gate, out=output)
|
||||
|
||||
assert cap.num_graphs == 2
|
||||
assert cap.num_eager_breaks == 1
|
||||
|
||||
for value in (1.0, 7.0, 19.0):
|
||||
x.fill_(value)
|
||||
cap.replay()
|
||||
cuda_capture_stream.synchronize()
|
||||
torch.testing.assert_close(
|
||||
output,
|
||||
torch.full_like(output, value * 7.0 + 3.0),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Replay ordering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
@@ -8,6 +11,7 @@ from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.spec_decode.llm_base_proposer import (
|
||||
SpecDecodeBaseProposer,
|
||||
compute_probs_and_sample_next_token,
|
||||
)
|
||||
|
||||
@@ -68,3 +72,21 @@ def test_compute_probs_and_sample_next_token_uses_fp64_exponential_race():
|
||||
|
||||
assert torch.equal(actual_ids, expected_ids)
|
||||
assert torch.allclose(actual_probs, probs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("architecture", "expected"),
|
||||
[
|
||||
("DeepSeekMTPModel", True),
|
||||
("KimiK3MTPModel", True),
|
||||
("MiniMaxM3ForCausalLM", False),
|
||||
],
|
||||
)
|
||||
def test_mtp_model_returns_tuple(architecture: str, expected: bool):
|
||||
proposer = object.__new__(SpecDecodeBaseProposer)
|
||||
proposer.method = "mtp"
|
||||
proposer.draft_model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(architectures=[architecture])
|
||||
)
|
||||
|
||||
assert proposer.model_returns_tuple() is expected
|
||||
|
||||
Reference in New Issue
Block a user