[New model] Kimi K3 (#50000)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-07-30 03:49:58 -07:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex Wei Zhao
parent 0c64be8873
commit aeeb36b1f1
82 changed files with 2927 additions and 1345 deletions
+32
View File
@@ -25,6 +25,9 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantDesc,
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
deepgemm_moe_permute,
)
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
)
@@ -41,6 +44,35 @@ from vllm.utils.deep_gemm import (
BLOCK_SIZE = [128, 128]
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
activations, scales = per_token_group_quant_fp8(
hidden_states,
group_size=128,
use_ue8m0=True,
)
topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64)
_, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute(
aq=activations,
aq_scale=scales,
topk_ids=topk_ids,
local_num_experts=2,
expert_map=None,
expert_tokens_meta=None,
)
padding = expert_ids < 0
assert padding.any()
torch.testing.assert_close(
permuted_scales[padding],
torch.zeros_like(permuted_scales[padding]),
rtol=0,
atol=0,
)
def make_block_quant_fp8_weights(
e: int,
n: int,
+18
View File
@@ -1026,6 +1026,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"moonshotai/Kimi-K2.5",
trust_remote_code=True,
),
"KimiK3ForConditionalGeneration": _HfExamplesInfo(
"moonshotai/Kimi-K3",
trust_remote_code=True,
),
"KimiVLForConditionalGeneration": _HfExamplesInfo(
"moonshotai/Kimi-VL-A3B-Instruct",
extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
@@ -1466,6 +1470,14 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
is_available_online=False,
use_original_num_layers=True, # DSpark has >1 draft block
),
"K3DSparkModel": _HfExamplesInfo(
"moonshotai/Kimi-K3",
speculative_model="Inferact/Kimi-K3-DSpark",
use_original_num_layers=True, # DSpark has >1 draft block
trust_remote_code=True,
# FIXME: Investigate the NVML failure in CI.
is_available_online=False,
),
"Qwen3DSparkModel": _HfExamplesInfo(
"Qwen/Qwen3-8B",
speculative_model="deepseek-ai/dspark_qwen3_8b_block7",
@@ -1664,6 +1676,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
trust_remote_code=True,
max_model_len=4096,
),
"KimiK3MTPModel": _HfExamplesInfo(
"moonshotai/Kimi-K3",
speculative_model="moonshotai/Kimi-K3",
trust_remote_code=True,
is_available_online=False,
),
"LongCatFlashMTPModel": _HfExamplesInfo(
"meituan-longcat/LongCat-Flash-Chat",
trust_remote_code=True,
+143
View File
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.model_executor.models.registry import ModelRegistry
from vllm.models.kimi_k3.nvidia import dspark_mla
from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel
def test_dspark_mla_uses_compile_free_model_entrypoint():
assert ModelRegistry._try_load_model_cls("K3DSparkModel") is K3DSparkForCausalLM
assert not issubclass(K3DSparkModel, TorchCompileWithNoGuardsWrapper)
@pytest.mark.parametrize(
("checkpoint_name", "runtime_name", "shard_id"),
[
(
"layers.0.self_attn.q_a_proj.weight",
"model.layers.0.self_attn.fused_qkv_a_proj.weight",
0,
),
(
"layers.0.self_attn.kv_a_proj_with_mqa.weight",
"model.layers.0.self_attn.fused_qkv_a_proj.weight",
1,
),
(
"layers.0.mlp.gate_proj.weight",
"model.layers.0.mlp.gate_up_proj.weight",
0,
),
(
"layers.0.mlp.up_proj.weight",
"model.layers.0.mlp.gate_up_proj.weight",
1,
),
("context_proj.weight", "model.context_proj.weight", None),
],
)
def test_dspark_mla_checkpoint_weight_mapping(checkpoint_name, runtime_name, shard_id):
assert K3DSparkForCausalLM.hf_to_vllm_mapper._map_name_with_shard(
checkpoint_name
) == (runtime_name, shard_id)
def test_dspark_mla_shares_frozen_target_weights_and_skips_training_head():
assert not K3DSparkForCausalLM.has_own_embed_tokens
assert not K3DSparkForCausalLM.has_own_lm_head
assert set(K3DSparkForCausalLM.checkpoint_skip_substrs) == {
"confidence_head",
"embed_tokens",
"lm_head",
}
@pytest.mark.cpu_test
def test_dspark_markov_head_is_replicated(
monkeypatch: pytest.MonkeyPatch,
):
from vllm.model_executor.layers import logits_processor, vocab_parallel_embedding
monkeypatch.setattr(
vocab_parallel_embedding, "get_tensor_model_parallel_rank", lambda: 3
)
monkeypatch.setattr(
vocab_parallel_embedding,
"get_tensor_model_parallel_world_size",
lambda: 8,
)
monkeypatch.setattr(
logits_processor,
"get_current_vllm_config",
lambda: SimpleNamespace(model_config=None),
)
head = DSparkMarkovHead(128, 128, 8, prefix="markov_head")
assert head.markov_w2.tp_size == 1
assert head.markov_w1.weight.shape == (128, 8)
assert head.markov_w2.weight.shape == (128, 8)
def fail_collective(*args, **kwargs):
raise AssertionError("replicated Markov head must not invoke TP collectives")
monkeypatch.setattr(
vocab_parallel_embedding,
"tensor_model_parallel_all_reduce",
fail_collective,
)
logits_processor = LogitsProcessor(128)
monkeypatch.setattr(logits_processor, "_gather_logits", fail_collective)
markov_embed = head.embed(torch.tensor([1, 2]))
bias = head.bias(markov_embed, logits_processor)
assert markov_embed.shape == (2, 8)
assert bias.shape == (2, 128)
@pytest.mark.cpu_test
def test_k3_dspark_uses_replicated_markov_head(monkeypatch: pytest.MonkeyPatch):
markov_head_calls = []
class DummyModule(nn.Module):
def __init__(self, *args, **kwargs):
super().__init__()
def make_markov_head(*args, **kwargs):
markov_head_calls.append((args, kwargs))
return DummyModule()
monkeypatch.setattr(dspark_mla, "get_draft_quant_config", lambda _: None)
monkeypatch.setattr(dspark_mla, "ReplicatedLinear", DummyModule)
monkeypatch.setattr(dspark_mla, "RMSNorm", DummyModule)
monkeypatch.setattr(dspark_mla, "K3DSparkDecoderLayer", DummyModule)
monkeypatch.setattr(dspark_mla, "DSparkMarkovHead", make_markov_head)
config = SimpleNamespace(
target_hidden_size=16,
num_target_layers=2,
hidden_size=8,
rms_norm_eps=1e-6,
num_hidden_layers=1,
vocab_size=128,
draft_vocab_size=128,
markov_rank=4,
)
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(hf_config=config)
)
)
K3DSparkModel(vllm_config=vllm_config, start_layer_id=0, prefix="model")
assert len(markov_head_calls) == 1
+20
View File
@@ -1235,6 +1235,26 @@ def test_vllm_config_defaults_are_none():
assert getattr(config.compilation_config, k) is None
def test_validate_mamba_align_subblock_prefill():
"""Align mode permits configured prefill chunks smaller than a block."""
config = SimpleNamespace(
cache_config=SimpleNamespace(
block_size=11392,
mamba_cache_mode="align",
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
),
scheduler_config=SimpleNamespace(
max_num_batched_tokens=8192,
long_prefill_token_threshold=4096,
disable_chunked_mm_input=False,
),
)
VllmConfig.validate_block_size(config)
@pytest.mark.parametrize(
("model_id", "compilation_config", "optimization_level"),
[
@@ -0,0 +1,165 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
from vllm.transformers_utils.config import get_config
from vllm.transformers_utils.configs.k3_dspark import K3DSparkConfig
def _write_dspark_config(path, **overrides):
path.mkdir()
config = {
"architectures": ["K3DSparkModel"],
"model_type": "k3_dspark",
"hidden_size": 7168,
"intermediate_size": 14336,
"num_hidden_layers": 5,
"num_attention_heads": 64,
"num_key_value_heads": 64,
"q_lora_rank": 1536,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"vocab_size": 163840,
"rms_norm_eps": 1e-5,
"max_position_embeddings": 32768,
"rope_theta": 50000.0,
"num_target_layers": 5,
"target_hidden_size": 7168,
"target_num_hidden_layers": 93,
"target_layer_ids": [2, 23, 47, 71, 89],
"markov_rank": 256,
"draft_vocab_size": 163840,
"torch_dtype": "bfloat16",
}
config.update(overrides)
(path / "config.json").write_text(json.dumps(config))
def _write_target_config(path):
path.mkdir()
config = {
"architectures": ["LlamaForCausalLM"],
"model_type": "llama",
"hidden_size": 7168,
"intermediate_size": 14336,
"num_hidden_layers": 93,
"num_attention_heads": 56,
"num_key_value_heads": 8,
"vocab_size": 163840,
"max_position_embeddings": 32768,
"torch_dtype": "bfloat16",
}
(path / "config.json").write_text(json.dumps(config))
def test_dspark_mla_config_loads_from_local_json(tmp_path):
draft_path = tmp_path / "draft"
_write_dspark_config(draft_path)
config = get_config(draft_path, trust_remote_code=False)
assert isinstance(config, K3DSparkConfig)
assert config.model_type == "k3_dspark"
assert config.architectures == ["K3DSparkModel"]
assert config.hidden_act == "silu"
assert config.rope_parameters == {
"rope_type": "default",
"rope_theta": 50000.0,
}
assert config.n_routed_experts == 0
assert config.draft_vocab_size == config.vocab_size
@pytest.mark.parametrize(
"overrides",
[
{"mla_use_nope": True},
{"mla_use_output_gate": True},
{"mla_use_qk_norm": True},
{"dspark_bonus_anchor": True},
{"q_lora_rank": None},
{"draft_vocab_size": 8192},
{"target_layer_ids": []},
{"num_target_layers": 4},
],
)
def test_dspark_mla_rejects_unsupported_checkpoint_options(tmp_path, overrides):
draft_path = tmp_path / "draft"
_write_dspark_config(draft_path, **overrides)
with pytest.raises(ValueError, match="MLA DSpark"):
get_config(draft_path, trust_remote_code=False)
def test_dspark_mla_uses_latent_kv_geometry(tmp_path):
draft_path = tmp_path / "draft"
_write_dspark_config(draft_path)
model_config = ModelConfig(
model=str(draft_path),
tokenizer_mode="skip",
runner="draft",
max_model_len=32768,
)
assert model_config.is_deepseek_mla
assert model_config.use_mla
assert model_config.get_head_size() == 576
# external_launcher skips ParallelConfig's local-GPU-count check so the
# config logic can be exercised at TP8 on a single-GPU test node.
parallel_config = ParallelConfig(
tensor_parallel_size=8, distributed_executor_backend="external_launcher"
)
assert model_config.get_num_kv_heads(parallel_config) == 1
assert model_config.get_num_attention_heads(parallel_config) == 8
assert model_config.get_num_experts() == 0
def test_dspark_mla_speculative_config_preserves_architecture(tmp_path):
target_path = tmp_path / "target"
draft_path = tmp_path / "draft"
_write_target_config(target_path)
_write_dspark_config(draft_path)
target_config = ModelConfig(
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
)
speculative_config = SpeculativeConfig(
model=str(draft_path),
method="dspark",
num_speculative_tokens=8,
target_model_config=target_config,
target_parallel_config=ParallelConfig(),
)
assert speculative_config.parallel_drafting
assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"]
assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark"
assert speculative_config.draft_model_config.use_mla
def test_dspark_mla_rejects_decode_context_parallelism(tmp_path):
target_path = tmp_path / "target"
draft_path = tmp_path / "draft"
_write_target_config(target_path)
_write_dspark_config(draft_path)
target_config = ModelConfig(
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
)
with pytest.raises(ValueError, match="does not currently support decode context"):
SpeculativeConfig(
model=str(draft_path),
method="dspark",
num_speculative_tokens=8,
target_model_config=target_config,
target_parallel_config=ParallelConfig(
tensor_parallel_size=2,
decode_context_parallel_size=2,
distributed_executor_backend="external_launcher",
),
)
+5 -1
View File
@@ -73,7 +73,11 @@ DEVICE_TYPE = current_platform.device_type
def test_mla_kv_cache_spec_uses_layer_cache_dtype(
cache_dtype: str, expected_quant_mode: KVQuantMode
):
layer = SimpleNamespace(kv_cache_dtype=cache_dtype, head_size=576)
layer = SimpleNamespace(
kv_cache_dtype=cache_dtype,
head_size=576,
non_causal_multi_token_decode=False,
)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=64), model_config=None
)
+126
View File
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonMetadata,
MLACommonMetadataBuilder,
QueryLenSupport,
)
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import MLAAttentionSpec
class _NonCausalMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
supports_non_causal_multi_token_decode = True
def _metadata(
query_start_loc: list[int], num_tokens: int | None = None
) -> CommonAttentionMetadata:
num_reqs = len(query_start_loc) - 1
num_tokens = query_start_loc[-1] if num_tokens is None else num_tokens
return CommonAttentionMetadata(
query_start_loc=torch.tensor(query_start_loc, dtype=torch.int32),
query_start_loc_cpu=torch.tensor(query_start_loc, dtype=torch.int32),
seq_lens=torch.arange(1, num_reqs + 1, dtype=torch.int32) * 100 + 8,
num_reqs=num_reqs,
num_actual_tokens=num_tokens,
max_query_len=max(
end - start for start, end in zip(query_start_loc, query_start_loc[1:])
),
max_seq_len=num_reqs * 100 + 8,
block_table_tensor=torch.arange(num_reqs * 3, dtype=torch.int32).view(
num_reqs, 3
),
slot_mapping=torch.arange(num_tokens),
causal=False,
seq_lens_cpu_upper_bound=None,
)
def _builder(marked: bool = True) -> _NonCausalMLAMetadataBuilder:
builder = object.__new__(_NonCausalMLAMetadataBuilder)
builder.device = torch.device("cpu")
builder.reorder_batch_threshold = 1
builder.query_len_support = QueryLenSupport.SINGLE_ONLY
builder.non_causal_multi_token_decode = marked
builder.dcp_world_size = 1
builder.metadata_cls = MLACommonMetadata
builder.model_config = SimpleNamespace(
dtype=torch.bfloat16, get_head_size=lambda: 576
)
return builder
def test_noncausal_block_uses_decode_without_cpu_lengths():
common_metadata = _metadata([0, 8, 16])
metadata = _builder().build(0, common_metadata)
assert metadata.num_decodes == 2
assert metadata.num_decode_tokens == 16
assert metadata.num_prefills == 0
assert metadata.prefill is None
assert metadata.decode is not None
assert metadata.decode.block_table.shape == (2, 3)
assert metadata.decode.seq_lens.shape == (2,)
assert torch.equal(metadata.decode.block_table, common_metadata.block_table_tensor)
assert torch.equal(metadata.decode.seq_lens, common_metadata.seq_lens)
assert not metadata.causal
def test_noncausal_support_is_explicit_and_uniform():
with pytest.raises(ValueError, match="explicitly supported"):
_builder(marked=False).build(0, _metadata([0, 8, 16]))
with pytest.raises(ValueError, match="uniform query block"):
_builder().build(0, _metadata([0, 3, 8]))
def test_noncausal_block_allows_trailing_cudagraph_padding():
common_metadata = _metadata([0, 8, 16, 16], num_tokens=24)
common_metadata.seq_lens[-1] = 0
metadata = _builder().build(0, common_metadata)
assert metadata.num_decodes == 3
assert metadata.num_decode_tokens == 24
assert metadata.decode is not None
assert metadata.decode.seq_lens.tolist() == [108, 208, 0]
def test_noncausal_block_rejects_non_trailing_padding():
with pytest.raises(ValueError, match="uniform query block"):
_builder().build(0, _metadata([0, 8, 8, 16], num_tokens=24))
def test_noncausal_decode_metadata_keeps_live_request_buffers():
common_metadata = _metadata([0, 8, 16])
metadata = _builder().build(0, common_metadata)
assert metadata.decode is not None
assert metadata.decode.seq_lens.data_ptr() == common_metadata.seq_lens.data_ptr()
assert (
metadata.decode.block_table.data_ptr()
== common_metadata.block_table_tensor.data_ptr()
)
def test_mla_cache_marker_is_promoted_to_group_capability():
kwargs = {
"block_size": 64,
"num_kv_heads": 1,
"head_size": 576,
"dtype": torch.bfloat16,
}
marked = MLAAttentionSpec(**kwargs, non_causal_multi_token_decode=True)
unmarked = MLAAttentionSpec(**kwargs)
assert MLAAttentionSpec.merge([marked, marked]).non_causal_multi_token_decode
assert MLAAttentionSpec.merge([marked, unmarked]).non_causal_multi_token_decode
assert not MLAAttentionSpec.merge(
[unmarked, unmarked]
).non_causal_multi_token_decode
@@ -40,6 +40,8 @@ def test_mamba_align_split_partial_tail_schedule():
hash_block_size = 32
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
hash_block_size=hash_block_size,
mamba_partial_cache_hit=True,
@@ -75,6 +77,78 @@ def test_mamba_align_split_partial_tail_schedule():
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
def test_mamba_align_split_when_block_exceeds_scheduling_budget():
"""Sub-block chunks make progress only when no step can fit a full block."""
block_size = 11392
token_budget = 8192
prompt_length = 30000
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
split = Scheduler._mamba_block_aligned_split
mock.max_num_scheduled_tokens = block_size
assert split(self=mock, request=req, num_new_tokens=token_budget) == 0
mock.max_num_scheduled_tokens = token_budget
scheduled_chunks = []
while req.num_computed_tokens < prompt_length:
num_new_tokens = min(token_budget, prompt_length - req.num_computed_tokens)
num_scheduled_tokens = split(
self=mock,
request=req,
num_new_tokens=num_new_tokens,
)
assert 0 < num_scheduled_tokens <= token_budget
scheduled_chunks.append(num_scheduled_tokens)
req.num_computed_tokens += num_scheduled_tokens
assert scheduled_chunks == [8192, 3200, 8192, 3200, 7216]
def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
"""A long-prefill cap below the block size permits sub-block progress."""
block_size = 512
token_budget = 8192
long_prefill_threshold = 384
prompt_length = 1300
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(
long_prefill_token_threshold=long_prefill_threshold
),
use_eagle=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
split = Scheduler._mamba_block_aligned_split
scheduled_chunks = []
while req.num_computed_tokens < prompt_length:
num_new_tokens = min(
long_prefill_threshold,
prompt_length - req.num_computed_tokens,
)
num_scheduled_tokens = split(
self=mock,
request=req,
num_new_tokens=num_new_tokens,
)
assert 0 < num_scheduled_tokens <= long_prefill_threshold
scheduled_chunks.append(num_scheduled_tokens)
req.num_computed_tokens += num_scheduled_tokens
assert scheduled_chunks == [384, 128, 384, 128, 276]
def test_hybrid_mamba_align_partial_hash_hit():
hash_block_size = 2
mamba_block_size = 2 * hash_block_size
+2
View File
@@ -1078,6 +1078,8 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
# Create minimal mock with just the needed attributes
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=3 * block_size,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
hash_block_size=block_size,
mamba_partial_cache_hit=False,
@@ -333,6 +333,54 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
assert torch.equal(x, torch.full((4,), 15.0, device="cuda"))
def test_eager_attention_inside_multistream_overlap(cuda_capture_stream):
"""Handle an eager attention break inside a multi-stream overlap region."""
from vllm.compilation.breakable_cudagraph import (
BreakableCUDAGraphCapture,
eager_break_during_capture,
)
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
x = torch.zeros(1024, device="cuda")
output = torch.empty_like(x)
aux_stream = torch.cuda.Stream()
main_event = torch.cuda.Event()
aux_event = torch.cuda.Event()
@eager_break_during_capture
def attention(query: torch.Tensor, out: torch.Tensor) -> None:
torch.add(query, 3.0, out=out)
def attention_frontend() -> torch.Tensor:
query = x * 2.0
attention_output = torch.empty_like(x)
attention(query, attention_output)
return attention_output
cap = BreakableCUDAGraphCapture()
with cap:
attention_output, gate = maybe_execute_in_parallel(
attention_frontend,
lambda: x * 5.0,
main_event,
aux_event,
aux_stream,
)
torch.add(attention_output, gate, out=output)
assert cap.num_graphs == 2
assert cap.num_eager_breaks == 1
for value in (1.0, 7.0, 19.0):
x.fill_(value)
cap.replay()
cuda_capture_stream.synchronize()
torch.testing.assert_close(
output,
torch.full_like(output, value * 7.0 + 3.0),
)
# ---------------------------------------------------------------------------
# Replay ordering
# ---------------------------------------------------------------------------
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
@@ -8,6 +11,7 @@ from vllm.utils.torch_utils import set_random_seed
from vllm.v1.sample.logits_processor import LogitsProcessors
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.spec_decode.llm_base_proposer import (
SpecDecodeBaseProposer,
compute_probs_and_sample_next_token,
)
@@ -68,3 +72,21 @@ def test_compute_probs_and_sample_next_token_uses_fp64_exponential_race():
assert torch.equal(actual_ids, expected_ids)
assert torch.allclose(actual_probs, probs)
@pytest.mark.parametrize(
("architecture", "expected"),
[
("DeepSeekMTPModel", True),
("KimiK3MTPModel", True),
("MiniMaxM3ForCausalLM", False),
],
)
def test_mtp_model_returns_tuple(architecture: str, expected: bool):
proposer = object.__new__(SpecDecodeBaseProposer)
proposer.method = "mtp"
proposer.draft_model_config = SimpleNamespace(
hf_config=SimpleNamespace(architectures=[architecture])
)
assert proposer.model_returns_tuple() is expected