Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-29 12:54:18 +07:00
264 changed files with 13095 additions and 1541 deletions
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""``get_top_tokens`` on the DeepSeek V4 MTP drafter.
The speculator opts into ``use_local_argmax_reduction`` by probing the draft
model for ``get_top_tokens``, and expects it to select the same greedy draft
token the full-vocab argmax would. V4 reaches its LM head through hc_head and
the shared head's RMSNorm, so both methods have to share that transform.
"""
from types import SimpleNamespace
from unittest import mock
import pytest
import torch
from transformers import PreTrainedConfig
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import HCHeadOp
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip(
"get_top_tokens is implemented on the ROCm DeepSeek V4 MTP drafter only.",
allow_module_level=True,
)
from vllm.models.deepseek_v4.amd.mtp import ( # noqa: E402
DeepSeekV4MTP,
DeepSeekV4MultiTokenPredictor,
)
DEVICE_TYPE = current_platform.device_type
START_LAYER_IDX = 61
HIDDEN_SIZE = 4096
HC_MULT = 4
RMS_NORM_EPS = 1e-6
HC_EPS = 1e-6
_EMBEDDING = "vllm.model_executor.layers.vocab_parallel_embedding"
def _make_predictor(vocab_size: int) -> DeepSeekV4MultiTokenPredictor:
config = PreTrainedConfig()
config.hidden_size = HIDDEN_SIZE
config.vocab_size = vocab_size
config.rms_norm_eps = RMS_NORM_EPS
# A single vocab shard, so the reduction returns the local argmax without
# touching the process group.
with (
mock.patch(f"{_EMBEDDING}.get_tensor_model_parallel_rank", return_value=0),
mock.patch(
f"{_EMBEDDING}.get_tensor_model_parallel_world_size", return_value=1
),
):
shared_head = SharedHead(config=config, prefix="shared_head")
generator = torch.Generator().manual_seed(7)
with torch.no_grad():
shared_head.head.weight.normal_(0.0, 0.02, generator=generator)
# The default norm weight is all ones, and the argmax is invariant to
# the norm's per-row rescale, so a uniform gain would hide a skipped
# shared_head. Vary it so the comparison below can see one.
shared_head.norm.weight.normal_(1.0, 0.5, generator=generator)
shared_head = shared_head.to(device=DEVICE_TYPE, dtype=torch.bfloat16)
hc_head_fn = torch.randn(
HC_MULT, HC_MULT * HIDDEN_SIZE, dtype=torch.float32, generator=generator
)
# Non-trivial gate scale and bias: hc_head mixes the hc_mult streams with
# these, and defaults of 0 would make a skipped hc_head harder to see.
hc_head_scale = torch.randn(1, dtype=torch.float32, generator=generator) * 0.1
hc_head_base = torch.randn(HC_MULT, dtype=torch.float32, generator=generator) * 0.1
mtp_layer = SimpleNamespace(
config=config,
hc_mult=HC_MULT,
hc_eps=HC_EPS,
rms_norm_eps=RMS_NORM_EPS,
hc_head_op=HCHeadOp(),
hc_head_fn=(hc_head_fn * 1e-4).to(DEVICE_TYPE),
hc_head_scale=hc_head_scale.to(DEVICE_TYPE),
hc_head_base=hc_head_base.to(DEVICE_TYPE),
shared_head=shared_head,
)
# Only the attributes the two methods read; a real predictor would also
# build the MLA/MoE draft block, which is irrelevant here.
predictor = object.__new__(DeepSeekV4MultiTokenPredictor)
torch.nn.Module.__init__(predictor)
predictor.layers = {str(START_LAYER_IDX): mtp_layer}
predictor.mtp_start_layer_idx = START_LAYER_IDX
predictor.num_mtp_layers = 1
predictor.logits_processor = LogitsProcessor(vocab_size)
return predictor
def _hidden_states(num_tokens: int, seed: int) -> torch.Tensor:
"""The flat pre-hc_head residual the MTP layer returns."""
generator = torch.Generator().manual_seed(seed)
hidden_states = torch.randn(
num_tokens, HC_MULT * HIDDEN_SIZE, dtype=torch.float32, generator=generator
)
return hidden_states.to(device=DEVICE_TYPE, dtype=torch.bfloat16)
def _assert_selects_max_logit(top: torch.Tensor, logits: torch.Tensor) -> None:
"""The selected token must carry the row's maximum logit.
hc_head emits bf16, so rows can tie for the maximum and then no single
index is the right answer; compare the values the two paths select.
"""
assert top.shape == (logits.shape[0],)
selected = logits.gather(1, top.unsqueeze(1)).squeeze(1)
assert torch.equal(selected, logits.max(dim=-1).values)
def test_get_top_tokens_matches_full_argmax(default_vllm_config):
"""Same draft token as the path taken when the reduction is off.
``get_top_tokens`` has to run the same pre-head transform as
``compute_logits``: hc_head over the (T, hc_mult, D) residual, then the
shared head's RMSNorm. Dropping either one returns a different draft token
instead of failing, which is what this comparison catches.
"""
predictor = _make_predictor(vocab_size=512)
for num_tokens in (1, 16, 128):
hidden_states = _hidden_states(num_tokens, seed=11)
logits = predictor.compute_logits(hidden_states)
top = predictor.get_top_tokens(hidden_states)
assert top.dtype == torch.int64
_assert_selects_max_logit(top, logits)
def test_get_top_tokens_honors_padded_vocab(default_vllm_config):
"""Padding entries above ``org_vocab_size`` must never win the argmax."""
vocab_size = 500
predictor = _make_predictor(vocab_size)
head = predictor.layers[str(START_LAYER_IDX)].shared_head.head
assert head.shard_indices.num_org_vocab_padding > 0
hidden_states = _hidden_states(32, seed=13)
logits = predictor.compute_logits(hidden_states)
top = predictor.get_top_tokens(hidden_states)
_assert_selects_max_logit(top, logits)
assert (top < vocab_size).all()
def test_wrapper_forwards_spec_step_idx():
"""``DeepSeekV4MTP`` is what the speculator probes, so it needs the method."""
assert hasattr(DeepSeekV4MTP, "get_top_tokens")
hidden_states = torch.empty(0)
wrapper = object.__new__(DeepSeekV4MTP)
wrapper.model = SimpleNamespace(get_top_tokens=lambda *args: args)
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states, 2) == (hidden_states, 2)
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states) == (hidden_states, 0)
+6 -6
View File
@@ -126,7 +126,7 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
num_reqs,
num_tokens,
max_query_len,
has_prefill=False,
decode_graph_eligible=True,
)
# The scheduler should mark every one of these shapes as a uniform
@@ -262,14 +262,14 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
# The batch shape is indistinguishable from a full batch of spec decodes.
assert (
get_uniform_decode_token_count(
num_reqs, num_tokens, decode_query_len, has_prefill=False
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True
)
== decode_query_len
)
# Two of the requests are decode_query_len tokens into a longer prompt.
uniform_tok_count = get_uniform_decode_token_count(
num_reqs, num_tokens, decode_query_len, has_prefill=True
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=False
)
assert uniform_tok_count is None
desc = manager.dispatch(
@@ -283,7 +283,7 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
# The same shape with every request decoding still gets its FULL graph.
uniform_tok_count = get_uniform_decode_token_count(
num_reqs, num_tokens, decode_query_len, has_prefill=False
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True
)
assert uniform_tok_count == decode_query_len
desc = manager.dispatch(
@@ -336,7 +336,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
num_reqs,
num_tokens,
max_query_len,
has_prefill=False,
decode_graph_eligible=True,
)
assert uniform_tok_count == max_query_len
@@ -401,7 +401,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
num_reqs,
num_tokens,
max_query_len,
has_prefill=False,
decode_graph_eligible=True,
)
assert uniform_tok_count == max_query_len
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import SimpleNamespace
import pytest
import torch
@@ -10,6 +11,10 @@ from vllm.v1.watermarking import GumbelWatermarker
from vllm.v1.watermarking.spec_decode import watermarked_rejection_sample
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
RejectionSampler,
gather_draft_sampled,
)
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
rejection_sample,
)
@@ -669,6 +674,81 @@ def test_placeholder_blocks_later_draft_tokens(use_block_verification: bool):
)
def test_verify_rejects_unproposed_drafts():
"""Zero drafts padded onto a step starting in the prefill must be rejected."""
torch.manual_seed(0)
device = "cuda"
num_trials = 256
K = 8
# Row 0: token 0 is very unlikely. Rows >= 1 follow the zero inputs and
# predict token 0. The never-written draft logits row is uniform.
target_logits_1d = torch.randn(VOCAB_SIZE, device=device)
target_logits_1d[0] = -20.0
inputs = _build_rejection_sample_inputs(
target_logits_1d,
torch.zeros(VOCAB_SIZE, device=device),
K,
temperature=1.0,
num_trials=num_trials,
)
target_rest = torch.randn(VOCAB_SIZE, device=device)
target_rest[0] = 20.0
inputs["target_logits"].view(num_trials, K + 1, VOCAB_SIZE)[:, 1:] = target_rest
inputs["draft_sampled"].view(num_trials, K + 1)[:, 1:] = 0
# Each request's first logit is the last prefill token, as in the first
# decode step after a P/D remote KV load.
first_pos = inputs["pos"].view(num_trials, K + 1)[:, 0]
prefill_len = first_pos + 1
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = SimpleNamespace(
apply_sampling_params=lambda logits, *args: logits,
sampling_states=SimpleNamespace(
temperature=SimpleNamespace(gpu=inputs["temperature"]),
seeds=SimpleNamespace(gpu=inputs["seed"]),
),
use_fp64_gumbel=False,
)
rejection_sampler.num_speculative_steps = K
rejection_sampler.synthetic_conditional_rates = None
rejection_sampler.use_block_verification = True
rejection_sampler.watermark_key = None
def verify() -> tuple[torch.Tensor, torch.Tensor]:
draft_sampled, pos = gather_draft_sampled(
inputs["draft_sampled"],
inputs["pos"],
torch.arange(num_trials * (K + 1), device=device),
inputs["expanded_idx_mapping"],
inputs["expanded_local_pos"],
prefill_len,
)
_, sampled, num_sampled = rejection_sampler._verify(
inputs["target_logits"],
inputs["draft_logits"],
draft_sampled,
pos,
inputs["cu_num_logits"],
inputs["idx_mapping"],
None,
inputs["expanded_idx_mapping"],
inputs["expanded_local_pos"],
None,
)
return sampled, num_sampled
sampled, num_sampled = verify()
assert (num_sampled == 1).all()
# Resampled from the target at row 0, not from a residual.
assert (sampled[:, 0] != 0).all()
# Once past the prefill, block verification accepts the same zero drafts.
prefill_len.copy_(first_pos)
_, num_sampled = verify()
assert (num_sampled == K + 1).all()
@pytest.mark.parametrize(
"num_speculative_steps,temperature",
[