Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``get_top_tokens`` on the DeepSeek V4 MTP drafter.
|
||||
|
||||
The speculator opts into ``use_local_argmax_reduction`` by probing the draft
|
||||
model for ``get_top_tokens``, and expects it to select the same greedy draft
|
||||
token the full-vocab argmax would. V4 reaches its LM head through hc_head and
|
||||
the shared head's RMSNorm, so both methods have to share that transform.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import HCHeadOp
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"get_top_tokens is implemented on the ROCm DeepSeek V4 MTP drafter only.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.models.deepseek_v4.amd.mtp import ( # noqa: E402
|
||||
DeepSeekV4MTP,
|
||||
DeepSeekV4MultiTokenPredictor,
|
||||
)
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
START_LAYER_IDX = 61
|
||||
HIDDEN_SIZE = 4096
|
||||
HC_MULT = 4
|
||||
RMS_NORM_EPS = 1e-6
|
||||
HC_EPS = 1e-6
|
||||
_EMBEDDING = "vllm.model_executor.layers.vocab_parallel_embedding"
|
||||
|
||||
|
||||
def _make_predictor(vocab_size: int) -> DeepSeekV4MultiTokenPredictor:
|
||||
config = PreTrainedConfig()
|
||||
config.hidden_size = HIDDEN_SIZE
|
||||
config.vocab_size = vocab_size
|
||||
config.rms_norm_eps = RMS_NORM_EPS
|
||||
|
||||
# A single vocab shard, so the reduction returns the local argmax without
|
||||
# touching the process group.
|
||||
with (
|
||||
mock.patch(f"{_EMBEDDING}.get_tensor_model_parallel_rank", return_value=0),
|
||||
mock.patch(
|
||||
f"{_EMBEDDING}.get_tensor_model_parallel_world_size", return_value=1
|
||||
),
|
||||
):
|
||||
shared_head = SharedHead(config=config, prefix="shared_head")
|
||||
|
||||
generator = torch.Generator().manual_seed(7)
|
||||
with torch.no_grad():
|
||||
shared_head.head.weight.normal_(0.0, 0.02, generator=generator)
|
||||
# The default norm weight is all ones, and the argmax is invariant to
|
||||
# the norm's per-row rescale, so a uniform gain would hide a skipped
|
||||
# shared_head. Vary it so the comparison below can see one.
|
||||
shared_head.norm.weight.normal_(1.0, 0.5, generator=generator)
|
||||
shared_head = shared_head.to(device=DEVICE_TYPE, dtype=torch.bfloat16)
|
||||
|
||||
hc_head_fn = torch.randn(
|
||||
HC_MULT, HC_MULT * HIDDEN_SIZE, dtype=torch.float32, generator=generator
|
||||
)
|
||||
# Non-trivial gate scale and bias: hc_head mixes the hc_mult streams with
|
||||
# these, and defaults of 0 would make a skipped hc_head harder to see.
|
||||
hc_head_scale = torch.randn(1, dtype=torch.float32, generator=generator) * 0.1
|
||||
hc_head_base = torch.randn(HC_MULT, dtype=torch.float32, generator=generator) * 0.1
|
||||
|
||||
mtp_layer = SimpleNamespace(
|
||||
config=config,
|
||||
hc_mult=HC_MULT,
|
||||
hc_eps=HC_EPS,
|
||||
rms_norm_eps=RMS_NORM_EPS,
|
||||
hc_head_op=HCHeadOp(),
|
||||
hc_head_fn=(hc_head_fn * 1e-4).to(DEVICE_TYPE),
|
||||
hc_head_scale=hc_head_scale.to(DEVICE_TYPE),
|
||||
hc_head_base=hc_head_base.to(DEVICE_TYPE),
|
||||
shared_head=shared_head,
|
||||
)
|
||||
|
||||
# Only the attributes the two methods read; a real predictor would also
|
||||
# build the MLA/MoE draft block, which is irrelevant here.
|
||||
predictor = object.__new__(DeepSeekV4MultiTokenPredictor)
|
||||
torch.nn.Module.__init__(predictor)
|
||||
predictor.layers = {str(START_LAYER_IDX): mtp_layer}
|
||||
predictor.mtp_start_layer_idx = START_LAYER_IDX
|
||||
predictor.num_mtp_layers = 1
|
||||
predictor.logits_processor = LogitsProcessor(vocab_size)
|
||||
return predictor
|
||||
|
||||
|
||||
def _hidden_states(num_tokens: int, seed: int) -> torch.Tensor:
|
||||
"""The flat pre-hc_head residual the MTP layer returns."""
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
hidden_states = torch.randn(
|
||||
num_tokens, HC_MULT * HIDDEN_SIZE, dtype=torch.float32, generator=generator
|
||||
)
|
||||
return hidden_states.to(device=DEVICE_TYPE, dtype=torch.bfloat16)
|
||||
|
||||
|
||||
def _assert_selects_max_logit(top: torch.Tensor, logits: torch.Tensor) -> None:
|
||||
"""The selected token must carry the row's maximum logit.
|
||||
|
||||
hc_head emits bf16, so rows can tie for the maximum and then no single
|
||||
index is the right answer; compare the values the two paths select.
|
||||
"""
|
||||
assert top.shape == (logits.shape[0],)
|
||||
selected = logits.gather(1, top.unsqueeze(1)).squeeze(1)
|
||||
assert torch.equal(selected, logits.max(dim=-1).values)
|
||||
|
||||
|
||||
def test_get_top_tokens_matches_full_argmax(default_vllm_config):
|
||||
"""Same draft token as the path taken when the reduction is off.
|
||||
|
||||
``get_top_tokens`` has to run the same pre-head transform as
|
||||
``compute_logits``: hc_head over the (T, hc_mult, D) residual, then the
|
||||
shared head's RMSNorm. Dropping either one returns a different draft token
|
||||
instead of failing, which is what this comparison catches.
|
||||
"""
|
||||
predictor = _make_predictor(vocab_size=512)
|
||||
|
||||
for num_tokens in (1, 16, 128):
|
||||
hidden_states = _hidden_states(num_tokens, seed=11)
|
||||
|
||||
logits = predictor.compute_logits(hidden_states)
|
||||
top = predictor.get_top_tokens(hidden_states)
|
||||
|
||||
assert top.dtype == torch.int64
|
||||
_assert_selects_max_logit(top, logits)
|
||||
|
||||
|
||||
def test_get_top_tokens_honors_padded_vocab(default_vllm_config):
|
||||
"""Padding entries above ``org_vocab_size`` must never win the argmax."""
|
||||
vocab_size = 500
|
||||
predictor = _make_predictor(vocab_size)
|
||||
head = predictor.layers[str(START_LAYER_IDX)].shared_head.head
|
||||
assert head.shard_indices.num_org_vocab_padding > 0
|
||||
|
||||
hidden_states = _hidden_states(32, seed=13)
|
||||
logits = predictor.compute_logits(hidden_states)
|
||||
top = predictor.get_top_tokens(hidden_states)
|
||||
|
||||
_assert_selects_max_logit(top, logits)
|
||||
assert (top < vocab_size).all()
|
||||
|
||||
|
||||
def test_wrapper_forwards_spec_step_idx():
|
||||
"""``DeepSeekV4MTP`` is what the speculator probes, so it needs the method."""
|
||||
assert hasattr(DeepSeekV4MTP, "get_top_tokens")
|
||||
|
||||
hidden_states = torch.empty(0)
|
||||
wrapper = object.__new__(DeepSeekV4MTP)
|
||||
wrapper.model = SimpleNamespace(get_top_tokens=lambda *args: args)
|
||||
|
||||
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states, 2) == (hidden_states, 2)
|
||||
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states) == (hidden_states, 0)
|
||||
@@ -126,7 +126,7 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
|
||||
# The scheduler should mark every one of these shapes as a uniform
|
||||
@@ -262,14 +262,14 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
# The batch shape is indistinguishable from a full batch of spec decodes.
|
||||
assert (
|
||||
get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=False
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True
|
||||
)
|
||||
== decode_query_len
|
||||
)
|
||||
|
||||
# Two of the requests are decode_query_len tokens into a longer prompt.
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=True
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=False
|
||||
)
|
||||
assert uniform_tok_count is None
|
||||
desc = manager.dispatch(
|
||||
@@ -283,7 +283,7 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
|
||||
# The same shape with every request decoding still gets its FULL graph.
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=False
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True
|
||||
)
|
||||
assert uniform_tok_count == decode_query_len
|
||||
desc = manager.dispatch(
|
||||
@@ -336,7 +336,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
@@ -401,7 +401,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -10,6 +11,10 @@ from vllm.v1.watermarking import GumbelWatermarker
|
||||
from vllm.v1.watermarking.spec_decode import watermarked_rejection_sample
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
|
||||
RejectionSampler,
|
||||
gather_draft_sampled,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
|
||||
rejection_sample,
|
||||
)
|
||||
@@ -669,6 +674,81 @@ def test_placeholder_blocks_later_draft_tokens(use_block_verification: bool):
|
||||
)
|
||||
|
||||
|
||||
def test_verify_rejects_unproposed_drafts():
|
||||
"""Zero drafts padded onto a step starting in the prefill must be rejected."""
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
num_trials = 256
|
||||
K = 8
|
||||
|
||||
# Row 0: token 0 is very unlikely. Rows >= 1 follow the zero inputs and
|
||||
# predict token 0. The never-written draft logits row is uniform.
|
||||
target_logits_1d = torch.randn(VOCAB_SIZE, device=device)
|
||||
target_logits_1d[0] = -20.0
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
torch.zeros(VOCAB_SIZE, device=device),
|
||||
K,
|
||||
temperature=1.0,
|
||||
num_trials=num_trials,
|
||||
)
|
||||
target_rest = torch.randn(VOCAB_SIZE, device=device)
|
||||
target_rest[0] = 20.0
|
||||
inputs["target_logits"].view(num_trials, K + 1, VOCAB_SIZE)[:, 1:] = target_rest
|
||||
inputs["draft_sampled"].view(num_trials, K + 1)[:, 1:] = 0
|
||||
|
||||
# Each request's first logit is the last prefill token, as in the first
|
||||
# decode step after a P/D remote KV load.
|
||||
first_pos = inputs["pos"].view(num_trials, K + 1)[:, 0]
|
||||
prefill_len = first_pos + 1
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = SimpleNamespace(
|
||||
apply_sampling_params=lambda logits, *args: logits,
|
||||
sampling_states=SimpleNamespace(
|
||||
temperature=SimpleNamespace(gpu=inputs["temperature"]),
|
||||
seeds=SimpleNamespace(gpu=inputs["seed"]),
|
||||
),
|
||||
use_fp64_gumbel=False,
|
||||
)
|
||||
rejection_sampler.num_speculative_steps = K
|
||||
rejection_sampler.synthetic_conditional_rates = None
|
||||
rejection_sampler.use_block_verification = True
|
||||
rejection_sampler.watermark_key = None
|
||||
|
||||
def verify() -> tuple[torch.Tensor, torch.Tensor]:
|
||||
draft_sampled, pos = gather_draft_sampled(
|
||||
inputs["draft_sampled"],
|
||||
inputs["pos"],
|
||||
torch.arange(num_trials * (K + 1), device=device),
|
||||
inputs["expanded_idx_mapping"],
|
||||
inputs["expanded_local_pos"],
|
||||
prefill_len,
|
||||
)
|
||||
_, sampled, num_sampled = rejection_sampler._verify(
|
||||
inputs["target_logits"],
|
||||
inputs["draft_logits"],
|
||||
draft_sampled,
|
||||
pos,
|
||||
inputs["cu_num_logits"],
|
||||
inputs["idx_mapping"],
|
||||
None,
|
||||
inputs["expanded_idx_mapping"],
|
||||
inputs["expanded_local_pos"],
|
||||
None,
|
||||
)
|
||||
return sampled, num_sampled
|
||||
|
||||
sampled, num_sampled = verify()
|
||||
assert (num_sampled == 1).all()
|
||||
# Resampled from the target at row 0, not from a residual.
|
||||
assert (sampled[:, 0] != 0).all()
|
||||
|
||||
# Once past the prefill, block verification accepts the same zero drafts.
|
||||
prefill_len.copy_(first_pos)
|
||||
_, num_sampled = verify()
|
||||
assert (num_sampled == K + 1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_speculative_steps,temperature",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user