[ModelRunner V2] Speculative Decoding NGram GPU Implementations (#40704)
Signed-off-by: PatchouliTaisa <patchychen@tencent.com> Signed-off-by: PatchouliTaisa <pyramkar@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Xuanan Chen <xuananchenc@nvidia.com> Co-authored-by: PatchouliTaisa <patchychen@tencent.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Xuanan Chen <xuananchenc@nvidia.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
This commit is contained in:
co-authored by
PatchouliTaisa
Nick Hill
Xuanan Chen
Giancarlo Delfin
parent
dac35c5b66
commit
819852df6b
@@ -922,12 +922,13 @@ def test_dsa_models_select_matching_mtp(model_type, expected_architecture):
|
||||
assert hf_config.architectures == [expected_architecture]
|
||||
|
||||
|
||||
def test_v2_model_runner_supports_extract_hidden_states():
|
||||
@pytest.mark.parametrize("method", ["extract_hidden_states", "ngram", "ngram_gpu"])
|
||||
def test_v2_model_runner_supports_speculative_method(method):
|
||||
config = VllmConfig()
|
||||
config.speculative_config = cast(
|
||||
SpeculativeConfig,
|
||||
SimpleNamespace(
|
||||
method="extract_hidden_states",
|
||||
method=method,
|
||||
parallel_drafting=False,
|
||||
enable_adaptive_verification=False,
|
||||
),
|
||||
|
||||
@@ -36,6 +36,39 @@ def test_ngram_max_len(num_speculative_tokens: int, vllm_runner):
|
||||
runner.llm.generate(_PROMPTS, sampling_params)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
|
||||
@pytest.mark.parametrize("method", ["ngram", "ngram_gpu"])
|
||||
def test_ngram_gpu_max_len(
|
||||
method: str,
|
||||
num_speculative_tokens: int,
|
||||
vllm_runner,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""V2 n-gram decoding stops at max_model_len."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
trust_remote_code=False,
|
||||
max_model_len=100,
|
||||
enable_chunked_prefill=None,
|
||||
enforce_eager=True, # For faster initialization.
|
||||
speculative_config={
|
||||
"method": method,
|
||||
"prompt_lookup_max": 5,
|
||||
"prompt_lookup_min": 3,
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
},
|
||||
) as runner:
|
||||
assert runner.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
sampling_params = SamplingParams(max_tokens=100, ignore_eos=True)
|
||||
outputs = runner.llm.generate(_PROMPTS, sampling_params)
|
||||
for output in outputs:
|
||||
assert output.prompt_token_ids is not None
|
||||
assert (
|
||||
len(output.prompt_token_ids) + len(output.outputs[0].token_ids) == 100
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
|
||||
@pytest.mark.parametrize("attn_backend", get_attn_backend_list_based_on_platform())
|
||||
def test_eagle_max_len(
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the V2 GPU-accelerated n-gram speculator.
|
||||
|
||||
These tests target the Triton proposer in
|
||||
``vllm.v1.worker.gpu.spec_decode.ngram.speculator`` and complement the CPU
|
||||
``NgramProposer`` tests in ``test_ngram.py``. The GPU speculator follows a
|
||||
slightly different policy than the CPU one: when multiple n-gram matches of
|
||||
the same length exist, the GPU kernel picks the right-most (most recent)
|
||||
match inside the active context, whereas the CPU implementation returns the
|
||||
left-most. The expectations below reflect the GPU behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.ngram.speculator import NgramGPUSpeculator
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip(
|
||||
"CUDA required for NgramGPUSpeculator tests",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
min_n: int,
|
||||
max_n: int,
|
||||
k: int,
|
||||
max_num_seqs: int = 8,
|
||||
max_model_len: int = 64,
|
||||
method: str = "ngram_gpu",
|
||||
) -> VllmConfig:
|
||||
model_config = ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
max_model_len=max_model_len,
|
||||
enforce_eager=True,
|
||||
)
|
||||
scheduler_config = SchedulerConfig.default_factory(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
speculative_config = SpeculativeConfig(
|
||||
method=method,
|
||||
prompt_lookup_min=min_n,
|
||||
prompt_lookup_max=max_n,
|
||||
num_speculative_tokens=k,
|
||||
)
|
||||
return VllmConfig(
|
||||
model_config=model_config,
|
||||
scheduler_config=scheduler_config,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
|
||||
def _make_request_state(cfg: VllmConfig) -> RequestState:
|
||||
assert cfg.speculative_config is not None
|
||||
return RequestState(
|
||||
max_num_reqs=cfg.scheduler_config.max_num_seqs,
|
||||
max_model_len=cfg.model_config.max_model_len,
|
||||
max_num_batched_tokens=cfg.scheduler_config.max_num_batched_tokens,
|
||||
num_speculative_steps=cfg.speculative_config.num_speculative_tokens,
|
||||
vocab_size=cfg.model_config.get_vocab_size(),
|
||||
device=DEVICE,
|
||||
use_dense_all_token_ids=True,
|
||||
)
|
||||
|
||||
|
||||
def _make_speculator(
|
||||
min_n: int,
|
||||
max_n: int,
|
||||
k: int,
|
||||
max_num_seqs: int = 8,
|
||||
max_model_len: int = 32,
|
||||
) -> NgramGPUSpeculator:
|
||||
cfg = _make_vllm_config(
|
||||
min_n=min_n,
|
||||
max_n=max_n,
|
||||
k=k,
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
return NgramGPUSpeculator(cfg, DEVICE, _make_request_state(cfg))
|
||||
|
||||
|
||||
def _propose(
|
||||
spec: NgramGPUSpeculator,
|
||||
rows: list[list[int]],
|
||||
seq_lens: list[int] | None = None,
|
||||
num_sampled: list[int] | None = None,
|
||||
last_sampled: list[int] | None = None,
|
||||
slots: list[int] | None = None,
|
||||
) -> list[list[int]]:
|
||||
"""Place each batch row at a request slot and run propose().
|
||||
|
||||
Returns the drafts as python lists in batch order.
|
||||
"""
|
||||
B = len(rows)
|
||||
if seq_lens is None:
|
||||
seq_lens = [len(r) for r in rows]
|
||||
if num_sampled is None:
|
||||
num_sampled = [1] * B
|
||||
if last_sampled is None:
|
||||
last_sampled = [0] * B
|
||||
if slots is None:
|
||||
slots = list(range(B))
|
||||
|
||||
max_num_reqs = spec.max_num_reqs
|
||||
all_token_ids = spec.req_states.all_token_ids.gpu
|
||||
total_len = spec.req_states.total_len.gpu
|
||||
all_token_ids.zero_()
|
||||
total_len.zero_()
|
||||
last_sampled_t = torch.zeros((max_num_reqs, 1), dtype=torch.int64, device=DEVICE)
|
||||
for row, slot, seq_len, last in zip(rows, slots, seq_lens, last_sampled):
|
||||
if row:
|
||||
all_token_ids[slot, : len(row)] = torch.tensor(
|
||||
row, dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
total_len[slot] = seq_len
|
||||
last_sampled_t[slot, 0] = last
|
||||
|
||||
idx_mapping = torch.tensor(slots, dtype=torch.int64, device=DEVICE)
|
||||
input_batch = SimpleNamespace(num_reqs=B, idx_mapping=idx_mapping)
|
||||
|
||||
drafts = spec.propose(
|
||||
input_batch=input_batch,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
last_hidden_states=torch.empty(0, device=DEVICE),
|
||||
aux_hidden_states=None,
|
||||
num_sampled=torch.tensor(num_sampled, dtype=torch.int32, device=DEVICE),
|
||||
num_rejected=torch.zeros(B, dtype=torch.int32, device=DEVICE),
|
||||
last_sampled=last_sampled_t,
|
||||
next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE),
|
||||
temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE),
|
||||
seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE),
|
||||
dp_sync=None,
|
||||
)
|
||||
return drafts.cpu().tolist()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_model_len", [32, 300])
|
||||
def test_no_match_clears_previous_proposal(max_model_len):
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2, max_model_len=max_model_len)
|
||||
row = [0] * (max_model_len - 5) + [1, 2, 3, 1, 2]
|
||||
assert _propose(spec, [row]) == [[3, 1]]
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 5]], last_sampled=[42])
|
||||
assert drafts == [[42, 42]]
|
||||
|
||||
|
||||
def test_no_4gram_match_only():
|
||||
"""No 4-gram match in [1,2,3,4,1,2,3] → 0 valid drafts."""
|
||||
spec = _make_speculator(min_n=4, max_n=4, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 1, 2, 3]], last_sampled=[7])
|
||||
assert drafts == [[7, 7]]
|
||||
|
||||
|
||||
def test_falls_back_to_3gram_when_4gram_missing():
|
||||
"""No 4-gram match but a 3-gram match exists → propose [4, 1]."""
|
||||
spec = _make_speculator(min_n=3, max_n=4, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 1, 2, 3]])
|
||||
assert drafts == [[4, 1]]
|
||||
|
||||
|
||||
def test_prefers_longer_ngram():
|
||||
"""Prefer a 4-gram match over a more recent 3-gram match."""
|
||||
spec = _make_speculator(min_n=3, max_n=4, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 50, 51, 2, 3, 4, 60, 61, 1, 2, 3, 4]])
|
||||
assert drafts == [[50, 51]]
|
||||
|
||||
|
||||
def test_picks_longest_match_among_2_3_4_grams():
|
||||
"""Prefer a 3-gram match over a more recent 2-gram match."""
|
||||
spec = _make_speculator(min_n=2, max_n=4, k=2)
|
||||
drafts = _propose(spec, [[2, 3, 4, 50, 51, 3, 4, 60, 61, 1, 2, 3, 4]])
|
||||
assert drafts == [[50, 51]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_model_len", [32, 128, 257, 1025])
|
||||
def test_picks_rightmost_when_multiple_matches(max_model_len):
|
||||
"""Pick the last valid match across blocks, ignoring trailing tokens."""
|
||||
spec = _make_speculator(min_n=3, max_n=3, k=2, max_model_len=max_model_len)
|
||||
padding = [0] * (spec.block_l - 5) if spec.n_blocks > 1 else []
|
||||
row = [1, 2, 3, 100] + padding + [1, 2, 3, 200, 1, 2, 3, 300, 1, 2, 3]
|
||||
drafts = _propose(spec, [row + [1, 2, 3, 999, 1, 2, 3]], seq_lens=[len(row)])
|
||||
assert drafts == [[300, 1]]
|
||||
|
||||
|
||||
def test_short_context_yields_zero_valid():
|
||||
"""The only length-2 window overlaps the suffix itself → no match."""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2)
|
||||
drafts = _propose(spec, [[5, 6]], last_sampled=[99])
|
||||
assert drafts == [[99, 99]]
|
||||
|
||||
|
||||
def test_zero_sampled_disables_proposal():
|
||||
"""num_sampled==0 disables proposals for that request regardless of match."""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 1, 2]], num_sampled=[0], last_sampled=[77])
|
||||
assert drafts == [[77, 77]]
|
||||
|
||||
|
||||
def test_tail_falls_back_when_few_tokens_after_match():
|
||||
"""Fewer than k tokens after the match → the tail falls back.
|
||||
|
||||
Tokens: [1, 2, 1, 2] (seq_len=4). Suffix (1, 2) matches at position 0
|
||||
(the match at position 2 is the suffix itself and is excluded). With
|
||||
k=3, only 2 slots map to tokens inside the context.
|
||||
"""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=3)
|
||||
drafts = _propose(spec, [[1, 2, 1, 2]], last_sampled=[55])
|
||||
assert drafts == [[1, 2, 55]]
|
||||
|
||||
|
||||
def test_multibatch_mixed():
|
||||
"""Mixed batch: row 0 matches, row 1 has no match."""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2)
|
||||
drafts = _propose(
|
||||
spec,
|
||||
[[1, 2, 3, 1, 2], [4, 5, 6]],
|
||||
last_sampled=[10, 20],
|
||||
)
|
||||
assert drafts[0] == [3, 1]
|
||||
assert drafts[1] == [20, 20]
|
||||
|
||||
|
||||
def test_multibatch_independent_choice_of_n():
|
||||
"""Each row independently picks its longest matched n."""
|
||||
spec = _make_speculator(min_n=2, max_n=3, k=2)
|
||||
drafts = _propose(
|
||||
spec,
|
||||
[
|
||||
[9, 1, 2, 3, 8, 1, 2, 3], # 3-gram (1,2,3) at idx 1 → [8, 1]
|
||||
[7, 1, 2, 9, 1, 2], # 2-gram (1,2) at idx 1 → [9, 1]
|
||||
],
|
||||
)
|
||||
assert drafts[0] == [8, 1]
|
||||
assert drafts[1] == [9, 1]
|
||||
|
||||
|
||||
def test_min_n_eq_1():
|
||||
"""min_n=max_n=1 — single-token n-grams always match if context > 1."""
|
||||
spec = _make_speculator(min_n=1, max_n=1, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 1]])
|
||||
assert drafts == [[2, 3]]
|
||||
|
||||
|
||||
def test_noncontiguous_idx_mapping():
|
||||
"""propose() reads token rows in place via idx_mapping (non-contiguous)."""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2)
|
||||
drafts = _propose(
|
||||
spec,
|
||||
[[7, 8, 9, 7, 8], [1, 2, 3, 1, 2]],
|
||||
slots=[3, 0],
|
||||
)
|
||||
assert drafts == [[9, 7], [3, 1]]
|
||||
|
||||
|
||||
def test_construction_validates_speculative_config():
|
||||
spec = _make_speculator(min_n=2, max_n=3, k=2)
|
||||
assert spec.min_n == 2
|
||||
assert spec.max_n == 3
|
||||
assert spec.num_speculative_steps == 2
|
||||
# No-op hooks must not raise.
|
||||
spec.init_cudagraph_manager(None)
|
||||
spec.capture()
|
||||
@@ -45,7 +45,8 @@ def test_init_speculator_dispatches_extract_hidden_states(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(spec_module, "ExtractHiddenStatesSpeculator", fake_speculator)
|
||||
|
||||
assert init_speculator(vllm_config, device) == (vllm_config, device)
|
||||
speculator = init_speculator(vllm_config, device, cast(Any, None))
|
||||
assert speculator == (vllm_config, device)
|
||||
|
||||
|
||||
def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch):
|
||||
|
||||
@@ -337,3 +337,23 @@ def test_capture_model_profile_only_skips_lock(monkeypatch):
|
||||
runner.capture_model(profile_only=True)
|
||||
|
||||
assert lock_calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target_buffer", ["absent", "none", "tensor"])
|
||||
def test_get_drafter_hidden_states_tolerates_missing_target_buffer(target_buffer):
|
||||
"""Targets allocate the MTP hidden buffer only for hidden-state drafters."""
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
hidden_states = torch.zeros(4, 8)
|
||||
buffer = torch.arange(16 * 8, dtype=torch.float32).view(16, 8)
|
||||
if target_buffer == "absent":
|
||||
runner.model = SimpleNamespace()
|
||||
else:
|
||||
returned = buffer if target_buffer == "tensor" else None
|
||||
runner.model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned)
|
||||
|
||||
out = runner._get_drafter_hidden_states(hidden_states)
|
||||
|
||||
if target_buffer == "tensor":
|
||||
assert torch.equal(out, buffer[:4])
|
||||
else:
|
||||
assert out is hidden_states
|
||||
|
||||
@@ -1088,10 +1088,13 @@ class VllmBackend:
|
||||
# Honors opt-outs such as CompilationMode.NONE or VLLM_DISABLE_COMPILE_CACHE.
|
||||
disable_cache = not is_compile_cache_enabled(self.inductor_config)
|
||||
|
||||
# TODO(patchy): ngram gpu kernel will cause vllm torch compile cache errors.
|
||||
# TODO(patchy): the V1 torch.compile ngram-gpu kernel causes vllm
|
||||
# torch compile cache errors. The V2 implementation is pure Triton and
|
||||
# does not need the cache disabled.
|
||||
is_ngram_gpu_enabled = (
|
||||
vllm_config.speculative_config is not None
|
||||
and vllm_config.speculative_config.use_ngram_gpu()
|
||||
and not vllm_config.use_v2_model_runner
|
||||
)
|
||||
disable_cache = disable_cache or is_ngram_gpu_enabled
|
||||
|
||||
|
||||
@@ -1970,6 +1970,9 @@ class SpeculativeConfig:
|
||||
def use_ngram_gpu(self) -> bool:
|
||||
return self.method == "ngram_gpu"
|
||||
|
||||
def use_ngram(self) -> bool:
|
||||
return self.method in ("ngram", "ngram_gpu")
|
||||
|
||||
def use_multi_module_mtp(self) -> bool:
|
||||
if self.method != "mtp" or self.draft_model_config is None:
|
||||
return False
|
||||
|
||||
@@ -3140,9 +3140,6 @@ class VllmConfig:
|
||||
|
||||
if speculative_config is not None:
|
||||
if speculative_config.method in (
|
||||
# https://github.com/vllm-project/vllm/pull/40704
|
||||
"ngram",
|
||||
"ngram_gpu",
|
||||
"suffix",
|
||||
"medusa",
|
||||
"mlp_speculator",
|
||||
|
||||
@@ -454,8 +454,8 @@ class ElasticEPScalingExecutor:
|
||||
torch.compiler.reset()
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
reset_compile_wrapper(self.worker.model_runner.get_model())
|
||||
if speculator is not None:
|
||||
reset_compile_wrapper(speculator.model)
|
||||
if draft_model := getattr(speculator, "model", None):
|
||||
reset_compile_wrapper(draft_model)
|
||||
|
||||
gc.collect()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -265,13 +265,32 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.encoder_cache = EncoderCache()
|
||||
self.ec_connector = get_ec_connector(vllm_config, self.encoder_cache)
|
||||
|
||||
self.num_speculative_steps = vllm_config.num_speculative_tokens
|
||||
num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens)
|
||||
use_dense_all_token_ids = (
|
||||
self.speculative_config is not None and self.speculative_config.use_ngram()
|
||||
)
|
||||
|
||||
# General request states.
|
||||
self.req_states = RequestState(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_model_len=self.max_model_len,
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
num_speculative_steps=self.num_speculative_steps,
|
||||
vocab_size=self.vocab_size,
|
||||
device=self.device,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
use_dense_all_token_ids=use_dense_all_token_ids,
|
||||
)
|
||||
|
||||
# Speculative decoding.
|
||||
self.speculator = None
|
||||
self.use_aux_hidden_state_outputs = False
|
||||
self.num_speculative_steps = vllm_config.num_speculative_tokens
|
||||
if self.speculative_config is not None:
|
||||
if self.is_last_pp_rank:
|
||||
self.speculator = init_speculator(self.vllm_config, self.device)
|
||||
self.speculator = init_speculator(
|
||||
self.vllm_config, self.device, self.req_states
|
||||
)
|
||||
|
||||
if self.speculative_config.method in (
|
||||
"eagle3",
|
||||
@@ -291,20 +310,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.is_pooling_model = self.model_config.runner_type == "pooling"
|
||||
self.pooling_runner: PoolingRunner | None = None
|
||||
|
||||
num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens)
|
||||
|
||||
self.step_timing = StepTimingCollector()
|
||||
|
||||
# General request states.
|
||||
self.req_states = RequestState(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_model_len=self.max_model_len,
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
num_speculative_steps=self.num_speculative_steps,
|
||||
vocab_size=self.vocab_size,
|
||||
device=self.device,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
)
|
||||
self.adaptive_verification: AdaptiveVerificationManager | None = None
|
||||
self.input_buffers = InputBuffers(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -909,14 +915,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
mm_inputs = [], all_false
|
||||
|
||||
# Let the target override the hidden state fed to the drafter
|
||||
# (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The
|
||||
# target returns a persistent buffer sized at max_num_batched_tokens;
|
||||
# slice to the active token count that propose() expects.
|
||||
spec_hidden_states = hidden_states
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
|
||||
spec_hidden_states = self._get_drafter_hidden_states(hidden_states)
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler, input_batch.idx_mapping
|
||||
@@ -2075,6 +2074,24 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
return None
|
||||
|
||||
def _get_drafter_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Hidden states fed to the drafter.
|
||||
|
||||
Targets such as DeepSeek V4 expose the pre-hc_head residual through
|
||||
get_mtp_target_hidden_states(). The buffer is sized at
|
||||
max_num_batched_tokens and only allocated for drafters that consume
|
||||
target hidden states, so None means "use the regular hidden states".
|
||||
"""
|
||||
get_target_hidden_states = getattr(
|
||||
self.model, "get_mtp_target_hidden_states", None
|
||||
)
|
||||
if get_target_hidden_states is None:
|
||||
return hidden_states
|
||||
target_hidden_states = get_target_hidden_states()
|
||||
if target_hidden_states is None:
|
||||
return hidden_states
|
||||
return target_hidden_states[: hidden_states.shape[0]]
|
||||
|
||||
@torch.inference_mode()
|
||||
@step_eplb_after()
|
||||
def sample_tokens(
|
||||
@@ -2220,14 +2237,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculator.observe_verification(
|
||||
input_batch.idx_mapping, num_sampled, num_rejected
|
||||
)
|
||||
# Let the target override the hidden state fed to the drafter
|
||||
# (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The
|
||||
# target returns a persistent buffer sized at max_num_batched_tokens;
|
||||
# slice to the active token count that propose() expects.
|
||||
spec_hidden_states = draft_hidden_states
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
|
||||
spec_hidden_states = self._get_drafter_hidden_states(draft_hidden_states)
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler, input_batch.idx_mapping
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
|
||||
def init_speculator(
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
req_states: "RequestState",
|
||||
):
|
||||
"""Build the speculator for this config."""
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
if speculative_config.method == "extract_hidden_states":
|
||||
@@ -66,5 +76,11 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
)
|
||||
|
||||
return PlainDraftModelSpeculator(vllm_config, device)
|
||||
elif speculative_config.use_ngram():
|
||||
from vllm.v1.worker.gpu.spec_decode.ngram.speculator import (
|
||||
NgramGPUSpeculator,
|
||||
)
|
||||
|
||||
return NgramGPUSpeculator(vllm_config, device, req_states)
|
||||
else:
|
||||
raise NotImplementedError(f"{speculative_config.method} is not supported yet.")
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import BaseSpeculator
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _ngram_scan_kernel(
|
||||
token_ids_ptr, # *int32 [max_num_reqs, token_ids_stride]
|
||||
token_ids_stride,
|
||||
idx_mapping_ptr, # *int64 [B] batch_idx -> req_state_idx
|
||||
total_len_ptr, # *int32 [max_num_reqs]
|
||||
num_sampled_ptr, # *int32 [B]
|
||||
scratch_ptr, # *int64 [B, scratch_stride] (output)
|
||||
scratch_stride,
|
||||
L, # int64 scalar (= max_model_len)
|
||||
MIN_N: tl.constexpr,
|
||||
MAX_N: tl.constexpr,
|
||||
MAX_N_PO2: tl.constexpr,
|
||||
BLOCK_L: tl.constexpr,
|
||||
):
|
||||
b = tl.program_id(0).to(tl.int64)
|
||||
blk = tl.program_id(1).to(tl.int64)
|
||||
Lp1 = tl.cast(L, tl.int64) + 1
|
||||
|
||||
req_state_idx = tl.load(idx_mapping_ptr + b).to(tl.int64)
|
||||
seq_len = tl.load(total_len_ptr + req_state_idx).to(tl.int64)
|
||||
num_sampled = tl.load(num_sampled_ptr + b)
|
||||
eligible_row = (num_sampled > 0) & (seq_len >= MIN_N)
|
||||
|
||||
scratch_off = b * scratch_stride + blk
|
||||
|
||||
# Ineligible rows, and blocks fully past the last candidate match
|
||||
# position, write 0 and exit.
|
||||
if not (eligible_row & (blk * BLOCK_L <= seq_len - MIN_N - 1)):
|
||||
tl.store(scratch_ptr + scratch_off, tl.zeros((), tl.int64))
|
||||
return
|
||||
|
||||
row_off = req_state_idx * token_ids_stride
|
||||
|
||||
# Load the length-MAX_N suffix once into registers.
|
||||
suf_iota = tl.arange(0, MAX_N_PO2).to(tl.int64)
|
||||
suf_pos = seq_len - MAX_N + suf_iota
|
||||
suf_in_range = (suf_iota < MAX_N) & (suf_pos >= 0) & (suf_pos < seq_len)
|
||||
suffix = tl.load(
|
||||
token_ids_ptr + row_off + suf_pos,
|
||||
mask=suf_in_range,
|
||||
other=-1,
|
||||
).to(tl.int32)
|
||||
|
||||
pos_iota = tl.arange(0, BLOCK_L).to(tl.int64)
|
||||
pos = blk * BLOCK_L + pos_iota # ascending
|
||||
|
||||
best_score = tl.zeros([BLOCK_L], dtype=tl.int64)
|
||||
|
||||
for n_iter in tl.static_range(MIN_N, MAX_N + 1):
|
||||
max_pos_n = seq_len - n_iter - 1
|
||||
match = (pos >= 0) & (pos <= max_pos_n)
|
||||
for j in tl.static_range(0, n_iter):
|
||||
tok = tl.load(
|
||||
token_ids_ptr + row_off + (pos + j),
|
||||
mask=match,
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
suf_idx = (MAX_N - n_iter) + j
|
||||
suf_val = tl.sum(tl.where(suf_iota == suf_idx, suffix, 0))
|
||||
match = match & (tok == suf_val)
|
||||
|
||||
# Pack (n, pos) so a single max yields longest-n, rightmost-pos.
|
||||
cand = n_iter * Lp1 + pos + 1
|
||||
best_score = tl.where(match, cand, best_score)
|
||||
|
||||
block_best = tl.max(best_score, axis=0)
|
||||
tl.store(scratch_ptr + scratch_off, block_best)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _ngram_finalize_kernel(
|
||||
token_ids_ptr, # *int32 [max_num_reqs, token_ids_stride]
|
||||
token_ids_stride,
|
||||
idx_mapping_ptr, # *int64 [B]
|
||||
total_len_ptr, # *int32 [max_num_reqs]
|
||||
num_sampled_ptr, # *int32 [B]
|
||||
last_sampled_ptr, # *int64 [max_num_reqs]
|
||||
scratch_ptr, # *int64 [B, scratch_stride]
|
||||
scratch_stride,
|
||||
drafts_ptr, # *int64 [B, K] (output)
|
||||
L,
|
||||
N_BLOCKS,
|
||||
K: tl.constexpr,
|
||||
K_PO2: tl.constexpr,
|
||||
N_BLOCKS_PO2: tl.constexpr,
|
||||
):
|
||||
b = tl.program_id(0).to(tl.int64)
|
||||
Lp1 = tl.cast(L, tl.int64) + 1
|
||||
NB = tl.cast(N_BLOCKS, tl.int64)
|
||||
|
||||
req_state_idx = tl.load(idx_mapping_ptr + b).to(tl.int64)
|
||||
|
||||
nb_iota = tl.arange(0, N_BLOCKS_PO2).to(tl.int64)
|
||||
nb_in_range = nb_iota < NB
|
||||
block_scores = tl.load(
|
||||
scratch_ptr + b * scratch_stride + nb_iota,
|
||||
mask=nb_in_range,
|
||||
other=0,
|
||||
)
|
||||
score = tl.max(block_scores, axis=0)
|
||||
|
||||
seq_len = tl.load(total_len_ptr + req_state_idx).to(tl.int64)
|
||||
num_sampled = tl.load(num_sampled_ptr + b)
|
||||
last_tok = tl.load(last_sampled_ptr + req_state_idx)
|
||||
|
||||
has_match = score > 0
|
||||
s1 = score - 1
|
||||
best_n = tl.where(has_match, s1 // Lp1, tl.zeros_like(s1))
|
||||
best_pos = tl.where(has_match, s1 - best_n * Lp1, tl.zeros_like(s1))
|
||||
draft_start = tl.where(has_match, best_pos + best_n, tl.zeros_like(s1))
|
||||
|
||||
tokens_avail = tl.maximum(seq_len - draft_start, 0)
|
||||
write_ok = (num_sampled > 0) & has_match
|
||||
|
||||
row_off = req_state_idx * token_ids_stride
|
||||
k_iota = tl.arange(0, K_PO2).to(tl.int64)
|
||||
k_in_range = k_iota < K
|
||||
gather_idx = tl.minimum(draft_start + k_iota, tl.cast(L, tl.int64) - 1)
|
||||
slot_valid = (k_iota < tokens_avail) & write_ok & k_in_range
|
||||
gathered = tl.load(
|
||||
token_ids_ptr + row_off + gather_idx,
|
||||
mask=slot_valid,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
# Invalid slots fall back to the last sampled token; they are verified as
|
||||
# ordinary (rejectable) drafts, so the fill value only affects efficiency.
|
||||
out = tl.where(slot_valid, gathered, last_tok)
|
||||
tl.store(drafts_ptr + b * K + k_iota, out, mask=k_in_range)
|
||||
|
||||
|
||||
class NgramGPUSpeculator(BaseSpeculator):
|
||||
"""V2-compatible GPU n-gram speculator."""
|
||||
|
||||
supports_mm_inputs = False
|
||||
draft_logits = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
req_states: RequestState,
|
||||
):
|
||||
if not HAS_TRITON:
|
||||
raise RuntimeError("ngram_gpu speculative decoding requires Triton.")
|
||||
spec = vllm_config.speculative_config
|
||||
assert spec is not None
|
||||
assert spec.prompt_lookup_min is not None, (
|
||||
"prompt_lookup_min must be configured for ngram_gpu"
|
||||
)
|
||||
assert spec.prompt_lookup_max is not None, (
|
||||
"prompt_lookup_max must be configured for ngram_gpu"
|
||||
)
|
||||
assert 1 <= spec.prompt_lookup_min <= spec.prompt_lookup_max
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.req_states = req_states
|
||||
self.speculative_config = spec
|
||||
self.num_speculative_steps: int = spec.num_speculative_tokens
|
||||
|
||||
self.min_n: int = spec.prompt_lookup_min
|
||||
self.max_n: int = spec.prompt_lookup_max
|
||||
|
||||
self.max_num_reqs: int = vllm_config.scheduler_config.max_num_seqs
|
||||
self.max_model_len: int = vllm_config.model_config.max_model_len
|
||||
|
||||
L = self.max_model_len
|
||||
if L >= 1024:
|
||||
self.block_l = 256
|
||||
elif L >= 256:
|
||||
self.block_l = 128
|
||||
elif L >= 64:
|
||||
self.block_l = 64
|
||||
else:
|
||||
self.block_l = max(16, triton.next_power_of_2(max(L, 1)))
|
||||
self.n_blocks = triton.cdiv(L, self.block_l)
|
||||
|
||||
self.scratch = torch.zeros(
|
||||
(self.max_num_reqs, self.n_blocks), dtype=torch.int64, device=device
|
||||
)
|
||||
# Batch-ordered draft output, scattered into RequestState.draft_tokens
|
||||
# by the model runner (same contract as the model-based speculators).
|
||||
self.drafts = torch.zeros(
|
||||
(self.max_num_reqs, self.num_speculative_steps),
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
del cudagraph_mode
|
||||
|
||||
def capture(self) -> None:
|
||||
return None
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
attn_metadata: Any,
|
||||
slot_mappings: Any,
|
||||
last_hidden_states: torch.Tensor,
|
||||
aux_hidden_states: list[torch.Tensor] | None,
|
||||
num_sampled: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
last_sampled: torch.Tensor,
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
is_profile: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
if dummy_run:
|
||||
return self.drafts[:num_reqs]
|
||||
|
||||
req_states = self.req_states
|
||||
token_ids = req_states.all_token_ids.gpu
|
||||
idx_mapping = input_batch.idx_mapping
|
||||
|
||||
_ngram_scan_kernel[(num_reqs, self.n_blocks)](
|
||||
token_ids,
|
||||
token_ids.stride(0),
|
||||
idx_mapping,
|
||||
req_states.total_len.gpu,
|
||||
num_sampled,
|
||||
self.scratch,
|
||||
self.scratch.stride(0),
|
||||
self.max_model_len,
|
||||
self.min_n,
|
||||
self.max_n,
|
||||
max(1, triton.next_power_of_2(self.max_n)),
|
||||
self.block_l,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
_ngram_finalize_kernel[(num_reqs,)](
|
||||
token_ids,
|
||||
token_ids.stride(0),
|
||||
idx_mapping,
|
||||
req_states.total_len.gpu,
|
||||
num_sampled,
|
||||
last_sampled.view(-1),
|
||||
self.scratch,
|
||||
self.scratch.stride(0),
|
||||
self.drafts,
|
||||
self.max_model_len,
|
||||
self.n_blocks,
|
||||
self.num_speculative_steps,
|
||||
max(1, triton.next_power_of_2(self.num_speculative_steps)),
|
||||
max(1, triton.next_power_of_2(self.n_blocks)),
|
||||
num_warps=2,
|
||||
num_stages=1,
|
||||
)
|
||||
return self.drafts[:num_reqs]
|
||||
@@ -16,6 +16,7 @@ class RequestState:
|
||||
vocab_size: int,
|
||||
device: torch.device,
|
||||
num_prefill_lookahead: int = 1,
|
||||
use_dense_all_token_ids: bool = False,
|
||||
):
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_model_len = max_model_len
|
||||
@@ -30,12 +31,14 @@ class RequestState:
|
||||
|
||||
# NOTE(woosuk): This tensor can be extremely large (e.g., several GBs)
|
||||
# depending on the configured max_num_reqs and max_model_len.
|
||||
# To save GPU memory, we use UVA instead of GPU for this tensor.
|
||||
# To save GPU memory, we use UVA instead of GPU by default, but
|
||||
# ngram_gpu benefits from dense device residency because it scans
|
||||
# active rows repeatedly during proposal.
|
||||
self.all_token_ids = StagedWriteTensor(
|
||||
(self.max_num_reqs, self.max_model_len),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
uva_instead_of_gpu=True,
|
||||
uva_instead_of_gpu=not use_dense_all_token_ids,
|
||||
)
|
||||
# NOTE(woosuk): Distinguish clearly between prompt_len and prefill_len:
|
||||
# - prompt_len: Number of tokens in the user-provided prompt.
|
||||
|
||||
Reference in New Issue
Block a user