[ModelRunner V2] Speculative Decoding NGram GPU Implementations (#40704)

Signed-off-by: PatchouliTaisa <patchychen@tencent.com>
Signed-off-by: PatchouliTaisa <pyramkar@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Xuanan Chen <xuananchenc@nvidia.com>
Co-authored-by: PatchouliTaisa <patchychen@tencent.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Xuanan Chen <xuananchenc@nvidia.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
This commit is contained in:
PatchyTIS
2026-10-06 13:32:59 -07:00
committed by GitHub
co-authored by PatchouliTaisa Nick Hill Xuanan Chen Giancarlo Delfin
parent dac35c5b66
commit 819852df6b
14 changed files with 687 additions and 43 deletions
+3 -2
View File
@@ -922,12 +922,13 @@ def test_dsa_models_select_matching_mtp(model_type, expected_architecture):
assert hf_config.architectures == [expected_architecture]
def test_v2_model_runner_supports_extract_hidden_states():
@pytest.mark.parametrize("method", ["extract_hidden_states", "ngram", "ngram_gpu"])
def test_v2_model_runner_supports_speculative_method(method):
config = VllmConfig()
config.speculative_config = cast(
SpeculativeConfig,
SimpleNamespace(
method="extract_hidden_states",
method=method,
parallel_drafting=False,
enable_adaptive_verification=False,
),
+33
View File
@@ -36,6 +36,39 @@ def test_ngram_max_len(num_speculative_tokens: int, vllm_runner):
runner.llm.generate(_PROMPTS, sampling_params)
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
@pytest.mark.parametrize("method", ["ngram", "ngram_gpu"])
def test_ngram_gpu_max_len(
method: str,
num_speculative_tokens: int,
vllm_runner,
monkeypatch: pytest.MonkeyPatch,
):
"""V2 n-gram decoding stops at max_model_len."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
"facebook/opt-125m",
trust_remote_code=False,
max_model_len=100,
enable_chunked_prefill=None,
enforce_eager=True, # For faster initialization.
speculative_config={
"method": method,
"prompt_lookup_max": 5,
"prompt_lookup_min": 3,
"num_speculative_tokens": num_speculative_tokens,
},
) as runner:
assert runner.llm.llm_engine.vllm_config.use_v2_model_runner
sampling_params = SamplingParams(max_tokens=100, ignore_eos=True)
outputs = runner.llm.generate(_PROMPTS, sampling_params)
for output in outputs:
assert output.prompt_token_ids is not None
assert (
len(output.prompt_token_ids) + len(output.outputs[0].token_ids) == 100
)
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
@pytest.mark.parametrize("attn_backend", get_attn_backend_list_based_on_platform())
def test_eagle_max_len(
+279
View File
@@ -0,0 +1,279 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the V2 GPU-accelerated n-gram speculator.
These tests target the Triton proposer in
``vllm.v1.worker.gpu.spec_decode.ngram.speculator`` and complement the CPU
``NgramProposer`` tests in ``test_ngram.py``. The GPU speculator follows a
slightly different policy than the CPU one: when multiple n-gram matches of
the same length exist, the GPU kernel picks the right-most (most recent)
match inside the active context, whereas the CPU implementation returns the
left-most. The expectations below reflect the GPU behavior.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from vllm.config import (
ModelConfig,
SchedulerConfig,
SpeculativeConfig,
VllmConfig,
)
from vllm.v1.worker.gpu.spec_decode.ngram.speculator import NgramGPUSpeculator
from vllm.v1.worker.gpu.states import RequestState
if not torch.cuda.is_available():
pytest.skip(
"CUDA required for NgramGPUSpeculator tests",
allow_module_level=True,
)
DEVICE = torch.device("cuda")
def _make_vllm_config(
min_n: int,
max_n: int,
k: int,
max_num_seqs: int = 8,
max_model_len: int = 64,
method: str = "ngram_gpu",
) -> VllmConfig:
model_config = ModelConfig(
model="facebook/opt-125m",
max_model_len=max_model_len,
enforce_eager=True,
)
scheduler_config = SchedulerConfig.default_factory(
max_num_seqs=max_num_seqs,
max_model_len=max_model_len,
)
speculative_config = SpeculativeConfig(
method=method,
prompt_lookup_min=min_n,
prompt_lookup_max=max_n,
num_speculative_tokens=k,
)
return VllmConfig(
model_config=model_config,
scheduler_config=scheduler_config,
speculative_config=speculative_config,
)
def _make_request_state(cfg: VllmConfig) -> RequestState:
assert cfg.speculative_config is not None
return RequestState(
max_num_reqs=cfg.scheduler_config.max_num_seqs,
max_model_len=cfg.model_config.max_model_len,
max_num_batched_tokens=cfg.scheduler_config.max_num_batched_tokens,
num_speculative_steps=cfg.speculative_config.num_speculative_tokens,
vocab_size=cfg.model_config.get_vocab_size(),
device=DEVICE,
use_dense_all_token_ids=True,
)
def _make_speculator(
min_n: int,
max_n: int,
k: int,
max_num_seqs: int = 8,
max_model_len: int = 32,
) -> NgramGPUSpeculator:
cfg = _make_vllm_config(
min_n=min_n,
max_n=max_n,
k=k,
max_num_seqs=max_num_seqs,
max_model_len=max_model_len,
)
return NgramGPUSpeculator(cfg, DEVICE, _make_request_state(cfg))
def _propose(
spec: NgramGPUSpeculator,
rows: list[list[int]],
seq_lens: list[int] | None = None,
num_sampled: list[int] | None = None,
last_sampled: list[int] | None = None,
slots: list[int] | None = None,
) -> list[list[int]]:
"""Place each batch row at a request slot and run propose().
Returns the drafts as python lists in batch order.
"""
B = len(rows)
if seq_lens is None:
seq_lens = [len(r) for r in rows]
if num_sampled is None:
num_sampled = [1] * B
if last_sampled is None:
last_sampled = [0] * B
if slots is None:
slots = list(range(B))
max_num_reqs = spec.max_num_reqs
all_token_ids = spec.req_states.all_token_ids.gpu
total_len = spec.req_states.total_len.gpu
all_token_ids.zero_()
total_len.zero_()
last_sampled_t = torch.zeros((max_num_reqs, 1), dtype=torch.int64, device=DEVICE)
for row, slot, seq_len, last in zip(rows, slots, seq_lens, last_sampled):
if row:
all_token_ids[slot, : len(row)] = torch.tensor(
row, dtype=torch.int32, device=DEVICE
)
total_len[slot] = seq_len
last_sampled_t[slot, 0] = last
idx_mapping = torch.tensor(slots, dtype=torch.int64, device=DEVICE)
input_batch = SimpleNamespace(num_reqs=B, idx_mapping=idx_mapping)
drafts = spec.propose(
input_batch=input_batch,
attn_metadata=None,
slot_mappings=None,
last_hidden_states=torch.empty(0, device=DEVICE),
aux_hidden_states=None,
num_sampled=torch.tensor(num_sampled, dtype=torch.int32, device=DEVICE),
num_rejected=torch.zeros(B, dtype=torch.int32, device=DEVICE),
last_sampled=last_sampled_t,
next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE),
temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE),
seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE),
dp_sync=None,
)
return drafts.cpu().tolist()
@pytest.mark.parametrize("max_model_len", [32, 300])
def test_no_match_clears_previous_proposal(max_model_len):
spec = _make_speculator(min_n=2, max_n=2, k=2, max_model_len=max_model_len)
row = [0] * (max_model_len - 5) + [1, 2, 3, 1, 2]
assert _propose(spec, [row]) == [[3, 1]]
drafts = _propose(spec, [[1, 2, 3, 4, 5]], last_sampled=[42])
assert drafts == [[42, 42]]
def test_no_4gram_match_only():
"""No 4-gram match in [1,2,3,4,1,2,3] → 0 valid drafts."""
spec = _make_speculator(min_n=4, max_n=4, k=2)
drafts = _propose(spec, [[1, 2, 3, 4, 1, 2, 3]], last_sampled=[7])
assert drafts == [[7, 7]]
def test_falls_back_to_3gram_when_4gram_missing():
"""No 4-gram match but a 3-gram match exists → propose [4, 1]."""
spec = _make_speculator(min_n=3, max_n=4, k=2)
drafts = _propose(spec, [[1, 2, 3, 4, 1, 2, 3]])
assert drafts == [[4, 1]]
def test_prefers_longer_ngram():
"""Prefer a 4-gram match over a more recent 3-gram match."""
spec = _make_speculator(min_n=3, max_n=4, k=2)
drafts = _propose(spec, [[1, 2, 3, 4, 50, 51, 2, 3, 4, 60, 61, 1, 2, 3, 4]])
assert drafts == [[50, 51]]
def test_picks_longest_match_among_2_3_4_grams():
"""Prefer a 3-gram match over a more recent 2-gram match."""
spec = _make_speculator(min_n=2, max_n=4, k=2)
drafts = _propose(spec, [[2, 3, 4, 50, 51, 3, 4, 60, 61, 1, 2, 3, 4]])
assert drafts == [[50, 51]]
@pytest.mark.parametrize("max_model_len", [32, 128, 257, 1025])
def test_picks_rightmost_when_multiple_matches(max_model_len):
"""Pick the last valid match across blocks, ignoring trailing tokens."""
spec = _make_speculator(min_n=3, max_n=3, k=2, max_model_len=max_model_len)
padding = [0] * (spec.block_l - 5) if spec.n_blocks > 1 else []
row = [1, 2, 3, 100] + padding + [1, 2, 3, 200, 1, 2, 3, 300, 1, 2, 3]
drafts = _propose(spec, [row + [1, 2, 3, 999, 1, 2, 3]], seq_lens=[len(row)])
assert drafts == [[300, 1]]
def test_short_context_yields_zero_valid():
"""The only length-2 window overlaps the suffix itself → no match."""
spec = _make_speculator(min_n=2, max_n=2, k=2)
drafts = _propose(spec, [[5, 6]], last_sampled=[99])
assert drafts == [[99, 99]]
def test_zero_sampled_disables_proposal():
"""num_sampled==0 disables proposals for that request regardless of match."""
spec = _make_speculator(min_n=2, max_n=2, k=2)
drafts = _propose(spec, [[1, 2, 3, 1, 2]], num_sampled=[0], last_sampled=[77])
assert drafts == [[77, 77]]
def test_tail_falls_back_when_few_tokens_after_match():
"""Fewer than k tokens after the match → the tail falls back.
Tokens: [1, 2, 1, 2] (seq_len=4). Suffix (1, 2) matches at position 0
(the match at position 2 is the suffix itself and is excluded). With
k=3, only 2 slots map to tokens inside the context.
"""
spec = _make_speculator(min_n=2, max_n=2, k=3)
drafts = _propose(spec, [[1, 2, 1, 2]], last_sampled=[55])
assert drafts == [[1, 2, 55]]
def test_multibatch_mixed():
"""Mixed batch: row 0 matches, row 1 has no match."""
spec = _make_speculator(min_n=2, max_n=2, k=2)
drafts = _propose(
spec,
[[1, 2, 3, 1, 2], [4, 5, 6]],
last_sampled=[10, 20],
)
assert drafts[0] == [3, 1]
assert drafts[1] == [20, 20]
def test_multibatch_independent_choice_of_n():
"""Each row independently picks its longest matched n."""
spec = _make_speculator(min_n=2, max_n=3, k=2)
drafts = _propose(
spec,
[
[9, 1, 2, 3, 8, 1, 2, 3], # 3-gram (1,2,3) at idx 1 → [8, 1]
[7, 1, 2, 9, 1, 2], # 2-gram (1,2) at idx 1 → [9, 1]
],
)
assert drafts[0] == [8, 1]
assert drafts[1] == [9, 1]
def test_min_n_eq_1():
"""min_n=max_n=1 — single-token n-grams always match if context > 1."""
spec = _make_speculator(min_n=1, max_n=1, k=2)
drafts = _propose(spec, [[1, 2, 3, 4, 1]])
assert drafts == [[2, 3]]
def test_noncontiguous_idx_mapping():
"""propose() reads token rows in place via idx_mapping (non-contiguous)."""
spec = _make_speculator(min_n=2, max_n=2, k=2)
drafts = _propose(
spec,
[[7, 8, 9, 7, 8], [1, 2, 3, 1, 2]],
slots=[3, 0],
)
assert drafts == [[9, 7], [3, 1]]
def test_construction_validates_speculative_config():
spec = _make_speculator(min_n=2, max_n=3, k=2)
assert spec.min_n == 2
assert spec.max_n == 3
assert spec.num_speculative_steps == 2
# No-op hooks must not raise.
spec.init_cudagraph_manager(None)
spec.capture()
@@ -45,7 +45,8 @@ def test_init_speculator_dispatches_extract_hidden_states(monkeypatch):
monkeypatch.setattr(spec_module, "ExtractHiddenStatesSpeculator", fake_speculator)
assert init_speculator(vllm_config, device) == (vllm_config, device)
speculator = init_speculator(vllm_config, device, cast(Any, None))
assert speculator == (vllm_config, device)
def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch):
@@ -337,3 +337,23 @@ def test_capture_model_profile_only_skips_lock(monkeypatch):
runner.capture_model(profile_only=True)
assert lock_calls == []
@pytest.mark.parametrize("target_buffer", ["absent", "none", "tensor"])
def test_get_drafter_hidden_states_tolerates_missing_target_buffer(target_buffer):
"""Targets allocate the MTP hidden buffer only for hidden-state drafters."""
runner = GPUModelRunner.__new__(GPUModelRunner)
hidden_states = torch.zeros(4, 8)
buffer = torch.arange(16 * 8, dtype=torch.float32).view(16, 8)
if target_buffer == "absent":
runner.model = SimpleNamespace()
else:
returned = buffer if target_buffer == "tensor" else None
runner.model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned)
out = runner._get_drafter_hidden_states(hidden_states)
if target_buffer == "tensor":
assert torch.equal(out, buffer[:4])
else:
assert out is hidden_states
+4 -1
View File
@@ -1088,10 +1088,13 @@ class VllmBackend:
# Honors opt-outs such as CompilationMode.NONE or VLLM_DISABLE_COMPILE_CACHE.
disable_cache = not is_compile_cache_enabled(self.inductor_config)
# TODO(patchy): ngram gpu kernel will cause vllm torch compile cache errors.
# TODO(patchy): the V1 torch.compile ngram-gpu kernel causes vllm
# torch compile cache errors. The V2 implementation is pure Triton and
# does not need the cache disabled.
is_ngram_gpu_enabled = (
vllm_config.speculative_config is not None
and vllm_config.speculative_config.use_ngram_gpu()
and not vllm_config.use_v2_model_runner
)
disable_cache = disable_cache or is_ngram_gpu_enabled
+3
View File
@@ -1970,6 +1970,9 @@ class SpeculativeConfig:
def use_ngram_gpu(self) -> bool:
return self.method == "ngram_gpu"
def use_ngram(self) -> bool:
return self.method in ("ngram", "ngram_gpu")
def use_multi_module_mtp(self) -> bool:
if self.method != "mtp" or self.draft_model_config is None:
return False
-3
View File
@@ -3140,9 +3140,6 @@ class VllmConfig:
if speculative_config is not None:
if speculative_config.method in (
# https://github.com/vllm-project/vllm/pull/40704
"ngram",
"ngram_gpu",
"suffix",
"medusa",
"mlp_speculator",
@@ -454,8 +454,8 @@ class ElasticEPScalingExecutor:
torch.compiler.reset()
with set_current_vllm_config(self.worker.vllm_config):
reset_compile_wrapper(self.worker.model_runner.get_model())
if speculator is not None:
reset_compile_wrapper(speculator.model)
if draft_model := getattr(speculator, "model", None):
reset_compile_wrapper(draft_model)
gc.collect()
torch.accelerator.synchronize()
+41 -31
View File
@@ -265,13 +265,32 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.encoder_cache = EncoderCache()
self.ec_connector = get_ec_connector(vllm_config, self.encoder_cache)
self.num_speculative_steps = vllm_config.num_speculative_tokens
num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens)
use_dense_all_token_ids = (
self.speculative_config is not None and self.speculative_config.use_ngram()
)
# General request states.
self.req_states = RequestState(
max_num_reqs=self.max_num_reqs,
max_model_len=self.max_model_len,
max_num_batched_tokens=self.max_num_tokens,
num_speculative_steps=self.num_speculative_steps,
vocab_size=self.vocab_size,
device=self.device,
num_prefill_lookahead=num_prefill_lookahead,
use_dense_all_token_ids=use_dense_all_token_ids,
)
# Speculative decoding.
self.speculator = None
self.use_aux_hidden_state_outputs = False
self.num_speculative_steps = vllm_config.num_speculative_tokens
if self.speculative_config is not None:
if self.is_last_pp_rank:
self.speculator = init_speculator(self.vllm_config, self.device)
self.speculator = init_speculator(
self.vllm_config, self.device, self.req_states
)
if self.speculative_config.method in (
"eagle3",
@@ -291,20 +310,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.is_pooling_model = self.model_config.runner_type == "pooling"
self.pooling_runner: PoolingRunner | None = None
num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens)
self.step_timing = StepTimingCollector()
# General request states.
self.req_states = RequestState(
max_num_reqs=self.max_num_reqs,
max_model_len=self.max_model_len,
max_num_batched_tokens=self.max_num_tokens,
num_speculative_steps=self.num_speculative_steps,
vocab_size=self.vocab_size,
device=self.device,
num_prefill_lookahead=num_prefill_lookahead,
)
self.adaptive_verification: AdaptiveVerificationManager | None = None
self.input_buffers = InputBuffers(
max_num_reqs=self.max_num_reqs,
@@ -909,14 +915,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
mm_inputs = [], all_false
# Let the target override the hidden state fed to the drafter
# (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The
# target returns a persistent buffer sized at max_num_batched_tokens;
# slice to the active token count that propose() expects.
spec_hidden_states = hidden_states
if hasattr(self.model, "get_mtp_target_hidden_states"):
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
spec_hidden_states = self._get_drafter_hidden_states(hidden_states)
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler, input_batch.idx_mapping
@@ -2075,6 +2074,24 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
return None
def _get_drafter_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Hidden states fed to the drafter.
Targets such as DeepSeek V4 expose the pre-hc_head residual through
get_mtp_target_hidden_states(). The buffer is sized at
max_num_batched_tokens and only allocated for drafters that consume
target hidden states, so None means "use the regular hidden states".
"""
get_target_hidden_states = getattr(
self.model, "get_mtp_target_hidden_states", None
)
if get_target_hidden_states is None:
return hidden_states
target_hidden_states = get_target_hidden_states()
if target_hidden_states is None:
return hidden_states
return target_hidden_states[: hidden_states.shape[0]]
@torch.inference_mode()
@step_eplb_after()
def sample_tokens(
@@ -2220,14 +2237,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator.observe_verification(
input_batch.idx_mapping, num_sampled, num_rejected
)
# Let the target override the hidden state fed to the drafter
# (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The
# target returns a persistent buffer sized at max_num_batched_tokens;
# slice to the active token count that propose() expects.
spec_hidden_states = draft_hidden_states
if hasattr(self.model, "get_mtp_target_hidden_states"):
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
spec_hidden_states = self._get_drafter_hidden_states(draft_hidden_states)
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler, input_batch.idx_mapping
+17 -1
View File
@@ -1,11 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import torch
from vllm.config import VllmConfig
if TYPE_CHECKING:
from vllm.v1.worker.gpu.states import RequestState
def init_speculator(vllm_config: VllmConfig, device: torch.device):
def init_speculator(
vllm_config: VllmConfig,
device: torch.device,
req_states: "RequestState",
):
"""Build the speculator for this config."""
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
if speculative_config.method == "extract_hidden_states":
@@ -66,5 +76,11 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
)
return PlainDraftModelSpeculator(vllm_config, device)
elif speculative_config.use_ngram():
from vllm.v1.worker.gpu.spec_decode.ngram.speculator import (
NgramGPUSpeculator,
)
return NgramGPUSpeculator(vllm_config, device, req_states)
else:
raise NotImplementedError(f"{speculative_config.method} is not supported yet.")
@@ -0,0 +1,278 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.triton_utils import HAS_TRITON, tl, triton
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.spec_decode.speculator import BaseSpeculator
if TYPE_CHECKING:
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.worker.gpu.dp_utils import DPSyncState
from vllm.v1.worker.gpu.states import RequestState
@triton.jit
def _ngram_scan_kernel(
token_ids_ptr, # *int32 [max_num_reqs, token_ids_stride]
token_ids_stride,
idx_mapping_ptr, # *int64 [B] batch_idx -> req_state_idx
total_len_ptr, # *int32 [max_num_reqs]
num_sampled_ptr, # *int32 [B]
scratch_ptr, # *int64 [B, scratch_stride] (output)
scratch_stride,
L, # int64 scalar (= max_model_len)
MIN_N: tl.constexpr,
MAX_N: tl.constexpr,
MAX_N_PO2: tl.constexpr,
BLOCK_L: tl.constexpr,
):
b = tl.program_id(0).to(tl.int64)
blk = tl.program_id(1).to(tl.int64)
Lp1 = tl.cast(L, tl.int64) + 1
req_state_idx = tl.load(idx_mapping_ptr + b).to(tl.int64)
seq_len = tl.load(total_len_ptr + req_state_idx).to(tl.int64)
num_sampled = tl.load(num_sampled_ptr + b)
eligible_row = (num_sampled > 0) & (seq_len >= MIN_N)
scratch_off = b * scratch_stride + blk
# Ineligible rows, and blocks fully past the last candidate match
# position, write 0 and exit.
if not (eligible_row & (blk * BLOCK_L <= seq_len - MIN_N - 1)):
tl.store(scratch_ptr + scratch_off, tl.zeros((), tl.int64))
return
row_off = req_state_idx * token_ids_stride
# Load the length-MAX_N suffix once into registers.
suf_iota = tl.arange(0, MAX_N_PO2).to(tl.int64)
suf_pos = seq_len - MAX_N + suf_iota
suf_in_range = (suf_iota < MAX_N) & (suf_pos >= 0) & (suf_pos < seq_len)
suffix = tl.load(
token_ids_ptr + row_off + suf_pos,
mask=suf_in_range,
other=-1,
).to(tl.int32)
pos_iota = tl.arange(0, BLOCK_L).to(tl.int64)
pos = blk * BLOCK_L + pos_iota # ascending
best_score = tl.zeros([BLOCK_L], dtype=tl.int64)
for n_iter in tl.static_range(MIN_N, MAX_N + 1):
max_pos_n = seq_len - n_iter - 1
match = (pos >= 0) & (pos <= max_pos_n)
for j in tl.static_range(0, n_iter):
tok = tl.load(
token_ids_ptr + row_off + (pos + j),
mask=match,
other=0,
).to(tl.int32)
suf_idx = (MAX_N - n_iter) + j
suf_val = tl.sum(tl.where(suf_iota == suf_idx, suffix, 0))
match = match & (tok == suf_val)
# Pack (n, pos) so a single max yields longest-n, rightmost-pos.
cand = n_iter * Lp1 + pos + 1
best_score = tl.where(match, cand, best_score)
block_best = tl.max(best_score, axis=0)
tl.store(scratch_ptr + scratch_off, block_best)
@triton.jit
def _ngram_finalize_kernel(
token_ids_ptr, # *int32 [max_num_reqs, token_ids_stride]
token_ids_stride,
idx_mapping_ptr, # *int64 [B]
total_len_ptr, # *int32 [max_num_reqs]
num_sampled_ptr, # *int32 [B]
last_sampled_ptr, # *int64 [max_num_reqs]
scratch_ptr, # *int64 [B, scratch_stride]
scratch_stride,
drafts_ptr, # *int64 [B, K] (output)
L,
N_BLOCKS,
K: tl.constexpr,
K_PO2: tl.constexpr,
N_BLOCKS_PO2: tl.constexpr,
):
b = tl.program_id(0).to(tl.int64)
Lp1 = tl.cast(L, tl.int64) + 1
NB = tl.cast(N_BLOCKS, tl.int64)
req_state_idx = tl.load(idx_mapping_ptr + b).to(tl.int64)
nb_iota = tl.arange(0, N_BLOCKS_PO2).to(tl.int64)
nb_in_range = nb_iota < NB
block_scores = tl.load(
scratch_ptr + b * scratch_stride + nb_iota,
mask=nb_in_range,
other=0,
)
score = tl.max(block_scores, axis=0)
seq_len = tl.load(total_len_ptr + req_state_idx).to(tl.int64)
num_sampled = tl.load(num_sampled_ptr + b)
last_tok = tl.load(last_sampled_ptr + req_state_idx)
has_match = score > 0
s1 = score - 1
best_n = tl.where(has_match, s1 // Lp1, tl.zeros_like(s1))
best_pos = tl.where(has_match, s1 - best_n * Lp1, tl.zeros_like(s1))
draft_start = tl.where(has_match, best_pos + best_n, tl.zeros_like(s1))
tokens_avail = tl.maximum(seq_len - draft_start, 0)
write_ok = (num_sampled > 0) & has_match
row_off = req_state_idx * token_ids_stride
k_iota = tl.arange(0, K_PO2).to(tl.int64)
k_in_range = k_iota < K
gather_idx = tl.minimum(draft_start + k_iota, tl.cast(L, tl.int64) - 1)
slot_valid = (k_iota < tokens_avail) & write_ok & k_in_range
gathered = tl.load(
token_ids_ptr + row_off + gather_idx,
mask=slot_valid,
other=0,
).to(tl.int64)
# Invalid slots fall back to the last sampled token; they are verified as
# ordinary (rejectable) drafts, so the fill value only affects efficiency.
out = tl.where(slot_valid, gathered, last_tok)
tl.store(drafts_ptr + b * K + k_iota, out, mask=k_in_range)
class NgramGPUSpeculator(BaseSpeculator):
"""V2-compatible GPU n-gram speculator."""
supports_mm_inputs = False
draft_logits = None
def __init__(
self,
vllm_config: VllmConfig,
device: torch.device,
req_states: RequestState,
):
if not HAS_TRITON:
raise RuntimeError("ngram_gpu speculative decoding requires Triton.")
spec = vllm_config.speculative_config
assert spec is not None
assert spec.prompt_lookup_min is not None, (
"prompt_lookup_min must be configured for ngram_gpu"
)
assert spec.prompt_lookup_max is not None, (
"prompt_lookup_max must be configured for ngram_gpu"
)
assert 1 <= spec.prompt_lookup_min <= spec.prompt_lookup_max
self.vllm_config = vllm_config
self.device = device
self.req_states = req_states
self.speculative_config = spec
self.num_speculative_steps: int = spec.num_speculative_tokens
self.min_n: int = spec.prompt_lookup_min
self.max_n: int = spec.prompt_lookup_max
self.max_num_reqs: int = vllm_config.scheduler_config.max_num_seqs
self.max_model_len: int = vllm_config.model_config.max_model_len
L = self.max_model_len
if L >= 1024:
self.block_l = 256
elif L >= 256:
self.block_l = 128
elif L >= 64:
self.block_l = 64
else:
self.block_l = max(16, triton.next_power_of_2(max(L, 1)))
self.n_blocks = triton.cdiv(L, self.block_l)
self.scratch = torch.zeros(
(self.max_num_reqs, self.n_blocks), dtype=torch.int64, device=device
)
# Batch-ordered draft output, scattered into RequestState.draft_tokens
# by the model runner (same contract as the model-based speculators).
self.drafts = torch.zeros(
(self.max_num_reqs, self.num_speculative_steps),
dtype=torch.int64,
device=device,
)
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
del cudagraph_mode
def capture(self) -> None:
return None
@torch.inference_mode()
def propose(
self,
input_batch: InputBatch,
attn_metadata: Any,
slot_mappings: Any,
last_hidden_states: torch.Tensor,
aux_hidden_states: list[torch.Tensor] | None,
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
last_sampled: torch.Tensor,
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
dp_sync: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
if dummy_run:
return self.drafts[:num_reqs]
req_states = self.req_states
token_ids = req_states.all_token_ids.gpu
idx_mapping = input_batch.idx_mapping
_ngram_scan_kernel[(num_reqs, self.n_blocks)](
token_ids,
token_ids.stride(0),
idx_mapping,
req_states.total_len.gpu,
num_sampled,
self.scratch,
self.scratch.stride(0),
self.max_model_len,
self.min_n,
self.max_n,
max(1, triton.next_power_of_2(self.max_n)),
self.block_l,
num_warps=4,
num_stages=2,
)
_ngram_finalize_kernel[(num_reqs,)](
token_ids,
token_ids.stride(0),
idx_mapping,
req_states.total_len.gpu,
num_sampled,
last_sampled.view(-1),
self.scratch,
self.scratch.stride(0),
self.drafts,
self.max_model_len,
self.n_blocks,
self.num_speculative_steps,
max(1, triton.next_power_of_2(self.num_speculative_steps)),
max(1, triton.next_power_of_2(self.n_blocks)),
num_warps=2,
num_stages=1,
)
return self.drafts[:num_reqs]
+5 -2
View File
@@ -16,6 +16,7 @@ class RequestState:
vocab_size: int,
device: torch.device,
num_prefill_lookahead: int = 1,
use_dense_all_token_ids: bool = False,
):
self.max_num_reqs = max_num_reqs
self.max_model_len = max_model_len
@@ -30,12 +31,14 @@ class RequestState:
# NOTE(woosuk): This tensor can be extremely large (e.g., several GBs)
# depending on the configured max_num_reqs and max_model_len.
# To save GPU memory, we use UVA instead of GPU for this tensor.
# To save GPU memory, we use UVA instead of GPU by default, but
# ngram_gpu benefits from dense device residency because it scans
# active rows repeatedly during proposal.
self.all_token_ids = StagedWriteTensor(
(self.max_num_reqs, self.max_model_len),
dtype=torch.int32,
device=device,
uva_instead_of_gpu=True,
uva_instead_of_gpu=not use_dense_all_token_ids,
)
# NOTE(woosuk): Distinguish clearly between prompt_len and prefill_len:
# - prompt_len: Number of tokens in the user-provided prompt.