From 819852df6bffb873304c421f7091b7f8133fa87b Mon Sep 17 00:00:00 2001 From: PatchyTIS <58251192+PatchouliTIS@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:32:59 +0800 Subject: [PATCH] [ModelRunner V2] Speculative Decoding NGram GPU Implementations (#40704) Signed-off-by: PatchouliTaisa Signed-off-by: PatchouliTaisa Signed-off-by: Nick Hill Signed-off-by: Xuanan Chen Co-authored-by: PatchouliTaisa Co-authored-by: Nick Hill Co-authored-by: Xuanan Chen Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> --- tests/test_config.py | 5 +- tests/v1/spec_decode/test_max_len.py | 33 +++ tests/v1/spec_decode/test_ngram_gpu.py | 279 ++++++++++++++++++ ...st_gpu_extract_hidden_states_speculator.py | 3 +- tests/v1/worker/test_gpu_model_runner_v2.py | 20 ++ vllm/compilation/backends.py | 5 +- vllm/config/speculative.py | 3 + vllm/config/vllm.py | 3 - .../distributed/elastic_ep/elastic_execute.py | 4 +- vllm/v1/worker/gpu/model_runner.py | 72 +++-- vllm/v1/worker/gpu/spec_decode/__init__.py | 18 +- .../worker/gpu/spec_decode/ngram/__init__.py | 0 .../gpu/spec_decode/ngram/speculator.py | 278 +++++++++++++++++ vllm/v1/worker/gpu/states.py | 7 +- 14 files changed, 687 insertions(+), 43 deletions(-) create mode 100644 tests/v1/spec_decode/test_ngram_gpu.py create mode 100644 vllm/v1/worker/gpu/spec_decode/ngram/__init__.py create mode 100644 vllm/v1/worker/gpu/spec_decode/ngram/speculator.py diff --git a/tests/test_config.py b/tests/test_config.py index a0c64099dc..0098ecdc3d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -922,12 +922,13 @@ def test_dsa_models_select_matching_mtp(model_type, expected_architecture): assert hf_config.architectures == [expected_architecture] -def test_v2_model_runner_supports_extract_hidden_states(): +@pytest.mark.parametrize("method", ["extract_hidden_states", "ngram", "ngram_gpu"]) +def test_v2_model_runner_supports_speculative_method(method): config = VllmConfig() config.speculative_config = cast( SpeculativeConfig, SimpleNamespace( - method="extract_hidden_states", + method=method, parallel_drafting=False, enable_adaptive_verification=False, ), diff --git a/tests/v1/spec_decode/test_max_len.py b/tests/v1/spec_decode/test_max_len.py index 81f8424193..17dd318563 100644 --- a/tests/v1/spec_decode/test_max_len.py +++ b/tests/v1/spec_decode/test_max_len.py @@ -36,6 +36,39 @@ def test_ngram_max_len(num_speculative_tokens: int, vllm_runner): runner.llm.generate(_PROMPTS, sampling_params) +@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10]) +@pytest.mark.parametrize("method", ["ngram", "ngram_gpu"]) +def test_ngram_gpu_max_len( + method: str, + num_speculative_tokens: int, + vllm_runner, + monkeypatch: pytest.MonkeyPatch, +): + """V2 n-gram decoding stops at max_model_len.""" + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + with vllm_runner( + "facebook/opt-125m", + trust_remote_code=False, + max_model_len=100, + enable_chunked_prefill=None, + enforce_eager=True, # For faster initialization. + speculative_config={ + "method": method, + "prompt_lookup_max": 5, + "prompt_lookup_min": 3, + "num_speculative_tokens": num_speculative_tokens, + }, + ) as runner: + assert runner.llm.llm_engine.vllm_config.use_v2_model_runner + sampling_params = SamplingParams(max_tokens=100, ignore_eos=True) + outputs = runner.llm.generate(_PROMPTS, sampling_params) + for output in outputs: + assert output.prompt_token_ids is not None + assert ( + len(output.prompt_token_ids) + len(output.outputs[0].token_ids) == 100 + ) + + @pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10]) @pytest.mark.parametrize("attn_backend", get_attn_backend_list_based_on_platform()) def test_eagle_max_len( diff --git a/tests/v1/spec_decode/test_ngram_gpu.py b/tests/v1/spec_decode/test_ngram_gpu.py new file mode 100644 index 0000000000..2359f0cdef --- /dev/null +++ b/tests/v1/spec_decode/test_ngram_gpu.py @@ -0,0 +1,279 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the V2 GPU-accelerated n-gram speculator. + +These tests target the Triton proposer in +``vllm.v1.worker.gpu.spec_decode.ngram.speculator`` and complement the CPU +``NgramProposer`` tests in ``test_ngram.py``. The GPU speculator follows a +slightly different policy than the CPU one: when multiple n-gram matches of +the same length exist, the GPU kernel picks the right-most (most recent) +match inside the active context, whereas the CPU implementation returns the +left-most. The expectations below reflect the GPU behavior. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.config import ( + ModelConfig, + SchedulerConfig, + SpeculativeConfig, + VllmConfig, +) +from vllm.v1.worker.gpu.spec_decode.ngram.speculator import NgramGPUSpeculator +from vllm.v1.worker.gpu.states import RequestState + +if not torch.cuda.is_available(): + pytest.skip( + "CUDA required for NgramGPUSpeculator tests", + allow_module_level=True, + ) + +DEVICE = torch.device("cuda") + + +def _make_vllm_config( + min_n: int, + max_n: int, + k: int, + max_num_seqs: int = 8, + max_model_len: int = 64, + method: str = "ngram_gpu", +) -> VllmConfig: + model_config = ModelConfig( + model="facebook/opt-125m", + max_model_len=max_model_len, + enforce_eager=True, + ) + scheduler_config = SchedulerConfig.default_factory( + max_num_seqs=max_num_seqs, + max_model_len=max_model_len, + ) + speculative_config = SpeculativeConfig( + method=method, + prompt_lookup_min=min_n, + prompt_lookup_max=max_n, + num_speculative_tokens=k, + ) + return VllmConfig( + model_config=model_config, + scheduler_config=scheduler_config, + speculative_config=speculative_config, + ) + + +def _make_request_state(cfg: VllmConfig) -> RequestState: + assert cfg.speculative_config is not None + return RequestState( + max_num_reqs=cfg.scheduler_config.max_num_seqs, + max_model_len=cfg.model_config.max_model_len, + max_num_batched_tokens=cfg.scheduler_config.max_num_batched_tokens, + num_speculative_steps=cfg.speculative_config.num_speculative_tokens, + vocab_size=cfg.model_config.get_vocab_size(), + device=DEVICE, + use_dense_all_token_ids=True, + ) + + +def _make_speculator( + min_n: int, + max_n: int, + k: int, + max_num_seqs: int = 8, + max_model_len: int = 32, +) -> NgramGPUSpeculator: + cfg = _make_vllm_config( + min_n=min_n, + max_n=max_n, + k=k, + max_num_seqs=max_num_seqs, + max_model_len=max_model_len, + ) + return NgramGPUSpeculator(cfg, DEVICE, _make_request_state(cfg)) + + +def _propose( + spec: NgramGPUSpeculator, + rows: list[list[int]], + seq_lens: list[int] | None = None, + num_sampled: list[int] | None = None, + last_sampled: list[int] | None = None, + slots: list[int] | None = None, +) -> list[list[int]]: + """Place each batch row at a request slot and run propose(). + + Returns the drafts as python lists in batch order. + """ + B = len(rows) + if seq_lens is None: + seq_lens = [len(r) for r in rows] + if num_sampled is None: + num_sampled = [1] * B + if last_sampled is None: + last_sampled = [0] * B + if slots is None: + slots = list(range(B)) + + max_num_reqs = spec.max_num_reqs + all_token_ids = spec.req_states.all_token_ids.gpu + total_len = spec.req_states.total_len.gpu + all_token_ids.zero_() + total_len.zero_() + last_sampled_t = torch.zeros((max_num_reqs, 1), dtype=torch.int64, device=DEVICE) + for row, slot, seq_len, last in zip(rows, slots, seq_lens, last_sampled): + if row: + all_token_ids[slot, : len(row)] = torch.tensor( + row, dtype=torch.int32, device=DEVICE + ) + total_len[slot] = seq_len + last_sampled_t[slot, 0] = last + + idx_mapping = torch.tensor(slots, dtype=torch.int64, device=DEVICE) + input_batch = SimpleNamespace(num_reqs=B, idx_mapping=idx_mapping) + + drafts = spec.propose( + input_batch=input_batch, + attn_metadata=None, + slot_mappings=None, + last_hidden_states=torch.empty(0, device=DEVICE), + aux_hidden_states=None, + num_sampled=torch.tensor(num_sampled, dtype=torch.int32, device=DEVICE), + num_rejected=torch.zeros(B, dtype=torch.int32, device=DEVICE), + last_sampled=last_sampled_t, + next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE), + temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE), + seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE), + dp_sync=None, + ) + return drafts.cpu().tolist() + + +@pytest.mark.parametrize("max_model_len", [32, 300]) +def test_no_match_clears_previous_proposal(max_model_len): + spec = _make_speculator(min_n=2, max_n=2, k=2, max_model_len=max_model_len) + row = [0] * (max_model_len - 5) + [1, 2, 3, 1, 2] + assert _propose(spec, [row]) == [[3, 1]] + drafts = _propose(spec, [[1, 2, 3, 4, 5]], last_sampled=[42]) + assert drafts == [[42, 42]] + + +def test_no_4gram_match_only(): + """No 4-gram match in [1,2,3,4,1,2,3] → 0 valid drafts.""" + spec = _make_speculator(min_n=4, max_n=4, k=2) + drafts = _propose(spec, [[1, 2, 3, 4, 1, 2, 3]], last_sampled=[7]) + assert drafts == [[7, 7]] + + +def test_falls_back_to_3gram_when_4gram_missing(): + """No 4-gram match but a 3-gram match exists → propose [4, 1].""" + spec = _make_speculator(min_n=3, max_n=4, k=2) + drafts = _propose(spec, [[1, 2, 3, 4, 1, 2, 3]]) + assert drafts == [[4, 1]] + + +def test_prefers_longer_ngram(): + """Prefer a 4-gram match over a more recent 3-gram match.""" + spec = _make_speculator(min_n=3, max_n=4, k=2) + drafts = _propose(spec, [[1, 2, 3, 4, 50, 51, 2, 3, 4, 60, 61, 1, 2, 3, 4]]) + assert drafts == [[50, 51]] + + +def test_picks_longest_match_among_2_3_4_grams(): + """Prefer a 3-gram match over a more recent 2-gram match.""" + spec = _make_speculator(min_n=2, max_n=4, k=2) + drafts = _propose(spec, [[2, 3, 4, 50, 51, 3, 4, 60, 61, 1, 2, 3, 4]]) + assert drafts == [[50, 51]] + + +@pytest.mark.parametrize("max_model_len", [32, 128, 257, 1025]) +def test_picks_rightmost_when_multiple_matches(max_model_len): + """Pick the last valid match across blocks, ignoring trailing tokens.""" + spec = _make_speculator(min_n=3, max_n=3, k=2, max_model_len=max_model_len) + padding = [0] * (spec.block_l - 5) if spec.n_blocks > 1 else [] + row = [1, 2, 3, 100] + padding + [1, 2, 3, 200, 1, 2, 3, 300, 1, 2, 3] + drafts = _propose(spec, [row + [1, 2, 3, 999, 1, 2, 3]], seq_lens=[len(row)]) + assert drafts == [[300, 1]] + + +def test_short_context_yields_zero_valid(): + """The only length-2 window overlaps the suffix itself → no match.""" + spec = _make_speculator(min_n=2, max_n=2, k=2) + drafts = _propose(spec, [[5, 6]], last_sampled=[99]) + assert drafts == [[99, 99]] + + +def test_zero_sampled_disables_proposal(): + """num_sampled==0 disables proposals for that request regardless of match.""" + spec = _make_speculator(min_n=2, max_n=2, k=2) + drafts = _propose(spec, [[1, 2, 3, 1, 2]], num_sampled=[0], last_sampled=[77]) + assert drafts == [[77, 77]] + + +def test_tail_falls_back_when_few_tokens_after_match(): + """Fewer than k tokens after the match → the tail falls back. + + Tokens: [1, 2, 1, 2] (seq_len=4). Suffix (1, 2) matches at position 0 + (the match at position 2 is the suffix itself and is excluded). With + k=3, only 2 slots map to tokens inside the context. + """ + spec = _make_speculator(min_n=2, max_n=2, k=3) + drafts = _propose(spec, [[1, 2, 1, 2]], last_sampled=[55]) + assert drafts == [[1, 2, 55]] + + +def test_multibatch_mixed(): + """Mixed batch: row 0 matches, row 1 has no match.""" + spec = _make_speculator(min_n=2, max_n=2, k=2) + drafts = _propose( + spec, + [[1, 2, 3, 1, 2], [4, 5, 6]], + last_sampled=[10, 20], + ) + assert drafts[0] == [3, 1] + assert drafts[1] == [20, 20] + + +def test_multibatch_independent_choice_of_n(): + """Each row independently picks its longest matched n.""" + spec = _make_speculator(min_n=2, max_n=3, k=2) + drafts = _propose( + spec, + [ + [9, 1, 2, 3, 8, 1, 2, 3], # 3-gram (1,2,3) at idx 1 → [8, 1] + [7, 1, 2, 9, 1, 2], # 2-gram (1,2) at idx 1 → [9, 1] + ], + ) + assert drafts[0] == [8, 1] + assert drafts[1] == [9, 1] + + +def test_min_n_eq_1(): + """min_n=max_n=1 — single-token n-grams always match if context > 1.""" + spec = _make_speculator(min_n=1, max_n=1, k=2) + drafts = _propose(spec, [[1, 2, 3, 4, 1]]) + assert drafts == [[2, 3]] + + +def test_noncontiguous_idx_mapping(): + """propose() reads token rows in place via idx_mapping (non-contiguous).""" + spec = _make_speculator(min_n=2, max_n=2, k=2) + drafts = _propose( + spec, + [[7, 8, 9, 7, 8], [1, 2, 3, 1, 2]], + slots=[3, 0], + ) + assert drafts == [[9, 7], [3, 1]] + + +def test_construction_validates_speculative_config(): + spec = _make_speculator(min_n=2, max_n=3, k=2) + assert spec.min_n == 2 + assert spec.max_n == 3 + assert spec.num_speculative_steps == 2 + # No-op hooks must not raise. + spec.init_cudagraph_manager(None) + spec.capture() diff --git a/tests/v1/worker/test_gpu_extract_hidden_states_speculator.py b/tests/v1/worker/test_gpu_extract_hidden_states_speculator.py index 74464f542f..5fa44d3475 100644 --- a/tests/v1/worker/test_gpu_extract_hidden_states_speculator.py +++ b/tests/v1/worker/test_gpu_extract_hidden_states_speculator.py @@ -45,7 +45,8 @@ def test_init_speculator_dispatches_extract_hidden_states(monkeypatch): monkeypatch.setattr(spec_module, "ExtractHiddenStatesSpeculator", fake_speculator) - assert init_speculator(vllm_config, device) == (vllm_config, device) + speculator = init_speculator(vllm_config, device, cast(Any, None)) + assert speculator == (vllm_config, device) def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch): diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 0caa132d7e..a10f54a97c 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -337,3 +337,23 @@ def test_capture_model_profile_only_skips_lock(monkeypatch): runner.capture_model(profile_only=True) assert lock_calls == [] + + +@pytest.mark.parametrize("target_buffer", ["absent", "none", "tensor"]) +def test_get_drafter_hidden_states_tolerates_missing_target_buffer(target_buffer): + """Targets allocate the MTP hidden buffer only for hidden-state drafters.""" + runner = GPUModelRunner.__new__(GPUModelRunner) + hidden_states = torch.zeros(4, 8) + buffer = torch.arange(16 * 8, dtype=torch.float32).view(16, 8) + if target_buffer == "absent": + runner.model = SimpleNamespace() + else: + returned = buffer if target_buffer == "tensor" else None + runner.model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned) + + out = runner._get_drafter_hidden_states(hidden_states) + + if target_buffer == "tensor": + assert torch.equal(out, buffer[:4]) + else: + assert out is hidden_states diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py index 43252ce3d8..d985d8de75 100644 --- a/vllm/compilation/backends.py +++ b/vllm/compilation/backends.py @@ -1088,10 +1088,13 @@ class VllmBackend: # Honors opt-outs such as CompilationMode.NONE or VLLM_DISABLE_COMPILE_CACHE. disable_cache = not is_compile_cache_enabled(self.inductor_config) - # TODO(patchy): ngram gpu kernel will cause vllm torch compile cache errors. + # TODO(patchy): the V1 torch.compile ngram-gpu kernel causes vllm + # torch compile cache errors. The V2 implementation is pure Triton and + # does not need the cache disabled. is_ngram_gpu_enabled = ( vllm_config.speculative_config is not None and vllm_config.speculative_config.use_ngram_gpu() + and not vllm_config.use_v2_model_runner ) disable_cache = disable_cache or is_ngram_gpu_enabled diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 47ffb3b7f7..cd42206f98 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -1970,6 +1970,9 @@ class SpeculativeConfig: def use_ngram_gpu(self) -> bool: return self.method == "ngram_gpu" + def use_ngram(self) -> bool: + return self.method in ("ngram", "ngram_gpu") + def use_multi_module_mtp(self) -> bool: if self.method != "mtp" or self.draft_model_config is None: return False diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index b847da4320..8fc89f1591 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -3140,9 +3140,6 @@ class VllmConfig: if speculative_config is not None: if speculative_config.method in ( - # https://github.com/vllm-project/vllm/pull/40704 - "ngram", - "ngram_gpu", "suffix", "medusa", "mlp_speculator", diff --git a/vllm/distributed/elastic_ep/elastic_execute.py b/vllm/distributed/elastic_ep/elastic_execute.py index 10b5e88d29..182a4b9bb4 100644 --- a/vllm/distributed/elastic_ep/elastic_execute.py +++ b/vllm/distributed/elastic_ep/elastic_execute.py @@ -454,8 +454,8 @@ class ElasticEPScalingExecutor: torch.compiler.reset() with set_current_vllm_config(self.worker.vllm_config): reset_compile_wrapper(self.worker.model_runner.get_model()) - if speculator is not None: - reset_compile_wrapper(speculator.model) + if draft_model := getattr(speculator, "model", None): + reset_compile_wrapper(draft_model) gc.collect() torch.accelerator.synchronize() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index e42917918a..c3ac543ff2 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -265,13 +265,32 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.encoder_cache = EncoderCache() self.ec_connector = get_ec_connector(vllm_config, self.encoder_cache) + self.num_speculative_steps = vllm_config.num_speculative_tokens + num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens) + use_dense_all_token_ids = ( + self.speculative_config is not None and self.speculative_config.use_ngram() + ) + + # General request states. + self.req_states = RequestState( + max_num_reqs=self.max_num_reqs, + max_model_len=self.max_model_len, + max_num_batched_tokens=self.max_num_tokens, + num_speculative_steps=self.num_speculative_steps, + vocab_size=self.vocab_size, + device=self.device, + num_prefill_lookahead=num_prefill_lookahead, + use_dense_all_token_ids=use_dense_all_token_ids, + ) + # Speculative decoding. self.speculator = None self.use_aux_hidden_state_outputs = False - self.num_speculative_steps = vllm_config.num_speculative_tokens if self.speculative_config is not None: if self.is_last_pp_rank: - self.speculator = init_speculator(self.vllm_config, self.device) + self.speculator = init_speculator( + self.vllm_config, self.device, self.req_states + ) if self.speculative_config.method in ( "eagle3", @@ -291,20 +310,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.is_pooling_model = self.model_config.runner_type == "pooling" self.pooling_runner: PoolingRunner | None = None - num_prefill_lookahead = max(1, vllm_config.num_prefill_lookahead_tokens) - self.step_timing = StepTimingCollector() - - # General request states. - self.req_states = RequestState( - max_num_reqs=self.max_num_reqs, - max_model_len=self.max_model_len, - max_num_batched_tokens=self.max_num_tokens, - num_speculative_steps=self.num_speculative_steps, - vocab_size=self.vocab_size, - device=self.device, - num_prefill_lookahead=num_prefill_lookahead, - ) self.adaptive_verification: AdaptiveVerificationManager | None = None self.input_buffers = InputBuffers( max_num_reqs=self.max_num_reqs, @@ -909,14 +915,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) mm_inputs = [], all_false - # Let the target override the hidden state fed to the drafter - # (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The - # target returns a persistent buffer sized at max_num_batched_tokens; - # slice to the active token count that propose() expects. - spec_hidden_states = hidden_states - if hasattr(self.model, "get_mtp_target_hidden_states"): - pre_hc_hidden_states = self.model.get_mtp_target_hidden_states() - spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr] + spec_hidden_states = self._get_drafter_hidden_states(hidden_states) if isinstance(self.sampler, GPUWatermarkSampler): self.speculator.prepare_watermarking( self.sampler, input_batch.idx_mapping @@ -2075,6 +2074,24 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) return None + def _get_drafter_hidden_states(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Hidden states fed to the drafter. + + Targets such as DeepSeek V4 expose the pre-hc_head residual through + get_mtp_target_hidden_states(). The buffer is sized at + max_num_batched_tokens and only allocated for drafters that consume + target hidden states, so None means "use the regular hidden states". + """ + get_target_hidden_states = getattr( + self.model, "get_mtp_target_hidden_states", None + ) + if get_target_hidden_states is None: + return hidden_states + target_hidden_states = get_target_hidden_states() + if target_hidden_states is None: + return hidden_states + return target_hidden_states[: hidden_states.shape[0]] + @torch.inference_mode() @step_eplb_after() def sample_tokens( @@ -2220,14 +2237,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.speculator.observe_verification( input_batch.idx_mapping, num_sampled, num_rejected ) - # Let the target override the hidden state fed to the drafter - # (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The - # target returns a persistent buffer sized at max_num_batched_tokens; - # slice to the active token count that propose() expects. - spec_hidden_states = draft_hidden_states - if hasattr(self.model, "get_mtp_target_hidden_states"): - pre_hc_hidden_states = self.model.get_mtp_target_hidden_states() - spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)] + spec_hidden_states = self._get_drafter_hidden_states(draft_hidden_states) if isinstance(self.sampler, GPUWatermarkSampler): self.speculator.prepare_watermarking( self.sampler, input_batch.idx_mapping diff --git a/vllm/v1/worker/gpu/spec_decode/__init__.py b/vllm/v1/worker/gpu/spec_decode/__init__.py index 0616f12dd5..902a4f085c 100644 --- a/vllm/v1/worker/gpu/spec_decode/__init__.py +++ b/vllm/v1/worker/gpu/spec_decode/__init__.py @@ -1,11 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import TYPE_CHECKING + import torch from vllm.config import VllmConfig +if TYPE_CHECKING: + from vllm.v1.worker.gpu.states import RequestState -def init_speculator(vllm_config: VllmConfig, device: torch.device): + +def init_speculator( + vllm_config: VllmConfig, + device: torch.device, + req_states: "RequestState", +): + """Build the speculator for this config.""" speculative_config = vllm_config.speculative_config assert speculative_config is not None if speculative_config.method == "extract_hidden_states": @@ -66,5 +76,11 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device): ) return PlainDraftModelSpeculator(vllm_config, device) + elif speculative_config.use_ngram(): + from vllm.v1.worker.gpu.spec_decode.ngram.speculator import ( + NgramGPUSpeculator, + ) + + return NgramGPUSpeculator(vllm_config, device, req_states) else: raise NotImplementedError(f"{speculative_config.method} is not supported yet.") diff --git a/vllm/v1/worker/gpu/spec_decode/ngram/__init__.py b/vllm/v1/worker/gpu/spec_decode/ngram/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/v1/worker/gpu/spec_decode/ngram/speculator.py b/vllm/v1/worker/gpu/spec_decode/ngram/speculator.py new file mode 100644 index 0000000000..404f576274 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/ngram/speculator.py @@ -0,0 +1,278 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import torch + +from vllm.config import VllmConfig +from vllm.triton_utils import HAS_TRITON, tl, triton +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.spec_decode.speculator import BaseSpeculator + +if TYPE_CHECKING: + from vllm.config.compilation import CUDAGraphMode + from vllm.v1.worker.gpu.dp_utils import DPSyncState + from vllm.v1.worker.gpu.states import RequestState + + +@triton.jit +def _ngram_scan_kernel( + token_ids_ptr, # *int32 [max_num_reqs, token_ids_stride] + token_ids_stride, + idx_mapping_ptr, # *int64 [B] batch_idx -> req_state_idx + total_len_ptr, # *int32 [max_num_reqs] + num_sampled_ptr, # *int32 [B] + scratch_ptr, # *int64 [B, scratch_stride] (output) + scratch_stride, + L, # int64 scalar (= max_model_len) + MIN_N: tl.constexpr, + MAX_N: tl.constexpr, + MAX_N_PO2: tl.constexpr, + BLOCK_L: tl.constexpr, +): + b = tl.program_id(0).to(tl.int64) + blk = tl.program_id(1).to(tl.int64) + Lp1 = tl.cast(L, tl.int64) + 1 + + req_state_idx = tl.load(idx_mapping_ptr + b).to(tl.int64) + seq_len = tl.load(total_len_ptr + req_state_idx).to(tl.int64) + num_sampled = tl.load(num_sampled_ptr + b) + eligible_row = (num_sampled > 0) & (seq_len >= MIN_N) + + scratch_off = b * scratch_stride + blk + + # Ineligible rows, and blocks fully past the last candidate match + # position, write 0 and exit. + if not (eligible_row & (blk * BLOCK_L <= seq_len - MIN_N - 1)): + tl.store(scratch_ptr + scratch_off, tl.zeros((), tl.int64)) + return + + row_off = req_state_idx * token_ids_stride + + # Load the length-MAX_N suffix once into registers. + suf_iota = tl.arange(0, MAX_N_PO2).to(tl.int64) + suf_pos = seq_len - MAX_N + suf_iota + suf_in_range = (suf_iota < MAX_N) & (suf_pos >= 0) & (suf_pos < seq_len) + suffix = tl.load( + token_ids_ptr + row_off + suf_pos, + mask=suf_in_range, + other=-1, + ).to(tl.int32) + + pos_iota = tl.arange(0, BLOCK_L).to(tl.int64) + pos = blk * BLOCK_L + pos_iota # ascending + + best_score = tl.zeros([BLOCK_L], dtype=tl.int64) + + for n_iter in tl.static_range(MIN_N, MAX_N + 1): + max_pos_n = seq_len - n_iter - 1 + match = (pos >= 0) & (pos <= max_pos_n) + for j in tl.static_range(0, n_iter): + tok = tl.load( + token_ids_ptr + row_off + (pos + j), + mask=match, + other=0, + ).to(tl.int32) + suf_idx = (MAX_N - n_iter) + j + suf_val = tl.sum(tl.where(suf_iota == suf_idx, suffix, 0)) + match = match & (tok == suf_val) + + # Pack (n, pos) so a single max yields longest-n, rightmost-pos. + cand = n_iter * Lp1 + pos + 1 + best_score = tl.where(match, cand, best_score) + + block_best = tl.max(best_score, axis=0) + tl.store(scratch_ptr + scratch_off, block_best) + + +@triton.jit +def _ngram_finalize_kernel( + token_ids_ptr, # *int32 [max_num_reqs, token_ids_stride] + token_ids_stride, + idx_mapping_ptr, # *int64 [B] + total_len_ptr, # *int32 [max_num_reqs] + num_sampled_ptr, # *int32 [B] + last_sampled_ptr, # *int64 [max_num_reqs] + scratch_ptr, # *int64 [B, scratch_stride] + scratch_stride, + drafts_ptr, # *int64 [B, K] (output) + L, + N_BLOCKS, + K: tl.constexpr, + K_PO2: tl.constexpr, + N_BLOCKS_PO2: tl.constexpr, +): + b = tl.program_id(0).to(tl.int64) + Lp1 = tl.cast(L, tl.int64) + 1 + NB = tl.cast(N_BLOCKS, tl.int64) + + req_state_idx = tl.load(idx_mapping_ptr + b).to(tl.int64) + + nb_iota = tl.arange(0, N_BLOCKS_PO2).to(tl.int64) + nb_in_range = nb_iota < NB + block_scores = tl.load( + scratch_ptr + b * scratch_stride + nb_iota, + mask=nb_in_range, + other=0, + ) + score = tl.max(block_scores, axis=0) + + seq_len = tl.load(total_len_ptr + req_state_idx).to(tl.int64) + num_sampled = tl.load(num_sampled_ptr + b) + last_tok = tl.load(last_sampled_ptr + req_state_idx) + + has_match = score > 0 + s1 = score - 1 + best_n = tl.where(has_match, s1 // Lp1, tl.zeros_like(s1)) + best_pos = tl.where(has_match, s1 - best_n * Lp1, tl.zeros_like(s1)) + draft_start = tl.where(has_match, best_pos + best_n, tl.zeros_like(s1)) + + tokens_avail = tl.maximum(seq_len - draft_start, 0) + write_ok = (num_sampled > 0) & has_match + + row_off = req_state_idx * token_ids_stride + k_iota = tl.arange(0, K_PO2).to(tl.int64) + k_in_range = k_iota < K + gather_idx = tl.minimum(draft_start + k_iota, tl.cast(L, tl.int64) - 1) + slot_valid = (k_iota < tokens_avail) & write_ok & k_in_range + gathered = tl.load( + token_ids_ptr + row_off + gather_idx, + mask=slot_valid, + other=0, + ).to(tl.int64) + # Invalid slots fall back to the last sampled token; they are verified as + # ordinary (rejectable) drafts, so the fill value only affects efficiency. + out = tl.where(slot_valid, gathered, last_tok) + tl.store(drafts_ptr + b * K + k_iota, out, mask=k_in_range) + + +class NgramGPUSpeculator(BaseSpeculator): + """V2-compatible GPU n-gram speculator.""" + + supports_mm_inputs = False + draft_logits = None + + def __init__( + self, + vllm_config: VllmConfig, + device: torch.device, + req_states: RequestState, + ): + if not HAS_TRITON: + raise RuntimeError("ngram_gpu speculative decoding requires Triton.") + spec = vllm_config.speculative_config + assert spec is not None + assert spec.prompt_lookup_min is not None, ( + "prompt_lookup_min must be configured for ngram_gpu" + ) + assert spec.prompt_lookup_max is not None, ( + "prompt_lookup_max must be configured for ngram_gpu" + ) + assert 1 <= spec.prompt_lookup_min <= spec.prompt_lookup_max + + self.vllm_config = vllm_config + self.device = device + self.req_states = req_states + self.speculative_config = spec + self.num_speculative_steps: int = spec.num_speculative_tokens + + self.min_n: int = spec.prompt_lookup_min + self.max_n: int = spec.prompt_lookup_max + + self.max_num_reqs: int = vllm_config.scheduler_config.max_num_seqs + self.max_model_len: int = vllm_config.model_config.max_model_len + + L = self.max_model_len + if L >= 1024: + self.block_l = 256 + elif L >= 256: + self.block_l = 128 + elif L >= 64: + self.block_l = 64 + else: + self.block_l = max(16, triton.next_power_of_2(max(L, 1))) + self.n_blocks = triton.cdiv(L, self.block_l) + + self.scratch = torch.zeros( + (self.max_num_reqs, self.n_blocks), dtype=torch.int64, device=device + ) + # Batch-ordered draft output, scattered into RequestState.draft_tokens + # by the model runner (same contract as the model-based speculators). + self.drafts = torch.zeros( + (self.max_num_reqs, self.num_speculative_steps), + dtype=torch.int64, + device=device, + ) + + def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: + del cudagraph_mode + + def capture(self) -> None: + return None + + @torch.inference_mode() + def propose( + self, + input_batch: InputBatch, + attn_metadata: Any, + slot_mappings: Any, + last_hidden_states: torch.Tensor, + aux_hidden_states: list[torch.Tensor] | None, + num_sampled: torch.Tensor, + num_rejected: torch.Tensor, + last_sampled: torch.Tensor, + next_prefill_tokens: torch.Tensor, + temperature: torch.Tensor, + seeds: torch.Tensor, + dp_sync: DPSyncState | None = None, + dummy_run: bool = False, + skip_attn_for_dummy_run: bool = False, + mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, + is_profile: bool = False, + ) -> torch.Tensor: + num_reqs = input_batch.num_reqs + if dummy_run: + return self.drafts[:num_reqs] + + req_states = self.req_states + token_ids = req_states.all_token_ids.gpu + idx_mapping = input_batch.idx_mapping + + _ngram_scan_kernel[(num_reqs, self.n_blocks)]( + token_ids, + token_ids.stride(0), + idx_mapping, + req_states.total_len.gpu, + num_sampled, + self.scratch, + self.scratch.stride(0), + self.max_model_len, + self.min_n, + self.max_n, + max(1, triton.next_power_of_2(self.max_n)), + self.block_l, + num_warps=4, + num_stages=2, + ) + + _ngram_finalize_kernel[(num_reqs,)]( + token_ids, + token_ids.stride(0), + idx_mapping, + req_states.total_len.gpu, + num_sampled, + last_sampled.view(-1), + self.scratch, + self.scratch.stride(0), + self.drafts, + self.max_model_len, + self.n_blocks, + self.num_speculative_steps, + max(1, triton.next_power_of_2(self.num_speculative_steps)), + max(1, triton.next_power_of_2(self.n_blocks)), + num_warps=2, + num_stages=1, + ) + return self.drafts[:num_reqs] diff --git a/vllm/v1/worker/gpu/states.py b/vllm/v1/worker/gpu/states.py index db27bf013f..16260b0ddb 100644 --- a/vllm/v1/worker/gpu/states.py +++ b/vllm/v1/worker/gpu/states.py @@ -16,6 +16,7 @@ class RequestState: vocab_size: int, device: torch.device, num_prefill_lookahead: int = 1, + use_dense_all_token_ids: bool = False, ): self.max_num_reqs = max_num_reqs self.max_model_len = max_model_len @@ -30,12 +31,14 @@ class RequestState: # NOTE(woosuk): This tensor can be extremely large (e.g., several GBs) # depending on the configured max_num_reqs and max_model_len. - # To save GPU memory, we use UVA instead of GPU for this tensor. + # To save GPU memory, we use UVA instead of GPU by default, but + # ngram_gpu benefits from dense device residency because it scans + # active rows repeatedly during proposal. self.all_token_ids = StagedWriteTensor( (self.max_num_reqs, self.max_model_len), dtype=torch.int32, device=device, - uva_instead_of_gpu=True, + uva_instead_of_gpu=not use_dense_all_token_ids, ) # NOTE(woosuk): Distinguish clearly between prompt_len and prefill_len: # - prompt_len: Number of tokens in the user-provided prompt.