Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -36,6 +36,39 @@ def test_ngram_max_len(num_speculative_tokens: int, vllm_runner):
|
||||
runner.llm.generate(_PROMPTS, sampling_params)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
|
||||
@pytest.mark.parametrize("method", ["ngram", "ngram_gpu"])
|
||||
def test_ngram_gpu_max_len(
|
||||
method: str,
|
||||
num_speculative_tokens: int,
|
||||
vllm_runner,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""V2 n-gram decoding stops at max_model_len."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
trust_remote_code=False,
|
||||
max_model_len=100,
|
||||
enable_chunked_prefill=None,
|
||||
enforce_eager=True, # For faster initialization.
|
||||
speculative_config={
|
||||
"method": method,
|
||||
"prompt_lookup_max": 5,
|
||||
"prompt_lookup_min": 3,
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
},
|
||||
) as runner:
|
||||
assert runner.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
sampling_params = SamplingParams(max_tokens=100, ignore_eos=True)
|
||||
outputs = runner.llm.generate(_PROMPTS, sampling_params)
|
||||
for output in outputs:
|
||||
assert output.prompt_token_ids is not None
|
||||
assert (
|
||||
len(output.prompt_token_ids) + len(output.outputs[0].token_ids) == 100
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1, 3, 10])
|
||||
@pytest.mark.parametrize("attn_backend", get_attn_backend_list_based_on_platform())
|
||||
def test_eagle_max_len(
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.config import StructuredOutputsConfig, VllmConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
|
||||
from vllm.parser.engine.parser_engine import ReasoningEnd
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
@@ -113,12 +114,12 @@ class _EngineReasonerStub(ParserEngineReasoningAdapter):
|
||||
def reasoning_end_token_ids(self):
|
||||
return self._end_token_ids
|
||||
|
||||
def find_reasoning_end_offset(self, token_ids):
|
||||
def find_reasoning_end(self, token_ids):
|
||||
self.windows.append(list(token_ids))
|
||||
for offset, token in enumerate(token_ids):
|
||||
if token in self._end_token_ids:
|
||||
return offset
|
||||
return len(token_ids)
|
||||
return ReasoningEnd(offset, False)
|
||||
return ReasoningEnd(len(token_ids), False)
|
||||
|
||||
def is_reasoning_end(self, input_ids):
|
||||
return any(token in self._end_token_ids for token in input_ids)
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the V2 GPU-accelerated n-gram speculator.
|
||||
|
||||
These tests target the Triton proposer in
|
||||
``vllm.v1.worker.gpu.spec_decode.ngram.speculator`` and complement the CPU
|
||||
``NgramProposer`` tests in ``test_ngram.py``. The GPU speculator follows a
|
||||
slightly different policy than the CPU one: when multiple n-gram matches of
|
||||
the same length exist, the GPU kernel picks the right-most (most recent)
|
||||
match inside the active context, whereas the CPU implementation returns the
|
||||
left-most. The expectations below reflect the GPU behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.ngram.speculator import NgramGPUSpeculator
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip(
|
||||
"CUDA required for NgramGPUSpeculator tests",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
min_n: int,
|
||||
max_n: int,
|
||||
k: int,
|
||||
max_num_seqs: int = 8,
|
||||
max_model_len: int = 64,
|
||||
method: str = "ngram_gpu",
|
||||
) -> VllmConfig:
|
||||
model_config = ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
max_model_len=max_model_len,
|
||||
enforce_eager=True,
|
||||
)
|
||||
scheduler_config = SchedulerConfig.default_factory(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
speculative_config = SpeculativeConfig(
|
||||
method=method,
|
||||
prompt_lookup_min=min_n,
|
||||
prompt_lookup_max=max_n,
|
||||
num_speculative_tokens=k,
|
||||
)
|
||||
return VllmConfig(
|
||||
model_config=model_config,
|
||||
scheduler_config=scheduler_config,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
|
||||
def _make_request_state(cfg: VllmConfig) -> RequestState:
|
||||
assert cfg.speculative_config is not None
|
||||
return RequestState(
|
||||
max_num_reqs=cfg.scheduler_config.max_num_seqs,
|
||||
max_model_len=cfg.model_config.max_model_len,
|
||||
max_num_batched_tokens=cfg.scheduler_config.max_num_batched_tokens,
|
||||
num_speculative_steps=cfg.speculative_config.num_speculative_tokens,
|
||||
vocab_size=cfg.model_config.get_vocab_size(),
|
||||
device=DEVICE,
|
||||
use_dense_all_token_ids=True,
|
||||
)
|
||||
|
||||
|
||||
def _make_speculator(
|
||||
min_n: int,
|
||||
max_n: int,
|
||||
k: int,
|
||||
max_num_seqs: int = 8,
|
||||
max_model_len: int = 32,
|
||||
) -> NgramGPUSpeculator:
|
||||
cfg = _make_vllm_config(
|
||||
min_n=min_n,
|
||||
max_n=max_n,
|
||||
k=k,
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
return NgramGPUSpeculator(cfg, DEVICE, _make_request_state(cfg))
|
||||
|
||||
|
||||
def _propose(
|
||||
spec: NgramGPUSpeculator,
|
||||
rows: list[list[int]],
|
||||
seq_lens: list[int] | None = None,
|
||||
num_sampled: list[int] | None = None,
|
||||
last_sampled: list[int] | None = None,
|
||||
slots: list[int] | None = None,
|
||||
) -> list[list[int]]:
|
||||
"""Place each batch row at a request slot and run propose().
|
||||
|
||||
Returns the drafts as python lists in batch order.
|
||||
"""
|
||||
B = len(rows)
|
||||
if seq_lens is None:
|
||||
seq_lens = [len(r) for r in rows]
|
||||
if num_sampled is None:
|
||||
num_sampled = [1] * B
|
||||
if last_sampled is None:
|
||||
last_sampled = [0] * B
|
||||
if slots is None:
|
||||
slots = list(range(B))
|
||||
|
||||
max_num_reqs = spec.max_num_reqs
|
||||
all_token_ids = spec.req_states.all_token_ids.gpu
|
||||
total_len = spec.req_states.total_len.gpu
|
||||
all_token_ids.zero_()
|
||||
total_len.zero_()
|
||||
last_sampled_t = torch.zeros((max_num_reqs, 1), dtype=torch.int64, device=DEVICE)
|
||||
for row, slot, seq_len, last in zip(rows, slots, seq_lens, last_sampled):
|
||||
if row:
|
||||
all_token_ids[slot, : len(row)] = torch.tensor(
|
||||
row, dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
total_len[slot] = seq_len
|
||||
last_sampled_t[slot, 0] = last
|
||||
|
||||
idx_mapping = torch.tensor(slots, dtype=torch.int64, device=DEVICE)
|
||||
input_batch = SimpleNamespace(num_reqs=B, idx_mapping=idx_mapping)
|
||||
|
||||
drafts = spec.propose(
|
||||
input_batch=input_batch,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
last_hidden_states=torch.empty(0, device=DEVICE),
|
||||
aux_hidden_states=None,
|
||||
num_sampled=torch.tensor(num_sampled, dtype=torch.int32, device=DEVICE),
|
||||
num_rejected=torch.zeros(B, dtype=torch.int32, device=DEVICE),
|
||||
last_sampled=last_sampled_t,
|
||||
next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE),
|
||||
temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE),
|
||||
seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE),
|
||||
dp_sync=None,
|
||||
)
|
||||
return drafts.cpu().tolist()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_model_len", [32, 300])
|
||||
def test_no_match_clears_previous_proposal(max_model_len):
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2, max_model_len=max_model_len)
|
||||
row = [0] * (max_model_len - 5) + [1, 2, 3, 1, 2]
|
||||
assert _propose(spec, [row]) == [[3, 1]]
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 5]], last_sampled=[42])
|
||||
assert drafts == [[42, 42]]
|
||||
|
||||
|
||||
def test_no_4gram_match_only():
|
||||
"""No 4-gram match in [1,2,3,4,1,2,3] → 0 valid drafts."""
|
||||
spec = _make_speculator(min_n=4, max_n=4, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 1, 2, 3]], last_sampled=[7])
|
||||
assert drafts == [[7, 7]]
|
||||
|
||||
|
||||
def test_falls_back_to_3gram_when_4gram_missing():
|
||||
"""No 4-gram match but a 3-gram match exists → propose [4, 1]."""
|
||||
spec = _make_speculator(min_n=3, max_n=4, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 1, 2, 3]])
|
||||
assert drafts == [[4, 1]]
|
||||
|
||||
|
||||
def test_prefers_longer_ngram():
|
||||
"""Prefer a 4-gram match over a more recent 3-gram match."""
|
||||
spec = _make_speculator(min_n=3, max_n=4, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 50, 51, 2, 3, 4, 60, 61, 1, 2, 3, 4]])
|
||||
assert drafts == [[50, 51]]
|
||||
|
||||
|
||||
def test_picks_longest_match_among_2_3_4_grams():
|
||||
"""Prefer a 3-gram match over a more recent 2-gram match."""
|
||||
spec = _make_speculator(min_n=2, max_n=4, k=2)
|
||||
drafts = _propose(spec, [[2, 3, 4, 50, 51, 3, 4, 60, 61, 1, 2, 3, 4]])
|
||||
assert drafts == [[50, 51]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_model_len", [32, 128, 257, 1025])
|
||||
def test_picks_rightmost_when_multiple_matches(max_model_len):
|
||||
"""Pick the last valid match across blocks, ignoring trailing tokens."""
|
||||
spec = _make_speculator(min_n=3, max_n=3, k=2, max_model_len=max_model_len)
|
||||
padding = [0] * (spec.block_l - 5) if spec.n_blocks > 1 else []
|
||||
row = [1, 2, 3, 100] + padding + [1, 2, 3, 200, 1, 2, 3, 300, 1, 2, 3]
|
||||
drafts = _propose(spec, [row + [1, 2, 3, 999, 1, 2, 3]], seq_lens=[len(row)])
|
||||
assert drafts == [[300, 1]]
|
||||
|
||||
|
||||
def test_short_context_yields_zero_valid():
|
||||
"""The only length-2 window overlaps the suffix itself → no match."""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2)
|
||||
drafts = _propose(spec, [[5, 6]], last_sampled=[99])
|
||||
assert drafts == [[99, 99]]
|
||||
|
||||
|
||||
def test_zero_sampled_disables_proposal():
|
||||
"""num_sampled==0 disables proposals for that request regardless of match."""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 1, 2]], num_sampled=[0], last_sampled=[77])
|
||||
assert drafts == [[77, 77]]
|
||||
|
||||
|
||||
def test_tail_falls_back_when_few_tokens_after_match():
|
||||
"""Fewer than k tokens after the match → the tail falls back.
|
||||
|
||||
Tokens: [1, 2, 1, 2] (seq_len=4). Suffix (1, 2) matches at position 0
|
||||
(the match at position 2 is the suffix itself and is excluded). With
|
||||
k=3, only 2 slots map to tokens inside the context.
|
||||
"""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=3)
|
||||
drafts = _propose(spec, [[1, 2, 1, 2]], last_sampled=[55])
|
||||
assert drafts == [[1, 2, 55]]
|
||||
|
||||
|
||||
def test_multibatch_mixed():
|
||||
"""Mixed batch: row 0 matches, row 1 has no match."""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2)
|
||||
drafts = _propose(
|
||||
spec,
|
||||
[[1, 2, 3, 1, 2], [4, 5, 6]],
|
||||
last_sampled=[10, 20],
|
||||
)
|
||||
assert drafts[0] == [3, 1]
|
||||
assert drafts[1] == [20, 20]
|
||||
|
||||
|
||||
def test_multibatch_independent_choice_of_n():
|
||||
"""Each row independently picks its longest matched n."""
|
||||
spec = _make_speculator(min_n=2, max_n=3, k=2)
|
||||
drafts = _propose(
|
||||
spec,
|
||||
[
|
||||
[9, 1, 2, 3, 8, 1, 2, 3], # 3-gram (1,2,3) at idx 1 → [8, 1]
|
||||
[7, 1, 2, 9, 1, 2], # 2-gram (1,2) at idx 1 → [9, 1]
|
||||
],
|
||||
)
|
||||
assert drafts[0] == [8, 1]
|
||||
assert drafts[1] == [9, 1]
|
||||
|
||||
|
||||
def test_min_n_eq_1():
|
||||
"""min_n=max_n=1 — single-token n-grams always match if context > 1."""
|
||||
spec = _make_speculator(min_n=1, max_n=1, k=2)
|
||||
drafts = _propose(spec, [[1, 2, 3, 4, 1]])
|
||||
assert drafts == [[2, 3]]
|
||||
|
||||
|
||||
def test_noncontiguous_idx_mapping():
|
||||
"""propose() reads token rows in place via idx_mapping (non-contiguous)."""
|
||||
spec = _make_speculator(min_n=2, max_n=2, k=2)
|
||||
drafts = _propose(
|
||||
spec,
|
||||
[[7, 8, 9, 7, 8], [1, 2, 3, 1, 2]],
|
||||
slots=[3, 0],
|
||||
)
|
||||
assert drafts == [[9, 7], [3, 1]]
|
||||
|
||||
|
||||
def test_construction_validates_speculative_config():
|
||||
spec = _make_speculator(min_n=2, max_n=3, k=2)
|
||||
assert spec.min_n == 2
|
||||
assert spec.max_n == 3
|
||||
assert spec.num_speculative_steps == 2
|
||||
# No-op hooks must not raise.
|
||||
spec.init_cudagraph_manager(None)
|
||||
spec.capture()
|
||||
Reference in New Issue
Block a user