[MyPy][3/N] Fix MyPy errors in test groups (part 3) (#55939)

Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Taimys
2026-09-30 11:06:07 +00:00
committed by GitHub
co-authored by Codex Luyi Xiao Taimys Harry Mellor Claude Opus 5.5
parent c4fa6f36a7
commit 5225f1bacb
118 changed files with 948 additions and 605 deletions
+54 -71
View File
@@ -3,7 +3,7 @@
import random
from collections.abc import Callable
from typing import NamedTuple, TypeAlias
from typing import NamedTuple, NoReturn, TypeAlias
import numpy as np
import pytest
@@ -13,6 +13,7 @@ from tests.utils import create_new_process_for_each_test
from tests.v1.sample.utils import (
LogitsprocsTestFakes,
create_fake_logits,
create_mock_reasoning_config,
create_penalty_tensor,
create_prompt_tokens_tensor,
fake_apply_logitsprocs,
@@ -103,14 +104,6 @@ class LogitsProcsRequestParams:
return f"MyClass({summ})"
class MockReasoningConfig:
"""Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests."""
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
reasoning_end_token_ids = [THINK_END_TOKEN_ID]
enabled = True
def _generate_fake_sampling_metadata(
num_output_tokens: int,
batch_size: int,
@@ -131,7 +124,9 @@ def _generate_fake_sampling_metadata(
)
vllm_config = VllmConfig()
vllm_config.reasoning_config = MockReasoningConfig()
vllm_config.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
logitsprocs = build_logitsprocs(
vllm_config=vllm_config,
@@ -242,7 +237,7 @@ def _raise_error_invalid(
request_params: LogitsProcsRequestParams,
step_idx: int,
err_cls: type[Exception] = ValueError,
) -> None:
) -> NoReturn:
raise err_cls(
f"Validation failed for step={step_idx}, "
f"batch_index={batch_index}, "
@@ -268,6 +263,7 @@ def _logit_bias_validate(
) -> None:
"""Validate logit bias logitproc applied correctly."""
logit_bias = request_params.params.logit_bias
assert logit_bias is not None
logits_old = test_fakes.logits[persistent_batch[batch_index].workload_index].cpu()
logits_new = logits_new[batch_index].cpu()
for token_id in range(VOCAB_SIZE):
@@ -368,9 +364,7 @@ def _min_tokens_validate(
ref_num_out_tokens = len(request_params.out_tokens)
min_reached = ref_num_out_tokens >= MIN_TOKENS_LEN_THRESHOLD
ref_all_stop_token_ids = request_params.params.all_stop_token_ids
mt_lp: MinTokensLogitsProcessor = next(
test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor)
)
mt_lp = next(test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor))
assert isinstance(mt_lp, MinTokensLogitsProcessor)
min_tok = mt_lp.min_toks.get(batch_index, None)
@@ -689,7 +683,7 @@ class LogitsprocTestHelpers(NamedTuple):
gen_request_fxn: Callable | None = None
logitsprocs_test_mapping = {
logitsprocs_test_mapping: dict[LogitprocType, LogitsprocTestHelpers] = {
STR_NO_LOGITPROC: LogitsprocTestHelpers(eval_fxn=_none_validate),
LogitBiasLogitsProcessor: LogitsprocTestHelpers(
gen_request_fxn=_logit_bias_params, eval_fxn=_logit_bias_validate
@@ -706,22 +700,22 @@ logitsprocs_test_mapping = {
}
def _get_test_cases() -> list[list[str]]:
def _get_test_cases() -> list[list[LogitprocType]]:
"""Each test case is a set of logitsprocs."""
logitsprocs_types = list(logitsprocs_test_mapping.keys())
logitsprocs_types: list[LogitprocType] = list(logitsprocs_test_mapping.keys())
# Isolate thinking-budget handling from other processors to avoid cross-talk.
thinking_id: LogitprocType = STR_THINKING_BUDGET
other_processors = [
other_processors: list[LogitprocType] = [
p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id
]
return (
[[STR_NO_LOGITPROC]]
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
+ [other_processors]
+ [[thinking_id]]
test_cases: list[list[LogitprocType]] = [[STR_NO_LOGITPROC]]
test_cases.extend(
[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors
)
test_cases.extend([other_processors, [thinking_id]])
return test_cases
def _generate_fake_step_update(
@@ -774,6 +768,7 @@ def _generate_fake_step_update(
for add_req_params in workload_params[wdx : (wdx + num_step_add_replace)]:
# Replace as many removed requests as possible with added requests
add_remove_idx = batch_update_builder.pop_removed()
assert add_remove_idx is not None
batch_update_builder.added.append(
(
add_remove_idx,
@@ -976,13 +971,6 @@ def test_logitsprocs(
step_idx += 1
class MockReasoningNoEndTokens:
"""Reasoning config with no end token ids (disables enforcement in holder)."""
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
reasoning_end_token_ids: list[int] = []
def test_maybe_create_thinking_budget_holder_without_reasoning():
cfg = VllmConfig()
assert cfg.reasoning_config is None
@@ -1000,7 +988,9 @@ def test_maybe_create_thinking_budget_holder_without_reasoning():
def test_thinking_budget_holder_has_tracked_after_sync_add():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1030,7 +1020,9 @@ def test_thinking_budget_holder_has_tracked_after_sync_add():
def test_thinking_budget_holder_sync_remove_clears_state():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1060,7 +1052,9 @@ def test_thinking_budget_holder_sync_remove_clears_state():
def test_thinking_budget_holder_sync_add_without_budget_drops_row():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1081,7 +1075,9 @@ def test_thinking_budget_holder_sync_add_without_budget_drops_row():
def test_thinking_budget_holder_swap_exchanges_state():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1125,7 +1121,9 @@ def test_thinking_budget_holder_swap_exchanges_state():
def test_thinking_budget_holder_unidirectional_move():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1163,7 +1161,9 @@ def test_thinking_budget_holder_unidirectional_move():
def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1197,7 +1197,7 @@ def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
def test_thinking_budget_holder_spec_mode_tensor_layout():
h = ThinkingBudgetStateHolder(
MockReasoningConfig(),
create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]),
8,
2,
torch.device("cpu"),
@@ -1209,7 +1209,7 @@ def test_thinking_budget_holder_spec_mode_tensor_layout():
def test_thinking_budget_holder_empty_end_tokens_disables_row():
vc = VllmConfig()
vc.reasoning_config = MockReasoningNoEndTokens()
vc.reasoning_config = create_mock_reasoning_config([THINK_START_TOKEN_ID], [])
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1253,7 +1253,9 @@ def test_thinking_budget_enforced_without_penalties():
passing an empty list (the pre-fix behavior) prevents budget enforcement.
"""
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
budget = 3 # allow 3 thinking tokens
h = ThinkingBudgetStateHolder(
@@ -1346,7 +1348,11 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
"""Test thinking budget enforced for a long thinking run,
then a natural end marker."""
h = ThinkingBudgetStateHolder(
MockReasoningConfig(), 8, 0, torch.device("cpu"), False
create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]),
8,
0,
torch.device("cpu"),
False,
)
h.sync_batch(
BatchUpdate(
@@ -1356,8 +1362,8 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
moved=(),
)
)
start = MockReasoningConfig.reasoning_start_token_ids
end = MockReasoningConfig.reasoning_end_token_ids
start = [THINK_START_TOKEN_ID]
end = [THINK_END_TOKEN_ID]
out: list[int] = list(start)
h.update_state([out], None, None)
@@ -1396,13 +1402,9 @@ class TestThinkingBudgetReentry:
@staticmethod
def _make_holder(end_token_ids: list[int]) -> ThinkingBudgetStateHolder:
class FakeReasoningConfig:
reasoning_start_token_ids = [TestThinkingBudgetReentry.THINK_START]
reasoning_end_token_ids: list[int] = []
enabled = True
cfg = FakeReasoningConfig()
cfg.reasoning_end_token_ids = end_token_ids
cfg = create_mock_reasoning_config(
[TestThinkingBudgetReentry.THINK_START], end_token_ids
)
return ThinkingBudgetStateHolder(
reasoning_config=cfg,
max_num_seqs=8,
@@ -1526,21 +1528,12 @@ class TestThinkingBudgetNaturalEndReentry:
@staticmethod
def _make_holder(end_token_ids):
from dataclasses import dataclass
from vllm.v1.sample.thinking_budget_state import (
ThinkingBudgetStateHolder,
)
@dataclass
class FakeReasoningConfig:
reasoning_start_token_ids: list[int]
reasoning_end_token_ids: list[int]
enabled: bool = True
cfg = FakeReasoningConfig(
reasoning_start_token_ids=[TestThinkingBudgetNaturalEndReentry.THINK_START],
reasoning_end_token_ids=end_token_ids,
cfg = create_mock_reasoning_config(
[TestThinkingBudgetNaturalEndReentry.THINK_START], end_token_ids
)
return ThinkingBudgetStateHolder(
reasoning_config=cfg,
@@ -1830,21 +1823,11 @@ class TestThinkingBudgetNaturalEndReentry:
natural </think> exit, continue_thinking is cleared and Block 2
enforcement works correctly.
"""
from dataclasses import dataclass
from unittest.mock import MagicMock
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
@dataclass
class FakeReasoningConfig:
reasoning_start_token_ids: list[int]
reasoning_end_token_ids: list[int]
enabled: bool = True
cfg = FakeReasoningConfig(
reasoning_start_token_ids=[self.THINK_START],
reasoning_end_token_ids=self.THINK_END_SINGLE,
)
cfg = create_mock_reasoning_config([self.THINK_START], self.THINK_END_SINGLE)
holder = ThinkingBudgetStateHolder(
reasoning_config=cfg,
max_num_seqs=8,
+10 -6
View File
@@ -57,15 +57,20 @@ prompts = [
class DummyLogitsProcessor(LogitsProcessor):
"""Fake logit processor to support unit testing and examples."""
@classmethod
def validate_params(cls, params: SamplingParams):
target_token: int | None = params.extra_args and params.extra_args.get(
"target_token"
@staticmethod
def _get_target_token(params: SamplingParams) -> int | None:
target_token = (
params.extra_args.get(DUMMY_LOGITPROC_ARG) if params.extra_args else None
)
if target_token is not None and not isinstance(target_token, int):
raise VLLMValidationError(
f"target_token value {target_token} {type(target_token)} is not int"
)
return target_token
@classmethod
def validate_params(cls, params: SamplingParams):
cls._get_target_token(params)
def __init__(
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
@@ -78,8 +83,7 @@ class DummyLogitsProcessor(LogitsProcessor):
def update_state(self, batch_update: BatchUpdate | None):
def extract_extra_arg(params: SamplingParams) -> int | None:
self.validate_params(params)
return params.extra_args and params.extra_args.get("target_token")
return self._get_target_token(params)
process_dict_updates(
self.req_info,
@@ -341,6 +341,7 @@ def test_batch_allowed_needs_one_shared_set():
assert states.batch_allowed([]) is None
shared = states.batch_allowed([0])
assert shared is not None
assert shared.tolist() == [1, 2, 3]
assert shared.dtype == torch.int64
assert states.batch_allowed([0, 0]) is shared
@@ -34,6 +34,8 @@ def test_stats_match_reference(vocab: int, dtype: torch.dtype):
)
assert torch.equal(argmax, ref_argmax)
torch.testing.assert_close(entropy, ref_entropy, atol=2e-3, rtol=1e-3)
assert probs is not None
assert ref_probs is not None
torch.testing.assert_close(probs.float(), ref_probs.float(), atol=5e-3, rtol=2e-2)
assert probs.dtype == torch.bfloat16
assert probs.shape == logits.shape
@@ -100,6 +102,7 @@ def test_masked_logits_keep_a_finite_entropy():
assert entropy[0].item() < 1e-6 and entropy[1].item() < 1e-6
assert abs(entropy[2].item() - math.log(2.0)) < 1e-4
assert torch.equal(argmax[[0, 1, 3]], torch.tensor([3, 3, 3], device="cuda"))
assert probs is not None
assert probs[0, 3].item() == 1.0 and probs[0].sum().item() == 1.0
_, _, ref_entropy, _ = sample_row_stats_reference(logits, temps, 1, None)
assert torch.isfinite(ref_entropy).all()
+6 -3
View File
@@ -340,7 +340,7 @@ def _make_v2_state(
device=device,
all_token_ids=SimpleNamespace(gpu=all_tokens),
)
state = DryState(_make_vllm_config(), req_states)
state = DryState(_make_vllm_config(), req_states) # type: ignore[arg-type]
if tokenizer is not None:
# Re-resolve the default set now that a tokenizer is installed.
state._tokenizer = tokenizer
@@ -446,7 +446,7 @@ def test_spec_decode_is_refused_at_construction():
)
config = _make_vllm_config(speculative_config=SimpleNamespace())
with pytest.raises(ValueError, match="speculative decoding"):
DryState(config, req_states)
DryState(config, req_states) # type: ignore[arg-type]
def test_v2_draft_expanded_logits_raise():
@@ -917,7 +917,10 @@ def test_loads_through_the_loader_by_fqcn():
total_len=None,
)
procs = loader.build_custom_logits_processors(
_make_vllm_config(), req_states, False, [DRY_FQCN]
_make_vllm_config(),
req_states, # type: ignore[arg-type]
False,
[DRY_FQCN],
)
assert [type(p) for p in procs] == [DryState]
+21 -7
View File
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
)
@@ -32,10 +33,17 @@ class _FakeLmHead:
self.tp_size = 1
def _fake_lm_head(
weight: torch.Tensor, quantized: bool = False, shard_indices: object | None = None
) -> VocabParallelEmbedding:
# The dtype tests only need the projection fields, not distributed setup.
return _FakeLmHead(weight, quantized, shard_indices) # type: ignore[return-value]
def _build_processor(vocab_size: int) -> LogitsProcessor:
lp = LogitsProcessor(vocab_size)
# The TP gather is orthogonal to the dtype behavior under test.
lp._gather_logits = lambda logits: logits
lp._gather_logits = lambda logits: logits # type: ignore[method-assign]
return lp
@@ -47,8 +55,9 @@ def test_fp32_head_runs_projection_in_fp32(default_vllm_config):
hidden_states = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
assert logits is not None
assert logits.dtype == torch.float32
assert torch.isfinite(logits).all()
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
@@ -66,8 +75,9 @@ def test_non_fp32_head_dtype_uses_cast_path(default_vllm_config):
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
assert logits is not None
assert logits.dtype == torch.float16
expected = torch.nn.functional.linear(hidden_states.half(), weight.half())
torch.testing.assert_close(logits, expected)
@@ -82,7 +92,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config):
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
lm_head = _FakeLmHead(weight)
lm_head = _fake_lm_head(weight)
with mock.patch.object(
lm_head.quant_method,
@@ -94,6 +104,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config):
logits = lp._get_logits(hidden_states, lm_head, None)
apply_mock.assert_called_once()
assert logits is not None
assert logits.dtype == torch.bfloat16
@@ -120,9 +131,10 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config):
with mock.patch(
"vllm.model_executor.layers.logits_processor.F.linear"
) as linear_mock:
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
linear_mock.assert_not_called()
assert logits is not None
assert logits.dtype == torch.float32
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
torch.testing.assert_close(logits, expected)
@@ -131,7 +143,7 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config):
def test_fp32_head_rejects_quantized_lm_head(default_vllm_config):
lp = _build_processor(64)
lp.head_dtype = torch.float32
lm_head = _FakeLmHead(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True)
lm_head = _fake_lm_head(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True)
with pytest.raises(ValueError, match="unquantized"):
lp._get_logits(torch.randn(4, 16, dtype=torch.bfloat16), lm_head, None)
@@ -197,7 +209,7 @@ def test_get_top_tokens_honors_head_dtype(default_vllm_config):
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
lm_head = _FakeLmHead(
lm_head = _fake_lm_head(
weight,
shard_indices=types.SimpleNamespace(
num_org_vocab_padding=0, org_vocab_start_index=0
@@ -236,7 +248,9 @@ def test_fp32_head_e2e_no_nan():
for output in outputs:
for completion in output.outputs:
assert completion.logprobs is not None
for token_id, position in zip(completion.token_ids, completion.logprobs):
assert position is not None
# The sampled token survived filtering, so its logprob is finite.
assert math.isfinite(position[token_id].logprob)
# No returned logprob is NaN.
+33 -17
View File
@@ -24,6 +24,7 @@ from vllm import SamplingParams
from vllm.config.model import LogprobsMode
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
from vllm.logprobs import Logprob
from vllm.platforms import current_platform
from vllm.v1.engine.input_processor import InputProcessor
@@ -145,7 +146,7 @@ def _repeat_logprob_config(
def _run_and_validate(
vllm_model: VllmRunner,
test_prompts: list[str],
vllm_sampling_params: SamplingParams,
vllm_sampling_params: SamplingParams | list[SamplingParams],
hf_logprobs: list[list[torch.Tensor]],
hf_outputs: list[tuple[list[int], str]],
logprob_prompt_logprob_list: BatchLogprobsSpecType,
@@ -163,11 +164,12 @@ def _run_and_validate(
# Extract request-level (prompt)logprobs config
num_top_logprobs, num_top_prompt_logprobs = logprob_prompt_logprob
assert vllm_result.prompt_token_ids is not None
# Test whether sampled token output is consistent between vLLM and HF
# vLLM prompt+completion should match HF output
if temperature == 0.0:
assert (
vllm_result.prompt_token_ids + vllm_result.outputs[0].token_ids
vllm_result.prompt_token_ids + list(vllm_result.outputs[0].token_ids)
== hf_output[0]
)
else:
@@ -191,7 +193,9 @@ def _run_and_validate(
# Confirm that the output token appears among the logprobs
assert token_id in logprobs
token_in_topk = logprobs[token_id].rank <= num_top_logprobs
rank = logprobs[token_id].rank
assert rank is not None
token_in_topk = rank <= num_top_logprobs
# If the output token is not included in the top K
# logprob, it can return 1 more data
@@ -207,8 +211,10 @@ def _run_and_validate(
output_text = vllm_result.outputs[0].text
output_string_from_most_likely_tokens_lst: list[str] = []
for top_logprobs in vllm_result.outputs[0].logprobs:
top_logprob = next(iter(top_logprobs.values()))
for output_logprobs in vllm_result.outputs[0].logprobs:
assert output_logprobs is not None
top_logprob = next(iter(output_logprobs.values()))
assert top_logprob.decoded_token is not None
output_string_from_most_likely_tokens_lst.append(
top_logprob.decoded_token
)
@@ -226,8 +232,9 @@ def _run_and_validate(
# Compare vLLM sample logprobs to HF
vllm_sample_logprobs = vllm_result.outputs[0].logprobs
for i, top_logprobs in enumerate(vllm_sample_logprobs):
for token_id, sample_logprob in top_logprobs.items():
for i, sample_logprobs in enumerate(vllm_sample_logprobs):
assert sample_logprobs is not None
for token_id, sample_logprob in sample_logprobs.items():
if temperature == 0.0 or i == 0:
logprob = sample_logprob.logprob
torch.testing.assert_close(
@@ -271,9 +278,9 @@ def _run_and_validate(
# Confirm that the prompt token appears among the logprobs
assert prompt_token_id in prompt_logprobs
token_in_topk = (
prompt_logprobs[prompt_token_id].rank <= num_top_prompt_logprobs
)
prompt_rank = prompt_logprobs[prompt_token_id].rank
assert prompt_rank is not None
token_in_topk = prompt_rank <= num_top_prompt_logprobs
# If the prompt token is not included in the top K
# logprob, it can return 1 more data
@@ -294,9 +301,10 @@ def _run_and_validate(
# 1:.
vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
for token_id, logprob in vllm_prompt_logprob_dict.items():
assert vllm_prompt_logprob_dict is not None
for token_id, prompt_lp in vllm_prompt_logprob_dict.items():
torch.testing.assert_close(
logprob.logprob,
prompt_lp.logprob,
hf_logprob[0][i][token_id].item(),
atol=2e-2,
rtol=2e-2,
@@ -478,12 +486,12 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
with patch.object(SamplingParams, "verify"):
with pytest.raises(VLLMValidationError, match="V2 model runner"):
InputProcessor._validate_params(processor, params, ("generate",))
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
processor.vllm_config.use_v2_model_runner = True
InputProcessor._validate_params(processor, params, ("generate",))
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
processor.vllm_config.cache_config.kv_sharing_fast_prefill = True
with pytest.raises(VLLMValidationError, match="fast-prefill"):
InputProcessor._validate_params(processor, params, ("generate",))
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
def test_none_logprobs(vllm_model, example_prompts):
@@ -607,7 +615,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
total_token_with_logprobs = 0
positive_values = 0
for output in results[0].outputs:
assert output.logprobs is not None
for logprobs in output.logprobs:
assert logprobs is not None
for token_id in logprobs:
logprob = logprobs[token_id]
if logprobs_mode in ("raw_logprobs", "processed_logprobs"):
@@ -648,6 +658,7 @@ def test_prompt_logprobs_mode():
)
assert results[0].prompt_logprobs is not None
assert results[0].prompt_logprobs[1] is not None
assert results[0].prompt_token_ids is not None
tok_id = results[0].prompt_token_ids[1]
values[mode] = results[0].prompt_logprobs[1][tok_id].logprob
finally:
@@ -1228,10 +1239,12 @@ def test_spec_decode_logprobs(
[prompt, prompt], [sampling_params, penalty_sampling_params]
)
# Collect logprobs outputs from reference LLM.
ref_logprobs = []
ref_logprobs: list[Logprob] = []
for results in ref_results:
for output in results.outputs:
assert output.logprobs is not None
for logprobs in output.logprobs:
assert logprobs is not None
ref_logprobs.extend(logprobs.values())
del ref_llm
torch.accelerator.empty_cache()
@@ -1249,10 +1262,12 @@ def test_spec_decode_logprobs(
[prompt, prompt], [sampling_params, penalty_sampling_params]
)
# Collect logprobs outputs from spec decode LLM.
spec_logprobs = []
spec_logprobs: list[Logprob] = []
for results in spec_results:
for output in results.outputs:
assert output.logprobs is not None
for logprobs in output.logprobs:
assert logprobs is not None
spec_logprobs.extend(logprobs.values())
del spec_llm
torch.accelerator.empty_cache()
@@ -1326,6 +1341,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
"Unexpected number of prompt logprob positions"
)
assert sampling_params.prompt_logprobs is not None
# Each position should have the requested number of logprobs
for pos, logprobs_dict in enumerate(prompt_logprobs):
if logprobs_dict is not None: # First token may be None
+1
View File
@@ -333,6 +333,7 @@ def test_sampler_repetition_penalty(
for batch_idx in range(batch_size):
non_penalized_token_id = logits[batch_idx].argmax().item()
penalized_token_id = logits[batch_idx].argmin().item()
assert sampling_metadata.prompt_token_ids is not None
prompt_tokens = sampling_metadata.prompt_token_ids[batch_idx][:].tolist()
output_tokens = sampling_metadata.output_token_ids[batch_idx]
if repetition_penalty > 1.0:
@@ -4,6 +4,7 @@
import torch
from tests.v1.sample.utils import create_mock_reasoning_config
from vllm.sampling_params import SamplingParams
from vllm.v1.sample.logits_processor.interface import (
BatchUpdate,
@@ -12,14 +13,9 @@ from vllm.v1.sample.logits_processor.interface import (
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
class _MockReasoningConfig:
reasoning_start_token_ids = [151667]
reasoning_end_token_ids = [151668]
def _make_holder() -> ThinkingBudgetStateHolder:
return ThinkingBudgetStateHolder(
_MockReasoningConfig(),
create_mock_reasoning_config([151667], [151668]),
8,
0,
torch.device("cpu"),
+2 -1
View File
@@ -104,7 +104,8 @@ def test_trace_decode_token_ids_rejects_speculative_decoding():
params = SamplingParams(trace_decode_token_ids=[1])
with pytest.raises(ValueError, match="not supported with speculative decoding"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=object()
_make_model_config(vocab_size=100),
speculative_config=object(), # type: ignore[arg-type] # Only non-None presence is tested.
)
+17 -1
View File
@@ -4,16 +4,28 @@
from collections.abc import Iterator
from enum import Enum
from typing import NamedTuple
from unittest.mock import MagicMock
import regex as re
import torch
from vllm import CompletionOutput
from vllm.config import ReasoningConfig
from vllm.utils.torch_utils import make_tensor_with_pad
from vllm.v1.sample.logits_processor import BatchUpdate, LogitsProcessor
from vllm.v1.sample.metadata import SamplingMetadata
def create_mock_reasoning_config(
start_token_ids: list[int], end_token_ids: list[int]
) -> ReasoningConfig:
return MagicMock(
spec=ReasoningConfig,
reasoning_start_token_ids=start_token_ids,
reasoning_end_token_ids=end_token_ids,
)
class BatchLogprobsComposition(Enum):
"""Types of logprobs configs to include in test batch."""
@@ -141,7 +153,11 @@ def compute_correct_cumulative_logprob(completion_output: CompletionOutput) -> f
token_ids = completion_output.token_ids
logprobs = completion_output.logprobs
assert logprobs is not None
return sum([lp[tok_id].logprob for tok_id, lp in zip(token_ids, logprobs)])
values: list[float] = []
for tok_id, lp in zip(token_ids, logprobs):
assert lp is not None
values.append(lp[tok_id].logprob)
return sum(values)
def create_fake_logits(batch_size: int, vocab_size: int) -> torch.Tensor:
@@ -25,9 +25,9 @@ def make_manager(
num_reqs, num_steps = confidences.shape
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
manager.num_speculative_steps = num_steps
manager._stale_confidences = [SimpleNamespace(np=confidences)]
manager._stale_confidences = [SimpleNamespace(np=confidences)] # type: ignore[list-item] # CPU-only confidence buffer.
manager._stale_idx = 0
manager.req_states = SimpleNamespace(
manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture.
req_id_to_index={"low": 0, "high": 1},
num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32),
prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)),
@@ -153,11 +153,11 @@ def test_manager_checks_target_varlen_cudagraph_bound(
manager = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=True,
attn_groups=groups,
req_states=SimpleNamespace(num_speculative_steps=7),
req_states=SimpleNamespace(num_speculative_steps=7), # type: ignore[arg-type]
query_start_loc=object(),
num_bonus_tokens=1,
max_total_logits=1,
vllm_config=None,
vllm_config=None, # type: ignore[arg-type]
target_layer_names={"target"},
additional_attn_cg_support=additional,
)
@@ -176,6 +176,7 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
{"low": 3, "high": 3},
{"low": [1, 2], "high": [3, 4]},
)
assert manager._batch_budget is not None
valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget
assert draft_budget == 1
@@ -183,14 +184,18 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
assert num_non_draft_tokens == {"low": 1, "high": 1}
def test_profiled_batches_seed_cost_curves_via_consumer():
def test_profiled_batches_seed_cost_curves_via_consumer(monkeypatch):
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64)
manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture.
max_num_batched_tokens=4096, max_num_reqs=64
)
manager.num_speculative_steps = 7
manager.num_bonus_tokens = 1
curves: dict[str, list[tuple[int, float]]] = {}
manager.set_cost_curves = lambda draft, verify: curves.update(
draft=draft, verify=verify
monkeypatch.setattr(
manager,
"set_cost_curves",
lambda draft, verify: curves.update(draft=draft, verify=verify),
)
timings = [
@@ -258,6 +263,7 @@ def test_budget_caps_at_one_rejection_sampler_chunk():
{"low": 3, "high": 3},
{"low": [1, 2], "high": [3, 4]},
)
assert manager._batch_budget is not None
_, _, draft_budget = manager._batch_budget
assert draft_budget <= 1
@@ -285,6 +291,7 @@ def test_zero_budget_rebuilds_cpu_cu_num_logits():
{"low": 3, "high": 3, "prefill": 40},
{"low": [1, 2], "high": [3, 4]},
)
assert manager._batch_budget is not None
_, _, draft_budget = manager._batch_budget
assert draft_budget == 0
@@ -323,6 +330,7 @@ def test_zero_budget_keeps_one_grammar_row_per_scheduled_draft():
manager.get_num_tokens(
{"low": 3, "high": 3, "prefill": 40}, scheduled_spec_decode_tokens
)
assert manager._batch_budget is not None
assert manager._batch_budget[2] == 0
req_ids = ["low", "high", "prefill"]
+10 -8
View File
@@ -98,7 +98,7 @@ def test_draft_quant_exclusions_include_global_layer_indices():
]
)
_add_global_draft_layer_exclusions(quant_config, 88, 5)
_add_global_draft_layer_exclusions(quant_config, 88, 5) # type: ignore[arg-type]
assert "layers.88.mlp_conv*" in quant_config.exclude_modules
assert "*layers.92.self_attn.q_proj" in quant_config.exclude_modules
@@ -171,7 +171,7 @@ def test_selector_leaves_greedy_drafting_without_proposal_logits(monkeypatch):
allocating one here would claim a proposal the walk never sampled from.
"""
_stub_base(monkeypatch, None)
speculator = DFlash2Speculator(None, torch.device("cpu"))
speculator = DFlash2Speculator(None, torch.device("cpu")) # type: ignore[arg-type] # Base initialization is stubbed.
assert speculator.draft_logits is None
@@ -183,7 +183,7 @@ def test_selector_asks_for_fp32_proposal_logits():
candidate row often enough that the walk and the rejection sampler checking it
would no longer read the same distribution.
"""
dtype, fill = DFlash2Speculator.draft_logits_spec(None, None)
dtype, fill = DFlash2Speculator.draft_logits_spec(None, None) # type: ignore[arg-type] # Constant spec ignores self/config.
assert dtype is torch.float32
assert fill == float("-inf")
@@ -309,9 +309,9 @@ def test_candidate_model_decoder_layer_cls(monkeypatch, variant):
vllm_config.compilation_config = mock_current_vllm_config.compilation_config
# 3. Instantiate the model under meta device to avoid parameter allocation issues
with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"):
with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): # type: ignore[arg-type]
model_cls = DFlash2Qwen3Model if variant == "dflash2" else LiLiCorr
model = model_cls(vllm_config=vllm_config)
model = model_cls(vllm_config=vllm_config) # type: ignore[arg-type]
# 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass)
assert len(model.layers) == 2
@@ -350,7 +350,7 @@ def test_conv_projections_use_draft_quant_config(monkeypatch):
quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True
)
layer = DFlash2Qwen3DecoderLayer(
SimpleNamespace(
SimpleNamespace( # type: ignore[arg-type]
speculative_config=SimpleNamespace(num_speculative_tokens=7),
model_config=SimpleNamespace(dtype=torch.bfloat16),
),
@@ -434,13 +434,15 @@ def test_context_kv_uses_quantized_projection_fallback(monkeypatch):
for projection in projections
]
qwen3_dflash.DFlashQwen3Model._build_context_kv_buffers(
model, layers_attn, has_bias=False
model, # type: ignore[arg-type]
layers_attn,
has_bias=False,
)
assert model._fused_kv_weight is None
assert all(not hasattr(projection, "weight") for projection in projections)
actual_k, actual_v = qwen3_dflash.DFlashQwen3Model._project_context_kv(
model,
model, # type: ignore[arg-type]
context_states,
num_ctx=2,
num_layers=2,
@@ -141,9 +141,9 @@ def test_dflash_drafter_window_reserves_bonus_token():
speculative_config=_dflash_speculative_config(NUM_SPECULATIVE_TOKENS),
)
# window = 4, so 96 fits (96 + 4 == 100) but 97 does not (97 + 4 == 101)
assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96))
assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97))
assert not input_fits_in_drafter(dflash_runner, None) # no metadata
assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96)) # type: ignore[arg-type]
assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type]
assert not input_fits_in_drafter(dflash_runner, None) # type: ignore[arg-type] # No metadata.
# Other drafters don't reserve the bonus token, so 97 fits (97 + 3 == 100).
plain_runner = SimpleNamespace(
@@ -151,4 +151,4 @@ def test_dflash_drafter_window_reserves_bonus_token():
effective_drafter_max_model_len=100,
speculative_config=SimpleNamespace(use_dflash=lambda: False),
)
assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97))
assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type]
@@ -56,6 +56,7 @@ def _make_predictor(vocab_size: int) -> DeepSeekV4MultiTokenPredictor:
):
shared_head = SharedHead(config=config, prefix="shared_head")
assert shared_head.head is not None
generator = torch.Generator().manual_seed(7)
with torch.no_grad():
shared_head.head.weight.normal_(0.0, 0.02, generator=generator)
@@ -157,7 +158,9 @@ def test_wrapper_forwards_spec_step_idx():
hidden_states = torch.empty(0)
wrapper = object.__new__(DeepSeekV4MTP)
wrapper.model = SimpleNamespace(get_top_tokens=lambda *args: args)
wrapper.model = SimpleNamespace( # type: ignore[assignment]
get_top_tokens=lambda *args: args
)
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states, 2) == (hidden_states, 2)
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states) == (hidden_states, 0)
+4 -4
View File
@@ -25,7 +25,7 @@ def _create_vllm_config_for_dsd(
max_num_seqs: int,
max_spec_tokens: int,
*,
cudagraph_mode: str = "FULL_AND_PIECEWISE",
cudagraph_mode: CUDAGraphMode = CUDAGraphMode.FULL_AND_PIECEWISE,
use_dynamic_sd: bool = True,
num_spec_per_batch_size: list[tuple[int, int, int]] | None = None,
) -> MagicMock:
@@ -199,7 +199,7 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=512,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
use_dynamic_sd=True,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
@@ -316,7 +316,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=max_num_seqs,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
use_dynamic_sd=False,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
@@ -382,7 +382,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=max_num_seqs,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
use_dynamic_sd=True,
num_spec_per_batch_size=num_spec_per_batch_size,
)
+17 -11
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Literal
from unittest import mock
import numpy as np
@@ -31,6 +32,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.spec_decode.dflash import DFlashProposer
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -46,13 +48,13 @@ DEVICE_TYPE = current_platform.device_type
def _create_proposer(
method: str,
method: Literal["eagle", "eagle3", "draft_model", "dflash"],
num_speculative_tokens: int,
attention_backend: str | None = None,
parallel_drafting: bool = False,
rejection_sample_method: str = "standard",
draft_sample_method: str = "greedy",
) -> EagleProposer:
rejection_sample_method: Literal["standard", "synthetic", "block"] = "standard",
draft_sample_method: Literal["greedy", "probabilistic"] = "greedy",
) -> SpecDecodeBaseProposer:
# Method-dependent setup
if method == "eagle":
target_model_dir = model_dir
@@ -102,9 +104,14 @@ def _create_proposer(
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
),
attention_config=AttentionConfig(backend=attention_backend),
attention_config=AttentionConfig(
backend=AttentionBackendEnum[attention_backend]
if attention_backend
else None
),
)
proposer: SpecDecodeBaseProposer
if method == "dflash":
proposer = DFlashProposer(vllm_config=vllm_config, device=device)
elif "eagle" in method:
@@ -135,7 +142,7 @@ def test_prepare_next_token_ids():
)
mock_num_scheduled_tokens = {req_id: 0 for req_id in req_ids}
mock_requests = {}
mock_requests: dict[str, CachedRequestState] = {}
for req_id in req_ids:
mock_request = mock.MagicMock(spec=CachedRequestState)
# Each request will have a backup next token id of 10, 20, 30, 40
@@ -948,13 +955,12 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
attn_metadata_builder = attn_metadata_builder_cls(
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
layer_names=proposer._draft_attn_layer_names,
layer_names=sorted(proposer._draft_attn_layer_names),
vllm_config=proposer.vllm_config,
device=device,
)
# Mock runner and draft_attn_groups for attention metadata building
proposer.runner = mock.MagicMock()
# Mock draft_attn_groups for attention metadata building.
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
@@ -1054,11 +1060,10 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch):
)
attn_metadata_builder = attn_metadata_builder_cls(
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
layer_names=proposer._draft_attn_layer_names,
layer_names=sorted(proposer._draft_attn_layer_names),
vllm_config=proposer.vllm_config,
device=device,
)
proposer.runner = mock.MagicMock()
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
@@ -1133,6 +1138,7 @@ def test_set_inputs_first_pass_dflash():
num_speculative_tokens = 3
proposer = _create_proposer("dflash", num_speculative_tokens)
assert isinstance(proposer, DFlashProposer)
mask_token_id = proposer.parallel_drafting_token_id
# Setup batch with 3 requests
@@ -93,7 +93,7 @@ def _run_build(
with patch.object(base_speculator, "build_attn_metadata", fake_build_attn_metadata):
EagleSpeculator._build_uniform_attn_metadata(
fake, # type: ignore[arg-type]
fake,
batch_desc=batch_desc,
num_reqs=num_reqs,
num_query_per_req=num_query_per_req,
@@ -51,7 +51,7 @@ def _create_proposer(
target_parallel_config=ParallelConfig(),
method="extract_hidden_states",
num_speculative_tokens=num_speculative_tokens,
draft_model_config={
draft_model_config={ # type: ignore[arg-type]
"hf_config": {
"eagle_aux_hidden_state_layer_ids": layer_ids,
}
@@ -103,7 +103,7 @@ def test_proposer_initialization_missing_layer_ids():
target_parallel_config=ParallelConfig(),
method="extract_hidden_states",
num_speculative_tokens=1,
draft_model_config={
draft_model_config={ # type: ignore[arg-type]
"hf_config": {} # Missing eagle_aux_hidden_state_layer_ids
},
)
@@ -146,7 +146,7 @@ def test_prepare_next_token_ids_padded():
mock_input_batch.vocab_size = 100
mock_input_batch.num_tokens_no_spec = np.array([5] * num_requests)
mock_requests = {}
mock_requests: dict[str, CachedRequestState] = {}
for req_id in req_ids:
mock_request = mock.MagicMock(spec=CachedRequestState)
# Each request will have a backup next token id of 10, 20, 30, 40
@@ -363,7 +363,7 @@ def test_extract_hidden_states_text_only_config_regression():
target_parallel_config=ParallelConfig(),
method="extract_hidden_states",
num_speculative_tokens=1,
draft_model_config={
draft_model_config={ # type: ignore[arg-type]
"hf_config": {
"eagle_aux_hidden_state_layer_ids": [1, 2, 3, 4],
}
@@ -450,7 +450,7 @@ def test_extract_hidden_states_speculative_config_vlm():
target_parallel_config=ParallelConfig(),
method="extract_hidden_states",
num_speculative_tokens=1,
draft_model_config={
draft_model_config={ # type: ignore[arg-type]
"hf_config": {
"eagle_aux_hidden_state_layer_ids": [1, 2],
}
+38 -20
View File
@@ -11,6 +11,7 @@ from torch import nn
from vllm.config import (
CompilationConfig,
CompilationMode,
DeviceConfig,
LoadConfig,
VllmConfig,
@@ -122,7 +123,7 @@ def test_lilicorr_matches_exported_head(head_width, slots, dtype):
set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cuda"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
),
):
@@ -183,23 +184,23 @@ def test_runtime_length_keeps_trained_checkpoint_geometry(monkeypatch):
with set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cpu"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
):
for slots in (1, 3):
spec.num_speculative_tokens = slots
head = LiLiCorr(
vllm_config=SimpleNamespace(speculative_config=spec)
vllm_config=SimpleNamespace(speculative_config=spec) # type: ignore[arg-type]
).lilicorr
assert head.slot_embedding.shape == (1, 1, 3, 1, 8)
assert head.relative_slot_bias.shape == (2, 7)
spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = False
with pytest.raises(ValueError, match="lilicorr_enabled"):
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec))
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type]
spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = True
spec.num_speculative_tokens = 4
with pytest.raises(ValueError, match="trained block_size"):
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec))
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type]
@pytest.mark.parametrize("tp_size", [1, 2])
@@ -268,7 +269,11 @@ def test_candidates_use_global_partition_and_exclude_padding(monkeypatch, tp_siz
)
monkeypatch.setattr(logits_processor, "_topk", lambda x, k: x.topk(k, dim=-1))
ids, values = logits_processor.LogitsProcessor.get_top_k_tokens(
fake, lm_head, torch.empty(2, 1), 2, return_log_probs=True
fake, # type: ignore[arg-type]
lm_head, # type: ignore[arg-type]
torch.empty(2, 1),
2,
return_log_probs=True,
)
expected = torch.cat(valid_shards, -1).log_softmax(-1).topk(2)
torch.testing.assert_close(ids, expected.indices)
@@ -286,13 +291,13 @@ def test_context_anchor_is_last_committed_normalized_feature():
anchor_valid=torch.ones(4, dtype=torch.bool),
)
batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) # type: ignore[arg-type]
torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]]))
assert spec.anchor_valid.tolist() == [True, True, False, False]
assert not spec.anchor_hidden[2:].any()
# A smaller reordered batch must not retain an old request's anchor.
batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) # type: ignore[arg-type]
torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1]))
assert not spec.anchor_hidden[1:].any()
@@ -316,7 +321,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
monkeypatch.setattr(DFlashSpeculator, "__init__", init_base)
cls = LiLiCorrSpeculator if lilicorr else DFlash2Speculator
spec = cls(None, torch.device("cpu"))
spec = cls(None, torch.device("cpu")) # type: ignore[arg-type]
hidden = torch.arange(36).float().view(9, 4)
spec._run_model = lambda *args: hidden
spec.sample_indices = torch.tensor([1, 2, 4, 5, 7, 8])
@@ -329,7 +334,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
spec.draft_logits = None
spec.use_fp64_gumbel = False
spec.enable_adaptive_verification = True
spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9))
spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) # type: ignore[assignment]
candidates = torch.arange(24).view(6, 4)
log_probs = candidates.float().log_softmax(-1)
scores = torch.randn(3, 2, 4, 4)
@@ -349,22 +354,35 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
compute_candidates=lambda h: (candidates, log_probs),
model=SimpleNamespace(lilicorr=score, candidate_selector=score),
)
spec.target_embeddings = lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4)
if lilicorr:
assert isinstance(spec, LiLiCorrSpeculator)
monkeypatch.setattr(
spec,
"target_embeddings",
lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4),
raising=False,
)
spec.anchor_hidden.copy_(torch.arange(12).view(3, 4))
spec.anchor_valid[:2] = True
spec.candidate_sampler.sample = sample
spec._maybe_predict_acceptance = lambda *args: adaptive_inputs.extend(args)
monkeypatch.setattr(spec.candidate_sampler, "sample", sample)
monkeypatch.setattr(
spec, "_maybe_predict_acceptance", lambda *args: adaptive_inputs.extend(args)
)
spec._generate_draft(3, 9, None, None, None)
expected_ids = candidates.view(3, 2, 4)
expected_first = spec.target_embeddings(expected_ids) if lilicorr else expected_ids
expected_first = (
spec.target_embeddings(expected_ids)
if isinstance(spec, LiLiCorrSpeculator)
else expected_ids
)
torch.testing.assert_close(scoring_inputs[0], expected_first)
torch.testing.assert_close(scoring_inputs[1], log_probs.view(3, 2, 4))
torch.testing.assert_close(
scoring_inputs[2], hidden[spec.sample_indices].view(3, 2, 4)
)
if lilicorr:
assert isinstance(spec, LiLiCorrSpeculator)
torch.testing.assert_close(scoring_inputs[3], spec.anchor_hidden)
torch.testing.assert_close(scoring_inputs[4], spec.anchor_valid)
else:
@@ -472,7 +490,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads(
):
from vllm.model_executor.models.lilicorr import LiLiCorr, LiLiCorrForCausalLM
wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM)
wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM) # type: ignore[type-abstract]
nn.Module.__init__(wrapper)
wrapper.model = LiLiCorr.__new__(LiLiCorr)
nn.Module.__init__(wrapper.model)
@@ -480,7 +498,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads(
with set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cpu"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
):
wrapper.model.lilicorr = LiLiCorrHead(
@@ -615,10 +633,10 @@ def owned_head_model(monkeypatch):
with set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cpu"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
):
model = LiLiCorrForCausalLM(vllm_config=vllm_config)
model = LiLiCorrForCausalLM(vllm_config=vllm_config) # type: ignore[arg-type]
return model, vllm_config
return build
@@ -720,7 +738,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch
with set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cpu"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
):
head = LiLiCorrHead(
@@ -728,7 +746,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch
block_size=4,
rms_norm_eps=1e-6,
config=_config(hidden_size=8),
quant_config=quant_config,
quant_config=quant_config, # type: ignore[arg-type]
prefix="model.lilicorr",
)
with torch.no_grad():
@@ -16,6 +16,7 @@ from types import SimpleNamespace
import pytest
import vllm.v1.spec_decode.llm_base_proposer as llm_base_proposer
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.spec_decode.eagle import EagleProposer
SCHEDULER_BLOCK_SIZE = 256
@@ -52,7 +53,7 @@ def _make_proposer(
monkeypatch.setattr(llm_base_proposer, "AttentionGroup", _FakeAttentionGroup)
proposer = EagleProposer.__new__(EagleProposer)
proposer.vllm_config = None
proposer.vllm_config = None # type: ignore[assignment] # Patched backend discovery does not read config.
proposer.device = None
proposer._draft_attn_layer_names = set(layer_names)
proposer.kv_cache_gid = -1
@@ -61,10 +62,10 @@ def _make_proposer(
return proposer
def _make_kv_cache_config(layer_names: set[str]) -> SimpleNamespace:
def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig:
spec = SimpleNamespace(block_size=SCHEDULER_BLOCK_SIZE)
group = SimpleNamespace(layer_names=list(layer_names), kv_cache_spec=spec)
return SimpleNamespace(kv_cache_groups=[group])
return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value]
def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
@@ -81,7 +82,9 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
assert proposer.block_size == KERNEL_BLOCK_SIZE
assert proposer.block_size != SCHEDULER_BLOCK_SIZE
# The metadata builder keeps receiving the kernel block size as well.
assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE
group = proposer.draft_attn_groups[0]
assert isinstance(group, _FakeAttentionGroup)
assert group.kernel_block_size == KERNEL_BLOCK_SIZE
def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch):
@@ -85,7 +85,7 @@ def test_compute_probs_and_sample_next_token_uses_fp64_exponential_race():
def test_mtp_model_returns_tuple(architecture: str, expected: bool):
proposer = object.__new__(SpecDecodeBaseProposer)
proposer.method = "mtp"
proposer.draft_model_config = SimpleNamespace(
proposer.draft_model_config = SimpleNamespace( # type: ignore[assignment] # Architecture-only config.
hf_config=SimpleNamespace(architectures=[architecture])
)
-1
View File
@@ -220,7 +220,6 @@ def test_mtp_propose(num_speculative_tokens, monkeypatch):
device=device,
)
proposer.runner = mock.MagicMock()
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backend and engine-reasoner checks that are not manager-flow tests."""
from typing import Literal
import pytest
from transformers import AutoTokenizer
@@ -17,7 +19,9 @@ TOKENIZER = "gpt2"
NUM_SPEC_TOKENS = 4
def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
def _make_manager_and_request(
backend: Literal["xgrammar", "guidance"], prompt_str: str = '{"a": "b"}'
):
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
prompt = tokenizer.encode(prompt_str)
@@ -33,6 +37,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
sampling_params = SamplingParams(
structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),
)
assert sampling_params.structured_outputs is not None
sampling_params.structured_outputs._backend = backend
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
@@ -43,6 +48,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
pooling_params=None,
)
manager.grammar_init(request)
assert request.structured_output_request is not None
while not request.structured_output_request._check_grammar_completion():
continue
@@ -168,7 +168,7 @@ def test_watermarked_bonus_uses_target_key():
watermarker=GumbelWatermarker(key=42, context_width=2),
)
expected = GumbelWatermarker(key=42, context_width=2).sample(
target_logits.unsqueeze(0), contexts[1:], lambda _: None
target_logits.unsqueeze(0), contexts[1:]
)
assert num_sampled.item() == 2
@@ -196,7 +196,7 @@ def test_watermarked_recovery_uses_target_key():
watermarker=GumbelWatermarker(key=42, context_width=2),
)
expected = GumbelWatermarker(key=42, context_width=2).sample(
target_logits.unsqueeze(0), contexts[:1], lambda _: None
target_logits.unsqueeze(0), contexts[:1]
)
assert num_sampled.item() == 1
@@ -225,9 +225,7 @@ def test_watermarked_recovery_supports_smaller_draft_vocabulary():
watermarking=torch.tensor([True], device="cuda"),
watermarker=watermarker,
)
expected = watermarker.sample(
target_logits[:4].unsqueeze(0), contexts[:1], lambda _: None
)
expected = watermarker.sample(target_logits[:4].unsqueeze(0), contexts[:1])
assert num_sampled.item() == 1
assert sampled[0, 0] == expected.token_ids[0]
@@ -702,7 +700,7 @@ def test_verify_rejects_unproposed_drafts():
first_pos = inputs["pos"].view(num_trials, K + 1)[:, 0]
prefill_len = first_pos + 1
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = SimpleNamespace(
rejection_sampler.sampler = SimpleNamespace( # type: ignore[assignment]
apply_sampling_params=lambda logits, *args: logits,
sampling_states=SimpleNamespace(
temperature=SimpleNamespace(gpu=inputs["temperature"]),