[MyPy][3/N] Fix MyPy errors in test groups (part 3) (#55939)
Signed-off-by: Taimys <ffy2281693949@gmail.com> Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Co-authored-by: Taimys <ffy2281693949@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Luyi Xiao
Taimys
Harry Mellor
Claude Opus 5.5
parent
c4fa6f36a7
commit
5225f1bacb
@@ -3,7 +3,7 @@
|
||||
|
||||
import random
|
||||
from collections.abc import Callable
|
||||
from typing import NamedTuple, TypeAlias
|
||||
from typing import NamedTuple, NoReturn, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -13,6 +13,7 @@ from tests.utils import create_new_process_for_each_test
|
||||
from tests.v1.sample.utils import (
|
||||
LogitsprocsTestFakes,
|
||||
create_fake_logits,
|
||||
create_mock_reasoning_config,
|
||||
create_penalty_tensor,
|
||||
create_prompt_tokens_tensor,
|
||||
fake_apply_logitsprocs,
|
||||
@@ -103,14 +104,6 @@ class LogitsProcsRequestParams:
|
||||
return f"MyClass({summ})"
|
||||
|
||||
|
||||
class MockReasoningConfig:
|
||||
"""Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests."""
|
||||
|
||||
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
|
||||
reasoning_end_token_ids = [THINK_END_TOKEN_ID]
|
||||
enabled = True
|
||||
|
||||
|
||||
def _generate_fake_sampling_metadata(
|
||||
num_output_tokens: int,
|
||||
batch_size: int,
|
||||
@@ -131,7 +124,9 @@ def _generate_fake_sampling_metadata(
|
||||
)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.reasoning_config = MockReasoningConfig()
|
||||
vllm_config.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
|
||||
logitsprocs = build_logitsprocs(
|
||||
vllm_config=vllm_config,
|
||||
@@ -242,7 +237,7 @@ def _raise_error_invalid(
|
||||
request_params: LogitsProcsRequestParams,
|
||||
step_idx: int,
|
||||
err_cls: type[Exception] = ValueError,
|
||||
) -> None:
|
||||
) -> NoReturn:
|
||||
raise err_cls(
|
||||
f"Validation failed for step={step_idx}, "
|
||||
f"batch_index={batch_index}, "
|
||||
@@ -268,6 +263,7 @@ def _logit_bias_validate(
|
||||
) -> None:
|
||||
"""Validate logit bias logitproc applied correctly."""
|
||||
logit_bias = request_params.params.logit_bias
|
||||
assert logit_bias is not None
|
||||
logits_old = test_fakes.logits[persistent_batch[batch_index].workload_index].cpu()
|
||||
logits_new = logits_new[batch_index].cpu()
|
||||
for token_id in range(VOCAB_SIZE):
|
||||
@@ -368,9 +364,7 @@ def _min_tokens_validate(
|
||||
ref_num_out_tokens = len(request_params.out_tokens)
|
||||
min_reached = ref_num_out_tokens >= MIN_TOKENS_LEN_THRESHOLD
|
||||
ref_all_stop_token_ids = request_params.params.all_stop_token_ids
|
||||
mt_lp: MinTokensLogitsProcessor = next(
|
||||
test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor)
|
||||
)
|
||||
mt_lp = next(test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor))
|
||||
assert isinstance(mt_lp, MinTokensLogitsProcessor)
|
||||
min_tok = mt_lp.min_toks.get(batch_index, None)
|
||||
|
||||
@@ -689,7 +683,7 @@ class LogitsprocTestHelpers(NamedTuple):
|
||||
gen_request_fxn: Callable | None = None
|
||||
|
||||
|
||||
logitsprocs_test_mapping = {
|
||||
logitsprocs_test_mapping: dict[LogitprocType, LogitsprocTestHelpers] = {
|
||||
STR_NO_LOGITPROC: LogitsprocTestHelpers(eval_fxn=_none_validate),
|
||||
LogitBiasLogitsProcessor: LogitsprocTestHelpers(
|
||||
gen_request_fxn=_logit_bias_params, eval_fxn=_logit_bias_validate
|
||||
@@ -706,22 +700,22 @@ logitsprocs_test_mapping = {
|
||||
}
|
||||
|
||||
|
||||
def _get_test_cases() -> list[list[str]]:
|
||||
def _get_test_cases() -> list[list[LogitprocType]]:
|
||||
"""Each test case is a set of logitsprocs."""
|
||||
logitsprocs_types = list(logitsprocs_test_mapping.keys())
|
||||
logitsprocs_types: list[LogitprocType] = list(logitsprocs_test_mapping.keys())
|
||||
|
||||
# Isolate thinking-budget handling from other processors to avoid cross-talk.
|
||||
thinking_id: LogitprocType = STR_THINKING_BUDGET
|
||||
other_processors = [
|
||||
other_processors: list[LogitprocType] = [
|
||||
p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id
|
||||
]
|
||||
|
||||
return (
|
||||
[[STR_NO_LOGITPROC]]
|
||||
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
|
||||
+ [other_processors]
|
||||
+ [[thinking_id]]
|
||||
test_cases: list[list[LogitprocType]] = [[STR_NO_LOGITPROC]]
|
||||
test_cases.extend(
|
||||
[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors
|
||||
)
|
||||
test_cases.extend([other_processors, [thinking_id]])
|
||||
return test_cases
|
||||
|
||||
|
||||
def _generate_fake_step_update(
|
||||
@@ -774,6 +768,7 @@ def _generate_fake_step_update(
|
||||
for add_req_params in workload_params[wdx : (wdx + num_step_add_replace)]:
|
||||
# Replace as many removed requests as possible with added requests
|
||||
add_remove_idx = batch_update_builder.pop_removed()
|
||||
assert add_remove_idx is not None
|
||||
batch_update_builder.added.append(
|
||||
(
|
||||
add_remove_idx,
|
||||
@@ -976,13 +971,6 @@ def test_logitsprocs(
|
||||
step_idx += 1
|
||||
|
||||
|
||||
class MockReasoningNoEndTokens:
|
||||
"""Reasoning config with no end token ids (disables enforcement in holder)."""
|
||||
|
||||
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
|
||||
reasoning_end_token_ids: list[int] = []
|
||||
|
||||
|
||||
def test_maybe_create_thinking_budget_holder_without_reasoning():
|
||||
cfg = VllmConfig()
|
||||
assert cfg.reasoning_config is None
|
||||
@@ -1000,7 +988,9 @@ def test_maybe_create_thinking_budget_holder_without_reasoning():
|
||||
|
||||
def test_thinking_budget_holder_has_tracked_after_sync_add():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1030,7 +1020,9 @@ def test_thinking_budget_holder_has_tracked_after_sync_add():
|
||||
|
||||
def test_thinking_budget_holder_sync_remove_clears_state():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1060,7 +1052,9 @@ def test_thinking_budget_holder_sync_remove_clears_state():
|
||||
|
||||
def test_thinking_budget_holder_sync_add_without_budget_drops_row():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1081,7 +1075,9 @@ def test_thinking_budget_holder_sync_add_without_budget_drops_row():
|
||||
|
||||
def test_thinking_budget_holder_swap_exchanges_state():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1125,7 +1121,9 @@ def test_thinking_budget_holder_swap_exchanges_state():
|
||||
|
||||
def test_thinking_budget_holder_unidirectional_move():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1163,7 +1161,9 @@ def test_thinking_budget_holder_unidirectional_move():
|
||||
|
||||
def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1197,7 +1197,7 @@ def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
|
||||
|
||||
def test_thinking_budget_holder_spec_mode_tensor_layout():
|
||||
h = ThinkingBudgetStateHolder(
|
||||
MockReasoningConfig(),
|
||||
create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]),
|
||||
8,
|
||||
2,
|
||||
torch.device("cpu"),
|
||||
@@ -1209,7 +1209,7 @@ def test_thinking_budget_holder_spec_mode_tensor_layout():
|
||||
|
||||
def test_thinking_budget_holder_empty_end_tokens_disables_row():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningNoEndTokens()
|
||||
vc.reasoning_config = create_mock_reasoning_config([THINK_START_TOKEN_ID], [])
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1253,7 +1253,9 @@ def test_thinking_budget_enforced_without_penalties():
|
||||
passing an empty list (the pre-fix behavior) prevents budget enforcement.
|
||||
"""
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
budget = 3 # allow 3 thinking tokens
|
||||
|
||||
h = ThinkingBudgetStateHolder(
|
||||
@@ -1346,7 +1348,11 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
|
||||
"""Test thinking budget enforced for a long thinking run,
|
||||
then a natural end marker."""
|
||||
h = ThinkingBudgetStateHolder(
|
||||
MockReasoningConfig(), 8, 0, torch.device("cpu"), False
|
||||
create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]),
|
||||
8,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
@@ -1356,8 +1362,8 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
start = MockReasoningConfig.reasoning_start_token_ids
|
||||
end = MockReasoningConfig.reasoning_end_token_ids
|
||||
start = [THINK_START_TOKEN_ID]
|
||||
end = [THINK_END_TOKEN_ID]
|
||||
|
||||
out: list[int] = list(start)
|
||||
h.update_state([out], None, None)
|
||||
@@ -1396,13 +1402,9 @@ class TestThinkingBudgetReentry:
|
||||
|
||||
@staticmethod
|
||||
def _make_holder(end_token_ids: list[int]) -> ThinkingBudgetStateHolder:
|
||||
class FakeReasoningConfig:
|
||||
reasoning_start_token_ids = [TestThinkingBudgetReentry.THINK_START]
|
||||
reasoning_end_token_ids: list[int] = []
|
||||
enabled = True
|
||||
|
||||
cfg = FakeReasoningConfig()
|
||||
cfg.reasoning_end_token_ids = end_token_ids
|
||||
cfg = create_mock_reasoning_config(
|
||||
[TestThinkingBudgetReentry.THINK_START], end_token_ids
|
||||
)
|
||||
return ThinkingBudgetStateHolder(
|
||||
reasoning_config=cfg,
|
||||
max_num_seqs=8,
|
||||
@@ -1526,21 +1528,12 @@ class TestThinkingBudgetNaturalEndReentry:
|
||||
|
||||
@staticmethod
|
||||
def _make_holder(end_token_ids):
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.v1.sample.thinking_budget_state import (
|
||||
ThinkingBudgetStateHolder,
|
||||
)
|
||||
|
||||
@dataclass
|
||||
class FakeReasoningConfig:
|
||||
reasoning_start_token_ids: list[int]
|
||||
reasoning_end_token_ids: list[int]
|
||||
enabled: bool = True
|
||||
|
||||
cfg = FakeReasoningConfig(
|
||||
reasoning_start_token_ids=[TestThinkingBudgetNaturalEndReentry.THINK_START],
|
||||
reasoning_end_token_ids=end_token_ids,
|
||||
cfg = create_mock_reasoning_config(
|
||||
[TestThinkingBudgetNaturalEndReentry.THINK_START], end_token_ids
|
||||
)
|
||||
return ThinkingBudgetStateHolder(
|
||||
reasoning_config=cfg,
|
||||
@@ -1830,21 +1823,11 @@ class TestThinkingBudgetNaturalEndReentry:
|
||||
natural </think> exit, continue_thinking is cleared and Block 2
|
||||
enforcement works correctly.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
|
||||
|
||||
@dataclass
|
||||
class FakeReasoningConfig:
|
||||
reasoning_start_token_ids: list[int]
|
||||
reasoning_end_token_ids: list[int]
|
||||
enabled: bool = True
|
||||
|
||||
cfg = FakeReasoningConfig(
|
||||
reasoning_start_token_ids=[self.THINK_START],
|
||||
reasoning_end_token_ids=self.THINK_END_SINGLE,
|
||||
)
|
||||
cfg = create_mock_reasoning_config([self.THINK_START], self.THINK_END_SINGLE)
|
||||
holder = ThinkingBudgetStateHolder(
|
||||
reasoning_config=cfg,
|
||||
max_num_seqs=8,
|
||||
|
||||
@@ -57,15 +57,20 @@ prompts = [
|
||||
class DummyLogitsProcessor(LogitsProcessor):
|
||||
"""Fake logit processor to support unit testing and examples."""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
target_token: int | None = params.extra_args and params.extra_args.get(
|
||||
"target_token"
|
||||
@staticmethod
|
||||
def _get_target_token(params: SamplingParams) -> int | None:
|
||||
target_token = (
|
||||
params.extra_args.get(DUMMY_LOGITPROC_ARG) if params.extra_args else None
|
||||
)
|
||||
if target_token is not None and not isinstance(target_token, int):
|
||||
raise VLLMValidationError(
|
||||
f"target_token value {target_token} {type(target_token)} is not int"
|
||||
)
|
||||
return target_token
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
cls._get_target_token(params)
|
||||
|
||||
def __init__(
|
||||
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
|
||||
@@ -78,8 +83,7 @@ class DummyLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
def extract_extra_arg(params: SamplingParams) -> int | None:
|
||||
self.validate_params(params)
|
||||
return params.extra_args and params.extra_args.get("target_token")
|
||||
return self._get_target_token(params)
|
||||
|
||||
process_dict_updates(
|
||||
self.req_info,
|
||||
|
||||
@@ -341,6 +341,7 @@ def test_batch_allowed_needs_one_shared_set():
|
||||
assert states.batch_allowed([]) is None
|
||||
|
||||
shared = states.batch_allowed([0])
|
||||
assert shared is not None
|
||||
assert shared.tolist() == [1, 2, 3]
|
||||
assert shared.dtype == torch.int64
|
||||
assert states.batch_allowed([0, 0]) is shared
|
||||
|
||||
@@ -34,6 +34,8 @@ def test_stats_match_reference(vocab: int, dtype: torch.dtype):
|
||||
)
|
||||
assert torch.equal(argmax, ref_argmax)
|
||||
torch.testing.assert_close(entropy, ref_entropy, atol=2e-3, rtol=1e-3)
|
||||
assert probs is not None
|
||||
assert ref_probs is not None
|
||||
torch.testing.assert_close(probs.float(), ref_probs.float(), atol=5e-3, rtol=2e-2)
|
||||
assert probs.dtype == torch.bfloat16
|
||||
assert probs.shape == logits.shape
|
||||
@@ -100,6 +102,7 @@ def test_masked_logits_keep_a_finite_entropy():
|
||||
assert entropy[0].item() < 1e-6 and entropy[1].item() < 1e-6
|
||||
assert abs(entropy[2].item() - math.log(2.0)) < 1e-4
|
||||
assert torch.equal(argmax[[0, 1, 3]], torch.tensor([3, 3, 3], device="cuda"))
|
||||
assert probs is not None
|
||||
assert probs[0, 3].item() == 1.0 and probs[0].sum().item() == 1.0
|
||||
_, _, ref_entropy, _ = sample_row_stats_reference(logits, temps, 1, None)
|
||||
assert torch.isfinite(ref_entropy).all()
|
||||
|
||||
@@ -340,7 +340,7 @@ def _make_v2_state(
|
||||
device=device,
|
||||
all_token_ids=SimpleNamespace(gpu=all_tokens),
|
||||
)
|
||||
state = DryState(_make_vllm_config(), req_states)
|
||||
state = DryState(_make_vllm_config(), req_states) # type: ignore[arg-type]
|
||||
if tokenizer is not None:
|
||||
# Re-resolve the default set now that a tokenizer is installed.
|
||||
state._tokenizer = tokenizer
|
||||
@@ -446,7 +446,7 @@ def test_spec_decode_is_refused_at_construction():
|
||||
)
|
||||
config = _make_vllm_config(speculative_config=SimpleNamespace())
|
||||
with pytest.raises(ValueError, match="speculative decoding"):
|
||||
DryState(config, req_states)
|
||||
DryState(config, req_states) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_v2_draft_expanded_logits_raise():
|
||||
@@ -917,7 +917,10 @@ def test_loads_through_the_loader_by_fqcn():
|
||||
total_len=None,
|
||||
)
|
||||
procs = loader.build_custom_logits_processors(
|
||||
_make_vllm_config(), req_states, False, [DRY_FQCN]
|
||||
_make_vllm_config(),
|
||||
req_states, # type: ignore[arg-type]
|
||||
False,
|
||||
[DRY_FQCN],
|
||||
)
|
||||
assert [type(p) for p in procs] == [DryState]
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
|
||||
@@ -32,10 +33,17 @@ class _FakeLmHead:
|
||||
self.tp_size = 1
|
||||
|
||||
|
||||
def _fake_lm_head(
|
||||
weight: torch.Tensor, quantized: bool = False, shard_indices: object | None = None
|
||||
) -> VocabParallelEmbedding:
|
||||
# The dtype tests only need the projection fields, not distributed setup.
|
||||
return _FakeLmHead(weight, quantized, shard_indices) # type: ignore[return-value]
|
||||
|
||||
|
||||
def _build_processor(vocab_size: int) -> LogitsProcessor:
|
||||
lp = LogitsProcessor(vocab_size)
|
||||
# The TP gather is orthogonal to the dtype behavior under test.
|
||||
lp._gather_logits = lambda logits: logits
|
||||
lp._gather_logits = lambda logits: logits # type: ignore[method-assign]
|
||||
return lp
|
||||
|
||||
|
||||
@@ -47,8 +55,9 @@ def test_fp32_head_runs_projection_in_fp32(default_vllm_config):
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16)
|
||||
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
|
||||
|
||||
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
|
||||
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
|
||||
|
||||
assert logits is not None
|
||||
assert logits.dtype == torch.float32
|
||||
assert torch.isfinite(logits).all()
|
||||
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
|
||||
@@ -66,8 +75,9 @@ def test_non_fp32_head_dtype_uses_cast_path(default_vllm_config):
|
||||
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
|
||||
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
|
||||
|
||||
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
|
||||
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
|
||||
|
||||
assert logits is not None
|
||||
assert logits.dtype == torch.float16
|
||||
expected = torch.nn.functional.linear(hidden_states.half(), weight.half())
|
||||
torch.testing.assert_close(logits, expected)
|
||||
@@ -82,7 +92,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config):
|
||||
|
||||
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
|
||||
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
|
||||
lm_head = _FakeLmHead(weight)
|
||||
lm_head = _fake_lm_head(weight)
|
||||
|
||||
with mock.patch.object(
|
||||
lm_head.quant_method,
|
||||
@@ -94,6 +104,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config):
|
||||
logits = lp._get_logits(hidden_states, lm_head, None)
|
||||
|
||||
apply_mock.assert_called_once()
|
||||
assert logits is not None
|
||||
assert logits.dtype == torch.bfloat16
|
||||
|
||||
|
||||
@@ -120,9 +131,10 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config):
|
||||
with mock.patch(
|
||||
"vllm.model_executor.layers.logits_processor.F.linear"
|
||||
) as linear_mock:
|
||||
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
|
||||
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
|
||||
|
||||
linear_mock.assert_not_called()
|
||||
assert logits is not None
|
||||
assert logits.dtype == torch.float32
|
||||
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
|
||||
torch.testing.assert_close(logits, expected)
|
||||
@@ -131,7 +143,7 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config):
|
||||
def test_fp32_head_rejects_quantized_lm_head(default_vllm_config):
|
||||
lp = _build_processor(64)
|
||||
lp.head_dtype = torch.float32
|
||||
lm_head = _FakeLmHead(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True)
|
||||
lm_head = _fake_lm_head(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True)
|
||||
|
||||
with pytest.raises(ValueError, match="unquantized"):
|
||||
lp._get_logits(torch.randn(4, 16, dtype=torch.bfloat16), lm_head, None)
|
||||
@@ -197,7 +209,7 @@ def test_get_top_tokens_honors_head_dtype(default_vllm_config):
|
||||
|
||||
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
|
||||
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
|
||||
lm_head = _FakeLmHead(
|
||||
lm_head = _fake_lm_head(
|
||||
weight,
|
||||
shard_indices=types.SimpleNamespace(
|
||||
num_org_vocab_padding=0, org_vocab_start_index=0
|
||||
@@ -236,7 +248,9 @@ def test_fp32_head_e2e_no_nan():
|
||||
|
||||
for output in outputs:
|
||||
for completion in output.outputs:
|
||||
assert completion.logprobs is not None
|
||||
for token_id, position in zip(completion.token_ids, completion.logprobs):
|
||||
assert position is not None
|
||||
# The sampled token survived filtering, so its logprob is finite.
|
||||
assert math.isfinite(position[token_id].logprob)
|
||||
# No returned logprob is NaN.
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm import SamplingParams
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
@@ -145,7 +146,7 @@ def _repeat_logprob_config(
|
||||
def _run_and_validate(
|
||||
vllm_model: VllmRunner,
|
||||
test_prompts: list[str],
|
||||
vllm_sampling_params: SamplingParams,
|
||||
vllm_sampling_params: SamplingParams | list[SamplingParams],
|
||||
hf_logprobs: list[list[torch.Tensor]],
|
||||
hf_outputs: list[tuple[list[int], str]],
|
||||
logprob_prompt_logprob_list: BatchLogprobsSpecType,
|
||||
@@ -163,11 +164,12 @@ def _run_and_validate(
|
||||
# Extract request-level (prompt)logprobs config
|
||||
num_top_logprobs, num_top_prompt_logprobs = logprob_prompt_logprob
|
||||
|
||||
assert vllm_result.prompt_token_ids is not None
|
||||
# Test whether sampled token output is consistent between vLLM and HF
|
||||
# vLLM prompt+completion should match HF output
|
||||
if temperature == 0.0:
|
||||
assert (
|
||||
vllm_result.prompt_token_ids + vllm_result.outputs[0].token_ids
|
||||
vllm_result.prompt_token_ids + list(vllm_result.outputs[0].token_ids)
|
||||
== hf_output[0]
|
||||
)
|
||||
else:
|
||||
@@ -191,7 +193,9 @@ def _run_and_validate(
|
||||
|
||||
# Confirm that the output token appears among the logprobs
|
||||
assert token_id in logprobs
|
||||
token_in_topk = logprobs[token_id].rank <= num_top_logprobs
|
||||
rank = logprobs[token_id].rank
|
||||
assert rank is not None
|
||||
token_in_topk = rank <= num_top_logprobs
|
||||
|
||||
# If the output token is not included in the top K
|
||||
# logprob, it can return 1 more data
|
||||
@@ -207,8 +211,10 @@ def _run_and_validate(
|
||||
|
||||
output_text = vllm_result.outputs[0].text
|
||||
output_string_from_most_likely_tokens_lst: list[str] = []
|
||||
for top_logprobs in vllm_result.outputs[0].logprobs:
|
||||
top_logprob = next(iter(top_logprobs.values()))
|
||||
for output_logprobs in vllm_result.outputs[0].logprobs:
|
||||
assert output_logprobs is not None
|
||||
top_logprob = next(iter(output_logprobs.values()))
|
||||
assert top_logprob.decoded_token is not None
|
||||
output_string_from_most_likely_tokens_lst.append(
|
||||
top_logprob.decoded_token
|
||||
)
|
||||
@@ -226,8 +232,9 @@ def _run_and_validate(
|
||||
|
||||
# Compare vLLM sample logprobs to HF
|
||||
vllm_sample_logprobs = vllm_result.outputs[0].logprobs
|
||||
for i, top_logprobs in enumerate(vllm_sample_logprobs):
|
||||
for token_id, sample_logprob in top_logprobs.items():
|
||||
for i, sample_logprobs in enumerate(vllm_sample_logprobs):
|
||||
assert sample_logprobs is not None
|
||||
for token_id, sample_logprob in sample_logprobs.items():
|
||||
if temperature == 0.0 or i == 0:
|
||||
logprob = sample_logprob.logprob
|
||||
torch.testing.assert_close(
|
||||
@@ -271,9 +278,9 @@ def _run_and_validate(
|
||||
|
||||
# Confirm that the prompt token appears among the logprobs
|
||||
assert prompt_token_id in prompt_logprobs
|
||||
token_in_topk = (
|
||||
prompt_logprobs[prompt_token_id].rank <= num_top_prompt_logprobs
|
||||
)
|
||||
prompt_rank = prompt_logprobs[prompt_token_id].rank
|
||||
assert prompt_rank is not None
|
||||
token_in_topk = prompt_rank <= num_top_prompt_logprobs
|
||||
|
||||
# If the prompt token is not included in the top K
|
||||
# logprob, it can return 1 more data
|
||||
@@ -294,9 +301,10 @@ def _run_and_validate(
|
||||
# 1:.
|
||||
vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
|
||||
for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
|
||||
for token_id, logprob in vllm_prompt_logprob_dict.items():
|
||||
assert vllm_prompt_logprob_dict is not None
|
||||
for token_id, prompt_lp in vllm_prompt_logprob_dict.items():
|
||||
torch.testing.assert_close(
|
||||
logprob.logprob,
|
||||
prompt_lp.logprob,
|
||||
hf_logprob[0][i][token_id].item(),
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
@@ -478,12 +486,12 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
|
||||
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
with pytest.raises(VLLMValidationError, match="V2 model runner"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
|
||||
processor.vllm_config.use_v2_model_runner = True
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
|
||||
processor.vllm_config.cache_config.kv_sharing_fast_prefill = True
|
||||
with pytest.raises(VLLMValidationError, match="fast-prefill"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_none_logprobs(vllm_model, example_prompts):
|
||||
@@ -607,7 +615,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
|
||||
total_token_with_logprobs = 0
|
||||
positive_values = 0
|
||||
for output in results[0].outputs:
|
||||
assert output.logprobs is not None
|
||||
for logprobs in output.logprobs:
|
||||
assert logprobs is not None
|
||||
for token_id in logprobs:
|
||||
logprob = logprobs[token_id]
|
||||
if logprobs_mode in ("raw_logprobs", "processed_logprobs"):
|
||||
@@ -648,6 +658,7 @@ def test_prompt_logprobs_mode():
|
||||
)
|
||||
assert results[0].prompt_logprobs is not None
|
||||
assert results[0].prompt_logprobs[1] is not None
|
||||
assert results[0].prompt_token_ids is not None
|
||||
tok_id = results[0].prompt_token_ids[1]
|
||||
values[mode] = results[0].prompt_logprobs[1][tok_id].logprob
|
||||
finally:
|
||||
@@ -1228,10 +1239,12 @@ def test_spec_decode_logprobs(
|
||||
[prompt, prompt], [sampling_params, penalty_sampling_params]
|
||||
)
|
||||
# Collect logprobs outputs from reference LLM.
|
||||
ref_logprobs = []
|
||||
ref_logprobs: list[Logprob] = []
|
||||
for results in ref_results:
|
||||
for output in results.outputs:
|
||||
assert output.logprobs is not None
|
||||
for logprobs in output.logprobs:
|
||||
assert logprobs is not None
|
||||
ref_logprobs.extend(logprobs.values())
|
||||
del ref_llm
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1249,10 +1262,12 @@ def test_spec_decode_logprobs(
|
||||
[prompt, prompt], [sampling_params, penalty_sampling_params]
|
||||
)
|
||||
# Collect logprobs outputs from spec decode LLM.
|
||||
spec_logprobs = []
|
||||
spec_logprobs: list[Logprob] = []
|
||||
for results in spec_results:
|
||||
for output in results.outputs:
|
||||
assert output.logprobs is not None
|
||||
for logprobs in output.logprobs:
|
||||
assert logprobs is not None
|
||||
spec_logprobs.extend(logprobs.values())
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1326,6 +1341,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
|
||||
"Unexpected number of prompt logprob positions"
|
||||
)
|
||||
|
||||
assert sampling_params.prompt_logprobs is not None
|
||||
# Each position should have the requested number of logprobs
|
||||
for pos, logprobs_dict in enumerate(prompt_logprobs):
|
||||
if logprobs_dict is not None: # First token may be None
|
||||
|
||||
@@ -333,6 +333,7 @@ def test_sampler_repetition_penalty(
|
||||
for batch_idx in range(batch_size):
|
||||
non_penalized_token_id = logits[batch_idx].argmax().item()
|
||||
penalized_token_id = logits[batch_idx].argmin().item()
|
||||
assert sampling_metadata.prompt_token_ids is not None
|
||||
prompt_tokens = sampling_metadata.prompt_token_ids[batch_idx][:].tolist()
|
||||
output_tokens = sampling_metadata.output_token_ids[batch_idx]
|
||||
if repetition_penalty > 1.0:
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.sample.utils import create_mock_reasoning_config
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
BatchUpdate,
|
||||
@@ -12,14 +13,9 @@ from vllm.v1.sample.logits_processor.interface import (
|
||||
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
|
||||
|
||||
|
||||
class _MockReasoningConfig:
|
||||
reasoning_start_token_ids = [151667]
|
||||
reasoning_end_token_ids = [151668]
|
||||
|
||||
|
||||
def _make_holder() -> ThinkingBudgetStateHolder:
|
||||
return ThinkingBudgetStateHolder(
|
||||
_MockReasoningConfig(),
|
||||
create_mock_reasoning_config([151667], [151668]),
|
||||
8,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
|
||||
@@ -104,7 +104,8 @@ def test_trace_decode_token_ids_rejects_speculative_decoding():
|
||||
params = SamplingParams(trace_decode_token_ids=[1])
|
||||
with pytest.raises(ValueError, match="not supported with speculative decoding"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=object()
|
||||
_make_model_config(vocab_size=100),
|
||||
speculative_config=object(), # type: ignore[arg-type] # Only non-None presence is tested.
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -4,16 +4,28 @@
|
||||
from collections.abc import Iterator
|
||||
from enum import Enum
|
||||
from typing import NamedTuple
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm import CompletionOutput
|
||||
from vllm.config import ReasoningConfig
|
||||
from vllm.utils.torch_utils import make_tensor_with_pad
|
||||
from vllm.v1.sample.logits_processor import BatchUpdate, LogitsProcessor
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
|
||||
|
||||
def create_mock_reasoning_config(
|
||||
start_token_ids: list[int], end_token_ids: list[int]
|
||||
) -> ReasoningConfig:
|
||||
return MagicMock(
|
||||
spec=ReasoningConfig,
|
||||
reasoning_start_token_ids=start_token_ids,
|
||||
reasoning_end_token_ids=end_token_ids,
|
||||
)
|
||||
|
||||
|
||||
class BatchLogprobsComposition(Enum):
|
||||
"""Types of logprobs configs to include in test batch."""
|
||||
|
||||
@@ -141,7 +153,11 @@ def compute_correct_cumulative_logprob(completion_output: CompletionOutput) -> f
|
||||
token_ids = completion_output.token_ids
|
||||
logprobs = completion_output.logprobs
|
||||
assert logprobs is not None
|
||||
return sum([lp[tok_id].logprob for tok_id, lp in zip(token_ids, logprobs)])
|
||||
values: list[float] = []
|
||||
for tok_id, lp in zip(token_ids, logprobs):
|
||||
assert lp is not None
|
||||
values.append(lp[tok_id].logprob)
|
||||
return sum(values)
|
||||
|
||||
|
||||
def create_fake_logits(batch_size: int, vocab_size: int) -> torch.Tensor:
|
||||
|
||||
@@ -25,9 +25,9 @@ def make_manager(
|
||||
num_reqs, num_steps = confidences.shape
|
||||
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
|
||||
manager.num_speculative_steps = num_steps
|
||||
manager._stale_confidences = [SimpleNamespace(np=confidences)]
|
||||
manager._stale_confidences = [SimpleNamespace(np=confidences)] # type: ignore[list-item] # CPU-only confidence buffer.
|
||||
manager._stale_idx = 0
|
||||
manager.req_states = SimpleNamespace(
|
||||
manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture.
|
||||
req_id_to_index={"low": 0, "high": 1},
|
||||
num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32),
|
||||
prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)),
|
||||
@@ -153,11 +153,11 @@ def test_manager_checks_target_varlen_cudagraph_bound(
|
||||
manager = maybe_create_adaptive_verification_manager(
|
||||
enable_adaptive_verification=True,
|
||||
attn_groups=groups,
|
||||
req_states=SimpleNamespace(num_speculative_steps=7),
|
||||
req_states=SimpleNamespace(num_speculative_steps=7), # type: ignore[arg-type]
|
||||
query_start_loc=object(),
|
||||
num_bonus_tokens=1,
|
||||
max_total_logits=1,
|
||||
vllm_config=None,
|
||||
vllm_config=None, # type: ignore[arg-type]
|
||||
target_layer_names={"target"},
|
||||
additional_attn_cg_support=additional,
|
||||
)
|
||||
@@ -176,6 +176,7 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
|
||||
{"low": 3, "high": 3},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
assert manager._batch_budget is not None
|
||||
valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget
|
||||
|
||||
assert draft_budget == 1
|
||||
@@ -183,14 +184,18 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
|
||||
assert num_non_draft_tokens == {"low": 1, "high": 1}
|
||||
|
||||
|
||||
def test_profiled_batches_seed_cost_curves_via_consumer():
|
||||
def test_profiled_batches_seed_cost_curves_via_consumer(monkeypatch):
|
||||
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
|
||||
manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64)
|
||||
manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture.
|
||||
max_num_batched_tokens=4096, max_num_reqs=64
|
||||
)
|
||||
manager.num_speculative_steps = 7
|
||||
manager.num_bonus_tokens = 1
|
||||
curves: dict[str, list[tuple[int, float]]] = {}
|
||||
manager.set_cost_curves = lambda draft, verify: curves.update(
|
||||
draft=draft, verify=verify
|
||||
monkeypatch.setattr(
|
||||
manager,
|
||||
"set_cost_curves",
|
||||
lambda draft, verify: curves.update(draft=draft, verify=verify),
|
||||
)
|
||||
|
||||
timings = [
|
||||
@@ -258,6 +263,7 @@ def test_budget_caps_at_one_rejection_sampler_chunk():
|
||||
{"low": 3, "high": 3},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
assert manager._batch_budget is not None
|
||||
_, _, draft_budget = manager._batch_budget
|
||||
assert draft_budget <= 1
|
||||
|
||||
@@ -285,6 +291,7 @@ def test_zero_budget_rebuilds_cpu_cu_num_logits():
|
||||
{"low": 3, "high": 3, "prefill": 40},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
assert manager._batch_budget is not None
|
||||
_, _, draft_budget = manager._batch_budget
|
||||
assert draft_budget == 0
|
||||
|
||||
@@ -323,6 +330,7 @@ def test_zero_budget_keeps_one_grammar_row_per_scheduled_draft():
|
||||
manager.get_num_tokens(
|
||||
{"low": 3, "high": 3, "prefill": 40}, scheduled_spec_decode_tokens
|
||||
)
|
||||
assert manager._batch_budget is not None
|
||||
assert manager._batch_budget[2] == 0
|
||||
|
||||
req_ids = ["low", "high", "prefill"]
|
||||
|
||||
@@ -98,7 +98,7 @@ def test_draft_quant_exclusions_include_global_layer_indices():
|
||||
]
|
||||
)
|
||||
|
||||
_add_global_draft_layer_exclusions(quant_config, 88, 5)
|
||||
_add_global_draft_layer_exclusions(quant_config, 88, 5) # type: ignore[arg-type]
|
||||
|
||||
assert "layers.88.mlp_conv*" in quant_config.exclude_modules
|
||||
assert "*layers.92.self_attn.q_proj" in quant_config.exclude_modules
|
||||
@@ -171,7 +171,7 @@ def test_selector_leaves_greedy_drafting_without_proposal_logits(monkeypatch):
|
||||
allocating one here would claim a proposal the walk never sampled from.
|
||||
"""
|
||||
_stub_base(monkeypatch, None)
|
||||
speculator = DFlash2Speculator(None, torch.device("cpu"))
|
||||
speculator = DFlash2Speculator(None, torch.device("cpu")) # type: ignore[arg-type] # Base initialization is stubbed.
|
||||
|
||||
assert speculator.draft_logits is None
|
||||
|
||||
@@ -183,7 +183,7 @@ def test_selector_asks_for_fp32_proposal_logits():
|
||||
candidate row often enough that the walk and the rejection sampler checking it
|
||||
would no longer read the same distribution.
|
||||
"""
|
||||
dtype, fill = DFlash2Speculator.draft_logits_spec(None, None)
|
||||
dtype, fill = DFlash2Speculator.draft_logits_spec(None, None) # type: ignore[arg-type] # Constant spec ignores self/config.
|
||||
|
||||
assert dtype is torch.float32
|
||||
assert fill == float("-inf")
|
||||
@@ -309,9 +309,9 @@ def test_candidate_model_decoder_layer_cls(monkeypatch, variant):
|
||||
vllm_config.compilation_config = mock_current_vllm_config.compilation_config
|
||||
|
||||
# 3. Instantiate the model under meta device to avoid parameter allocation issues
|
||||
with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"):
|
||||
with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): # type: ignore[arg-type]
|
||||
model_cls = DFlash2Qwen3Model if variant == "dflash2" else LiLiCorr
|
||||
model = model_cls(vllm_config=vllm_config)
|
||||
model = model_cls(vllm_config=vllm_config) # type: ignore[arg-type]
|
||||
|
||||
# 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass)
|
||||
assert len(model.layers) == 2
|
||||
@@ -350,7 +350,7 @@ def test_conv_projections_use_draft_quant_config(monkeypatch):
|
||||
quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True
|
||||
)
|
||||
layer = DFlash2Qwen3DecoderLayer(
|
||||
SimpleNamespace(
|
||||
SimpleNamespace( # type: ignore[arg-type]
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=7),
|
||||
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||
),
|
||||
@@ -434,13 +434,15 @@ def test_context_kv_uses_quantized_projection_fallback(monkeypatch):
|
||||
for projection in projections
|
||||
]
|
||||
qwen3_dflash.DFlashQwen3Model._build_context_kv_buffers(
|
||||
model, layers_attn, has_bias=False
|
||||
model, # type: ignore[arg-type]
|
||||
layers_attn,
|
||||
has_bias=False,
|
||||
)
|
||||
assert model._fused_kv_weight is None
|
||||
assert all(not hasattr(projection, "weight") for projection in projections)
|
||||
|
||||
actual_k, actual_v = qwen3_dflash.DFlashQwen3Model._project_context_kv(
|
||||
model,
|
||||
model, # type: ignore[arg-type]
|
||||
context_states,
|
||||
num_ctx=2,
|
||||
num_layers=2,
|
||||
|
||||
@@ -141,9 +141,9 @@ def test_dflash_drafter_window_reserves_bonus_token():
|
||||
speculative_config=_dflash_speculative_config(NUM_SPECULATIVE_TOKENS),
|
||||
)
|
||||
# window = 4, so 96 fits (96 + 4 == 100) but 97 does not (97 + 4 == 101)
|
||||
assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96))
|
||||
assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97))
|
||||
assert not input_fits_in_drafter(dflash_runner, None) # no metadata
|
||||
assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96)) # type: ignore[arg-type]
|
||||
assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type]
|
||||
assert not input_fits_in_drafter(dflash_runner, None) # type: ignore[arg-type] # No metadata.
|
||||
|
||||
# Other drafters don't reserve the bonus token, so 97 fits (97 + 3 == 100).
|
||||
plain_runner = SimpleNamespace(
|
||||
@@ -151,4 +151,4 @@ def test_dflash_drafter_window_reserves_bonus_token():
|
||||
effective_drafter_max_model_len=100,
|
||||
speculative_config=SimpleNamespace(use_dflash=lambda: False),
|
||||
)
|
||||
assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97))
|
||||
assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type]
|
||||
|
||||
@@ -56,6 +56,7 @@ def _make_predictor(vocab_size: int) -> DeepSeekV4MultiTokenPredictor:
|
||||
):
|
||||
shared_head = SharedHead(config=config, prefix="shared_head")
|
||||
|
||||
assert shared_head.head is not None
|
||||
generator = torch.Generator().manual_seed(7)
|
||||
with torch.no_grad():
|
||||
shared_head.head.weight.normal_(0.0, 0.02, generator=generator)
|
||||
@@ -157,7 +158,9 @@ def test_wrapper_forwards_spec_step_idx():
|
||||
|
||||
hidden_states = torch.empty(0)
|
||||
wrapper = object.__new__(DeepSeekV4MTP)
|
||||
wrapper.model = SimpleNamespace(get_top_tokens=lambda *args: args)
|
||||
wrapper.model = SimpleNamespace( # type: ignore[assignment]
|
||||
get_top_tokens=lambda *args: args
|
||||
)
|
||||
|
||||
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states, 2) == (hidden_states, 2)
|
||||
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states) == (hidden_states, 0)
|
||||
|
||||
@@ -25,7 +25,7 @@ def _create_vllm_config_for_dsd(
|
||||
max_num_seqs: int,
|
||||
max_spec_tokens: int,
|
||||
*,
|
||||
cudagraph_mode: str = "FULL_AND_PIECEWISE",
|
||||
cudagraph_mode: CUDAGraphMode = CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
use_dynamic_sd: bool = True,
|
||||
num_spec_per_batch_size: list[tuple[int, int, int]] | None = None,
|
||||
) -> MagicMock:
|
||||
@@ -199,7 +199,7 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=512,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
use_dynamic_sd=True,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
@@ -316,7 +316,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
use_dynamic_sd=False,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
@@ -382,7 +382,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
use_dynamic_sd=True,
|
||||
num_spec_per_batch_size=num_spec_per_batch_size,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Literal
|
||||
from unittest import mock
|
||||
|
||||
import numpy as np
|
||||
@@ -31,6 +32,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.spec_decode.dflash import DFlashProposer
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
|
||||
@@ -46,13 +48,13 @@ DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def _create_proposer(
|
||||
method: str,
|
||||
method: Literal["eagle", "eagle3", "draft_model", "dflash"],
|
||||
num_speculative_tokens: int,
|
||||
attention_backend: str | None = None,
|
||||
parallel_drafting: bool = False,
|
||||
rejection_sample_method: str = "standard",
|
||||
draft_sample_method: str = "greedy",
|
||||
) -> EagleProposer:
|
||||
rejection_sample_method: Literal["standard", "synthetic", "block"] = "standard",
|
||||
draft_sample_method: Literal["greedy", "probabilistic"] = "greedy",
|
||||
) -> SpecDecodeBaseProposer:
|
||||
# Method-dependent setup
|
||||
if method == "eagle":
|
||||
target_model_dir = model_dir
|
||||
@@ -102,9 +104,14 @@ def _create_proposer(
|
||||
max_model_len=model_config.max_model_len,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
),
|
||||
attention_config=AttentionConfig(backend=attention_backend),
|
||||
attention_config=AttentionConfig(
|
||||
backend=AttentionBackendEnum[attention_backend]
|
||||
if attention_backend
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
proposer: SpecDecodeBaseProposer
|
||||
if method == "dflash":
|
||||
proposer = DFlashProposer(vllm_config=vllm_config, device=device)
|
||||
elif "eagle" in method:
|
||||
@@ -135,7 +142,7 @@ def test_prepare_next_token_ids():
|
||||
)
|
||||
|
||||
mock_num_scheduled_tokens = {req_id: 0 for req_id in req_ids}
|
||||
mock_requests = {}
|
||||
mock_requests: dict[str, CachedRequestState] = {}
|
||||
for req_id in req_ids:
|
||||
mock_request = mock.MagicMock(spec=CachedRequestState)
|
||||
# Each request will have a backup next token id of 10, 20, 30, 40
|
||||
@@ -948,13 +955,12 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
|
||||
|
||||
attn_metadata_builder = attn_metadata_builder_cls(
|
||||
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
|
||||
layer_names=proposer._draft_attn_layer_names,
|
||||
layer_names=sorted(proposer._draft_attn_layer_names),
|
||||
vllm_config=proposer.vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Mock runner and draft_attn_groups for attention metadata building
|
||||
proposer.runner = mock.MagicMock()
|
||||
# Mock draft_attn_groups for attention metadata building.
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
@@ -1054,11 +1060,10 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch):
|
||||
)
|
||||
attn_metadata_builder = attn_metadata_builder_cls(
|
||||
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
|
||||
layer_names=proposer._draft_attn_layer_names,
|
||||
layer_names=sorted(proposer._draft_attn_layer_names),
|
||||
vllm_config=proposer.vllm_config,
|
||||
device=device,
|
||||
)
|
||||
proposer.runner = mock.MagicMock()
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
@@ -1133,6 +1138,7 @@ def test_set_inputs_first_pass_dflash():
|
||||
|
||||
num_speculative_tokens = 3
|
||||
proposer = _create_proposer("dflash", num_speculative_tokens)
|
||||
assert isinstance(proposer, DFlashProposer)
|
||||
mask_token_id = proposer.parallel_drafting_token_id
|
||||
|
||||
# Setup batch with 3 requests
|
||||
|
||||
@@ -93,7 +93,7 @@ def _run_build(
|
||||
|
||||
with patch.object(base_speculator, "build_attn_metadata", fake_build_attn_metadata):
|
||||
EagleSpeculator._build_uniform_attn_metadata(
|
||||
fake, # type: ignore[arg-type]
|
||||
fake,
|
||||
batch_desc=batch_desc,
|
||||
num_reqs=num_reqs,
|
||||
num_query_per_req=num_query_per_req,
|
||||
|
||||
@@ -51,7 +51,7 @@ def _create_proposer(
|
||||
target_parallel_config=ParallelConfig(),
|
||||
method="extract_hidden_states",
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
draft_model_config={
|
||||
draft_model_config={ # type: ignore[arg-type]
|
||||
"hf_config": {
|
||||
"eagle_aux_hidden_state_layer_ids": layer_ids,
|
||||
}
|
||||
@@ -103,7 +103,7 @@ def test_proposer_initialization_missing_layer_ids():
|
||||
target_parallel_config=ParallelConfig(),
|
||||
method="extract_hidden_states",
|
||||
num_speculative_tokens=1,
|
||||
draft_model_config={
|
||||
draft_model_config={ # type: ignore[arg-type]
|
||||
"hf_config": {} # Missing eagle_aux_hidden_state_layer_ids
|
||||
},
|
||||
)
|
||||
@@ -146,7 +146,7 @@ def test_prepare_next_token_ids_padded():
|
||||
mock_input_batch.vocab_size = 100
|
||||
mock_input_batch.num_tokens_no_spec = np.array([5] * num_requests)
|
||||
|
||||
mock_requests = {}
|
||||
mock_requests: dict[str, CachedRequestState] = {}
|
||||
for req_id in req_ids:
|
||||
mock_request = mock.MagicMock(spec=CachedRequestState)
|
||||
# Each request will have a backup next token id of 10, 20, 30, 40
|
||||
@@ -363,7 +363,7 @@ def test_extract_hidden_states_text_only_config_regression():
|
||||
target_parallel_config=ParallelConfig(),
|
||||
method="extract_hidden_states",
|
||||
num_speculative_tokens=1,
|
||||
draft_model_config={
|
||||
draft_model_config={ # type: ignore[arg-type]
|
||||
"hf_config": {
|
||||
"eagle_aux_hidden_state_layer_ids": [1, 2, 3, 4],
|
||||
}
|
||||
@@ -450,7 +450,7 @@ def test_extract_hidden_states_speculative_config_vlm():
|
||||
target_parallel_config=ParallelConfig(),
|
||||
method="extract_hidden_states",
|
||||
num_speculative_tokens=1,
|
||||
draft_model_config={
|
||||
draft_model_config={ # type: ignore[arg-type]
|
||||
"hf_config": {
|
||||
"eagle_aux_hidden_state_layer_ids": [1, 2],
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ from torch import nn
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
DeviceConfig,
|
||||
LoadConfig,
|
||||
VllmConfig,
|
||||
@@ -122,7 +123,7 @@ def test_lilicorr_matches_exported_head(head_width, slots, dtype):
|
||||
set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cuda"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
),
|
||||
):
|
||||
@@ -183,23 +184,23 @@ def test_runtime_length_keeps_trained_checkpoint_geometry(monkeypatch):
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cpu"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
):
|
||||
for slots in (1, 3):
|
||||
spec.num_speculative_tokens = slots
|
||||
head = LiLiCorr(
|
||||
vllm_config=SimpleNamespace(speculative_config=spec)
|
||||
vllm_config=SimpleNamespace(speculative_config=spec) # type: ignore[arg-type]
|
||||
).lilicorr
|
||||
assert head.slot_embedding.shape == (1, 1, 3, 1, 8)
|
||||
assert head.relative_slot_bias.shape == (2, 7)
|
||||
spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = False
|
||||
with pytest.raises(ValueError, match="lilicorr_enabled"):
|
||||
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec))
|
||||
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type]
|
||||
spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = True
|
||||
spec.num_speculative_tokens = 4
|
||||
with pytest.raises(ValueError, match="trained block_size"):
|
||||
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec))
|
||||
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@@ -268,7 +269,11 @@ def test_candidates_use_global_partition_and_exclude_padding(monkeypatch, tp_siz
|
||||
)
|
||||
monkeypatch.setattr(logits_processor, "_topk", lambda x, k: x.topk(k, dim=-1))
|
||||
ids, values = logits_processor.LogitsProcessor.get_top_k_tokens(
|
||||
fake, lm_head, torch.empty(2, 1), 2, return_log_probs=True
|
||||
fake, # type: ignore[arg-type]
|
||||
lm_head, # type: ignore[arg-type]
|
||||
torch.empty(2, 1),
|
||||
2,
|
||||
return_log_probs=True,
|
||||
)
|
||||
expected = torch.cat(valid_shards, -1).log_softmax(-1).topk(2)
|
||||
torch.testing.assert_close(ids, expected.indices)
|
||||
@@ -286,13 +291,13 @@ def test_context_anchor_is_last_committed_normalized_feature():
|
||||
anchor_valid=torch.ones(4, dtype=torch.bool),
|
||||
)
|
||||
batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6]))
|
||||
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2]))
|
||||
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) # type: ignore[arg-type]
|
||||
torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]]))
|
||||
assert spec.anchor_valid.tolist() == [True, True, False, False]
|
||||
assert not spec.anchor_hidden[2:].any()
|
||||
# A smaller reordered batch must not retain an old request's anchor.
|
||||
batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3]))
|
||||
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1]))
|
||||
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) # type: ignore[arg-type]
|
||||
torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1]))
|
||||
assert not spec.anchor_hidden[1:].any()
|
||||
|
||||
@@ -316,7 +321,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
|
||||
|
||||
monkeypatch.setattr(DFlashSpeculator, "__init__", init_base)
|
||||
cls = LiLiCorrSpeculator if lilicorr else DFlash2Speculator
|
||||
spec = cls(None, torch.device("cpu"))
|
||||
spec = cls(None, torch.device("cpu")) # type: ignore[arg-type]
|
||||
hidden = torch.arange(36).float().view(9, 4)
|
||||
spec._run_model = lambda *args: hidden
|
||||
spec.sample_indices = torch.tensor([1, 2, 4, 5, 7, 8])
|
||||
@@ -329,7 +334,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
|
||||
spec.draft_logits = None
|
||||
spec.use_fp64_gumbel = False
|
||||
spec.enable_adaptive_verification = True
|
||||
spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9))
|
||||
spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) # type: ignore[assignment]
|
||||
candidates = torch.arange(24).view(6, 4)
|
||||
log_probs = candidates.float().log_softmax(-1)
|
||||
scores = torch.randn(3, 2, 4, 4)
|
||||
@@ -349,22 +354,35 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
|
||||
compute_candidates=lambda h: (candidates, log_probs),
|
||||
model=SimpleNamespace(lilicorr=score, candidate_selector=score),
|
||||
)
|
||||
spec.target_embeddings = lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4)
|
||||
if lilicorr:
|
||||
assert isinstance(spec, LiLiCorrSpeculator)
|
||||
monkeypatch.setattr(
|
||||
spec,
|
||||
"target_embeddings",
|
||||
lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4),
|
||||
raising=False,
|
||||
)
|
||||
spec.anchor_hidden.copy_(torch.arange(12).view(3, 4))
|
||||
spec.anchor_valid[:2] = True
|
||||
spec.candidate_sampler.sample = sample
|
||||
spec._maybe_predict_acceptance = lambda *args: adaptive_inputs.extend(args)
|
||||
monkeypatch.setattr(spec.candidate_sampler, "sample", sample)
|
||||
monkeypatch.setattr(
|
||||
spec, "_maybe_predict_acceptance", lambda *args: adaptive_inputs.extend(args)
|
||||
)
|
||||
spec._generate_draft(3, 9, None, None, None)
|
||||
|
||||
expected_ids = candidates.view(3, 2, 4)
|
||||
expected_first = spec.target_embeddings(expected_ids) if lilicorr else expected_ids
|
||||
expected_first = (
|
||||
spec.target_embeddings(expected_ids)
|
||||
if isinstance(spec, LiLiCorrSpeculator)
|
||||
else expected_ids
|
||||
)
|
||||
torch.testing.assert_close(scoring_inputs[0], expected_first)
|
||||
torch.testing.assert_close(scoring_inputs[1], log_probs.view(3, 2, 4))
|
||||
torch.testing.assert_close(
|
||||
scoring_inputs[2], hidden[spec.sample_indices].view(3, 2, 4)
|
||||
)
|
||||
if lilicorr:
|
||||
assert isinstance(spec, LiLiCorrSpeculator)
|
||||
torch.testing.assert_close(scoring_inputs[3], spec.anchor_hidden)
|
||||
torch.testing.assert_close(scoring_inputs[4], spec.anchor_valid)
|
||||
else:
|
||||
@@ -472,7 +490,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads(
|
||||
):
|
||||
from vllm.model_executor.models.lilicorr import LiLiCorr, LiLiCorrForCausalLM
|
||||
|
||||
wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM)
|
||||
wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM) # type: ignore[type-abstract]
|
||||
nn.Module.__init__(wrapper)
|
||||
wrapper.model = LiLiCorr.__new__(LiLiCorr)
|
||||
nn.Module.__init__(wrapper.model)
|
||||
@@ -480,7 +498,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads(
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cpu"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
):
|
||||
wrapper.model.lilicorr = LiLiCorrHead(
|
||||
@@ -615,10 +633,10 @@ def owned_head_model(monkeypatch):
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cpu"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
):
|
||||
model = LiLiCorrForCausalLM(vllm_config=vllm_config)
|
||||
model = LiLiCorrForCausalLM(vllm_config=vllm_config) # type: ignore[arg-type]
|
||||
return model, vllm_config
|
||||
|
||||
return build
|
||||
@@ -720,7 +738,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(
|
||||
device_config=DeviceConfig("cpu"),
|
||||
compilation_config=CompilationConfig(mode=0),
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
)
|
||||
):
|
||||
head = LiLiCorrHead(
|
||||
@@ -728,7 +746,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch
|
||||
block_size=4,
|
||||
rms_norm_eps=1e-6,
|
||||
config=_config(hidden_size=8),
|
||||
quant_config=quant_config,
|
||||
quant_config=quant_config, # type: ignore[arg-type]
|
||||
prefix="model.lilicorr",
|
||||
)
|
||||
with torch.no_grad():
|
||||
|
||||
@@ -16,6 +16,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
|
||||
import vllm.v1.spec_decode.llm_base_proposer as llm_base_proposer
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
|
||||
SCHEDULER_BLOCK_SIZE = 256
|
||||
@@ -52,7 +53,7 @@ def _make_proposer(
|
||||
monkeypatch.setattr(llm_base_proposer, "AttentionGroup", _FakeAttentionGroup)
|
||||
|
||||
proposer = EagleProposer.__new__(EagleProposer)
|
||||
proposer.vllm_config = None
|
||||
proposer.vllm_config = None # type: ignore[assignment] # Patched backend discovery does not read config.
|
||||
proposer.device = None
|
||||
proposer._draft_attn_layer_names = set(layer_names)
|
||||
proposer.kv_cache_gid = -1
|
||||
@@ -61,10 +62,10 @@ def _make_proposer(
|
||||
return proposer
|
||||
|
||||
|
||||
def _make_kv_cache_config(layer_names: set[str]) -> SimpleNamespace:
|
||||
def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig:
|
||||
spec = SimpleNamespace(block_size=SCHEDULER_BLOCK_SIZE)
|
||||
group = SimpleNamespace(layer_names=list(layer_names), kv_cache_spec=spec)
|
||||
return SimpleNamespace(kv_cache_groups=[group])
|
||||
return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value]
|
||||
|
||||
|
||||
def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
@@ -81,7 +82,9 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
assert proposer.block_size == KERNEL_BLOCK_SIZE
|
||||
assert proposer.block_size != SCHEDULER_BLOCK_SIZE
|
||||
# The metadata builder keeps receiving the kernel block size as well.
|
||||
assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE
|
||||
group = proposer.draft_attn_groups[0]
|
||||
assert isinstance(group, _FakeAttentionGroup)
|
||||
assert group.kernel_block_size == KERNEL_BLOCK_SIZE
|
||||
|
||||
|
||||
def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
@@ -85,7 +85,7 @@ def test_compute_probs_and_sample_next_token_uses_fp64_exponential_race():
|
||||
def test_mtp_model_returns_tuple(architecture: str, expected: bool):
|
||||
proposer = object.__new__(SpecDecodeBaseProposer)
|
||||
proposer.method = "mtp"
|
||||
proposer.draft_model_config = SimpleNamespace(
|
||||
proposer.draft_model_config = SimpleNamespace( # type: ignore[assignment] # Architecture-only config.
|
||||
hf_config=SimpleNamespace(architectures=[architecture])
|
||||
)
|
||||
|
||||
|
||||
@@ -220,7 +220,6 @@ def test_mtp_propose(num_speculative_tokens, monkeypatch):
|
||||
device=device,
|
||||
)
|
||||
|
||||
proposer.runner = mock.MagicMock()
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backend and engine-reasoner checks that are not manager-flow tests."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
@@ -17,7 +19,9 @@ TOKENIZER = "gpt2"
|
||||
NUM_SPEC_TOKENS = 4
|
||||
|
||||
|
||||
def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
|
||||
def _make_manager_and_request(
|
||||
backend: Literal["xgrammar", "guidance"], prompt_str: str = '{"a": "b"}'
|
||||
):
|
||||
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
|
||||
prompt = tokenizer.encode(prompt_str)
|
||||
|
||||
@@ -33,6 +37,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
|
||||
sampling_params = SamplingParams(
|
||||
structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),
|
||||
)
|
||||
assert sampling_params.structured_outputs is not None
|
||||
sampling_params.structured_outputs._backend = backend
|
||||
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
|
||||
|
||||
@@ -43,6 +48,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
|
||||
pooling_params=None,
|
||||
)
|
||||
manager.grammar_init(request)
|
||||
assert request.structured_output_request is not None
|
||||
while not request.structured_output_request._check_grammar_completion():
|
||||
continue
|
||||
|
||||
|
||||
@@ -168,7 +168,7 @@ def test_watermarked_bonus_uses_target_key():
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
expected = GumbelWatermarker(key=42, context_width=2).sample(
|
||||
target_logits.unsqueeze(0), contexts[1:], lambda _: None
|
||||
target_logits.unsqueeze(0), contexts[1:]
|
||||
)
|
||||
|
||||
assert num_sampled.item() == 2
|
||||
@@ -196,7 +196,7 @@ def test_watermarked_recovery_uses_target_key():
|
||||
watermarker=GumbelWatermarker(key=42, context_width=2),
|
||||
)
|
||||
expected = GumbelWatermarker(key=42, context_width=2).sample(
|
||||
target_logits.unsqueeze(0), contexts[:1], lambda _: None
|
||||
target_logits.unsqueeze(0), contexts[:1]
|
||||
)
|
||||
|
||||
assert num_sampled.item() == 1
|
||||
@@ -225,9 +225,7 @@ def test_watermarked_recovery_supports_smaller_draft_vocabulary():
|
||||
watermarking=torch.tensor([True], device="cuda"),
|
||||
watermarker=watermarker,
|
||||
)
|
||||
expected = watermarker.sample(
|
||||
target_logits[:4].unsqueeze(0), contexts[:1], lambda _: None
|
||||
)
|
||||
expected = watermarker.sample(target_logits[:4].unsqueeze(0), contexts[:1])
|
||||
|
||||
assert num_sampled.item() == 1
|
||||
assert sampled[0, 0] == expected.token_ids[0]
|
||||
@@ -702,7 +700,7 @@ def test_verify_rejects_unproposed_drafts():
|
||||
first_pos = inputs["pos"].view(num_trials, K + 1)[:, 0]
|
||||
prefill_len = first_pos + 1
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = SimpleNamespace(
|
||||
rejection_sampler.sampler = SimpleNamespace( # type: ignore[assignment]
|
||||
apply_sampling_params=lambda logits, *args: logits,
|
||||
sampling_states=SimpleNamespace(
|
||||
temperature=SimpleNamespace(gpu=inputs["temperature"]),
|
||||
|
||||
Reference in New Issue
Block a user