[MyPy][3/N] Fix MyPy errors in test groups (part 3) (#55939)

Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Taimys
2026-09-30 11:06:07 +00:00
committed by GitHub
co-authored by Codex Luyi Xiao Taimys Harry Mellor Claude Opus 5.5
parent c4fa6f36a7
commit 5225f1bacb
118 changed files with 948 additions and 605 deletions
@@ -25,9 +25,9 @@ def make_manager(
num_reqs, num_steps = confidences.shape
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
manager.num_speculative_steps = num_steps
manager._stale_confidences = [SimpleNamespace(np=confidences)]
manager._stale_confidences = [SimpleNamespace(np=confidences)] # type: ignore[list-item] # CPU-only confidence buffer.
manager._stale_idx = 0
manager.req_states = SimpleNamespace(
manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture.
req_id_to_index={"low": 0, "high": 1},
num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32),
prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)),
@@ -153,11 +153,11 @@ def test_manager_checks_target_varlen_cudagraph_bound(
manager = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=True,
attn_groups=groups,
req_states=SimpleNamespace(num_speculative_steps=7),
req_states=SimpleNamespace(num_speculative_steps=7), # type: ignore[arg-type]
query_start_loc=object(),
num_bonus_tokens=1,
max_total_logits=1,
vllm_config=None,
vllm_config=None, # type: ignore[arg-type]
target_layer_names={"target"},
additional_attn_cg_support=additional,
)
@@ -176,6 +176,7 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
{"low": 3, "high": 3},
{"low": [1, 2], "high": [3, 4]},
)
assert manager._batch_budget is not None
valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget
assert draft_budget == 1
@@ -183,14 +184,18 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
assert num_non_draft_tokens == {"low": 1, "high": 1}
def test_profiled_batches_seed_cost_curves_via_consumer():
def test_profiled_batches_seed_cost_curves_via_consumer(monkeypatch):
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64)
manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture.
max_num_batched_tokens=4096, max_num_reqs=64
)
manager.num_speculative_steps = 7
manager.num_bonus_tokens = 1
curves: dict[str, list[tuple[int, float]]] = {}
manager.set_cost_curves = lambda draft, verify: curves.update(
draft=draft, verify=verify
monkeypatch.setattr(
manager,
"set_cost_curves",
lambda draft, verify: curves.update(draft=draft, verify=verify),
)
timings = [
@@ -258,6 +263,7 @@ def test_budget_caps_at_one_rejection_sampler_chunk():
{"low": 3, "high": 3},
{"low": [1, 2], "high": [3, 4]},
)
assert manager._batch_budget is not None
_, _, draft_budget = manager._batch_budget
assert draft_budget <= 1
@@ -285,6 +291,7 @@ def test_zero_budget_rebuilds_cpu_cu_num_logits():
{"low": 3, "high": 3, "prefill": 40},
{"low": [1, 2], "high": [3, 4]},
)
assert manager._batch_budget is not None
_, _, draft_budget = manager._batch_budget
assert draft_budget == 0
@@ -323,6 +330,7 @@ def test_zero_budget_keeps_one_grammar_row_per_scheduled_draft():
manager.get_num_tokens(
{"low": 3, "high": 3, "prefill": 40}, scheduled_spec_decode_tokens
)
assert manager._batch_budget is not None
assert manager._batch_budget[2] == 0
req_ids = ["low", "high", "prefill"]
+10 -8
View File
@@ -98,7 +98,7 @@ def test_draft_quant_exclusions_include_global_layer_indices():
]
)
_add_global_draft_layer_exclusions(quant_config, 88, 5)
_add_global_draft_layer_exclusions(quant_config, 88, 5) # type: ignore[arg-type]
assert "layers.88.mlp_conv*" in quant_config.exclude_modules
assert "*layers.92.self_attn.q_proj" in quant_config.exclude_modules
@@ -171,7 +171,7 @@ def test_selector_leaves_greedy_drafting_without_proposal_logits(monkeypatch):
allocating one here would claim a proposal the walk never sampled from.
"""
_stub_base(monkeypatch, None)
speculator = DFlash2Speculator(None, torch.device("cpu"))
speculator = DFlash2Speculator(None, torch.device("cpu")) # type: ignore[arg-type] # Base initialization is stubbed.
assert speculator.draft_logits is None
@@ -183,7 +183,7 @@ def test_selector_asks_for_fp32_proposal_logits():
candidate row often enough that the walk and the rejection sampler checking it
would no longer read the same distribution.
"""
dtype, fill = DFlash2Speculator.draft_logits_spec(None, None)
dtype, fill = DFlash2Speculator.draft_logits_spec(None, None) # type: ignore[arg-type] # Constant spec ignores self/config.
assert dtype is torch.float32
assert fill == float("-inf")
@@ -309,9 +309,9 @@ def test_candidate_model_decoder_layer_cls(monkeypatch, variant):
vllm_config.compilation_config = mock_current_vllm_config.compilation_config
# 3. Instantiate the model under meta device to avoid parameter allocation issues
with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"):
with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): # type: ignore[arg-type]
model_cls = DFlash2Qwen3Model if variant == "dflash2" else LiLiCorr
model = model_cls(vllm_config=vllm_config)
model = model_cls(vllm_config=vllm_config) # type: ignore[arg-type]
# 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass)
assert len(model.layers) == 2
@@ -350,7 +350,7 @@ def test_conv_projections_use_draft_quant_config(monkeypatch):
quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True
)
layer = DFlash2Qwen3DecoderLayer(
SimpleNamespace(
SimpleNamespace( # type: ignore[arg-type]
speculative_config=SimpleNamespace(num_speculative_tokens=7),
model_config=SimpleNamespace(dtype=torch.bfloat16),
),
@@ -434,13 +434,15 @@ def test_context_kv_uses_quantized_projection_fallback(monkeypatch):
for projection in projections
]
qwen3_dflash.DFlashQwen3Model._build_context_kv_buffers(
model, layers_attn, has_bias=False
model, # type: ignore[arg-type]
layers_attn,
has_bias=False,
)
assert model._fused_kv_weight is None
assert all(not hasattr(projection, "weight") for projection in projections)
actual_k, actual_v = qwen3_dflash.DFlashQwen3Model._project_context_kv(
model,
model, # type: ignore[arg-type]
context_states,
num_ctx=2,
num_layers=2,
@@ -141,9 +141,9 @@ def test_dflash_drafter_window_reserves_bonus_token():
speculative_config=_dflash_speculative_config(NUM_SPECULATIVE_TOKENS),
)
# window = 4, so 96 fits (96 + 4 == 100) but 97 does not (97 + 4 == 101)
assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96))
assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97))
assert not input_fits_in_drafter(dflash_runner, None) # no metadata
assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96)) # type: ignore[arg-type]
assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type]
assert not input_fits_in_drafter(dflash_runner, None) # type: ignore[arg-type] # No metadata.
# Other drafters don't reserve the bonus token, so 97 fits (97 + 3 == 100).
plain_runner = SimpleNamespace(
@@ -151,4 +151,4 @@ def test_dflash_drafter_window_reserves_bonus_token():
effective_drafter_max_model_len=100,
speculative_config=SimpleNamespace(use_dflash=lambda: False),
)
assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97))
assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type]
@@ -56,6 +56,7 @@ def _make_predictor(vocab_size: int) -> DeepSeekV4MultiTokenPredictor:
):
shared_head = SharedHead(config=config, prefix="shared_head")
assert shared_head.head is not None
generator = torch.Generator().manual_seed(7)
with torch.no_grad():
shared_head.head.weight.normal_(0.0, 0.02, generator=generator)
@@ -157,7 +158,9 @@ def test_wrapper_forwards_spec_step_idx():
hidden_states = torch.empty(0)
wrapper = object.__new__(DeepSeekV4MTP)
wrapper.model = SimpleNamespace(get_top_tokens=lambda *args: args)
wrapper.model = SimpleNamespace( # type: ignore[assignment]
get_top_tokens=lambda *args: args
)
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states, 2) == (hidden_states, 2)
assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states) == (hidden_states, 0)
+4 -4
View File
@@ -25,7 +25,7 @@ def _create_vllm_config_for_dsd(
max_num_seqs: int,
max_spec_tokens: int,
*,
cudagraph_mode: str = "FULL_AND_PIECEWISE",
cudagraph_mode: CUDAGraphMode = CUDAGraphMode.FULL_AND_PIECEWISE,
use_dynamic_sd: bool = True,
num_spec_per_batch_size: list[tuple[int, int, int]] | None = None,
) -> MagicMock:
@@ -199,7 +199,7 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=512,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
use_dynamic_sd=True,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
@@ -316,7 +316,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=max_num_seqs,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
use_dynamic_sd=False,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
@@ -382,7 +382,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=max_num_seqs,
max_spec_tokens=max_spec_tokens,
cudagraph_mode="FULL_AND_PIECEWISE",
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
use_dynamic_sd=True,
num_spec_per_batch_size=num_spec_per_batch_size,
)
+17 -11
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Literal
from unittest import mock
import numpy as np
@@ -31,6 +32,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.spec_decode.dflash import DFlashProposer
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -46,13 +48,13 @@ DEVICE_TYPE = current_platform.device_type
def _create_proposer(
method: str,
method: Literal["eagle", "eagle3", "draft_model", "dflash"],
num_speculative_tokens: int,
attention_backend: str | None = None,
parallel_drafting: bool = False,
rejection_sample_method: str = "standard",
draft_sample_method: str = "greedy",
) -> EagleProposer:
rejection_sample_method: Literal["standard", "synthetic", "block"] = "standard",
draft_sample_method: Literal["greedy", "probabilistic"] = "greedy",
) -> SpecDecodeBaseProposer:
# Method-dependent setup
if method == "eagle":
target_model_dir = model_dir
@@ -102,9 +104,14 @@ def _create_proposer(
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
),
attention_config=AttentionConfig(backend=attention_backend),
attention_config=AttentionConfig(
backend=AttentionBackendEnum[attention_backend]
if attention_backend
else None
),
)
proposer: SpecDecodeBaseProposer
if method == "dflash":
proposer = DFlashProposer(vllm_config=vllm_config, device=device)
elif "eagle" in method:
@@ -135,7 +142,7 @@ def test_prepare_next_token_ids():
)
mock_num_scheduled_tokens = {req_id: 0 for req_id in req_ids}
mock_requests = {}
mock_requests: dict[str, CachedRequestState] = {}
for req_id in req_ids:
mock_request = mock.MagicMock(spec=CachedRequestState)
# Each request will have a backup next token id of 10, 20, 30, 40
@@ -948,13 +955,12 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
attn_metadata_builder = attn_metadata_builder_cls(
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
layer_names=proposer._draft_attn_layer_names,
layer_names=sorted(proposer._draft_attn_layer_names),
vllm_config=proposer.vllm_config,
device=device,
)
# Mock runner and draft_attn_groups for attention metadata building
proposer.runner = mock.MagicMock()
# Mock draft_attn_groups for attention metadata building.
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
@@ -1054,11 +1060,10 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch):
)
attn_metadata_builder = attn_metadata_builder_cls(
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
layer_names=proposer._draft_attn_layer_names,
layer_names=sorted(proposer._draft_attn_layer_names),
vllm_config=proposer.vllm_config,
device=device,
)
proposer.runner = mock.MagicMock()
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
@@ -1133,6 +1138,7 @@ def test_set_inputs_first_pass_dflash():
num_speculative_tokens = 3
proposer = _create_proposer("dflash", num_speculative_tokens)
assert isinstance(proposer, DFlashProposer)
mask_token_id = proposer.parallel_drafting_token_id
# Setup batch with 3 requests
@@ -93,7 +93,7 @@ def _run_build(
with patch.object(base_speculator, "build_attn_metadata", fake_build_attn_metadata):
EagleSpeculator._build_uniform_attn_metadata(
fake, # type: ignore[arg-type]
fake,
batch_desc=batch_desc,
num_reqs=num_reqs,
num_query_per_req=num_query_per_req,
@@ -51,7 +51,7 @@ def _create_proposer(
target_parallel_config=ParallelConfig(),
method="extract_hidden_states",
num_speculative_tokens=num_speculative_tokens,
draft_model_config={
draft_model_config={ # type: ignore[arg-type]
"hf_config": {
"eagle_aux_hidden_state_layer_ids": layer_ids,
}
@@ -103,7 +103,7 @@ def test_proposer_initialization_missing_layer_ids():
target_parallel_config=ParallelConfig(),
method="extract_hidden_states",
num_speculative_tokens=1,
draft_model_config={
draft_model_config={ # type: ignore[arg-type]
"hf_config": {} # Missing eagle_aux_hidden_state_layer_ids
},
)
@@ -146,7 +146,7 @@ def test_prepare_next_token_ids_padded():
mock_input_batch.vocab_size = 100
mock_input_batch.num_tokens_no_spec = np.array([5] * num_requests)
mock_requests = {}
mock_requests: dict[str, CachedRequestState] = {}
for req_id in req_ids:
mock_request = mock.MagicMock(spec=CachedRequestState)
# Each request will have a backup next token id of 10, 20, 30, 40
@@ -363,7 +363,7 @@ def test_extract_hidden_states_text_only_config_regression():
target_parallel_config=ParallelConfig(),
method="extract_hidden_states",
num_speculative_tokens=1,
draft_model_config={
draft_model_config={ # type: ignore[arg-type]
"hf_config": {
"eagle_aux_hidden_state_layer_ids": [1, 2, 3, 4],
}
@@ -450,7 +450,7 @@ def test_extract_hidden_states_speculative_config_vlm():
target_parallel_config=ParallelConfig(),
method="extract_hidden_states",
num_speculative_tokens=1,
draft_model_config={
draft_model_config={ # type: ignore[arg-type]
"hf_config": {
"eagle_aux_hidden_state_layer_ids": [1, 2],
}
+38 -20
View File
@@ -11,6 +11,7 @@ from torch import nn
from vllm.config import (
CompilationConfig,
CompilationMode,
DeviceConfig,
LoadConfig,
VllmConfig,
@@ -122,7 +123,7 @@ def test_lilicorr_matches_exported_head(head_width, slots, dtype):
set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cuda"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
),
):
@@ -183,23 +184,23 @@ def test_runtime_length_keeps_trained_checkpoint_geometry(monkeypatch):
with set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cpu"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
):
for slots in (1, 3):
spec.num_speculative_tokens = slots
head = LiLiCorr(
vllm_config=SimpleNamespace(speculative_config=spec)
vllm_config=SimpleNamespace(speculative_config=spec) # type: ignore[arg-type]
).lilicorr
assert head.slot_embedding.shape == (1, 1, 3, 1, 8)
assert head.relative_slot_bias.shape == (2, 7)
spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = False
with pytest.raises(ValueError, match="lilicorr_enabled"):
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec))
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type]
spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = True
spec.num_speculative_tokens = 4
with pytest.raises(ValueError, match="trained block_size"):
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec))
LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type]
@pytest.mark.parametrize("tp_size", [1, 2])
@@ -268,7 +269,11 @@ def test_candidates_use_global_partition_and_exclude_padding(monkeypatch, tp_siz
)
monkeypatch.setattr(logits_processor, "_topk", lambda x, k: x.topk(k, dim=-1))
ids, values = logits_processor.LogitsProcessor.get_top_k_tokens(
fake, lm_head, torch.empty(2, 1), 2, return_log_probs=True
fake, # type: ignore[arg-type]
lm_head, # type: ignore[arg-type]
torch.empty(2, 1),
2,
return_log_probs=True,
)
expected = torch.cat(valid_shards, -1).log_softmax(-1).topk(2)
torch.testing.assert_close(ids, expected.indices)
@@ -286,13 +291,13 @@ def test_context_anchor_is_last_committed_normalized_feature():
anchor_valid=torch.ones(4, dtype=torch.bool),
)
batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) # type: ignore[arg-type]
torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]]))
assert spec.anchor_valid.tolist() == [True, True, False, False]
assert not spec.anchor_hidden[2:].any()
# A smaller reordered batch must not retain an old request's anchor.
batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) # type: ignore[arg-type]
torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1]))
assert not spec.anchor_hidden[1:].any()
@@ -316,7 +321,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
monkeypatch.setattr(DFlashSpeculator, "__init__", init_base)
cls = LiLiCorrSpeculator if lilicorr else DFlash2Speculator
spec = cls(None, torch.device("cpu"))
spec = cls(None, torch.device("cpu")) # type: ignore[arg-type]
hidden = torch.arange(36).float().view(9, 4)
spec._run_model = lambda *args: hidden
spec.sample_indices = torch.tensor([1, 2, 4, 5, 7, 8])
@@ -329,7 +334,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
spec.draft_logits = None
spec.use_fp64_gumbel = False
spec.enable_adaptive_verification = True
spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9))
spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) # type: ignore[assignment]
candidates = torch.arange(24).view(6, 4)
log_probs = candidates.float().log_softmax(-1)
scores = torch.randn(3, 2, 4, 4)
@@ -349,22 +354,35 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil
compute_candidates=lambda h: (candidates, log_probs),
model=SimpleNamespace(lilicorr=score, candidate_selector=score),
)
spec.target_embeddings = lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4)
if lilicorr:
assert isinstance(spec, LiLiCorrSpeculator)
monkeypatch.setattr(
spec,
"target_embeddings",
lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4),
raising=False,
)
spec.anchor_hidden.copy_(torch.arange(12).view(3, 4))
spec.anchor_valid[:2] = True
spec.candidate_sampler.sample = sample
spec._maybe_predict_acceptance = lambda *args: adaptive_inputs.extend(args)
monkeypatch.setattr(spec.candidate_sampler, "sample", sample)
monkeypatch.setattr(
spec, "_maybe_predict_acceptance", lambda *args: adaptive_inputs.extend(args)
)
spec._generate_draft(3, 9, None, None, None)
expected_ids = candidates.view(3, 2, 4)
expected_first = spec.target_embeddings(expected_ids) if lilicorr else expected_ids
expected_first = (
spec.target_embeddings(expected_ids)
if isinstance(spec, LiLiCorrSpeculator)
else expected_ids
)
torch.testing.assert_close(scoring_inputs[0], expected_first)
torch.testing.assert_close(scoring_inputs[1], log_probs.view(3, 2, 4))
torch.testing.assert_close(
scoring_inputs[2], hidden[spec.sample_indices].view(3, 2, 4)
)
if lilicorr:
assert isinstance(spec, LiLiCorrSpeculator)
torch.testing.assert_close(scoring_inputs[3], spec.anchor_hidden)
torch.testing.assert_close(scoring_inputs[4], spec.anchor_valid)
else:
@@ -472,7 +490,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads(
):
from vllm.model_executor.models.lilicorr import LiLiCorr, LiLiCorrForCausalLM
wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM)
wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM) # type: ignore[type-abstract]
nn.Module.__init__(wrapper)
wrapper.model = LiLiCorr.__new__(LiLiCorr)
nn.Module.__init__(wrapper.model)
@@ -480,7 +498,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads(
with set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cpu"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
):
wrapper.model.lilicorr = LiLiCorrHead(
@@ -615,10 +633,10 @@ def owned_head_model(monkeypatch):
with set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cpu"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
):
model = LiLiCorrForCausalLM(vllm_config=vllm_config)
model = LiLiCorrForCausalLM(vllm_config=vllm_config) # type: ignore[arg-type]
return model, vllm_config
return build
@@ -720,7 +738,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch
with set_current_vllm_config(
VllmConfig(
device_config=DeviceConfig("cpu"),
compilation_config=CompilationConfig(mode=0),
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
)
):
head = LiLiCorrHead(
@@ -728,7 +746,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch
block_size=4,
rms_norm_eps=1e-6,
config=_config(hidden_size=8),
quant_config=quant_config,
quant_config=quant_config, # type: ignore[arg-type]
prefix="model.lilicorr",
)
with torch.no_grad():
@@ -16,6 +16,7 @@ from types import SimpleNamespace
import pytest
import vllm.v1.spec_decode.llm_base_proposer as llm_base_proposer
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.spec_decode.eagle import EagleProposer
SCHEDULER_BLOCK_SIZE = 256
@@ -52,7 +53,7 @@ def _make_proposer(
monkeypatch.setattr(llm_base_proposer, "AttentionGroup", _FakeAttentionGroup)
proposer = EagleProposer.__new__(EagleProposer)
proposer.vllm_config = None
proposer.vllm_config = None # type: ignore[assignment] # Patched backend discovery does not read config.
proposer.device = None
proposer._draft_attn_layer_names = set(layer_names)
proposer.kv_cache_gid = -1
@@ -61,10 +62,10 @@ def _make_proposer(
return proposer
def _make_kv_cache_config(layer_names: set[str]) -> SimpleNamespace:
def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig:
spec = SimpleNamespace(block_size=SCHEDULER_BLOCK_SIZE)
group = SimpleNamespace(layer_names=list(layer_names), kv_cache_spec=spec)
return SimpleNamespace(kv_cache_groups=[group])
return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value]
def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
@@ -81,7 +82,9 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
assert proposer.block_size == KERNEL_BLOCK_SIZE
assert proposer.block_size != SCHEDULER_BLOCK_SIZE
# The metadata builder keeps receiving the kernel block size as well.
assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE
group = proposer.draft_attn_groups[0]
assert isinstance(group, _FakeAttentionGroup)
assert group.kernel_block_size == KERNEL_BLOCK_SIZE
def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch):
@@ -85,7 +85,7 @@ def test_compute_probs_and_sample_next_token_uses_fp64_exponential_race():
def test_mtp_model_returns_tuple(architecture: str, expected: bool):
proposer = object.__new__(SpecDecodeBaseProposer)
proposer.method = "mtp"
proposer.draft_model_config = SimpleNamespace(
proposer.draft_model_config = SimpleNamespace( # type: ignore[assignment] # Architecture-only config.
hf_config=SimpleNamespace(architectures=[architecture])
)
-1
View File
@@ -220,7 +220,6 @@ def test_mtp_propose(num_speculative_tokens, monkeypatch):
device=device,
)
proposer.runner = mock.MagicMock()
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backend and engine-reasoner checks that are not manager-flow tests."""
from typing import Literal
import pytest
from transformers import AutoTokenizer
@@ -17,7 +19,9 @@ TOKENIZER = "gpt2"
NUM_SPEC_TOKENS = 4
def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
def _make_manager_and_request(
backend: Literal["xgrammar", "guidance"], prompt_str: str = '{"a": "b"}'
):
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
prompt = tokenizer.encode(prompt_str)
@@ -33,6 +37,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
sampling_params = SamplingParams(
structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),
)
assert sampling_params.structured_outputs is not None
sampling_params.structured_outputs._backend = backend
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
@@ -43,6 +48,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
pooling_params=None,
)
manager.grammar_init(request)
assert request.structured_output_request is not None
while not request.structured_output_request._check_grammar_completion():
continue
@@ -168,7 +168,7 @@ def test_watermarked_bonus_uses_target_key():
watermarker=GumbelWatermarker(key=42, context_width=2),
)
expected = GumbelWatermarker(key=42, context_width=2).sample(
target_logits.unsqueeze(0), contexts[1:], lambda _: None
target_logits.unsqueeze(0), contexts[1:]
)
assert num_sampled.item() == 2
@@ -196,7 +196,7 @@ def test_watermarked_recovery_uses_target_key():
watermarker=GumbelWatermarker(key=42, context_width=2),
)
expected = GumbelWatermarker(key=42, context_width=2).sample(
target_logits.unsqueeze(0), contexts[:1], lambda _: None
target_logits.unsqueeze(0), contexts[:1]
)
assert num_sampled.item() == 1
@@ -225,9 +225,7 @@ def test_watermarked_recovery_supports_smaller_draft_vocabulary():
watermarking=torch.tensor([True], device="cuda"),
watermarker=watermarker,
)
expected = watermarker.sample(
target_logits[:4].unsqueeze(0), contexts[:1], lambda _: None
)
expected = watermarker.sample(target_logits[:4].unsqueeze(0), contexts[:1])
assert num_sampled.item() == 1
assert sampled[0, 0] == expected.token_ids[0]
@@ -702,7 +700,7 @@ def test_verify_rejects_unproposed_drafts():
first_pos = inputs["pos"].view(num_trials, K + 1)[:, 0]
prefill_len = first_pos + 1
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = SimpleNamespace(
rejection_sampler.sampler = SimpleNamespace( # type: ignore[assignment]
apply_sampling_params=lambda logits, *args: logits,
sampling_states=SimpleNamespace(
temperature=SimpleNamespace(gpu=inputs["temperature"]),