[Model Runner V2] Support custom logits processors (#56497)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
David Cheung
2026-09-19 17:46:49 +00:00
committed by GitHub
co-authored by Claude Nick Hill
parent 751f6807d9
commit 5302d1fe40
32 changed files with 1215 additions and 510 deletions
+9
View File
@@ -652,6 +652,15 @@ def test_v2_model_runner_supports_extract_hidden_states():
assert config._get_v2_model_runner_unsupported_features() == []
def test_v2_model_runner_supports_custom_logits_processors():
config = VllmConfig()
config.model_config = cast(
ModelConfig, SimpleNamespace(logits_processors=["a.b:C"])
)
assert config._get_v2_model_runner_unsupported_features() == []
def test_dflash2_draft_forces_v2_model_runner():
"""A DFlash2 draft must reach the V2 speculator, the only one that runs its
candidate selector; on V1 it would draft as DFlash1 without raising."""
+8
View File
@@ -51,6 +51,14 @@ def test_non_diffusion_models_unaffected():
params.verify(MockModelConfig(), None, None, None)
def test_verify_leaves_logits_processors_to_admission():
"""verify() is runner-agnostic; LP validation lives in the admission
layer, so an unimportable FQCN must not fail verify()."""
SamplingParams().verify(
MockModelConfig(logits_processors=["no.such:Cls"]), None, None, None
)
@pytest.mark.parametrize("value", [-(2**63) - 1, 2**64])
def test_extra_args_rejects_nested_integer_overflow(value):
"""Reject extension values before they reach the engine transport."""
@@ -85,6 +85,7 @@ def _validate(enable_trace_replay: bool) -> None:
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
validate_logits_processors_params=lambda params: None,
)
params = SamplingParams(trace_decode_token_ids=[1, 2, 3])
with patch.object(SamplingParams, "verify"):
@@ -162,6 +162,9 @@ def test_custom_logitsprocs(monkeypatch, logitproc_source: CustomLogitprocSource
"""
# Test that logitproc info is passed to workers
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")
# These tests exercise the V1-interface logits processor; Model Runner V2
# rejects V1-interface processors at load time by design.
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
set_random_seed(40)
# Choose LLM args based on logitproc source
@@ -217,6 +220,9 @@ def test_custom_logitsprocs_req(monkeypatch):
"""
# Test that logitproc info is passed to workers
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")
# These tests exercise the V1-interface logits processor; Model Runner V2
# rejects V1-interface processors at load time by design.
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
set_random_seed(40)
_run_test(
{"logits_processors": [WrappedPerReqLogitsProcessor]}, logitproc_loaded=True
@@ -261,6 +267,9 @@ def test_rejects_custom_logitsprocs(
"""
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
# These tests exercise V1-interface rejection and V1 runner internals;
# Model Runner V2 rejects V1-interface processors at load time by design.
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
set_random_seed(40)
test_params: dict[str, dict[str, Any]] = {
@@ -79,6 +79,9 @@ def server(default_server_args, request, monkeypatch):
"""
# Test that logitproc info is passed to workers
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")
# These tests exercise the V1-interface logits processor; Model Runner V2
# rejects V1-interface processors at load time by design.
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
if request.param:
# Launch server, append FQCN argument, inject dummy logitproc module
+49 -6
View File
@@ -14,6 +14,10 @@ if not torch.cuda.is_available():
from vllm.sampling_params import SamplingParams
from vllm.v1.worker.gpu.sample.bad_words import BadWordsState
from vllm.v1.worker.gpu.sample.logits_processor import (
LogitsContext,
LogitsProcRequestState,
)
from vllm.v1.worker.gpu.states import RequestState
DEVICE = torch.device("cuda")
@@ -47,7 +51,7 @@ def _make_state(bad_words_token_ids: list[list[int]]) -> tuple[BadWordsState, in
req_states.apply_staged_writes()
req_idx = req_states.req_id_to_index["req"]
state = BadWordsState(req_states)
state = BadWordsState(None, LogitsProcRequestState.from_request_state(req_states))
state.add_request(req_idx, SamplingParams(_bad_words_token_ids=bad_words_token_ids))
state.apply_staged_writes()
return state, req_idx
@@ -61,16 +65,55 @@ def _apply(bad_words_token_ids: list[list[int]]) -> torch.Tensor:
expanded_idx_mapping = torch.tensor(
[req_idx] * num_logits, dtype=torch.int32, device=DEVICE
)
state.apply_bad_words(
state.apply(
logits,
expanded_idx_mapping,
idx_mapping_np,
torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE),
torch.tensor(LOCAL_POS, dtype=torch.int32, device=DEVICE),
LogitsContext(
expanded_idx_mapping=expanded_idx_mapping,
idx_mapping=torch.tensor([req_idx], dtype=torch.int32, device=DEVICE),
idx_mapping_np=idx_mapping_np,
expanded_local_pos=torch.tensor(
LOCAL_POS, dtype=torch.int32, device=DEVICE
),
input_ids=torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE),
pos=torch.zeros(num_logits, dtype=torch.int32, device=DEVICE),
seq_lens_upper_bound_np=np.full(1, num_logits, dtype=np.int64),
),
)
return logits.cpu()
def test_v2_bad_words_slot_reuse_clears_state():
"""A slot recycled from a bad-words request must not keep masking.
There is no removal hook by design; safety comes from add_request()
overwriting the slot's state unconditionally.
"""
state, req_idx = _make_state([[10, 11, 60]])
state.add_request(req_idx, SamplingParams())
state.apply_staged_writes()
assert int(state.num_bad_words.np[req_idx]) == 0
num_logits = len(INPUT_IDS)
idx_mapping = torch.tensor([req_idx], dtype=torch.int32, device=DEVICE)
out = state.apply(
torch.zeros((num_logits, VOCAB_SIZE), device=DEVICE),
LogitsContext(
expanded_idx_mapping=torch.tensor(
[req_idx] * num_logits, dtype=torch.int32, device=DEVICE
),
idx_mapping=idx_mapping,
idx_mapping_np=np.array([req_idx], dtype=np.intp),
expanded_local_pos=torch.tensor(
LOCAL_POS, dtype=torch.int32, device=DEVICE
),
input_ids=torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE),
pos=torch.zeros(num_logits, dtype=torch.int32, device=DEVICE),
seq_lens_upper_bound_np=np.full(1, num_logits, dtype=np.int64),
),
)
assert not torch.isinf(out).any()
def test_v2_bad_words_prefix_inside_draft_tokens():
"""A prefix matching entirely within the draft tokens must mask the bad
word's last token at the draft position that completes the prefix."""
+6
View File
@@ -183,6 +183,12 @@ def test_local_batch_partition(tp_size: int, seed: int):
).all()
assert (_np(local.logits_indices) == _np(batch.logits_indices)[rows]).all()
assert (_np(local.seq_lens) == _np(batch.seq_lens)[owned]).all()
# Indexed by batch position, like seq_lens: a consumer holding local row
# indices reads another request's length if this is not localised too.
assert (
_np(local.seq_lens_cpu_upper_bound)
== _np(batch.seq_lens_cpu_upper_bound)[owned]
).all()
assert batch.num_draft_tokens_per_req is not None
assert local.num_draft_tokens_per_req is not None
assert (
+29 -6
View File
@@ -14,6 +14,11 @@ if not torch.cuda.is_available():
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState
from vllm.v1.worker.gpu.sample.logits_processor import (
LogitsContext,
LogitsProcRequestState,
)
from vllm.v1.worker.gpu.states import RequestState
DEVICE = torch.device("cuda")
VOCAB_SIZE = 128
@@ -45,17 +50,35 @@ def _only_stop_token_left(num_rows: int) -> torch.Tensor:
def _apply(logits: torch.Tensor, structured: list[bool]) -> torch.Tensor:
"""Run the fused bias kernel with one logits row per request."""
state = LogitBiasState(max_num_reqs=4, device=DEVICE)
req_states = RequestState(
max_num_reqs=4,
max_model_len=64,
max_num_batched_tokens=16,
num_speculative_steps=1,
vocab_size=VOCAB_SIZE,
device=DEVICE,
)
# add_request() reads prompt_len per slot; every slot here uses the same one.
req_states.prompt_len.np[:] = PROMPT_LEN
state = LogitBiasState(None, LogitsProcRequestState.from_request_state(req_states))
for req_idx, is_structured in enumerate(structured):
state.add_request(req_idx, PROMPT_LEN, _params(is_structured))
state.add_request(req_idx, _params(is_structured))
state.apply_staged_writes()
num_reqs = len(structured)
state.apply_logit_bias(
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE)
state.apply(
logits,
torch.arange(num_reqs, dtype=torch.int32, device=DEVICE),
np.arange(num_reqs, dtype=np.intp),
torch.full((num_reqs,), POS, dtype=torch.int32, device=DEVICE),
LogitsContext(
expanded_idx_mapping=idx_mapping,
idx_mapping=idx_mapping,
idx_mapping_np=np.arange(num_reqs, dtype=np.intp),
expanded_local_pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
input_ids=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
pos=torch.full((num_reqs,), POS, dtype=torch.int32, device=DEVICE),
seq_lens_upper_bound_np=np.full(num_reqs, POS + 1, dtype=np.int64),
),
)
return logits.cpu()
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
import torch
pytest.importorskip("triton")
if not torch.cuda.is_available():
pytest.skip(
"CUDA required for Model Runner V2 logits processor tests",
allow_module_level=True,
)
from vllm.sampling_params import SamplingParams
from vllm.v1.worker.gpu.sample.logits_processor import (
LogitsContext,
LogitsProcessor,
)
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.states import RequestState
DEVICE = torch.device("cuda")
VOCAB_SIZE = 128
TARGET_TOKEN = 7
BIAS = 5.0
BOOST = 100.0
TEMPERATURE = 2.0
class MockReasoningConfig:
reasoning_start_token_ids = [90]
reasoning_end_token_ids = [91]
natural_reasoning_end_token_ids = [91]
class RecordingProcessor(LogitsProcessor):
"""Adds BOOST to TARGET_TOKEN and records every lifecycle callback."""
def __init__(self, vllm_config: Any, req_state: Any):
self.events: list[str] = []
self.seen_logits: torch.Tensor | None = None
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool:
self.events.append(f"add_request:{req_idx}")
return True
def apply_staged_writes(self) -> None:
self.events.append("apply_staged_writes")
def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor:
self.events.append("apply")
self.seen_logits = logits.clone()
logits[:, TARGET_TOKEN] += BOOST
return logits
def _make_sampler(processor: RecordingProcessor) -> Sampler:
req_states = RequestState(
max_num_reqs=4,
max_model_len=64,
max_num_batched_tokens=16,
num_speculative_steps=1,
vocab_size=VOCAB_SIZE,
device=DEVICE,
)
return Sampler(
vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()),
max_num_reqs=4,
vocab_size=VOCAB_SIZE,
device=DEVICE,
req_states=req_states,
custom_logits_processors=[processor],
)
def test_custom_processor_lifecycle_and_pipeline_position():
"""A custom processor's hooks fire in order, it runs after the built-in
bias stage, and before temperature scaling."""
proc = RecordingProcessor(None, None)
sampler = _make_sampler(proc)
sampler.add_request(
0,
SamplingParams(logit_bias={TARGET_TOKEN: BIAS}, temperature=TEMPERATURE),
)
sampler.apply_staged_writes()
num_reqs = 1
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE)
raw = torch.zeros(num_reqs, VOCAB_SIZE, device=DEVICE)
out = sampler.apply_sampling_params(
raw,
expanded_idx_mapping=idx_mapping,
idx_mapping=idx_mapping,
idx_mapping_np=np.arange(num_reqs, dtype=np.intp),
pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
input_ids=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
expanded_local_pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
seq_lens_upper_bound_np=np.ones(num_reqs, dtype=np.int64),
)
assert proc.events == [
"add_request:0",
"apply_staged_writes",
"apply",
]
# The processor sees the built-in bias applied but no temperature scaling.
assert proc.seen_logits is not None
assert proc.seen_logits[0, TARGET_TOKEN].item() == BIAS
# Its own modification is then temperature-scaled by the sampler.
assert out[0, TARGET_TOKEN].item() == (BIAS + BOOST) / TEMPERATURE
@@ -56,7 +56,16 @@ class FakeEplbState:
def _make_runner(**overrides: Any) -> Any:
runner: Any = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
runner.device = torch.device("cpu")
runner.model_config = SimpleNamespace(model="test-model")
runner.model_config = SimpleNamespace(model="test-model", logits_processors=None)
runner.req_states = SimpleNamespace(
device=runner.device,
max_num_reqs=8,
vocab_size=0,
all_token_ids=None,
prompt_len=None,
prefill_len=None,
total_len=None,
)
runner.load_config = SimpleNamespace(load_format="hf")
runner.parallel_config = SimpleNamespace(
enable_eplb=True,
@@ -47,6 +47,9 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
)
).to(device),
seq_lens_cpu_upper_bound=torch.from_numpy(
np.array([10, 20, 30, 40], dtype=np.int32)
),
)
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
+38 -7
View File
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from types import SimpleNamespace
import numpy as np
import pytest
import torch
@@ -10,6 +13,10 @@ if not torch.cuda.is_available():
pytest.skip("CUDA required for sampler flag tests", allow_module_level=True)
from vllm.sampling_params import SamplingParams
from vllm.v1.worker.gpu.sample.logits_processor import (
LogitsContext,
LogitsProcessor,
)
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.states import RequestState
@@ -23,7 +30,7 @@ class MockReasoningConfig:
natural_reasoning_end_token_ids = [91]
def _make_sampler() -> Sampler:
def _make_sampler(custom_logits_processors: Sequence[LogitsProcessor] = ()) -> Sampler:
req_states = RequestState(
max_num_reqs=4,
max_model_len=64,
@@ -33,11 +40,12 @@ def _make_sampler() -> Sampler:
device=DEVICE,
)
return Sampler(
vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()),
max_num_reqs=4,
vocab_size=VOCAB_SIZE,
device=DEVICE,
req_states=req_states,
reasoning_config=MockReasoningConfig(),
custom_logits_processors=custom_logits_processors,
)
@@ -65,23 +73,46 @@ def test_logits_processing_cache_matches_request_features(
sampling_params: SamplingParams, expected: bool
):
sampler = _make_sampler()
sampler.add_request(3, prompt_len=1, sampling_params=sampling_params)
sampler.add_request(3, sampling_params=sampling_params)
assert sampler.needs_logits_processing[3] == expected
def test_logits_processing_cache_is_overwritten_when_slot_is_reused():
sampler = _make_sampler()
sampler.add_request(3, 1, SamplingParams.for_sampler_warmup())
sampler.add_request(3, 1, SamplingParams())
sampler.add_request(3, SamplingParams.for_sampler_warmup())
sampler.add_request(3, SamplingParams())
assert not sampler.needs_logits_processing[3]
class _GateProcessor(LogitsProcessor):
"""Reports a fixed admission decision from add_request()."""
def __init__(self, admitted: bool):
self.admitted = admitted
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool:
return self.admitted
def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor:
raise AssertionError("the pipeline never runs in this test")
@pytest.mark.parametrize("admitted", [True, False])
def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
"""A custom processor's add_request() return value is OR-ed into the
per-request needs_logits_processing flag."""
sampler = _make_sampler(custom_logits_processors=[_GateProcessor(admitted)])
sampler.add_request(3, SamplingParams())
assert sampler.needs_logits_processing[3] == admitted
def test_logits_processing_cache_only_checks_active_requests():
sampler = _make_sampler()
sampler.add_request(0, 1, SamplingParams(temperature=0.0))
sampler.add_request(2, 1, SamplingParams.for_sampler_warmup())
sampler.add_request(0, SamplingParams(temperature=0.0))
sampler.add_request(2, SamplingParams.for_sampler_warmup())
sampling_only = np.array([0], dtype=np.int32)
with_processing = np.array([0, 2], dtype=np.int32)
+17 -7
View File
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import numpy as np
import pytest
import torch
@@ -12,6 +15,7 @@ if not torch.cuda.is_available():
)
from vllm.sampling_params import SamplingParams
from vllm.v1.worker.gpu.sample.logits_processor import LogitsContext
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState
from vllm.v1.worker.gpu.states import RequestState
@@ -75,11 +79,17 @@ def _apply(
idx_mapping_np = idx_mapping.cpu().numpy()
state.apply(
logits,
expanded_idx_mapping,
idx_mapping,
idx_mapping_np,
torch.tensor(input_ids, dtype=torch.int32, device=DEVICE),
torch.tensor(local_pos, dtype=torch.int32, device=DEVICE),
LogitsContext(
expanded_idx_mapping=expanded_idx_mapping,
idx_mapping=idx_mapping,
idx_mapping_np=idx_mapping_np,
expanded_local_pos=torch.tensor(
local_pos, dtype=torch.int32, device=DEVICE
),
input_ids=torch.tensor(input_ids, dtype=torch.int32, device=DEVICE),
pos=torch.zeros(len(input_ids), dtype=torch.int32, device=DEVICE),
seq_lens_upper_bound_np=np.full(1, len(input_ids), dtype=np.int64),
),
)
return logits.cpu()
@@ -190,15 +200,14 @@ def test_v2_greedy_sampling_applies_thinking_budget():
"""Greedy-only requests must not bypass thinking-budget processing."""
req_states = _make_req_states([1, START, 10, 11, 12], prompt_len=1)
sampler = Sampler(
vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()),
max_num_reqs=4,
vocab_size=VOCAB_SIZE,
device=DEVICE,
req_states=req_states,
reasoning_config=MockReasoningConfig(),
)
sampler.add_request(
req_idx=3,
prompt_len=1,
sampling_params=SamplingParams(
temperature=0.0,
thinking_token_budget=3,
@@ -219,6 +228,7 @@ def test_v2_greedy_sampling_applies_thinking_budget():
torch.tensor([4], dtype=torch.int32, device=DEVICE),
input_ids,
torch.tensor([0], dtype=torch.int32, device=DEVICE),
np.full(1, 4, dtype=np.int64),
)
assert out[0, END].item() == pytest.approx(1.0e9)
@@ -0,0 +1,228 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU-only tests for the V2 custom logits processor loader.
Load-time validation is pure Python (importlib + issubclass), so it runs
without CUDA. Pipeline behavior is covered by test_gpu_logits_processors.py.
"""
import subprocess
import sys
from enum import Enum, auto
from types import SimpleNamespace
from typing import Any
import pytest
import torch
import vllm.v1.worker.gpu.sample.logits_processor.loader as loader
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams
from vllm.v1.sample.logits_processor import (
LogitsProcessor as V1LogitsProcessor,
)
from vllm.v1.worker.gpu.sample.logits_processor import (
LogitsContext,
LogitsProcessor,
)
class DummyV2Processor(LogitsProcessor):
"""Records its constructor args so tests can assert how it was built."""
def __init__(self, vllm_config: Any, req_state: Any):
self.ctor_args = (vllm_config, req_state)
def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor:
return logits
class AltV2Processor(DummyV2Processor):
"""Second V2 processor, used to assert plugin/FQCN load ordering."""
class DummyV1Processor(V1LogitsProcessor):
"""A V1-interface processor. Never instantiated: the loader must reject
it at the issubclass check, before any constructor runs."""
def not_a_processor() -> None:
"""FQCN target that resolves but is not a class."""
def _fake_entrypoint(cls: type) -> SimpleNamespace:
return SimpleNamespace(
name=cls.__name__.lower(),
value=f"fake:{cls.__name__}",
load=lambda: cls,
)
@pytest.fixture(autouse=True)
def no_installed_plugins(monkeypatch: pytest.MonkeyPatch):
"""Isolate tests from entry points registered in the environment."""
monkeypatch.setattr(loader, "entry_points", lambda group: [])
class ProcSource(Enum):
CLASS = auto()
FQCN = auto()
ENTRYPOINT = auto()
@pytest.mark.parametrize("source", list(ProcSource))
def test_loads_v2_processors(monkeypatch: pytest.MonkeyPatch, source: ProcSource):
"""A V2 subclass loads via class object, FQCN string, or entrypoint."""
fake_req_states = SimpleNamespace(
device="cpu",
max_num_reqs=4,
vocab_size=128,
all_token_ids=None,
prompt_len=None,
prefill_len=None,
total_len=None,
)
custom: list[Any]
if source is ProcSource.CLASS:
custom = [DummyV2Processor]
elif source is ProcSource.FQCN:
custom = [f"{__name__}:DummyV2Processor"]
else:
monkeypatch.setattr(
loader, "entry_points", lambda group: [_fake_entrypoint(DummyV2Processor)]
)
# Plugins load before user-specified processors.
custom = [f"{__name__}:AltV2Processor"]
procs = loader.build_custom_logits_processors(None, fake_req_states, False, custom)
if source is ProcSource.ENTRYPOINT:
assert [type(p) for p in procs] == [DummyV2Processor, AltV2Processor]
else:
assert [type(p) for p in procs] == [DummyV2Processor]
vllm_config, req_state = procs[0].ctor_args
assert vllm_config is None
assert isinstance(req_state, loader.LogitsProcRequestState)
assert req_state.max_num_reqs == 4
@pytest.mark.parametrize(
("custom", "plugin_cls", "exc_type", "msg_fragment"),
[
pytest.param(
[DummyV1Processor],
None,
ValueError,
"not a subclass",
id="v1-class-object",
),
pytest.param(
[f"{__name__}:DummyV1Processor"],
None,
ValueError,
"not a subclass",
id="v1-fqcn",
),
pytest.param(
[],
DummyV1Processor,
ValueError,
"V1-interface plugins are not supported",
id="v1-entrypoint",
),
pytest.param(
[f"{__name__}:not_a_processor"],
None,
ValueError,
"must be a type",
id="non-type-fqcn",
),
pytest.param(
["no.such.module:Nope"],
None,
RuntimeError,
"no.such.module:Nope",
id="missing-module",
),
pytest.param(
["no_colon_here"],
None,
ValueError,
"Expected format",
id="fqcn-missing-colon",
),
pytest.param(
["too:many:colons"],
None,
ValueError,
"Expected format",
id="fqcn-extra-colons",
),
],
)
def test_rejects_invalid(
monkeypatch: pytest.MonkeyPatch,
custom: list,
plugin_cls: type | None,
exc_type: type[Exception],
msg_fragment: str,
):
"""V1 processors and malformed references fail at load time, not at
sampling time, with a message naming the offending reference."""
if plugin_cls is not None:
monkeypatch.setattr(
loader, "entry_points", lambda group: [_fake_entrypoint(plugin_cls)]
)
with pytest.raises(exc_type, match=msg_fragment):
loader.build_custom_logits_processors(None, None, False, custom)
def test_pooling_model_rejects_custom_logitsprocs():
"""Pooling models reject custom processors instead of ignoring them."""
with pytest.raises(ValueError, match="Pooling models do not support"):
loader.build_custom_logits_processors(None, None, True, [DummyV2Processor])
assert loader.build_custom_logits_processors(None, None, True, []) == []
class ValidatingProcessor(DummyV2Processor):
"""Rejects target_token=-1 via the optional validate_params hook."""
@classmethod
def validate_params(cls, sampling_params: SamplingParams) -> None:
if (sampling_params.extra_args or {}).get("target_token") == -1:
raise ValueError("target_token must not be -1")
def test_validate_params_runs_at_admission():
"""The factory loads classes up front; the returned validator runs
validate_params per request and wraps rejections as VLLMValidationError."""
loader._cached_load_v2_logitsprocs.cache_clear()
validate = loader.build_custom_logits_processors_params_validator(
[ValidatingProcessor]
)
validate(SamplingParams(extra_args={"target_token": 1}))
with pytest.raises(VLLMValidationError, match="target_token"):
validate(SamplingParams(extra_args={"target_token": -1}))
def test_validator_factory_loads_classes_eagerly():
"""A bad reference fails when the validator is built (at startup), not on
the first request."""
loader._cached_load_v2_logitsprocs.cache_clear()
with pytest.raises(RuntimeError, match="no.such.module:Nope"):
loader.build_custom_logits_processors_params_validator(["no.such.module:Nope"])
def test_loader_import_stays_frontend_safe():
"""The frontend process imports the loader to validate params; importing
it must not pull in model-runner side modules."""
code = (
"import sys\n"
"import vllm.v1.worker.gpu.sample.logits_processor.loader\n"
"assert 'vllm.sampling_params' not in sys.modules\n"
"assert 'vllm.v1.sample.logits_processor' not in sys.modules\n"
"assert 'vllm.v1.worker.gpu.states' not in sys.modules\n"
"assert 'vllm.v1.worker.gpu.buffer_utils' not in sys.modules\n"
)
subprocess.run([sys.executable, "-c", code], check=True)