[Model Runner V2] Support custom logits processors (#56497)
Signed-off-by: David Cheung <d7cheung@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude
Nick Hill
parent
751f6807d9
commit
5302d1fe40
@@ -652,6 +652,15 @@ def test_v2_model_runner_supports_extract_hidden_states():
|
||||
assert config._get_v2_model_runner_unsupported_features() == []
|
||||
|
||||
|
||||
def test_v2_model_runner_supports_custom_logits_processors():
|
||||
config = VllmConfig()
|
||||
config.model_config = cast(
|
||||
ModelConfig, SimpleNamespace(logits_processors=["a.b:C"])
|
||||
)
|
||||
|
||||
assert config._get_v2_model_runner_unsupported_features() == []
|
||||
|
||||
|
||||
def test_dflash2_draft_forces_v2_model_runner():
|
||||
"""A DFlash2 draft must reach the V2 speculator, the only one that runs its
|
||||
candidate selector; on V1 it would draft as DFlash1 without raising."""
|
||||
|
||||
@@ -51,6 +51,14 @@ def test_non_diffusion_models_unaffected():
|
||||
params.verify(MockModelConfig(), None, None, None)
|
||||
|
||||
|
||||
def test_verify_leaves_logits_processors_to_admission():
|
||||
"""verify() is runner-agnostic; LP validation lives in the admission
|
||||
layer, so an unimportable FQCN must not fail verify()."""
|
||||
SamplingParams().verify(
|
||||
MockModelConfig(logits_processors=["no.such:Cls"]), None, None, None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [-(2**63) - 1, 2**64])
|
||||
def test_extra_args_rejects_nested_integer_overflow(value):
|
||||
"""Reject extension values before they reach the engine transport."""
|
||||
|
||||
@@ -85,6 +85,7 @@ def _validate(enable_trace_replay: bool) -> None:
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
validate_logits_processors_params=lambda params: None,
|
||||
)
|
||||
params = SamplingParams(trace_decode_token_ids=[1, 2, 3])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
|
||||
@@ -162,6 +162,9 @@ def test_custom_logitsprocs(monkeypatch, logitproc_source: CustomLogitprocSource
|
||||
"""
|
||||
# Test that logitproc info is passed to workers
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")
|
||||
# These tests exercise the V1-interface logits processor; Model Runner V2
|
||||
# rejects V1-interface processors at load time by design.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
set_random_seed(40)
|
||||
|
||||
# Choose LLM args based on logitproc source
|
||||
@@ -217,6 +220,9 @@ def test_custom_logitsprocs_req(monkeypatch):
|
||||
"""
|
||||
# Test that logitproc info is passed to workers
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")
|
||||
# These tests exercise the V1-interface logits processor; Model Runner V2
|
||||
# rejects V1-interface processors at load time by design.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
set_random_seed(40)
|
||||
_run_test(
|
||||
{"logits_processors": [WrappedPerReqLogitsProcessor]}, logitproc_loaded=True
|
||||
@@ -261,6 +267,9 @@ def test_rejects_custom_logitsprocs(
|
||||
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
# These tests exercise V1-interface rejection and V1 runner internals;
|
||||
# Model Runner V2 rejects V1-interface processors at load time by design.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
set_random_seed(40)
|
||||
|
||||
test_params: dict[str, dict[str, Any]] = {
|
||||
|
||||
@@ -79,6 +79,9 @@ def server(default_server_args, request, monkeypatch):
|
||||
"""
|
||||
# Test that logitproc info is passed to workers
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")
|
||||
# These tests exercise the V1-interface logits processor; Model Runner V2
|
||||
# rejects V1-interface processors at load time by design.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
|
||||
if request.param:
|
||||
# Launch server, append FQCN argument, inject dummy logitproc module
|
||||
|
||||
@@ -14,6 +14,10 @@ if not torch.cuda.is_available():
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.worker.gpu.sample.bad_words import BadWordsState
|
||||
from vllm.v1.worker.gpu.sample.logits_processor import (
|
||||
LogitsContext,
|
||||
LogitsProcRequestState,
|
||||
)
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
@@ -47,7 +51,7 @@ def _make_state(bad_words_token_ids: list[list[int]]) -> tuple[BadWordsState, in
|
||||
req_states.apply_staged_writes()
|
||||
|
||||
req_idx = req_states.req_id_to_index["req"]
|
||||
state = BadWordsState(req_states)
|
||||
state = BadWordsState(None, LogitsProcRequestState.from_request_state(req_states))
|
||||
state.add_request(req_idx, SamplingParams(_bad_words_token_ids=bad_words_token_ids))
|
||||
state.apply_staged_writes()
|
||||
return state, req_idx
|
||||
@@ -61,16 +65,55 @@ def _apply(bad_words_token_ids: list[list[int]]) -> torch.Tensor:
|
||||
expanded_idx_mapping = torch.tensor(
|
||||
[req_idx] * num_logits, dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
state.apply_bad_words(
|
||||
state.apply(
|
||||
logits,
|
||||
expanded_idx_mapping,
|
||||
idx_mapping_np,
|
||||
torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE),
|
||||
torch.tensor(LOCAL_POS, dtype=torch.int32, device=DEVICE),
|
||||
LogitsContext(
|
||||
expanded_idx_mapping=expanded_idx_mapping,
|
||||
idx_mapping=torch.tensor([req_idx], dtype=torch.int32, device=DEVICE),
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
expanded_local_pos=torch.tensor(
|
||||
LOCAL_POS, dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
input_ids=torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE),
|
||||
pos=torch.zeros(num_logits, dtype=torch.int32, device=DEVICE),
|
||||
seq_lens_upper_bound_np=np.full(1, num_logits, dtype=np.int64),
|
||||
),
|
||||
)
|
||||
return logits.cpu()
|
||||
|
||||
|
||||
def test_v2_bad_words_slot_reuse_clears_state():
|
||||
"""A slot recycled from a bad-words request must not keep masking.
|
||||
|
||||
There is no removal hook by design; safety comes from add_request()
|
||||
overwriting the slot's state unconditionally.
|
||||
"""
|
||||
state, req_idx = _make_state([[10, 11, 60]])
|
||||
state.add_request(req_idx, SamplingParams())
|
||||
state.apply_staged_writes()
|
||||
assert int(state.num_bad_words.np[req_idx]) == 0
|
||||
|
||||
num_logits = len(INPUT_IDS)
|
||||
idx_mapping = torch.tensor([req_idx], dtype=torch.int32, device=DEVICE)
|
||||
out = state.apply(
|
||||
torch.zeros((num_logits, VOCAB_SIZE), device=DEVICE),
|
||||
LogitsContext(
|
||||
expanded_idx_mapping=torch.tensor(
|
||||
[req_idx] * num_logits, dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
idx_mapping=idx_mapping,
|
||||
idx_mapping_np=np.array([req_idx], dtype=np.intp),
|
||||
expanded_local_pos=torch.tensor(
|
||||
LOCAL_POS, dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
input_ids=torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE),
|
||||
pos=torch.zeros(num_logits, dtype=torch.int32, device=DEVICE),
|
||||
seq_lens_upper_bound_np=np.full(1, num_logits, dtype=np.int64),
|
||||
),
|
||||
)
|
||||
assert not torch.isinf(out).any()
|
||||
|
||||
|
||||
def test_v2_bad_words_prefix_inside_draft_tokens():
|
||||
"""A prefix matching entirely within the draft tokens must mask the bad
|
||||
word's last token at the draft position that completes the prefix."""
|
||||
|
||||
@@ -183,6 +183,12 @@ def test_local_batch_partition(tp_size: int, seed: int):
|
||||
).all()
|
||||
assert (_np(local.logits_indices) == _np(batch.logits_indices)[rows]).all()
|
||||
assert (_np(local.seq_lens) == _np(batch.seq_lens)[owned]).all()
|
||||
# Indexed by batch position, like seq_lens: a consumer holding local row
|
||||
# indices reads another request's length if this is not localised too.
|
||||
assert (
|
||||
_np(local.seq_lens_cpu_upper_bound)
|
||||
== _np(batch.seq_lens_cpu_upper_bound)[owned]
|
||||
).all()
|
||||
assert batch.num_draft_tokens_per_req is not None
|
||||
assert local.num_draft_tokens_per_req is not None
|
||||
assert (
|
||||
|
||||
@@ -14,6 +14,11 @@ if not torch.cuda.is_available():
|
||||
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState
|
||||
from vllm.v1.worker.gpu.sample.logits_processor import (
|
||||
LogitsContext,
|
||||
LogitsProcRequestState,
|
||||
)
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
VOCAB_SIZE = 128
|
||||
@@ -45,17 +50,35 @@ def _only_stop_token_left(num_rows: int) -> torch.Tensor:
|
||||
|
||||
def _apply(logits: torch.Tensor, structured: list[bool]) -> torch.Tensor:
|
||||
"""Run the fused bias kernel with one logits row per request."""
|
||||
state = LogitBiasState(max_num_reqs=4, device=DEVICE)
|
||||
req_states = RequestState(
|
||||
max_num_reqs=4,
|
||||
max_model_len=64,
|
||||
max_num_batched_tokens=16,
|
||||
num_speculative_steps=1,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
device=DEVICE,
|
||||
)
|
||||
# add_request() reads prompt_len per slot; every slot here uses the same one.
|
||||
req_states.prompt_len.np[:] = PROMPT_LEN
|
||||
|
||||
state = LogitBiasState(None, LogitsProcRequestState.from_request_state(req_states))
|
||||
for req_idx, is_structured in enumerate(structured):
|
||||
state.add_request(req_idx, PROMPT_LEN, _params(is_structured))
|
||||
state.add_request(req_idx, _params(is_structured))
|
||||
state.apply_staged_writes()
|
||||
|
||||
num_reqs = len(structured)
|
||||
state.apply_logit_bias(
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE)
|
||||
state.apply(
|
||||
logits,
|
||||
torch.arange(num_reqs, dtype=torch.int32, device=DEVICE),
|
||||
np.arange(num_reqs, dtype=np.intp),
|
||||
torch.full((num_reqs,), POS, dtype=torch.int32, device=DEVICE),
|
||||
LogitsContext(
|
||||
expanded_idx_mapping=idx_mapping,
|
||||
idx_mapping=idx_mapping,
|
||||
idx_mapping_np=np.arange(num_reqs, dtype=np.intp),
|
||||
expanded_local_pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
|
||||
input_ids=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
|
||||
pos=torch.full((num_reqs,), POS, dtype=torch.int32, device=DEVICE),
|
||||
seq_lens_upper_bound_np=np.full(num_reqs, POS + 1, dtype=np.int64),
|
||||
),
|
||||
)
|
||||
return logits.cpu()
|
||||
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytest.importorskip("triton")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip(
|
||||
"CUDA required for Model Runner V2 logits processor tests",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.worker.gpu.sample.logits_processor import (
|
||||
LogitsContext,
|
||||
LogitsProcessor,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
VOCAB_SIZE = 128
|
||||
TARGET_TOKEN = 7
|
||||
BIAS = 5.0
|
||||
BOOST = 100.0
|
||||
TEMPERATURE = 2.0
|
||||
|
||||
|
||||
class MockReasoningConfig:
|
||||
reasoning_start_token_ids = [90]
|
||||
reasoning_end_token_ids = [91]
|
||||
natural_reasoning_end_token_ids = [91]
|
||||
|
||||
|
||||
class RecordingProcessor(LogitsProcessor):
|
||||
"""Adds BOOST to TARGET_TOKEN and records every lifecycle callback."""
|
||||
|
||||
def __init__(self, vllm_config: Any, req_state: Any):
|
||||
self.events: list[str] = []
|
||||
self.seen_logits: torch.Tensor | None = None
|
||||
|
||||
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool:
|
||||
self.events.append(f"add_request:{req_idx}")
|
||||
return True
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
self.events.append("apply_staged_writes")
|
||||
|
||||
def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor:
|
||||
self.events.append("apply")
|
||||
self.seen_logits = logits.clone()
|
||||
logits[:, TARGET_TOKEN] += BOOST
|
||||
return logits
|
||||
|
||||
|
||||
def _make_sampler(processor: RecordingProcessor) -> Sampler:
|
||||
req_states = RequestState(
|
||||
max_num_reqs=4,
|
||||
max_model_len=64,
|
||||
max_num_batched_tokens=16,
|
||||
num_speculative_steps=1,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
device=DEVICE,
|
||||
)
|
||||
return Sampler(
|
||||
vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()),
|
||||
max_num_reqs=4,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
device=DEVICE,
|
||||
req_states=req_states,
|
||||
custom_logits_processors=[processor],
|
||||
)
|
||||
|
||||
|
||||
def test_custom_processor_lifecycle_and_pipeline_position():
|
||||
"""A custom processor's hooks fire in order, it runs after the built-in
|
||||
bias stage, and before temperature scaling."""
|
||||
proc = RecordingProcessor(None, None)
|
||||
sampler = _make_sampler(proc)
|
||||
sampler.add_request(
|
||||
0,
|
||||
SamplingParams(logit_bias={TARGET_TOKEN: BIAS}, temperature=TEMPERATURE),
|
||||
)
|
||||
sampler.apply_staged_writes()
|
||||
|
||||
num_reqs = 1
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE)
|
||||
raw = torch.zeros(num_reqs, VOCAB_SIZE, device=DEVICE)
|
||||
out = sampler.apply_sampling_params(
|
||||
raw,
|
||||
expanded_idx_mapping=idx_mapping,
|
||||
idx_mapping=idx_mapping,
|
||||
idx_mapping_np=np.arange(num_reqs, dtype=np.intp),
|
||||
pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
|
||||
input_ids=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
|
||||
expanded_local_pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE),
|
||||
seq_lens_upper_bound_np=np.ones(num_reqs, dtype=np.int64),
|
||||
)
|
||||
|
||||
assert proc.events == [
|
||||
"add_request:0",
|
||||
"apply_staged_writes",
|
||||
"apply",
|
||||
]
|
||||
# The processor sees the built-in bias applied but no temperature scaling.
|
||||
assert proc.seen_logits is not None
|
||||
assert proc.seen_logits[0, TARGET_TOKEN].item() == BIAS
|
||||
# Its own modification is then temperature-scaled by the sampler.
|
||||
assert out[0, TARGET_TOKEN].item() == (BIAS + BOOST) / TEMPERATURE
|
||||
@@ -56,7 +56,16 @@ class FakeEplbState:
|
||||
def _make_runner(**overrides: Any) -> Any:
|
||||
runner: Any = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
|
||||
runner.device = torch.device("cpu")
|
||||
runner.model_config = SimpleNamespace(model="test-model")
|
||||
runner.model_config = SimpleNamespace(model="test-model", logits_processors=None)
|
||||
runner.req_states = SimpleNamespace(
|
||||
device=runner.device,
|
||||
max_num_reqs=8,
|
||||
vocab_size=0,
|
||||
all_token_ids=None,
|
||||
prompt_len=None,
|
||||
prefill_len=None,
|
||||
total_len=None,
|
||||
)
|
||||
runner.load_config = SimpleNamespace(load_format="hf")
|
||||
runner.parallel_config = SimpleNamespace(
|
||||
enable_eplb=True,
|
||||
|
||||
@@ -47,6 +47,9 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
|
||||
)
|
||||
).to(device),
|
||||
seq_lens_cpu_upper_bound=torch.from_numpy(
|
||||
np.array([10, 20, 30, 40], dtype=np.int32)
|
||||
),
|
||||
)
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
@@ -10,6 +13,10 @@ if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for sampler flag tests", allow_module_level=True)
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.worker.gpu.sample.logits_processor import (
|
||||
LogitsContext,
|
||||
LogitsProcessor,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
@@ -23,7 +30,7 @@ class MockReasoningConfig:
|
||||
natural_reasoning_end_token_ids = [91]
|
||||
|
||||
|
||||
def _make_sampler() -> Sampler:
|
||||
def _make_sampler(custom_logits_processors: Sequence[LogitsProcessor] = ()) -> Sampler:
|
||||
req_states = RequestState(
|
||||
max_num_reqs=4,
|
||||
max_model_len=64,
|
||||
@@ -33,11 +40,12 @@ def _make_sampler() -> Sampler:
|
||||
device=DEVICE,
|
||||
)
|
||||
return Sampler(
|
||||
vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()),
|
||||
max_num_reqs=4,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
device=DEVICE,
|
||||
req_states=req_states,
|
||||
reasoning_config=MockReasoningConfig(),
|
||||
custom_logits_processors=custom_logits_processors,
|
||||
)
|
||||
|
||||
|
||||
@@ -65,23 +73,46 @@ def test_logits_processing_cache_matches_request_features(
|
||||
sampling_params: SamplingParams, expected: bool
|
||||
):
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(3, prompt_len=1, sampling_params=sampling_params)
|
||||
sampler.add_request(3, sampling_params=sampling_params)
|
||||
|
||||
assert sampler.needs_logits_processing[3] == expected
|
||||
|
||||
|
||||
def test_logits_processing_cache_is_overwritten_when_slot_is_reused():
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(3, 1, SamplingParams.for_sampler_warmup())
|
||||
sampler.add_request(3, 1, SamplingParams())
|
||||
sampler.add_request(3, SamplingParams.for_sampler_warmup())
|
||||
sampler.add_request(3, SamplingParams())
|
||||
|
||||
assert not sampler.needs_logits_processing[3]
|
||||
|
||||
|
||||
class _GateProcessor(LogitsProcessor):
|
||||
"""Reports a fixed admission decision from add_request()."""
|
||||
|
||||
def __init__(self, admitted: bool):
|
||||
self.admitted = admitted
|
||||
|
||||
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool:
|
||||
return self.admitted
|
||||
|
||||
def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor:
|
||||
raise AssertionError("the pipeline never runs in this test")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("admitted", [True, False])
|
||||
def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
|
||||
"""A custom processor's add_request() return value is OR-ed into the
|
||||
per-request needs_logits_processing flag."""
|
||||
sampler = _make_sampler(custom_logits_processors=[_GateProcessor(admitted)])
|
||||
sampler.add_request(3, SamplingParams())
|
||||
|
||||
assert sampler.needs_logits_processing[3] == admitted
|
||||
|
||||
|
||||
def test_logits_processing_cache_only_checks_active_requests():
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(0, 1, SamplingParams(temperature=0.0))
|
||||
sampler.add_request(2, 1, SamplingParams.for_sampler_warmup())
|
||||
sampler.add_request(0, SamplingParams(temperature=0.0))
|
||||
sampler.add_request(2, SamplingParams.for_sampler_warmup())
|
||||
|
||||
sampling_only = np.array([0], dtype=np.int32)
|
||||
with_processing = np.array([0, 2], dtype=np.int32)
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -12,6 +15,7 @@ if not torch.cuda.is_available():
|
||||
)
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.worker.gpu.sample.logits_processor import LogitsContext
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
@@ -75,11 +79,17 @@ def _apply(
|
||||
idx_mapping_np = idx_mapping.cpu().numpy()
|
||||
state.apply(
|
||||
logits,
|
||||
expanded_idx_mapping,
|
||||
idx_mapping,
|
||||
idx_mapping_np,
|
||||
torch.tensor(input_ids, dtype=torch.int32, device=DEVICE),
|
||||
torch.tensor(local_pos, dtype=torch.int32, device=DEVICE),
|
||||
LogitsContext(
|
||||
expanded_idx_mapping=expanded_idx_mapping,
|
||||
idx_mapping=idx_mapping,
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
expanded_local_pos=torch.tensor(
|
||||
local_pos, dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
input_ids=torch.tensor(input_ids, dtype=torch.int32, device=DEVICE),
|
||||
pos=torch.zeros(len(input_ids), dtype=torch.int32, device=DEVICE),
|
||||
seq_lens_upper_bound_np=np.full(1, len(input_ids), dtype=np.int64),
|
||||
),
|
||||
)
|
||||
return logits.cpu()
|
||||
|
||||
@@ -190,15 +200,14 @@ def test_v2_greedy_sampling_applies_thinking_budget():
|
||||
"""Greedy-only requests must not bypass thinking-budget processing."""
|
||||
req_states = _make_req_states([1, START, 10, 11, 12], prompt_len=1)
|
||||
sampler = Sampler(
|
||||
vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()),
|
||||
max_num_reqs=4,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
device=DEVICE,
|
||||
req_states=req_states,
|
||||
reasoning_config=MockReasoningConfig(),
|
||||
)
|
||||
sampler.add_request(
|
||||
req_idx=3,
|
||||
prompt_len=1,
|
||||
sampling_params=SamplingParams(
|
||||
temperature=0.0,
|
||||
thinking_token_budget=3,
|
||||
@@ -219,6 +228,7 @@ def test_v2_greedy_sampling_applies_thinking_budget():
|
||||
torch.tensor([4], dtype=torch.int32, device=DEVICE),
|
||||
input_ids,
|
||||
torch.tensor([0], dtype=torch.int32, device=DEVICE),
|
||||
np.full(1, 4, dtype=np.int64),
|
||||
)
|
||||
|
||||
assert out[0, END].item() == pytest.approx(1.0e9)
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only tests for the V2 custom logits processor loader.
|
||||
|
||||
Load-time validation is pure Python (importlib + issubclass), so it runs
|
||||
without CUDA. Pipeline behavior is covered by test_gpu_logits_processors.py.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from enum import Enum, auto
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.worker.gpu.sample.logits_processor.loader as loader
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
LogitsProcessor as V1LogitsProcessor,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.logits_processor import (
|
||||
LogitsContext,
|
||||
LogitsProcessor,
|
||||
)
|
||||
|
||||
|
||||
class DummyV2Processor(LogitsProcessor):
|
||||
"""Records its constructor args so tests can assert how it was built."""
|
||||
|
||||
def __init__(self, vllm_config: Any, req_state: Any):
|
||||
self.ctor_args = (vllm_config, req_state)
|
||||
|
||||
def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor:
|
||||
return logits
|
||||
|
||||
|
||||
class AltV2Processor(DummyV2Processor):
|
||||
"""Second V2 processor, used to assert plugin/FQCN load ordering."""
|
||||
|
||||
|
||||
class DummyV1Processor(V1LogitsProcessor):
|
||||
"""A V1-interface processor. Never instantiated: the loader must reject
|
||||
it at the issubclass check, before any constructor runs."""
|
||||
|
||||
|
||||
def not_a_processor() -> None:
|
||||
"""FQCN target that resolves but is not a class."""
|
||||
|
||||
|
||||
def _fake_entrypoint(cls: type) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
name=cls.__name__.lower(),
|
||||
value=f"fake:{cls.__name__}",
|
||||
load=lambda: cls,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def no_installed_plugins(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Isolate tests from entry points registered in the environment."""
|
||||
monkeypatch.setattr(loader, "entry_points", lambda group: [])
|
||||
|
||||
|
||||
class ProcSource(Enum):
|
||||
CLASS = auto()
|
||||
FQCN = auto()
|
||||
ENTRYPOINT = auto()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", list(ProcSource))
|
||||
def test_loads_v2_processors(monkeypatch: pytest.MonkeyPatch, source: ProcSource):
|
||||
"""A V2 subclass loads via class object, FQCN string, or entrypoint."""
|
||||
fake_req_states = SimpleNamespace(
|
||||
device="cpu",
|
||||
max_num_reqs=4,
|
||||
vocab_size=128,
|
||||
all_token_ids=None,
|
||||
prompt_len=None,
|
||||
prefill_len=None,
|
||||
total_len=None,
|
||||
)
|
||||
custom: list[Any]
|
||||
if source is ProcSource.CLASS:
|
||||
custom = [DummyV2Processor]
|
||||
elif source is ProcSource.FQCN:
|
||||
custom = [f"{__name__}:DummyV2Processor"]
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
loader, "entry_points", lambda group: [_fake_entrypoint(DummyV2Processor)]
|
||||
)
|
||||
# Plugins load before user-specified processors.
|
||||
custom = [f"{__name__}:AltV2Processor"]
|
||||
|
||||
procs = loader.build_custom_logits_processors(None, fake_req_states, False, custom)
|
||||
|
||||
if source is ProcSource.ENTRYPOINT:
|
||||
assert [type(p) for p in procs] == [DummyV2Processor, AltV2Processor]
|
||||
else:
|
||||
assert [type(p) for p in procs] == [DummyV2Processor]
|
||||
vllm_config, req_state = procs[0].ctor_args
|
||||
assert vllm_config is None
|
||||
assert isinstance(req_state, loader.LogitsProcRequestState)
|
||||
assert req_state.max_num_reqs == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("custom", "plugin_cls", "exc_type", "msg_fragment"),
|
||||
[
|
||||
pytest.param(
|
||||
[DummyV1Processor],
|
||||
None,
|
||||
ValueError,
|
||||
"not a subclass",
|
||||
id="v1-class-object",
|
||||
),
|
||||
pytest.param(
|
||||
[f"{__name__}:DummyV1Processor"],
|
||||
None,
|
||||
ValueError,
|
||||
"not a subclass",
|
||||
id="v1-fqcn",
|
||||
),
|
||||
pytest.param(
|
||||
[],
|
||||
DummyV1Processor,
|
||||
ValueError,
|
||||
"V1-interface plugins are not supported",
|
||||
id="v1-entrypoint",
|
||||
),
|
||||
pytest.param(
|
||||
[f"{__name__}:not_a_processor"],
|
||||
None,
|
||||
ValueError,
|
||||
"must be a type",
|
||||
id="non-type-fqcn",
|
||||
),
|
||||
pytest.param(
|
||||
["no.such.module:Nope"],
|
||||
None,
|
||||
RuntimeError,
|
||||
"no.such.module:Nope",
|
||||
id="missing-module",
|
||||
),
|
||||
pytest.param(
|
||||
["no_colon_here"],
|
||||
None,
|
||||
ValueError,
|
||||
"Expected format",
|
||||
id="fqcn-missing-colon",
|
||||
),
|
||||
pytest.param(
|
||||
["too:many:colons"],
|
||||
None,
|
||||
ValueError,
|
||||
"Expected format",
|
||||
id="fqcn-extra-colons",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
custom: list,
|
||||
plugin_cls: type | None,
|
||||
exc_type: type[Exception],
|
||||
msg_fragment: str,
|
||||
):
|
||||
"""V1 processors and malformed references fail at load time, not at
|
||||
sampling time, with a message naming the offending reference."""
|
||||
if plugin_cls is not None:
|
||||
monkeypatch.setattr(
|
||||
loader, "entry_points", lambda group: [_fake_entrypoint(plugin_cls)]
|
||||
)
|
||||
|
||||
with pytest.raises(exc_type, match=msg_fragment):
|
||||
loader.build_custom_logits_processors(None, None, False, custom)
|
||||
|
||||
|
||||
def test_pooling_model_rejects_custom_logitsprocs():
|
||||
"""Pooling models reject custom processors instead of ignoring them."""
|
||||
with pytest.raises(ValueError, match="Pooling models do not support"):
|
||||
loader.build_custom_logits_processors(None, None, True, [DummyV2Processor])
|
||||
assert loader.build_custom_logits_processors(None, None, True, []) == []
|
||||
|
||||
|
||||
class ValidatingProcessor(DummyV2Processor):
|
||||
"""Rejects target_token=-1 via the optional validate_params hook."""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, sampling_params: SamplingParams) -> None:
|
||||
if (sampling_params.extra_args or {}).get("target_token") == -1:
|
||||
raise ValueError("target_token must not be -1")
|
||||
|
||||
|
||||
def test_validate_params_runs_at_admission():
|
||||
"""The factory loads classes up front; the returned validator runs
|
||||
validate_params per request and wraps rejections as VLLMValidationError."""
|
||||
loader._cached_load_v2_logitsprocs.cache_clear()
|
||||
validate = loader.build_custom_logits_processors_params_validator(
|
||||
[ValidatingProcessor]
|
||||
)
|
||||
validate(SamplingParams(extra_args={"target_token": 1}))
|
||||
with pytest.raises(VLLMValidationError, match="target_token"):
|
||||
validate(SamplingParams(extra_args={"target_token": -1}))
|
||||
|
||||
|
||||
def test_validator_factory_loads_classes_eagerly():
|
||||
"""A bad reference fails when the validator is built (at startup), not on
|
||||
the first request."""
|
||||
loader._cached_load_v2_logitsprocs.cache_clear()
|
||||
with pytest.raises(RuntimeError, match="no.such.module:Nope"):
|
||||
loader.build_custom_logits_processors_params_validator(["no.such.module:Nope"])
|
||||
|
||||
|
||||
def test_loader_import_stays_frontend_safe():
|
||||
"""The frontend process imports the loader to validate params; importing
|
||||
it must not pull in model-runner side modules."""
|
||||
code = (
|
||||
"import sys\n"
|
||||
"import vllm.v1.worker.gpu.sample.logits_processor.loader\n"
|
||||
"assert 'vllm.sampling_params' not in sys.modules\n"
|
||||
"assert 'vllm.v1.sample.logits_processor' not in sys.modules\n"
|
||||
"assert 'vllm.v1.worker.gpu.states' not in sys.modules\n"
|
||||
"assert 'vllm.v1.worker.gpu.buffer_utils' not in sys.modules\n"
|
||||
)
|
||||
subprocess.run([sys.executable, "-c", code], check=True)
|
||||
Reference in New Issue
Block a user