[Model Runner V2] Support custom logits processors (#56497)
Signed-off-by: David Cheung <d7cheung@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude
Nick Hill
parent
751f6807d9
commit
5302d1fe40
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
@@ -10,6 +13,10 @@ if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for sampler flag tests", allow_module_level=True)
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.worker.gpu.sample.logits_processor import (
|
||||
LogitsContext,
|
||||
LogitsProcessor,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
@@ -23,7 +30,7 @@ class MockReasoningConfig:
|
||||
natural_reasoning_end_token_ids = [91]
|
||||
|
||||
|
||||
def _make_sampler() -> Sampler:
|
||||
def _make_sampler(custom_logits_processors: Sequence[LogitsProcessor] = ()) -> Sampler:
|
||||
req_states = RequestState(
|
||||
max_num_reqs=4,
|
||||
max_model_len=64,
|
||||
@@ -33,11 +40,12 @@ def _make_sampler() -> Sampler:
|
||||
device=DEVICE,
|
||||
)
|
||||
return Sampler(
|
||||
vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()),
|
||||
max_num_reqs=4,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
device=DEVICE,
|
||||
req_states=req_states,
|
||||
reasoning_config=MockReasoningConfig(),
|
||||
custom_logits_processors=custom_logits_processors,
|
||||
)
|
||||
|
||||
|
||||
@@ -65,23 +73,46 @@ def test_logits_processing_cache_matches_request_features(
|
||||
sampling_params: SamplingParams, expected: bool
|
||||
):
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(3, prompt_len=1, sampling_params=sampling_params)
|
||||
sampler.add_request(3, sampling_params=sampling_params)
|
||||
|
||||
assert sampler.needs_logits_processing[3] == expected
|
||||
|
||||
|
||||
def test_logits_processing_cache_is_overwritten_when_slot_is_reused():
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(3, 1, SamplingParams.for_sampler_warmup())
|
||||
sampler.add_request(3, 1, SamplingParams())
|
||||
sampler.add_request(3, SamplingParams.for_sampler_warmup())
|
||||
sampler.add_request(3, SamplingParams())
|
||||
|
||||
assert not sampler.needs_logits_processing[3]
|
||||
|
||||
|
||||
class _GateProcessor(LogitsProcessor):
|
||||
"""Reports a fixed admission decision from add_request()."""
|
||||
|
||||
def __init__(self, admitted: bool):
|
||||
self.admitted = admitted
|
||||
|
||||
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool:
|
||||
return self.admitted
|
||||
|
||||
def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor:
|
||||
raise AssertionError("the pipeline never runs in this test")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("admitted", [True, False])
|
||||
def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
|
||||
"""A custom processor's add_request() return value is OR-ed into the
|
||||
per-request needs_logits_processing flag."""
|
||||
sampler = _make_sampler(custom_logits_processors=[_GateProcessor(admitted)])
|
||||
sampler.add_request(3, SamplingParams())
|
||||
|
||||
assert sampler.needs_logits_processing[3] == admitted
|
||||
|
||||
|
||||
def test_logits_processing_cache_only_checks_active_requests():
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(0, 1, SamplingParams(temperature=0.0))
|
||||
sampler.add_request(2, 1, SamplingParams.for_sampler_warmup())
|
||||
sampler.add_request(0, SamplingParams(temperature=0.0))
|
||||
sampler.add_request(2, SamplingParams.for_sampler_warmup())
|
||||
|
||||
sampling_only = np.array([0], dtype=np.int32)
|
||||
with_processing = np.array([0, 2], dtype=np.int32)
|
||||
|
||||
Reference in New Issue
Block a user