[Model Runner V2] Support custom logits processors (#56497)
Signed-off-by: David Cheung <d7cheung@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude
Nick Hill
parent
751f6807d9
commit
5302d1fe40
@@ -2,7 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from functools import partial
|
||||
from typing import Any, Literal
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -51,6 +52,11 @@ class InputProcessor:
|
||||
self.speculative_config = vllm_config.speculative_config
|
||||
self.structured_outputs_config = vllm_config.structured_outputs_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
# Load the custom logits processor classes once; the returned callable
|
||||
# runs their validate_params hooks per request at admission.
|
||||
self.validate_logits_processors_params = (
|
||||
self._build_logits_processors_params_validator()
|
||||
)
|
||||
|
||||
self.generation_config_fields = model_config.try_get_generation_config()
|
||||
|
||||
@@ -81,6 +87,25 @@ class InputProcessor:
|
||||
def get_tokenizer(self) -> TokenizerLike:
|
||||
return self.renderer.get_tokenizer()
|
||||
|
||||
def _build_logits_processors_params_validator(
|
||||
self,
|
||||
) -> Callable[[SamplingParams], None]:
|
||||
"""Load the custom logits processor classes and return the per-request
|
||||
params validator for the active model runner."""
|
||||
custom_logitsprocs = self.model_config.logits_processors
|
||||
if self.vllm_config.use_v2_model_runner:
|
||||
from vllm.v1.worker.gpu.sample.logits_processor import (
|
||||
build_custom_logits_processors_params_validator,
|
||||
)
|
||||
|
||||
return build_custom_logits_processors_params_validator(custom_logitsprocs)
|
||||
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
validate_logits_processors_parameters,
|
||||
)
|
||||
|
||||
return partial(validate_logits_processors_parameters, custom_logitsprocs)
|
||||
|
||||
def _validate_params(
|
||||
self,
|
||||
params: SamplingParams | PoolingParams,
|
||||
@@ -101,6 +126,8 @@ class InputProcessor:
|
||||
self.tokenizer,
|
||||
)
|
||||
|
||||
self.validate_logits_processors_params(params)
|
||||
|
||||
if self.model_config.return_sampling_mask:
|
||||
if params.temperature <= 0:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user