[v1] Support allowed_token_ids in v1 Sampler (#13210)

Signed-off-by: Lu Fang <lufang@fb.com>
This commit is contained in:
Lu Fang
2025-02-22 14:13:05 +08:00
committed by GitHub
parent 8aca27fa11
commit bb78fb318e
7 changed files with 168 additions and 19 deletions
+14
View File
@@ -83,6 +83,19 @@ class Processor:
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
"not enabled!")
def _validate_allowed_token_ids(
self,
params: Union[SamplingParams, PoolingParams],
) -> None:
if not isinstance(params, SamplingParams):
return
if params.allowed_token_ids is None:
return
if not all(0 <= tid < self.model_config.vocab_size
for tid in params.allowed_token_ids):
raise ValueError(
"allowed_token_ids contains out-of-vocab token id")
def process_inputs(
self,
request_id: str,
@@ -100,6 +113,7 @@ class Processor:
self._validate_logprobs(params)
self._validate_lora(lora_request)
self._validate_allowed_token_ids(params)
if arrival_time is None:
arrival_time = time.time()