[V1] LogitsProcessor programming model (#16728)

Signed-off-by: Nick Hill <nhill@redhat.com>
Signed-off-by: Andrew Feldman <afeldman@neuralmagic.com>
Signed-off-by: Andrew Feldman <afeldman@redhat.com>
Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
afeldman-nm
2025-07-02 09:10:42 -07:00
committed by GitHub
co-authored by Nick Hill
parent c1909e7e8c
commit 48fb076cbc
13 changed files with 1401 additions and 393 deletions
+5 -6
View File
@@ -6,6 +6,8 @@ from typing import Optional
import torch
from vllm.v1.sample.logits_processor import LogitsProcessorManager
@dataclass
class SamplingMetadata:
@@ -16,7 +18,6 @@ class SamplingMetadata:
top_p: Optional[torch.Tensor]
top_k: Optional[torch.Tensor]
min_p: Optional[torch.Tensor]
generators: dict[int, torch.Generator]
@@ -31,14 +32,12 @@ class SamplingMetadata:
output_token_ids: list[list[int]]
# req_index -> (min_tokens, stop_token_ids)
min_tokens: dict[int, tuple[int, set[int]]]
logit_bias: list[Optional[dict[int, float]]]
# `allowed_token_ids_mask` is a 2D bool tensor of shape (max batch size,
# vocab size).
allowed_token_ids_mask: Optional[torch.Tensor]
# req_index -> bad_words_token_ids
bad_words_token_ids: dict[int, list[list[int]]]
# Loaded logits processors
logitsprocs: LogitsProcessorManager