Bound generation inputs before expensive work (#51447)

Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Clinton Thomas
2026-08-11 22:03:50 +08:00
committed by GitHub
co-authored by Lucas Bourtoule mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5b184f775a
commit b64a2708b0
11 changed files with 421 additions and 37 deletions
+13 -11
View File
@@ -3,14 +3,12 @@
import numpy as np
import torch
import vllm.envs as envs
from vllm.sampling_params import SamplingParams
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
from vllm.v1.worker.gpu.states import RequestState
MAX_BAD_WORDS_TOTAL_TOKENS = 1024 # Max total tokens for all bad words per request
MAX_NUM_BAD_WORDS = 128 # Max number of bad words per request
class BadWordsState:
def __init__(self, req_states: RequestState):
@@ -18,15 +16,17 @@ class BadWordsState:
self.max_num_reqs = req_states.max_num_reqs
self.device = req_states.device
# flattened bad word tokens: [max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS]
max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
# flattened bad word tokens: [max_num_reqs, VLLM_MAX_BAD_WORDS_TOTAL_TOKENS]
self.bad_word_token_ids = StagedWriteTensor(
(self.max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS),
(self.max_num_reqs, max_total_tokens),
dtype=torch.int32,
device=self.device,
)
# cumulative offsets of bad words: [max_num_reqs, MAX_NUM_BAD_WORDS + 1]
# cumulative offsets of bad words: [max_num_reqs, VLLM_MAX_NUM_BAD_WORDS + 1]
self.bad_word_offsets = StagedWriteTensor(
(self.max_num_reqs, MAX_NUM_BAD_WORDS + 1),
(self.max_num_reqs, max_num_bad_words + 1),
dtype=torch.int32,
device=self.device,
)
@@ -40,10 +40,11 @@ class BadWordsState:
return
num_bad_words = len(bad_words_token_ids)
if num_bad_words > MAX_NUM_BAD_WORDS:
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
if num_bad_words > max_num_bad_words:
raise ValueError(
f"Too many bad words: {num_bad_words}. "
f"The max number is {MAX_NUM_BAD_WORDS}."
f"The max number is {max_num_bad_words}."
)
# Flatten bad words and compute offsets
@@ -53,10 +54,11 @@ class BadWordsState:
flattened_tokens.extend(bad_word)
offsets.append(len(flattened_tokens))
if len(flattened_tokens) > MAX_BAD_WORDS_TOTAL_TOKENS:
max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS
if len(flattened_tokens) > max_total_tokens:
raise ValueError(
f"Too many total bad word tokens: {len(flattened_tokens)}. "
f"The max is {MAX_BAD_WORDS_TOTAL_TOKENS}."
f"The max is {max_total_tokens}."
)
# Stage writes