Bound generation inputs before expensive work (#51447)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5b184f775a
commit
b64a2708b0
@@ -3,14 +3,12 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
MAX_BAD_WORDS_TOTAL_TOKENS = 1024 # Max total tokens for all bad words per request
|
||||
MAX_NUM_BAD_WORDS = 128 # Max number of bad words per request
|
||||
|
||||
|
||||
class BadWordsState:
|
||||
def __init__(self, req_states: RequestState):
|
||||
@@ -18,15 +16,17 @@ class BadWordsState:
|
||||
self.max_num_reqs = req_states.max_num_reqs
|
||||
self.device = req_states.device
|
||||
|
||||
# flattened bad word tokens: [max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS]
|
||||
max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS
|
||||
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
|
||||
# flattened bad word tokens: [max_num_reqs, VLLM_MAX_BAD_WORDS_TOTAL_TOKENS]
|
||||
self.bad_word_token_ids = StagedWriteTensor(
|
||||
(self.max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS),
|
||||
(self.max_num_reqs, max_total_tokens),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
# cumulative offsets of bad words: [max_num_reqs, MAX_NUM_BAD_WORDS + 1]
|
||||
# cumulative offsets of bad words: [max_num_reqs, VLLM_MAX_NUM_BAD_WORDS + 1]
|
||||
self.bad_word_offsets = StagedWriteTensor(
|
||||
(self.max_num_reqs, MAX_NUM_BAD_WORDS + 1),
|
||||
(self.max_num_reqs, max_num_bad_words + 1),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
@@ -40,10 +40,11 @@ class BadWordsState:
|
||||
return
|
||||
|
||||
num_bad_words = len(bad_words_token_ids)
|
||||
if num_bad_words > MAX_NUM_BAD_WORDS:
|
||||
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
|
||||
if num_bad_words > max_num_bad_words:
|
||||
raise ValueError(
|
||||
f"Too many bad words: {num_bad_words}. "
|
||||
f"The max number is {MAX_NUM_BAD_WORDS}."
|
||||
f"The max number is {max_num_bad_words}."
|
||||
)
|
||||
|
||||
# Flatten bad words and compute offsets
|
||||
@@ -53,10 +54,11 @@ class BadWordsState:
|
||||
flattened_tokens.extend(bad_word)
|
||||
offsets.append(len(flattened_tokens))
|
||||
|
||||
if len(flattened_tokens) > MAX_BAD_WORDS_TOTAL_TOKENS:
|
||||
max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS
|
||||
if len(flattened_tokens) > max_total_tokens:
|
||||
raise ValueError(
|
||||
f"Too many total bad word tokens: {len(flattened_tokens)}. "
|
||||
f"The max is {MAX_BAD_WORDS_TOTAL_TOKENS}."
|
||||
f"The max is {max_total_tokens}."
|
||||
)
|
||||
|
||||
# Stage writes
|
||||
|
||||
Reference in New Issue
Block a user