[Refactor] Pass Renderer to Input Processor (#34329)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -356,8 +356,9 @@ class LLM:
|
||||
self.supported_tasks = supported_tasks
|
||||
|
||||
self.model_config = self.llm_engine.model_config
|
||||
self.input_processor = self.llm_engine.input_processor
|
||||
self.renderer = self.llm_engine.renderer
|
||||
self.io_processor = self.llm_engine.io_processor
|
||||
self.input_processor = self.llm_engine.input_processor
|
||||
|
||||
# Cache for __repr__ to avoid repeated collective_rpc calls
|
||||
self._cached_repr: str | None = None
|
||||
@@ -816,7 +817,7 @@ class LLM:
|
||||
A list of `TokensPrompts` objects containing the tokenized prompt
|
||||
after chat template interpolation, and the raw multi-modal inputs.
|
||||
"""
|
||||
renderer = self.llm_engine.renderer
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
parsed_prompts = [
|
||||
@@ -858,7 +859,7 @@ class LLM:
|
||||
A list of `TokensPrompts` objects containing the tokenized prompt
|
||||
after chat template interpolation, and the raw multi-modal inputs.
|
||||
"""
|
||||
renderer = self.llm_engine.renderer
|
||||
renderer = self.renderer
|
||||
|
||||
chat_params = ChatParams(
|
||||
chat_template=chat_template,
|
||||
|
||||
@@ -239,8 +239,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
try:
|
||||
renderer = self.engine_client.renderer
|
||||
tokenizer = renderer.tokenizer
|
||||
tokenizer = self.renderer.tokenizer
|
||||
|
||||
tool_parser = self.tool_parser
|
||||
|
||||
@@ -375,6 +374,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
max_model_len = self.model_config.max_model_len
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||
try:
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
@@ -387,7 +387,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
)
|
||||
|
||||
max_tokens = get_max_tokens(
|
||||
self.max_model_len,
|
||||
max_model_len,
|
||||
request.max_completion_tokens
|
||||
if request.max_completion_tokens is not None
|
||||
else request.max_tokens,
|
||||
|
||||
@@ -157,13 +157,14 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
max_model_len = self.model_config.max_model_len
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||
try:
|
||||
for i, engine_prompt in enumerate(engine_prompts):
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
max_tokens = get_max_tokens(
|
||||
self.max_model_len,
|
||||
max_model_len,
|
||||
request.max_tokens,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self.default_sampling_params,
|
||||
|
||||
@@ -242,11 +242,10 @@ class OpenAIServing:
|
||||
|
||||
self.log_error_stack = log_error_stack
|
||||
|
||||
self.input_processor = self.models.input_processor
|
||||
self.io_processor = self.models.io_processor
|
||||
self.renderer = self.models.renderer
|
||||
self.model_config = self.models.model_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
self.model_config = engine_client.model_config
|
||||
self.renderer = engine_client.renderer
|
||||
self.io_processor = engine_client.io_processor
|
||||
self.input_processor = engine_client.input_processor
|
||||
|
||||
async def beam_search(
|
||||
self,
|
||||
@@ -537,7 +536,7 @@ class OpenAIServing:
|
||||
|
||||
if (
|
||||
truncate_prompt_tokens is not None
|
||||
and truncate_prompt_tokens > self.max_model_len
|
||||
and truncate_prompt_tokens > self.model_config.max_model_len
|
||||
):
|
||||
return self.create_error_response(
|
||||
"truncate_prompt_tokens value is "
|
||||
@@ -844,6 +843,7 @@ class OpenAIServing:
|
||||
input_text: str,
|
||||
) -> TokensPrompt:
|
||||
token_num = len(input_ids)
|
||||
max_model_len = self.model_config.max_model_len
|
||||
|
||||
# Note: EmbeddingRequest, ClassificationRequest,
|
||||
# and ScoreRequest doesn't have max_tokens
|
||||
@@ -862,7 +862,7 @@ class OpenAIServing:
|
||||
):
|
||||
# Note: input length can be up to the entire model context length
|
||||
# since these requests don't generate tokens.
|
||||
if token_num > self.max_model_len:
|
||||
if token_num > max_model_len:
|
||||
operations: dict[type[AnyRequest], str] = {
|
||||
ScoreDataRequest: "score",
|
||||
ScoreTextRequest: "score",
|
||||
@@ -873,7 +873,7 @@ class OpenAIServing:
|
||||
operation = operations.get(type(request), "embedding generation")
|
||||
raise VLLMValidationError(
|
||||
f"This model's maximum context length is "
|
||||
f"{self.max_model_len} tokens. However, you requested "
|
||||
f"{max_model_len} tokens. However, you requested "
|
||||
f"{token_num} tokens in the input for {operation}. "
|
||||
f"Please reduce the length of the input.",
|
||||
parameter="input_tokens",
|
||||
@@ -898,22 +898,22 @@ class OpenAIServing:
|
||||
|
||||
# Note: input length can be up to model context length - 1 for
|
||||
# completion-like requests.
|
||||
if token_num >= self.max_model_len:
|
||||
if token_num >= max_model_len:
|
||||
raise VLLMValidationError(
|
||||
f"This model's maximum context length is "
|
||||
f"{self.max_model_len} tokens. However, your request has "
|
||||
f"{max_model_len} tokens. However, your request has "
|
||||
f"{token_num} input tokens. Please reduce the length of "
|
||||
"the input messages.",
|
||||
parameter="input_tokens",
|
||||
value=token_num,
|
||||
)
|
||||
|
||||
if max_tokens is not None and token_num + max_tokens > self.max_model_len:
|
||||
if max_tokens is not None and token_num + max_tokens > max_model_len:
|
||||
raise VLLMValidationError(
|
||||
"'max_tokens' or 'max_completion_tokens' is too large: "
|
||||
f"{max_tokens}. This model's maximum context length is "
|
||||
f"{self.max_model_len} tokens and your request has "
|
||||
f"{token_num} input tokens ({max_tokens} > {self.max_model_len}"
|
||||
f"{max_model_len} tokens and your request has "
|
||||
f"{token_num} input tokens ({max_tokens} > {max_model_len}"
|
||||
f" - {token_num}).",
|
||||
parameter="max_tokens",
|
||||
value=max_tokens,
|
||||
@@ -1089,6 +1089,7 @@ class OpenAIServing:
|
||||
priority: int = 0,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
orig_priority = priority
|
||||
@@ -1148,7 +1149,7 @@ class OpenAIServing:
|
||||
token_ids = context.render_for_completion()
|
||||
engine_prompt = TokensPrompt(prompt_token_ids=token_ids)
|
||||
|
||||
sampling_params.max_tokens = self.max_model_len - len(token_ids)
|
||||
sampling_params.max_tokens = max_model_len - len(token_ids)
|
||||
elif isinstance(context, ParsableContext):
|
||||
engine_prompts = await self._render_next_turn(
|
||||
context.request,
|
||||
@@ -1162,7 +1163,7 @@ class OpenAIServing:
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
self.max_model_len,
|
||||
max_model_len,
|
||||
context.request.max_output_tokens,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self.default_sampling_params, # type: ignore
|
||||
|
||||
@@ -59,11 +59,10 @@ class OpenAIServingModels:
|
||||
)
|
||||
self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock)
|
||||
|
||||
self.input_processor = self.engine_client.input_processor
|
||||
self.io_processor = self.engine_client.io_processor
|
||||
self.renderer = self.engine_client.renderer
|
||||
self.model_config = self.engine_client.model_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
self.renderer = self.engine_client.renderer
|
||||
self.io_processor = self.engine_client.io_processor
|
||||
self.input_processor = self.engine_client.input_processor
|
||||
|
||||
async def init_static_loras(self):
|
||||
"""Loads all static LoRA modules.
|
||||
@@ -96,12 +95,13 @@ class OpenAIServingModels:
|
||||
return self.base_model_paths[0].name
|
||||
|
||||
async def show_available_models(self) -> ModelList:
|
||||
"""Show available models. This includes the base model and all
|
||||
adapters"""
|
||||
"""Show available models. This includes the base model and all adapters."""
|
||||
max_model_len = self.model_config.max_model_len
|
||||
|
||||
model_cards = [
|
||||
ModelCard(
|
||||
id=base_model.name,
|
||||
max_model_len=self.max_model_len,
|
||||
max_model_len=max_model_len,
|
||||
root=base_model.model_path,
|
||||
permission=[ModelPermission()],
|
||||
)
|
||||
|
||||
@@ -296,10 +296,12 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
) -> ErrorResponse | None:
|
||||
"""Add validations to the input to the generator here."""
|
||||
prompt_len = self._extract_prompt_len(engine_prompt)
|
||||
if self.max_model_len <= prompt_len:
|
||||
max_model_len = self.model_config.max_model_len
|
||||
|
||||
if prompt_len >= max_model_len:
|
||||
error_message = (
|
||||
f"The engine prompt length {prompt_len} "
|
||||
f"exceeds the max_model_len {self.max_model_len}. "
|
||||
f"exceeds the max_model_len {max_model_len}. "
|
||||
"Please reduce prompt."
|
||||
)
|
||||
return self.create_error_response(
|
||||
@@ -414,6 +416,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
raw_request.state.request_metadata = request_metadata
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
max_model_len = self.model_config.max_model_len
|
||||
generators: list[AsyncGenerator[ConversationContext, None]] = []
|
||||
|
||||
builtin_tool_list: list[str] = []
|
||||
@@ -431,8 +434,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
assert len(builtin_tool_list) == 0
|
||||
available_tools = []
|
||||
try:
|
||||
renderer = self.engine_client.renderer
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
|
||||
for engine_prompt in engine_prompts:
|
||||
maybe_error = self._validate_generator_input(engine_prompt)
|
||||
@@ -440,7 +442,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
return maybe_error
|
||||
|
||||
default_max_tokens = get_max_tokens(
|
||||
self.max_model_len,
|
||||
max_model_len,
|
||||
request.max_output_tokens,
|
||||
self._extract_prompt_len(engine_prompt),
|
||||
self.default_sampling_params,
|
||||
|
||||
@@ -69,16 +69,8 @@ class OpenAIServingEmbedding(OpenAIServing):
|
||||
self.trust_request_chat_template = trust_request_chat_template
|
||||
|
||||
pooler_config = self.model_config.pooler_config
|
||||
|
||||
# Avoid repeated attribute lookups
|
||||
self.supports_chunked_processing = bool(
|
||||
pooler_config and pooler_config.enable_chunked_processing
|
||||
)
|
||||
self.max_embed_len = (
|
||||
pooler_config.max_embed_len
|
||||
if pooler_config and pooler_config.max_embed_len
|
||||
else None
|
||||
)
|
||||
assert pooler_config is not None
|
||||
self.pooler_config = pooler_config
|
||||
|
||||
async def _preprocess(
|
||||
self,
|
||||
@@ -240,7 +232,7 @@ class OpenAIServingEmbedding(OpenAIServing):
|
||||
"""Check if chunked processing should be used for this request."""
|
||||
return (
|
||||
isinstance(request, (EmbeddingCompletionRequest, EmbeddingChatRequest))
|
||||
and self.supports_chunked_processing
|
||||
and self.pooler_config.enable_chunked_processing
|
||||
)
|
||||
|
||||
async def _process_chunked_request(
|
||||
@@ -310,14 +302,14 @@ class OpenAIServingEmbedding(OpenAIServing):
|
||||
max_pos_embeddings = self._get_max_position_embeddings()
|
||||
|
||||
# Determine the effective max length for validation
|
||||
if self.max_embed_len is not None:
|
||||
if self.pooler_config.max_embed_len:
|
||||
# Use max_embed_len for validation instead of max_model_len
|
||||
length_type = "maximum embedding input length"
|
||||
max_length_value = self.max_embed_len
|
||||
max_length_value = self.pooler_config.max_embed_len
|
||||
else:
|
||||
# Fall back to max_model_len validation (original behavior)
|
||||
length_type = "maximum context length"
|
||||
max_length_value = self.max_model_len
|
||||
max_length_value = self.model_config.max_model_len
|
||||
|
||||
validation_error_msg = (
|
||||
"This model's {length_type} is {max_length_value} tokens. "
|
||||
|
||||
@@ -117,7 +117,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
tokens=input_ids,
|
||||
token_strs=token_strs,
|
||||
count=len(input_ids),
|
||||
max_model_len=self.max_model_len,
|
||||
max_model_len=self.model_config.max_model_len,
|
||||
)
|
||||
|
||||
async def create_detokenize(
|
||||
|
||||
Reference in New Issue
Block a user