[Refactor] Pass Renderer to Input Processor (#34329)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-02-11 19:38:11 -08:00
committed by GitHub
parent ced2a92f40
commit b96f7314b4
20 changed files with 107 additions and 106 deletions
+4 -3
View File
@@ -356,8 +356,9 @@ class LLM:
self.supported_tasks = supported_tasks
self.model_config = self.llm_engine.model_config
self.input_processor = self.llm_engine.input_processor
self.renderer = self.llm_engine.renderer
self.io_processor = self.llm_engine.io_processor
self.input_processor = self.llm_engine.input_processor
# Cache for __repr__ to avoid repeated collective_rpc calls
self._cached_repr: str | None = None
@@ -816,7 +817,7 @@ class LLM:
A list of `TokensPrompts` objects containing the tokenized prompt
after chat template interpolation, and the raw multi-modal inputs.
"""
renderer = self.llm_engine.renderer
renderer = self.renderer
model_config = self.model_config
parsed_prompts = [
@@ -858,7 +859,7 @@ class LLM:
A list of `TokensPrompts` objects containing the tokenized prompt
after chat template interpolation, and the raw multi-modal inputs.
"""
renderer = self.llm_engine.renderer
renderer = self.renderer
chat_params = ChatParams(
chat_template=chat_template,
@@ -239,8 +239,7 @@ class OpenAIServingChat(OpenAIServing):
raise self.engine_client.dead_error
try:
renderer = self.engine_client.renderer
tokenizer = renderer.tokenizer
tokenizer = self.renderer.tokenizer
tool_parser = self.tool_parser
@@ -375,6 +374,7 @@ class OpenAIServingChat(OpenAIServing):
data_parallel_rank = self._get_data_parallel_rank(raw_request)
# Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len
generators: list[AsyncGenerator[RequestOutput, None]] = []
try:
for i, engine_prompt in enumerate(engine_prompts):
@@ -387,7 +387,7 @@ class OpenAIServingChat(OpenAIServing):
)
max_tokens = get_max_tokens(
self.max_model_len,
max_model_len,
request.max_completion_tokens
if request.max_completion_tokens is not None
else request.max_tokens,
@@ -157,13 +157,14 @@ class OpenAIServingCompletion(OpenAIServing):
data_parallel_rank = self._get_data_parallel_rank(raw_request)
# Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len
generators: list[AsyncGenerator[RequestOutput, None]] = []
try:
for i, engine_prompt in enumerate(engine_prompts):
prompt_text = self._extract_prompt_text(engine_prompt)
max_tokens = get_max_tokens(
self.max_model_len,
max_model_len,
request.max_tokens,
self._extract_prompt_len(engine_prompt),
self.default_sampling_params,
+16 -15
View File
@@ -242,11 +242,10 @@ class OpenAIServing:
self.log_error_stack = log_error_stack
self.input_processor = self.models.input_processor
self.io_processor = self.models.io_processor
self.renderer = self.models.renderer
self.model_config = self.models.model_config
self.max_model_len = self.model_config.max_model_len
self.model_config = engine_client.model_config
self.renderer = engine_client.renderer
self.io_processor = engine_client.io_processor
self.input_processor = engine_client.input_processor
async def beam_search(
self,
@@ -537,7 +536,7 @@ class OpenAIServing:
if (
truncate_prompt_tokens is not None
and truncate_prompt_tokens > self.max_model_len
and truncate_prompt_tokens > self.model_config.max_model_len
):
return self.create_error_response(
"truncate_prompt_tokens value is "
@@ -844,6 +843,7 @@ class OpenAIServing:
input_text: str,
) -> TokensPrompt:
token_num = len(input_ids)
max_model_len = self.model_config.max_model_len
# Note: EmbeddingRequest, ClassificationRequest,
# and ScoreRequest doesn't have max_tokens
@@ -862,7 +862,7 @@ class OpenAIServing:
):
# Note: input length can be up to the entire model context length
# since these requests don't generate tokens.
if token_num > self.max_model_len:
if token_num > max_model_len:
operations: dict[type[AnyRequest], str] = {
ScoreDataRequest: "score",
ScoreTextRequest: "score",
@@ -873,7 +873,7 @@ class OpenAIServing:
operation = operations.get(type(request), "embedding generation")
raise VLLMValidationError(
f"This model's maximum context length is "
f"{self.max_model_len} tokens. However, you requested "
f"{max_model_len} tokens. However, you requested "
f"{token_num} tokens in the input for {operation}. "
f"Please reduce the length of the input.",
parameter="input_tokens",
@@ -898,22 +898,22 @@ class OpenAIServing:
# Note: input length can be up to model context length - 1 for
# completion-like requests.
if token_num >= self.max_model_len:
if token_num >= max_model_len:
raise VLLMValidationError(
f"This model's maximum context length is "
f"{self.max_model_len} tokens. However, your request has "
f"{max_model_len} tokens. However, your request has "
f"{token_num} input tokens. Please reduce the length of "
"the input messages.",
parameter="input_tokens",
value=token_num,
)
if max_tokens is not None and token_num + max_tokens > self.max_model_len:
if max_tokens is not None and token_num + max_tokens > max_model_len:
raise VLLMValidationError(
"'max_tokens' or 'max_completion_tokens' is too large: "
f"{max_tokens}. This model's maximum context length is "
f"{self.max_model_len} tokens and your request has "
f"{token_num} input tokens ({max_tokens} > {self.max_model_len}"
f"{max_model_len} tokens and your request has "
f"{token_num} input tokens ({max_tokens} > {max_model_len}"
f" - {token_num}).",
parameter="max_tokens",
value=max_tokens,
@@ -1089,6 +1089,7 @@ class OpenAIServing:
priority: int = 0,
trace_headers: Mapping[str, str] | None = None,
):
max_model_len = self.model_config.max_model_len
prompt_text = self._extract_prompt_text(engine_prompt)
orig_priority = priority
@@ -1148,7 +1149,7 @@ class OpenAIServing:
token_ids = context.render_for_completion()
engine_prompt = TokensPrompt(prompt_token_ids=token_ids)
sampling_params.max_tokens = self.max_model_len - len(token_ids)
sampling_params.max_tokens = max_model_len - len(token_ids)
elif isinstance(context, ParsableContext):
engine_prompts = await self._render_next_turn(
context.request,
@@ -1162,7 +1163,7 @@ class OpenAIServing:
prompt_text = self._extract_prompt_text(engine_prompt)
sampling_params.max_tokens = get_max_tokens(
self.max_model_len,
max_model_len,
context.request.max_output_tokens,
self._extract_prompt_len(engine_prompt),
self.default_sampling_params, # type: ignore
+7 -7
View File
@@ -59,11 +59,10 @@ class OpenAIServingModels:
)
self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock)
self.input_processor = self.engine_client.input_processor
self.io_processor = self.engine_client.io_processor
self.renderer = self.engine_client.renderer
self.model_config = self.engine_client.model_config
self.max_model_len = self.model_config.max_model_len
self.renderer = self.engine_client.renderer
self.io_processor = self.engine_client.io_processor
self.input_processor = self.engine_client.input_processor
async def init_static_loras(self):
"""Loads all static LoRA modules.
@@ -96,12 +95,13 @@ class OpenAIServingModels:
return self.base_model_paths[0].name
async def show_available_models(self) -> ModelList:
"""Show available models. This includes the base model and all
adapters"""
"""Show available models. This includes the base model and all adapters."""
max_model_len = self.model_config.max_model_len
model_cards = [
ModelCard(
id=base_model.name,
max_model_len=self.max_model_len,
max_model_len=max_model_len,
root=base_model.model_path,
permission=[ModelPermission()],
)
+7 -5
View File
@@ -296,10 +296,12 @@ class OpenAIServingResponses(OpenAIServing):
) -> ErrorResponse | None:
"""Add validations to the input to the generator here."""
prompt_len = self._extract_prompt_len(engine_prompt)
if self.max_model_len <= prompt_len:
max_model_len = self.model_config.max_model_len
if prompt_len >= max_model_len:
error_message = (
f"The engine prompt length {prompt_len} "
f"exceeds the max_model_len {self.max_model_len}. "
f"exceeds the max_model_len {max_model_len}. "
"Please reduce prompt."
)
return self.create_error_response(
@@ -414,6 +416,7 @@ class OpenAIServingResponses(OpenAIServing):
raw_request.state.request_metadata = request_metadata
# Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len
generators: list[AsyncGenerator[ConversationContext, None]] = []
builtin_tool_list: list[str] = []
@@ -431,8 +434,7 @@ class OpenAIServingResponses(OpenAIServing):
assert len(builtin_tool_list) == 0
available_tools = []
try:
renderer = self.engine_client.renderer
tokenizer = renderer.get_tokenizer()
tokenizer = self.renderer.get_tokenizer()
for engine_prompt in engine_prompts:
maybe_error = self._validate_generator_input(engine_prompt)
@@ -440,7 +442,7 @@ class OpenAIServingResponses(OpenAIServing):
return maybe_error
default_max_tokens = get_max_tokens(
self.max_model_len,
max_model_len,
request.max_output_tokens,
self._extract_prompt_len(engine_prompt),
self.default_sampling_params,
+6 -14
View File
@@ -69,16 +69,8 @@ class OpenAIServingEmbedding(OpenAIServing):
self.trust_request_chat_template = trust_request_chat_template
pooler_config = self.model_config.pooler_config
# Avoid repeated attribute lookups
self.supports_chunked_processing = bool(
pooler_config and pooler_config.enable_chunked_processing
)
self.max_embed_len = (
pooler_config.max_embed_len
if pooler_config and pooler_config.max_embed_len
else None
)
assert pooler_config is not None
self.pooler_config = pooler_config
async def _preprocess(
self,
@@ -240,7 +232,7 @@ class OpenAIServingEmbedding(OpenAIServing):
"""Check if chunked processing should be used for this request."""
return (
isinstance(request, (EmbeddingCompletionRequest, EmbeddingChatRequest))
and self.supports_chunked_processing
and self.pooler_config.enable_chunked_processing
)
async def _process_chunked_request(
@@ -310,14 +302,14 @@ class OpenAIServingEmbedding(OpenAIServing):
max_pos_embeddings = self._get_max_position_embeddings()
# Determine the effective max length for validation
if self.max_embed_len is not None:
if self.pooler_config.max_embed_len:
# Use max_embed_len for validation instead of max_model_len
length_type = "maximum embedding input length"
max_length_value = self.max_embed_len
max_length_value = self.pooler_config.max_embed_len
else:
# Fall back to max_model_len validation (original behavior)
length_type = "maximum context length"
max_length_value = self.max_model_len
max_length_value = self.model_config.max_model_len
validation_error_msg = (
"This model's {length_type} is {max_length_value} tokens. "
+1 -1
View File
@@ -117,7 +117,7 @@ class OpenAIServingTokenization(OpenAIServing):
tokens=input_ids,
token_strs=token_strs,
count=len(input_ids),
max_model_len=self.max_model_len,
max_model_len=self.model_config.max_model_len,
)
async def create_detokenize(