[Frontend] Replace return_mm_kwargs with an mm_payload enum on render requests (#61427)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -12,9 +12,11 @@ import vllm.envs as envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
AnyResponseFormat,
|
||||
MMPayload,
|
||||
PerRequestMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
resolve_return_mm_kwargs_alias,
|
||||
structured_outputs_from_response_format,
|
||||
validate_cache_salt,
|
||||
validate_structural_tag_response_format,
|
||||
@@ -197,15 +199,25 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
return_mm_kwargs: bool = Field(
|
||||
default=True,
|
||||
mm_payload: MMPayload = Field(
|
||||
default="tensors",
|
||||
description=(
|
||||
"If false, the render response's `features` set `kwargs_data` "
|
||||
"and `mm_metadata` to null, for callers that need only the token "
|
||||
"layout and item hashes, such as cache-aware routers. Do not send "
|
||||
"such a response to `/inference/v1/generate`, which reads a null "
|
||||
"`kwargs_data` as every item being cached. Only supported on the "
|
||||
"render endpoints; ignored on regular generation endpoints."
|
||||
"What the render response's `features` carry for multimodal "
|
||||
"items. 'tensors' serializes the processed tensors into "
|
||||
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
|
||||
"callers that need only the token layout and item hashes, such as "
|
||||
"cache-aware routers. Do not send a 'none' response to "
|
||||
"`/inference/v1/generate` which reads a null `kwargs_data` as "
|
||||
"every item being cached. Only supported on the render endpoints. "
|
||||
"Ignored on regular generation endpoints."
|
||||
),
|
||||
)
|
||||
return_mm_kwargs: bool | None = Field(
|
||||
default=None,
|
||||
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
|
||||
description=(
|
||||
"Deprecated alias: false is the same as `mm_payload='none'`. "
|
||||
"Ignored when `mm_payload` is set."
|
||||
),
|
||||
)
|
||||
|
||||
@@ -402,6 +414,11 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
routed_experts_prompt_start=self.routed_experts_prompt_start,
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
|
||||
return resolve_return_mm_kwargs_alias(data)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
|
||||
Reference in New Issue
Block a user