[responsesAPI] add better error messaging for long prompts (#25724)

Signed-off-by: Andrew Xia <axia@meta.com>
Signed-off-by: Andrew Xia <axia@fb.com>
Co-authored-by: Andrew Xia <axia@fb.com>
This commit is contained in:
Andrew Xia
2025-10-03 14:33:13 -07:00
committed by GitHub
co-authored by Andrew Xia
parent c1ffcb55da
commit 831b124151
2 changed files with 84 additions and 1 deletions
@@ -192,6 +192,23 @@ class OpenAIServingResponses(OpenAIServing):
self.tool_server = tool_server
def _validate_generator_input(
self,
engine_prompt: EngineTokensPrompt) -> Optional[ErrorResponse]:
"""Add validations to the input to the generator here."""
if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
error_message = (
"The engine prompt length"
f" {len(engine_prompt['prompt_token_ids'])} "
f"exceeds the max_model_len {self.max_model_len}. "
"Please reduce prompt.")
return self.create_error_response(
err_type="invalid_request_error",
message=error_message,
status_code=HTTPStatus.BAD_REQUEST,
)
return None
async def create_responses(
self,
request: ResponsesRequest,
@@ -287,8 +304,13 @@ class OpenAIServingResponses(OpenAIServing):
available_tools = []
try:
for i, engine_prompt in enumerate(engine_prompts):
maybe_error = self._validate_generator_input(engine_prompt)
if maybe_error is not None:
return maybe_error
default_max_tokens = self.max_model_len - len(
engine_prompt["prompt_token_ids"])
sampling_params = request.to_sampling_params(
default_max_tokens, self.default_sampling_params)