[Frontend] Make model optional on all /derender request classes (#51463)

Signed-off-by: Vinay Damodaran <vrdn@hey.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Vinay R Damodaran
2026-08-12 04:48:58 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent a53ad85913
commit 3ee2df3033
3 changed files with 37 additions and 14 deletions
+12 -8
View File
@@ -166,16 +166,17 @@ class ServingDerender(BaseServing):
total_tokens=prompt_tokens + completion_tokens,
)
model_name = request.model or self.models.model_name()
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
model_name,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
model=model_name,
created=int(time.time()),
choices=choices,
usage=usage,
@@ -231,17 +232,18 @@ class ServingDerender(BaseServing):
total_tokens=total_prompt_tokens + total_completion_tokens,
)
model_name = request.model or self.models.model_name()
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
model_name,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
model=model_name,
created=int(time.time()),
choices=choices,
usage=usage,
@@ -264,9 +266,10 @@ class ServingDerender(BaseServing):
if error_check_ret is not None:
return error_check_ret
model_name = request.model or self.models.model_name()
try:
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
model=request.model,
model=model_name,
generate_chunk=request.generate_chunk,
state=request.stream_state,
chat_request=request.chat_request,
@@ -284,7 +287,7 @@ class ServingDerender(BaseServing):
logger.debug(
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
request.model,
model_name,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
@@ -304,12 +307,13 @@ class ServingDerender(BaseServing):
if error_check_ret is not None:
return error_check_ret
model_name = request.model or self.models.model_name()
try:
(
chunk,
updated_state,
) = await self.online_derenderer.derender_completion_stream(
model=request.model,
model=model_name,
generate_chunk=request.generate_chunk,
state=request.stream_state,
prompt_tokens=request.prompt_tokens,
@@ -325,7 +329,7 @@ class ServingDerender(BaseServing):
logger.debug(
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
request.model,
model_name,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
@@ -277,8 +277,8 @@ class DerenderChatRequest(BaseModel):
# --8<-- [start:derender-chat-request]
stream: Literal[False] = False
model: str
"""Served model name."""
model: str | None = None
"""Served model name. Defaults to the server's served model name."""
generate_response: GenerateResponse
"""The complete token-in / token-out engine response to derender."""
@@ -311,8 +311,8 @@ class DerenderCompletionRequest(BaseModel):
# --8<-- [start:derender-completion-request]
stream: Literal[False] = False
model: str
"""Served model name."""
model: str | None = None
"""Served model name. Defaults to the server's served model name."""
generate_responses: list[GenerateResponse]
"""One response per prompt, parallel to the list[GenerateRequest]
@@ -428,7 +428,7 @@ class DerenderChatStreamRequest(BaseModel):
stream: Literal[True]
model: str
model: str | None = None
generate_chunk: GenerateStreamResponse
"""One SSE chunk from ``/inference/v1/generate`` (``stream=True``)."""
@@ -452,7 +452,7 @@ class DerenderCompletionStreamRequest(BaseModel):
stream: Literal[True]
model: str
model: str | None = None
generate_chunk: GenerateStreamResponse
"""One SSE chunk from ``/inference/v1/generate``."""