fix: use configured max_logprobs instead of hardcoded 20 in derender validation (#47834)
Signed-off-by: jperezde <jperezde@redhat.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
d3e69fd671
commit
b3e85be663
@@ -64,6 +64,8 @@ class ServingDerender(BaseServing):
|
||||
"""
|
||||
max_n = envs.VLLM_MAX_N_SEQUENCES
|
||||
max_model_len = self.model_config.max_model_len
|
||||
# See ModelConfig.max_logprobs for semantics and default value.
|
||||
max_logprobs = self.model_config.max_logprobs
|
||||
|
||||
if len(generate_responses) > max_n:
|
||||
return self.create_error_response(
|
||||
@@ -95,11 +97,16 @@ class ServingDerender(BaseServing):
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
for entry in choice.logprobs.content:
|
||||
if entry.top_logprobs and len(entry.top_logprobs) > 20:
|
||||
if (
|
||||
max_logprobs >= 0
|
||||
and entry.top_logprobs
|
||||
and len(entry.top_logprobs) > max_logprobs
|
||||
):
|
||||
return self.create_error_response(
|
||||
f"top_logprobs count "
|
||||
f"({len(entry.top_logprobs)}) in "
|
||||
f"choice {choice.index} exceeds maximum (20)."
|
||||
f"choice {choice.index} exceeds "
|
||||
f"max_logprobs ({max_logprobs})."
|
||||
)
|
||||
|
||||
if gen.prompt_logprobs and len(gen.prompt_logprobs) > max_model_len:
|
||||
|
||||
Reference in New Issue
Block a user