diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py index 880d939626..3167e00b96 100644 --- a/tests/entrypoints/scale_out/derender/test_derender.py +++ b/tests/entrypoints/scale_out/derender/test_derender.py @@ -622,7 +622,7 @@ async def test_derender_chat_oversized_logprobs_rejected(client): @pytest.mark.asyncio async def test_derender_chat_oversized_top_logprobs_rejected(client): - """top_logprobs count exceeding 20 returns 400.""" + """top_logprobs count exceeding max_logprobs (default 20) returns 400.""" oversized_top_logprobs = { "content": [ { @@ -654,7 +654,9 @@ async def test_derender_chat_oversized_top_logprobs_rejected(client): }, ) assert response.status_code == 400 - assert "top_logprobs count" in response.json()["error"]["message"] + msg = response.json()["error"]["message"] + assert "top_logprobs count" in msg + assert "max_logprobs" in msg @pytest.mark.asyncio diff --git a/vllm/entrypoints/scale_out/derender/serving.py b/vllm/entrypoints/scale_out/derender/serving.py index f46ca7f4a5..613ff65ad0 100644 --- a/vllm/entrypoints/scale_out/derender/serving.py +++ b/vllm/entrypoints/scale_out/derender/serving.py @@ -64,6 +64,8 @@ class ServingDerender(BaseServing): """ max_n = envs.VLLM_MAX_N_SEQUENCES max_model_len = self.model_config.max_model_len + # See ModelConfig.max_logprobs for semantics and default value. + max_logprobs = self.model_config.max_logprobs if len(generate_responses) > max_n: return self.create_error_response( @@ -95,11 +97,16 @@ class ServingDerender(BaseServing): f"max_model_len ({max_model_len})." ) for entry in choice.logprobs.content: - if entry.top_logprobs and len(entry.top_logprobs) > 20: + if ( + max_logprobs >= 0 + and entry.top_logprobs + and len(entry.top_logprobs) > max_logprobs + ): return self.create_error_response( f"top_logprobs count " f"({len(entry.top_logprobs)}) in " - f"choice {choice.index} exceeds maximum (20)." + f"choice {choice.index} exceeds " + f"max_logprobs ({max_logprobs})." ) if gen.prompt_logprobs and len(gen.prompt_logprobs) > max_model_len: