[Bugfix][Frontend] Support scalar priority and wire priority into LLM.chat() (#57243)

Signed-off-by: subhash polisetti <subhashr161347@gmail.com>
This commit is contained in:
Subhash Polisetti
2026-10-11 10:03:33 +00:00
committed by GitHub
parent 2339bd39d6
commit f425a21208
5 changed files with 65 additions and 12 deletions
+19
View File
@@ -65,6 +65,25 @@ def test_multi_chat(text_llm):
assert len(outputs) == 2
def test_chat_priority(text_llm):
messages = [
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Explain the concept of entropy."},
]
# A single value is applied to every conversation.
outputs = text_llm.chat(messages, priority=1)
assert len(outputs) == 1
outputs = text_llm.chat([messages, messages], priority=[0, 1])
assert len(outputs) == 2
# Exception raised, if the length of priority does not match the length of
# conversations
with pytest.raises(VLLMValidationError):
text_llm.chat([messages, messages], priority=[0])
def test_llm_chat_tokenization_no_double_bos(text_llm):
"""LLM.chat() should not add special tokens when using chat templates.
Check we get a single BOS token for llama chat.
+8
View File
@@ -75,6 +75,10 @@ def test_multiple_priority(llm: LLM):
outputs = llm.generate(PROMPTS, sampling_params=None, priority=[0] * len(PROMPTS))
assert len(PROMPTS) == len(outputs)
# Generate works when priority is a single value applied to every prompt
outputs = llm.generate(PROMPTS, sampling_params=None, priority=1)
assert len(PROMPTS) == len(outputs)
# Exception raised, if the length of priority does not match the length of prompts
with pytest.raises(VLLMValidationError):
outputs = llm.generate(
@@ -91,6 +95,10 @@ def test_single_prompt_priority(llm: LLM):
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=[0])
assert len(outputs) == 1
# Single string prompts should also accept a single priority value.
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=1)
assert len(outputs) == 1
def test_max_model_len(vllm_runner):
max_model_len = 20
@@ -55,3 +55,9 @@ def test_matching_lengths_pass_through(mixin: OfflineInferenceMixin):
]
assert mixin._lora_request_to_seq([lora], num_requests=1) == [lora]
assert mixin._priority_to_seq([3], num_requests=1) == [3]
def test_scalar_priority_broadcast(mixin: OfflineInferenceMixin):
assert mixin._priority_to_seq(5, num_requests=1) == [5]
assert mixin._priority_to_seq(5, num_requests=3) == [5, 5, 5]
assert mixin._priority_to_seq(None, num_requests=3) == [0, 0, 0]