[Bugfix][Frontend] Support scalar priority and wire priority into LLM.chat() (#57243)
Signed-off-by: subhash polisetti <subhashr161347@gmail.com>
This commit is contained in:
@@ -65,6 +65,25 @@ def test_multi_chat(text_llm):
|
||||
assert len(outputs) == 2
|
||||
|
||||
|
||||
def test_chat_priority(text_llm):
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant"},
|
||||
{"role": "user", "content": "Explain the concept of entropy."},
|
||||
]
|
||||
|
||||
# A single value is applied to every conversation.
|
||||
outputs = text_llm.chat(messages, priority=1)
|
||||
assert len(outputs) == 1
|
||||
|
||||
outputs = text_llm.chat([messages, messages], priority=[0, 1])
|
||||
assert len(outputs) == 2
|
||||
|
||||
# Exception raised, if the length of priority does not match the length of
|
||||
# conversations
|
||||
with pytest.raises(VLLMValidationError):
|
||||
text_llm.chat([messages, messages], priority=[0])
|
||||
|
||||
|
||||
def test_llm_chat_tokenization_no_double_bos(text_llm):
|
||||
"""LLM.chat() should not add special tokens when using chat templates.
|
||||
Check we get a single BOS token for llama chat.
|
||||
|
||||
@@ -75,6 +75,10 @@ def test_multiple_priority(llm: LLM):
|
||||
outputs = llm.generate(PROMPTS, sampling_params=None, priority=[0] * len(PROMPTS))
|
||||
assert len(PROMPTS) == len(outputs)
|
||||
|
||||
# Generate works when priority is a single value applied to every prompt
|
||||
outputs = llm.generate(PROMPTS, sampling_params=None, priority=1)
|
||||
assert len(PROMPTS) == len(outputs)
|
||||
|
||||
# Exception raised, if the length of priority does not match the length of prompts
|
||||
with pytest.raises(VLLMValidationError):
|
||||
outputs = llm.generate(
|
||||
@@ -91,6 +95,10 @@ def test_single_prompt_priority(llm: LLM):
|
||||
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=[0])
|
||||
assert len(outputs) == 1
|
||||
|
||||
# Single string prompts should also accept a single priority value.
|
||||
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=1)
|
||||
assert len(outputs) == 1
|
||||
|
||||
|
||||
def test_max_model_len(vllm_runner):
|
||||
max_model_len = 20
|
||||
|
||||
@@ -55,3 +55,9 @@ def test_matching_lengths_pass_through(mixin: OfflineInferenceMixin):
|
||||
]
|
||||
assert mixin._lora_request_to_seq([lora], num_requests=1) == [lora]
|
||||
assert mixin._priority_to_seq([3], num_requests=1) == [3]
|
||||
|
||||
|
||||
def test_scalar_priority_broadcast(mixin: OfflineInferenceMixin):
|
||||
assert mixin._priority_to_seq(5, num_requests=1) == [5]
|
||||
assert mixin._priority_to_seq(5, num_requests=3) == [5, 5, 5]
|
||||
assert mixin._priority_to_seq(None, num_requests=3) == [0, 0, 0]
|
||||
|
||||
Reference in New Issue
Block a user