[Bugfix][Frontend] Support scalar priority and wire priority into LLM.chat() (#57243)
Signed-off-by: subhash polisetti <subhashr161347@gmail.com>
This commit is contained in:
@@ -65,6 +65,25 @@ def test_multi_chat(text_llm):
|
||||
assert len(outputs) == 2
|
||||
|
||||
|
||||
def test_chat_priority(text_llm):
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant"},
|
||||
{"role": "user", "content": "Explain the concept of entropy."},
|
||||
]
|
||||
|
||||
# A single value is applied to every conversation.
|
||||
outputs = text_llm.chat(messages, priority=1)
|
||||
assert len(outputs) == 1
|
||||
|
||||
outputs = text_llm.chat([messages, messages], priority=[0, 1])
|
||||
assert len(outputs) == 2
|
||||
|
||||
# Exception raised, if the length of priority does not match the length of
|
||||
# conversations
|
||||
with pytest.raises(VLLMValidationError):
|
||||
text_llm.chat([messages, messages], priority=[0])
|
||||
|
||||
|
||||
def test_llm_chat_tokenization_no_double_bos(text_llm):
|
||||
"""LLM.chat() should not add special tokens when using chat templates.
|
||||
Check we get a single BOS token for llama chat.
|
||||
|
||||
@@ -75,6 +75,10 @@ def test_multiple_priority(llm: LLM):
|
||||
outputs = llm.generate(PROMPTS, sampling_params=None, priority=[0] * len(PROMPTS))
|
||||
assert len(PROMPTS) == len(outputs)
|
||||
|
||||
# Generate works when priority is a single value applied to every prompt
|
||||
outputs = llm.generate(PROMPTS, sampling_params=None, priority=1)
|
||||
assert len(PROMPTS) == len(outputs)
|
||||
|
||||
# Exception raised, if the length of priority does not match the length of prompts
|
||||
with pytest.raises(VLLMValidationError):
|
||||
outputs = llm.generate(
|
||||
@@ -91,6 +95,10 @@ def test_single_prompt_priority(llm: LLM):
|
||||
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=[0])
|
||||
assert len(outputs) == 1
|
||||
|
||||
# Single string prompts should also accept a single priority value.
|
||||
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=1)
|
||||
assert len(outputs) == 1
|
||||
|
||||
|
||||
def test_max_model_len(vllm_runner):
|
||||
max_model_len = 20
|
||||
|
||||
@@ -55,3 +55,9 @@ def test_matching_lengths_pass_through(mixin: OfflineInferenceMixin):
|
||||
]
|
||||
assert mixin._lora_request_to_seq([lora], num_requests=1) == [lora]
|
||||
assert mixin._priority_to_seq([3], num_requests=1) == [3]
|
||||
|
||||
|
||||
def test_scalar_priority_broadcast(mixin: OfflineInferenceMixin):
|
||||
assert mixin._priority_to_seq(5, num_requests=1) == [5]
|
||||
assert mixin._priority_to_seq(5, num_requests=3) == [5, 5, 5]
|
||||
assert mixin._priority_to_seq(None, num_requests=3) == [0, 0, 0]
|
||||
|
||||
@@ -281,10 +281,13 @@ class OfflineInferenceMixin:
|
||||
|
||||
def _priority_to_seq(
|
||||
self,
|
||||
priority: list[int] | None,
|
||||
priority: int | Sequence[int] | None,
|
||||
num_requests: int,
|
||||
) -> Sequence[int]:
|
||||
if priority is not None:
|
||||
if priority is None:
|
||||
priority = 0
|
||||
|
||||
if isinstance(priority, Sequence):
|
||||
if len(priority) != num_requests:
|
||||
raise VLLMValidationError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
@@ -293,7 +296,7 @@ class OfflineInferenceMixin:
|
||||
|
||||
return priority
|
||||
|
||||
return [0] * num_requests
|
||||
return [priority] * num_requests
|
||||
|
||||
def _add_completion_requests(
|
||||
self,
|
||||
@@ -304,7 +307,7 @@ class OfflineInferenceMixin:
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[str]:
|
||||
@@ -341,7 +344,7 @@ class OfflineInferenceMixin:
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
@@ -367,6 +370,7 @@ class OfflineInferenceMixin:
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
add_generation_prompt: bool = True,
|
||||
@@ -381,6 +385,7 @@ class OfflineInferenceMixin:
|
||||
params=params,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
@@ -402,7 +407,7 @@ class OfflineInferenceMixin:
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
add_generation_prompt: bool = True,
|
||||
|
||||
+21
-6
@@ -422,7 +422,7 @@ class LLM(
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[RequestOutput]:
|
||||
@@ -448,9 +448,9 @@ class LLM(
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
priority: The priority of the requests, if any.
|
||||
Only applicable when priority scheduling policy is enabled.
|
||||
If provided, must be a list of integers matching the length
|
||||
of `prompts`, where each priority value corresponds to the prompt
|
||||
at the same index.
|
||||
When it is a single value, it is applied to every prompt.
|
||||
When it is a list, the list must have the same length as the
|
||||
prompts and it is paired one by one with the prompt.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
mm_processor_kwargs: Overrides for `processor.__call__`.
|
||||
|
||||
@@ -486,7 +486,7 @@ class LLM(
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
@@ -502,6 +502,10 @@ class LLM(
|
||||
sampling_params: The sampling parameters for text generation.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
priority: The priority of the requests, if any.
|
||||
Only applicable when priority scheduling policy is enabled.
|
||||
When it is a single value, it is applied to every prompt.
|
||||
When it is a list, the list must have the same length as the
|
||||
prompts and it is paired one by one with the prompt.
|
||||
use_tqdm: If True, shows a tqdm progress bar while adding requests.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
mm_processor_kwargs: Overrides for `processor.__call__`.
|
||||
@@ -617,6 +621,7 @@ class LLM(
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
add_generation_prompt: bool = True,
|
||||
@@ -651,6 +656,11 @@ class LLM(
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
priority: The priority of the requests, if any.
|
||||
Only applicable when priority scheduling policy is enabled.
|
||||
When it is a single value, it is applied to every prompt.
|
||||
When it is a list, the list must have the same length as the
|
||||
prompts and it is paired one by one with the prompt.
|
||||
chat_template: The template to use for structuring the chat.
|
||||
If not provided, the model's default chat template will be used.
|
||||
chat_template_content_format: The format to render message content.
|
||||
@@ -695,6 +705,7 @@ class LLM(
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
@@ -712,7 +723,7 @@ class LLM(
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
priority: int | Sequence[int] | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
add_generation_prompt: bool = True,
|
||||
@@ -739,6 +750,10 @@ class LLM(
|
||||
conversations.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
priority: The priority of the requests, if any.
|
||||
Only applicable when priority scheduling policy is enabled.
|
||||
When it is a single value, it is applied to every prompt.
|
||||
When it is a list, the list must have the same length as the
|
||||
prompts and it is paired one by one with the prompt.
|
||||
chat_template: The template to use for structuring the chat.
|
||||
chat_template_content_format: The format to render message content.
|
||||
add_generation_prompt: If True, adds a generation template
|
||||
|
||||
Reference in New Issue
Block a user