[Bugfix][Frontend] Support scalar priority and wire priority into LLM.chat() (#57243)

Signed-off-by: subhash polisetti <subhashr161347@gmail.com>
This commit is contained in:
Subhash Polisetti
2026-10-11 10:03:33 +00:00
committed by GitHub
parent 2339bd39d6
commit f425a21208
5 changed files with 65 additions and 12 deletions
+19
View File
@@ -65,6 +65,25 @@ def test_multi_chat(text_llm):
assert len(outputs) == 2
def test_chat_priority(text_llm):
messages = [
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Explain the concept of entropy."},
]
# A single value is applied to every conversation.
outputs = text_llm.chat(messages, priority=1)
assert len(outputs) == 1
outputs = text_llm.chat([messages, messages], priority=[0, 1])
assert len(outputs) == 2
# Exception raised, if the length of priority does not match the length of
# conversations
with pytest.raises(VLLMValidationError):
text_llm.chat([messages, messages], priority=[0])
def test_llm_chat_tokenization_no_double_bos(text_llm):
"""LLM.chat() should not add special tokens when using chat templates.
Check we get a single BOS token for llama chat.
+8
View File
@@ -75,6 +75,10 @@ def test_multiple_priority(llm: LLM):
outputs = llm.generate(PROMPTS, sampling_params=None, priority=[0] * len(PROMPTS))
assert len(PROMPTS) == len(outputs)
# Generate works when priority is a single value applied to every prompt
outputs = llm.generate(PROMPTS, sampling_params=None, priority=1)
assert len(PROMPTS) == len(outputs)
# Exception raised, if the length of priority does not match the length of prompts
with pytest.raises(VLLMValidationError):
outputs = llm.generate(
@@ -91,6 +95,10 @@ def test_single_prompt_priority(llm: LLM):
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=[0])
assert len(outputs) == 1
# Single string prompts should also accept a single priority value.
outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=1)
assert len(outputs) == 1
def test_max_model_len(vllm_runner):
max_model_len = 20
@@ -55,3 +55,9 @@ def test_matching_lengths_pass_through(mixin: OfflineInferenceMixin):
]
assert mixin._lora_request_to_seq([lora], num_requests=1) == [lora]
assert mixin._priority_to_seq([3], num_requests=1) == [3]
def test_scalar_priority_broadcast(mixin: OfflineInferenceMixin):
assert mixin._priority_to_seq(5, num_requests=1) == [5]
assert mixin._priority_to_seq(5, num_requests=3) == [5, 5, 5]
assert mixin._priority_to_seq(None, num_requests=3) == [0, 0, 0]
+11 -6
View File
@@ -281,10 +281,13 @@ class OfflineInferenceMixin:
def _priority_to_seq(
self,
priority: list[int] | None,
priority: int | Sequence[int] | None,
num_requests: int,
) -> Sequence[int]:
if priority is not None:
if priority is None:
priority = 0
if isinstance(priority, Sequence):
if len(priority) != num_requests:
raise VLLMValidationError(
f"The lengths of prompts ({num_requests}) "
@@ -293,7 +296,7 @@ class OfflineInferenceMixin:
return priority
return [0] * num_requests
return [priority] * num_requests
def _add_completion_requests(
self,
@@ -304,7 +307,7 @@ class OfflineInferenceMixin:
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
priority: int | Sequence[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[str]:
@@ -341,7 +344,7 @@ class OfflineInferenceMixin:
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
priority: int | Sequence[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
):
@@ -367,6 +370,7 @@ class OfflineInferenceMixin:
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
add_generation_prompt: bool = True,
@@ -381,6 +385,7 @@ class OfflineInferenceMixin:
params=params,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
chat_template_kwargs=chat_template_kwargs,
@@ -402,7 +407,7 @@ class OfflineInferenceMixin:
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
add_generation_prompt: bool = True,
+21 -6
View File
@@ -422,7 +422,7 @@ class LLM(
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
priority: int | Sequence[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[RequestOutput]:
@@ -448,9 +448,9 @@ class LLM(
lora_request: LoRA request to use for generation, if any.
priority: The priority of the requests, if any.
Only applicable when priority scheduling policy is enabled.
If provided, must be a list of integers matching the length
of `prompts`, where each priority value corresponds to the prompt
at the same index.
When it is a single value, it is applied to every prompt.
When it is a list, the list must have the same length as the
prompts and it is paired one by one with the prompt.
tokenization_kwargs: Overrides for `tokenizer.encode`.
mm_processor_kwargs: Overrides for `processor.__call__`.
@@ -486,7 +486,7 @@ class LLM(
prompts: PromptType | Sequence[PromptType],
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
priority: int | Sequence[int] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
@@ -502,6 +502,10 @@ class LLM(
sampling_params: The sampling parameters for text generation.
lora_request: LoRA request to use for generation, if any.
priority: The priority of the requests, if any.
Only applicable when priority scheduling policy is enabled.
When it is a single value, it is applied to every prompt.
When it is a list, the list must have the same length as the
prompts and it is paired one by one with the prompt.
use_tqdm: If True, shows a tqdm progress bar while adding requests.
tokenization_kwargs: Overrides for `tokenizer.encode`.
mm_processor_kwargs: Overrides for `processor.__call__`.
@@ -617,6 +621,7 @@ class LLM(
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
add_generation_prompt: bool = True,
@@ -651,6 +656,11 @@ class LLM(
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
priority: The priority of the requests, if any.
Only applicable when priority scheduling policy is enabled.
When it is a single value, it is applied to every prompt.
When it is a list, the list must have the same length as the
prompts and it is paired one by one with the prompt.
chat_template: The template to use for structuring the chat.
If not provided, the model's default chat template will be used.
chat_template_content_format: The format to render message content.
@@ -695,6 +705,7 @@ class LLM(
output_type=RequestOutput,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
chat_template_kwargs=chat_template_kwargs,
@@ -712,7 +723,7 @@ class LLM(
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
add_generation_prompt: bool = True,
@@ -739,6 +750,10 @@ class LLM(
conversations.
lora_request: LoRA request to use for generation, if any.
priority: The priority of the requests, if any.
Only applicable when priority scheduling policy is enabled.
When it is a single value, it is applied to every prompt.
When it is a list, the list must have the same length as the
prompts and it is paired one by one with the prompt.
chat_template: The template to use for structuring the chat.
chat_template_content_format: The format to render message content.
add_generation_prompt: If True, adds a generation template