From f425a212081181e3d6ad2ca92daa74ba0f9e83a4 Mon Sep 17 00:00:00 2001 From: Subhash Polisetti <49185065+subhashpolisetti@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:03:33 -0700 Subject: [PATCH] [Bugfix][Frontend] Support scalar priority and wire priority into LLM.chat() (#57243) Signed-off-by: subhash polisetti --- tests/entrypoints/llm/test_chat.py | 19 +++++++++++++ tests/entrypoints/llm/test_generate.py | 8 ++++++ .../unit_tests/test_offline_utils.py | 6 +++++ vllm/entrypoints/common/offline.py | 17 +++++++----- vllm/entrypoints/llm.py | 27 ++++++++++++++----- 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/tests/entrypoints/llm/test_chat.py b/tests/entrypoints/llm/test_chat.py index 276426e692..8fdd41c511 100644 --- a/tests/entrypoints/llm/test_chat.py +++ b/tests/entrypoints/llm/test_chat.py @@ -65,6 +65,25 @@ def test_multi_chat(text_llm): assert len(outputs) == 2 +def test_chat_priority(text_llm): + messages = [ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Explain the concept of entropy."}, + ] + + # A single value is applied to every conversation. + outputs = text_llm.chat(messages, priority=1) + assert len(outputs) == 1 + + outputs = text_llm.chat([messages, messages], priority=[0, 1]) + assert len(outputs) == 2 + + # Exception raised, if the length of priority does not match the length of + # conversations + with pytest.raises(VLLMValidationError): + text_llm.chat([messages, messages], priority=[0]) + + def test_llm_chat_tokenization_no_double_bos(text_llm): """LLM.chat() should not add special tokens when using chat templates. Check we get a single BOS token for llama chat. diff --git a/tests/entrypoints/llm/test_generate.py b/tests/entrypoints/llm/test_generate.py index 4a2c59dc58..ccff4eb7a0 100644 --- a/tests/entrypoints/llm/test_generate.py +++ b/tests/entrypoints/llm/test_generate.py @@ -75,6 +75,10 @@ def test_multiple_priority(llm: LLM): outputs = llm.generate(PROMPTS, sampling_params=None, priority=[0] * len(PROMPTS)) assert len(PROMPTS) == len(outputs) + # Generate works when priority is a single value applied to every prompt + outputs = llm.generate(PROMPTS, sampling_params=None, priority=1) + assert len(PROMPTS) == len(outputs) + # Exception raised, if the length of priority does not match the length of prompts with pytest.raises(VLLMValidationError): outputs = llm.generate( @@ -91,6 +95,10 @@ def test_single_prompt_priority(llm: LLM): outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=[0]) assert len(outputs) == 1 + # Single string prompts should also accept a single priority value. + outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=1) + assert len(outputs) == 1 + def test_max_model_len(vllm_runner): max_model_len = 20 diff --git a/tests/entrypoints/unit_tests/test_offline_utils.py b/tests/entrypoints/unit_tests/test_offline_utils.py index a37969c9b3..8a0308ce3a 100644 --- a/tests/entrypoints/unit_tests/test_offline_utils.py +++ b/tests/entrypoints/unit_tests/test_offline_utils.py @@ -55,3 +55,9 @@ def test_matching_lengths_pass_through(mixin: OfflineInferenceMixin): ] assert mixin._lora_request_to_seq([lora], num_requests=1) == [lora] assert mixin._priority_to_seq([3], num_requests=1) == [3] + + +def test_scalar_priority_broadcast(mixin: OfflineInferenceMixin): + assert mixin._priority_to_seq(5, num_requests=1) == [5] + assert mixin._priority_to_seq(5, num_requests=3) == [5, 5, 5] + assert mixin._priority_to_seq(None, num_requests=3) == [0, 0, 0] diff --git a/vllm/entrypoints/common/offline.py b/vllm/entrypoints/common/offline.py index b33b8c34c5..1bb0bedccb 100644 --- a/vllm/entrypoints/common/offline.py +++ b/vllm/entrypoints/common/offline.py @@ -281,10 +281,13 @@ class OfflineInferenceMixin: def _priority_to_seq( self, - priority: list[int] | None, + priority: int | Sequence[int] | None, num_requests: int, ) -> Sequence[int]: - if priority is not None: + if priority is None: + priority = 0 + + if isinstance(priority, Sequence): if len(priority) != num_requests: raise VLLMValidationError( f"The lengths of prompts ({num_requests}) " @@ -293,7 +296,7 @@ class OfflineInferenceMixin: return priority - return [0] * num_requests + return [priority] * num_requests def _add_completion_requests( self, @@ -304,7 +307,7 @@ class OfflineInferenceMixin: *, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, - priority: list[int] | None = None, + priority: int | Sequence[int] | None = None, tokenization_kwargs: dict[str, Any] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, ) -> list[str]: @@ -341,7 +344,7 @@ class OfflineInferenceMixin: *, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, - priority: list[int] | None = None, + priority: int | Sequence[int] | None = None, tokenization_kwargs: dict[str, Any] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, ): @@ -367,6 +370,7 @@ class OfflineInferenceMixin: *, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, + priority: int | Sequence[int] | None = None, chat_template: str | None = None, chat_template_content_format: ChatTemplateContentFormatOption = "auto", add_generation_prompt: bool = True, @@ -381,6 +385,7 @@ class OfflineInferenceMixin: params=params, use_tqdm=use_tqdm, lora_request=lora_request, + priority=priority, chat_template=chat_template, chat_template_content_format=chat_template_content_format, chat_template_kwargs=chat_template_kwargs, @@ -402,7 +407,7 @@ class OfflineInferenceMixin: *, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, - priority: list[int] | None = None, + priority: int | Sequence[int] | None = None, chat_template: str | None = None, chat_template_content_format: ChatTemplateContentFormatOption = "auto", add_generation_prompt: bool = True, diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index ff7bd9c574..6b7f91d08e 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -422,7 +422,7 @@ class LLM( *, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, - priority: list[int] | None = None, + priority: int | Sequence[int] | None = None, tokenization_kwargs: dict[str, Any] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, ) -> list[RequestOutput]: @@ -448,9 +448,9 @@ class LLM( lora_request: LoRA request to use for generation, if any. priority: The priority of the requests, if any. Only applicable when priority scheduling policy is enabled. - If provided, must be a list of integers matching the length - of `prompts`, where each priority value corresponds to the prompt - at the same index. + When it is a single value, it is applied to every prompt. + When it is a list, the list must have the same length as the + prompts and it is paired one by one with the prompt. tokenization_kwargs: Overrides for `tokenizer.encode`. mm_processor_kwargs: Overrides for `processor.__call__`. @@ -486,7 +486,7 @@ class LLM( prompts: PromptType | Sequence[PromptType], sampling_params: SamplingParams | Sequence[SamplingParams] | None = None, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, - priority: list[int] | None = None, + priority: int | Sequence[int] | None = None, use_tqdm: bool | Callable[..., tqdm] = True, tokenization_kwargs: dict[str, Any] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, @@ -502,6 +502,10 @@ class LLM( sampling_params: The sampling parameters for text generation. lora_request: LoRA request to use for generation, if any. priority: The priority of the requests, if any. + Only applicable when priority scheduling policy is enabled. + When it is a single value, it is applied to every prompt. + When it is a list, the list must have the same length as the + prompts and it is paired one by one with the prompt. use_tqdm: If True, shows a tqdm progress bar while adding requests. tokenization_kwargs: Overrides for `tokenizer.encode`. mm_processor_kwargs: Overrides for `processor.__call__`. @@ -617,6 +621,7 @@ class LLM( sampling_params: SamplingParams | Sequence[SamplingParams] | None = None, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, + priority: int | Sequence[int] | None = None, chat_template: str | None = None, chat_template_content_format: ChatTemplateContentFormatOption = "auto", add_generation_prompt: bool = True, @@ -651,6 +656,11 @@ class LLM( it is used to create the progress bar. If `False`, no progress bar is created. lora_request: LoRA request to use for generation, if any. + priority: The priority of the requests, if any. + Only applicable when priority scheduling policy is enabled. + When it is a single value, it is applied to every prompt. + When it is a list, the list must have the same length as the + prompts and it is paired one by one with the prompt. chat_template: The template to use for structuring the chat. If not provided, the model's default chat template will be used. chat_template_content_format: The format to render message content. @@ -695,6 +705,7 @@ class LLM( output_type=RequestOutput, use_tqdm=use_tqdm, lora_request=lora_request, + priority=priority, chat_template=chat_template, chat_template_content_format=chat_template_content_format, chat_template_kwargs=chat_template_kwargs, @@ -712,7 +723,7 @@ class LLM( sampling_params: SamplingParams | Sequence[SamplingParams] | None = None, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, - priority: list[int] | None = None, + priority: int | Sequence[int] | None = None, chat_template: str | None = None, chat_template_content_format: ChatTemplateContentFormatOption = "auto", add_generation_prompt: bool = True, @@ -739,6 +750,10 @@ class LLM( conversations. lora_request: LoRA request to use for generation, if any. priority: The priority of the requests, if any. + Only applicable when priority scheduling policy is enabled. + When it is a single value, it is applied to every prompt. + When it is a list, the list must have the same length as the + prompts and it is paired one by one with the prompt. chat_template: The template to use for structuring the chat. chat_template_content_format: The format to render message content. add_generation_prompt: If True, adds a generation template