[Bugfix] Forward mm_processor_kwargs in offline generate APIs (#40251)

Signed-off-by: wuyingjun <wuyingjun_yewu@cmss.chinamobile.com>
This commit is contained in:
wuyingjun
2026-04-20 00:56:56 -07:00
committed by GitHub
parent e729cc823d
commit 77fd2c8631
2 changed files with 279 additions and 4 deletions
+38 -4
View File
@@ -452,6 +452,7 @@ class LLM:
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[RequestOutput]:
"""Generates the completions for the input prompts.
@@ -479,6 +480,7 @@ class LLM:
of `prompts`, where each priority value corresponds to the prompt
at the same index.
tokenization_kwargs: Overrides for `tokenizer.encode`.
mm_processor_kwargs: Overrides for `processor.__call__`.
Returns:
A list of `RequestOutput` objects containing the
@@ -503,6 +505,7 @@ class LLM:
lora_request=lora_request,
tokenization_kwargs=tokenization_kwargs,
priority=priority,
mm_processor_kwargs=mm_processor_kwargs,
)
def enqueue(
@@ -513,6 +516,7 @@ class LLM:
priority: list[int] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[str]:
"""Enqueue prompts for generation without waiting for completion.
@@ -527,6 +531,7 @@ class LLM:
priority: The priority of the requests, if any.
use_tqdm: If True, shows a tqdm progress bar while adding requests.
tokenization_kwargs: Overrides for `tokenizer.encode`.
mm_processor_kwargs: Overrides for `processor.__call__`.
Returns:
A list of request IDs for the enqueued requests.
@@ -545,6 +550,7 @@ class LLM:
lora_request=lora_request,
priority=priority,
tokenization_kwargs=tokenization_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
)
@overload
@@ -843,6 +849,7 @@ class LLM:
self,
prompts: Sequence[PromptType],
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
"""
Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
@@ -862,15 +869,29 @@ class LLM:
tok_params = renderer.default_cmpl_tok_params.with_kwargs(
**(tokenization_kwargs or {})
)
prompt_extras = (
None
if mm_processor_kwargs is None
else {"mm_processor_kwargs": mm_processor_kwargs}
)
return renderer.render_cmpl(parsed_prompts, tok_params)
return renderer.render_cmpl(
parsed_prompts,
tok_params,
prompt_extras=prompt_extras,
)
def _preprocess_cmpl_one(
self,
prompt: PromptType,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> EngineInput:
(engine_input,) = self._preprocess_cmpl([prompt], tokenization_kwargs)
(engine_input,) = self._preprocess_cmpl(
[prompt],
tokenization_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
)
return engine_input
def _preprocess_chat(
@@ -908,16 +929,22 @@ class LLM:
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
),
mm_processor_kwargs=mm_processor_kwargs,
)
tok_params = renderer.default_chat_tok_params.with_kwargs(
**(tokenization_kwargs or {})
)
prompt_extras = (
None
if mm_processor_kwargs is None
else {"mm_processor_kwargs": mm_processor_kwargs}
)
_, engine_inputs = renderer.render_chat(
conversations,
chat_params,
tok_params,
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
prompt_extras=prompt_extras,
)
return engine_inputs
@@ -1571,6 +1598,7 @@ class LLM:
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[str]:
seq_prompts = prompt_to_seq(prompts)
seq_params = self._params_to_seq(params, len(seq_prompts))
@@ -1579,7 +1607,11 @@ class LLM:
return self._render_and_add_requests(
prompts=(
self._preprocess_cmpl_one(prompt, tokenization_kwargs)
self._preprocess_cmpl_one(
prompt,
tokenization_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
)
for prompt in maybe_tqdm(
seq_prompts,
use_tqdm=use_tqdm,
@@ -1603,6 +1635,7 @@ class LLM:
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
):
self._add_completion_requests(
prompts=prompts,
@@ -1611,6 +1644,7 @@ class LLM:
lora_request=lora_request,
priority=priority,
tokenization_kwargs=tokenization_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
)
return self._run_engine(use_tqdm=use_tqdm, output_type=output_type)