[Bugfix] Forward mm_processor_kwargs in offline generate APIs (#40251)
Signed-off-by: wuyingjun <wuyingjun_yewu@cmss.chinamobile.com>
This commit is contained in:
+38
-4
@@ -452,6 +452,7 @@ class LLM:
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[RequestOutput]:
|
||||
"""Generates the completions for the input prompts.
|
||||
|
||||
@@ -479,6 +480,7 @@ class LLM:
|
||||
of `prompts`, where each priority value corresponds to the prompt
|
||||
at the same index.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
mm_processor_kwargs: Overrides for `processor.__call__`.
|
||||
|
||||
Returns:
|
||||
A list of `RequestOutput` objects containing the
|
||||
@@ -503,6 +505,7 @@ class LLM:
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
priority=priority,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
def enqueue(
|
||||
@@ -513,6 +516,7 @@ class LLM:
|
||||
priority: list[int] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[str]:
|
||||
"""Enqueue prompts for generation without waiting for completion.
|
||||
|
||||
@@ -527,6 +531,7 @@ class LLM:
|
||||
priority: The priority of the requests, if any.
|
||||
use_tqdm: If True, shows a tqdm progress bar while adding requests.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
mm_processor_kwargs: Overrides for `processor.__call__`.
|
||||
|
||||
Returns:
|
||||
A list of request IDs for the enqueued requests.
|
||||
@@ -545,6 +550,7 @@ class LLM:
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
@overload
|
||||
@@ -843,6 +849,7 @@ class LLM:
|
||||
self,
|
||||
prompts: Sequence[PromptType],
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
"""
|
||||
Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
|
||||
@@ -862,15 +869,29 @@ class LLM:
|
||||
tok_params = renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(tokenization_kwargs or {})
|
||||
)
|
||||
prompt_extras = (
|
||||
None
|
||||
if mm_processor_kwargs is None
|
||||
else {"mm_processor_kwargs": mm_processor_kwargs}
|
||||
)
|
||||
|
||||
return renderer.render_cmpl(parsed_prompts, tok_params)
|
||||
return renderer.render_cmpl(
|
||||
parsed_prompts,
|
||||
tok_params,
|
||||
prompt_extras=prompt_extras,
|
||||
)
|
||||
|
||||
def _preprocess_cmpl_one(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> EngineInput:
|
||||
(engine_input,) = self._preprocess_cmpl([prompt], tokenization_kwargs)
|
||||
(engine_input,) = self._preprocess_cmpl(
|
||||
[prompt],
|
||||
tokenization_kwargs,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return engine_input
|
||||
|
||||
def _preprocess_chat(
|
||||
@@ -908,16 +929,22 @@ class LLM:
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
),
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
tok_params = renderer.default_chat_tok_params.with_kwargs(
|
||||
**(tokenization_kwargs or {})
|
||||
)
|
||||
prompt_extras = (
|
||||
None
|
||||
if mm_processor_kwargs is None
|
||||
else {"mm_processor_kwargs": mm_processor_kwargs}
|
||||
)
|
||||
|
||||
_, engine_inputs = renderer.render_chat(
|
||||
conversations,
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
|
||||
prompt_extras=prompt_extras,
|
||||
)
|
||||
|
||||
return engine_inputs
|
||||
@@ -1571,6 +1598,7 @@ class LLM:
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[str]:
|
||||
seq_prompts = prompt_to_seq(prompts)
|
||||
seq_params = self._params_to_seq(params, len(seq_prompts))
|
||||
@@ -1579,7 +1607,11 @@ class LLM:
|
||||
|
||||
return self._render_and_add_requests(
|
||||
prompts=(
|
||||
self._preprocess_cmpl_one(prompt, tokenization_kwargs)
|
||||
self._preprocess_cmpl_one(
|
||||
prompt,
|
||||
tokenization_kwargs,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
for prompt in maybe_tqdm(
|
||||
seq_prompts,
|
||||
use_tqdm=use_tqdm,
|
||||
@@ -1603,6 +1635,7 @@ class LLM:
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
self._add_completion_requests(
|
||||
prompts=prompts,
|
||||
@@ -1611,6 +1644,7 @@ class LLM:
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return self._run_engine(use_tqdm=use_tqdm, output_type=output_type)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user