[Misc] Forward request-level prompt extras for cross-encoder scoring (#46939)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Wentao Ye
parent
6eac8e0070
commit
fa1fa968c4
@@ -10,6 +10,7 @@ from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
|
|||||||
from vllm.inputs import EngineInput
|
from vllm.inputs import EngineInput
|
||||||
from vllm.renderers import TokenizeParams
|
from vllm.renderers import TokenizeParams
|
||||||
from vllm.renderers.hf import safe_apply_chat_template
|
from vllm.renderers.hf import safe_apply_chat_template
|
||||||
|
from vllm.renderers.inputs.preprocess import extract_target_prompt
|
||||||
from vllm.tasks import PoolingTask
|
from vllm.tasks import PoolingTask
|
||||||
from vllm.utils.mistral import is_mistral_tokenizer
|
from vllm.utils.mistral import is_mistral_tokenizer
|
||||||
|
|
||||||
@@ -433,8 +434,16 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
|||||||
max_tokens_per_doc: int = 0,
|
max_tokens_per_doc: int = 0,
|
||||||
prompt_extras: dict[str, Any] | None = None,
|
prompt_extras: dict[str, Any] | None = None,
|
||||||
) -> tuple[Sequence[EngineInput], list[PoolingParams]]:
|
) -> tuple[Sequence[EngineInput], list[PoolingParams]]:
|
||||||
# todo: support prompt_extras
|
|
||||||
arrival_time = time.time()
|
arrival_time = time.time()
|
||||||
|
engine_prompt_extras = (
|
||||||
|
{
|
||||||
|
k: v
|
||||||
|
for k in ("mm_processor_kwargs", "cache_salt")
|
||||||
|
if (v := prompt_extras.get(k)) is not None
|
||||||
|
}
|
||||||
|
if prompt_extras
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
data_1 = scoring_data.data_1
|
data_1 = scoring_data.data_1
|
||||||
data_2 = scoring_data.data_2
|
data_2 = scoring_data.data_2
|
||||||
@@ -472,6 +481,9 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
|||||||
pooling_params_list.append(pooling_params)
|
pooling_params_list.append(pooling_params)
|
||||||
|
|
||||||
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
|
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
|
||||||
|
if engine_prompt_extras:
|
||||||
|
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
|
||||||
|
target_prompt.update(engine_prompt_extras)
|
||||||
engine_inputs.append(
|
engine_inputs.append(
|
||||||
self.renderer.process_for_engine(engine_prompt, arrival_time)
|
self.renderer.process_for_engine(engine_prompt, arrival_time)
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user