[Bugfix][Pooling] Preserve cache_salt for chunked embeddings (#47696)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
@@ -142,13 +142,16 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
)
|
||||
|
||||
prompt_token_ids = cast(list[int], token_ids)
|
||||
cache_salt = cast(str | None, engine_input["prompts"].get("cache_salt"))
|
||||
|
||||
for chunk_idx, chunk_tokens in enumerate(
|
||||
chunk_list(prompt_token_ids, max_model_len)
|
||||
):
|
||||
chunked_engine_inputs.append(
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=chunk_tokens),
|
||||
prompts=tokens_input(
|
||||
prompt_token_ids=chunk_tokens, cache_salt=cache_salt
|
||||
),
|
||||
params=engine_input["params"],
|
||||
lora_requests=engine_input["lora_requests"],
|
||||
priorities=engine_input["priorities"],
|
||||
|
||||
Reference in New Issue
Block a user