feat(multimodal): support externally processed mm_kwargs with cache injection (#39502)

Signed-off-by: Krish Hung <krishung5@gmail.com>
Signed-off-by: krishung5 <krish@nvidia.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kris Hung
2026-04-21 11:31:09 +00:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 28c222157b
commit 766cb65d00
2 changed files with 232 additions and 0 deletions
+39
View File
@@ -172,6 +172,45 @@ class InputProcessor:
return mm_hash
return f"{lora_request.lora_name}:{mm_hash}"
def inject_into_mm_cache(
self,
mm_hashes: dict[str, list[str]],
mm_kwargs: dict[str, list],
) -> None:
"""Inject pre-processed mm_kwargs into the processor cache.
Call this when mm_kwargs have already been through the HF processor
externally (e.g. by a frontend that transfers pre-processed tensors
to the backend). This ensures MM cache hit rate metrics are reported
accurately and avoids redundant processing on subsequent requests
with the same images.
Uses ``get_and_update_item()`` with an empty prompt_updates list,
since token expansion has already been handled externally.
"""
cache = self.renderer.mm_processor_cache
if cache is None:
return
try:
for modality, hashes in mm_hashes.items():
items = mm_kwargs.get(modality, [])
for i, mm_hash in enumerate(hashes):
if i < len(items) and items[i] is not None:
# Insert into cache via get_and_update_item.
# Use the returned item (may be an address for SHM
# cache or the original item for LRU cache).
items[i], _ = cache.get_and_update_item(
(items[i], []),
mm_hash,
)
# Update cache stats to reflect the externally processed items
self.renderer.update_mm_cache_stats()
except Exception:
logger.warning(
"Failed to inject mm_kwargs into processor cache",
exc_info=True,
)
@staticmethod
def assign_request_id(request: EngineCoreRequest):
"""Replace the externally supplied request ID with an internal request ID