Encoder cache extension hooks (#48218)

Signed-off-by: hotTea <958436561@qq.com>
Signed-off-by: hanxi-java <634498162@qq.com>
Co-authored-by: hanxi-java <634498162@qq.com>
Co-authored-by: 韩熙 <63780107+hanxi-java@users.noreply.github.com>
This commit is contained in:
hotTea
2026-07-24 02:52:00 -07:00
committed by GitHub
co-authored by hanxi-java 韩熙
parent 163ecba377
commit 833483f357
8 changed files with 106 additions and 15 deletions
@@ -46,6 +46,7 @@ def _gather(features, cached, *, num_scheduled, shift, num_computed=0):
input_batch=SimpleNamespace(req_ids=["req0"]),
requests={"req0": req_state},
encoder_cache=encoder_cache,
_get_encoder_output_from_cache=lambda mm_hash: encoder_cache.get(mm_hash),
is_multimodal_pruning_enabled=False,
uses_mrope=False,
)
+3
View File
@@ -11,6 +11,7 @@ from vllm.config.compilation import (
)
from vllm.config.device import DeviceConfig
from vllm.config.diffusion import DiffusionConfig
from vllm.config.ec_manager_config import EncoderCacheManagerConfig
from vllm.config.ec_transfer import ECTransferConfig
from vllm.config.kernel import KernelConfig
from vllm.config.kv_events import KVEventsConfig
@@ -75,6 +76,8 @@ __all__ = [
"DeviceConfig",
# From vllm.config.diffusion
"DiffusionConfig",
# From vllm.config.ec_manager_config
"EncoderCacheManagerConfig",
# From vllm.config.ec_transfer
"ECTransferConfig",
# From vllm.config.kernel
+29
View File
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC
from vllm.config.utils import config
from vllm.utils.import_utils import resolve_obj_by_qualname
@config
class EncoderCacheManagerConfig:
encoder_cache_manager_cls: str | None = None
"""The qualified class name of the custom encoder cache manager.
"""
def get_encoder_cache_manager_obj(self):
cls_path = self.encoder_cache_manager_cls
if cls_path is None:
return None
return resolve_obj_by_qualname(cls_path)
class EncoderCacheManagerMetadata(ABC): # noqa: B024
"""
Abstract Metadata used to communicate between the
Scheduler EncoderCacheManager and Worker EncoderCacheManager.
"""
pass
+5
View File
@@ -32,6 +32,7 @@ from .cache import CacheConfig
from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode
from .device import DeviceConfig
from .diffusion import DiffusionConfig
from .ec_manager_config import EncoderCacheManagerConfig
from .ec_transfer import ECTransferConfig
from .kernel import KernelConfig
from .kv_events import KVEventsConfig
@@ -367,6 +368,10 @@ class VllmConfig:
"""The configurations for event publishing."""
ec_transfer_config: ECTransferConfig | None = None
"""The configurations for distributed EC cache transfer."""
ec_manager_config: EncoderCacheManagerConfig = Field(
default_factory=EncoderCacheManagerConfig
)
"""The configurations for custom encoder cache manager."""
reasoning_config: ReasoningConfig | None = None
"""The configurations for reasoning model."""
# some opaque config, only used to provide additional information
+4
View File
@@ -5,6 +5,7 @@ from collections import OrderedDict
from collections.abc import Mapping
from typing import TYPE_CHECKING
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
from vllm.logger import init_logger
from vllm.v1.request import Request
@@ -265,6 +266,9 @@ class EncoderCacheManager:
self.freed = []
return freed
def get_manager_metadata(self) -> EncoderCacheManagerMetadata | None:
return None
def compute_mm_encoder_budget(
scheduler_config: "SchedulerConfig",
+4 -1
View File
@@ -5,6 +5,8 @@ from dataclasses import dataclass
from functools import cached_property
from typing import TYPE_CHECKING
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
if TYPE_CHECKING:
import numpy as np
import numpy.typing as npt
@@ -246,7 +248,8 @@ class SchedulerOutput:
# EC Cache Connector metadata
ec_connector_metadata: ECConnectorMetadata | None = None
# EC Cache Manager metadata
ec_manager_metadata: EncoderCacheManagerMetadata | None = None
# Block IDs freshly allocated from the pool during this scheduling step.
# The worker zeros the corresponding GPU memory before the blocks are used,
# preventing stale NaN/data from corrupting attention or SSM computation.
+10 -6
View File
@@ -227,12 +227,15 @@ class Scheduler(SchedulerInterface):
mm_budget.encoder_compute_budget if mm_budget else 0
)
encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0
self.encoder_cache_manager = (
EncoderDecoderCacheManager(cache_size=encoder_cache_size)
if self.is_encoder_decoder
else EncoderCacheManager(cache_size=encoder_cache_size)
)
manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj()
if manager_cls_obj is not None:
self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size)
else:
self.encoder_cache_manager = (
EncoderDecoderCacheManager(cache_size=encoder_cache_size)
if self.is_encoder_decoder
else EncoderCacheManager(cache_size=encoder_cache_size)
)
speculative_config = vllm_config.speculative_config
self.use_eagle = False
self.num_spec_tokens = vllm_config.num_speculative_tokens
@@ -1151,6 +1154,7 @@ class Scheduler(SchedulerInterface):
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
kv_cache_block_copies=pending_kv_cache_block_copies,
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
)
# NOTE(Kuntai): this function is designed for multiple purposes:
+50 -8
View File
@@ -37,6 +37,7 @@ from vllm.config import (
update_config,
)
from vllm.config.cache import CacheConfig
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
from vllm.config.model import PROCESSED_LOGPROBS_MODES
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
from vllm.distributed.eplb.eplb_state import EplbState
@@ -1164,6 +1165,22 @@ class GPUModelRunner(
self.async_output_copy_stream = stream
return stream
def _on_request_state_removed(
self,
req_id: str,
req_state: CachedRequestState | None,
) -> None:
"""Hook for platform runners to clean request-scoped side caches."""
del req_id, req_state
def _process_encoder_cache_scheduler_output(
self,
scheduler_output: "SchedulerOutput",
) -> None:
"""Apply scheduler-side encoder cache lifecycle updates."""
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self.encoder_cache.pop(mm_hash, None)
def _update_states(self, scheduler_output: "SchedulerOutput") -> Callable | None:
"""Update the cached states and the persistent batch with the scheduler
output.
@@ -1176,7 +1193,8 @@ class GPUModelRunner(
"""
# Remove finished requests from the cached states.
for req_id in scheduler_output.finished_req_ids:
self.requests.pop(req_id, None)
req_state = self.requests.pop(req_id, None)
self._on_request_state_removed(req_id, req_state)
self.num_prompt_logprobs.pop(req_id, None)
self.late_interaction_runner.on_requests_finished(
scheduler_output.finished_req_ids
@@ -1202,8 +1220,7 @@ class GPUModelRunner(
)
# Free the cached encoder outputs.
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self.encoder_cache.pop(mm_hash, None)
self._process_encoder_cache_scheduler_output(scheduler_output)
# Remove the unscheduled requests from the persistent batch.
# NOTE(woosuk): The unscheduled requests are either preempted requests
@@ -2958,6 +2975,18 @@ class GPUModelRunner(
return mm_hashes, mm_kwargs, mm_lora_refs
def _cache_encoder_output(
self,
mm_hash: str,
output: torch.Tensor,
ec_manager_metadata: "EncoderCacheManagerMetadata | None",
free_encoder_mm_hashes: list[str],
) -> None:
"""Store an encoder output for later multimodal embedding gather."""
del ec_manager_metadata, free_encoder_mm_hashes
self.encoder_cache[mm_hash] = output
self.maybe_save_ec_to_connector(self.encoder_cache, mm_hash)
def _execute_mm_encoder(
self, scheduler_output: "SchedulerOutput"
) -> list[torch.Tensor]:
@@ -2983,8 +3012,12 @@ class GPUModelRunner(
pe_tensor = mm_kwargs[i][1]["embedding"].data
assert isinstance(pe_tensor, torch.Tensor)
self.encoder_cache[mm_hashes[i]] = pe_tensor.to(self.device)
self.maybe_save_ec_to_connector(self.encoder_cache, mm_hashes[i])
self._cache_encoder_output(
mm_hashes[i],
pe_tensor.to(self.device),
scheduler_output.ec_manager_metadata,
scheduler_output.free_encoder_mm_hashes,
)
# Filter out `prompt_embeds` items from mm_kwargs/mm_hashes/mm_lora_refs
# since they don't require further encoder processing.
mm_hashes = [h for i, h in enumerate(mm_hashes) if i not in pe_indices]
@@ -3161,12 +3194,21 @@ class GPUModelRunner(
# Cache the encoder outputs by mm_hash
for mm_hash, output in zip(mm_hashes, encoder_outputs):
self.encoder_cache[mm_hash] = output
self._cache_encoder_output(
mm_hash,
output,
scheduler_output.ec_manager_metadata,
scheduler_output.free_encoder_mm_hashes,
)
logger.debug("Finish execute for mm hash %s", mm_hash)
self.maybe_save_ec_to_connector(self.encoder_cache, mm_hash)
return encoder_outputs
def _get_encoder_output_from_cache(self, mm_hash: str) -> torch.Tensor | None:
"""Return a cached encoder output for multimodal
embedding gather."""
return self.encoder_cache.get(mm_hash, None)
def _gather_mm_embeddings(
self,
scheduler_output: "SchedulerOutput",
@@ -3220,7 +3262,7 @@ class GPUModelRunner(
continue
mm_hash = mm_feature.identifier
encoder_output = self.encoder_cache.get(mm_hash, None)
encoder_output = self._get_encoder_output_from_cache(mm_hash)
if encoder_output is None:
# A feature starting at/after the processed boundary is only
# reached via the drafter's +1 look-ahead and might not be