Encoder cache extension hooks (#48218)
Signed-off-by: hotTea <958436561@qq.com> Signed-off-by: hanxi-java <634498162@qq.com> Co-authored-by: hanxi-java <634498162@qq.com> Co-authored-by: 韩熙 <63780107+hanxi-java@users.noreply.github.com>
This commit is contained in:
@@ -46,6 +46,7 @@ def _gather(features, cached, *, num_scheduled, shift, num_computed=0):
|
||||
input_batch=SimpleNamespace(req_ids=["req0"]),
|
||||
requests={"req0": req_state},
|
||||
encoder_cache=encoder_cache,
|
||||
_get_encoder_output_from_cache=lambda mm_hash: encoder_cache.get(mm_hash),
|
||||
is_multimodal_pruning_enabled=False,
|
||||
uses_mrope=False,
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.config.compilation import (
|
||||
)
|
||||
from vllm.config.device import DeviceConfig
|
||||
from vllm.config.diffusion import DiffusionConfig
|
||||
from vllm.config.ec_manager_config import EncoderCacheManagerConfig
|
||||
from vllm.config.ec_transfer import ECTransferConfig
|
||||
from vllm.config.kernel import KernelConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
@@ -75,6 +76,8 @@ __all__ = [
|
||||
"DeviceConfig",
|
||||
# From vllm.config.diffusion
|
||||
"DiffusionConfig",
|
||||
# From vllm.config.ec_manager_config
|
||||
"EncoderCacheManagerConfig",
|
||||
# From vllm.config.ec_transfer
|
||||
"ECTransferConfig",
|
||||
# From vllm.config.kernel
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC
|
||||
|
||||
from vllm.config.utils import config
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
|
||||
@config
|
||||
class EncoderCacheManagerConfig:
|
||||
encoder_cache_manager_cls: str | None = None
|
||||
"""The qualified class name of the custom encoder cache manager.
|
||||
"""
|
||||
|
||||
def get_encoder_cache_manager_obj(self):
|
||||
cls_path = self.encoder_cache_manager_cls
|
||||
if cls_path is None:
|
||||
return None
|
||||
return resolve_obj_by_qualname(cls_path)
|
||||
|
||||
|
||||
class EncoderCacheManagerMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Abstract Metadata used to communicate between the
|
||||
Scheduler EncoderCacheManager and Worker EncoderCacheManager.
|
||||
"""
|
||||
|
||||
pass
|
||||
@@ -32,6 +32,7 @@ from .cache import CacheConfig
|
||||
from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode
|
||||
from .device import DeviceConfig
|
||||
from .diffusion import DiffusionConfig
|
||||
from .ec_manager_config import EncoderCacheManagerConfig
|
||||
from .ec_transfer import ECTransferConfig
|
||||
from .kernel import KernelConfig
|
||||
from .kv_events import KVEventsConfig
|
||||
@@ -367,6 +368,10 @@ class VllmConfig:
|
||||
"""The configurations for event publishing."""
|
||||
ec_transfer_config: ECTransferConfig | None = None
|
||||
"""The configurations for distributed EC cache transfer."""
|
||||
ec_manager_config: EncoderCacheManagerConfig = Field(
|
||||
default_factory=EncoderCacheManagerConfig
|
||||
)
|
||||
"""The configurations for custom encoder cache manager."""
|
||||
reasoning_config: ReasoningConfig | None = None
|
||||
"""The configurations for reasoning model."""
|
||||
# some opaque config, only used to provide additional information
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections import OrderedDict
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -265,6 +266,9 @@ class EncoderCacheManager:
|
||||
self.freed = []
|
||||
return freed
|
||||
|
||||
def get_manager_metadata(self) -> EncoderCacheManagerMetadata | None:
|
||||
return None
|
||||
|
||||
|
||||
def compute_mm_encoder_budget(
|
||||
scheduler_config: "SchedulerConfig",
|
||||
|
||||
@@ -5,6 +5,8 @@ from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
@@ -246,7 +248,8 @@ class SchedulerOutput:
|
||||
|
||||
# EC Cache Connector metadata
|
||||
ec_connector_metadata: ECConnectorMetadata | None = None
|
||||
|
||||
# EC Cache Manager metadata
|
||||
ec_manager_metadata: EncoderCacheManagerMetadata | None = None
|
||||
# Block IDs freshly allocated from the pool during this scheduling step.
|
||||
# The worker zeros the corresponding GPU memory before the blocks are used,
|
||||
# preventing stale NaN/data from corrupting attention or SSM computation.
|
||||
|
||||
@@ -227,12 +227,15 @@ class Scheduler(SchedulerInterface):
|
||||
mm_budget.encoder_compute_budget if mm_budget else 0
|
||||
)
|
||||
encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0
|
||||
self.encoder_cache_manager = (
|
||||
EncoderDecoderCacheManager(cache_size=encoder_cache_size)
|
||||
if self.is_encoder_decoder
|
||||
else EncoderCacheManager(cache_size=encoder_cache_size)
|
||||
)
|
||||
|
||||
manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj()
|
||||
if manager_cls_obj is not None:
|
||||
self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size)
|
||||
else:
|
||||
self.encoder_cache_manager = (
|
||||
EncoderDecoderCacheManager(cache_size=encoder_cache_size)
|
||||
if self.is_encoder_decoder
|
||||
else EncoderCacheManager(cache_size=encoder_cache_size)
|
||||
)
|
||||
speculative_config = vllm_config.speculative_config
|
||||
self.use_eagle = False
|
||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||
@@ -1151,6 +1154,7 @@ class Scheduler(SchedulerInterface):
|
||||
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
|
||||
kv_cache_block_copies=pending_kv_cache_block_copies,
|
||||
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
|
||||
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
|
||||
)
|
||||
|
||||
# NOTE(Kuntai): this function is designed for multiple purposes:
|
||||
|
||||
@@ -37,6 +37,7 @@ from vllm.config import (
|
||||
update_config,
|
||||
)
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES
|
||||
from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
@@ -1164,6 +1165,22 @@ class GPUModelRunner(
|
||||
self.async_output_copy_stream = stream
|
||||
return stream
|
||||
|
||||
def _on_request_state_removed(
|
||||
self,
|
||||
req_id: str,
|
||||
req_state: CachedRequestState | None,
|
||||
) -> None:
|
||||
"""Hook for platform runners to clean request-scoped side caches."""
|
||||
del req_id, req_state
|
||||
|
||||
def _process_encoder_cache_scheduler_output(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
) -> None:
|
||||
"""Apply scheduler-side encoder cache lifecycle updates."""
|
||||
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
||||
self.encoder_cache.pop(mm_hash, None)
|
||||
|
||||
def _update_states(self, scheduler_output: "SchedulerOutput") -> Callable | None:
|
||||
"""Update the cached states and the persistent batch with the scheduler
|
||||
output.
|
||||
@@ -1176,7 +1193,8 @@ class GPUModelRunner(
|
||||
"""
|
||||
# Remove finished requests from the cached states.
|
||||
for req_id in scheduler_output.finished_req_ids:
|
||||
self.requests.pop(req_id, None)
|
||||
req_state = self.requests.pop(req_id, None)
|
||||
self._on_request_state_removed(req_id, req_state)
|
||||
self.num_prompt_logprobs.pop(req_id, None)
|
||||
self.late_interaction_runner.on_requests_finished(
|
||||
scheduler_output.finished_req_ids
|
||||
@@ -1202,8 +1220,7 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
# Free the cached encoder outputs.
|
||||
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
||||
self.encoder_cache.pop(mm_hash, None)
|
||||
self._process_encoder_cache_scheduler_output(scheduler_output)
|
||||
|
||||
# Remove the unscheduled requests from the persistent batch.
|
||||
# NOTE(woosuk): The unscheduled requests are either preempted requests
|
||||
@@ -2958,6 +2975,18 @@ class GPUModelRunner(
|
||||
|
||||
return mm_hashes, mm_kwargs, mm_lora_refs
|
||||
|
||||
def _cache_encoder_output(
|
||||
self,
|
||||
mm_hash: str,
|
||||
output: torch.Tensor,
|
||||
ec_manager_metadata: "EncoderCacheManagerMetadata | None",
|
||||
free_encoder_mm_hashes: list[str],
|
||||
) -> None:
|
||||
"""Store an encoder output for later multimodal embedding gather."""
|
||||
del ec_manager_metadata, free_encoder_mm_hashes
|
||||
self.encoder_cache[mm_hash] = output
|
||||
self.maybe_save_ec_to_connector(self.encoder_cache, mm_hash)
|
||||
|
||||
def _execute_mm_encoder(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> list[torch.Tensor]:
|
||||
@@ -2983,8 +3012,12 @@ class GPUModelRunner(
|
||||
pe_tensor = mm_kwargs[i][1]["embedding"].data
|
||||
assert isinstance(pe_tensor, torch.Tensor)
|
||||
|
||||
self.encoder_cache[mm_hashes[i]] = pe_tensor.to(self.device)
|
||||
self.maybe_save_ec_to_connector(self.encoder_cache, mm_hashes[i])
|
||||
self._cache_encoder_output(
|
||||
mm_hashes[i],
|
||||
pe_tensor.to(self.device),
|
||||
scheduler_output.ec_manager_metadata,
|
||||
scheduler_output.free_encoder_mm_hashes,
|
||||
)
|
||||
# Filter out `prompt_embeds` items from mm_kwargs/mm_hashes/mm_lora_refs
|
||||
# since they don't require further encoder processing.
|
||||
mm_hashes = [h for i, h in enumerate(mm_hashes) if i not in pe_indices]
|
||||
@@ -3161,12 +3194,21 @@ class GPUModelRunner(
|
||||
|
||||
# Cache the encoder outputs by mm_hash
|
||||
for mm_hash, output in zip(mm_hashes, encoder_outputs):
|
||||
self.encoder_cache[mm_hash] = output
|
||||
self._cache_encoder_output(
|
||||
mm_hash,
|
||||
output,
|
||||
scheduler_output.ec_manager_metadata,
|
||||
scheduler_output.free_encoder_mm_hashes,
|
||||
)
|
||||
logger.debug("Finish execute for mm hash %s", mm_hash)
|
||||
self.maybe_save_ec_to_connector(self.encoder_cache, mm_hash)
|
||||
|
||||
return encoder_outputs
|
||||
|
||||
def _get_encoder_output_from_cache(self, mm_hash: str) -> torch.Tensor | None:
|
||||
"""Return a cached encoder output for multimodal
|
||||
embedding gather."""
|
||||
return self.encoder_cache.get(mm_hash, None)
|
||||
|
||||
def _gather_mm_embeddings(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
@@ -3220,7 +3262,7 @@ class GPUModelRunner(
|
||||
continue
|
||||
|
||||
mm_hash = mm_feature.identifier
|
||||
encoder_output = self.encoder_cache.get(mm_hash, None)
|
||||
encoder_output = self._get_encoder_output_from_cache(mm_hash)
|
||||
if encoder_output is None:
|
||||
# A feature starting at/after the processed boundary is only
|
||||
# reached via the drafter's +1 look-ahead and might not be
|
||||
|
||||
Reference in New Issue
Block a user