[Hybrid] Enable mamba prefix cache "align" mode with async scheduling (#33997)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
This commit is contained in:
@@ -648,11 +648,6 @@ class VllmConfig:
|
||||
"`external_launcher` distributed executor backend, but you chose "
|
||||
f"`{executor_backend}`."
|
||||
)
|
||||
if self.cache_config.mamba_cache_mode != "none":
|
||||
raise ValueError(
|
||||
"Currently, async scheduling is not compatible with "
|
||||
"prefix caching for Mamba models."
|
||||
)
|
||||
elif self.scheduler_config.async_scheduling is None:
|
||||
# Enable async scheduling unless there is an incompatible option.
|
||||
if (
|
||||
@@ -685,13 +680,6 @@ class VllmConfig:
|
||||
scope="local",
|
||||
)
|
||||
self.scheduler_config.async_scheduling = False
|
||||
elif self.cache_config.mamba_cache_mode != "none":
|
||||
logger.warning_once(
|
||||
"Async scheduling is not compatible with "
|
||||
"prefix caching for Mamba models and will be disabled.",
|
||||
scope="local",
|
||||
)
|
||||
self.scheduler_config.async_scheduling = False
|
||||
else:
|
||||
self.scheduler_config.async_scheduling = True
|
||||
|
||||
|
||||
@@ -814,6 +814,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
|
||||
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
assert isinstance(self.kv_cache_spec, MambaSpec)
|
||||
|
||||
# NOTE (tdoublep) with async scheduling, the num_computed_tokens can contain
|
||||
# draft tokens from the previous step that may or may not be rejected later.
|
||||
# This can make us think we are further ahead in the sequence than we actually
|
||||
# are, so let's assume that all tokens are rejected so we don't free blocks
|
||||
# that we might actually need.
|
||||
num_computed_tokens = max(0, num_computed_tokens - self.num_speculative_blocks)
|
||||
|
||||
super().remove_skipped_blocks(request_id, num_computed_tokens)
|
||||
if self.mamba_cache_mode == "align":
|
||||
# `last_state_block_idx` refers to the block index allocated two steps ago.
|
||||
@@ -879,6 +887,9 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# We can ignore lookahead tokens because current draft models don't have
|
||||
# mamba layers.
|
||||
num_tokens = num_tokens_main_model
|
||||
|
||||
# NOTE(tdouble): this is an over-estimate of how many blocks we need because
|
||||
# num_tokens can include draft tokens that will later be rejected.
|
||||
num_required_blocks = (
|
||||
cdiv(num_tokens, self.block_size) + self.num_speculative_blocks
|
||||
)
|
||||
@@ -922,6 +933,8 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# mamba layers.
|
||||
num_tokens = num_tokens_main_model
|
||||
req_blocks: list[KVCacheBlock] = self.req_to_blocks[request_id]
|
||||
# NOTE(tdouble): this is an over-estimate of how many blocks we need because
|
||||
# num_tokens can include draft tokens that will later be rejected.
|
||||
num_required_blocks = (
|
||||
cdiv(num_tokens, self.block_size) + self.num_speculative_blocks
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState
|
||||
@@ -142,7 +143,11 @@ def preprocess_mamba(
|
||||
# if num_computed_tokens is 0, prev_state_idx will be -1
|
||||
prev_state_idx = (req_state.num_computed_tokens - 1) // block_size
|
||||
|
||||
num_blocks = len(req_state.block_ids[mamba_group_ids[0]])
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
num_blocks: int = (
|
||||
cdiv(req_state.num_computed_tokens + num_scheduled_tokens, block_size)
|
||||
+ num_speculative_blocks
|
||||
)
|
||||
|
||||
# We always save the current running state at the last
|
||||
# (1 + num_speculative_blocks) block.
|
||||
|
||||
Reference in New Issue
Block a user