[V1][Hybrid] Mamba Prefix Caching with align mode (#30877)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
co-authored by
Chen Zhang
parent
fec9da0af4
commit
5206e5e28c
@@ -152,7 +152,11 @@ from vllm.v1.spec_decode.ngram_proposer import NgramProposer
|
||||
from vllm.v1.spec_decode.suffix_decoding import SuffixDecodingProposer
|
||||
from vllm.v1.structured_output.utils import apply_grammar_bitmask
|
||||
from vllm.v1.utils import CpuGpuBuffer, record_function_or_nullcontext
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
from vllm.v1.worker import mamba_utils
|
||||
from vllm.v1.worker.cp_utils import (
|
||||
check_attention_cp_compatibility,
|
||||
get_total_cp_world_size,
|
||||
)
|
||||
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
|
||||
from vllm.v1.worker.ec_connector_model_runner_mixin import ECConnectorModelRunnerMixin
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
@@ -688,6 +692,7 @@ class GPUModelRunner(
|
||||
# Ephemeral state transferred between execute_model() and sample_tokens().
|
||||
self.execute_model_state: ExecuteModelState | None = None
|
||||
self.kv_connector_output: KVConnectorOutput | None = None
|
||||
self.mamba_state_idx: dict[str, int] = {}
|
||||
self.layerwise_nvtx_hooks_registered = False
|
||||
|
||||
def update_max_model_len(self, max_model_len: int) -> None:
|
||||
@@ -1075,7 +1080,7 @@ class GPUModelRunner(
|
||||
self.input_batch.refresh_metadata()
|
||||
|
||||
def _update_states_after_model_execute(
|
||||
self, output_token_ids: torch.Tensor
|
||||
self, output_token_ids: torch.Tensor, scheduler_output: "SchedulerOutput"
|
||||
) -> None:
|
||||
"""Update the cached states after model execution.
|
||||
|
||||
@@ -1111,6 +1116,16 @@ class GPUModelRunner(
|
||||
)
|
||||
for i, num_tokens in enumerate(num_accepted_tokens):
|
||||
self.input_batch.num_accepted_tokens_cpu[i] = num_tokens
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
mamba_utils.postprocess_mamba(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.mamba_state_idx,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
)
|
||||
|
||||
def _init_mrope_positions(self, req_state: CachedRequestState):
|
||||
model = self.get_model()
|
||||
@@ -2751,7 +2766,6 @@ class GPUModelRunner(
|
||||
logits,
|
||||
sampling_metadata,
|
||||
)
|
||||
self._update_states_after_model_execute(sampler_output.sampled_token_ids)
|
||||
return sampler_output
|
||||
|
||||
def _bookkeeping_sync(
|
||||
@@ -3237,6 +3251,18 @@ class GPUModelRunner(
|
||||
|
||||
pad_attn = cudagraph_mode == CUDAGraphMode.FULL
|
||||
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
mamba_utils.preprocess_mamba(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.cache_config,
|
||||
self.mamba_state_idx,
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
)
|
||||
|
||||
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
|
||||
ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices
|
||||
|
||||
@@ -3423,6 +3449,10 @@ class GPUModelRunner(
|
||||
with record_function_or_nullcontext("gpu_model_runner: sample"):
|
||||
sampler_output = self._sample(logits, spec_decode_metadata)
|
||||
|
||||
self._update_states_after_model_execute(
|
||||
sampler_output.sampled_token_ids, scheduler_output
|
||||
)
|
||||
|
||||
self._draft_token_ids = None
|
||||
self._draft_token_req_ids = None
|
||||
self.input_batch.prev_sampled_token_ids = None
|
||||
@@ -5322,6 +5352,24 @@ class GPUModelRunner(
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups
|
||||
if not isinstance(kv_cache_group.kv_cache_spec, EncoderOnlyAttentionSpec)
|
||||
]
|
||||
max_num_blocks = []
|
||||
max_model_len = max(self.max_model_len, self.max_encoder_len)
|
||||
for i, kv_cache_group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if isinstance(kv_cache_group.kv_cache_spec, EncoderOnlyAttentionSpec):
|
||||
continue
|
||||
max_num_blocks_per_req = cdiv(
|
||||
max_model_len, block_sizes[i] * get_total_cp_world_size()
|
||||
)
|
||||
if isinstance(kv_cache_group.kv_cache_spec, MambaSpec):
|
||||
mamba_blocks_per_req = (
|
||||
max_num_blocks_per_req
|
||||
if self.cache_config.enable_prefix_caching
|
||||
else 1
|
||||
) + kv_cache_group.kv_cache_spec.num_speculative_blocks
|
||||
max_num_blocks_per_req = max(
|
||||
max_num_blocks_per_req, mamba_blocks_per_req
|
||||
)
|
||||
max_num_blocks.append(max_num_blocks_per_req)
|
||||
|
||||
if block_sizes != [self.cache_config.block_size] or kernel_block_sizes != [
|
||||
self.cache_config.block_size
|
||||
@@ -5333,18 +5381,18 @@ class GPUModelRunner(
|
||||
)
|
||||
self.input_batch = InputBatch(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_model_len=max(self.max_model_len, self.max_encoder_len),
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks_per_req=max_num_blocks,
|
||||
is_spec_decode=bool(self.vllm_config.speculative_config),
|
||||
logitsprocs=self.input_batch.logitsprocs,
|
||||
logitsprocs_need_output_token_ids=self.input_batch.logitsprocs_need_output_token_ids,
|
||||
is_pooling_model=self.is_pooling_model,
|
||||
num_speculative_tokens=self.num_spec_tokens,
|
||||
)
|
||||
|
||||
def _allocate_kv_cache_tensors(
|
||||
|
||||
Reference in New Issue
Block a user