[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -52,6 +52,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_slot_mappings_by_layer,
|
||||
@@ -205,6 +206,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Draft tokens propagation - for spec-dec + struct outputs.
|
||||
self.draft_tokens_handler = DraftTokensHandler(self.device)
|
||||
|
||||
self.pcp_manager: pcp.PCPManager | None = None
|
||||
|
||||
# Pooling models.
|
||||
self.is_pooling_model = self.model_config.runner_type == "pooling"
|
||||
self.pooling_runner: PoolingRunner | None = None
|
||||
@@ -456,6 +459,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cp_rank=self.dcp_rank,
|
||||
cp_interleave=self.cp_interleave,
|
||||
)
|
||||
self.pcp_manager = pcp.maybe_build_pcp_manager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
self.supports_mm_inputs,
|
||||
self.req_states,
|
||||
self.block_tables,
|
||||
)
|
||||
initialize_mamba_ssu_backend(
|
||||
self.vllm_config.mamba_config, self.kv_cache_config
|
||||
)
|
||||
@@ -1008,7 +1018,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# prompt_lens is only used in R-SWA case.
|
||||
prompt_lens = self.req_states.prompt_len.gpu[idx_mapping]
|
||||
|
||||
return InputBatch(
|
||||
input_batch = InputBatch(
|
||||
req_ids=req_ids,
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_after_padding=num_reqs_padded,
|
||||
@@ -1040,10 +1050,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
has_structured_output_reqs=scheduler_output.has_structured_output_requests,
|
||||
prompt_lens=prompt_lens,
|
||||
)
|
||||
return pcp.maybe_partition_pcp_batch(self.pcp_manager, input_batch)
|
||||
|
||||
def prepare_attn(
|
||||
self, input_batch: InputBatch
|
||||
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
|
||||
if self.pcp_manager is not None:
|
||||
return self.pcp_manager.prepare_attn(input_batch)
|
||||
|
||||
# Block tables: num_kv_cache_groups x [num_reqs_padded, max_num_blocks].
|
||||
block_tables = self.block_tables.gather_block_tables(
|
||||
input_batch.idx_mapping,
|
||||
@@ -1063,8 +1077,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self, input_batch: InputBatch
|
||||
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
|
||||
block_tables = self.block_tables.get_dummy_block_tables(input_batch.num_reqs)
|
||||
slot_mappings = self.block_tables.get_dummy_slot_mappings(
|
||||
input_batch.num_tokens
|
||||
slot_mappings = pcp.maybe_get_pcp_dummy_slot_mappings(
|
||||
self.pcp_manager, self.block_tables, input_batch.num_tokens
|
||||
)
|
||||
return block_tables, slot_mappings
|
||||
|
||||
@@ -1412,6 +1426,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
|
||||
# Last rank: sample tokens
|
||||
hidden_states, input_batch = pcp.maybe_restore_pcp_for_sampling(
|
||||
self.pcp_manager, hidden_states, input_batch
|
||||
)
|
||||
|
||||
sampler_output, num_sampled, num_rejected = self.sample(
|
||||
hidden_states, input_batch, grammar_output
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user