[Core] MRV2 support for fast-prefill (#56145)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-09-10 10:36:04 -07:00
committed by GitHub
parent 7470082f57
commit e6cb56337b
12 changed files with 233 additions and 12 deletions
+23 -1
View File
@@ -89,6 +89,7 @@ from vllm.v1.worker.gpu.async_utils import (
StepTimingCollector,
)
from vllm.v1.worker.gpu.attn_utils import (
FastPrefillHelper,
build_slot_mappings_by_layer,
get_kv_cache_spec,
init_attn_backend,
@@ -321,6 +322,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_num_tokens=self.max_num_tokens,
device=self.device,
)
self.fast_prefill: FastPrefillHelper | None = None
if self.use_pp:
self.pp_handler = PPHandler(
max_num_reqs=self.max_num_reqs,
@@ -610,10 +612,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
for group in self.kv_cache_config.kv_cache_groups
for layer_name in group.layer_names
} - self.speculator.draft_attn_layer_names
draft_attn_layer_names = None
if isinstance(self.speculator, DraftModelSpeculator):
draft_attn_layer_names = self.speculator.draft_attn_layer_names
self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend(
self.kv_cache_config,
self.vllm_config,
self.device,
draft_layer_names=draft_attn_layer_names,
)
additional_attn_cg_support = self.model_state.get_additional_cg_support()
attn_cg_support = attn_cg_support.narrow(*additional_attn_cg_support)
@@ -665,7 +671,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
initialize_mamba_ssu_backend(
self.vllm_config.mamba_config,
self.kv_cache_config,
use_replayssm=self.vllm_config.cache_config.use_replayssm,
use_replayssm=self.cache_config.use_replayssm,
)
if self.adaptive_verification is not None:
self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
@@ -687,6 +693,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
lora_capture_cases=self.lora_capture_cases,
varlen_decode=self.adaptive_verification is not None,
)
if self.cache_config.kv_sharing_fast_prefill and self.pcp_manager is None:
self.fast_prefill = FastPrefillHelper(
self.cudagraph_manager, self.max_num_tokens
)
check_attention_cp_compatibility(self.vllm_config)
if isinstance(self.speculator, DraftModelSpeculator):
# HACK(woosuk)
@@ -1339,6 +1349,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.model_state.num_new_sampled_tokens_per_step,
)
fast_prefill = None
if self.fast_prefill is not None:
fast_prefill = self.fast_prefill.prepare(
logits_indices,
num_reqs,
cu_num_logits_np,
batch_req_state.has_prefill,
batch_desc,
)
# CPU upper bound on seq_lens; padded entries left at zero.
num_computed_tokens_np = self.req_states.num_computed_tokens_np[idx_mapping_np]
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
@@ -1385,6 +1405,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cu_num_logits_np=cu_num_logits_np,
has_structured_output_reqs=scheduler_output.has_structured_output_requests,
prompt_lens=prompt_lens,
fast_prefill=fast_prefill,
max_query_len=(
int(num_scheduled_tokens_upper_bound.max())
if adaptive_verification is not None
@@ -2127,6 +2148,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
memory is reclaimable when running in the same process."""
torch.accelerator.synchronize()
self.cudagraph_manager = None
self.fast_prefill = None
if hasattr(self, "kv_caches"):
self.kv_caches.clear()
if hasattr(self, "attn_groups"):