[Core] MRV2 support for fast-prefill (#56145)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -89,6 +89,7 @@ from vllm.v1.worker.gpu.async_utils import (
|
||||
StepTimingCollector,
|
||||
)
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
FastPrefillHelper,
|
||||
build_slot_mappings_by_layer,
|
||||
get_kv_cache_spec,
|
||||
init_attn_backend,
|
||||
@@ -321,6 +322,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_num_tokens=self.max_num_tokens,
|
||||
device=self.device,
|
||||
)
|
||||
self.fast_prefill: FastPrefillHelper | None = None
|
||||
if self.use_pp:
|
||||
self.pp_handler = PPHandler(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -610,10 +612,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
for group in self.kv_cache_config.kv_cache_groups
|
||||
for layer_name in group.layer_names
|
||||
} - self.speculator.draft_attn_layer_names
|
||||
draft_attn_layer_names = None
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
draft_attn_layer_names = self.speculator.draft_attn_layer_names
|
||||
self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend(
|
||||
self.kv_cache_config,
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
draft_layer_names=draft_attn_layer_names,
|
||||
)
|
||||
additional_attn_cg_support = self.model_state.get_additional_cg_support()
|
||||
attn_cg_support = attn_cg_support.narrow(*additional_attn_cg_support)
|
||||
@@ -665,7 +671,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
initialize_mamba_ssu_backend(
|
||||
self.vllm_config.mamba_config,
|
||||
self.kv_cache_config,
|
||||
use_replayssm=self.vllm_config.cache_config.use_replayssm,
|
||||
use_replayssm=self.cache_config.use_replayssm,
|
||||
)
|
||||
if self.adaptive_verification is not None:
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
@@ -687,6 +693,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
lora_capture_cases=self.lora_capture_cases,
|
||||
varlen_decode=self.adaptive_verification is not None,
|
||||
)
|
||||
if self.cache_config.kv_sharing_fast_prefill and self.pcp_manager is None:
|
||||
self.fast_prefill = FastPrefillHelper(
|
||||
self.cudagraph_manager, self.max_num_tokens
|
||||
)
|
||||
check_attention_cp_compatibility(self.vllm_config)
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
# HACK(woosuk)
|
||||
@@ -1339,6 +1349,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.model_state.num_new_sampled_tokens_per_step,
|
||||
)
|
||||
|
||||
fast_prefill = None
|
||||
if self.fast_prefill is not None:
|
||||
fast_prefill = self.fast_prefill.prepare(
|
||||
logits_indices,
|
||||
num_reqs,
|
||||
cu_num_logits_np,
|
||||
batch_req_state.has_prefill,
|
||||
batch_desc,
|
||||
)
|
||||
|
||||
# CPU upper bound on seq_lens; padded entries left at zero.
|
||||
num_computed_tokens_np = self.req_states.num_computed_tokens_np[idx_mapping_np]
|
||||
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
|
||||
@@ -1385,6 +1405,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cu_num_logits_np=cu_num_logits_np,
|
||||
has_structured_output_reqs=scheduler_output.has_structured_output_requests,
|
||||
prompt_lens=prompt_lens,
|
||||
fast_prefill=fast_prefill,
|
||||
max_query_len=(
|
||||
int(num_scheduled_tokens_upper_bound.max())
|
||||
if adaptive_verification is not None
|
||||
@@ -2127,6 +2148,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
memory is reclaimable when running in the same process."""
|
||||
torch.accelerator.synchronize()
|
||||
self.cudagraph_manager = None
|
||||
self.fast_prefill = None
|
||||
if hasattr(self, "kv_caches"):
|
||||
self.kv_caches.clear()
|
||||
if hasattr(self, "attn_groups"):
|
||||
|
||||
Reference in New Issue
Block a user