[Core] Replace routing replay with device cache and async D2H pipeline (#39917)
Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
@@ -54,7 +54,11 @@ from vllm.lora.layers import LoRAMapping, LoRAMappingType
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
extract_routed_experts_for_current_batch,
|
||||
free_routing_buffers,
|
||||
get_global_experts_capturer,
|
||||
init_routed_experts_capturer_with_shared_cache,
|
||||
issue_routing_d2h_copy,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
@@ -1092,6 +1096,12 @@ class GPUModelRunner(
|
||||
for req_id in scheduler_output.finished_req_ids:
|
||||
self.input_batch.remove_request(req_id)
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
free_routing_buffers(
|
||||
scheduler_output.finished_req_ids,
|
||||
scheduler_output.preempted_req_ids,
|
||||
)
|
||||
|
||||
# Zero GPU memory for freshly allocated cache blocks to prevent
|
||||
# stale NaN/data from corrupting attention or SSM computation.
|
||||
if scheduler_output.new_block_ids_to_zero:
|
||||
@@ -2175,10 +2185,6 @@ class GPUModelRunner(
|
||||
block_table_gid_0 = _get_block_table(0)
|
||||
slot_mapping_gid_0 = slot_mappings[0]
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
attn_gid = self.routed_experts_attn_gid
|
||||
slot_mapping_attn = slot_mappings[attn_gid]
|
||||
self.slot_mapping = slot_mapping_attn[:num_tokens].cpu().numpy()
|
||||
num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[
|
||||
:num_reqs_padded
|
||||
]
|
||||
@@ -3859,11 +3865,9 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
capturer = get_global_experts_capturer()
|
||||
if capturer is not None:
|
||||
capturer.clear_buffer() # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
capturer.finalize_pending_copy()
|
||||
|
||||
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
|
||||
# the modification has influence on the scheduler_output in engine core process.
|
||||
@@ -4373,6 +4377,14 @@ class GPUModelRunner(
|
||||
scheduler_output.total_num_scheduled_tokens,
|
||||
)
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
issue_routing_d2h_copy(
|
||||
input_batch_req_ids=self.input_batch.req_ids,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
positions=self.positions,
|
||||
positions_cpu=self._positions_cpu,
|
||||
)
|
||||
|
||||
if propose_drafts_after_bookkeeping:
|
||||
# ngram and other speculative decoding methods use the sampled
|
||||
# tokens on the CPU, so they are run after bookkeeping.
|
||||
@@ -4392,12 +4404,15 @@ class GPUModelRunner(
|
||||
self.kv_connector_output = None
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"):
|
||||
routed_experts_dict = None
|
||||
if self.routed_experts_initialized:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.save_captured_experts(indices=self.slot_mapping) # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
routed_experts_dict = extract_routed_experts_for_current_batch(
|
||||
req_ids=req_ids_output_copy,
|
||||
requests=self.requests,
|
||||
req_id_to_index=self.input_batch.req_id_to_index,
|
||||
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
|
||||
max_model_len=self.max_model_len,
|
||||
)
|
||||
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=req_ids_output_copy,
|
||||
@@ -4411,6 +4426,7 @@ class GPUModelRunner(
|
||||
else None,
|
||||
num_nans_in_logits=num_nans_in_logits,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
routed_experts_dict=routed_experts_dict,
|
||||
)
|
||||
|
||||
if not self.use_async_scheduling:
|
||||
@@ -6138,6 +6154,7 @@ class GPUModelRunner(
|
||||
"Skipping CUDA graph capture. To turn on CUDA graph capture, "
|
||||
"ensure `cudagraph_mode` was not manually set to `NONE`"
|
||||
)
|
||||
self.init_routed_experts_capturer()
|
||||
return 0
|
||||
|
||||
# Initialize encoder CUDA graph manager if enabled.
|
||||
@@ -6171,6 +6188,13 @@ class GPUModelRunner(
|
||||
|
||||
start_time = time.perf_counter()
|
||||
|
||||
# Initialize the routed experts capturer once before any CUDA graph
|
||||
# capture. Must happen before graphs are captured so the buffer
|
||||
# address is baked into the graph. Do NOT call this inside
|
||||
# _capture_cudagraphs() -- creating the capturer twice replaces
|
||||
# the device buffer, causing graphs to write to a dead buffer.
|
||||
self.init_routed_experts_capturer()
|
||||
|
||||
# Trigger CUDA graph capture for specific shapes.
|
||||
# Capture the large shapes first so that the smaller shapes
|
||||
# can reuse the memory pool allocated for the large shapes.
|
||||
@@ -6965,45 +6989,40 @@ class GPUModelRunner(
|
||||
"Initializing routed experts capturer, enable_return_routed_experts: %s",
|
||||
self.model_config.enable_return_routed_experts,
|
||||
)
|
||||
routed_experts_capturer = RoutedExpertsCapturer.create()
|
||||
self.routed_experts_attn_gid = self._get_attention_kv_cache_gid()
|
||||
min_block_size = min(
|
||||
[
|
||||
group.kv_cache_spec.block_size
|
||||
for group in self.kv_cache_config.kv_cache_groups
|
||||
]
|
||||
)
|
||||
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||
self.max_num_kv_tokens = (
|
||||
self.kv_cache_config.num_blocks // num_groups
|
||||
) * min_block_size
|
||||
dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size
|
||||
pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
|
||||
if pcp_size * dcp_size > 1:
|
||||
self.max_num_kv_tokens *= pcp_size * dcp_size
|
||||
from vllm.distributed import get_tp_group
|
||||
|
||||
routed_experts_capturer.init_buffer(
|
||||
if hasattr(self.model_config.hf_text_config, "n_shared_experts"):
|
||||
num_fused_shared_experts = 1
|
||||
else:
|
||||
num_fused_shared_experts = 0
|
||||
|
||||
tp_group = get_tp_group()
|
||||
init_routed_experts_capturer_with_shared_cache(
|
||||
enable=self.model_config.enable_return_routed_experts,
|
||||
model_config=self.model_config,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
|
||||
max_num_kv_tokens=self.max_num_kv_tokens,
|
||||
vllm_config=self.vllm_config,
|
||||
max_model_len=self.max_model_len,
|
||||
device=self.device,
|
||||
rank=tp_group.rank_in_group,
|
||||
world_size=tp_group.world_size,
|
||||
)
|
||||
self._bind_routed_experts_capturer(routed_experts_capturer)
|
||||
self._bind_routed_experts_capturer()
|
||||
self.routed_experts_initialized = True
|
||||
|
||||
def _bind_routed_experts_capturer(self, capturer: RoutedExpertsCapturer) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
BaseRouter,
|
||||
# Pinned CPU buffer for async positions D2H (avoids sync .cpu() call)
|
||||
self._positions_cpu = torch.empty(
|
||||
self.scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.long,
|
||||
pin_memory=True,
|
||||
)
|
||||
|
||||
for module in self.compilation_config.static_forward_context.values():
|
||||
if isinstance(module, FusedMoE) and isinstance(module.router, BaseRouter):
|
||||
layer_id = module.layer_id
|
||||
def _bind_routed_experts_capturer(self) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
bind_routing_capture_to_model,
|
||||
)
|
||||
|
||||
def _capture_fn(topk_ids, _layer_id=layer_id, _capturer=capturer):
|
||||
_capturer.capture(_layer_id, topk_ids)
|
||||
|
||||
module.router.set_capture_fn(_capture_fn)
|
||||
bind_routing_capture_to_model(self.model)
|
||||
|
||||
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user