[Feature] Support recording expert indices for rollout router replay (#28284)
Signed-off-by: xhx1022 <1737006628@qq.com> Signed-off-by: Hongxin Xu <70438206+xhx1022@users.noreply.github.com> Signed-off-by: arlenxu <arlenxu@tencent.com> Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com> Co-authored-by: arlenxu <arlenxu@tencent.com>
This commit is contained in:
co-authored by
22quinn
arlenxu
parent
0565f1fdec
commit
49e6b86c91
@@ -50,6 +50,9 @@ from vllm.forward_context import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping, LoRAMappingType
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import (
|
||||
MRotaryEmbedding,
|
||||
XDRotaryEmbedding,
|
||||
@@ -1633,6 +1636,8 @@ class GPUModelRunner(
|
||||
return blk_table_tensor, slot_mapping
|
||||
|
||||
block_table_gid_0, slot_mapping_gid_0 = _get_block_table_and_slot_mapping(0)
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
self.slot_mapping = slot_mapping_gid_0[:num_tokens].cpu().numpy()
|
||||
cm_base = CommonAttentionMetadata(
|
||||
query_start_loc=self.query_start_loc.gpu[: num_reqs_padded + 1],
|
||||
query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs_padded + 1],
|
||||
@@ -3112,6 +3117,13 @@ class GPUModelRunner(
|
||||
"after execute_model() returns None."
|
||||
)
|
||||
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.clear_buffer() # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
|
||||
if scheduler_output.preempted_req_ids and has_kv_transfer_group():
|
||||
get_kv_transfer_group().handle_preemptions(
|
||||
scheduler_output.preempted_req_ids
|
||||
@@ -3485,6 +3497,13 @@ class GPUModelRunner(
|
||||
self.eplb_step()
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"):
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.save_captured_experts(indices=self.slot_mapping) # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=req_ids_output_copy,
|
||||
req_id_to_index=req_id_to_index_output_copy,
|
||||
@@ -5646,6 +5665,28 @@ class GPUModelRunner(
|
||||
kv_transfer_group.register_kv_caches(kv_caches)
|
||||
kv_transfer_group.set_host_xfer_buffer_ops(copy_kv_blocks)
|
||||
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
self.init_routed_experts_capturer()
|
||||
|
||||
def init_routed_experts_capturer(self):
|
||||
logger.info(
|
||||
"Initializing routed experts capturer, enable_return_routed_experts: %s",
|
||||
self.model_config.enable_return_routed_experts,
|
||||
)
|
||||
routed_experts_capturer = RoutedExpertsCapturer.create()
|
||||
block_size = self.cache_config.block_size
|
||||
self.max_num_kv_tokens = (
|
||||
self.kv_cache_config.num_blocks // len(self.kv_cache_config.kv_cache_groups)
|
||||
+ 1
|
||||
) * block_size
|
||||
|
||||
routed_experts_capturer.init_buffer(
|
||||
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
|
||||
max_num_kv_tokens=self.max_num_kv_tokens,
|
||||
model_config=self.model_config,
|
||||
instance_id=self.vllm_config.instance_id,
|
||||
)
|
||||
|
||||
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
|
||||
"""
|
||||
Add encoder-only layers to the KV cache config.
|
||||
|
||||
Reference in New Issue
Block a user