[Feature] Support recording expert indices for rollout router replay (#28284)
Signed-off-by: xhx1022 <1737006628@qq.com> Signed-off-by: Hongxin Xu <70438206+xhx1022@users.noreply.github.com> Signed-off-by: arlenxu <arlenxu@tencent.com> Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com> Co-authored-by: arlenxu <arlenxu@tencent.com>
This commit is contained in:
co-authored by
22quinn
arlenxu
parent
0565f1fdec
commit
49e6b86c91
@@ -6,6 +6,8 @@ from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm import envs
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.config import VllmConfig
|
||||
@@ -24,6 +26,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsReader,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||
from vllm.v1.core.encoder_cache_manager import (
|
||||
EncoderCacheManager,
|
||||
@@ -219,11 +224,31 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
|
||||
self.use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
|
||||
|
||||
self.perf_metrics: ModelMetrics | None = None
|
||||
if self.log_stats and vllm_config.observability_config.enable_mfu_metrics:
|
||||
self.perf_metrics = ModelMetrics(vllm_config)
|
||||
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
assert self.dcp_world_size == 1 and self.pcp_world_size == 1, (
|
||||
"enable_return_routed_experts does not support context parallelism "
|
||||
"(dcp_world_size > 1 or pcp_world_size > 1)"
|
||||
)
|
||||
|
||||
self.routed_experts_reader = RoutedExpertsReader.create()
|
||||
|
||||
assert len(kv_cache_config.kv_cache_groups) > 0, (
|
||||
"enable_return_routed_experts requires at least one kv cache group"
|
||||
)
|
||||
self.max_num_kv_tokens = (
|
||||
kv_cache_config.num_blocks // len(kv_cache_config.kv_cache_groups) + 1
|
||||
) * self.block_size
|
||||
|
||||
self.routed_experts_reader.attach_buffer(
|
||||
max_num_kv_tokens=self.max_num_kv_tokens,
|
||||
model_config=self.vllm_config.model_config,
|
||||
instance_id=self.vllm_config.instance_id,
|
||||
)
|
||||
|
||||
def schedule(self) -> SchedulerOutput:
|
||||
# NOTE(woosuk) on the scheduling algorithm:
|
||||
# There's no "decoding phase" nor "prefill phase" in the scheduler.
|
||||
@@ -1162,7 +1187,30 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
stopped = True
|
||||
|
||||
routed_experts = None
|
||||
if stopped:
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
kv_blocks = self.kv_cache_manager.get_blocks(request.request_id)
|
||||
block_ids = kv_blocks.get_block_ids()[0]
|
||||
num_tokens = request.num_tokens - 1
|
||||
|
||||
# compute slot mapping
|
||||
block_ids_array = np.array(block_ids, dtype=np.int32)
|
||||
num_blocks = len(block_ids)
|
||||
block_size = self.block_size
|
||||
|
||||
# generate block offsets
|
||||
block_offsets = np.arange(0, block_size)
|
||||
|
||||
# compute slot mapping: slot = block_id * block_size + offset
|
||||
slot_mapping = (
|
||||
block_offsets.reshape((1, block_size))
|
||||
+ block_ids_array.reshape((num_blocks, 1)) * block_size
|
||||
).flatten()[:num_tokens]
|
||||
|
||||
routed_experts = self.routed_experts_reader.get_routed_experts(
|
||||
indices=slot_mapping
|
||||
)
|
||||
kv_transfer_params = self._free_request(request)
|
||||
if status_before_stop == RequestStatus.RUNNING:
|
||||
stopped_running_reqs.add(request)
|
||||
@@ -1209,6 +1257,7 @@ class Scheduler(SchedulerInterface):
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
trace_headers=request.trace_headers,
|
||||
num_cached_tokens=request.num_cached_tokens,
|
||||
routed_experts=routed_experts,
|
||||
num_nans_in_logits=request.num_nans_in_logits,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user