[Core] Replace routing replay with device cache and async D2H pipeline (#39917)
Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
@@ -1157,6 +1157,12 @@ class VllmConfig:
|
||||
if envs.VLLM_USE_V2_MODEL_RUNNER:
|
||||
self._validate_v2_model_runner()
|
||||
|
||||
if (
|
||||
self.model_config is not None
|
||||
and self.model_config.enable_return_routed_experts
|
||||
):
|
||||
self._validate_return_routed_experts()
|
||||
|
||||
# Re-compute compile ranges after platform-specific config updates
|
||||
# (e.g., XPU may lower max_num_batched_tokens when MLA is enabled)
|
||||
self._set_compile_ranges()
|
||||
@@ -1846,6 +1852,51 @@ class VllmConfig:
|
||||
+ ", ".join(unsupported)
|
||||
)
|
||||
|
||||
def _validate_return_routed_experts(self) -> None:
|
||||
"""Reject parallelism configurations not yet validated with
|
||||
--enable-return-routed-experts.
|
||||
|
||||
Validated scope (PR #39917): TP, EP, DP, single-node and multi-node,
|
||||
prefix caching, and speculative decoding (MTP validated end-to-end;
|
||||
Eagle/Eagle3/Ngram/Medusa supported by construction since the
|
||||
routing buffer is bound only to the target model and verified-token
|
||||
routing lands at the correct positions during the main forward).
|
||||
|
||||
Out-of-scope (block until validated): PP > 1, prefill context
|
||||
parallelism (PCP) > 1, decode context parallelism (DCP) > 1,
|
||||
async scheduling.
|
||||
"""
|
||||
unsupported: list[str] = []
|
||||
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
unsupported.append(
|
||||
"pipeline parallelism "
|
||||
f"(pipeline_parallel_size="
|
||||
f"{self.parallel_config.pipeline_parallel_size})"
|
||||
)
|
||||
if self.parallel_config.prefill_context_parallel_size > 1:
|
||||
unsupported.append(
|
||||
"prefill context parallelism "
|
||||
f"(prefill_context_parallel_size="
|
||||
f"{self.parallel_config.prefill_context_parallel_size})"
|
||||
)
|
||||
if self.parallel_config.decode_context_parallel_size > 1:
|
||||
unsupported.append(
|
||||
"decode context parallelism "
|
||||
f"(decode_context_parallel_size="
|
||||
f"{self.parallel_config.decode_context_parallel_size})"
|
||||
)
|
||||
if self.scheduler_config.async_scheduling:
|
||||
unsupported.append("async scheduling")
|
||||
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"--enable-return-routed-experts is not yet validated with: "
|
||||
+ ", ".join(unsupported)
|
||||
+ ". Disable these features or omit "
|
||||
"--enable-return-routed-experts."
|
||||
)
|
||||
|
||||
def validate_block_size(self) -> None:
|
||||
"""Validate block_size against DCP and mamba constraints.
|
||||
|
||||
|
||||
@@ -92,12 +92,16 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
# not part of the OpenAI spec but is useful for tracing the tokens
|
||||
# in agent scenarios
|
||||
token_ids: list[int] | None = None
|
||||
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
|
||||
|
||||
|
||||
class ChatCompletionResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
|
||||
object: Literal["chat.completion"] = "chat.completion"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
prompt_routed_experts: list[list[list[int]]] | None = (
|
||||
None # [prompt_len, num_layers, top_k]
|
||||
)
|
||||
model: str
|
||||
choices: list[ChatCompletionResponseChoice]
|
||||
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
|
||||
|
||||
@@ -1088,6 +1088,11 @@ class OpenAIServingChat(OpenAIServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=(
|
||||
output.routed_experts.tolist()
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
choices.append(choice_data)
|
||||
continue
|
||||
@@ -1309,6 +1314,11 @@ class OpenAIServingChat(OpenAIServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=(
|
||||
output.routed_experts.tolist()
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
|
||||
|
||||
@@ -1348,6 +1358,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
prompt_routed_experts = None
|
||||
if final_res.prompt_routed_experts is not None:
|
||||
prompt_routed_experts = final_res.prompt_routed_experts.tolist()
|
||||
|
||||
response = ChatCompletionResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
@@ -1360,6 +1374,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
final_res.prompt_token_ids if request.return_token_ids else None
|
||||
),
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
# Log complete response if output logging is enabled
|
||||
|
||||
@@ -468,12 +468,16 @@ class CompletionResponseChoice(OpenAIBaseModel):
|
||||
token_ids: list[int] | None = None # For response
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_ids: list[int] | None = None # For prompt
|
||||
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
|
||||
|
||||
|
||||
class CompletionResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
|
||||
object: Literal["text_completion"] = "text_completion"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
prompt_routed_experts: list[list[list[int]]] | None = (
|
||||
None # [prompt_len, num_layers, top_k]
|
||||
)
|
||||
model: str
|
||||
choices: list[CompletionResponseChoice]
|
||||
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
|
||||
|
||||
@@ -542,6 +542,11 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=(
|
||||
output.routed_experts.tolist()
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
choices.append(choice_data)
|
||||
|
||||
@@ -565,8 +570,13 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
)
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
prompt_routed_experts = None
|
||||
if final_res_batch:
|
||||
kv_transfer_params = final_res_batch[0].kv_transfer_params
|
||||
pre = final_res_batch[0].prompt_routed_experts
|
||||
if pre is not None:
|
||||
prompt_routed_experts = pre.tolist()
|
||||
|
||||
return CompletionResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
@@ -575,6 +585,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
usage=usage,
|
||||
system_fingerprint=self.system_fingerprint,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
def _create_completion_logprobs(
|
||||
|
||||
@@ -246,6 +246,9 @@ class FusedMoE(PluggableLayer):
|
||||
not supported by the router (or the experts).
|
||||
"""
|
||||
|
||||
# Auto-incrementing layer ID for routing replay buffer binding.
|
||||
_next_moe_layer_id: int = 0
|
||||
|
||||
# --8<-- [end:fused_moe]
|
||||
|
||||
def __init__(
|
||||
@@ -290,6 +293,10 @@ class FusedMoE(PluggableLayer):
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# Assign unique layer ID for routing replay buffer binding.
|
||||
self.moe_layer_id = FusedMoE._next_moe_layer_id
|
||||
FusedMoE._next_moe_layer_id += 1
|
||||
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
self.params_dtype = params_dtype
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -451,6 +451,10 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
|
||||
)
|
||||
|
||||
# Get routing replay buffer from persistent layer attribute
|
||||
# (set by bind_routing_capture_to_model during capturer init)
|
||||
routing_replay_out = getattr(layer, "_routing_replay_out", None)
|
||||
|
||||
if self._quant_method.is_monolithic:
|
||||
fused_out = self._quant_method.apply_monolithic(
|
||||
layer=layer,
|
||||
@@ -465,6 +469,10 @@ class MoERunner(MoERunnerInterface):
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
# Write routing data for non-monolithic path (Triton, etc.)
|
||||
if routing_replay_out is not None:
|
||||
routing_replay_out[: topk_ids.shape[0]].copy_(topk_ids.to(torch.int16))
|
||||
|
||||
# Passing shared_experts_input in case SharedExpertsOrder is
|
||||
# MK_INTERNAL_OVERLAPPED.
|
||||
fused_out = self._quant_method.apply(
|
||||
|
||||
@@ -121,6 +121,7 @@ class RequestOutput:
|
||||
num_cached_tokens: int | None = None,
|
||||
*,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
prompt_routed_experts: np.ndarray | None = None,
|
||||
# Forward compatibility, code that uses args added in new release can
|
||||
# still run with older versions of vLLM without breaking.
|
||||
**kwargs: Any,
|
||||
@@ -141,12 +142,15 @@ class RequestOutput:
|
||||
self.encoder_prompt_token_ids = encoder_prompt_token_ids
|
||||
self.num_cached_tokens = num_cached_tokens
|
||||
self.kv_transfer_params = kv_transfer_params
|
||||
self.prompt_routed_experts = prompt_routed_experts
|
||||
|
||||
def add(self, next_output: "RequestOutput", aggregate: bool) -> None:
|
||||
"""Merge subsequent RequestOutput into this one"""
|
||||
|
||||
self.finished |= next_output.finished
|
||||
self.kv_transfer_params = next_output.kv_transfer_params
|
||||
if next_output.prompt_routed_experts is not None:
|
||||
self.prompt_routed_experts = next_output.prompt_routed_experts
|
||||
|
||||
for next_completion in next_output.outputs:
|
||||
for i, completion in enumerate(self.outputs):
|
||||
|
||||
@@ -7,8 +7,6 @@ from collections.abc import Iterable
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm import envs
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.config import VllmConfig
|
||||
@@ -27,9 +25,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsReader,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||
from vllm.multimodal.encoder_budget import MultiModalBudget
|
||||
from vllm.v1.core.encoder_cache_manager import (
|
||||
@@ -52,7 +47,7 @@ from vllm.v1.core.sched.request_queue import (
|
||||
)
|
||||
from vllm.v1.core.sched.utils import check_stop, remove_all
|
||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
|
||||
from vllm.v1.metrics.stats import PrefixCacheStats, SchedulerStats
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
@@ -260,43 +255,6 @@ class Scheduler(SchedulerInterface):
|
||||
if self.log_stats and vllm_config.observability_config.enable_mfu_metrics:
|
||||
self.perf_metrics = ModelMetrics(vllm_config)
|
||||
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
assert self.dcp_world_size == 1 and self.pcp_world_size == 1, (
|
||||
"enable_return_routed_experts does not support context parallelism "
|
||||
"(dcp_world_size > 1 or pcp_world_size > 1)"
|
||||
)
|
||||
|
||||
self.routed_experts_reader = RoutedExpertsReader.create()
|
||||
|
||||
assert len(kv_cache_config.kv_cache_groups) > 0, (
|
||||
"enable_return_routed_experts requires at least one kv cache group"
|
||||
)
|
||||
# Find the attention group for routed experts indexing.
|
||||
self.routed_experts_attn_gid = 0
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if isinstance(group.kv_cache_spec, AttentionSpec):
|
||||
self.routed_experts_attn_gid = gid
|
||||
break
|
||||
min_block_size = min(
|
||||
[
|
||||
group.kv_cache_spec.block_size
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
]
|
||||
)
|
||||
num_groups = len(kv_cache_config.kv_cache_groups)
|
||||
self.max_num_kv_tokens = (
|
||||
kv_cache_config.num_blocks // num_groups
|
||||
) * min_block_size
|
||||
dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size
|
||||
pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
|
||||
if pcp_size * dcp_size > 1:
|
||||
self.max_num_kv_tokens *= pcp_size * dcp_size
|
||||
|
||||
self.routed_experts_reader.attach_buffer(
|
||||
max_num_kv_tokens=self.max_num_kv_tokens,
|
||||
vllm_config=self.vllm_config,
|
||||
)
|
||||
|
||||
self._pause_state: PauseState = PauseState.UNPAUSED
|
||||
|
||||
def _mamba_block_aligned_split(
|
||||
@@ -1415,11 +1373,15 @@ class Scheduler(SchedulerInterface):
|
||||
request.resumable = False
|
||||
stopped = True
|
||||
|
||||
# Get routing data from ModelRunnerOutput (via worker D2H pipeline)
|
||||
routed_experts = None
|
||||
if (
|
||||
model_runner_output.routed_experts_dict is not None
|
||||
and req_id in model_runner_output.routed_experts_dict
|
||||
):
|
||||
routed_experts = model_runner_output.routed_experts_dict[req_id]
|
||||
finish_reason = None
|
||||
if stopped:
|
||||
routed_experts = self._get_routed_experts(request)
|
||||
|
||||
# Capture finish_reason BEFORE _handle_stopped_request, which may
|
||||
# reset the status to WAITING for streaming requests that continue.
|
||||
finish_reason = request.get_finished_reason()
|
||||
@@ -1594,31 +1556,6 @@ class Scheduler(SchedulerInterface):
|
||||
self._enqueue_waiting_request(request)
|
||||
return False
|
||||
|
||||
def _get_routed_experts(self, request: Request) -> np.ndarray | None:
|
||||
if not self.vllm_config.model_config.enable_return_routed_experts:
|
||||
return None
|
||||
|
||||
kv_blocks = self.kv_cache_manager.get_blocks(request.request_id)
|
||||
block_ids = kv_blocks.get_block_ids()[self.routed_experts_attn_gid]
|
||||
num_tokens = request.num_tokens - 1
|
||||
|
||||
# compute slot mapping using attention group's block_size
|
||||
block_ids_array = np.array(block_ids, dtype=np.int32)
|
||||
num_blocks = len(block_ids)
|
||||
attn_group = self.kv_cache_config.kv_cache_groups[self.routed_experts_attn_gid]
|
||||
block_size = attn_group.kv_cache_spec.block_size
|
||||
|
||||
# generate block offsets
|
||||
block_offsets = np.arange(0, block_size)
|
||||
|
||||
# compute slot mapping: slot = block_id * block_size + offset
|
||||
slot_mapping = (
|
||||
block_offsets.reshape((1, block_size))
|
||||
+ block_ids_array.reshape((num_blocks, 1)) * block_size
|
||||
).flatten()[:num_tokens]
|
||||
|
||||
return self.routed_experts_reader.get_routed_experts(indices=slot_mapping)
|
||||
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int]
|
||||
) -> tuple[list[int], bool]:
|
||||
|
||||
@@ -11,6 +11,9 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
split_routed_experts,
|
||||
)
|
||||
from vllm.outputs import (
|
||||
STREAM_FINISHED,
|
||||
CompletionOutput,
|
||||
@@ -314,8 +317,24 @@ class RequestState:
|
||||
finished,
|
||||
)
|
||||
|
||||
# Split routing data into prompt and generation portions.
|
||||
# Prompt routing lives on RequestOutput (shared across n>1
|
||||
# completions); generation routing lives on each CompletionOutput.
|
||||
prompt_routed_experts = None
|
||||
gen_routed_experts = None
|
||||
if routed_experts is not None:
|
||||
prompt_len = len(self.prompt_token_ids) if self.prompt_token_ids else 0
|
||||
num_gen = (
|
||||
self.detokenizer.num_output_tokens()
|
||||
if self.detokenizer is not None
|
||||
else None
|
||||
)
|
||||
prompt_routed_experts, gen_routed_experts = split_routed_experts(
|
||||
routed_experts, prompt_len, num_gen
|
||||
)
|
||||
|
||||
output = self._new_completion_output(
|
||||
new_token_ids, finish_reason, stop_reason, routed_experts
|
||||
new_token_ids, finish_reason, stop_reason, gen_routed_experts
|
||||
)
|
||||
|
||||
if self.parent_req is None:
|
||||
@@ -327,7 +346,11 @@ class RequestState:
|
||||
external_req_id = self.parent_req.external_req_id
|
||||
|
||||
return self._new_request_output(
|
||||
external_req_id, outputs, finished, kv_transfer_params
|
||||
external_req_id,
|
||||
outputs,
|
||||
finished,
|
||||
kv_transfer_params,
|
||||
prompt_routed_experts,
|
||||
)
|
||||
|
||||
def _new_request_output(
|
||||
@@ -336,6 +359,7 @@ class RequestState:
|
||||
outputs: list[CompletionOutput] | list[PoolingOutput],
|
||||
finished: bool,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
prompt_routed_experts: np.ndarray | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
# If prompt embeds were used, put placeholder prompt token ids
|
||||
prompt_token_ids = self.prompt_token_ids
|
||||
@@ -371,6 +395,7 @@ class RequestState:
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
metrics=self.stats,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
def _new_completion_output(
|
||||
|
||||
@@ -198,6 +198,9 @@ class ModelRunnerOutput:
|
||||
# req_id -> num_nans_in_logits
|
||||
num_nans_in_logits: dict[str, int] | None = None
|
||||
|
||||
# req_id -> routed experts ndarray of shape (seq_len, num_moe_layers, top_k)
|
||||
routed_experts_dict: dict[str, np.ndarray] | None = None
|
||||
|
||||
# information related to cudagraph execution
|
||||
cudagraph_stats: CUDAGraphStat | None = None
|
||||
|
||||
|
||||
@@ -54,7 +54,11 @@ from vllm.lora.layers import LoRAMapping, LoRAMappingType
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
extract_routed_experts_for_current_batch,
|
||||
free_routing_buffers,
|
||||
get_global_experts_capturer,
|
||||
init_routed_experts_capturer_with_shared_cache,
|
||||
issue_routing_d2h_copy,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
@@ -1092,6 +1096,12 @@ class GPUModelRunner(
|
||||
for req_id in scheduler_output.finished_req_ids:
|
||||
self.input_batch.remove_request(req_id)
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
free_routing_buffers(
|
||||
scheduler_output.finished_req_ids,
|
||||
scheduler_output.preempted_req_ids,
|
||||
)
|
||||
|
||||
# Zero GPU memory for freshly allocated cache blocks to prevent
|
||||
# stale NaN/data from corrupting attention or SSM computation.
|
||||
if scheduler_output.new_block_ids_to_zero:
|
||||
@@ -2175,10 +2185,6 @@ class GPUModelRunner(
|
||||
block_table_gid_0 = _get_block_table(0)
|
||||
slot_mapping_gid_0 = slot_mappings[0]
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
attn_gid = self.routed_experts_attn_gid
|
||||
slot_mapping_attn = slot_mappings[attn_gid]
|
||||
self.slot_mapping = slot_mapping_attn[:num_tokens].cpu().numpy()
|
||||
num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[
|
||||
:num_reqs_padded
|
||||
]
|
||||
@@ -3859,11 +3865,9 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
capturer = get_global_experts_capturer()
|
||||
if capturer is not None:
|
||||
capturer.clear_buffer() # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
capturer.finalize_pending_copy()
|
||||
|
||||
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
|
||||
# the modification has influence on the scheduler_output in engine core process.
|
||||
@@ -4373,6 +4377,14 @@ class GPUModelRunner(
|
||||
scheduler_output.total_num_scheduled_tokens,
|
||||
)
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
issue_routing_d2h_copy(
|
||||
input_batch_req_ids=self.input_batch.req_ids,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
positions=self.positions,
|
||||
positions_cpu=self._positions_cpu,
|
||||
)
|
||||
|
||||
if propose_drafts_after_bookkeeping:
|
||||
# ngram and other speculative decoding methods use the sampled
|
||||
# tokens on the CPU, so they are run after bookkeeping.
|
||||
@@ -4392,12 +4404,15 @@ class GPUModelRunner(
|
||||
self.kv_connector_output = None
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"):
|
||||
routed_experts_dict = None
|
||||
if self.routed_experts_initialized:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.save_captured_experts(indices=self.slot_mapping) # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
routed_experts_dict = extract_routed_experts_for_current_batch(
|
||||
req_ids=req_ids_output_copy,
|
||||
requests=self.requests,
|
||||
req_id_to_index=self.input_batch.req_id_to_index,
|
||||
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
|
||||
max_model_len=self.max_model_len,
|
||||
)
|
||||
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=req_ids_output_copy,
|
||||
@@ -4411,6 +4426,7 @@ class GPUModelRunner(
|
||||
else None,
|
||||
num_nans_in_logits=num_nans_in_logits,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
routed_experts_dict=routed_experts_dict,
|
||||
)
|
||||
|
||||
if not self.use_async_scheduling:
|
||||
@@ -6138,6 +6154,7 @@ class GPUModelRunner(
|
||||
"Skipping CUDA graph capture. To turn on CUDA graph capture, "
|
||||
"ensure `cudagraph_mode` was not manually set to `NONE`"
|
||||
)
|
||||
self.init_routed_experts_capturer()
|
||||
return 0
|
||||
|
||||
# Initialize encoder CUDA graph manager if enabled.
|
||||
@@ -6171,6 +6188,13 @@ class GPUModelRunner(
|
||||
|
||||
start_time = time.perf_counter()
|
||||
|
||||
# Initialize the routed experts capturer once before any CUDA graph
|
||||
# capture. Must happen before graphs are captured so the buffer
|
||||
# address is baked into the graph. Do NOT call this inside
|
||||
# _capture_cudagraphs() -- creating the capturer twice replaces
|
||||
# the device buffer, causing graphs to write to a dead buffer.
|
||||
self.init_routed_experts_capturer()
|
||||
|
||||
# Trigger CUDA graph capture for specific shapes.
|
||||
# Capture the large shapes first so that the smaller shapes
|
||||
# can reuse the memory pool allocated for the large shapes.
|
||||
@@ -6965,45 +6989,40 @@ class GPUModelRunner(
|
||||
"Initializing routed experts capturer, enable_return_routed_experts: %s",
|
||||
self.model_config.enable_return_routed_experts,
|
||||
)
|
||||
routed_experts_capturer = RoutedExpertsCapturer.create()
|
||||
self.routed_experts_attn_gid = self._get_attention_kv_cache_gid()
|
||||
min_block_size = min(
|
||||
[
|
||||
group.kv_cache_spec.block_size
|
||||
for group in self.kv_cache_config.kv_cache_groups
|
||||
]
|
||||
)
|
||||
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||
self.max_num_kv_tokens = (
|
||||
self.kv_cache_config.num_blocks // num_groups
|
||||
) * min_block_size
|
||||
dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size
|
||||
pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
|
||||
if pcp_size * dcp_size > 1:
|
||||
self.max_num_kv_tokens *= pcp_size * dcp_size
|
||||
from vllm.distributed import get_tp_group
|
||||
|
||||
routed_experts_capturer.init_buffer(
|
||||
if hasattr(self.model_config.hf_text_config, "n_shared_experts"):
|
||||
num_fused_shared_experts = 1
|
||||
else:
|
||||
num_fused_shared_experts = 0
|
||||
|
||||
tp_group = get_tp_group()
|
||||
init_routed_experts_capturer_with_shared_cache(
|
||||
enable=self.model_config.enable_return_routed_experts,
|
||||
model_config=self.model_config,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
|
||||
max_num_kv_tokens=self.max_num_kv_tokens,
|
||||
vllm_config=self.vllm_config,
|
||||
max_model_len=self.max_model_len,
|
||||
device=self.device,
|
||||
rank=tp_group.rank_in_group,
|
||||
world_size=tp_group.world_size,
|
||||
)
|
||||
self._bind_routed_experts_capturer(routed_experts_capturer)
|
||||
self._bind_routed_experts_capturer()
|
||||
self.routed_experts_initialized = True
|
||||
|
||||
def _bind_routed_experts_capturer(self, capturer: RoutedExpertsCapturer) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
BaseRouter,
|
||||
# Pinned CPU buffer for async positions D2H (avoids sync .cpu() call)
|
||||
self._positions_cpu = torch.empty(
|
||||
self.scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.long,
|
||||
pin_memory=True,
|
||||
)
|
||||
|
||||
for module in self.compilation_config.static_forward_context.values():
|
||||
if isinstance(module, FusedMoE) and isinstance(module.router, BaseRouter):
|
||||
layer_id = module.layer_id
|
||||
def _bind_routed_experts_capturer(self) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
bind_routing_capture_to_model,
|
||||
)
|
||||
|
||||
def _capture_fn(topk_ids, _layer_id=layer_id, _capturer=capturer):
|
||||
_capturer.capture(_layer_id, topk_ids)
|
||||
|
||||
module.router.set_capture_fn(_capture_fn)
|
||||
bind_routing_capture_to_model(self.model)
|
||||
|
||||
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user