[MRV2] Enable routed-experts capture (#50721)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -15,6 +15,7 @@ from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig
|
||||
from vllm.v1.outputs import RoutedExpertsTensors
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -78,8 +79,7 @@ class RoutedExpertsCapturer:
|
||||
Invariants:
|
||||
- One instance per worker; shape is fixed at init and covers the
|
||||
worst-case step (``max_num_batched_tokens`` tokens).
|
||||
- :meth:`clear_buffer` is called at the start of every step, so
|
||||
unused slots stay zero.
|
||||
- Every routed layer overwrites the current step's token rows.
|
||||
- ``device_buffer.dtype`` is ``torch.int32``.
|
||||
"""
|
||||
|
||||
@@ -87,6 +87,7 @@ class RoutedExpertsCapturer:
|
||||
self,
|
||||
max_num_batched_tokens: int,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
hf_config = vllm_config.model_config.hf_text_config
|
||||
num_experts_per_tok = _get_num_experts_per_tok(hf_config)
|
||||
@@ -116,6 +117,8 @@ class RoutedExpertsCapturer:
|
||||
)
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
# KV cache group whose slot layout the routing data is keyed by.
|
||||
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
|
||||
|
||||
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
|
||||
"""Capture expert routing decisions for a specific layer.
|
||||
@@ -205,30 +208,102 @@ class RoutedExpertsCapturer:
|
||||
f"tp_size={self.tp_size})"
|
||||
)
|
||||
|
||||
# Defensive: model may expose more layers than the capture buffer
|
||||
# was sized for (unusual, but guards against miss-config).
|
||||
if layer_id >= self.device_buffer.shape[1]:
|
||||
return
|
||||
raise IndexError(
|
||||
f"routed-experts layer {layer_id} exceeds capture buffer "
|
||||
f"layer count {self.device_buffer.shape[1]}"
|
||||
)
|
||||
|
||||
self.device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[
|
||||
start_loc:end_loc, :
|
||||
]
|
||||
|
||||
def clear_buffer(self) -> None:
|
||||
"""Zero the device buffer. Called at the start of every step so
|
||||
slots belonging to finished / preempted tokens don't leak into
|
||||
the next step.
|
||||
"""
|
||||
self.device_buffer.zero_()
|
||||
|
||||
def get_device_buffer(self) -> torch.Tensor:
|
||||
"""Return the underlying device buffer so the model runner can
|
||||
issue the D2H copy. The tensor is shared; callers must either
|
||||
clone or fully drain it before the next forward pass runs
|
||||
:meth:`clear_buffer`.
|
||||
clone or fully drain it before the next forward pass overwrites it.
|
||||
"""
|
||||
return self.device_buffer
|
||||
|
||||
def get_routed_experts(
|
||||
self, slot_mappings: torch.Tensor, num_tokens: int
|
||||
) -> RoutedExpertsTensors:
|
||||
"""Snapshot this step's routing data and its attention slot mapping.
|
||||
|
||||
Both tensors are cloned since the capture buffer and the shared
|
||||
``slot_mappings`` are overwritten by the next step, which may race
|
||||
with an in-flight D2H copy.
|
||||
|
||||
Args:
|
||||
slot_mappings: Per-KV-cache-group slot mappings for this step,
|
||||
shape ``(num_kv_cache_groups, max_num_batched_tokens)``.
|
||||
num_tokens: Total number of tokens scheduled in this step.
|
||||
"""
|
||||
return RoutedExpertsTensors(
|
||||
routing_data=self.device_buffer[:num_tokens].clone(),
|
||||
slot_mapping=slot_mappings[self.attn_gid, :num_tokens].clone(),
|
||||
)
|
||||
|
||||
|
||||
def bind_routed_experts_capturer(
|
||||
model: torch.nn.Module,
|
||||
capturer: RoutedExpertsCapturer,
|
||||
) -> None:
|
||||
"""Attach capture callbacks to the target model's MoE routers."""
|
||||
from vllm.model_executor.layers.fused_moe.layer import MoERunner
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
|
||||
num_bound = 0
|
||||
for module in model.modules():
|
||||
if not isinstance(module, MoERunner):
|
||||
continue
|
||||
layer_id = module.layer_id
|
||||
|
||||
def capture_fn(
|
||||
topk_ids: torch.Tensor,
|
||||
layer_id: int = layer_id,
|
||||
capturer: RoutedExpertsCapturer = capturer,
|
||||
) -> None:
|
||||
capturer.capture(layer_id, topk_ids)
|
||||
|
||||
quant_method = module._quant_method
|
||||
moe_kernel = getattr(quant_method, "moe_kernel", None)
|
||||
impl = getattr(moe_kernel, "impl", None)
|
||||
fused_experts = getattr(impl, "fused_experts", None)
|
||||
if quant_method.is_monolithic:
|
||||
if not (
|
||||
isinstance(fused_experts, FusedMoEExpertsMonolithic)
|
||||
and fused_experts.supports_routing_replay_capture()
|
||||
):
|
||||
raise ValueError(
|
||||
"Routed-experts capture is not supported with monolithic "
|
||||
f"MoE kernel {type(fused_experts).__name__}."
|
||||
)
|
||||
fused_experts.set_capture_fn(capture_fn)
|
||||
num_bound += 1
|
||||
elif isinstance(module.router, BaseRouter):
|
||||
module.router.set_capture_fn(capture_fn)
|
||||
num_bound += 1
|
||||
else:
|
||||
raise ValueError(
|
||||
"Routed-experts capture is not supported with router "
|
||||
f"{type(module.router).__name__}."
|
||||
)
|
||||
|
||||
if num_bound == 0:
|
||||
raise ValueError("No supported MoE router found for routed-experts capture.")
|
||||
|
||||
|
||||
def get_routed_experts_attn_gid(kv_cache_config: KVCacheConfig) -> int:
|
||||
"""Return the full-attention KV cache group used for routed experts."""
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if isinstance(group.kv_cache_spec, FullAttentionSpec):
|
||||
return gid
|
||||
raise ValueError("Routed-experts capture requires a full-attention KV cache group.")
|
||||
|
||||
|
||||
class RoutedExpertsManager:
|
||||
"""Scheduler-side slot-indexed buffer for routed experts.
|
||||
@@ -260,17 +335,8 @@ class RoutedExpertsManager:
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
# Pick the attention group for block/slot mapping. We require
|
||||
# a FullAttentionSpec group rather than any AttentionSpec to
|
||||
# stay consistent with the worker-side lookup in
|
||||
# ``GPUModelRunner._get_attention_kv_cache_gid``; hybrid models
|
||||
# (Mamba / linear attention) also have other AttentionSpec
|
||||
# groups whose slot layout differs.
|
||||
self.attn_gid = next(
|
||||
gid
|
||||
for gid, g in enumerate(kv_cache_config.kv_cache_groups)
|
||||
if isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
)
|
||||
# Hybrid models also have KV groups whose slot layout differs.
|
||||
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
|
||||
attn_group = kv_cache_config.kv_cache_groups[self.attn_gid]
|
||||
self.block_size = attn_group.kv_cache_spec.block_size
|
||||
|
||||
|
||||
Reference in New Issue
Block a user