[Model] Support R3 capture with DeepGEMM MegaMoE (#51831)
Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -6,6 +6,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from functools import partial
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -20,6 +23,12 @@ from vllm.v1.outputs import RoutedExpertsTensors
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RoutedExpertsCaptureSource(Protocol):
|
||||
layer_id: int
|
||||
capture_fn: Callable[[torch.Tensor], None] | None
|
||||
|
||||
|
||||
def _get_routed_experts_shape(vllm_config: VllmConfig) -> tuple[int, int, int]:
|
||||
model_config = vllm_config.model_config
|
||||
num_layers = model_config.get_total_num_hidden_layers()
|
||||
@@ -234,6 +243,10 @@ def bind_routed_experts_capturer(
|
||||
|
||||
num_bound = 0
|
||||
for module in model.modules():
|
||||
if isinstance(module, RoutedExpertsCaptureSource):
|
||||
module.capture_fn = partial(capturer.capture, module.layer_id)
|
||||
num_bound += 1
|
||||
continue
|
||||
if not isinstance(module, MoERunner):
|
||||
continue
|
||||
layer_id = module.layer_id
|
||||
|
||||
Reference in New Issue
Block a user