Fix Transformers backend FP8 MoE and remove some boilerplate (#46820)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1088,8 +1088,6 @@ class AXK1ForCausalLM(
|
|||||||
self.set_moe_parameters()
|
self.set_moe_parameters()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.models.interfaces import (
|
from vllm.model_executor.models.interfaces import (
|
||||||
EagleModelMixin,
|
EagleModelMixin,
|
||||||
|
MixtureOfExperts,
|
||||||
SupportsEagle3,
|
SupportsEagle3,
|
||||||
SupportsLoRA,
|
SupportsLoRA,
|
||||||
SupportsPP,
|
SupportsPP,
|
||||||
@@ -595,7 +596,9 @@ class AfmoeModel(nn.Module, EagleModelMixin):
|
|||||||
return loaded_params
|
return loaded_params
|
||||||
|
|
||||||
|
|
||||||
class AfmoeForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA):
|
class AfmoeForCausalLM(
|
||||||
|
nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA, MixtureOfExperts
|
||||||
|
):
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping = {
|
||||||
"qkv_proj": [
|
"qkv_proj": [
|
||||||
"q_proj",
|
"q_proj",
|
||||||
@@ -635,8 +638,6 @@ class AfmoeForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA):
|
|||||||
self.make_empty_intermediate_tensors = (
|
self.make_empty_intermediate_tensors = (
|
||||||
self.model.make_empty_intermediate_tensors
|
self.model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.num_moe_layers = config.num_hidden_layers - config.num_dense_layers
|
self.num_moe_layers = config.num_hidden_layers - config.num_dense_layers
|
||||||
self.num_expert_groups = config.n_group
|
self.num_expert_groups = config.n_group
|
||||||
@@ -663,21 +664,24 @@ class AfmoeForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA):
|
|||||||
self.num_shared_experts = example_moe.n_shared_experts
|
self.num_shared_experts = example_moe.n_shared_experts
|
||||||
self.num_redundant_experts = example_moe.n_redundant_experts
|
self.num_redundant_experts = example_moe.n_redundant_experts
|
||||||
|
|
||||||
def set_eplb_state(
|
def update_physical_experts_metadata(
|
||||||
self,
|
self,
|
||||||
expert_load_view: torch.Tensor,
|
num_physical_experts: int,
|
||||||
logical_to_physical_map: torch.Tensor,
|
num_local_physical_experts: int,
|
||||||
logical_replica_count: torch.Tensor,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
for layer_idx, layer in enumerate(self.moe_layers):
|
assert self.num_local_physical_experts == num_local_physical_experts
|
||||||
# Register the expert weights.
|
self.num_physical_experts = num_physical_experts
|
||||||
self.expert_weights.append(layer.get_expert_weights())
|
self.num_local_physical_experts = num_local_physical_experts
|
||||||
layer.set_eplb_state(
|
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
||||||
moe_layer_idx=layer_idx,
|
for layer in self.model.layers:
|
||||||
expert_load_view=expert_load_view,
|
if isinstance(layer, PPMissingLayer):
|
||||||
logical_to_physical_map=logical_to_physical_map,
|
continue
|
||||||
logical_replica_count=logical_replica_count,
|
if layer.moe_enabled:
|
||||||
)
|
moe = layer.mlp
|
||||||
|
moe.n_local_physical_experts = num_local_physical_experts
|
||||||
|
moe.n_physical_experts = num_physical_experts
|
||||||
|
moe.n_redundant_experts = self.num_redundant_experts
|
||||||
|
moe.experts.update_expert_map()
|
||||||
|
|
||||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||||
return self.model.embed_input_ids(input_ids)
|
return self.model.embed_input_ids(input_ids)
|
||||||
|
|||||||
@@ -217,7 +217,6 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
|||||||
self.set_moe_parameters()
|
self.set_moe_parameters()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||||
self.num_expert_groups = self.config.n_group
|
self.num_expert_groups = self.config.n_group
|
||||||
|
|
||||||
|
|||||||
@@ -1684,8 +1684,6 @@ class DeepseekV2ForCausalLM(
|
|||||||
self.set_moe_parameters()
|
self.set_moe_parameters()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
|
|||||||
@@ -656,8 +656,6 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
|
|||||||
self.model.make_empty_intermediate_tensors
|
self.model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
|
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
moe_layers_indices = [
|
moe_layers_indices = [
|
||||||
i
|
i
|
||||||
|
|||||||
@@ -1562,7 +1562,6 @@ class Gemma4ForCausalLM(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# --- MixtureOfExperts protocol ---
|
# --- MixtureOfExperts protocol ---
|
||||||
self.expert_weights: list[list[torch.Tensor]] = []
|
|
||||||
self.moe_layers: list[nn.Module] = []
|
self.moe_layers: list[nn.Module] = []
|
||||||
example_moe: Gemma4MoE | None = None
|
example_moe: Gemma4MoE | None = None
|
||||||
|
|
||||||
|
|||||||
@@ -1114,7 +1114,6 @@ class Gemma4ForConditionalGeneration(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# --- MixtureOfExperts delegation to language_model ---
|
# --- MixtureOfExperts delegation to language_model ---
|
||||||
self.expert_weights = self.language_model.expert_weights
|
|
||||||
self.moe_layers = self.language_model.moe_layers
|
self.moe_layers = self.language_model.moe_layers
|
||||||
self.num_moe_layers = self.language_model.num_moe_layers
|
self.num_moe_layers = self.language_model.num_moe_layers
|
||||||
self.num_logical_experts = self.language_model.num_logical_experts
|
self.num_logical_experts = self.language_model.num_logical_experts
|
||||||
|
|||||||
@@ -655,8 +655,6 @@ class Glm4MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, Glm4MixtureOfExper
|
|||||||
self.make_empty_intermediate_tensors = (
|
self.make_empty_intermediate_tensors = (
|
||||||
self.model.make_empty_intermediate_tensors
|
self.model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.num_moe_layers = config.num_hidden_layers - config.first_k_dense_replace
|
self.num_moe_layers = config.num_hidden_layers - config.first_k_dense_replace
|
||||||
self.num_expert_groups = config.n_group
|
self.num_expert_groups = config.n_group
|
||||||
|
|||||||
@@ -573,8 +573,6 @@ class Glm4MoeLiteForCausalLM(
|
|||||||
self.set_moe_parameters()
|
self.set_moe_parameters()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
|
|||||||
@@ -209,8 +209,6 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
|||||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||||
)
|
)
|
||||||
|
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||||
self.num_expert_groups = self.config.n_group
|
self.num_expert_groups = self.config.n_group
|
||||||
|
|||||||
@@ -195,8 +195,6 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
|
|||||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||||
)
|
)
|
||||||
|
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.num_moe_layers = self.config.num_nextn_predict_layers
|
self.num_moe_layers = self.config.num_nextn_predict_layers
|
||||||
self.num_expert_groups = self.config.n_group
|
self.num_expert_groups = self.config.n_group
|
||||||
|
|||||||
@@ -134,7 +134,6 @@ class GlmOcrMTP(nn.Module, SupportsPP):
|
|||||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||||
)
|
)
|
||||||
|
|
||||||
self.expert_weights = []
|
|
||||||
self.num_layers = self.config.num_nextn_predict_layers
|
self.num_layers = self.config.num_nextn_predict_layers
|
||||||
for layer in self.model.layers.values():
|
for layer in self.model.layers.values():
|
||||||
assert isinstance(layer, GlmOcrMultiTokenPredictorLayer)
|
assert isinstance(layer, GlmOcrMultiTokenPredictorLayer)
|
||||||
|
|||||||
@@ -991,7 +991,6 @@ class HunYuanMoEV1Base(HunyuanV1ModelBase, MixtureOfExperts):
|
|||||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.expert_weights = []
|
|
||||||
self.num_expert_groups = 1
|
self.num_expert_groups = 1
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_layer = None
|
example_layer = None
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
|||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.transformers_utils.configs.hy_v3 import HYV3Config
|
from vllm.transformers_utils.configs.hy_v3 import HYV3Config
|
||||||
|
|
||||||
from .interfaces import SupportsLoRA, SupportsPP
|
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||||
from .utils import (
|
from .utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
PPMissingLayer,
|
PPMissingLayer,
|
||||||
@@ -392,7 +392,7 @@ class HYV3DecoderLayer(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
@support_torch_compile
|
@support_torch_compile
|
||||||
class HYV3Model(nn.Module):
|
class HYV3Model(nn.Module, MixtureOfExperts):
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
@@ -429,7 +429,6 @@ class HYV3Model(nn.Module):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.expert_weights = []
|
|
||||||
self.num_expert_groups = 1
|
self.num_expert_groups = 1
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_layer = None
|
example_layer = None
|
||||||
|
|||||||
@@ -29,36 +29,34 @@ from torch import Tensor
|
|||||||
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
||||||
from typing_extensions import Self, TypeIs
|
from typing_extensions import Self, TypeIs
|
||||||
|
|
||||||
from vllm.config import ModelConfig, SpeechToTextConfig, SpeechToTextParams
|
|
||||||
from vllm.inputs import PromptType, TokensPrompt
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.tasks import ScoreType
|
|
||||||
from vllm.utils.collection_utils import common_prefix
|
from vllm.utils.collection_utils import common_prefix
|
||||||
from vllm.utils.func_utils import supports_kw
|
from vllm.utils.func_utils import supports_kw
|
||||||
|
|
||||||
from .interfaces_base import VllmModel
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import (
|
||||||
|
ModelConfig,
|
||||||
|
SpeechToTextConfig,
|
||||||
|
SpeechToTextParams,
|
||||||
|
VllmConfig,
|
||||||
|
)
|
||||||
|
from vllm.inputs import PromptType, TokensPrompt
|
||||||
from vllm.lora.model_manager import LoRAModelManager
|
from vllm.lora.model_manager import LoRAModelManager
|
||||||
|
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||||
|
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||||
|
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||||
from vllm.model_executor.models.utils import WeightsMapper
|
from vllm.model_executor.models.utils import WeightsMapper
|
||||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||||
from vllm.multimodal.registry import _ProcessorFactories
|
from vllm.multimodal.registry import _ProcessorFactories
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
|
from vllm.tasks import ScoreType
|
||||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||||
EncoderCudaGraphCaptureInputs,
|
EncoderCudaGraphCaptureInputs,
|
||||||
EncoderCudaGraphConfig,
|
EncoderCudaGraphConfig,
|
||||||
EncoderCudaGraphReplayBuffers,
|
EncoderCudaGraphReplayBuffers,
|
||||||
EncoderItemSpec,
|
EncoderItemSpec,
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
VllmConfig = object
|
|
||||||
WeightsMapper = object
|
|
||||||
MultiModalFeatureSpec = object
|
|
||||||
_ProcessorFactories = object
|
|
||||||
IntermediateTensors = object
|
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -89,7 +87,7 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
|
|||||||
|
|
||||||
|
|
||||||
# Cache results of `SupportsMultiModal.get_language_model`
|
# Cache results of `SupportsMultiModal.get_language_model`
|
||||||
_language_model_by_module = dict[nn.Module, VllmModel]()
|
_language_model_by_module = dict[nn.Module, "VllmModel"]()
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
@runtime_checkable
|
||||||
@@ -123,7 +121,7 @@ class SupportsMultiModal(Protocol):
|
|||||||
in their raw form and not input embeddings.
|
in their raw form and not input embeddings.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_processor_factory: ClassVar[_ProcessorFactories]
|
_processor_factory: ClassVar["_ProcessorFactories"]
|
||||||
"""
|
"""
|
||||||
Set internally by `MultiModalRegistry.register_processor`.
|
Set internally by `MultiModalRegistry.register_processor`.
|
||||||
"""
|
"""
|
||||||
@@ -175,7 +173,7 @@ class SupportsMultiModal(Protocol):
|
|||||||
self._has_oov_mm_tokens,
|
self._has_oov_mm_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_language_model(self) -> VllmModel:
|
def get_language_model(self) -> "VllmModel":
|
||||||
"""
|
"""
|
||||||
Returns the underlying language model used for text generation.
|
Returns the underlying language model used for text generation.
|
||||||
|
|
||||||
@@ -216,7 +214,7 @@ class SupportsMultiModal(Protocol):
|
|||||||
@contextmanager
|
@contextmanager
|
||||||
def _mark_language_model(
|
def _mark_language_model(
|
||||||
self,
|
self,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: "VllmConfig",
|
||||||
*,
|
*,
|
||||||
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
|
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
|
||||||
):
|
):
|
||||||
@@ -251,7 +249,7 @@ class SupportsMultiModal(Protocol):
|
|||||||
@contextmanager
|
@contextmanager
|
||||||
def _mark_tower_model(
|
def _mark_tower_model(
|
||||||
self,
|
self,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: "VllmConfig",
|
||||||
modalities: set[str] | str,
|
modalities: set[str] | str,
|
||||||
*,
|
*,
|
||||||
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
|
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
|
||||||
@@ -295,7 +293,7 @@ class SupportsMultiModal(Protocol):
|
|||||||
@contextmanager
|
@contextmanager
|
||||||
def _mark_composite_model(
|
def _mark_composite_model(
|
||||||
self,
|
self,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: "VllmConfig",
|
||||||
*,
|
*,
|
||||||
language_targets: type[nn.Module] | tuple[type[nn.Module], ...],
|
language_targets: type[nn.Module] | tuple[type[nn.Module], ...],
|
||||||
tower_targets: dict[str, type[nn.Module] | tuple[type[nn.Module], ...]],
|
tower_targets: dict[str, type[nn.Module] | tuple[type[nn.Module], ...]],
|
||||||
@@ -513,7 +511,7 @@ class SupportsScoreTemplate(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def post_process_tokens(cls, prompt: TokensPrompt) -> None:
|
def post_process_tokens(cls, prompt: "TokensPrompt") -> None:
|
||||||
"""
|
"""
|
||||||
Perform architecture-specific manipulations on the input tokens.
|
Perform architecture-specific manipulations on the input tokens.
|
||||||
"""
|
"""
|
||||||
@@ -633,7 +631,7 @@ class SupportsPP(Protocol):
|
|||||||
batch_size: int,
|
batch_size: int,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
) -> IntermediateTensors:
|
) -> "IntermediateTensors":
|
||||||
"""Called when PP rank > 0 for profiling purposes."""
|
"""Called when PP rank > 0 for profiling purposes."""
|
||||||
...
|
...
|
||||||
|
|
||||||
@@ -642,8 +640,8 @@ class SupportsPP(Protocol):
|
|||||||
input_ids: Tensor | None,
|
input_ids: Tensor | None,
|
||||||
positions: Tensor,
|
positions: Tensor,
|
||||||
*,
|
*,
|
||||||
intermediate_tensors: IntermediateTensors | None,
|
intermediate_tensors: "IntermediateTensors | None",
|
||||||
) -> IntermediateTensors | None:
|
) -> "IntermediateTensors | None":
|
||||||
"""
|
"""
|
||||||
Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when
|
Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when
|
||||||
PP rank > 0.
|
PP rank > 0.
|
||||||
@@ -665,15 +663,15 @@ class _SupportsPPType(Protocol):
|
|||||||
batch_size: int,
|
batch_size: int,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
) -> IntermediateTensors: ...
|
) -> "IntermediateTensors": ...
|
||||||
|
|
||||||
def forward(
|
def forward(
|
||||||
self,
|
self,
|
||||||
input_ids: Tensor | None,
|
input_ids: Tensor | None,
|
||||||
positions: Tensor,
|
positions: Tensor,
|
||||||
*,
|
*,
|
||||||
intermediate_tensors: IntermediateTensors | None,
|
intermediate_tensors: "IntermediateTensors | None",
|
||||||
) -> Tensor | IntermediateTensors: ...
|
) -> "Tensor | IntermediateTensors": ...
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
@@ -803,7 +801,7 @@ class IsHybrid(Protocol):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def get_mamba_state_shape_from_config(
|
def get_mamba_state_shape_from_config(
|
||||||
cls,
|
cls,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: "VllmConfig",
|
||||||
) -> tuple[tuple[int, int], tuple[int, int, int]]:
|
) -> tuple[tuple[int, int], tuple[int, int, int]]:
|
||||||
"""Calculate shapes for Mamba's convolutional and state caches.
|
"""Calculate shapes for Mamba's convolutional and state caches.
|
||||||
|
|
||||||
@@ -818,7 +816,7 @@ class IsHybrid(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, ...]:
|
def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]:
|
||||||
"""Calculate copy-function callables for each Mamba state.
|
"""Calculate copy-function callables for each Mamba state.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -883,7 +881,7 @@ class MixtureOfExperts(Protocol):
|
|||||||
num_redundant_experts: int
|
num_redundant_experts: int
|
||||||
"""Number of redundant experts in this model."""
|
"""Number of redundant experts in this model."""
|
||||||
|
|
||||||
moe_layers: Iterable[nn.Module]
|
moe_layers: Iterable["MoERunner"]
|
||||||
"""List of MoE layers in this model."""
|
"""List of MoE layers in this model."""
|
||||||
|
|
||||||
def set_eplb_state(
|
def set_eplb_state(
|
||||||
@@ -908,6 +906,7 @@ class MixtureOfExperts(Protocol):
|
|||||||
logical_to_physical_map: Mapping from logical to physical experts.
|
logical_to_physical_map: Mapping from logical to physical experts.
|
||||||
logical_replica_count: Count of replicas for each logical expert.
|
logical_replica_count: Count of replicas for each logical expert.
|
||||||
"""
|
"""
|
||||||
|
self.expert_weights = []
|
||||||
for layer_idx, layer in enumerate(self.moe_layers):
|
for layer_idx, layer in enumerate(self.moe_layers):
|
||||||
# Register the expert weights.
|
# Register the expert weights.
|
||||||
self.expert_weights.append(layer.get_expert_weights())
|
self.expert_weights.append(layer.get_expert_weights())
|
||||||
@@ -982,7 +981,7 @@ def supports_mamba_prefix_caching(
|
|||||||
class SupportsCrossEncoding(Protocol):
|
class SupportsCrossEncoding(Protocol):
|
||||||
"""The interface required for all models that support cross encoding."""
|
"""The interface required for all models that support cross encoding."""
|
||||||
|
|
||||||
score_type: ClassVar[ScoreType] = "cross-encoder"
|
score_type: ClassVar["ScoreType"] = "cross-encoder"
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
@runtime_checkable
|
||||||
@@ -994,13 +993,13 @@ class SupportsLateInteraction(Protocol):
|
|||||||
MaxSim (max over document tokens, sum over query tokens).
|
MaxSim (max over document tokens, sum over query tokens).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
score_type: ClassVar[ScoreType] = "late-interaction"
|
score_type: ClassVar["ScoreType"] = "late-interaction"
|
||||||
|
|
||||||
|
|
||||||
class SupportsQuant:
|
class SupportsQuant:
|
||||||
"""The interface required for all models that support quantization."""
|
"""The interface required for all models that support quantization."""
|
||||||
|
|
||||||
hf_to_vllm_mapper: ClassVar[WeightsMapper | None] = None
|
hf_to_vllm_mapper: ClassVar["WeightsMapper | None"] = None
|
||||||
packed_modules_mapping: ClassVar[dict[str, list[str]] | None] = None
|
packed_modules_mapping: ClassVar[dict[str, list[str]] | None] = None
|
||||||
quant_config: QuantizationConfig | None = None
|
quant_config: QuantizationConfig | None = None
|
||||||
|
|
||||||
@@ -1054,8 +1053,8 @@ class SupportsRealtime(Protocol):
|
|||||||
cls,
|
cls,
|
||||||
audio_stream: AsyncGenerator[np.ndarray, None],
|
audio_stream: AsyncGenerator[np.ndarray, None],
|
||||||
input_stream: asyncio.Queue[list[int]],
|
input_stream: asyncio.Queue[list[int]],
|
||||||
model_config: ModelConfig,
|
model_config: "ModelConfig",
|
||||||
) -> AsyncGenerator[PromptType, None]: ...
|
) -> AsyncGenerator["PromptType", None]: ...
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
@@ -1124,8 +1123,8 @@ class SupportsTranscription(Protocol):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def get_generation_prompt(
|
def get_generation_prompt(
|
||||||
cls,
|
cls,
|
||||||
stt_params: SpeechToTextParams,
|
stt_params: "SpeechToTextParams",
|
||||||
) -> PromptType:
|
) -> "PromptType":
|
||||||
"""Get the prompt for the ASR model.
|
"""Get the prompt for the ASR model.
|
||||||
The model has control over the construction, as long as it
|
The model has control over the construction, as long as it
|
||||||
returns a valid PromptType."""
|
returns a valid PromptType."""
|
||||||
@@ -1163,8 +1162,8 @@ class SupportsTranscription(Protocol):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_speech_to_text_config(
|
def get_speech_to_text_config(
|
||||||
cls, model_config: ModelConfig, task_type: Literal["transcribe", "translate"]
|
cls, model_config: "ModelConfig", task_type: Literal["transcribe", "translate"]
|
||||||
) -> SpeechToTextConfig:
|
) -> "SpeechToTextConfig":
|
||||||
"""Get the speech to text config for the ASR model."""
|
"""Get the speech to text config for the ASR model."""
|
||||||
...
|
...
|
||||||
|
|
||||||
@@ -1172,8 +1171,8 @@ class SupportsTranscription(Protocol):
|
|||||||
def get_num_audio_tokens(
|
def get_num_audio_tokens(
|
||||||
cls,
|
cls,
|
||||||
audio_duration_s: float,
|
audio_duration_s: float,
|
||||||
stt_config: SpeechToTextConfig,
|
stt_config: "SpeechToTextConfig",
|
||||||
model_config: ModelConfig,
|
model_config: "ModelConfig",
|
||||||
) -> int | None:
|
) -> int | None:
|
||||||
"""
|
"""
|
||||||
Map from audio duration to number of audio tokens produced by the ASR
|
Map from audio duration to number of audio tokens produced by the ASR
|
||||||
@@ -1202,8 +1201,8 @@ class SupportsTranscription(Protocol):
|
|||||||
def get_language_detection_prompt(
|
def get_language_detection_prompt(
|
||||||
cls,
|
cls,
|
||||||
audio: np.ndarray,
|
audio: np.ndarray,
|
||||||
stt_config: SpeechToTextConfig,
|
stt_config: "SpeechToTextConfig",
|
||||||
) -> PromptType:
|
) -> "PromptType":
|
||||||
"""Return a prompt that triggers language detection.
|
"""Return a prompt that triggers language detection.
|
||||||
|
|
||||||
Only needs to be implemented when
|
Only needs to be implemented when
|
||||||
|
|||||||
@@ -513,8 +513,6 @@ class InternS1ProMoeMixtureOfExperts(MixtureOfExperts):
|
|||||||
moe.experts.update_expert_map()
|
moe.experts.update_expert_map()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_moe = None
|
example_moe = None
|
||||||
for layer in self.language_model.model.layers:
|
for layer in self.language_model.model.layers:
|
||||||
|
|||||||
@@ -692,7 +692,6 @@ class Lfm2MoeForCausalLM(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_layer = None
|
example_layer = None
|
||||||
|
|||||||
@@ -729,8 +729,6 @@ class Llama4ForCausalLM(LlamaForCausalLM, MixtureOfExperts):
|
|||||||
self.set_moe_parameters()
|
self.set_moe_parameters()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_moe = None
|
example_moe = None
|
||||||
for layer in self.model.layers:
|
for layer in self.model.layers:
|
||||||
|
|||||||
@@ -227,8 +227,6 @@ class MellumForCausalLM(Qwen3MoeForCausalLM):
|
|||||||
self.model.make_empty_intermediate_tensors
|
self.model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
|
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_layer = None
|
example_layer = None
|
||||||
for layer in self.model.layers:
|
for layer in self.model.layers:
|
||||||
|
|||||||
@@ -512,7 +512,6 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
|
|||||||
self.model.make_empty_intermediate_tensors
|
self.model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
|
|
||||||
self.expert_weights = []
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_moe = None
|
example_moe = None
|
||||||
|
|
||||||
|
|||||||
@@ -886,7 +886,6 @@ class NemotronHForCausalLM(
|
|||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
if self.model.has_moe:
|
if self.model.has_moe:
|
||||||
self.expert_weights = []
|
|
||||||
self.num_expert_groups = config.n_group
|
self.num_expert_groups = config.n_group
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
|
|||||||
@@ -1289,7 +1289,6 @@ class OpenPanguMoEModel(OpenPanguModelBase, MixtureOfExperts):
|
|||||||
config = vllm_config.model_config.hf_config
|
config = vllm_config.model_config.hf_config
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.expert_weights = []
|
|
||||||
self.num_moe_layers = config.num_hidden_layers - config.first_k_dense_replace
|
self.num_moe_layers = config.num_hidden_layers - config.first_k_dense_replace
|
||||||
self.num_expert_groups = 1
|
self.num_expert_groups = 1
|
||||||
|
|
||||||
|
|||||||
@@ -751,7 +751,7 @@ class Param2MoEMixtureOfExperts(MixtureOfExperts):
|
|||||||
logical_to_physical_map: torch.Tensor,
|
logical_to_physical_map: torch.Tensor,
|
||||||
logical_replica_count: torch.Tensor,
|
logical_replica_count: torch.Tensor,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.expert_weights.clear()
|
self.expert_weights = []
|
||||||
for layer_idx, layer in enumerate(self.moe_layers):
|
for layer_idx, layer in enumerate(self.moe_layers):
|
||||||
if hasattr(layer, "get_expert_weights"):
|
if hasattr(layer, "get_expert_weights"):
|
||||||
self.expert_weights.append(layer.get_expert_weights())
|
self.expert_weights.append(layer.get_expert_weights())
|
||||||
@@ -832,7 +832,6 @@ class Param2MoEForCausalLM(
|
|||||||
self.model.make_empty_intermediate_tensors
|
self.model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
|
|
||||||
self.expert_weights: list[torch.Tensor] = []
|
|
||||||
self.num_moe_layers: int = 0
|
self.num_moe_layers: int = 0
|
||||||
self.moe_layers: list = []
|
self.moe_layers: list = []
|
||||||
self.moe_mlp_layers: list = []
|
self.moe_mlp_layers: list = []
|
||||||
|
|||||||
@@ -765,8 +765,6 @@ class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts):
|
|||||||
moe.experts.update_expert_map()
|
moe.experts.update_expert_map()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_moe = None
|
example_moe = None
|
||||||
for layer in self.language_model.model.layers:
|
for layer in self.language_model.model.layers:
|
||||||
|
|||||||
@@ -703,7 +703,6 @@ class Qwen3MoeForCausalLM(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_layer = None
|
example_layer = None
|
||||||
|
|||||||
@@ -763,8 +763,6 @@ class QwenNextMixtureOfExperts(MixtureOfExperts):
|
|||||||
moe.experts.update_expert_map()
|
moe.experts.update_expert_map()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_moe = None
|
example_moe = None
|
||||||
for layer in self.model.layers:
|
for layer in self.model.layers:
|
||||||
|
|||||||
@@ -373,8 +373,6 @@ class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts):
|
|||||||
moe.experts.update_expert_map()
|
moe.experts.update_expert_map()
|
||||||
|
|
||||||
def set_moe_parameters(self):
|
def set_moe_parameters(self):
|
||||||
self.expert_weights = []
|
|
||||||
|
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_moe = None
|
example_moe = None
|
||||||
for layer in self.language_model.model.layers:
|
for layer in self.language_model.model.layers:
|
||||||
|
|||||||
@@ -919,17 +919,17 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.moe_layers: list[FusedMoEBlock] = []
|
self.moe_layers: list[MoERunner] = []
|
||||||
|
example_layer: FusedMoEBlock | None = None
|
||||||
for layer in self.model.layers:
|
for layer in self.model.layers:
|
||||||
if isinstance(layer, PPMissingLayer):
|
if isinstance(layer, PPMissingLayer):
|
||||||
continue
|
continue
|
||||||
assert isinstance(layer, Step3p5DecoderLayer)
|
assert isinstance(layer, Step3p5DecoderLayer)
|
||||||
if hasattr(layer, "moe") and isinstance(layer.moe, FusedMoEBlock):
|
if hasattr(layer, "moe") and isinstance(layer.moe, FusedMoEBlock):
|
||||||
self.moe_layers.append(layer.moe)
|
example_layer = layer.moe
|
||||||
|
self.moe_layers.append(layer.moe.experts)
|
||||||
|
|
||||||
self.expert_weights = []
|
|
||||||
assert len(self.moe_layers) > 0, "No MoE layers found in the model."
|
assert len(self.moe_layers) > 0, "No MoE layers found in the model."
|
||||||
example_layer = self.moe_layers[0]
|
|
||||||
self.num_moe_layers = len(self.moe_layers)
|
self.num_moe_layers = len(self.moe_layers)
|
||||||
self.num_expert_groups = 1
|
self.num_expert_groups = 1
|
||||||
self.num_shared_experts = 0
|
self.num_shared_experts = 0
|
||||||
@@ -959,24 +959,6 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
|||||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||||
return self.model.embed_tokens(input_ids)
|
return self.model.embed_tokens(input_ids)
|
||||||
|
|
||||||
def set_eplb_state(
|
|
||||||
self,
|
|
||||||
expert_load_view: torch.Tensor,
|
|
||||||
logical_to_physical_map: torch.Tensor,
|
|
||||||
logical_replica_count: torch.Tensor,
|
|
||||||
) -> None:
|
|
||||||
for layer_idx, layer in enumerate(self.moe_layers):
|
|
||||||
experts = layer.experts
|
|
||||||
assert isinstance(experts, MoERunner)
|
|
||||||
# Register the expert weights.
|
|
||||||
self.expert_weights.append(experts.get_expert_weights())
|
|
||||||
experts.set_eplb_state(
|
|
||||||
moe_layer_idx=layer_idx,
|
|
||||||
expert_load_view=expert_load_view,
|
|
||||||
logical_to_physical_map=logical_to_physical_map,
|
|
||||||
logical_replica_count=logical_replica_count,
|
|
||||||
)
|
|
||||||
|
|
||||||
def update_physical_experts_metadata(
|
def update_physical_experts_metadata(
|
||||||
self,
|
self,
|
||||||
num_physical_experts: int,
|
num_physical_experts: int,
|
||||||
|
|||||||
@@ -28,14 +28,9 @@ from vllm.config.utils import getattr_iter
|
|||||||
from vllm.distributed import get_dp_group, get_ep_group
|
from vllm.distributed import get_dp_group, get_ep_group
|
||||||
from vllm.forward_context import ForwardContext, get_forward_context
|
from vllm.forward_context import ForwardContext, get_forward_context
|
||||||
from vllm.model_executor.custom_op import PluggableLayer
|
from vllm.model_executor.custom_op import PluggableLayer
|
||||||
from vllm.model_executor.layers.fused_moe import (
|
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
|
||||||
FusedMoE,
|
|
||||||
MoERunner,
|
|
||||||
fused_moe_make_expert_params_mapping,
|
|
||||||
)
|
|
||||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||||
from vllm.model_executor.models.utils import maybe_prefix
|
from vllm.model_executor.models.utils import maybe_prefix
|
||||||
from vllm.platforms import current_platform
|
|
||||||
from vllm.utils.torch_utils import direct_register_custom_op
|
from vllm.utils.torch_utils import direct_register_custom_op
|
||||||
|
|
||||||
from .utils import log_replacement
|
from .utils import log_replacement
|
||||||
@@ -52,7 +47,7 @@ class TransformersMoEState:
|
|||||||
|
|
||||||
# --8<-- [start:transformers_fused_moe]
|
# --8<-- [start:transformers_fused_moe]
|
||||||
@PluggableLayer.register("transformers_fused_moe")
|
@PluggableLayer.register("transformers_fused_moe")
|
||||||
class TransformersFusedMoE(MoERunner):
|
class TransformersMoERunner(MoERunner):
|
||||||
"""Custom FusedMoE for the Transformers modeling backend."""
|
"""Custom FusedMoE for the Transformers modeling backend."""
|
||||||
|
|
||||||
# --8<-- [end:transformers_fused_moe]
|
# --8<-- [end:transformers_fused_moe]
|
||||||
@@ -93,7 +88,7 @@ class TransformersFusedMoE(MoERunner):
|
|||||||
return self.routed_experts.load_weights(weights)
|
return self.routed_experts.load_weights(weights)
|
||||||
|
|
||||||
|
|
||||||
def transformers_moe_forward(
|
def _transformers_moe_forward(
|
||||||
hidden_states: torch.Tensor,
|
hidden_states: torch.Tensor,
|
||||||
topk_ids: torch.Tensor,
|
topk_ids: torch.Tensor,
|
||||||
topk_weights: torch.Tensor,
|
topk_weights: torch.Tensor,
|
||||||
@@ -106,7 +101,7 @@ def transformers_moe_forward(
|
|||||||
return self._forward_super(hidden_states, topk_weights)
|
return self._forward_super(hidden_states, topk_weights)
|
||||||
|
|
||||||
|
|
||||||
def transformers_moe_forward_fake(
|
def _transformers_moe_forward_fake(
|
||||||
hidden_states: torch.Tensor,
|
hidden_states: torch.Tensor,
|
||||||
topk_ids: torch.Tensor,
|
topk_ids: torch.Tensor,
|
||||||
topk_weights: torch.Tensor,
|
topk_weights: torch.Tensor,
|
||||||
@@ -117,10 +112,9 @@ def transformers_moe_forward_fake(
|
|||||||
|
|
||||||
direct_register_custom_op(
|
direct_register_custom_op(
|
||||||
op_name="transformers_moe_forward",
|
op_name="transformers_moe_forward",
|
||||||
op_func=transformers_moe_forward,
|
op_func=_transformers_moe_forward,
|
||||||
mutates_args=["hidden_states"],
|
mutates_args=["hidden_states"],
|
||||||
fake_impl=transformers_moe_forward_fake,
|
fake_impl=_transformers_moe_forward_fake,
|
||||||
dispatch_key=current_platform.dispatch_key,
|
|
||||||
tags=(torch.Tag.needs_fixed_stride_order,),
|
tags=(torch.Tag.needs_fixed_stride_order,),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -131,20 +125,6 @@ class MoEMixin(MixtureOfExperts):
|
|||||||
# Skip MixtureOfExperts.__init__ and call the next class in MRO
|
# Skip MixtureOfExperts.__init__ and call the next class in MRO
|
||||||
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
|
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
|
||||||
|
|
||||||
def set_eplb_state(
|
|
||||||
self,
|
|
||||||
expert_load_view: torch.Tensor,
|
|
||||||
logical_to_physical_map: torch.Tensor,
|
|
||||||
logical_replica_count: torch.Tensor,
|
|
||||||
):
|
|
||||||
for moe_layer_idx, mlp_layer in enumerate(self.mlp_moe_layers):
|
|
||||||
mlp_layer.experts.set_eplb_state(
|
|
||||||
moe_layer_idx=moe_layer_idx,
|
|
||||||
expert_load_view=expert_load_view,
|
|
||||||
logical_to_physical_map=logical_to_physical_map,
|
|
||||||
logical_replica_count=logical_replica_count,
|
|
||||||
)
|
|
||||||
|
|
||||||
def update_physical_experts_metadata(
|
def update_physical_experts_metadata(
|
||||||
self,
|
self,
|
||||||
num_physical_experts: int,
|
num_physical_experts: int,
|
||||||
@@ -154,7 +134,7 @@ class MoEMixin(MixtureOfExperts):
|
|||||||
self.num_physical_experts = num_physical_experts
|
self.num_physical_experts = num_physical_experts
|
||||||
self.num_local_physical_experts = num_local_physical_experts
|
self.num_local_physical_experts = num_local_physical_experts
|
||||||
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
||||||
for mlp in self.mlp_moe_layers:
|
for mlp in self.mlp_layers:
|
||||||
mlp.n_local_physical_experts = num_local_physical_experts
|
mlp.n_local_physical_experts = num_local_physical_experts
|
||||||
mlp.n_physical_experts = num_physical_experts
|
mlp.n_physical_experts = num_physical_experts
|
||||||
mlp.n_redundant_experts = self.num_redundant_experts
|
mlp.n_redundant_experts = self.num_redundant_experts
|
||||||
@@ -185,7 +165,7 @@ class MoEMixin(MixtureOfExperts):
|
|||||||
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
||||||
for gate_proj, down_proj, up_proj in ckpt_names:
|
for gate_proj, down_proj, up_proj in ckpt_names:
|
||||||
expert_mapping.extend(
|
expert_mapping.extend(
|
||||||
fused_moe_make_expert_params_mapping(
|
RoutedExperts.make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name=gate_proj,
|
ckpt_gate_proj_name=gate_proj,
|
||||||
ckpt_down_proj_name=down_proj,
|
ckpt_down_proj_name=down_proj,
|
||||||
@@ -248,10 +228,8 @@ class MoEMixin(MixtureOfExperts):
|
|||||||
# MixtureOfExperts mixin settings
|
# MixtureOfExperts mixin settings
|
||||||
ep_size = get_ep_group().world_size
|
ep_size = get_ep_group().world_size
|
||||||
|
|
||||||
self.mlp_moe_layers = [] # Used for MixtureOfExperts methods
|
self.mlp_layers = [] # Used for MixtureOfExperts methods
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
self.expert_weights = []
|
|
||||||
self.num_moe_layers = 0
|
|
||||||
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
|
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
|
||||||
self.num_logical_experts = num_experts
|
self.num_logical_experts = num_experts
|
||||||
self.num_physical_experts = num_experts + num_redundant_experts
|
self.num_physical_experts = num_experts + num_redundant_experts
|
||||||
@@ -335,19 +313,18 @@ class MoEMixin(MixtureOfExperts):
|
|||||||
custom_routing_function,
|
custom_routing_function,
|
||||||
moe_state=moe_state,
|
moe_state=moe_state,
|
||||||
),
|
),
|
||||||
runner_cls=TransformersFusedMoE,
|
runner_cls=TransformersMoERunner,
|
||||||
runner_args={"moe_state": moe_state},
|
runner_args={"moe_state": moe_state},
|
||||||
)
|
)
|
||||||
mlp.experts = fused_experts
|
mlp.experts = fused_experts
|
||||||
log_replacement(qual_name, experts, fused_experts)
|
log_replacement(qual_name, experts, fused_experts)
|
||||||
# Update MixtureOfExperts mixin state
|
# Update MixtureOfExperts mixin state
|
||||||
self.mlp_moe_layers.append(mlp)
|
self.mlp_layers.append(mlp)
|
||||||
self.moe_layers.append(fused_experts)
|
self.moe_layers.append(fused_experts)
|
||||||
self.expert_weights.append(fused_experts.get_expert_weights())
|
|
||||||
self.num_moe_layers += 1
|
|
||||||
else:
|
else:
|
||||||
_recursive_replace(child_module, prefix=qual_name)
|
_recursive_replace(child_module, prefix=qual_name)
|
||||||
|
|
||||||
_recursive_replace(self.model, prefix="model")
|
_recursive_replace(self.model, prefix="model")
|
||||||
|
self.num_moe_layers = len(self.moe_layers)
|
||||||
# Continue with the replacement of layers in Base
|
# Continue with the replacement of layers in Base
|
||||||
super().recursive_replace()
|
super().recursive_replace()
|
||||||
|
|||||||
Reference in New Issue
Block a user