[MoE Refactor] Rename FusedMoE.make_expert_params_mapping to fused_moe_make_expert_params_mapping (#40671)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
|||||||
from vllm.model_executor.layers.fused_moe.layer import (
|
from vllm.model_executor.layers.fused_moe.layer import (
|
||||||
FusedMoE,
|
FusedMoE,
|
||||||
FusedMoeWeightScaleSupported,
|
FusedMoeWeightScaleSupported,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||||
FusedMoEActivationFormat,
|
FusedMoEActivationFormat,
|
||||||
@@ -65,6 +66,7 @@ __all__ = [
|
|||||||
"RoutingMethodType",
|
"RoutingMethodType",
|
||||||
"activation_without_mul",
|
"activation_without_mul",
|
||||||
"apply_moe_activation",
|
"apply_moe_activation",
|
||||||
|
"fused_moe_make_expert_params_mapping",
|
||||||
"override_config",
|
"override_config",
|
||||||
"get_config",
|
"get_config",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1618,6 +1618,25 @@ class FusedMoE(PluggableLayer):
|
|||||||
return s
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
# This is a temporary forwarding method which will be removed/modified layer.
|
||||||
|
def fused_moe_make_expert_params_mapping(
|
||||||
|
model: torch.nn.Module,
|
||||||
|
ckpt_gate_proj_name: str,
|
||||||
|
ckpt_down_proj_name: str,
|
||||||
|
ckpt_up_proj_name: str,
|
||||||
|
num_experts: int,
|
||||||
|
num_redundant_experts: int = 0,
|
||||||
|
) -> list[tuple[str, str, int, str]]:
|
||||||
|
return FusedMoE.make_expert_params_mapping(
|
||||||
|
model,
|
||||||
|
ckpt_gate_proj_name,
|
||||||
|
ckpt_down_proj_name,
|
||||||
|
ckpt_up_proj_name,
|
||||||
|
num_experts,
|
||||||
|
num_redundant_experts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# Mark the FusedMoE weight_loader as supporting MoE-specific parameters
|
# Mark the FusedMoE weight_loader as supporting MoE-specific parameters
|
||||||
# to avoid expensive runtime reflection in model loading code
|
# to avoid expensive runtime reflection in model loading code
|
||||||
FusedMoE.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
FusedMoE.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
||||||
|
|||||||
@@ -42,7 +42,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
@@ -916,7 +919,7 @@ class AXK1ForCausalLM(
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
@@ -950,7 +953,7 @@ class AXK1ForCausalLM(
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -18,7 +18,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
@@ -479,7 +482,7 @@ class AfmoeModel(nn.Module, EagleModelMixin):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -18,7 +18,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_experts,
|
||||||
|
fused_topk,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
|
|||||||
@@ -14,7 +14,9 @@ from vllm.config.multimodal import BaseDummyOptions
|
|||||||
from vllm.distributed import get_tensor_model_parallel_rank
|
from vllm.distributed import get_tensor_model_parallel_rank
|
||||||
from vllm.inputs import MultiModalDataDict
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
|
|||||||
@@ -41,7 +41,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -461,7 +464,7 @@ class BailingMoeModel(nn.Module):
|
|||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -21,7 +21,10 @@ from vllm.model_executor.layers.fla.ops.layernorm_guard import (
|
|||||||
RMSNormGated,
|
RMSNormGated,
|
||||||
layernorm_fn,
|
layernorm_fn,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
@@ -990,7 +993,7 @@ class BailingMoeV25Model(nn.Module):
|
|||||||
|
|
||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
"""Get expert parameter mapping for MoE layers."""
|
"""Get expert parameter mapping for MoE layers."""
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -15,7 +15,9 @@ from vllm.distributed import (
|
|||||||
get_tensor_model_parallel_world_size,
|
get_tensor_model_parallel_world_size,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
ReplicatedLinear,
|
ReplicatedLinear,
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ import torch.nn as nn
|
|||||||
|
|
||||||
from vllm.compilation.decorators import support_torch_compile
|
from vllm.compilation.decorators import support_torch_compile
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
@@ -105,7 +107,7 @@ class DeepseekV2Model(nn.Module):
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -11,7 +11,9 @@ from vllm._aiter_ops import rocm_aiter_ops
|
|||||||
from vllm.compilation.decorators import support_torch_compile
|
from vllm.compilation.decorators import support_torch_compile
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
@@ -252,7 +254,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
|||||||
]
|
]
|
||||||
stacked_params_mapping.extend(indexer_fused_mapping)
|
stacked_params_mapping.extend(indexer_fused_mapping)
|
||||||
|
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ from vllm.model_executor.layers.fused_moe import (
|
|||||||
FusedMoE,
|
FusedMoE,
|
||||||
GateLinear,
|
GateLinear,
|
||||||
RoutingMethodType,
|
RoutingMethodType,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -1432,7 +1433,7 @@ class DeepseekV2ForCausalLM(
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
@@ -1474,7 +1475,7 @@ class DeepseekV2ForCausalLM(
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -40,7 +40,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -413,7 +416,7 @@ class Dots1Model(nn.Module):
|
|||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -42,7 +42,10 @@ from vllm.distributed import (
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -485,7 +488,7 @@ class Ernie4_5_MoeModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -36,7 +36,10 @@ from vllm.config import CacheConfig, VllmConfig
|
|||||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -649,7 +652,7 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP):
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -30,7 +30,10 @@ from vllm.distributed import (
|
|||||||
get_pp_group,
|
get_pp_group,
|
||||||
get_tensor_model_parallel_world_size,
|
get_tensor_model_parallel_world_size,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
@@ -326,7 +329,7 @@ class ExaoneMoeModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ from torch import nn
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||||
from vllm.model_executor.models.olmoe import OlmoeAttention, OlmoeForCausalLM
|
from vllm.model_executor.models.olmoe import OlmoeAttention, OlmoeForCausalLM
|
||||||
|
|||||||
@@ -37,7 +37,10 @@ from vllm.forward_context import get_forward_context
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import GeluAndMul
|
from vllm.model_executor.layers.activation import GeluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE, GateLinear
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
GateLinear,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
|
|||||||
@@ -42,7 +42,10 @@ from vllm.distributed import (
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -466,7 +469,7 @@ class Glm4MoeModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -41,7 +41,9 @@ from vllm.distributed import (
|
|||||||
get_pp_group,
|
get_pp_group,
|
||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
@@ -308,7 +310,7 @@ class Glm4MoeLiteModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
@@ -334,7 +336,7 @@ class Glm4MoeLiteModel(nn.Module):
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
@@ -616,7 +618,7 @@ class Glm4MoeLiteForCausalLM(
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -32,7 +32,10 @@ from transformers import PretrainedConfig
|
|||||||
|
|
||||||
from vllm._aiter_ops import rocm_aiter_ops
|
from vllm._aiter_ops import rocm_aiter_ops
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
@@ -260,7 +263,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
|||||||
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
||||||
]
|
]
|
||||||
|
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -31,7 +31,10 @@ import torch.nn as nn
|
|||||||
from transformers import PretrainedConfig
|
from transformers import PretrainedConfig
|
||||||
|
|
||||||
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
|
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
@@ -247,7 +250,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -20,7 +20,10 @@ from vllm.distributed import (
|
|||||||
tensor_model_parallel_all_gather,
|
tensor_model_parallel_all_gather,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -331,7 +334,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
|||||||
# Params for weights, weight scales, activation scales
|
# Params for weights, weight scales, activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
# NOTE: this is only used for quark.
|
# NOTE: this is only used for quark.
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="w1",
|
ckpt_gate_proj_name="w1",
|
||||||
ckpt_down_proj_name="w2",
|
ckpt_down_proj_name="w2",
|
||||||
|
|||||||
@@ -39,7 +39,10 @@ from vllm.distributed import (
|
|||||||
tensor_model_parallel_all_gather,
|
tensor_model_parallel_all_gather,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -351,7 +354,7 @@ class GraniteMoeModel(nn.Module):
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="w1",
|
ckpt_gate_proj_name="w1",
|
||||||
ckpt_down_proj_name="w2",
|
ckpt_down_proj_name="w2",
|
||||||
|
|||||||
@@ -38,7 +38,10 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import GeluAndMul
|
from vllm.model_executor.layers.activation import GeluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -519,7 +522,7 @@ class Grok1Model(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Map expert parameter names to standard names
|
# Map expert parameter names to standard names
|
||||||
num_experts = _get_num_experts(self.config)
|
num_experts = _get_num_experts(self.config)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name=self.ckpt_gate_proj_name,
|
ckpt_gate_proj_name=self.ckpt_gate_proj_name,
|
||||||
ckpt_down_proj_name=self.ckpt_down_proj_name,
|
ckpt_down_proj_name=self.ckpt_down_proj_name,
|
||||||
|
|||||||
@@ -42,7 +42,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
@@ -712,7 +715,7 @@ class HunYuanModel(nn.Module, EagleModelMixin):
|
|||||||
if _is_moe(self.config):
|
if _is_moe(self.config):
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -41,7 +41,9 @@ from vllm.distributed import (
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
|||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.distributed.parallel_state import get_pp_group
|
from vllm.distributed.parallel_state import get_pp_group
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -378,7 +381,7 @@ class JambaModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.kda import KimiDeltaAttention
|
from vllm.model_executor.layers.kda import KimiDeltaAttention
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -476,7 +479,7 @@ class KimiLinearModel(nn.Module):
|
|||||||
if self.config.is_moe:
|
if self.config.is_moe:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="w1",
|
ckpt_gate_proj_name="w1",
|
||||||
ckpt_down_proj_name="w2",
|
ckpt_down_proj_name="w2",
|
||||||
|
|||||||
@@ -15,7 +15,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -482,7 +485,7 @@ class Lfm2MoeModel(nn.Module):
|
|||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="w1",
|
ckpt_gate_proj_name="w1",
|
||||||
ckpt_down_proj_name="w2",
|
ckpt_down_proj_name="w2",
|
||||||
|
|||||||
@@ -36,7 +36,10 @@ from vllm.model_executor.layers.attention import (
|
|||||||
Attention,
|
Attention,
|
||||||
ChunkedLocalAttention,
|
ChunkedLocalAttention,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -414,7 +417,7 @@ class Llama4Model(LlamaModel):
|
|||||||
params_dict: The dictionary of module parameters.
|
params_dict: The dictionary of module parameters.
|
||||||
loaded_params: The set of already loaded parameters.
|
loaded_params: The set of already loaded parameters.
|
||||||
expert_params_mapping: The mapping of expert parameters. Must be
|
expert_params_mapping: The mapping of expert parameters. Must be
|
||||||
generated by FusedMoE.make_expert_params_mapping().
|
generated by fused_moe_make_expert_params_mapping().
|
||||||
fused: Whether the expert weights are fused into a single weight
|
fused: Whether the expert weights are fused into a single weight
|
||||||
tensor or are separate weight tensors for each expert.
|
tensor or are separate weight tensors for each expert.
|
||||||
When fused is True, loaded_weight should have shape of:
|
When fused is True, loaded_weight should have shape of:
|
||||||
@@ -554,7 +557,7 @@ class Llama4Model(LlamaModel):
|
|||||||
fused_experts_params = False
|
fused_experts_params = False
|
||||||
# Expert parameter mapping for the case where the expert weights are
|
# Expert parameter mapping for the case where the expert weights are
|
||||||
# not fused into a single weight tensor.
|
# not fused into a single weight tensor.
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
@@ -564,7 +567,7 @@ class Llama4Model(LlamaModel):
|
|||||||
)
|
)
|
||||||
# Expert parameter mapping for the case where the expert weights are
|
# Expert parameter mapping for the case where the expert weights are
|
||||||
# fused into a single weight tensor.
|
# fused into a single weight tensor.
|
||||||
expert_params_mapping_fused = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping_fused = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_up_proj",
|
ckpt_gate_proj_name="gate_up_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -46,7 +46,10 @@ from vllm.config import CacheConfig, VllmConfig
|
|||||||
from vllm.distributed import get_pp_group
|
from vllm.distributed import get_pp_group
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -622,7 +625,7 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -22,7 +22,10 @@ from vllm.distributed import (
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -511,7 +514,7 @@ class MiMoV2Model(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -38,7 +38,10 @@ from vllm.distributed import (
|
|||||||
get_tensor_model_parallel_world_size,
|
get_tensor_model_parallel_world_size,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -393,7 +396,7 @@ class MiniMaxM2Model(nn.Module, EagleModelMixin):
|
|||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="w1",
|
ckpt_gate_proj_name="w1",
|
||||||
ckpt_down_proj_name="w2",
|
ckpt_down_proj_name="w2",
|
||||||
|
|||||||
@@ -24,7 +24,9 @@ from vllm.distributed.parallel_state import (
|
|||||||
from vllm.forward_context import get_forward_context
|
from vllm.forward_context import get_forward_context
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
|
|||||||
@@ -40,7 +40,10 @@ from vllm.distributed import (
|
|||||||
get_tensor_model_parallel_world_size,
|
get_tensor_model_parallel_world_size,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -364,7 +367,7 @@ class MixtralModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="w1",
|
ckpt_gate_proj_name="w1",
|
||||||
ckpt_down_proj_name="w2",
|
ckpt_down_proj_name="w2",
|
||||||
|
|||||||
@@ -40,7 +40,9 @@ from vllm.config.multimodal import BaseDummyOptions
|
|||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.inputs import MultiModalDataDict
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -1072,7 +1074,7 @@ class Llama4ForConditionalGeneration(
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.fused_moe import (
|
|||||||
FusedMoE,
|
FusedMoE,
|
||||||
GateLinear,
|
GateLinear,
|
||||||
activation_without_mul,
|
activation_without_mul,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
@@ -652,7 +653,7 @@ class NemotronHModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
if self.has_moe:
|
if self.has_moe:
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
# - FusedMoe.w1 (aka gate_proj) should be up_proj since that's
|
# - FusedMoe.w1 (aka gate_proj) should be up_proj since that's
|
||||||
# what the activation is applied to
|
# what the activation is applied to
|
||||||
# - FusedMoe.w3 (aka up_proj) should be ignored since we're
|
# - FusedMoe.w3 (aka up_proj) should be ignored since we're
|
||||||
|
|||||||
@@ -11,7 +11,9 @@ import torch.nn as nn
|
|||||||
from vllm.compilation.decorators import support_torch_compile
|
from vllm.compilation.decorators import support_torch_compile
|
||||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||||
from vllm.config.parallel import ParallelConfig
|
from vllm.config.parallel import ParallelConfig
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
@@ -399,7 +401,7 @@ class NemotronHMTP(nn.Module, SupportsPP):
|
|||||||
if getattr(self.config, "model_type", None) == "nemotron_h_puzzle":
|
if getattr(self.config, "model_type", None) == "nemotron_h_puzzle":
|
||||||
num_experts = self.config.mtp_n_routed_experts
|
num_experts = self.config.mtp_n_routed_experts
|
||||||
if num_experts is not None:
|
if num_experts is not None:
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="up_proj",
|
ckpt_gate_proj_name="up_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -32,7 +32,10 @@ from vllm.distributed import (
|
|||||||
from vllm.distributed.utils import split_tensor_along_last_dim
|
from vllm.distributed.utils import split_tensor_along_last_dim
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
@@ -336,7 +339,7 @@ class OlmoeModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -44,7 +44,10 @@ from vllm.model_executor.layers.attention import (
|
|||||||
Attention,
|
Attention,
|
||||||
StaticSinkAttention,
|
StaticSinkAttention,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
@@ -1149,7 +1152,7 @@ class OpenPanguModel(nn.Module):
|
|||||||
]
|
]
|
||||||
has_experts = hasattr(self.config, "n_routed_experts")
|
has_experts = hasattr(self.config, "n_routed_experts")
|
||||||
if has_experts:
|
if has_experts:
|
||||||
expert_merge_mapping = FusedMoE.make_expert_params_mapping(
|
expert_merge_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -28,7 +28,9 @@ from vllm.config import VllmConfig
|
|||||||
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
@@ -147,7 +149,7 @@ class OpenPanguMTP(nn.Module):
|
|||||||
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1),
|
||||||
]
|
]
|
||||||
|
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -32,7 +32,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -690,7 +693,7 @@ class Param2MoEModel(nn.Module):
|
|||||||
return loaded_params
|
return loaded_params
|
||||||
|
|
||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -35,7 +35,10 @@ from vllm.compilation.decorators import support_torch_compile
|
|||||||
from vllm.config import CacheConfig, VllmConfig
|
from vllm.config import CacheConfig, VllmConfig
|
||||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
ReplicatedLinear,
|
ReplicatedLinear,
|
||||||
@@ -514,7 +517,7 @@ class PhiMoEModel(nn.Module):
|
|||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="w1",
|
ckpt_gate_proj_name="w1",
|
||||||
ckpt_down_proj_name="w2",
|
ckpt_down_proj_name="w2",
|
||||||
|
|||||||
@@ -40,7 +40,10 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -418,7 +421,7 @@ class Qwen2MoeModel(nn.Module):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -12,7 +12,9 @@ from vllm.compilation.decorators import support_torch_compile
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.distributed.parallel_state import get_pp_group
|
from vllm.distributed.parallel_state import get_pp_group
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
@@ -194,7 +196,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -43,7 +43,10 @@ from vllm.distributed import (
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
MergedColumnParallelLinear,
|
||||||
@@ -516,7 +519,7 @@ class Qwen3MoeModel(nn.Module, EagleModelMixin):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -23,7 +23,10 @@ from vllm.distributed import (
|
|||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import (
|
from vllm.model_executor.layers.layernorm import (
|
||||||
GemmaRMSNorm as Qwen3NextRMSNorm,
|
GemmaRMSNorm as Qwen3NextRMSNorm,
|
||||||
)
|
)
|
||||||
@@ -533,7 +536,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
|||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -11,7 +11,9 @@ from vllm.compilation.decorators import support_torch_compile
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.distributed.parallel_state import get_pp_group
|
from vllm.distributed.parallel_state import get_pp_group
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
@@ -145,7 +147,7 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
|||||||
|
|
||||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||||
# (param_name, weight_name, expert_id, shard_id)
|
# (param_name, weight_name, expert_id, shard_id)
|
||||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -35,7 +35,10 @@ from vllm.distributed import (
|
|||||||
get_tensor_model_parallel_world_size,
|
get_tensor_model_parallel_world_size,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
@@ -529,7 +532,7 @@ class SarvamMLAModel(nn.Module):
|
|||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||||
return FusedMoE.make_expert_params_mapping(
|
return fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -18,7 +18,9 @@ from vllm.distributed import (
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
|
|||||||
@@ -23,7 +23,10 @@ from vllm.distributed import (
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
|
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||||
from vllm.model_executor.layers.linear import (
|
from vllm.model_executor.layers.linear import (
|
||||||
ColumnParallelLinear,
|
ColumnParallelLinear,
|
||||||
@@ -637,7 +640,7 @@ class Step3p5Model(nn.Module):
|
|||||||
]
|
]
|
||||||
|
|
||||||
# New per-expert format: .moe.experts.E.gate_proj.weight_packed [out, in]
|
# New per-expert format: .moe.experts.E.gate_proj.weight_packed [out, in]
|
||||||
per_expert_mapping = FusedMoE.make_expert_params_mapping(
|
per_expert_mapping = fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name="gate_proj",
|
ckpt_gate_proj_name="gate_proj",
|
||||||
ckpt_down_proj_name="down_proj",
|
ckpt_down_proj_name="down_proj",
|
||||||
|
|||||||
@@ -25,7 +25,10 @@ from vllm.config.utils import getattr_iter
|
|||||||
from vllm.distributed import get_dp_group, get_ep_group
|
from vllm.distributed import get_dp_group, get_ep_group
|
||||||
from vllm.forward_context import ForwardContext, get_forward_context
|
from vllm.forward_context import ForwardContext, get_forward_context
|
||||||
from vllm.model_executor.custom_op import PluggableLayer
|
from vllm.model_executor.custom_op import PluggableLayer
|
||||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
|
FusedMoE,
|
||||||
|
fused_moe_make_expert_params_mapping,
|
||||||
|
)
|
||||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||||
from vllm.model_executor.models.utils import maybe_prefix
|
from vllm.model_executor.models.utils import maybe_prefix
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
@@ -179,7 +182,7 @@ class MoEMixin(MixtureOfExperts):
|
|||||||
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
||||||
for gate_proj, down_proj, up_proj in ckpt_names:
|
for gate_proj, down_proj, up_proj in ckpt_names:
|
||||||
expert_mapping.extend(
|
expert_mapping.extend(
|
||||||
FusedMoE.make_expert_params_mapping(
|
fused_moe_make_expert_params_mapping(
|
||||||
self,
|
self,
|
||||||
ckpt_gate_proj_name=gate_proj,
|
ckpt_gate_proj_name=gate_proj,
|
||||||
ckpt_down_proj_name=down_proj,
|
ckpt_down_proj_name=down_proj,
|
||||||
|
|||||||
Reference in New Issue
Block a user