[Model][MoE] Load MegaMoE experts directly into the DeepGEMM layout (#60096)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Aaron Hao <hao-aaron@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Aaron Hao <hao-aaron@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-10-13 11:52:08 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Aaron Hao
parent 624a5ddafd
commit c5bda658d2
11 changed files with 564 additions and 891 deletions
+8 -9
View File
@@ -46,7 +46,7 @@ from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
@@ -507,6 +507,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts." in name:
experts = self.model.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
@@ -574,17 +581,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
self.process_weights_after_loading()
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
def _finalize_moe(self) -> None:
for layer in self.model.layers:
layer.ffn.finalize_mega_moe_weights()
def process_weights_after_loading(self) -> None:
self._finalize_moe()
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
+90 -226
View File
@@ -33,9 +33,7 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
DeepGemmMegaMoEBackend,
get_deep_gemm_mega_moe_backend,
ue8m0_uint8_to_float,
)
from vllm.model_executor.layers.fused_moe.router.base_router import (
eplb_map_to_physical_and_record,
@@ -225,6 +223,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
hidden_size: int,
intermediate_size: int,
num_shared_experts: int = 0,
shared_experts: nn.Module | None = None,
prefix: str = "",
num_logical_experts: int | None = None,
):
@@ -238,7 +237,6 @@ class DeepseekV4MegaMoEExperts(nn.Module):
self.top_k = top_k
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_shared_experts = num_shared_experts
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.num_logical_experts = (
@@ -247,64 +245,33 @@ class DeepseekV4MegaMoEExperts(nn.Module):
self.eplb_state = EplbLayerState()
weight_attrs = {"weight_loader": self.weight_loader}
self.w13_weight = nn.Parameter(
torch.zeros(
num_local_experts,
2 * intermediate_size,
hidden_size // 2,
dtype=torch.uint8,
),
requires_grad=False,
device = torch.get_default_device()
if device.type != current_platform.device_type: # meta under ipc_cache
device = torch.device(
current_platform.device_type, torch.accelerator.current_device_index()
)
self.backend = get_deep_gemm_mega_moe_backend(
device, hidden_size, intermediate_size
)
set_weight_attrs(self.w13_weight, weight_attrs)
self.w13_weight_scale = nn.Parameter(
torch.zeros(
num_local_experts,
2 * intermediate_size,
hidden_size // 32,
dtype=torch.uint8,
),
requires_grad=False,
if num_shared_experts and not self.backend.supports_shared_experts(
shared_experts
):
num_shared_experts = 0
self.num_shared_experts = num_shared_experts
self.backend.create_weights(
self,
num_local_experts=num_local_experts,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
num_shared_experts=num_shared_experts,
)
set_weight_attrs(self.w13_weight_scale, weight_attrs)
for name, param in self.named_parameters(recurse=False):
shared = name.startswith("shared_")
loader = self.shared_weight_loader if shared else self.weight_loader
set_weight_attrs(param, {"weight_loader": loader})
self.w13_weight_scale.quant_method = "block"
self.w2_weight = nn.Parameter(
torch.zeros(
num_local_experts,
hidden_size,
intermediate_size // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
set_weight_attrs(self.w2_weight, weight_attrs)
self.w2_weight_scale = nn.Parameter(
torch.zeros(
num_local_experts,
hidden_size,
intermediate_size // 32,
dtype=torch.uint8,
),
requires_grad=False,
)
set_weight_attrs(self.w2_weight_scale, weight_attrs)
self.w2_weight_scale.quant_method = "block"
self._transformed_l1_weights: tuple[torch.Tensor, torch.Tensor] | None = None
self._transformed_l2_weights: tuple[torch.Tensor, torch.Tensor] | None = None
self._transformed_shared_l1_weights: (
tuple[torch.Tensor, torch.Tensor] | None
) = None
self._transformed_shared_l2_weights: (
tuple[torch.Tensor, torch.Tensor] | None
) = None
self._backend: DeepGemmMegaMoEBackend | None = None
# Register in the static forward context so the custom-op wrapper
# can look up this module by name from within a torch.compile graph.
compilation_config = vllm_config.compilation_config
@@ -332,144 +299,57 @@ class DeepseekV4MegaMoEExperts(nn.Module):
return_success: bool = False,
) -> bool | None:
local_expert_ids = self._map_global_expert_id(expert_id)
if not local_expert_ids:
param_prefix = "w2_" if shard_id == "w2" else "w13_"
if not local_expert_ids or param_prefix not in weight_name:
return False if return_success else None
loaded_any = False
is_scale = weight_name.endswith("weight_scale")
for local_expert_id in local_expert_ids:
expert_data = param.data[local_expert_id]
if shard_id in ("w1", "w3"):
if "w13_" not in weight_name:
continue
shard_offset = 0 if shard_id == "w1" else self.intermediate_size
expert_data = expert_data.narrow(
0, shard_offset, self.intermediate_size
)
elif shard_id == "w2":
if "w2_" not in weight_name:
continue
else:
raise ValueError(f"Unsupported expert shard id: {shard_id}")
if expert_data.shape != loaded_weight.shape:
raise ValueError(
f"DeepSeek V4 MegaMoE expert weight shape mismatch for "
f"{weight_name}: parameter shard {tuple(expert_data.shape)} "
f"vs checkpoint {tuple(loaded_weight.shape)}"
)
expert_data.copy_(loaded_weight)
loaded_any = True
if return_success:
return loaded_any
return None
@staticmethod
def _ue8m0_uint8_to_float(sf: torch.Tensor) -> torch.Tensor:
return ue8m0_uint8_to_float(sf)
def _ensure_backend(self) -> DeepGemmMegaMoEBackend:
"""Lazily resolve the MegaMoE backend.
The backend is looked up via :func:`get_deep_gemm_mega_moe_backend`, which
performs all runtime capability checks. Failure propagates directly
to the caller. This must be called before :meth:`finalize_weights`
releases the loader-side parameters.
"""
if self._backend is not None:
return self._backend
assert self.w13_weight is not None, (
"_ensure_backend() must be called before finalize_weights() "
"releases the loader-side parameters."
)
self._backend = get_deep_gemm_mega_moe_backend(
self.w13_weight.device,
self.hidden_size,
self.intermediate_size,
)
return self._backend
def _check_runtime_supported(self) -> None:
"""Preserved for API compatibility; delegates to the backend check."""
self._ensure_backend()
def _finalize_shared_expert_weights(self, shared_experts: DeepseekV4MLP) -> None:
"""Delegate shared-expert weight transform to the backend and update state.
On success, populates ``self._transformed_shared_l{1,2}_weights``. If
the backend cannot fuse shared experts (returns ``None``), disables
native fusion by setting ``self.num_shared_experts = 0``.
"""
backend = self._ensure_backend()
result = backend.transform_shared_expert_weights(
shared_experts=shared_experts,
num_shared_experts=self.num_shared_experts,
hidden_size=self.hidden_size,
intermediate_size=self.intermediate_size,
prefix=self.prefix,
)
if result is None:
self.num_shared_experts = 0
return
(
self._transformed_shared_l1_weights,
self._transformed_shared_l2_weights,
) = result
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
backend = self._ensure_backend()
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
if self._transformed_l1_weights is None:
self._transformed_l1_weights, self._transformed_l2_weights = (
backend.transform_weights(
w13_weight=self.w13_weight.data,
w13_weight_scale=self.w13_weight_scale.data,
w2_weight=self.w2_weight.data,
w2_weight_scale=self.w2_weight_scale.data,
num_local_experts=self.num_local_experts,
hidden_size=self.hidden_size,
intermediate_size=self.intermediate_size,
)
self.backend.load_weight(
param.data[local_expert_id], loaded_weight, shard_id, is_scale
)
# Drop the original loader-side parameters: the MegaMoE kernels only
# consume the transformed views above. transform_weights_for_mega_moe
# allocates a fresh tensor for the L1 weight (see
# _interleave_l1_weights) and fresh SF tensors for L1/L2; the L2
# weight is the only tensor that aliases the original storage, and
# _transformed_l2_weights still holds it, so the storage stays live
# after we drop the Parameter.
self.w13_weight = None
self.w13_weight_scale = None
self.w2_weight = None
self.w2_weight_scale = None
return True if return_success else None
if shared_experts is None or self.num_shared_experts == 0:
return
if self._transformed_shared_l1_weights is not None:
return
if not backend.supports_shared_experts(deep_gemm):
logger.warning_once(
"Disabling native MegaMoE shared-expert fusion because the "
"installed DeepGEMM Python API is older than the vLLM "
"source. Rebuild the vendored _deep_gemm_C extension to enable it.",
)
self.num_shared_experts = 0
return
def load_shared_expert_weight(self, name: str, loaded_weight: torch.Tensor) -> str:
"""Load one shared-expert checkpoint tensor; return the parameter name."""
if ".w2." in name or ".down_proj." in name:
param_name = "shared_l2_weight"
elif ".w1." in name or ".w3." in name:
param_name = "shared_l1_weight"
else:
raise ValueError(f"Unexpected shared-expert tensor: {name}")
if "weight_scale" in name:
param_name += "_scale"
param = getattr(self, param_name)
param.weight_loader(param, loaded_weight, name)
return f"{name.partition('shared_experts.')[0]}experts.{param_name}"
self._finalize_shared_expert_weights(shared_experts)
def shared_weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
) -> None:
shard_id = (
"w1" if ".w1." in weight_name else "w3" if ".w3." in weight_name else "w2"
)
is_scale = "weight_scale" in weight_name
self.backend.load_weight(param.data, loaded_weight, shard_id, is_scale)
@property
def _transformed_l1_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
return self.w13_weight, self.w13_weight_scale
@property
def _transformed_l2_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
return self.w2_weight, self.w2_weight_scale
@property
def has_fused_shared_experts(self) -> bool:
return self._transformed_shared_l1_weights is not None
return self.num_shared_experts > 0
def get_symm_buffer(self):
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
backend = self._ensure_backend()
backend = self.backend
group = get_ep_group().device_group
device = torch.accelerator.current_device_index()
key = (
@@ -515,10 +395,6 @@ class DeepseekV4MegaMoEExperts(nn.Module):
)
def get_expert_weights(self) -> list[torch.Tensor]:
self.finalize_weights()
assert self._transformed_l1_weights is not None
assert self._transformed_l2_weights is not None
def _to_eplb_view(name: str, t: torch.Tensor) -> torch.Tensor:
"""Return a (num_local_experts, -1) view with contiguous memory layout."""
assert t.shape[0] == self.num_local_experts
@@ -601,7 +477,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
else None,
)
backend = self._ensure_backend()
backend = self.backend
symm_buffer = self.get_symm_buffer()
shared_x_sf = None
shared_block_m = None
@@ -630,28 +506,21 @@ class DeepseekV4MegaMoEExperts(nn.Module):
hidden_quant=backend.hidden_quant,
)
assert self._transformed_l1_weights is not None
assert self._transformed_l2_weights is not None
if self.has_fused_shared_experts:
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
l2_weights=self._transformed_l2_weights,
symm_buffer=symm_buffer,
shared_l1_weights=self._transformed_shared_l1_weights,
shared_l2_weights=self._transformed_shared_l2_weights,
activation_clamp=activation_clamp,
fast_math=fast_math,
)
else:
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
l2_weights=self._transformed_l2_weights,
symm_buffer=symm_buffer,
activation_clamp=activation_clamp,
fast_math=fast_math,
)
fused = self.has_fused_shared_experts
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
l2_weights=self._transformed_l2_weights,
symm_buffer=symm_buffer,
shared_l1_weights=(
(self.shared_l1_weight, self.shared_l1_weight_scale) if fused else None
),
shared_l2_weights=(
(self.shared_l2_weight, self.shared_l2_weight_scale) if fused else None
),
activation_clamp=activation_clamp,
fast_math=fast_math,
)
return y
@@ -842,9 +711,12 @@ class DeepseekV4MoE(nn.Module):
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
shared_experts=self.shared_experts,
prefix=f"{prefix}.experts",
)
self.experts = DeepseekV4MegaMoEExperts(vllm_config, **expert_kwargs)
if self.experts.has_fused_shared_experts:
self.shared_experts = None
def _init_fused_moe_experts(
self,
@@ -911,12 +783,7 @@ class DeepseekV4MoE(nn.Module):
return self._forward_fused_moe(hidden_states, input_ids)
org_shape = hidden_states.shape
# The MegaMoE experts module owns the backend; resolve and assert it.
self.experts._ensure_backend()
assert self.experts._backend is not None, (
"MegaMoE backend must be resolved before routing."
)
mega_backend = self.experts._backend
mega_backend = self.experts.backend
# Small local padded batches favor GateLinear; 128-expert gates cross earlier.
gate_threshold = 1 if self.gate.weight.shape[0] == 128 else 16
if (
@@ -983,10 +850,7 @@ class DeepseekV4MoE(nn.Module):
activation_clamp=activation_clamp,
)
if (
self.shared_experts is not None
and not self.experts.has_fused_shared_experts
):
if self.shared_experts is not None:
shared_output = self.shared_experts(hidden_states)
final_hidden_states += shared_output
@@ -1004,10 +868,6 @@ class DeepseekV4MoE(nn.Module):
return final_hidden_states.view(org_shape)
def finalize_mega_moe_weights(self) -> None:
if self.use_native_mega_moe:
self.experts.finalize_weights(self.shared_experts)
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
"""Pick the CUDA sparse-MLA attention class for the configured backend.
@@ -1559,6 +1419,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
)
for name, loaded_weight in weights:
if ".shared_experts." in name:
if is_pp_missing_parameter(name, self):
continue
experts = self.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if pad_shared_expert and ".shared_experts." in name:
loaded_weight = self._pad_shared_expert_weight(
self.quant_config, name, loaded_weight
@@ -1688,10 +1557,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
num_experts=self.config.n_routed_experts,
)
def finalize_mega_moe_weights(self) -> None:
for layer in islice(self.layers, self.start_layer, self.end_layer):
layer.ffn.finalize_mega_moe_weights()
def finalize_mhc_broadcast_weights(self) -> None:
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
return
@@ -1891,7 +1756,6 @@ class DeepseekV4ForCausalLM(
return loaded_params
def process_weights_after_loading(self) -> None:
self.model.finalize_mega_moe_weights()
self.model.finalize_mhc_broadcast_weights()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
+7 -8
View File
@@ -436,6 +436,13 @@ class DeepSeekV4MTP(nn.Module):
else ".weight_scale_inv"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts." in name:
experts = self.model.layers[str(spec_layer)].mtp_block.ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
@@ -529,17 +536,9 @@ class DeepSeekV4MTP(nn.Module):
f"Use a checkpoint that includes MTP layer weights, "
f"or disable speculative decoding."
)
self.process_weights_after_loading()
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
return loaded_params
def finalize_mega_moe_weights(self) -> None:
for layer in self.model.layers.values():
layer.mtp_block.ffn.finalize_mega_moe_weights()
def process_weights_after_loading(self) -> None:
self.finalize_mega_moe_weights()
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
+8 -9
View File
@@ -44,7 +44,7 @@ from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
@@ -431,6 +431,13 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
else f".{self.linear_scale_name}"
)
name = name.removesuffix(".scale") + suffix
if ".shared_experts." in name:
experts = self.model.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if ".shared_experts.w2" in name:
name = name.replace(".shared_experts.w2", ".shared_experts.down_proj")
if self.pad_shared_expert and ".shared_experts." in name:
@@ -492,17 +499,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
if self.model.confidence_head is not None and not loaded_confidence_head:
self.model.confidence_head = None
self.process_weights_after_loading()
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
def _finalize_moe(self) -> None:
for layer in self.model.layers:
layer.ffn.finalize_mega_moe_weights()
def process_weights_after_loading(self) -> None:
self._finalize_moe()
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
+9 -5
View File
@@ -1275,6 +1275,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# Vision weights are loaded by the outer multimodal wrapper.
logger.warning_once("Skipping non-text weight: %s", name)
continue
if ".shared_experts." in name:
if is_pp_missing_parameter(name, self):
continue
experts = self.layers[extract_layer_index(name)].ffn.experts
if getattr(experts, "has_fused_shared_experts", False):
loaded_params.add(
experts.load_shared_expert_weight(name, loaded_weight)
)
continue
if pad_shared_expert and ".shared_experts." in name:
loaded_weight = self._pad_shared_expert_weight(
self.quant_config, name, loaded_weight
@@ -1406,10 +1415,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
num_experts=self.config.n_routed_experts,
)
def finalize_mega_moe_weights(self) -> None:
for layer in islice(self.layers, self.start_layer, self.end_layer):
layer.ffn.finalize_mega_moe_weights()
def finalize_mhc_broadcast_weights(self) -> None:
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
return
@@ -1680,7 +1685,6 @@ class DeepseekV41LLMForCausalLM(
return loaded_params
def process_weights_after_loading(self) -> None:
self.model.finalize_mega_moe_weights()
self.model.finalize_mhc_broadcast_weights()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
+3 -69
View File
@@ -95,10 +95,7 @@ from vllm.models.common.ops.sequence_parallel import (
sp_reduce_scatter,
sp_shard,
)
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4MegaMoEExperts,
DeepseekV4MLP,
)
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
from vllm.models.kimi_k3.nvidia.latent_moe_runner import (
@@ -351,10 +348,6 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
self.activation = activation
self.activation_beta = activation_beta
self.activation_linear_beta = activation_linear_beta
self.register_buffer("_mega_l1_packed", None, persistent=False)
self.register_buffer("_mega_l1_scale", None, persistent=False)
self.register_buffer("_mega_l2_packed", None, persistent=False)
self.register_buffer("_mega_l2_scale", None, persistent=False)
def synchronize_first_launch(self) -> None:
ep_group = get_ep_group()
@@ -366,51 +359,11 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
torch.distributed.barrier(group=ep_group.cpu_group)
self._synchronized_ep_groups.add(key)
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
if self._transformed_l1_weights is not None:
return
# Weight cache IPC engine: the daemon exported the transformed
# buffers; reuse them zero-copy and drop the raw packed params.
if self._mega_l1_packed is not None:
self._transformed_l1_weights = (self._mega_l1_packed, self._mega_l1_scale)
self._transformed_l2_weights = (self._mega_l2_packed, self._mega_l2_scale)
self._drop_raw_mega_weights()
return
self._check_runtime_supported()
backend = self._ensure_backend()
self._transformed_l1_weights, self._transformed_l2_weights = (
backend.transform_weights(
w13_weight=self.w13_weight.data,
w13_weight_scale=self.w13_weight_scale.data,
w2_weight=self.w2_weight.data,
w2_weight_scale=self.w2_weight_scale.data,
num_local_experts=self.num_local_experts,
hidden_size=self.hidden_size,
intermediate_size=self.intermediate_size,
activation=self.activation,
)
)
l1_packed, l1_scale = self._transformed_l1_weights
l2_packed, l2_scale = self._transformed_l2_weights
self.register_buffer("_mega_l1_packed", l1_packed, persistent=False)
self.register_buffer("_mega_l1_scale", l1_scale, persistent=False)
self.register_buffer("_mega_l2_packed", l2_packed, persistent=False)
self.register_buffer("_mega_l2_scale", l2_scale, persistent=False)
self._drop_raw_mega_weights()
def _drop_raw_mega_weights(self) -> None:
self.w13_weight = None
self.w13_weight_scale = None
self.w2_weight = None
self.w2_weight_scale = None
def get_symm_buffer(self):
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
backend = self._ensure_backend()
backend = self.backend
group = get_ep_group().device_group
device = torch.accelerator.current_device_index()
key = (
@@ -485,7 +438,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
else None,
)
backend = self._ensure_backend()
backend = self.backend
symm_buffer = self.get_symm_buffer()
prepare_megamoe_inputs(
hidden_states,
@@ -498,9 +451,6 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
is_padding=is_padding,
hidden_quant=backend.hidden_quant,
)
self.finalize_weights()
assert self._transformed_l1_weights is not None
assert self._transformed_l2_weights is not None
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
@@ -1573,11 +1523,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
loaded_params.add(name)
return loaded_params
def finalize_mega_moe_weights(self) -> None:
for module in self.modules():
if isinstance(module, KimiMoE) and module.use_mega_moe:
module.experts.finalize_weights()
class KimiLinearForCausalLM(
nn.Module,
@@ -1700,14 +1645,6 @@ class KimiLinearForCausalLM(
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def process_weights_after_loading(self) -> None:
# A parent AutoWeightsLoader may invoke load_weights repeatedly for
# non-contiguous streamed prefixes. Finalize only after the full stream.
self.model.finalize_mega_moe_weights()
# The fused MultiHeadLatentAttention's process_weights_after_loading
# (W_UK_T / W_UV absorption) is driven by the loader's generic post-load
# hook for any AttentionLayerBase, so no manual trigger is needed here.
def get_spec_layer_idx_from_weight_name(
config: KimiLinearConfig, weight_name: str
@@ -2154,6 +2091,3 @@ class KimiK3ForConditionalGeneration(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def process_weights_after_loading(self) -> None:
self.language_model.process_weights_after_loading()
-5
View File
@@ -421,11 +421,6 @@ class KimiK3MTP(nn.Module):
f"MTP layer weights, or disable speculative decoding."
)
if use_mega_moe:
for module in self.modules():
if isinstance(module, KimiMoE) and module.use_mega_moe:
module.experts.finalize_weights()
return loaded_params
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: