[Bugfix][RL] Load attention sinks through weight_loader for layerwise reload (#57797)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Ronald
2026-10-14 10:17:58 +00:00
committed by GitHub
co-authored by aoshen02
parent f9b65c30c7
commit 19aba3e4d5
30 changed files with 403 additions and 476 deletions
-15
View File
@@ -26,10 +26,6 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
@@ -413,12 +409,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -474,11 +464,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if ".shared_experts.w2" in name:
name = name.replace(
".shared_experts.w2", ".shared_experts.down_proj"
-16
View File
@@ -1265,14 +1265,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
return inv_name
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1362,14 +1354,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
-18
View File
@@ -19,10 +19,6 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -384,14 +380,6 @@ class DeepSeekV4MTP(nn.Module):
# KeyError downstream rather than be silently rewritten.
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
expert_mapping = fused_moe_make_expert_params_mapping(
@@ -489,12 +477,6 @@ class DeepSeekV4MTP(nn.Module):
loaded_params.add(name_mapped)
break
continue
elif "attn_sink" in name:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if ".shared_experts.w2" in name:
name = name.replace(
+2 -4
View File
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
from vllm.models.common.ops.fused_qk_rmsnorm import (
_FUSED_Q_KV_RMSNORM_KERNEL,
fused_q_kv_rmsnorm,
@@ -240,10 +241,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads)
# Sink padded to the same head count, initialized to -inf (no sink
# effect). Weight loading fills the first n_local_heads slots.
self.attn_sink = nn.Parameter(
torch.full((self.padded_heads,), -float("inf"), dtype=torch.float32),
requires_grad=False,
)
self.attn_sink = make_attention_sink(self.padded_heads, torch.float32)
self.fused_wqa_wkv = MergedColumnParallelLinear(
self.hidden_size,
-16
View File
@@ -663,14 +663,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
return inv_name
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -755,14 +747,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
-15
View File
@@ -18,10 +18,6 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
@@ -484,12 +480,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
weights = _duplicate_context_wkv_weights(weights, len(self.model.layers))
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
@@ -565,11 +555,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
-17
View File
@@ -14,7 +14,6 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_ep_group,
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.distributed.eplb.eplb_state import EplbLayerState
@@ -1399,14 +1398,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1490,14 +1481,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
-18
View File
@@ -21,10 +21,6 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
@@ -386,14 +382,6 @@ class DeepSeekV4MTP(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
if first_layer.mtp_block.ffn.use_native_mega_moe:
@@ -497,12 +485,6 @@ class DeepSeekV4MTP(nn.Module):
loaded_params.add(name_mapped)
break
continue
elif "attn_sink" in name:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if name.endswith(".ffn.gate.bias"):
# ``e_score_correction_bias`` lives on the gate
-17
View File
@@ -15,10 +15,6 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
@@ -322,12 +318,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -383,13 +373,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
if name not in params_dict:
continue
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if ".shared_experts.w2" in name:
name = name.replace(
".shared_experts.w2", ".shared_experts.down_proj"
-16
View File
@@ -1149,14 +1149,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1219,14 +1211,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
-18
View File
@@ -19,10 +19,6 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -335,14 +331,6 @@ class DeepSeekV4MTP(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
if first_layer.mtp_block.ffn.use_mega_moe:
@@ -441,12 +429,6 @@ class DeepSeekV4MTP(nn.Module):
loaded_params.add(name_mapped)
break
continue
elif "attn_sink" in name:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if ".shared_experts.w2" in name:
name = name.replace(
-15
View File
@@ -18,10 +18,6 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.torch import mhc_post_torch
@@ -420,12 +416,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -487,11 +477,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
-17
View File
@@ -12,7 +12,6 @@ import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
@@ -985,14 +984,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
return inv_name
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1073,14 +1064,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
+2 -4
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.layers.sparse_mqa_indexer import SparseMQAIndexer
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
from vllm.models.deepseek_v41.common.ops import (
@@ -356,10 +357,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads)
# Sink padded to the same head count, initialized to -inf (no sink
# effect). Weight loading fills the first n_local_heads slots.
self.attn_sink = nn.Parameter(
torch.full((self.padded_heads,), -float("inf"), dtype=torch.float32),
requires_grad=False,
)
self.attn_sink = make_attention_sink(self.padded_heads, torch.float32)
self.fused_wqa_wkv = MergedColumnParallelLinear(
self.hidden_size,
-15
View File
@@ -19,10 +19,6 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
@@ -409,12 +405,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params: set[str] = set()
loaded_confidence_head = False
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // tp_size
head_start = n_local_head * tp_rank
head_end = n_local_head * (tp_rank + 1)
for name, loaded_weight in weights:
mapped = self._remap_dspark_name(name)
if mapped is None:
@@ -483,11 +473,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
loaded_params.add(name)
break
else:
if "attn_sink" in name:
narrow = loaded_weight[head_start:head_end]
params_dict[name][: narrow.shape[0]].copy_(narrow)
loaded_params.add(name)
continue
if name.endswith(".ffn.gate.bias"):
name = name.replace(
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
-17
View File
@@ -15,7 +15,6 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_engram_dp_size,
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
@@ -1251,14 +1250,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
n_head = self.config.num_attention_heads
n_local_head = n_head // tp_size
head_rank_start = n_local_head * tp_rank
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
@@ -1346,14 +1337,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
break
loaded_params.add(name_mapped)
continue
elif "attn_sink" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
else:
if is_pp_missing_parameter(name, self):
continue
+26 -46
View File
@@ -36,8 +36,6 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
@@ -435,12 +433,6 @@ class HYV4Model(nn.Module):
split_expert_params_mapping = self.get_expert_mapping()
loaded_params: set[str] = set()
# Sink weights are sharded like the q/k/v linears.
sink_tp_size = get_tensor_model_parallel_world_size()
sink_tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // sink_tp_size
head_rank_start = n_local_head * sink_tp_rank
head_rank_end = n_local_head * (sink_tp_rank + 1)
base_layer = (
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
)
@@ -572,48 +564,36 @@ class HYV4Model(nn.Module):
if loaded_expert_param_names:
loaded_params.update(loaded_expert_param_names)
continue
if "learnable_sink_param" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
with torch.no_grad():
params_dict[name][:n].copy_(narrow_weight)
else:
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace(
"gate.e_score_correction_bias", "expert_bias"
)
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace("gate.e_score_correction_bias", "expert_bias")
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_pp_missing_parameter(name, self):
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
if _should_skip_missing_param(name):
continue
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
if name not in params_dict:
if _should_skip_missing_param(name):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
return loaded_params
+6 -7
View File
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionBackend, AttentionType
@@ -493,13 +494,11 @@ class HYV4MLAAttention(nn.Module):
if self.learnable_sink:
sink_backend = self._resolve_sink_backend(kv_cache_dtype)
enable_sink = sink_backend is not None
self.learnable_sink_param = nn.Parameter(
torch.empty(
self.num_local_heads,
# The kernels require fp32 sinks; the disabled path keeps
# the checkpoint dtype since the value is never consumed.
dtype=torch.float32 if enable_sink else torch.bfloat16,
)
# The kernels require fp32 sinks; the disabled path keeps the
# checkpoint dtype since the value is never consumed.
self.learnable_sink_param = make_attention_sink(
self.num_local_heads,
torch.float32 if enable_sink else torch.bfloat16,
)
if enable_sink:
sinks = self.learnable_sink_param
+26 -46
View File
@@ -35,8 +35,6 @@ from transformers import PreTrainedConfig
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
@@ -433,12 +431,6 @@ class HYV4Model(nn.Module):
split_expert_params_mapping = self.get_expert_mapping()
loaded_params: set[str] = set()
# Sink weights are sharded like the q/k/v linears.
sink_tp_size = get_tensor_model_parallel_world_size()
sink_tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // sink_tp_size
head_rank_start = n_local_head * sink_tp_rank
head_rank_end = n_local_head * (sink_tp_rank + 1)
base_layer = (
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
)
@@ -570,48 +562,36 @@ class HYV4Model(nn.Module):
if loaded_expert_param_names:
loaded_params.update(loaded_expert_param_names)
continue
if "learnable_sink_param" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
with torch.no_grad():
params_dict[name][:n].copy_(narrow_weight)
else:
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace(
"gate.e_score_correction_bias", "expert_bias"
)
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace("gate.e_score_correction_bias", "expert_bias")
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_pp_missing_parameter(name, self):
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
if _should_skip_missing_param(name):
continue
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
if name not in params_dict:
if _should_skip_missing_param(name):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
return loaded_params
-18
View File
@@ -11,10 +11,6 @@ from torch import nn
from transformers import PreTrainedConfig
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -744,11 +740,6 @@ class HYV4MTP(nn.Module):
}
num_experts = getattr(self.config, "n_routed_experts", 0)
sink_tp_size = get_tensor_model_parallel_world_size()
sink_tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // sink_tp_size
head_rank_start = n_local_head * sink_tp_rank
head_rank_end = n_local_head * (sink_tp_rank + 1)
# Routed-expert weights come in two checkpoint layouts:
# split: mlp.experts.<id>.gate_proj.weight
@@ -883,15 +874,6 @@ class HYV4MTP(nn.Module):
if is_loaded:
continue
if "learnable_sink_param" in name:
if name in params_dict:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
with torch.no_grad():
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue