[Bugfix][RL] Load attention sinks through weight_loader for layerwise reload (#57797)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -26,10 +26,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
@@ -413,12 +409,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -474,11 +464,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
|
||||
@@ -1265,14 +1265,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1362,14 +1354,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -19,10 +19,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -384,14 +380,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
# KeyError downstream rather than be silently rewritten.
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
@@ -489,12 +477,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
|
||||
from vllm.models.common.ops.fused_qk_rmsnorm import (
|
||||
_FUSED_Q_KV_RMSNORM_KERNEL,
|
||||
fused_q_kv_rmsnorm,
|
||||
@@ -240,10 +241,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads)
|
||||
# Sink padded to the same head count, initialized to -inf (no sink
|
||||
# effect). Weight loading fills the first n_local_heads slots.
|
||||
self.attn_sink = nn.Parameter(
|
||||
torch.full((self.padded_heads,), -float("inf"), dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.attn_sink = make_attention_sink(self.padded_heads, torch.float32)
|
||||
|
||||
self.fused_wqa_wkv = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
|
||||
@@ -663,14 +663,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -755,14 +747,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -18,10 +18,6 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
@@ -484,12 +480,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
weights = _duplicate_context_wkv_weights(weights, len(self.model.layers))
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
@@ -565,11 +555,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
|
||||
@@ -14,7 +14,6 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
@@ -1399,14 +1398,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1490,14 +1481,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -21,10 +21,6 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
@@ -386,14 +382,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
if first_layer.mtp_block.ffn.use_native_mega_moe:
|
||||
@@ -497,12 +485,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
# ``e_score_correction_bias`` lives on the gate
|
||||
|
||||
@@ -15,10 +15,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
@@ -322,12 +318,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -383,13 +373,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
if name not in params_dict:
|
||||
continue
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
|
||||
@@ -1149,14 +1149,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1219,14 +1211,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -19,10 +19,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -335,14 +331,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
if first_layer.mtp_block.ffn.use_mega_moe:
|
||||
@@ -441,12 +429,6 @@ class DeepSeekV4MTP(nn.Module):
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
|
||||
@@ -18,10 +18,6 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.torch import mhc_post_torch
|
||||
@@ -420,12 +416,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -487,11 +477,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
|
||||
@@ -12,7 +12,6 @@ import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
@@ -985,14 +984,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1073,14 +1064,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.layers.sparse_mqa_indexer import SparseMQAIndexer
|
||||
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
@@ -356,10 +357,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads)
|
||||
# Sink padded to the same head count, initialized to -inf (no sink
|
||||
# effect). Weight loading fills the first n_local_heads slots.
|
||||
self.attn_sink = nn.Parameter(
|
||||
torch.full((self.padded_heads,), -float("inf"), dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.attn_sink = make_attention_sink(self.padded_heads, torch.float32)
|
||||
|
||||
self.fused_wqa_wkv = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
|
||||
@@ -19,10 +19,6 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
@@ -409,12 +405,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
loaded_confidence_head = False
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // tp_size
|
||||
head_start = n_local_head * tp_rank
|
||||
head_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._remap_dspark_name(name)
|
||||
if mapped is None:
|
||||
@@ -483,11 +473,6 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if "attn_sink" in name:
|
||||
narrow = loaded_weight[head_start:head_end]
|
||||
params_dict[name][: narrow.shape[0]].copy_(narrow)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(
|
||||
".ffn.gate.bias", ".ffn.gate.e_score_correction_bias"
|
||||
|
||||
@@ -15,7 +15,6 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_engram_dp_size,
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
@@ -1251,14 +1250,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
@@ -1346,14 +1337,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
break
|
||||
loaded_params.add(name_mapped)
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -36,8 +36,6 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
@@ -435,12 +433,6 @@ class HYV4Model(nn.Module):
|
||||
split_expert_params_mapping = self.get_expert_mapping()
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# Sink weights are sharded like the q/k/v linears.
|
||||
sink_tp_size = get_tensor_model_parallel_world_size()
|
||||
sink_tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // sink_tp_size
|
||||
head_rank_start = n_local_head * sink_tp_rank
|
||||
head_rank_end = n_local_head * (sink_tp_rank + 1)
|
||||
base_layer = (
|
||||
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
||||
)
|
||||
@@ -572,48 +564,36 @@ class HYV4Model(nn.Module):
|
||||
if loaded_expert_param_names:
|
||||
loaded_params.update(loaded_expert_param_names)
|
||||
continue
|
||||
if "learnable_sink_param" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
with torch.no_grad():
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace(
|
||||
"gate.e_score_correction_bias", "expert_bias"
|
||||
)
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace("gate.e_score_correction_bias", "expert_bias")
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionType
|
||||
@@ -493,13 +494,11 @@ class HYV4MLAAttention(nn.Module):
|
||||
if self.learnable_sink:
|
||||
sink_backend = self._resolve_sink_backend(kv_cache_dtype)
|
||||
enable_sink = sink_backend is not None
|
||||
self.learnable_sink_param = nn.Parameter(
|
||||
torch.empty(
|
||||
self.num_local_heads,
|
||||
# The kernels require fp32 sinks; the disabled path keeps
|
||||
# the checkpoint dtype since the value is never consumed.
|
||||
dtype=torch.float32 if enable_sink else torch.bfloat16,
|
||||
)
|
||||
# The kernels require fp32 sinks; the disabled path keeps the
|
||||
# checkpoint dtype since the value is never consumed.
|
||||
self.learnable_sink_param = make_attention_sink(
|
||||
self.num_local_heads,
|
||||
torch.float32 if enable_sink else torch.bfloat16,
|
||||
)
|
||||
if enable_sink:
|
||||
sinks = self.learnable_sink_param
|
||||
|
||||
@@ -35,8 +35,6 @@ from transformers import PreTrainedConfig
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
@@ -433,12 +431,6 @@ class HYV4Model(nn.Module):
|
||||
split_expert_params_mapping = self.get_expert_mapping()
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# Sink weights are sharded like the q/k/v linears.
|
||||
sink_tp_size = get_tensor_model_parallel_world_size()
|
||||
sink_tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // sink_tp_size
|
||||
head_rank_start = n_local_head * sink_tp_rank
|
||||
head_rank_end = n_local_head * (sink_tp_rank + 1)
|
||||
base_layer = (
|
||||
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
||||
)
|
||||
@@ -570,48 +562,36 @@ class HYV4Model(nn.Module):
|
||||
if loaded_expert_param_names:
|
||||
loaded_params.update(loaded_expert_param_names)
|
||||
continue
|
||||
if "learnable_sink_param" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
with torch.no_grad():
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace(
|
||||
"gate.e_score_correction_bias", "expert_bias"
|
||||
)
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace("gate.e_score_correction_bias", "expert_bias")
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
@@ -11,10 +11,6 @@ from torch import nn
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -744,11 +740,6 @@ class HYV4MTP(nn.Module):
|
||||
}
|
||||
|
||||
num_experts = getattr(self.config, "n_routed_experts", 0)
|
||||
sink_tp_size = get_tensor_model_parallel_world_size()
|
||||
sink_tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // sink_tp_size
|
||||
head_rank_start = n_local_head * sink_tp_rank
|
||||
head_rank_end = n_local_head * (sink_tp_rank + 1)
|
||||
|
||||
# Routed-expert weights come in two checkpoint layouts:
|
||||
# split: mlp.experts.<id>.gate_proj.weight
|
||||
@@ -883,15 +874,6 @@ class HYV4MTP(nn.Module):
|
||||
if is_loaded:
|
||||
continue
|
||||
|
||||
if "learnable_sink_param" in name:
|
||||
if name in params_dict:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
with torch.no_grad():
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user