Revert "[Models] Fuse Qwen3.5 GDN's qkvz_proj and ba_proj" (#34683)

This commit is contained in:
Jiangyun Zhu
2026-02-17 01:29:27 -08:00
committed by GitHub
parent c464b57374
commit 1d65283e95
3 changed files with 182 additions and 87 deletions
+6 -28
View File
@@ -685,13 +685,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
self,
param: Parameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
loaded_shard_id: int | None = None,
):
if isinstance(loaded_shard_id, tuple):
raise NotImplementedError(
"Shard id with multiple indices is not supported in weight_loader, "
"please use weight_loader_v2 instead."
)
# Special case for GGUF
# initialize GGUF param after we know the quantize type
is_gguf_weight = getattr(param, "is_gguf_weight", False)
@@ -830,10 +825,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
param_data.copy_(loaded_weight)
def _load_fused_module_from_checkpoint(
self,
param: BasevLLMParameter,
loaded_weight: torch.Tensor,
output_sizes: list[int] | None = None,
self, param: BasevLLMParameter, loaded_weight: torch.Tensor
):
"""
Handle special case for models where MLP layers are already
@@ -847,8 +839,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
current_shard_offset = 0
shard_offsets: list[tuple[int, int, int]] = []
output_sizes = output_sizes or self.output_sizes
for i, output_size in enumerate(output_sizes):
for i, output_size in enumerate(self.output_sizes):
shard_offsets.append((i, current_shard_offset, output_size))
current_shard_offset += output_size
@@ -873,30 +864,17 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
self,
param: BasevLLMParameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
loaded_shard_id: int | None = None,
):
if loaded_shard_id is None or isinstance(loaded_shard_id, tuple):
if loaded_shard_id is None:
if isinstance(param, PerTensorScaleParameter):
param.load_merged_column_weight(loaded_weight=loaded_weight, shard_id=0)
return
elif type(param) in (RowvLLMParameter, BasevLLMParameter):
param.load_merged_column_weight(loaded_weight=loaded_weight)
return
output_sizes = (
[self.output_sizes[idx] for idx in loaded_shard_id]
if loaded_shard_id
else None
)
if isinstance(param, BlockQuantScaleParameter):
weight_block_size = getattr(self, "weight_block_size", None)
output_sizes = [
adjust_block_scale_shard(weight_block_size, size, 0)[0]
for size in (output_sizes or self.output_sizes)
]
# TODO: @dsikka - move to parameter.py
self._load_fused_module_from_checkpoint(
param, loaded_weight, output_sizes=output_sizes
)
self._load_fused_module_from_checkpoint(param, loaded_weight)
return
assert loaded_shard_id < len(self.output_sizes)