[Bugfix] Fuse Qwen3.5 in_qkvz_proj forwarding with LoRA enabled (#37912)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -195,9 +195,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
# There are two LoRA layers
|
||||
# the output_sizes in MergedColumnParallelLinear is not sharded by tp
|
||||
# we need to divide it by the tp_size to get correct slices size
|
||||
output_sizes = self.base_layer.output_sizes
|
||||
self.output_sizes = self.base_layer.output_sizes
|
||||
self.output_slices = tuple(
|
||||
divide(output_size, self.tp_size) for output_size in output_sizes
|
||||
divide(output_size, self.tp_size) for output_size in self.output_sizes
|
||||
)
|
||||
self.n_slices = len(self.output_slices)
|
||||
self.output_ids = (self.tp_rank,) * self.n_slices
|
||||
@@ -261,6 +261,42 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
]
|
||||
return sliced_lora_b
|
||||
|
||||
def expand_packed_lora(
|
||||
self,
|
||||
lora_a: list[torch.Tensor],
|
||||
lora_b: list[torch.Tensor],
|
||||
) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
|
||||
"""
|
||||
Expand packed adapter groups when they don't match n_slices.
|
||||
E.g. in_proj_qkv (covers Q+K+V) + in_proj_z
|
||||
"""
|
||||
expanded_a: list[torch.Tensor] = []
|
||||
expanded_b: list[torch.Tensor] = []
|
||||
start_idx = 0
|
||||
for a_i, b_i in zip(lora_a, lora_b):
|
||||
# Determine which output slices this b_i covers.
|
||||
b_rows, cu_rows, covered = b_i.shape[0], 0, 0
|
||||
for i in range(start_idx, self.n_slices):
|
||||
cu_rows += self.output_sizes[i]
|
||||
if cu_rows == b_rows:
|
||||
covered = i - start_idx + 1
|
||||
break
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Cannot determine how to split lora_b with {b_rows} rows "
|
||||
f"into {self.n_slices} slices with output sizes "
|
||||
f"{self.output_sizes} starting from index {start_idx}."
|
||||
)
|
||||
# Split b_i into per-slice tensors and replicate a_i for each.
|
||||
start = 0
|
||||
for j in range(covered):
|
||||
size = self.output_sizes[start_idx + j]
|
||||
expanded_b.append(b_i[start : start + size, :])
|
||||
expanded_a.append(a_i)
|
||||
start += size
|
||||
start_idx += covered
|
||||
return expanded_a, expanded_b
|
||||
|
||||
def set_lora(
|
||||
self,
|
||||
index: int,
|
||||
@@ -269,6 +305,12 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
):
|
||||
self.reset_lora(index)
|
||||
|
||||
# Expand packed adapter groups when they don't match n_slices.
|
||||
# E.g. in_proj_qkv (covers Q+K+V) + in_proj_z as 2 groups for a
|
||||
# 4-slice layer: split b_qkv by output_sizes and replicate a_qkv.
|
||||
if isinstance(lora_b, list) and len(lora_b) != self.n_slices:
|
||||
lora_a, lora_b = self.expand_packed_lora(lora_a, lora_b)
|
||||
|
||||
if self.tp_size > 1:
|
||||
lora_a = self.slice_lora_a(lora_a)
|
||||
lora_b = self.slice_lora_b(lora_b)
|
||||
@@ -497,18 +539,14 @@ class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLo
|
||||
def slice_lora_a(
|
||||
self, lora_a: list[torch.Tensor | None]
|
||||
) -> list[torch.Tensor | None]:
|
||||
# NOTE: lora_a contains 2 subloras, and each sublora could be None.
|
||||
output_shard_size = self.lora_a_stacked[0].shape[2]
|
||||
output_start_idx = self.tp_rank * output_shard_size
|
||||
lora_a = [
|
||||
lora_a[0][output_start_idx : output_start_idx + output_shard_size, :]
|
||||
if lora_a[0] is not None
|
||||
else None,
|
||||
lora_a[1][output_start_idx : output_start_idx + output_shard_size, :]
|
||||
if lora_a[1] is not None
|
||||
else None,
|
||||
return [
|
||||
lora_a_i[output_start_idx : output_start_idx + output_shard_size, :]
|
||||
if (lora_a_i := lora_a[i]) is not None
|
||||
else None
|
||||
for i in range(len(lora_a))
|
||||
]
|
||||
return lora_a
|
||||
|
||||
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
|
||||
return _mcp_apply(x, bias, self)
|
||||
|
||||
@@ -563,11 +563,16 @@ class LoRAModelManager:
|
||||
else:
|
||||
parts = module_name.split(".")
|
||||
replacements = self.packed_modules_mapping[parts[-1]]
|
||||
n_slices = getattr(module, "n_slices", len(replacements))
|
||||
if module.__class__.__name__ == "FusedMoEWithLoRA":
|
||||
replacements = replacements[
|
||||
: len(module.lora_a_stacked) // self.lora_slots
|
||||
]
|
||||
subloras: list[LoRALayerWeights | None] = []
|
||||
# HACK: overrides replacements for qkvz = qkv + z case.
|
||||
# Any better methods to handle this case?
|
||||
if n_slices != len(replacements):
|
||||
replacements = [f"slice_{i}" for i in range(n_slices)]
|
||||
for i, r in enumerate(replacements):
|
||||
lora = LoRALayerWeights.create_dummy_lora_weights(
|
||||
module_name + "." + r,
|
||||
|
||||
@@ -263,7 +263,6 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
config: Qwen3NextConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
create_in_proj_qkvz: bool = True,
|
||||
gqa_interleaved_layout=False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -323,32 +322,14 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
# we need to create qkvz_proj adaptively here.
|
||||
# When create_in_proj_qkvz is False (e.g. LoRA enabled in Qwen3.5),
|
||||
# in_proj_qkv and in_proj_z are created separately instead.
|
||||
self.has_lora_projections = not create_in_proj_qkvz
|
||||
if create_in_proj_qkvz:
|
||||
self.in_proj_qkvz = self.create_qkvz_proj(
|
||||
hidden_size=self.hidden_size,
|
||||
key_dim=self.key_dim,
|
||||
value_dim=self.value_dim,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvz",
|
||||
)
|
||||
else:
|
||||
# LoRA case (Qwen3.5 only): keep q/k/v and z as separate modules
|
||||
# so that LoRA adapters can be applied independently.
|
||||
self.in_proj_qkv = MergedColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_sizes=[self.key_dim, self.key_dim, self.value_dim],
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkv",
|
||||
)
|
||||
self.in_proj_z = ColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_size=self.value_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.in_proj_z",
|
||||
)
|
||||
self.in_proj_qkvz = self.create_qkvz_proj(
|
||||
hidden_size=self.hidden_size,
|
||||
key_dim=self.key_dim,
|
||||
value_dim=self.value_dim,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvz",
|
||||
)
|
||||
|
||||
# ba_proj doesn't support blockwise fp8 quantization.
|
||||
# Qwen3-Next and Qwen3.5 have different in_proj_ba checkpoint
|
||||
# layouts, so we use a factory method to create the projection.
|
||||
@@ -707,7 +688,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
):
|
||||
"""ROCm forward using AITER Triton fused projection+attention when
|
||||
available, otherwise falling back to the generic CUDA path."""
|
||||
if not self.has_lora_projections and GDN_AITER_TRITON_AVAILABLE:
|
||||
if GDN_AITER_TRITON_AVAILABLE:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
projected_states_ba, _ = self.in_proj_ba(hidden_states)
|
||||
@@ -752,37 +733,27 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
# ============================================================
|
||||
# Part 1: Input Projection
|
||||
# ============================================================
|
||||
if self.has_lora_projections:
|
||||
# LoRA path (Qwen3.5 only): separate in_proj_qkv and in_proj_z
|
||||
mixed_qkv, _ = self.in_proj_qkv(hidden_states)
|
||||
ba, _ = self.in_proj_ba(hidden_states)
|
||||
z, _ = self.in_proj_z(hidden_states)
|
||||
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
ba, _ = self.in_proj_ba(hidden_states)
|
||||
|
||||
if self.gqa_interleaved_layout:
|
||||
# Qwen3-Next: unpack the interleaved GQA layout
|
||||
query, key, value, z, b, a = self.fix_query_key_value_ordering(
|
||||
mixed_qkvz, ba
|
||||
)
|
||||
query, key, value = map(
|
||||
lambda x: rearrange(x, "l p d -> l (p d)"), (query, key, value)
|
||||
)
|
||||
mixed_qkv = torch.cat((query, key, value), dim=-1)
|
||||
else:
|
||||
# Qwen3.5: weights are already in [q, k, v, z] and [b, a] order
|
||||
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
||||
z_size = self.value_dim // self.tp_size
|
||||
mixed_qkv, z = mixed_qkvz.split([qkv_size, z_size], dim=-1)
|
||||
z = z.reshape(z.size(0), -1, self.head_v_dim)
|
||||
b, a = ba.chunk(2, dim=-1)
|
||||
b = b.contiguous()
|
||||
a = a.contiguous()
|
||||
else:
|
||||
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
ba, _ = self.in_proj_ba(hidden_states)
|
||||
|
||||
if self.gqa_interleaved_layout:
|
||||
# Qwen3-Next: unpack the interleaved GQA layout
|
||||
query, key, value, z, b, a = self.fix_query_key_value_ordering(
|
||||
mixed_qkvz, ba
|
||||
)
|
||||
query, key, value = map(
|
||||
lambda x: rearrange(x, "l p d -> l (p d)"), (query, key, value)
|
||||
)
|
||||
mixed_qkv = torch.cat((query, key, value), dim=-1)
|
||||
else:
|
||||
# Qwen3.5: weights are already in [q, k, v, z] and [b, a] order
|
||||
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
||||
z_size = self.value_dim // self.tp_size
|
||||
mixed_qkv, z = mixed_qkvz.split([qkv_size, z_size], dim=-1)
|
||||
z = z.reshape(z.size(0), -1, self.head_v_dim)
|
||||
b, a = ba.chunk(2, dim=-1)
|
||||
b = b.contiguous()
|
||||
a = a.contiguous()
|
||||
|
||||
# ============================================================
|
||||
# Part 2: Core Attention (Custom Op)
|
||||
@@ -822,8 +793,6 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
"""
|
||||
num_tokens = hidden_states.size(0)
|
||||
|
||||
assert not self.has_lora_projections, "lora isn't supported on XPU."
|
||||
|
||||
# ============================================================
|
||||
# Part 1: Input Projection
|
||||
# ============================================================
|
||||
|
||||
@@ -138,7 +138,6 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
create_in_proj_qkvz=vllm_config.lora_config is None,
|
||||
)
|
||||
elif self.layer_type == "full_attention":
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
@@ -217,7 +216,6 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
self.num_redundant_experts = eplb_config.num_redundant_experts
|
||||
|
||||
self.config = config
|
||||
self.enable_lora = vllm_config.lora_config is not None
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
@@ -276,6 +274,9 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
# GDN
|
||||
("in_proj_qkvz", "in_proj_qkv", (0, 1, 2)),
|
||||
("in_proj_qkvz", "in_proj_z", 3),
|
||||
# self attention
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
@@ -287,21 +288,6 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
("in_proj_ba", "in_proj_a", 1),
|
||||
]
|
||||
|
||||
if self.enable_lora:
|
||||
stacked_params_mapping.extend(
|
||||
[
|
||||
("in_proj_qkv", "in_proj_qkv", (0, 1, 2)),
|
||||
("in_proj_z", "in_proj_z", 0),
|
||||
]
|
||||
)
|
||||
else:
|
||||
stacked_params_mapping.extend(
|
||||
[
|
||||
("in_proj_qkvz", "in_proj_qkv", (0, 1, 2)),
|
||||
("in_proj_qkvz", "in_proj_z", 3),
|
||||
]
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
@@ -352,10 +338,7 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
if param_name == "in_proj_z" and self.enable_lora:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
@@ -485,15 +468,6 @@ class Qwen3_5ForCausalLMBase(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
# When LoRA is enabled, GDN uses separate in_proj_qkv and in_proj_z
|
||||
# instead of merged in_proj_qkvz; pack mapping must match.
|
||||
if vllm_config.lora_config:
|
||||
base = getattr(Qwen3_5ForCausalLMBase, "packed_modules_mapping", {})
|
||||
self.packed_modules_mapping = {k: list(v) for k, v in base.items()}
|
||||
self.packed_modules_mapping.pop("in_proj_qkvz", None)
|
||||
self.packed_modules_mapping["in_proj_qkv"] = ["in_proj_qkv"]
|
||||
self.packed_modules_mapping["in_proj_z"] = ["in_proj_z"]
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head = self.model.embed_tokens
|
||||
@@ -586,7 +560,6 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
||||
# protocols have not __init__ method, so we need to use nn.Module.__init__
|
||||
nn.Module.__init__(self)
|
||||
self.update_packed_mapping(enable_lora=vllm_config.lora_config is not None)
|
||||
config: Qwen3_5Config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
@@ -614,17 +587,6 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def update_packed_mapping(self, enable_lora: bool):
|
||||
# When LoRA is enabled, GDN uses separate in_proj_qkv and in_proj_z
|
||||
if enable_lora:
|
||||
base = getattr(
|
||||
Qwen3_5ForConditionalGeneration, "packed_modules_mapping", {}
|
||||
)
|
||||
self.packed_modules_mapping = {k: list(v) for k, v in base.items()}
|
||||
self.packed_modules_mapping.pop("in_proj_qkvz", None)
|
||||
self.packed_modules_mapping["in_proj_qkv"] = ["in_proj_qkv"]
|
||||
self.packed_modules_mapping["in_proj_z"] = ["in_proj_z"]
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -811,7 +773,6 @@ class Qwen3_5MoeForConditionalGeneration(
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
||||
# protocols have not __init__ method, so we need to use nn.Module.__init__
|
||||
nn.Module.__init__(self)
|
||||
self.update_packed_mapping(enable_lora=vllm_config.lora_config is not None)
|
||||
config: Qwen3_5MoeConfig = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
|
||||
Reference in New Issue
Block a user