[CI] Fix Transformers modeling backend LoRA test (#47832)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -258,7 +258,8 @@ def _apply_qkv_fuser_with_stubs(module: nn.Module, fuser: QKVFuser):
|
|||||||
merged.weight.copy_(torch.cat([q.weight, k.weight, v.weight], dim=0))
|
merged.weight.copy_(torch.cat([q.weight, k.weight, v.weight], dim=0))
|
||||||
if q.bias is not None:
|
if q.bias is not None:
|
||||||
merged.bias.copy_(torch.cat([q.bias, k.bias, v.bias], dim=0))
|
merged.bias.copy_(torch.cat([q.bias, k.bias, v.bias], dim=0))
|
||||||
merged.split_sizes = [q.out_features, k.out_features, v.out_features]
|
merged.output_sizes = [q.out_features, k.out_features, v.out_features]
|
||||||
|
merged.tp_size = 1
|
||||||
setattr(module, fuser.merged_name, merged)
|
setattr(module, fuser.merged_name, merged)
|
||||||
for name in (fuser.q_name, fuser.k_name, fuser.v_name):
|
for name in (fuser.q_name, fuser.k_name, fuser.v_name):
|
||||||
delattr(module, name)
|
delattr(module, name)
|
||||||
@@ -332,7 +333,8 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
|
|||||||
# original semantics (branches, kwargs, attribute reads)
|
# original semantics (branches, kwargs, attribute reads)
|
||||||
code = fuser.fused_forward.__code__
|
code = fuser.fused_forward.__code__
|
||||||
names = code.co_names
|
names = code.co_names
|
||||||
assert "qkv_proj" in names and "split_sizes" in names and "o_proj" in names
|
assert "qkv_proj" in names and "output_sizes" in names and "o_proj" in names
|
||||||
|
assert "tp_size" in names
|
||||||
assert not {"q_proj", "k_proj", "v_proj"} & set(names)
|
assert not {"q_proj", "k_proj", "v_proj"} & set(names)
|
||||||
if attn_cls is FakeAttention:
|
if attn_cls is FakeAttention:
|
||||||
assert "update" in names # the cache branch survives
|
assert "update" in names # the cache branch survives
|
||||||
|
|||||||
@@ -1000,16 +1000,12 @@ class QKVParallelLinear(ColumnParallelLinear):
|
|||||||
self.num_kv_heads = divide(self.total_num_kv_heads, tp_size)
|
self.num_kv_heads = divide(self.total_num_kv_heads, tp_size)
|
||||||
self.num_kv_head_replicas = 1
|
self.num_kv_head_replicas = 1
|
||||||
input_size = self.hidden_size
|
input_size = self.hidden_size
|
||||||
output_size = (
|
|
||||||
self.num_heads * self.head_size
|
|
||||||
+ self.num_kv_heads * self.head_size
|
|
||||||
+ self.num_kv_heads * self.v_head_size
|
|
||||||
) * tp_size
|
|
||||||
self.output_sizes = [
|
self.output_sizes = [
|
||||||
self.num_heads * self.head_size * tp_size, # q_proj
|
self.num_heads * self.head_size * tp_size, # q_proj
|
||||||
self.num_kv_heads * self.head_size * tp_size, # k_proj
|
self.num_kv_heads * self.head_size * tp_size, # k_proj
|
||||||
self.num_kv_heads * self.v_head_size * tp_size, # v_proj
|
self.num_kv_heads * self.v_head_size * tp_size, # v_proj
|
||||||
]
|
]
|
||||||
|
output_size = sum(self.output_sizes)
|
||||||
|
|
||||||
super().__init__(
|
super().__init__(
|
||||||
input_size=input_size,
|
input_size=input_size,
|
||||||
|
|||||||
@@ -127,15 +127,14 @@ class QKVFuser(StackedFuser):
|
|||||||
if len({id(block) for block, _ in blocks}) != 1:
|
if len({id(block) for block, _ in blocks}) != 1:
|
||||||
raise ValueError("projection calls are in different blocks")
|
raise ValueError("projection calls are in different blocks")
|
||||||
|
|
||||||
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(self.qkv.split_sizes, -1)
|
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1)
|
||||||
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
|
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
|
||||||
temps = [f"{name}_fused" for name in (self.q_name, self.k_name, self.v_name)]
|
temps = [f"{name}_fused" for name in (self.q_name, self.k_name, self.v_name)]
|
||||||
if names & set(temps):
|
if names & set(temps):
|
||||||
raise ValueError("fused temporaries would shadow existing names")
|
raise ValueError("fused temporaries would shadow existing names")
|
||||||
merged = f"self.{self.merged_name}"
|
merged = f"self.{self.merged_name}"
|
||||||
template = (
|
sections = f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
|
||||||
f"{', '.join(temps)} = {merged}(__arg__).split({merged}.split_sizes, -1)"
|
template = f"{', '.join(temps)} = {merged}(__arg__).split({sections}, -1)"
|
||||||
)
|
|
||||||
assign = ast.parse(template).body[0]
|
assign = ast.parse(template).body[0]
|
||||||
arg = next(
|
arg = next(
|
||||||
node
|
node
|
||||||
@@ -198,13 +197,6 @@ class QKVFuser(StackedFuser):
|
|||||||
self.merged_name,
|
self.merged_name,
|
||||||
merged,
|
merged,
|
||||||
)
|
)
|
||||||
# The rewritten forward splits the merged projection into this rank's
|
|
||||||
# shard sizes (see `update_forward`)
|
|
||||||
merged.split_sizes = [
|
|
||||||
merged.num_heads * merged.head_size,
|
|
||||||
merged.num_kv_heads * merged.head_size,
|
|
||||||
merged.num_kv_heads * merged.v_head_size,
|
|
||||||
]
|
|
||||||
setattr(module, self.merged_name, merged)
|
setattr(module, self.merged_name, merged)
|
||||||
# Drop the consumed submodules so their (meta) params are not expected.
|
# Drop the consumed submodules so their (meta) params are not expected.
|
||||||
for name in (self.q_name, self.k_name, self.v_name):
|
for name in (self.q_name, self.k_name, self.v_name):
|
||||||
|
|||||||
Reference in New Issue
Block a user