diff --git a/tests/models/transformers/fusers/test_linear.py b/tests/models/transformers/fusers/test_linear.py index 3ead599234..546dafb792 100644 --- a/tests/models/transformers/fusers/test_linear.py +++ b/tests/models/transformers/fusers/test_linear.py @@ -258,7 +258,8 @@ def _apply_qkv_fuser_with_stubs(module: nn.Module, fuser: QKVFuser): merged.weight.copy_(torch.cat([q.weight, k.weight, v.weight], dim=0)) if q.bias is not None: merged.bias.copy_(torch.cat([q.bias, k.bias, v.bias], dim=0)) - merged.split_sizes = [q.out_features, k.out_features, v.out_features] + merged.output_sizes = [q.out_features, k.out_features, v.out_features] + merged.tp_size = 1 setattr(module, fuser.merged_name, merged) for name in (fuser.q_name, fuser.k_name, fuser.v_name): delattr(module, name) @@ -332,7 +333,8 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads): # original semantics (branches, kwargs, attribute reads) code = fuser.fused_forward.__code__ names = code.co_names - assert "qkv_proj" in names and "split_sizes" in names and "o_proj" in names + assert "qkv_proj" in names and "output_sizes" in names and "o_proj" in names + assert "tp_size" in names assert not {"q_proj", "k_proj", "v_proj"} & set(names) if attn_cls is FakeAttention: assert "update" in names # the cache branch survives diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py index e487b91e98..d92b9fc7d0 100644 --- a/vllm/model_executor/layers/linear.py +++ b/vllm/model_executor/layers/linear.py @@ -1000,16 +1000,12 @@ class QKVParallelLinear(ColumnParallelLinear): self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) self.num_kv_head_replicas = 1 input_size = self.hidden_size - output_size = ( - self.num_heads * self.head_size - + self.num_kv_heads * self.head_size - + self.num_kv_heads * self.v_head_size - ) * tp_size self.output_sizes = [ self.num_heads * self.head_size * tp_size, # q_proj self.num_kv_heads * self.head_size * tp_size, # k_proj self.num_kv_heads * self.v_head_size * tp_size, # v_proj ] + output_size = sum(self.output_sizes) super().__init__( input_size=input_size, diff --git a/vllm/model_executor/models/transformers/fusers/qkv.py b/vllm/model_executor/models/transformers/fusers/qkv.py index 03f5a0b49e..010a0018ac 100644 --- a/vllm/model_executor/models/transformers/fusers/qkv.py +++ b/vllm/model_executor/models/transformers/fusers/qkv.py @@ -127,15 +127,14 @@ class QKVFuser(StackedFuser): if len({id(block) for block, _ in blocks}) != 1: raise ValueError("projection calls are in different blocks") - # q(x), k(x), v(x) -> q, k, v = qkv(x).split(self.qkv.split_sizes, -1) + # q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1) names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)} temps = [f"{name}_fused" for name in (self.q_name, self.k_name, self.v_name)] if names & set(temps): raise ValueError("fused temporaries would shadow existing names") merged = f"self.{self.merged_name}" - template = ( - f"{', '.join(temps)} = {merged}(__arg__).split({merged}.split_sizes, -1)" - ) + sections = f"[s // {merged}.tp_size for s in {merged}.output_sizes]" + template = f"{', '.join(temps)} = {merged}(__arg__).split({sections}, -1)" assign = ast.parse(template).body[0] arg = next( node @@ -198,13 +197,6 @@ class QKVFuser(StackedFuser): self.merged_name, merged, ) - # The rewritten forward splits the merged projection into this rank's - # shard sizes (see `update_forward`) - merged.split_sizes = [ - merged.num_heads * merged.head_size, - merged.num_kv_heads * merged.head_size, - merged.num_kv_heads * merged.v_head_size, - ] setattr(module, self.merged_name, merged) # Drop the consumed submodules so their (meta) params are not expected. for name in (self.q_name, self.k_name, self.v_name):