[Model] Fix Kimi-K3 MLA with disabled context parallelism (#50404)
Signed-off-by: Tasos Varoudis <varoudis@archtech.gr> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
parent
bb543efdfd
commit
c416f15710
@@ -301,6 +301,11 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
indexer=None,
|
||||
)
|
||||
if getattr(self.impl, "dcp_world_size", -1) < 1:
|
||||
# FlashAttention requires the cp_world_size is positive and the cp_rank
|
||||
# is non negative; manually set here if not set by caller (-1 is unset)
|
||||
self.impl.dcp_world_size = 1
|
||||
self.impl.dcp_rank = 0
|
||||
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
|
||||
Reference in New Issue
Block a user