Pad to 64 not 128 Q heads in flashMLA sparse (#60029)
Signed-off-by: Patryk Saffer <patryk.saffer@mistral.ai> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Misha Goin
parent
d1f3d8b870
commit
e43db1f5e2
@@ -228,9 +228,9 @@ class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
|
|||||||
-1, 1, kv_c_and_k_pe_cache.shape[-1]
|
-1, 1, kv_c_and_k_pe_cache.shape[-1]
|
||||||
)
|
)
|
||||||
|
|
||||||
# NOTE(Chen): kernel requires num_local_head to be a multiple of
|
# NOTE(Chen): kernel requires num_local_head to be a multiple of 64.
|
||||||
# 64 on hopper and 128 on blackwell. Pad from q's head count, not
|
# Pad from q's head count, not self.num_heads: under DCP the heads are
|
||||||
# self.num_heads: under DCP the heads are all-gathered before this.
|
# all-gathered before this.
|
||||||
if actual_num_heads is None:
|
if actual_num_heads is None:
|
||||||
actual_num_heads = q.shape[1]
|
actual_num_heads = q.shape[1]
|
||||||
padded_num_heads = (
|
padded_num_heads = (
|
||||||
|
|||||||
@@ -730,10 +730,8 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
|||||||
**mla_args,
|
**mla_args,
|
||||||
)
|
)
|
||||||
self.softmax_scale = scale
|
self.softmax_scale = scale
|
||||||
# Prefill BF16 kernel requires 64 on Hopper, 128 on Blackwell
|
# Prefill BF16 kernel requires 64 heads on Hopper and Blackwell.
|
||||||
self.prefill_padding = (
|
self.prefill_padding = 64
|
||||||
128 if current_platform.is_device_capability_family(100) else 64
|
|
||||||
)
|
|
||||||
self.fp8_decode_padded_heads = self._compute_fp8_decode_padded_heads(num_heads)
|
self.fp8_decode_padded_heads = self._compute_fp8_decode_padded_heads(num_heads)
|
||||||
|
|
||||||
# The kernels read queries as wide as the cache rows. A NoPE model on
|
# The kernels read queries as wide as the cache rows. A NoPE model on
|
||||||
@@ -1323,9 +1321,9 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
|||||||
-1, 1, kv_c_and_k_pe_cache.shape[-1]
|
-1, 1, kv_c_and_k_pe_cache.shape[-1]
|
||||||
)
|
)
|
||||||
|
|
||||||
# NOTE(Chen): kernel requires num_local_head to be a multiple of
|
# NOTE(Chen): kernel requires num_local_head to be a multiple of 64.
|
||||||
# 64 on hopper and 128 on blackwell. Pad from q's head count, not
|
# Pad from q's head count, not self.num_heads: under DCP the heads are
|
||||||
# self.num_heads: under DCP the heads are all-gathered before this.
|
# all-gathered before this.
|
||||||
if actual_num_heads is None:
|
if actual_num_heads is None:
|
||||||
actual_num_heads = q.shape[1]
|
actual_num_heads = q.shape[1]
|
||||||
padded_num_heads = (
|
padded_num_heads = (
|
||||||
|
|||||||
Reference in New Issue
Block a user