[Bugfix][ROCm][MLA] Pass q/kv dtypes to get_mla_metadata_v1 in FP8 decode (#46997)
Signed-off-by: pei.zhang <pei.zhang@amd.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -199,6 +199,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
else:
|
||||
kv_cache_dtype_str = "bf16"
|
||||
kv_dtype = dtypes.d_dtypes.get(kv_cache_dtype_str, dtypes.bf16)
|
||||
# Persist for get_mla_metadata_v1 (decode build): omitting these causes
|
||||
# wrong split/reduce metadata for the gfx950 fp8 nhead=32 fold path.
|
||||
self._mla_q_dtype = q_dtype
|
||||
self._mla_kv_dtype = kv_dtype
|
||||
(
|
||||
(work_meta_data_size, work_meta_data_type),
|
||||
(work_indptr_size, work_indptr_type),
|
||||
@@ -534,6 +538,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
max_seqlen_qo=max_qo_len,
|
||||
uni_seqlen_qo=max_qo_len,
|
||||
fast_mode=True,
|
||||
dtype_q=self._mla_q_dtype,
|
||||
dtype_kv=self._mla_kv_dtype,
|
||||
)
|
||||
has_persistent_metadata = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user