[ROCm][Bugfix] Fix AITER preshuffled FP8 block-scale kernel (#56433)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
@@ -564,10 +564,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
|
||||
self.is_amx_bmm_enabled = getattr(self.impl, "uses_amx_bmm", False)
|
||||
# AMX reads kv_b_proj's weight directly and never calls it live; the
|
||||
# reference CPU MLA backend calls it but isn't perf-critical. Skip
|
||||
# the packed-kernel dispatch either way.
|
||||
kv_b_proj._cpu_skip_gemm_dispatch = True
|
||||
# MLA reads this weight directly to build W_UK/W_UV, so a backend must
|
||||
# not relayout it at load time.
|
||||
kv_b_proj.skip_weight_relayout = True
|
||||
self.use_direct_call = not current_platform.opaque_attention_op()
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
|
||||
@@ -204,9 +204,9 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if current_platform.is_cpu():
|
||||
# MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): not
|
||||
# perf-critical, so skip packing and use a plain fallback.
|
||||
if getattr(layer, "_cpu_skip_gemm_dispatch", False):
|
||||
# MLA's kv_b_proj (see `skip_weight_relayout`): not perf-critical,
|
||||
# so skip packing and use a plain fallback.
|
||||
if getattr(layer, "skip_weight_relayout", False):
|
||||
layer.cpu_linear = torch.nn.functional.linear
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user