[Model][Hardware][AMD]: Part 1/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_AITER_FA, and ROCM_AITER_UNIFIED_ATTN (#42749)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
@@ -136,10 +136,12 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) ->
|
||||
|
||||
# We also keep q/k/v_scale on host (cpu) memory for attention
|
||||
# backends that require the scales to be on host instead of on device.
|
||||
# e.g. Flashinfer
|
||||
# e.g. Flashinfer & AITER
|
||||
layer._q_scale_float = 1.0
|
||||
layer._k_scale_float = 1.0
|
||||
layer._v_scale_float = 1.0
|
||||
layer._k_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
|
||||
layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
|
||||
layer._prob_scale_float = 1.0
|
||||
|
||||
# Initialize q/k/v range constants used by calc_kv_scales
|
||||
@@ -586,6 +588,8 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
self._q_scale_float = self._q_scale.item()
|
||||
self._k_scale_float = self._k_scale.item()
|
||||
self._v_scale_float = self._v_scale.item()
|
||||
self._k_scale_cpu.fill_(self._k_scale_float)
|
||||
self._v_scale_cpu.fill_(self._v_scale_float)
|
||||
# We only calculate the scales once
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
|
||||
@@ -1024,6 +1024,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self._q_scale_float = self._q_scale.item()
|
||||
self._k_scale_float = self._k_scale.item()
|
||||
self._v_scale_float = self._v_scale.item()
|
||||
self._k_scale_cpu.fill_(self._k_scale_float)
|
||||
self._v_scale_cpu.fill_(self._v_scale_float)
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
|
||||
@@ -1200,6 +1200,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
|
||||
layer._v_scale_float = _to_scalar(layer.v_scale)
|
||||
layer._q_scale_float = _to_scalar(layer.q_scale)
|
||||
|
||||
# Sync host (cpu) scale copies read by AITER fused kernels.
|
||||
layer._k_scale_cpu.fill_(layer._k_scale_float)
|
||||
layer._v_scale_cpu.fill_(layer._v_scale_float)
|
||||
|
||||
# Discard all placeholders.
|
||||
del layer.k_scale
|
||||
del layer.v_scale
|
||||
|
||||
@@ -144,6 +144,9 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
layer._v_scale.copy_(v_scale)
|
||||
layer._k_scale_float = k_scale
|
||||
layer._v_scale_float = v_scale
|
||||
# Sync host (cpu) scale copies read by AITER fused kernels.
|
||||
layer._k_scale_cpu.fill_(k_scale)
|
||||
layer._v_scale_cpu.fill_(v_scale)
|
||||
if k_scale == 1.0 and v_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
|
||||
logger.warning_once(
|
||||
"Using KV cache scaling factor 1.0 for fp8_e4m3. "
|
||||
|
||||
Reference in New Issue
Block a user