[Model][Hardware][AMD]: Part 1/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_AITER_FA, and ROCM_AITER_UNIFIED_ATTN (#42749)

Signed-off-by: Jack Hu <Jack.Hu@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Jack Hu
2026-07-16 17:39:04 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent ab0a20d151
commit 2cab53ddee
15 changed files with 1361 additions and 4 deletions
@@ -1200,6 +1200,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
layer._v_scale_float = _to_scalar(layer.v_scale)
layer._q_scale_float = _to_scalar(layer.q_scale)
# Sync host (cpu) scale copies read by AITER fused kernels.
layer._k_scale_cpu.fill_(layer._k_scale_float)
layer._v_scale_cpu.fill_(layer._v_scale_float)
# Discard all placeholders.
del layer.k_scale
del layer.v_scale
@@ -144,6 +144,9 @@ class BaseKVCacheMethod(QuantizeMethodBase):
layer._v_scale.copy_(v_scale)
layer._k_scale_float = k_scale
layer._v_scale_float = v_scale
# Sync host (cpu) scale copies read by AITER fused kernels.
layer._k_scale_cpu.fill_(k_scale)
layer._v_scale_cpu.fill_(v_scale)
if k_scale == 1.0 and v_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
logger.warning_once(
"Using KV cache scaling factor 1.0 for fp8_e4m3. "