[Model][Hardware][AMD]: Part 1/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_AITER_FA, and ROCM_AITER_UNIFIED_ATTN (#42749)

Signed-off-by: Jack Hu <Jack.Hu@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Jack Hu
2026-07-16 17:39:04 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent ab0a20d151
commit 2cab53ddee
15 changed files with 1361 additions and 4 deletions
+41
View File
@@ -17,6 +17,10 @@ from vllm.config.model import ModelConfig
from vllm.model_executor.kernels.linear.scaled_mm import (
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.attention.attention import (
Attention,
set_default_quant_scales,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.quantization.fp8 import (
Fp8Config,
@@ -24,6 +28,7 @@ from vllm.model_executor.layers.quantization.fp8 import (
Fp8LinearMethod,
Fp8MoEMethod,
)
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineLinearMethod,
)
@@ -422,6 +427,42 @@ def test_fp8_reloading(
method.process_weights_after_loading(layer)
@pytest.mark.parametrize("source", ["checkpoint", "runtime_calc"])
def test_kv_cache_scale_sync_to_host_copies(source):
"""Test device-to-host sync of the k/v quantization scales, for both the
checkpoint-load and runtime-calc paths that produce them.
"""
layer = torch.nn.Module()
set_default_quant_scales(layer, register_buffer=True)
layer.kv_cache_dtype = "fp8"
if source == "checkpoint":
# Scales come from the checkpoint, so runtime calc is disabled.
layer.calculate_kv_scales = False
method = BaseKVCacheMethod(quant_config=None)
method.create_weights(layer)
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
method.process_weights_after_loading(layer)
else:
# First forward computes distinct, non-unity scales from live k/v.
layer.calculate_kv_scales = True
query = torch.full((4, 8), 10.0)
key = torch.full((4, 8), 60.0)
value = torch.full((4, 8), 50.0)
Attention.calc_kv_scales(layer, query, key, value)
assert layer._k_scale_float != 1.0
assert layer._v_scale_float != 1.0
# Host copy must mirror both the float and the device scale tensor.
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale_float)
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale_float)
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale.item())
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale.item())
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",