[Model][Hardware][AMD]: Part 1/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_AITER_FA, and ROCM_AITER_UNIFIED_ATTN (#42749)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
@@ -17,6 +17,10 @@ from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
Attention,
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
@@ -24,6 +28,7 @@ from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8LinearMethod,
|
||||
Fp8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
@@ -422,6 +427,42 @@ def test_fp8_reloading(
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["checkpoint", "runtime_calc"])
|
||||
def test_kv_cache_scale_sync_to_host_copies(source):
|
||||
"""Test device-to-host sync of the k/v quantization scales, for both the
|
||||
checkpoint-load and runtime-calc paths that produce them.
|
||||
"""
|
||||
layer = torch.nn.Module()
|
||||
set_default_quant_scales(layer, register_buffer=True)
|
||||
layer.kv_cache_dtype = "fp8"
|
||||
|
||||
if source == "checkpoint":
|
||||
# Scales come from the checkpoint, so runtime calc is disabled.
|
||||
layer.calculate_kv_scales = False
|
||||
method = BaseKVCacheMethod(quant_config=None)
|
||||
method.create_weights(layer)
|
||||
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
|
||||
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
|
||||
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
|
||||
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
|
||||
method.process_weights_after_loading(layer)
|
||||
else:
|
||||
# First forward computes distinct, non-unity scales from live k/v.
|
||||
layer.calculate_kv_scales = True
|
||||
query = torch.full((4, 8), 10.0)
|
||||
key = torch.full((4, 8), 60.0)
|
||||
value = torch.full((4, 8), 50.0)
|
||||
Attention.calc_kv_scales(layer, query, key, value)
|
||||
|
||||
assert layer._k_scale_float != 1.0
|
||||
assert layer._v_scale_float != 1.0
|
||||
# Host copy must mirror both the float and the device scale tensor.
|
||||
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale_float)
|
||||
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale_float)
|
||||
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale.item())
|
||||
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale.item())
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
|
||||
Reference in New Issue
Block a user