diff --git a/tests/compile/passes/test_rope_kvcache_fusion.py b/tests/compile/passes/test_rope_kvcache_fusion.py index b27adfc46f..b24b9b5e61 100644 --- a/tests/compile/passes/test_rope_kvcache_fusion.py +++ b/tests/compile/passes/test_rope_kvcache_fusion.py @@ -3,11 +3,13 @@ import pytest import torch +from torch._higher_order_ops import auto_functionalized import vllm.config from tests.compile.backend import TestBackend from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops +from vllm.compilation.passes.fusion import rope_kvcache_fusion from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP from vllm.compilation.passes.fusion.rope_kvcache_fusion import RopeKVCacheFusionPass from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass @@ -24,6 +26,7 @@ from vllm.config import ( PassConfig, VllmConfig, ) +from vllm.config.utils import Range from vllm.forward_context import get_forward_context, set_forward_context from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding @@ -40,6 +43,20 @@ VLLM_UNIFIED_KV_CACHE_UPDATE_OP = torch.ops.vllm.unified_kv_cache_update FP8_DTYPE = current_platform.fp8_dtype() +def test_rope_kvcache_fusion_default_keeps_large_ranges_unfused(): + vllm_config = VllmConfig( + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + pass_config=PassConfig(fuse_rope_kvcache=True), + ), + ) + fusion_pass = RopeKVCacheFusionPass(vllm_config) + + assert fusion_pass.is_applicable_for_range(Range(1, 256)) + assert not fusion_pass.is_applicable_for_range(Range(257, 11650)) + assert not fusion_pass.is_applicable_for_range(Range(11651, 16384)) + + class QKRoPEKVCacheTestModel(torch.nn.Module): def __init__( self, @@ -184,6 +201,51 @@ class QKRoPEKVCacheTestModel(torch.nn.Module): return [torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default] +class QKRoPEStaticQKVCacheTestModel(QKRoPEKVCacheTestModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.q_scale = torch.ones((), dtype=torch.float32, device=self.device) + + def forward( + self, qkv: torch.Tensor, positions: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # Create copy so inplace ops do not modify the original tensors + qkv = qkv.clone() + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + + q_fp8 = torch.empty(q.shape, device=q.device, dtype=FP8_DTYPE) + _, q_fp8 = auto_functionalized( + torch.ops._C.static_scaled_fp8_quant.default, + result=q_fp8, + input=q, + scale=self.q_scale, + group_shape=(-1, -1), + ) + q = q_fp8.view(-1, self.num_heads, self.head_size) + k = k.view(-1, self.num_kv_heads, self.head_size) + v = v.view(-1, self.num_kv_heads, self.head_size) + kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( + k, v, _encode_layer_name(self.layer_name) + ) + return q, k, v, kv_cache_dummy_dep + + def ops_in_model_before(self) -> list[torch._ops.OpOverload]: + ops = [] + if self.enable_rope_custom_op: + if rocm_aiter_ops.is_triton_rotary_embed_enabled(): + ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default) + else: + ops.append(ROTARY_OP) + else: + ops.append(INDEX_SELECT_OP) + ops.append(torch.ops.vllm.unified_kv_cache_update.default) + return ops + + def ops_in_model_after(self) -> list[torch._ops.OpOverload]: + return [torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default] + + @pytest.mark.parametrize( "attn_backend", [ @@ -326,3 +388,198 @@ def test_rope_kvcache_fusion( atol=ATOL, rtol=RTOL, ) + + +@pytest.mark.parametrize( + "attn_backend", + [AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN], +) +@pytest.mark.parametrize("enable_rope_custom_op", [True]) +@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False]) +@pytest.mark.parametrize("num_heads", [64]) +@pytest.mark.parametrize("num_kv_heads", [8]) +@pytest.mark.parametrize("head_size", [64]) +@pytest.mark.parametrize("block_size", [16]) +@pytest.mark.parametrize("is_neox", [True, False]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +@pytest.mark.skipif( + not is_aiter_found_and_supported(), + reason="Only test on ROCm with AITER installed and supported", +) +@pytest.mark.skipif( + not hasattr(torch.ops._C, "static_scaled_fp8_quant"), + reason="static fp8 quant op not available on this build", +) +def test_rope_static_qquant_kvcache_fusion( + attn_backend: AttentionBackendEnum, + enable_rope_custom_op: bool, + enable_aiter_triton_rope: bool, + num_heads: int, + num_kv_heads: int, + head_size: int, + block_size: int, + is_neox: bool, + dtype: torch.dtype, + kv_cache_dtype: str, + monkeypatch: pytest.MonkeyPatch, +): + torch.set_default_device("cuda") + torch.set_default_dtype(dtype) + torch.manual_seed(0) + + custom_ops: list[str] = [] + if enable_rope_custom_op: + custom_ops.append("+rotary_embedding") + + vllm_config = VllmConfig( + model_config=ModelConfig(dtype=dtype), + cache_config=CacheConfig( + block_size=block_size, + cache_dtype=kv_cache_dtype, + ), + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + custom_ops=custom_ops, + pass_config=PassConfig( + fuse_rope_kvcache=True, + eliminate_noops=True, + ), + ), + ) + + with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m: + m.setenv("VLLM_ROCM_USE_AITER", "1") + m.setenv( + "VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0" + ) + rocm_aiter_ops.refresh_env_variables() + + model = QKRoPEStaticQKVCacheTestModel( + vllm_config=vllm_config, + attn_backend=attn_backend, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_size=head_size, + is_neox=is_neox, + dtype=dtype, + device=torch.get_default_device(), + ) + + fusion_pass = RopeKVCacheFusionPass(vllm_config) + passes = [ + NoOpEliminationPass(vllm_config), + SplitCoalescingPass(vllm_config), + ScatterSplitReplacementPass(vllm_config), + fusion_pass, + PostCleanupPass(vllm_config), + ] + backend = TestBackend(*passes) + + T = 5 + qkv = torch.randn( + T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype + ) + pos = torch.arange(T, dtype=torch.long) + + qkv_unfused = qkv.clone() + pos_unfused = pos.clone() + + with set_forward_context(None, vllm_config): + forward_context = get_forward_context() + attn_metadata = model.build_attn_metadata(T) + forward_context.slot_mapping = { + model.layer_name: attn_metadata.slot_mapping + } + q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused) + attn_layer = forward_context.no_compile_layers[model.layer_name] + kv_cache_unfused = attn_layer.kv_cache + del dummy + + torch._dynamo.mark_dynamic(qkv, 0) + torch._dynamo.mark_dynamic(pos, 0) + with set_forward_context(None, vllm_config): + model_fused = torch.compile(model, backend=backend) + forward_context = get_forward_context() + attn_metadata = model_fused.build_attn_metadata(T) + forward_context.slot_mapping = { + model.layer_name: attn_metadata.slot_mapping + } + q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos) + attn_layer = forward_context.no_compile_layers[model.layer_name] + kv_cache_fused = attn_layer.kv_cache + del dummy + + assert fusion_pass.matched_count == 1 + backend.check_before_ops(model.ops_in_model_before()) + backend.check_after_ops(model.ops_in_model_after()) + static_quant_pre = backend.op_count( + torch.ops._C.static_scaled_fp8_quant.default, before=True + ) + static_quant_post = backend.op_count( + torch.ops._C.static_scaled_fp8_quant.default + ) + assert static_quant_pre > 0 + # The replacement still emits static quant, so count is expected to + # remain non-zero after fusion. + assert static_quant_post > 0 + + # Negative control: without the static-Q pattern, the generic RoPE+KV + # pattern cannot match this rope -> static-quant -> kv graph, so the + # fusion above is attributable solely to RopeStaticQQuantKVCachePattern. + # This is a structural property independent of the rope/dtype/neox axes, + # so run the (extra compile) check only once on a representative combo. + if is_neox and enable_aiter_triton_rope and kv_cache_dtype == "auto": + m.setattr( + rope_kvcache_fusion, + "_supports_static_q_fp8_quant_fusion", + lambda: False, + ) + generic_pass = RopeKVCacheFusionPass(vllm_config) + generic_backend = TestBackend( + NoOpEliminationPass(vllm_config), + SplitCoalescingPass(vllm_config), + ScatterSplitReplacementPass(vllm_config), + generic_pass, + PostCleanupPass(vllm_config), + ) + # Reset dynamo so the model is recompiled through generic_backend + # instead of reusing the cached compilation from above. + torch._dynamo.reset() + with set_forward_context(None, vllm_config): + model_generic = torch.compile(model, backend=generic_backend) + forward_context = get_forward_context() + attn_metadata = model_generic.build_attn_metadata(T) + forward_context.slot_mapping = { + model.layer_name: attn_metadata.slot_mapping + } + model_generic(qkv, pos) + # op_count reads the post-pass graph, so it also confirms the pass ran + # (a no-op pass would raise instead of silently passing on count 0). + assert generic_pass.matched_count == 0 + assert ( + generic_backend.op_count( + torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default + ) + == 0 + ) + + if dtype == torch.float16: + ATOL, RTOL = (2e-3, 2e-3) + else: + ATOL, RTOL = (1e-2, 1e-2) + + torch.testing.assert_close( + q_unfused.to(torch.float32), + q_fused.to(torch.float32), + atol=ATOL, + rtol=RTOL, + ) + torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL) + torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL) + torch.testing.assert_close( + kv_cache_unfused.view(dtype), + kv_cache_fused.view(dtype), + atol=ATOL, + rtol=RTOL, + ) diff --git a/vllm/compilation/passes/fusion/rope_kvcache_fusion.py b/vllm/compilation/passes/fusion/rope_kvcache_fusion.py index bc6754188a..fa641897bf 100644 --- a/vllm/compilation/passes/fusion/rope_kvcache_fusion.py +++ b/vllm/compilation/passes/fusion/rope_kvcache_fusion.py @@ -15,6 +15,7 @@ from vllm.model_executor.layers.attention.attention import ( Attention, get_attention_context, ) +from vllm.platforms import current_platform from vllm.utils.torch_utils import ( _USE_LAYERNAME, LayerNameType, @@ -34,6 +35,12 @@ from .rms_quant_fusion import ( ) logger = init_logger(__name__) +FP8_DTYPE = current_platform.fp8_dtype() +STATIC_FP8_QUANT_OP = getattr(torch.ops._C, "static_scaled_fp8_quant", None) + + +def _supports_static_q_fp8_quant_fusion() -> bool: + return STATIC_FP8_QUANT_OP is not None and FP8_DTYPE is not None def fused_rope_and_unified_kv_cache_update_impl( @@ -90,6 +97,185 @@ direct_register_custom_op( ) +class RopeStaticQQuantKVCachePattern: + """ + Fuse rope + static Q fp8 quant while preserving explicit KV-cache update + dependency ordering. + """ + + FUSED_ROPE_KV_OP = torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default + + def __init__( + self, + layer: Attention, + is_neox: bool, + ) -> None: + self.layer_name = layer.layer_name + self.num_heads = layer.num_heads + self.num_kv_heads = layer.num_kv_heads + self.head_size = layer.head_size + self.head_size_v = layer.head_size_v + self.is_neox = is_neox + + self.q_size = self.num_heads * self.head_size + self.k_size = self.num_kv_heads * self.head_size + self.v_size = self.num_kv_heads * self.head_size_v + + self.rope_matcher = MatcherRotaryEmbedding( + is_neox=self.is_neox, + head_size=self.head_size, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + ) + + def get_inputs(self) -> list: + T = 5 + L = 4096 + qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size) + positions = empty_i64(T) + cos_sin_cache = empty_bf16(L, self.head_size) + q_scale = torch.empty((), dtype=torch.float32, device=qkv.device) + inputs: list = [qkv, positions, cos_sin_cache, q_scale] + if _USE_LAYERNAME: + inputs.append(_encode_layer_name(self.layer_name)) + return inputs + + def _mk_pattern_with_layer_name_input(self, _ln): + def pattern(qkv, positions, cos_sin_cache, q_scale, layer_name): + q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + q, k = self.rope_matcher(positions, q, k, cos_sin_cache) + q_fp8 = torch.empty(q.shape, device=q.device, dtype=FP8_DTYPE) + _, q_fp8 = auto_functionalized( + torch.ops._C.static_scaled_fp8_quant.default, + result=q_fp8, + input=q, + scale=q_scale, + group_shape=(-1, -1), + ) + q_view = q_fp8.view(-1, self.num_heads, self.head_size) + k_view = k.view(-1, self.num_kv_heads, self.head_size) + v_view = v.view(-1, self.num_kv_heads, self.head_size_v) + kv_cache_dummy = torch.ops.vllm.unified_kv_cache_update( + k_view, v_view, layer_name + ) + return kv_cache_dummy, q_view, k_view, v_view + + def replacement(qkv, positions, cos_sin_cache, q_scale, layer_name): + q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + q_view = q.view(-1, self.num_heads, self.head_size) + k_view = k.view(-1, self.num_kv_heads, self.head_size) + v_view = v.view(-1, self.num_kv_heads, self.head_size_v) + rope_kv_results = auto_functionalized( + self.FUSED_ROPE_KV_OP, + query=q_view, + key=k_view, + value=v_view, + positions=positions, + cos_sin_cache=cos_sin_cache, + is_neox=self.is_neox, + layer_name=layer_name, + ) + kv_cache_dummy = rope_kv_results[0] + q_after = rope_kv_results[1] + k_after = rope_kv_results[2] + q_after_flat = q_after.view(-1, self.q_size) + q_fp8 = torch.empty( + q_after_flat.shape, device=q_after_flat.device, dtype=FP8_DTYPE + ) + _, q_fp8 = auto_functionalized( + torch.ops._C.static_scaled_fp8_quant.default, + result=q_fp8, + input=q_after_flat, + scale=q_scale, + group_shape=(-1, -1), + ) + return ( + kv_cache_dummy, + q_fp8.view(-1, self.num_heads, self.head_size), + k_after, + v_view, + ) + + return pattern, replacement + + def _mk_pattern_with_layer_name_closure(self, _ln): + def pattern(qkv, positions, cos_sin_cache, q_scale): + q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + q, k = self.rope_matcher(positions, q, k, cos_sin_cache) + q_fp8 = torch.empty(q.shape, device=q.device, dtype=FP8_DTYPE) + _, q_fp8 = auto_functionalized( + torch.ops._C.static_scaled_fp8_quant.default, + result=q_fp8, + input=q, + scale=q_scale, + group_shape=(-1, -1), + ) + q_view = q_fp8.view(-1, self.num_heads, self.head_size) + k_view = k.view(-1, self.num_kv_heads, self.head_size) + v_view = v.view(-1, self.num_kv_heads, self.head_size_v) + kv_cache_dummy = torch.ops.vllm.unified_kv_cache_update(k_view, v_view, _ln) + return kv_cache_dummy, q_view, k_view, v_view + + def replacement(qkv, positions, cos_sin_cache, q_scale): + q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + q_view = q.view(-1, self.num_heads, self.head_size) + k_view = k.view(-1, self.num_kv_heads, self.head_size) + v_view = v.view(-1, self.num_kv_heads, self.head_size_v) + rope_kv_results = auto_functionalized( + self.FUSED_ROPE_KV_OP, + query=q_view, + key=k_view, + value=v_view, + positions=positions, + cos_sin_cache=cos_sin_cache, + is_neox=self.is_neox, + layer_name=_ln, + ) + kv_cache_dummy = rope_kv_results[0] + q_after = rope_kv_results[1] + k_after = rope_kv_results[2] + q_after_flat = q_after.view(-1, self.q_size) + q_fp8 = torch.empty( + q_after_flat.shape, device=q_after_flat.device, dtype=FP8_DTYPE + ) + _, q_fp8 = auto_functionalized( + torch.ops._C.static_scaled_fp8_quant.default, + result=q_fp8, + input=q_after_flat, + scale=q_scale, + group_shape=(-1, -1), + ) + return ( + kv_cache_dummy, + q_fp8.view(-1, self.num_heads, self.head_size), + k_after, + v_view, + ) + + return pattern, replacement + + def register(self, pm_pass: PatternMatcherPass) -> None: + _ln = _encode_layer_name(self.layer_name) + + if _USE_LAYERNAME: + pattern, replacement = self._mk_pattern_with_layer_name_input(_ln) + else: + pattern, replacement = self._mk_pattern_with_layer_name_closure(_ln) + + def fwd_and_view_to_reshape(*args, **kwargs) -> fx.GraphModule: + gm = pm.fwd_only(*args, **kwargs) + view_to_reshape(gm) + return gm + + pm.register_replacement( + pattern, + replacement, + self.get_inputs(), + fwd_and_view_to_reshape, + pm_pass, + ) + + class RopeReshapeKVCachePattern: """ This pattern matches the following unfused inplace ops: @@ -253,6 +439,11 @@ class RopeKVCacheFusionPass(VllmPatternMatcherPass): for _, layer in attn_layers.items(): if layer.impl.fused_rope_kvcache_supported(): for is_neox in [True, False]: + if _supports_static_q_fp8_quant_fusion(): + RopeStaticQQuantKVCachePattern( + layer=layer, + is_neox=is_neox, + ).register(self.patterns) RopeReshapeKVCachePattern( layer=layer, is_neox=is_neox, @@ -274,4 +465,6 @@ class RopeKVCacheFusionPass(VllmPatternMatcherPass): return compile_range.end <= self.max_token_num def uuid(self) -> str: - return VllmInductorPass.hash_source(self, RopeReshapeKVCachePattern) + return VllmInductorPass.hash_source( + self, RopeStaticQQuantKVCachePattern, RopeReshapeKVCachePattern + )