[Bugfix][Spec Decode] Support heterogeneous QK fusion geometry (#48671)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
aoshen02
2026-07-14 22:37:10 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 6472131298
commit 3b39fd284a
2 changed files with 98 additions and 19 deletions
@@ -212,3 +212,73 @@ def test_qk_norm_rope_fusion(
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on CUDA and ROCm platforms",
)
def test_qk_norm_rope_fusion_with_heterogeneous_draft_model():
"""The fusion pass should support target and draft attention geometries."""
if not hasattr(torch.ops._C, "fused_qk_norm_rope"):
pytest.skip("fused_qk_norm_rope custom op not available")
dtype = torch.bfloat16
torch.set_default_device(current_platform.device_type)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm", "+rotary_embedding"],
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
eliminate_noops=True,
),
),
)
with (
set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
target_model = QKNormRoPETestModel(
num_heads=32,
num_kv_heads=8,
head_dim=128,
eps=1e-6,
is_neox=True,
vllm_config=vllm_config,
dtype=dtype,
prefix="model.layers.0.self_attn.attn",
)
draft_model = QKNormRoPETestModel(
num_heads=16,
num_kv_heads=8,
head_dim=128,
eps=1e-6,
is_neox=True,
vllm_config=vllm_config,
dtype=dtype,
prefix="draft_model.layers.0.self_attn.attn",
)
fusion_pass = QKNormRoPEFusionPass(vllm_config)
backend = TestBackend(
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
)
for model in (target_model, draft_model):
num_tokens = 5
qkv = torch.randn(num_tokens, model.q_size + 2 * model.kv_size)
positions = torch.arange(num_tokens, dtype=torch.long, device=qkv.device)
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(positions, 0)
torch.compile(model, backend=backend)(qkv, positions)
assert fusion_pass.matched_count == 1