[ROCm][Qwen4Exp] Use fused PLE Triton kernels on the AMD backend (#60021)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Mikko Tukiainen
2026-10-08 15:47:56 +00:00
committed by GitHub
co-authored by Cursor
parent 0fc0cefb4e
commit e2414ec1db
8 changed files with 277 additions and 590 deletions
+53 -4
View File
@@ -45,6 +45,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
from vllm.platforms import current_platform
from vllm.utils.torch_utils import weak_ref_tensor
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
@@ -1102,7 +1103,7 @@ def test_fused_ngram_ids_correctness(
contexts: list[list[int]],
first_token_id: int,
) -> None:
from vllm.models.qwen4_exp.nvidia.ops.ple import ple_ngram_ids
from vllm.models.qwen4_exp.common.ops.ple import ple_ngram_ids
device = torch.device("cuda")
query_start_loc = torch.tensor(
@@ -1857,15 +1858,20 @@ def _make_conv_metadata(
),
],
)
@pytest.mark.parametrize(
"layer_cls", [Qwen4ExpPLELayer, Qwen4ExpPLELayerAMD], ids=["nvidia", "amd"]
)
def test_fused_conv_correctness(
case: _ConvBatchCase,
state_layout: str,
layer_cls: type[nn.Module],
) -> None:
device = torch.device("cuda")
metadata, num_real_tokens = _make_conv_metadata(case, device)
if case.state_index_stride > 1:
assert metadata.state_indices_tensor.stride(0) == case.state_index_stride
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
# layer_cls is type[nn.Module], so layer_cls.__new__ is type.__new__.
module = object.__new__(layer_cls)
nn.Module.__init__(module)
module.conv_state_len = (case.kernel_size - 1) * case.dilation
module.short_conv_dilation = case.dilation
@@ -1938,7 +1944,7 @@ def test_fused_conv_correctness(
def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
import math
from vllm.models.qwen4_exp.nvidia.ops.ple import ple_gate
from vllm.models.qwen4_exp.common.ops.ple import ple_gate
device = torch.device("cuda")
hc, h = 4, 2560
@@ -2038,7 +2044,6 @@ def _build_amd_ngram_embedding(
embedding_dim=config.ple_embed_dim,
ple_dense_layer_id=0,
max_total_tokens=4,
max_num_reqs=2,
prefix="test.ple_embedding",
layer_name="test.ple",
quant_config=(
@@ -2224,3 +2229,47 @@ def test_amd_pinned_embedding_output_written_under_cudagraph_capture(
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="only the ROCm backend runs the n-gram embedding under torch.compile",
)
@pytest.mark.parametrize(
("query_lens", "contexts"),
[([3, 1], [[5, 6], [0, 7]]), ([1, 0, 2], [[1, 2], [3, 4], [0, 0]])],
ids=["two-requests", "empty-request"],
)
def test_amd_ngram_embedding_matches_reference_under_compile(
monkeypatch: pytest.MonkeyPatch,
query_lens: list[int],
contexts: list[list[int]],
) -> None:
"""The AMD n-gram ID op must run under Dynamo with a symbolic request count."""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=False, fp8_checkpoint=False
)
device = torch.device("cuda:0")
query_start_loc = torch.tensor(
[0, *accumulate(query_lens)], dtype=torch.int32, device=device
)
input_ids = torch.tensor(
[9, 0, 11, 12][: sum(query_lens)], dtype=torch.int32, device=device
)
ngram_context = torch.tensor(contexts, dtype=torch.int32, device=device)
compiled = torch.compile(module, fullgraph=True, dynamic=True)
output = compiled(input_ids, query_start_loc, ngram_context)
ngram_ids = _reference_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
module.layer_multipliers,
module.ngram_heads_vocab_sizes,
module.ngram_heads_offsets,
module.eos_token_id,
module.heads_per_ngram,
)
expected = loaded_weight[ngram_ids.cpu()].flatten(-2).to(device)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)