[Bugfix][Qwen4Exp] Keep pinned PLE prefetch ids out of the CUDA graph pool (#58489)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-09-25 10:51:50 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3059155b47
commit 48d8880d09
2 changed files with 42 additions and 0 deletions
+35
View File
@@ -35,6 +35,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
from vllm.utils.torch_utils import weak_ref_tensor
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
)
@@ -885,6 +886,40 @@ def test_fused_ngram_ids_correctness(
assert torch.all((actual >= offsets) & (actual < offsets + sizes))
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused PLE needs CUDA")
def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
"""Eager breaks hand the side-stream lookup weak refs, so the ids must stay
valid after start_prefetch returns rather than be reused from the pool."""
device = torch.device("cuda")
query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
input_ids = torch.arange(20, 25, dtype=torch.int32, device=device)
ngram_context = torch.tensor([[11, 12], [13, 14]], dtype=torch.int32, device=device)
params = _ngram_hash_params(device, ngram_context.shape[1])
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
nn.Module.__init__(module)
for name in ("layer_multipliers", "ngram_heads_vocab_sizes", "ngram_heads_offsets"):
module.register_buffer(name, params[name])
module.eos_token_id = params["eos_token_id"]
module.heads_per_ngram = params["heads_per_ngram"]
num_heads = params["ngram_heads_vocab_sizes"].numel()
module._prefetch_ids = torch.empty(8, num_heads, dtype=torch.long, device=device)
prefetched: list[torch.Tensor] = []
module.ngram_embedding = SimpleNamespace(
supports_prefetch=True,
start_prefetch=lambda _, ids: prefetched.append(weak_ref_tensor(ids)),
)
expected = _reference_ngram_ids(input_ids, query_start_loc, ngram_context, **params)
# A fresh pool makes the next same-size allocation reuse any freed ids,
# like a later CUDA graph segment would.
pool = torch.cuda.MemPool()
with torch.cuda.use_mem_pool(pool):
module.start_prefetch(None, input_ids, query_start_loc, ngram_context)
torch.full_like(expected, -1)
assert torch.equal(prefetched[0], expected)
def _short_conv_dilated_decode_pytorch(
x_d: torch.Tensor,
conv_state: torch.Tensor,
@@ -716,6 +716,12 @@ class Qwen4ExpNGramEmbedding(nn.Module):
max_total_tokens=max_total_tokens,
data_parallel_rank=data_parallel_rank,
)
if self.ngram_embedding.supports_prefetch:
# The side-stream lookup outlives eager-break args, whose
# graph-pool storage later segments may reuse.
self._prefetch_ids = torch.empty(
max_total_tokens, self.ngram_heads, dtype=torch.long
)
weight = self.ngram_embedding.weight
logger.info(
"Initialized PLE embedding %s: quantization_method=%s, "
@@ -864,6 +870,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
input_ids,
query_start_loc,
ngram_context,
output=self._prefetch_ids[: input_ids.numel()],
)
embedding.start_prefetch(hidden_states, ngram_ids)