[Bugfix][Qwen4Exp] Keep pinned PLE prefetch ids out of the CUDA graph pool (#58489)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3059155b47
commit
48d8880d09
@@ -35,6 +35,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
|
||||
from vllm.utils.torch_utils import weak_ref_tensor
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionMetadata,
|
||||
)
|
||||
@@ -885,6 +886,40 @@ def test_fused_ngram_ids_correctness(
|
||||
assert torch.all((actual >= offsets) & (actual < offsets + sizes))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused PLE needs CUDA")
|
||||
def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
|
||||
"""Eager breaks hand the side-stream lookup weak refs, so the ids must stay
|
||||
valid after start_prefetch returns rather than be reused from the pool."""
|
||||
device = torch.device("cuda")
|
||||
query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
|
||||
input_ids = torch.arange(20, 25, dtype=torch.int32, device=device)
|
||||
ngram_context = torch.tensor([[11, 12], [13, 14]], dtype=torch.int32, device=device)
|
||||
params = _ngram_hash_params(device, ngram_context.shape[1])
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
for name in ("layer_multipliers", "ngram_heads_vocab_sizes", "ngram_heads_offsets"):
|
||||
module.register_buffer(name, params[name])
|
||||
module.eos_token_id = params["eos_token_id"]
|
||||
module.heads_per_ngram = params["heads_per_ngram"]
|
||||
num_heads = params["ngram_heads_vocab_sizes"].numel()
|
||||
module._prefetch_ids = torch.empty(8, num_heads, dtype=torch.long, device=device)
|
||||
prefetched: list[torch.Tensor] = []
|
||||
module.ngram_embedding = SimpleNamespace(
|
||||
supports_prefetch=True,
|
||||
start_prefetch=lambda _, ids: prefetched.append(weak_ref_tensor(ids)),
|
||||
)
|
||||
expected = _reference_ngram_ids(input_ids, query_start_loc, ngram_context, **params)
|
||||
|
||||
# A fresh pool makes the next same-size allocation reuse any freed ids,
|
||||
# like a later CUDA graph segment would.
|
||||
pool = torch.cuda.MemPool()
|
||||
with torch.cuda.use_mem_pool(pool):
|
||||
module.start_prefetch(None, input_ids, query_start_loc, ngram_context)
|
||||
torch.full_like(expected, -1)
|
||||
|
||||
assert torch.equal(prefetched[0], expected)
|
||||
|
||||
|
||||
def _short_conv_dilated_decode_pytorch(
|
||||
x_d: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
|
||||
@@ -716,6 +716,12 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
max_total_tokens=max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
)
|
||||
if self.ngram_embedding.supports_prefetch:
|
||||
# The side-stream lookup outlives eager-break args, whose
|
||||
# graph-pool storage later segments may reuse.
|
||||
self._prefetch_ids = torch.empty(
|
||||
max_total_tokens, self.ngram_heads, dtype=torch.long
|
||||
)
|
||||
weight = self.ngram_embedding.weight
|
||||
logger.info(
|
||||
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||
@@ -864,6 +870,7 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
output=self._prefetch_ids[: input_ids.numel()],
|
||||
)
|
||||
embedding.start_prefetch(hidden_states, ngram_ids)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user