diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index b3038524eb..d271877dfc 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -609,6 +609,32 @@ def test_pinned_embedding_forward_finalizes_prefetched_output( assert torch.equal(output, expected) +def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None: + """Forward before any start_prefetch must fail loudly, not read garbage.""" + embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) + nn.Module.__init__(embedding) + embedding._prefetch_buffer = None + embedding._prefetch_stream = None + embedding._output_dim = 6 + hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16) + + with pytest.raises(RuntimeError, match="prior start_prefetch"): + embedding(hidden_states) + + +def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None: + """_finalize_prefetch before any start_prefetch must fail loudly.""" + embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) + nn.Module.__init__(embedding) + embedding._prefetch_buffer = None + embedding._prefetch_stream = None + embedding.tp_size = 1 + output = torch.zeros(2, 6, dtype=torch.bfloat16) + + with pytest.raises(RuntimeError, match="prior start_prefetch"): + embedding._finalize_prefetch(torch.empty(4, 2, 3), output) + + def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None: embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) nn.Module.__init__(embedding) @@ -723,6 +749,67 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva( ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """start_prefetch lazily allocates the stream/buffer, then forward consumes it. + + Exercises the full pinned-prefetch contract end to end: first call + allocates the side stream and buffer with the constructor geometry, + and forward finalizes the lookup into the returned output. + """ + _mock_etp_group(monkeypatch) + monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + embedding_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + parameter_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + + with torch.device("cuda:0"): + embedding = Qwen4ExpPLEPinnedHostEmbedding( + 4, + 3, + params_dtype=torch.bfloat16, + padding_size=1, + prefix="test.ple_embedding", + embedding_method=Qwen4ExpPLEUnquantizedEmbeddingMethod(), + num_ngram_heads=2, + max_total_tokens=4, + ) + assert embedding._prefetch_buffer is None + assert embedding._prefetch_stream is None + + loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3) + loaded_weight = loaded_weight.to(torch.bfloat16) + copy_ple_embedding_shard_( + embedding.weight, + loaded_weight, + checkpoint_start=0, + tp_start=0, + tp_end=4, + ) + + ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0") + hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0") + embedding.start_prefetch(hidden_states, ngram_ids) + + buffer = embedding._prefetch_buffer + assert buffer is not None + assert buffer.shape == (4, 2, 3) + assert buffer.dtype == torch.bfloat16 + assert buffer.device.type == "cuda" + assert embedding._prefetch_stream is not None + + output = embedding(hidden_states) + + expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + + def test_ple_fp8_embedding_supports_mixed_precision_config() -> None: prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding" quant_config = ModelOptMixedPrecisionConfig.from_config( @@ -2004,6 +2091,25 @@ def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale( ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") +def test_amd_pinned_embedding_prefetch_machinery_is_lazy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The pinned embedding must not allocate the prefetch stream/buffer in init. + + Paths that never prefetch (AMD uses the synchronous pinned-lookup op) + keep the stream/buffer unallocated for the process lifetime. NVIDIA's + first ``start_prefetch`` — the eager profile run, always before cudagraph + capture — allocates them lazily. + """ + module, _ = _build_amd_ngram_embedding( + monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False + ) + embedding = module.ngram_embedding + assert embedding._prefetch_buffer is None + assert embedding._prefetch_stream is None + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") @pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"]) def test_amd_pinned_embedding_output_written_under_compile( diff --git a/vllm/models/qwen4_exp/common/ngram_embedding.py b/vllm/models/qwen4_exp/common/ngram_embedding.py index 4d1d358958..e7371770f8 100644 --- a/vllm/models/qwen4_exp/common/ngram_embedding.py +++ b/vllm/models/qwen4_exp/common/ngram_embedding.py @@ -7,6 +7,7 @@ n-gram embedding table can be kept in pinned host memory and looked up through Unified Virtual Addressing on any CUDA-alike platform. """ +import threading from abc import ABC, abstractmethod from typing import ClassVar @@ -415,14 +416,11 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): ) self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight) self._block_d = triton.next_power_of_2(self.embedding_dim) - self._prefetch_stream = torch.cuda.Stream(device=self._uva_weight.device) - self._prefetch_buffer = torch.empty( - max_total_tokens * self.etp_data_parallel_size, - num_ngram_heads, - self.embedding_dim, - dtype=self.weight.dtype, - device=self._uva_weight.device, - ) + self._prefetch_stream: torch.cuda.Stream | None = None + self._prefetch_buffer: torch.Tensor | None = None + self._prefetch_alloc_lock = threading.Lock() + self._prefetch_rows = max_total_tokens * self.etp_data_parallel_size + self._num_ngram_heads = num_ngram_heads self._output_dim = num_ngram_heads * self.embedding_dim def allocate_embedding_weight( @@ -506,10 +504,42 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): ngram_ids: torch.Tensor, ) -> None: """Gather ETP IDs and launch their UVA lookup on the side stream.""" + buffer = self._prefetch_buffer + if buffer is None: + # First use allocates. The eager profile run always precedes + # cudagraph capture, so allocation never happens mid-capture; + # the lock keeps concurrent first callers from tearing the + # stream/buffer pair. + with self._prefetch_alloc_lock: + buffer = self._prefetch_buffer + if buffer is None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "pinned PLE prefetch buffer must be allocated " + "eagerly, before cudagraph capture" + ) + self._prefetch_stream = torch.cuda.Stream( + device=self._uva_weight.device + ) + buffer = torch.empty( + self._prefetch_rows, + self._num_ngram_heads, + self.embedding_dim, + dtype=self.weight.dtype, + device=self._uva_weight.device, + ) + self._prefetch_buffer = buffer + prefetch_stream = self._prefetch_stream + if prefetch_stream is None: + raise RuntimeError("pinned PLE prefetch stream was not allocated") slot_size, _ = self._get_dp_gather_slot(ngram_ids.shape[0]) gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) - active_output = self._prefetch_buffer[: gathered_ids.shape[0]] - prefetch_stream = self._prefetch_stream + if gathered_ids.shape[0] > buffer.shape[0]: + raise ValueError( + f"pinned PLE prefetch buffer holds {buffer.shape[0]} rows, " + f"but the batch needs {gathered_ids.shape[0]}" + ) + active_output = buffer[: gathered_ids.shape[0]] prefetch_stream.wait_stream(torch.cuda.current_stream()) gathered_ids.record_stream(prefetch_stream) with torch.cuda.stream(prefetch_stream): @@ -522,7 +552,10 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): output: torch.Tensor, ) -> None: """Join the side stream, reduce ETP shards, and select local rows.""" - torch.cuda.current_stream().wait_stream(self._prefetch_stream) + prefetch_stream = self._prefetch_stream + if prefetch_stream is None: + raise RuntimeError("pinned PLE finalize requires a prior start_prefetch") + torch.cuda.current_stream().wait_stream(prefetch_stream) slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0]) active_output = prefetch_output[: slot_size * self.etp_data_parallel_size] embeddings = self._reduce_etp_embeddings(active_output) @@ -535,8 +568,9 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: """Finish the pinned lookup into graph-owned output storage.""" - output = self._prefetch_buffer.new_empty( - (hidden_states.shape[0], self._output_dim) - ) - self._finalize_prefetch(self._prefetch_buffer, output) + buffer = self._prefetch_buffer + if buffer is None: + raise RuntimeError("pinned PLE lookup requires a prior start_prefetch") + output = buffer.new_empty((hidden_states.shape[0], self._output_dim)) + self._finalize_prefetch(buffer, output) return output