[ROCm][Perf] Allocate the pinned PLE prefetch buffer lazily (#58797)
Signed-off-by: Mathew Odden <modden@redhat.com> Co-authored-by: opencode+glm-5.3+vllm <opencode+glm-5.3+vllm@example.com>
This commit is contained in:
co-authored by
opencode+glm-5.3+vllm
parent
50319916a3
commit
91dab0eb7f
@@ -609,6 +609,32 @@ def test_pinned_embedding_forward_finalizes_prefetched_output(
|
||||
assert torch.equal(output, expected)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None:
|
||||
"""Forward before any start_prefetch must fail loudly, not read garbage."""
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = None
|
||||
embedding._prefetch_stream = None
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(RuntimeError, match="prior start_prefetch"):
|
||||
embedding(hidden_states)
|
||||
|
||||
|
||||
def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None:
|
||||
"""_finalize_prefetch before any start_prefetch must fail loudly."""
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = None
|
||||
embedding._prefetch_stream = None
|
||||
embedding.tp_size = 1
|
||||
output = torch.zeros(2, 6, dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(RuntimeError, match="prior start_prefetch"):
|
||||
embedding._finalize_prefetch(torch.empty(4, 2, 3), output)
|
||||
|
||||
|
||||
def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None:
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
@@ -723,6 +749,67 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""start_prefetch lazily allocates the stream/buffer, then forward consumes it.
|
||||
|
||||
Exercises the full pinned-prefetch contract end to end: first call
|
||||
allocates the side stream and buffer with the constructor geometry,
|
||||
and forward finalizes the lookup into the returned output.
|
||||
"""
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
|
||||
with torch.device("cuda:0"):
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding(
|
||||
4,
|
||||
3,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=Qwen4ExpPLEUnquantizedEmbeddingMethod(),
|
||||
num_ngram_heads=2,
|
||||
max_total_tokens=4,
|
||||
)
|
||||
assert embedding._prefetch_buffer is None
|
||||
assert embedding._prefetch_stream is None
|
||||
|
||||
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
|
||||
loaded_weight = loaded_weight.to(torch.bfloat16)
|
||||
copy_ple_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0")
|
||||
embedding.start_prefetch(hidden_states, ngram_ids)
|
||||
|
||||
buffer = embedding._prefetch_buffer
|
||||
assert buffer is not None
|
||||
assert buffer.shape == (4, 2, 3)
|
||||
assert buffer.dtype == torch.bfloat16
|
||||
assert buffer.device.type == "cuda"
|
||||
assert embedding._prefetch_stream is not None
|
||||
|
||||
output = embedding(hidden_states)
|
||||
|
||||
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptMixedPrecisionConfig.from_config(
|
||||
@@ -2004,6 +2091,25 @@ def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
def test_amd_pinned_embedding_prefetch_machinery_is_lazy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The pinned embedding must not allocate the prefetch stream/buffer in init.
|
||||
|
||||
Paths that never prefetch (AMD uses the synchronous pinned-lookup op)
|
||||
keep the stream/buffer unallocated for the process lifetime. NVIDIA's
|
||||
first ``start_prefetch`` — the eager profile run, always before cudagraph
|
||||
capture — allocates them lazily.
|
||||
"""
|
||||
module, _ = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False
|
||||
)
|
||||
embedding = module.ngram_embedding
|
||||
assert embedding._prefetch_buffer is None
|
||||
assert embedding._prefetch_stream is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
|
||||
def test_amd_pinned_embedding_output_written_under_compile(
|
||||
|
||||
@@ -7,6 +7,7 @@ n-gram embedding table can be kept in pinned host memory and looked up through
|
||||
Unified Virtual Addressing on any CUDA-alike platform.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import ClassVar
|
||||
|
||||
@@ -415,14 +416,11 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
)
|
||||
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
|
||||
self._block_d = triton.next_power_of_2(self.embedding_dim)
|
||||
self._prefetch_stream = torch.cuda.Stream(device=self._uva_weight.device)
|
||||
self._prefetch_buffer = torch.empty(
|
||||
max_total_tokens * self.etp_data_parallel_size,
|
||||
num_ngram_heads,
|
||||
self.embedding_dim,
|
||||
dtype=self.weight.dtype,
|
||||
device=self._uva_weight.device,
|
||||
)
|
||||
self._prefetch_stream: torch.cuda.Stream | None = None
|
||||
self._prefetch_buffer: torch.Tensor | None = None
|
||||
self._prefetch_alloc_lock = threading.Lock()
|
||||
self._prefetch_rows = max_total_tokens * self.etp_data_parallel_size
|
||||
self._num_ngram_heads = num_ngram_heads
|
||||
self._output_dim = num_ngram_heads * self.embedding_dim
|
||||
|
||||
def allocate_embedding_weight(
|
||||
@@ -506,10 +504,42 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
ngram_ids: torch.Tensor,
|
||||
) -> None:
|
||||
"""Gather ETP IDs and launch their UVA lookup on the side stream."""
|
||||
buffer = self._prefetch_buffer
|
||||
if buffer is None:
|
||||
# First use allocates. The eager profile run always precedes
|
||||
# cudagraph capture, so allocation never happens mid-capture;
|
||||
# the lock keeps concurrent first callers from tearing the
|
||||
# stream/buffer pair.
|
||||
with self._prefetch_alloc_lock:
|
||||
buffer = self._prefetch_buffer
|
||||
if buffer is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"pinned PLE prefetch buffer must be allocated "
|
||||
"eagerly, before cudagraph capture"
|
||||
)
|
||||
self._prefetch_stream = torch.cuda.Stream(
|
||||
device=self._uva_weight.device
|
||||
)
|
||||
buffer = torch.empty(
|
||||
self._prefetch_rows,
|
||||
self._num_ngram_heads,
|
||||
self.embedding_dim,
|
||||
dtype=self.weight.dtype,
|
||||
device=self._uva_weight.device,
|
||||
)
|
||||
self._prefetch_buffer = buffer
|
||||
prefetch_stream = self._prefetch_stream
|
||||
if prefetch_stream is None:
|
||||
raise RuntimeError("pinned PLE prefetch stream was not allocated")
|
||||
slot_size, _ = self._get_dp_gather_slot(ngram_ids.shape[0])
|
||||
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
|
||||
active_output = self._prefetch_buffer[: gathered_ids.shape[0]]
|
||||
prefetch_stream = self._prefetch_stream
|
||||
if gathered_ids.shape[0] > buffer.shape[0]:
|
||||
raise ValueError(
|
||||
f"pinned PLE prefetch buffer holds {buffer.shape[0]} rows, "
|
||||
f"but the batch needs {gathered_ids.shape[0]}"
|
||||
)
|
||||
active_output = buffer[: gathered_ids.shape[0]]
|
||||
prefetch_stream.wait_stream(torch.cuda.current_stream())
|
||||
gathered_ids.record_stream(prefetch_stream)
|
||||
with torch.cuda.stream(prefetch_stream):
|
||||
@@ -522,7 +552,10 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
"""Join the side stream, reduce ETP shards, and select local rows."""
|
||||
torch.cuda.current_stream().wait_stream(self._prefetch_stream)
|
||||
prefetch_stream = self._prefetch_stream
|
||||
if prefetch_stream is None:
|
||||
raise RuntimeError("pinned PLE finalize requires a prior start_prefetch")
|
||||
torch.cuda.current_stream().wait_stream(prefetch_stream)
|
||||
slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0])
|
||||
active_output = prefetch_output[: slot_size * self.etp_data_parallel_size]
|
||||
embeddings = self._reduce_etp_embeddings(active_output)
|
||||
@@ -535,8 +568,9 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Finish the pinned lookup into graph-owned output storage."""
|
||||
output = self._prefetch_buffer.new_empty(
|
||||
(hidden_states.shape[0], self._output_dim)
|
||||
)
|
||||
self._finalize_prefetch(self._prefetch_buffer, output)
|
||||
buffer = self._prefetch_buffer
|
||||
if buffer is None:
|
||||
raise RuntimeError("pinned PLE lookup requires a prior start_prefetch")
|
||||
output = buffer.new_empty((hidden_states.shape[0], self._output_dim))
|
||||
self._finalize_prefetch(buffer, output)
|
||||
return output
|
||||
|
||||
Reference in New Issue
Block a user