[Bugfix] Pin EPLB and MLA host-to-device transfer buffers (#56138)

Signed-off-by: khluu <khluu000@gmail.com>
Signed-off-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Kevin H. Luu
2026-09-11 01:57:47 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent d0dfe587d5
commit 1cf6555214
4 changed files with 81 additions and 5 deletions
+25
View File
@@ -7,6 +7,8 @@ import pytest
import torch
import torch.distributed
import vllm.distributed.eplb.eplb_communicator as eplb_comm
import vllm.utils.gpu_sync_debug as gsd
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed.eplb.eplb_communicator import (
create_eplb_communicator,
@@ -25,6 +27,29 @@ from vllm.distributed.parallel_state import (
from .eplb_utils import distributed_run, set_env_vars_and_device
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_gloo_receive_staging_does_not_force_gpu_sync(monkeypatch):
"""Gloo receives must reach the GPU without an implicit pageable copy."""
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
gsd._install_copy_checkers()
monkeypatch.setattr(eplb_comm, "is_local_first_rank", lambda: False)
monkeypatch.setattr(eplb_comm, "P2POp", lambda op, tensor, peer, group: tensor)
def receive(tensors):
for tensor in tensors:
tensor.fill_(7)
return []
monkeypatch.setattr(eplb_comm, "batch_isend_irecv", receive)
communicator = eplb_comm.TorchDistGlooStagedEplbCommunicator(cpu_group=None)
dst = torch.empty(32, device="cuda")
communicator.add_recv([dst], src_rank=1, expert_id=0)
gsd.with_gpu_sync_check(communicator.execute)()
torch.testing.assert_close(dst.cpu(), torch.full((32,), 7.0))
def create_expert_indices_with_redundancy(
num_layers: int,
num_logical_experts: int,
@@ -8,14 +8,20 @@ chunk never covers a prefill without context (which is why the partial no
longer needs an empty-span masking pass).
"""
from types import SimpleNamespace
import pytest
import torch
import vllm.utils.gpu_sync_debug as gsd
from vllm.model_executor.layers.attention.mla_attention import (
build_mla_chunked_context_metadata,
init_mla_context_partial,
reorg_kvcache,
)
from vllm.model_executor.layers.attention.sparse_mla_attention import (
SparseMLACommonMetadataBuilder,
)
BLOCK_SIZE = 16
@@ -45,6 +51,40 @@ def build_chunked_context(
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_sparse_context_lengths_do_not_force_gpu_sync(monkeypatch):
"""Sparse MLA must pin computed context lengths before the H2D copy."""
builder = SimpleNamespace(
chunked_prefill_workspace=torch.empty((2048, 1)),
chunked_prefill_workspace_size=1024,
kv_cache_spec=SimpleNamespace(block_size=BLOCK_SIZE),
device=torch.device("cuda"),
dcp_world_size=1,
dcp_local_block_size=1,
dcp_virtual_block_size=1,
dcp_manager=None,
)
common_metadata = SimpleNamespace(
seq_lens_cpu_upper_bound=torch.tensor([17, 2052, 324], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 1, 5, 9], dtype=torch.int32),
)
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
gsd._install_copy_checkers()
metadata = gsd.with_gpu_sync_check(
SparseMLACommonMetadataBuilder._build_chunked_context_fields
)(
builder,
common_metadata,
num_decodes=1,
num_prefills=2,
prefill_query_lens_cpu=torch.tensor([4, 4], dtype=torch.int32),
)
assert metadata is not None
assert metadata.context_lens.device.type == "cuda"
assert metadata.context_lens.tolist() == [2048, 320]
@pytest.mark.parametrize(
"context_lens,workspace_size",
[