From 1cf6555214446cfb98c42ff5d07baab784925c1b Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Fri, 11 Sep 2026 01:57:47 -0700 Subject: [PATCH] [Bugfix] Pin EPLB and MLA host-to-device transfer buffers (#56138) Signed-off-by: khluu Signed-off-by: Kevin H. Luu Co-authored-by: OpenAI Codex --- tests/distributed/test_eplb_execute.py | 25 ++++++++++++ tests/v1/attention/test_mla_context_chunks.py | 40 +++++++++++++++++++ vllm/distributed/eplb/eplb_communicator.py | 5 ++- .../layers/attention/sparse_mla_attention.py | 16 ++++++-- 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/tests/distributed/test_eplb_execute.py b/tests/distributed/test_eplb_execute.py index 21fa057fd2..1ff41f5df2 100644 --- a/tests/distributed/test_eplb_execute.py +++ b/tests/distributed/test_eplb_execute.py @@ -7,6 +7,8 @@ import pytest import torch import torch.distributed +import vllm.distributed.eplb.eplb_communicator as eplb_comm +import vllm.utils.gpu_sync_debug as gsd from vllm.config import VllmConfig, set_current_vllm_config from vllm.distributed.eplb.eplb_communicator import ( create_eplb_communicator, @@ -25,6 +27,29 @@ from vllm.distributed.parallel_state import ( from .eplb_utils import distributed_run, set_env_vars_and_device +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_gloo_receive_staging_does_not_force_gpu_sync(monkeypatch): + """Gloo receives must reach the GPU without an implicit pageable copy.""" + monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error") + monkeypatch.setattr(gsd, "_sync_check_enabled", True) + gsd._install_copy_checkers() + monkeypatch.setattr(eplb_comm, "is_local_first_rank", lambda: False) + + monkeypatch.setattr(eplb_comm, "P2POp", lambda op, tensor, peer, group: tensor) + + def receive(tensors): + for tensor in tensors: + tensor.fill_(7) + return [] + + monkeypatch.setattr(eplb_comm, "batch_isend_irecv", receive) + communicator = eplb_comm.TorchDistGlooStagedEplbCommunicator(cpu_group=None) + dst = torch.empty(32, device="cuda") + communicator.add_recv([dst], src_rank=1, expert_id=0) + gsd.with_gpu_sync_check(communicator.execute)() + torch.testing.assert_close(dst.cpu(), torch.full((32,), 7.0)) + + def create_expert_indices_with_redundancy( num_layers: int, num_logical_experts: int, diff --git a/tests/v1/attention/test_mla_context_chunks.py b/tests/v1/attention/test_mla_context_chunks.py index 091079acc6..2ed73b2c98 100644 --- a/tests/v1/attention/test_mla_context_chunks.py +++ b/tests/v1/attention/test_mla_context_chunks.py @@ -8,14 +8,20 @@ chunk never covers a prefill without context (which is why the partial no longer needs an empty-span masking pass). """ +from types import SimpleNamespace + import pytest import torch +import vllm.utils.gpu_sync_debug as gsd from vllm.model_executor.layers.attention.mla_attention import ( build_mla_chunked_context_metadata, init_mla_context_partial, reorg_kvcache, ) +from vllm.model_executor.layers.attention.sparse_mla_attention import ( + SparseMLACommonMetadataBuilder, +) BLOCK_SIZE = 16 @@ -45,6 +51,40 @@ def build_chunked_context( ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_sparse_context_lengths_do_not_force_gpu_sync(monkeypatch): + """Sparse MLA must pin computed context lengths before the H2D copy.""" + builder = SimpleNamespace( + chunked_prefill_workspace=torch.empty((2048, 1)), + chunked_prefill_workspace_size=1024, + kv_cache_spec=SimpleNamespace(block_size=BLOCK_SIZE), + device=torch.device("cuda"), + dcp_world_size=1, + dcp_local_block_size=1, + dcp_virtual_block_size=1, + dcp_manager=None, + ) + common_metadata = SimpleNamespace( + seq_lens_cpu_upper_bound=torch.tensor([17, 2052, 324], dtype=torch.int32), + query_start_loc_cpu=torch.tensor([0, 1, 5, 9], dtype=torch.int32), + ) + monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error") + monkeypatch.setattr(gsd, "_sync_check_enabled", True) + gsd._install_copy_checkers() + metadata = gsd.with_gpu_sync_check( + SparseMLACommonMetadataBuilder._build_chunked_context_fields + )( + builder, + common_metadata, + num_decodes=1, + num_prefills=2, + prefill_query_lens_cpu=torch.tensor([4, 4], dtype=torch.int32), + ) + assert metadata is not None + assert metadata.context_lens.device.type == "cuda" + assert metadata.context_lens.tolist() == [2048, 320] + + @pytest.mark.parametrize( "context_lens,workspace_size", [ diff --git a/vllm/distributed/eplb/eplb_communicator.py b/vllm/distributed/eplb/eplb_communicator.py index 77362e593c..162c0f5843 100644 --- a/vllm/distributed/eplb/eplb_communicator.py +++ b/vllm/distributed/eplb/eplb_communicator.py @@ -34,6 +34,7 @@ from vllm.distributed.utils import is_weak_contiguous from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import PIN_MEMORY logger = init_logger(__name__) @@ -204,7 +205,9 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator): ) ) continue - cpu_tensor = torch.empty_like(tensor, device="cpu") + cpu_tensor = torch.empty_like( + tensor, device="cpu", pin_memory=PIN_MEMORY + ) p2p_ops.append( P2POp( torch.distributed.irecv, diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index ab6946132f..8c2bb72bf4 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -28,7 +28,11 @@ from vllm.model_executor.layers.attention.mla_attention import ( from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.flashinfer import has_flashinfer -from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor +from vllm.utils.torch_utils import ( + PIN_MEMORY, + is_quantized_kv_cache, + np_to_pinned_tensor, +) from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder from vllm.v1.attention.backends.fa_utils import get_flash_attn_version from vllm.v1.attention.backends.utils import split_decodes_and_prefills @@ -241,9 +245,13 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound assert seq_lens_cpu is not None - context_lens_cpu = ( - seq_lens_cpu[num_decodes : num_decodes + num_prefills] - - prefill_query_lens_cpu + context_lens_cpu = torch.empty( + num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY + ) + torch.subtract( + seq_lens_cpu[num_decodes : num_decodes + num_prefills], + prefill_query_lens_cpu, + out=context_lens_cpu, ) qsl_cpu = common_attn_metadata.query_start_loc_cpu prefill_query_start_loc_cpu = qsl_cpu[num_decodes:] - qsl_cpu[num_decodes]