[Bugfix] Pin EPLB and MLA host-to-device transfer buffers (#56138)
Signed-off-by: khluu <khluu000@gmail.com> Signed-off-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
d0dfe587d5
commit
1cf6555214
@@ -7,6 +7,8 @@ import pytest
|
||||
import torch
|
||||
import torch.distributed
|
||||
|
||||
import vllm.distributed.eplb.eplb_communicator as eplb_comm
|
||||
import vllm.utils.gpu_sync_debug as gsd
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.eplb.eplb_communicator import (
|
||||
create_eplb_communicator,
|
||||
@@ -25,6 +27,29 @@ from vllm.distributed.parallel_state import (
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_gloo_receive_staging_does_not_force_gpu_sync(monkeypatch):
|
||||
"""Gloo receives must reach the GPU without an implicit pageable copy."""
|
||||
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
|
||||
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
|
||||
gsd._install_copy_checkers()
|
||||
monkeypatch.setattr(eplb_comm, "is_local_first_rank", lambda: False)
|
||||
|
||||
monkeypatch.setattr(eplb_comm, "P2POp", lambda op, tensor, peer, group: tensor)
|
||||
|
||||
def receive(tensors):
|
||||
for tensor in tensors:
|
||||
tensor.fill_(7)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(eplb_comm, "batch_isend_irecv", receive)
|
||||
communicator = eplb_comm.TorchDistGlooStagedEplbCommunicator(cpu_group=None)
|
||||
dst = torch.empty(32, device="cuda")
|
||||
communicator.add_recv([dst], src_rank=1, expert_id=0)
|
||||
gsd.with_gpu_sync_check(communicator.execute)()
|
||||
torch.testing.assert_close(dst.cpu(), torch.full((32,), 7.0))
|
||||
|
||||
|
||||
def create_expert_indices_with_redundancy(
|
||||
num_layers: int,
|
||||
num_logical_experts: int,
|
||||
|
||||
@@ -8,14 +8,20 @@ chunk never covers a prefill without context (which is why the partial no
|
||||
longer needs an empty-span masking pass).
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.utils.gpu_sync_debug as gsd
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
build_mla_chunked_context_metadata,
|
||||
init_mla_context_partial,
|
||||
reorg_kvcache,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
SparseMLACommonMetadataBuilder,
|
||||
)
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
|
||||
@@ -45,6 +51,40 @@ def build_chunked_context(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_sparse_context_lengths_do_not_force_gpu_sync(monkeypatch):
|
||||
"""Sparse MLA must pin computed context lengths before the H2D copy."""
|
||||
builder = SimpleNamespace(
|
||||
chunked_prefill_workspace=torch.empty((2048, 1)),
|
||||
chunked_prefill_workspace_size=1024,
|
||||
kv_cache_spec=SimpleNamespace(block_size=BLOCK_SIZE),
|
||||
device=torch.device("cuda"),
|
||||
dcp_world_size=1,
|
||||
dcp_local_block_size=1,
|
||||
dcp_virtual_block_size=1,
|
||||
dcp_manager=None,
|
||||
)
|
||||
common_metadata = SimpleNamespace(
|
||||
seq_lens_cpu_upper_bound=torch.tensor([17, 2052, 324], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 1, 5, 9], dtype=torch.int32),
|
||||
)
|
||||
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
|
||||
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
|
||||
gsd._install_copy_checkers()
|
||||
metadata = gsd.with_gpu_sync_check(
|
||||
SparseMLACommonMetadataBuilder._build_chunked_context_fields
|
||||
)(
|
||||
builder,
|
||||
common_metadata,
|
||||
num_decodes=1,
|
||||
num_prefills=2,
|
||||
prefill_query_lens_cpu=torch.tensor([4, 4], dtype=torch.int32),
|
||||
)
|
||||
assert metadata is not None
|
||||
assert metadata.context_lens.device.type == "cuda"
|
||||
assert metadata.context_lens.tolist() == [2048, 320]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"context_lens,workspace_size",
|
||||
[
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.distributed.utils import is_weak_contiguous
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -204,7 +205,9 @@ class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
|
||||
)
|
||||
)
|
||||
continue
|
||||
cpu_tensor = torch.empty_like(tensor, device="cpu")
|
||||
cpu_tensor = torch.empty_like(
|
||||
tensor, device="cpu", pin_memory=PIN_MEMORY
|
||||
)
|
||||
p2p_ops.append(
|
||||
P2POp(
|
||||
torch.distributed.irecv,
|
||||
|
||||
@@ -28,7 +28,11 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
is_quantized_kv_cache,
|
||||
np_to_pinned_tensor,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
@@ -241,9 +245,13 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
context_lens_cpu = (
|
||||
seq_lens_cpu[num_decodes : num_decodes + num_prefills]
|
||||
- prefill_query_lens_cpu
|
||||
context_lens_cpu = torch.empty(
|
||||
num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY
|
||||
)
|
||||
torch.subtract(
|
||||
seq_lens_cpu[num_decodes : num_decodes + num_prefills],
|
||||
prefill_query_lens_cpu,
|
||||
out=context_lens_cpu,
|
||||
)
|
||||
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
prefill_query_start_loc_cpu = qsl_cpu[num_decodes:] - qsl_cpu[num_decodes]
|
||||
|
||||
Reference in New Issue
Block a user