Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -70,20 +70,21 @@ def test_moe_sp_rejects_pipeline_parallel() -> None:
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
|
||||
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
|
||||
pytest.raises(ValueError, match="MoE SP requires PP=1"),
|
||||
):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
|
||||
|
||||
|
||||
@spawn_new_process_for_each_test
|
||||
@pytest.mark.parametrize("backend", ["amd", "nvidia"])
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
|
||||
backend: str, use_sp: bool
|
||||
) -> None:
|
||||
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
|
||||
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
|
||||
model.use_hc_sequence_parallel = use_sp
|
||||
model.use_sequence_parallel = False
|
||||
torch.nn.Module.__init__(model)
|
||||
@@ -98,7 +99,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
def decoder(**kwargs):
|
||||
"""Check that the embedding residual shares the hidden state's token slice."""
|
||||
hidden = kwargs["hidden_states"]
|
||||
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
|
||||
if backend == "nvidia":
|
||||
prev_block_output = kwargs["prev_block_output"]
|
||||
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
|
||||
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
|
||||
|
||||
model.layers = [decoder]
|
||||
@@ -120,9 +123,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
|
||||
|
||||
with (
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
|
||||
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
|
||||
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
|
||||
@@ -14,8 +14,8 @@ from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA or ROCm", allow_module_level=True)
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import (
|
||||
@@ -38,14 +38,19 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
from .test_ple import _ConvBatchCase, _make_conv_metadata
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.qwen4_exp.amd import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.amd.model import Qwen4ExpDecoderLayer
|
||||
else:
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
|
||||
|
||||
class _TPAttentionProjection(nn.Module):
|
||||
"""Keep real TP communication without setting up an attention KV cache."""
|
||||
@@ -168,7 +173,8 @@ def _check_decoder_ple(rank: int) -> None:
|
||||
set_current_vllm_config(config),
|
||||
set_forward_context({layer.ple.prefix: metadata}, config),
|
||||
):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
if hasattr(layer.ple, "start_prefetch"):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
output = layer(
|
||||
*inputs,
|
||||
positions,
|
||||
@@ -177,7 +183,10 @@ def _check_decoder_ple(rank: int) -> None:
|
||||
ngram_context=context,
|
||||
)
|
||||
if config.parallel_config.enable_hc_sp:
|
||||
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
# ROCm all-gather requires contiguous inputs.
|
||||
output = tuple(
|
||||
sp_all_gather(x.contiguous())[:num_tokens] for x in output
|
||||
)
|
||||
# A later MoE call may reuse workspace-backed output storage.
|
||||
outputs.append(tuple(x.clone() for x in output))
|
||||
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
|
||||
@@ -223,11 +232,38 @@ def _check_moe_sp(rank: int, config: VllmConfig) -> None:
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
actual = tuple(sp_all_gather(x.contiguous())[:num_tokens] for x in output)
|
||||
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
def _check_compiled_sp_ops() -> None:
|
||||
"""Compiled SP collectives stay correct for token counts unseen at trace time."""
|
||||
from vllm.models.qwen4_exp.amd.ops.sequence_parallel import (
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
|
||||
def collectives(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
gathered = sp_all_gather(sp_shard(x) * 2)[: x.shape[0]]
|
||||
reduced = sp_all_gather(sp_reduce_scatter(x))[: x.shape[0]]
|
||||
return gathered, reduced
|
||||
|
||||
# vLLM drops Dynamo guards, so one trace must serve every token count.
|
||||
compiled = torch.compile(
|
||||
collectives,
|
||||
fullgraph=True,
|
||||
options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
|
||||
)
|
||||
traced = torch.randn(8, 16)
|
||||
torch._dynamo.mark_dynamic(traced, 0)
|
||||
compiled(traced)
|
||||
for num_tokens in (7, 3, 1):
|
||||
torch.manual_seed(num_tokens)
|
||||
x = torch.randn(num_tokens, 16)
|
||||
torch.testing.assert_close(compiled(x), collectives(x), atol=0, rtol=0)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool, compiled_ops: bool = False) -> None:
|
||||
"""Initialize rank-local runtime state and run one equivalence check."""
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
@@ -247,6 +283,8 @@ def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
if moe_sp:
|
||||
_check_moe_sp(rank, config)
|
||||
elif compiled_ops:
|
||||
_check_compiled_sp_ops()
|
||||
else:
|
||||
_check_decoder_ple(rank)
|
||||
finally:
|
||||
@@ -265,3 +303,10 @@ def test_moe_sp_equivalence(monkeypatch) -> None:
|
||||
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
|
||||
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
|
||||
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only SP ops")
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_compiled_sp_ops_handle_untraced_token_counts() -> None:
|
||||
"""Padding in compiled SP collectives must not specialize on the traced size."""
|
||||
mp.spawn(_run_sp, args=(get_open_port(), False, True), nprocs=2)
|
||||
|
||||
Reference in New Issue
Block a user