Revert "[DSpark] Support pipeline-parallel targets in aggregated serving (#56956)" (#58484)

This commit is contained in:
Nick Hill
2026-09-24 08:48:36 -07:00
committed by GitHub
parent d636456b1f
commit 09fe178dba
11 changed files with 28 additions and 629 deletions
@@ -339,136 +339,6 @@ def test_dsv4_fast_topk(
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
def test_dsv4_fast_topk_padding_uint32_falls_back(monkeypatch: pytest.MonkeyPatch):
"""Padded rows need the -1 sentinel, which uint32 cannot represent: the
router must skip the dsv4 fast path and still route the real rows."""
torch.manual_seed(0)
num_tokens = 17
num_experts = 256
hidden_states = torch.randn((num_tokens, 64), dtype=torch.float32, device="cuda")
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
is_padding = torch.zeros(num_tokens, dtype=torch.bool, device="cuda")
is_padding[1::2] = True
gating_output[is_padding] = float("nan")
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.router."
"fused_topk_bias_router._get_padding_mask",
lambda _: is_padding,
)
# uint32 + padding would trip dsv4_topk's signed-indices assertion; the
# generic path must take over instead.
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=correction_bias,
topk=6,
renormalize=True,
indices_type=torch.uint32,
routed_scaling_factor=1.5,
)
assert topk_ids.dtype == torch.uint32
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output[~is_padding],
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
)
# uint32 CUDA tensors do not support boolean-mask indexing; widen first.
torch.testing.assert_close(
topk_ids.to(torch.int64)[~is_padding],
topk_ids_ref.to(torch.int64),
atol=0,
rtol=0,
)
torch.testing.assert_close(
topk_weights[~is_padding], topk_weights_ref, atol=2e-5, rtol=2e-5
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
def test_dsv4_fast_topk_padding(monkeypatch: pytest.MonkeyPatch):
"""Verify the DSV4 fast path removes graph-padding rows from routing."""
torch.manual_seed(0)
num_tokens = 17
num_experts = 256
hidden_states = torch.randn((num_tokens, 64), dtype=torch.float32, device="cuda")
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
is_padding = torch.zeros(num_tokens, dtype=torch.bool, device="cuda")
is_padding[1::2] = True
gating_output[is_padding] = float("nan")
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.router."
"fused_topk_bias_router._get_padding_mask",
lambda _: is_padding,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=correction_bias,
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
)
assert torch.equal(topk_ids[is_padding], torch.full_like(topk_ids[is_padding], -1))
assert torch.equal(
topk_weights[is_padding], torch.zeros_like(topk_weights[is_padding])
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output[~is_padding],
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
)
torch.testing.assert_close(topk_ids[~is_padding], topk_ids_ref, atol=0, rtol=0)
torch.testing.assert_close(
topk_weights[~is_padding], topk_weights_ref, atol=2e-5, rtol=2e-5
)
# The mask buffer is persistent under CUDA graph replay, but its contents
# change with every batch. Verify that the kernel reads those contents at
# runtime rather than specializing on the mask captured above.
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_weights, graph_ids = dsv4_topk(
gating_output,
correction_bias,
torch.int32,
1.5,
is_padding=is_padding,
)
is_padding.logical_not_()
graph.replay()
assert torch.equal(
graph_ids[is_padding], torch.full_like(graph_ids[is_padding], -1)
)
assert torch.equal(
graph_weights[is_padding], torch.zeros_like(graph_weights[is_padding])
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from types import SimpleNamespace
import numpy as np
@@ -9,9 +8,7 @@ import pytest
import torch
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.spec_decode.dflash import speculator as dflash_speculator
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
DFlashSpeculator,
prepare_dflash_inputs,
)
@@ -36,7 +33,6 @@ def _run_prepare(
input_buffers = SimpleNamespace(
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
query_start_loc=torch.full(
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
),
@@ -144,114 +140,6 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
assert out.temperature[2].item() == 1.0
assert out.seeds[2].item() == 17
assert not out.input_buffers.is_padding[:3].any()
assert out.input_buffers.is_padding[3:].all()
assert out.input_buffers.input_ids[3:].cpu().tolist() == [0] * 13
assert out.input_buffers.positions[3:].cpu().tolist() == [0] * 13
def test_prepare_dflash_inputs_compacts_noncontiguous_request_slots():
device = torch.device("cuda")
max_num_reqs = 4
max_num_tokens = 16
num_speculative_steps = 3
input_buffers = SimpleNamespace(
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
query_start_loc=torch.full(
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
),
seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device),
)
input_batch = SimpleNamespace(
num_reqs=2,
num_scheduled_tokens=np.array([4, 4], dtype=np.int32),
positions=torch.tensor(
[10, 11, 12, 13, 20, 21, 22, 23],
dtype=torch.int64,
device=device,
),
query_start_loc=torch.tensor([0, 4, 8], dtype=torch.int32, device=device),
# Active batch rows are compact, while request state remains in slots 3 and 1.
idx_mapping=torch.tensor([3, 1], dtype=torch.int32, device=device),
)
query_slot_mapping = torch.full(
(max_num_tokens,), -2, dtype=torch.int64, device=device
)
context_positions = torch.full(
(max_num_tokens,), -1, dtype=torch.int64, device=device
)
context_slot_mapping = torch.full(
(max_num_tokens,), -2, dtype=torch.int64, device=device
)
sample_indices = torch.full(
(max_num_reqs * num_speculative_steps,),
-1,
dtype=torch.int64,
device=device,
)
sample_pos = torch.full_like(sample_indices, -1)
sample_idx_mapping = torch.full(
sample_indices.shape, -1, dtype=torch.int32, device=device
)
temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device)
seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device)
input_temperature = torch.tensor(
[0.0, 0.5, 0.0, 1.0], dtype=torch.float32, device=device
)
input_seeds = torch.tensor([0, 11, 0, 33], dtype=torch.int64, device=device)
last_sampled = torch.tensor([0, 77, 0, 99], dtype=torch.int64, device=device)
next_prefill_tokens = torch.zeros_like(last_sampled)
block_table = torch.tensor(
[[0, 0, 7, 8, 9, 10, 11, 12], [0, 0, 13, 14, 15, 16, 17, 18]],
dtype=torch.int32,
device=device,
)
prepare_dflash_inputs(
input_buffers,
query_slot_mapping,
context_positions,
context_slot_mapping,
sample_indices,
sample_pos,
sample_idx_mapping,
temperature,
seeds,
input_batch,
torch.tensor([1, 1], dtype=torch.int32, device=device),
torch.tensor([2, 1], dtype=torch.int32, device=device),
last_sampled,
next_prefill_tokens,
input_temperature,
input_seeds,
block_table,
4,
0,
1,
1,
123,
num_speculative_steps,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
128,
sample_from_anchor=True,
)
torch.accelerator.synchronize()
# Query rows follow compact batch order, but every persistent state lookup
# follows idx_mapping instead of accidentally using the compact row index.
assert input_buffers.input_ids[:6].cpu().tolist() == [99, 123, 123, 77, 123, 123]
assert input_buffers.positions[:6].cpu().tolist() == [12, 13, 14, 23, 24, 25]
assert sample_indices[:6].cpu().tolist() == [0, 1, 2, 3, 4, 5]
assert sample_idx_mapping[:6].cpu().tolist() == [3, 3, 3, 1, 1, 1]
assert temperature.cpu().tolist() == [0.0, 0.5, 0.0, 1.0]
assert seeds.cpu().tolist() == [0, 11, 0, 33]
assert not input_buffers.is_padding[:6].any()
assert input_buffers.is_padding[6:].all()
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
out = _run_prepare(
@@ -286,40 +174,3 @@ def test_prepare_dflash_inputs_never_writes_the_null_block():
PAD_SLOT_ID,
PAD_SLOT_ID,
]
def test_dflash_forward_context_receives_draft_padding_mask(monkeypatch):
device = torch.device("cuda")
input_buffers = SimpleNamespace(
input_ids=torch.tensor([11, 12, 0, 0], dtype=torch.int32, device=device),
positions=torch.tensor([7, 8, 0, 0], dtype=torch.int64, device=device),
is_padding=torch.tensor([False, False, True, True], device=device),
)
observed = None
@contextmanager
def fake_set_forward_context(*args, **kwargs):
nonlocal observed
observed = kwargs["is_padding"].clone()
yield
monkeypatch.setattr(
dflash_speculator, "set_forward_context", fake_set_forward_context
)
speculator = SimpleNamespace(
input_buffers=input_buffers,
vllm_config=SimpleNamespace(),
model=lambda **kwargs: kwargs["input_ids"],
)
result = DFlashSpeculator._run_model(
speculator,
num_tokens=4,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=None,
)
assert result.tolist() == [11, 12, 0, 0]
assert observed is not None
assert observed.tolist() == [False, False, True, True]
+3 -149
View File
@@ -5,35 +5,16 @@
from unittest.mock import Mock
import numpy as np
import pytest
import torch
from vllm.v1.worker.gpu import model_runner, pp_utils
from vllm.v1.worker.gpu import pp_utils
def _cuda_handler(max_sample_len=6):
handler = object.__new__(pp_utils.PPHandler)
handler.is_last_rank = True
handler.disabled = False
handler.max_sample_len = max_sample_len
handler.last_rank = 1
handler.broadcast_group = Mock()
handler.device = torch.device("cuda")
handler.main_stream = torch.cuda.current_stream()
handler.broadcast_stream = torch.cuda.Stream()
return handler
def _batch(num_computed, prefill_len, num_scheduled, idx_mapping=None):
num_reqs = len(num_computed)
if idx_mapping is None:
idx_mapping = list(range(num_reqs))
def _batch(num_computed, prefill_len, num_scheduled):
return Mock(
num_reqs=num_reqs,
num_reqs=len(num_computed),
num_computed_tokens_np=np.array(num_computed, dtype=np.int32),
prefill_len_np=np.array(prefill_len, dtype=np.int32),
num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32),
idx_mapping=torch.tensor(idx_mapping, dtype=torch.int64),
)
@@ -100,130 +81,3 @@ def test_decode_row_ahead_of_a_prefill_chunk():
assert mask is not None
assert mask.tolist() == [True, False]
def test_disabled_handler_skips_broadcast_and_receive(monkeypatch):
"""While disabled (warmup), neither side enqueues a broadcast op."""
sent = []
monkeypatch.setattr(
pp_utils.torch.distributed,
"broadcast",
lambda *args, **kwargs: sent.append((args, kwargs)),
)
handler = object.__new__(pp_utils.PPHandler)
handler.set_disabled(True)
handler.is_last_rank = False
assert handler.receive(Mock()) is False
handler.is_last_rank = True
assert handler.broadcast(Mock(), Mock(), Mock(), Mock()) is None
assert sent == []
handler.set_disabled(False)
assert handler.disabled is False
def test_alloc_combined_keeps_unbind_views_16_byte_aligned():
"""Triton specializes on pointer alignment: an unaligned `num_rejected`
would compile a second `_post_update_kernel` variant at serving time,
where the in-flight broadcast NCCL kernel can block the module load."""
for num_reqs in range(1, 9):
combined = pp_utils._alloc_combined(num_reqs, torch.device("cpu"))
num_sampled, num_rejected = combined.unbind(dim=0)
assert num_sampled.data_ptr() % 16 == 0
assert num_rejected.data_ptr() % 16 == 0
assert combined.shape[1] >= num_reqs
def test_warmup_pp_decode_update_matches_serving_specialization(monkeypatch):
"""The warmup launch must hit the same triton specialization as serving.
A mismatch means the first real ``update_pp_decode_requests`` recompiles
mid-serving, where the in-flight broadcast NCCL kernel blocks the CUDA
module load and deadlocks the pipeline.
"""
calls = []
monkeypatch.setattr(model_runner, "post_update", lambda *args: calls.append(args))
runner = object.__new__(model_runner.GPUModelRunner)
runner.device = torch.device("cpu")
runner.pp_handler = Mock(max_sample_len=3)
runner.req_states = Mock()
runner.warmup_pp_decode_update()
assert len(calls) == 1
args = calls[0]
idx_mapping, _, _, output_bin_counts = args[:4]
sampled_tokens, num_sampled, num_rejected, query_start_loc = args[4:8]
broadcast_drafts, draft_tokens_out = args[10:12]
assert idx_mapping.tolist() == [-1] and idx_mapping.dtype == torch.int64
assert output_bin_counts is None
assert query_start_loc is None
assert sampled_tokens.shape == (1, 3) and sampled_tokens.dtype == torch.int64
assert num_sampled.dtype == torch.int32
assert num_rejected.dtype == torch.int32
# Spec-enabled PP handlers receive drafts over the broadcast; the warmup
# must compile that specialization (non-None draft pointers) too.
assert broadcast_drafts.shape == (1, 2) and broadcast_drafts.dtype == torch.int64
assert draft_tokens_out is runner.req_states.draft_tokens
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA stream")
def test_broadcast_pads_plain_sampler_rows_to_max_sample_len(monkeypatch):
"""The wire shape must not depend on whether the batch carried drafts:
the receiver always allocates [num_reqs, max_sample_len], and a NCCL
broadcast with mismatched counts hangs the receiver."""
sent = []
monkeypatch.setattr(
pp_utils.torch.distributed,
"broadcast",
lambda tensor, **kw: sent.append(tensor),
)
handler = _cuda_handler()
batch = _batch(num_computed=[10], prefill_len=[8], num_scheduled=[1])
handler.broadcast(
torch.zeros(1, 1, dtype=torch.int64, device="cuda"), # plain sampler
torch.ones(1, dtype=torch.int32, device="cuda"),
torch.zeros(1, dtype=torch.int32, device="cuda"),
batch,
)
assert sent[0].shape == (1, 6)
assert sent[1].shape == (2, 4)
torch.accelerator.synchronize()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA stream")
def test_broadcast_drafts_gathers_fresh_rows_from_the_table(monkeypatch):
sent = []
monkeypatch.setattr(
pp_utils.torch.distributed,
"broadcast",
lambda tensor, **kw: sent.append(tensor),
)
handler = _cuda_handler()
# The batch's single row maps to request-state row 2.
batch = _batch(num_computed=[10], prefill_len=[8], num_scheduled=[1])
batch.idx_mapping = torch.tensor([2], dtype=torch.int64, device="cuda")
table = torch.arange(20, dtype=torch.int64, device="cuda").view(4, 5)
handler.broadcast_drafts(table, batch)
assert sent[0].shape == (1, 5)
# The payload is a gather into a fresh tensor: propose() overwrites its
# persistent buffer on the next step, possibly before this send completes.
assert sent[0].data_ptr() != table.data_ptr()
assert sent[0].cpu().tolist() == [table[2].cpu().tolist()]
# An all-prefill batch sends nothing (receive() enqueues nothing either).
sent.clear()
prefill_batch = _batch(num_computed=[0], prefill_len=[4096], num_scheduled=[448])
prefill_batch.idx_mapping = prefill_batch.idx_mapping.cuda()
handler.broadcast_drafts(table, prefill_batch)
assert sent == []
torch.accelerator.synchronize()