[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)

Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chris Eastwood
2026-09-11 17:55:21 +00:00
committed by GitHub
co-authored by Nick Hill mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9a35c081e8
commit 6fe67cbbf3
12 changed files with 167 additions and 49 deletions
@@ -24,6 +24,7 @@ from vllm.config import set_current_vllm_config
from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha
from vllm.model_executor.layers.attention.sparse_mla_attention import (
GLOBAL_TOPK_MASK_MAX_BYTES,
SparseMLACommonImpl,
_masked_mha_workspace_fits,
_topk_mask_shape,
)
@@ -44,6 +45,9 @@ import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as flashinfer_sparse
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
from vllm.model_executor.layers.attention.sparse_mla_attention import (
SharedTopkIndicesBuffer,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
@@ -1864,3 +1868,44 @@ def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
assert out.is_contiguous()
assert not out.isnan().any()
assert not lse.isnan().any()
def test_sparse_impl_observes_repointed_indexer_buffer():
"""The MTP proposer repoints the draft's indexer at the target model's buffer
after the backend impl is built, so the impl must resolve the buffer per read.
Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
impl = object.__new__(FlashInferMLASparseImpl)
own = torch.zeros(4, 8, dtype=torch.int32)
target = torch.ones(4, 8, dtype=torch.int32)
indexer = SimpleNamespace(topk_indices_buffer=own)
impl.init_topk_indices_buffer(indexer, None)
assert impl.topk_indices_buffer is own
indexer.topk_indices_buffer = target
assert impl.topk_indices_buffer is target
def test_explicit_topk_buffer_supersedes_indexer():
"""Backbone skip-topk layers have no indexer, and the proposer also assigns the
shared buffer directly onto draft submodules."""
impl = object.__new__(FlashInferMLASparseImpl)
indexer = SimpleNamespace(
topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
)
impl.init_topk_indices_buffer(indexer, None)
explicit = torch.ones(2, 2, dtype=torch.int32)
impl.topk_indices_buffer = explicit
assert impl.topk_indices_buffer is explicit
def test_sparse_mla_common_impl_resolves_buffer_lazily():
"""Guards the whole SparseMLACommonImpl family at once: a backend that
snapshots the buffer instead silently loses MTP buffer sharing."""
assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
"topk_indices_buffer must stay a lazily-resolved property"
)
@@ -56,14 +56,7 @@ INLINE_CONFIGS = [
enable_expert_parallel=True,
enable_eplb=True,
),
InlineConfig(
id="pp2",
pp_size=2,
skip_reason=(
"DeepSeek MTP pipeline-parallel support is in flight upstream; "
"see https://github.com/vllm-project/vllm/pull/44698"
),
),
InlineConfig(id="pp2", pp_size=2),
]