[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)
Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nick Hill
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9a35c081e8
commit
6fe67cbbf3
@@ -24,6 +24,7 @@ from vllm.config import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES,
|
||||
SparseMLACommonImpl,
|
||||
_masked_mha_workspace_fits,
|
||||
_topk_mask_shape,
|
||||
)
|
||||
@@ -44,6 +45,9 @@ import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as flashinfer_sparse
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
SharedTopkIndicesBuffer,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseImpl,
|
||||
@@ -1864,3 +1868,44 @@ def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
|
||||
assert out.is_contiguous()
|
||||
assert not out.isnan().any()
|
||||
assert not lse.isnan().any()
|
||||
|
||||
|
||||
def test_sparse_impl_observes_repointed_indexer_buffer():
|
||||
"""The MTP proposer repoints the draft's indexer at the target model's buffer
|
||||
after the backend impl is built, so the impl must resolve the buffer per read.
|
||||
Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
own = torch.zeros(4, 8, dtype=torch.int32)
|
||||
target = torch.ones(4, 8, dtype=torch.int32)
|
||||
indexer = SimpleNamespace(topk_indices_buffer=own)
|
||||
impl.init_topk_indices_buffer(indexer, None)
|
||||
|
||||
assert impl.topk_indices_buffer is own
|
||||
|
||||
indexer.topk_indices_buffer = target
|
||||
|
||||
assert impl.topk_indices_buffer is target
|
||||
|
||||
|
||||
def test_explicit_topk_buffer_supersedes_indexer():
|
||||
"""Backbone skip-topk layers have no indexer, and the proposer also assigns the
|
||||
shared buffer directly onto draft submodules."""
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
indexer = SimpleNamespace(
|
||||
topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
|
||||
)
|
||||
impl.init_topk_indices_buffer(indexer, None)
|
||||
explicit = torch.ones(2, 2, dtype=torch.int32)
|
||||
|
||||
impl.topk_indices_buffer = explicit
|
||||
|
||||
assert impl.topk_indices_buffer is explicit
|
||||
|
||||
|
||||
def test_sparse_mla_common_impl_resolves_buffer_lazily():
|
||||
"""Guards the whole SparseMLACommonImpl family at once: a backend that
|
||||
snapshots the buffer instead silently loses MTP buffer sharing."""
|
||||
assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
|
||||
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
|
||||
"topk_indices_buffer must stay a lazily-resolved property"
|
||||
)
|
||||
|
||||
@@ -56,14 +56,7 @@ INLINE_CONFIGS = [
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
),
|
||||
InlineConfig(
|
||||
id="pp2",
|
||||
pp_size=2,
|
||||
skip_reason=(
|
||||
"DeepSeek MTP pipeline-parallel support is in flight upstream; "
|
||||
"see https://github.com/vllm-project/vllm/pull/44698"
|
||||
),
|
||||
),
|
||||
InlineConfig(id="pp2", pp_size=2),
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user