[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)

Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chris Eastwood
2026-09-11 17:55:21 +00:00
committed by GitHub
co-authored by Nick Hill mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9a35c081e8
commit 6fe67cbbf3
12 changed files with 167 additions and 49 deletions
@@ -501,7 +501,40 @@ def _build_topk_mask(
return out[:batch_size, :max_q_len]
class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
class SharedTopkIndicesBuffer:
"""Resolves the shared top-k index buffer for sparse MLA implementations.
The indexer owns the buffer, but `LLMBaseProposer.load_model` repoints the
draft's indexers at the target's buffer after the impls are constructed, so
it must be resolved per read rather than snapshotted. Backbone skip-topk
layers have no indexer and pass the buffer explicitly.
"""
_indexer: object | None = None
_topk_indices_buffer: torch.Tensor | None = None
def init_topk_indices_buffer(
self,
indexer: object | None,
topk_indices_buffer: torch.Tensor | None,
) -> None:
self._indexer = indexer
self._topk_indices_buffer = topk_indices_buffer
@property
def topk_indices_buffer(self) -> torch.Tensor | None:
if self._indexer is not None:
return self._indexer.topk_indices_buffer # type: ignore[attr-defined]
return self._topk_indices_buffer
@topk_indices_buffer.setter
def topk_indices_buffer(self, buffer: torch.Tensor | None) -> None:
# An explicit assignment supersedes the indexer.
self._indexer = None
self._topk_indices_buffer = buffer
class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic[T]):
"""Sparse MLA base with dense and masked-MHA prefill paths."""
is_sparse = True
@@ -543,14 +576,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
kv_b_proj,
)
# The indexer carries the shared buffer for normal layers and tests;
# the explicitly-passed buffer covers backbone skip layers, whose
# indexer is not constructed (see deepseek_v2.py).
self.topk_indices_buffer: torch.Tensor | None = (
indexer.topk_indices_buffer # type: ignore[attr-defined]
if indexer is not None
else topk_indices_buffer
)
self.init_topk_indices_buffer(indexer, topk_indices_buffer)
self._use_flashinfer_concat_mla_k = (
has_flashinfer()
and which("ninja") is not None