[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)
Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nick Hill
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9a35c081e8
commit
6fe67cbbf3
@@ -501,7 +501,40 @@ def _build_topk_mask(
|
||||
return out[:batch_size, :max_q_len]
|
||||
|
||||
|
||||
class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
class SharedTopkIndicesBuffer:
|
||||
"""Resolves the shared top-k index buffer for sparse MLA implementations.
|
||||
|
||||
The indexer owns the buffer, but `LLMBaseProposer.load_model` repoints the
|
||||
draft's indexers at the target's buffer after the impls are constructed, so
|
||||
it must be resolved per read rather than snapshotted. Backbone skip-topk
|
||||
layers have no indexer and pass the buffer explicitly.
|
||||
"""
|
||||
|
||||
_indexer: object | None = None
|
||||
_topk_indices_buffer: torch.Tensor | None = None
|
||||
|
||||
def init_topk_indices_buffer(
|
||||
self,
|
||||
indexer: object | None,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
) -> None:
|
||||
self._indexer = indexer
|
||||
self._topk_indices_buffer = topk_indices_buffer
|
||||
|
||||
@property
|
||||
def topk_indices_buffer(self) -> torch.Tensor | None:
|
||||
if self._indexer is not None:
|
||||
return self._indexer.topk_indices_buffer # type: ignore[attr-defined]
|
||||
return self._topk_indices_buffer
|
||||
|
||||
@topk_indices_buffer.setter
|
||||
def topk_indices_buffer(self, buffer: torch.Tensor | None) -> None:
|
||||
# An explicit assignment supersedes the indexer.
|
||||
self._indexer = None
|
||||
self._topk_indices_buffer = buffer
|
||||
|
||||
|
||||
class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic[T]):
|
||||
"""Sparse MLA base with dense and masked-MHA prefill paths."""
|
||||
|
||||
is_sparse = True
|
||||
@@ -543,14 +576,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
kv_b_proj,
|
||||
)
|
||||
|
||||
# The indexer carries the shared buffer for normal layers and tests;
|
||||
# the explicitly-passed buffer covers backbone skip layers, whose
|
||||
# indexer is not constructed (see deepseek_v2.py).
|
||||
self.topk_indices_buffer: torch.Tensor | None = (
|
||||
indexer.topk_indices_buffer # type: ignore[attr-defined]
|
||||
if indexer is not None
|
||||
else topk_indices_buffer
|
||||
)
|
||||
self.init_topk_indices_buffer(indexer, topk_indices_buffer)
|
||||
self._use_flashinfer_concat_mla_k = (
|
||||
has_flashinfer()
|
||||
and which("ninja") is not None
|
||||
|
||||
Reference in New Issue
Block a user