[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)
Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nick Hill
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9a35c081e8
commit
6fe67cbbf3
@@ -182,7 +182,8 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
# DSA is always sparse (has index_topk); allocate the shared top-k
|
||||
# buffer the indexer writes and the sparse MLA backend reads.
|
||||
self.is_v32 = True
|
||||
topk_indices_buffer = torch.empty(
|
||||
# On the model, not a local: the MTP proposer shares it with the draft.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
@@ -207,7 +208,7 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
lambda prefix: DeepseekV32DecoderLayer(
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user