[Spec Decode][Perf] Replicate DSpark Markov head across TP ranks (#49731)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Michael Goin
2026-07-29 11:43:51 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent a0c092ee72
commit d6247d7173
4 changed files with 94 additions and 23 deletions
+15 -7
View File
@@ -24,7 +24,6 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model
@@ -40,6 +39,10 @@ class DSparkMarkovHead(nn.Module):
``vocab_size``); ``markov_w2`` projects it to a draft-vocab bias
(``draft_vocab_size``) added to the base draft logits. The two sizes
coincide for full-vocab drafts.
Both weights are replicated because the head runs sequentially for every
draft position. Sharding them would add an all-reduce and a full-vocab
gather to each position.
"""
def __init__(
@@ -50,19 +53,24 @@ class DSparkMarkovHead(nn.Module):
prefix: str,
) -> None:
super().__init__()
# TODO(ben): profile for which (if any) it makes sense to replicate or TP-shard
self.markov_w1 = VocabParallelEmbedding(
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w1")
)
self.markov_w1 = nn.Embedding(vocab_size, markov_rank)
self.markov_w2 = ParallelLMHead(
draft_vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
draft_vocab_size,
markov_rank,
bias=False,
prefix=maybe_prefix(prefix, "markov_w2"),
disable_tp=True,
)
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
"""r-dim Markov embedding of ``token_ids`` ([B] -> [B, r])."""
return self.markov_w1(token_ids)
def bias(self, markov_embed: torch.Tensor, logits_processor) -> torch.Tensor:
def bias(
self,
markov_embed: torch.Tensor,
logits_processor: LogitsProcessor,
) -> torch.Tensor:
"""Vocab-size transition bias from a Markov embedding ([B, r] -> [B, V])."""
return logits_processor(self.markov_w2, markov_embed)