[Spec Decode][Perf] Replicate DSpark Markov head across TP ranks (#49731)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a0c092ee72
commit
d6247d7173
@@ -24,7 +24,6 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model
|
||||
@@ -40,6 +39,10 @@ class DSparkMarkovHead(nn.Module):
|
||||
``vocab_size``); ``markov_w2`` projects it to a draft-vocab bias
|
||||
(``draft_vocab_size``) added to the base draft logits. The two sizes
|
||||
coincide for full-vocab drafts.
|
||||
|
||||
Both weights are replicated because the head runs sequentially for every
|
||||
draft position. Sharding them would add an all-reduce and a full-vocab
|
||||
gather to each position.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -50,19 +53,24 @@ class DSparkMarkovHead(nn.Module):
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# TODO(ben): profile for which (if any) it makes sense to replicate or TP-shard
|
||||
self.markov_w1 = VocabParallelEmbedding(
|
||||
vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w1")
|
||||
)
|
||||
self.markov_w1 = nn.Embedding(vocab_size, markov_rank)
|
||||
self.markov_w2 = ParallelLMHead(
|
||||
draft_vocab_size, markov_rank, prefix=maybe_prefix(prefix, "markov_w2")
|
||||
draft_vocab_size,
|
||||
markov_rank,
|
||||
bias=False,
|
||||
prefix=maybe_prefix(prefix, "markov_w2"),
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
def embed(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""r-dim Markov embedding of ``token_ids`` ([B] -> [B, r])."""
|
||||
return self.markov_w1(token_ids)
|
||||
|
||||
def bias(self, markov_embed: torch.Tensor, logits_processor) -> torch.Tensor:
|
||||
def bias(
|
||||
self,
|
||||
markov_embed: torch.Tensor,
|
||||
logits_processor: LogitsProcessor,
|
||||
) -> torch.Tensor:
|
||||
"""Vocab-size transition bias from a Markov embedding ([B, r] -> [B, V])."""
|
||||
return logits_processor(self.markov_w2, markov_embed)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user