[Spec Decode][Perf] Replicate DSpark Markov head across TP ranks (#49731)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Michael Goin
2026-07-29 11:43:51 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent a0c092ee72
commit d6247d7173
4 changed files with 94 additions and 23 deletions
+52 -8
View File
@@ -14,6 +14,7 @@ import torch
from vllm import LLM, SamplingParams
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
)
@@ -28,6 +29,7 @@ class _FakeLmHead:
self.weight = weight
self.quant_method = object() if quantized else UnquantizedEmbeddingMethod()
self.shard_indices = shard_indices
self.tp_size = 1
def _build_processor(vocab_size: int) -> LogitsProcessor:
@@ -135,11 +137,59 @@ def test_fp32_head_rejects_quantized_lm_head(default_vllm_config):
lp._get_logits(torch.randn(4, 16, dtype=torch.bfloat16), lm_head, None)
def test_replicated_lm_head_skips_tp_communication_and_preserves_processing(
default_vllm_config,
):
from unittest import mock
vocab_size, hidden_size = 12, 8
soft_cap, scale = 2.0, 0.5
lp = LogitsProcessor(
vocab_size,
soft_cap=soft_cap,
scale=scale,
)
lp.head_dtype = torch.float32
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
world_size_getter = (
"vllm.model_executor.layers.vocab_parallel_embedding."
"get_tensor_model_parallel_world_size"
)
with mock.patch(world_size_getter, return_value=2):
lm_head = ParallelLMHead(
vocab_size,
hidden_size,
params_dtype=torch.bfloat16,
disable_tp=True,
)
lm_head.weight_loader(lm_head.weight, weight)
assert lm_head.tp_size == 1
with mock.patch.object(lp, "_gather_logits") as gather_mock:
logits = lp(lm_head, hidden_states)
gather_mock.assert_not_called()
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
expected = torch.tanh(expected / soft_cap) * soft_cap * scale
torch.testing.assert_close(logits, expected)
all_gather_path = (
"vllm.model_executor.layers.logits_processor.tensor_model_parallel_all_gather"
)
with mock.patch(all_gather_path) as all_gather:
top = lp.get_top_tokens(lm_head, hidden_states)
all_gather.assert_not_called()
assert torch.equal(top, expected.argmax(dim=-1))
def test_get_top_tokens_honors_head_dtype(default_vllm_config):
# The spec-decode local-argmax path (get_top_tokens) must run the lm_head
# in head_dtype too, not just _get_logits.
import types
from unittest import mock
vocab_size, hidden_size = 64, 16
lp = _build_processor(vocab_size)
@@ -154,13 +204,7 @@ def test_get_top_tokens_honors_head_dtype(default_vllm_config):
),
)
with mock.patch(
"vllm.model_executor.layers.logits_processor."
"get_tensor_model_parallel_world_size",
return_value=1,
):
top = lp.get_top_tokens(lm_head, hidden_states, None)
top = lp.get_top_tokens(lm_head, hidden_states, None)
expected = torch.nn.functional.linear(hidden_states.float(), weight.float()).argmax(
dim=-1
)