[Spec Decode] DFlash2: local convolution + candidate selector (#52816)
This commit is contained in:
@@ -1501,6 +1501,14 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env
|
||||
max_num_seqs=32,
|
||||
),
|
||||
"DFlash2DraftModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3.8-27B",
|
||||
speculative_model="z-lab/Qwen3.8-27B-DFlash2",
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True,
|
||||
max_model_len=8192,
|
||||
max_num_seqs=32,
|
||||
),
|
||||
"DFlashLagunaForCausalLM": _HfExamplesInfo(
|
||||
"poolside/Laguna-XS-2.1-NVFP4",
|
||||
speculative_model="poolside/Laguna-XS-2.1-DFlash-NVFP4",
|
||||
|
||||
@@ -199,6 +199,7 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
|
||||
),
|
||||
)
|
||||
config._dflash_needs_multi_kv_group = lambda: False
|
||||
config._is_dflash2_draft = lambda: False
|
||||
config._is_default_v2_model_runner_model = lambda: (
|
||||
VllmConfig._is_default_v2_model_runner_model(config)
|
||||
)
|
||||
@@ -292,6 +293,29 @@ def test_v2_model_runner_supports_extract_hidden_states():
|
||||
assert config._get_v2_model_runner_unsupported_features() == []
|
||||
|
||||
|
||||
def test_dflash2_draft_forces_v2_model_runner():
|
||||
"""A DFlash2 draft must reach the V2 speculator, the only one that runs its
|
||||
candidate selector; on V1 it would draft as DFlash1 without raising."""
|
||||
|
||||
def config(method, architectures):
|
||||
return SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
method=method,
|
||||
draft_model_config=SimpleNamespace(architectures=architectures),
|
||||
)
|
||||
)
|
||||
|
||||
assert VllmConfig._is_dflash2_draft(config("dflash", ["DFlash2DraftModel"]))
|
||||
assert not VllmConfig._is_dflash2_draft(config("dflash", ["DFlashDraftModel"]))
|
||||
assert not VllmConfig._is_dflash2_draft(config("eagle", ["DFlash2DraftModel"]))
|
||||
assert not VllmConfig._is_dflash2_draft(SimpleNamespace(speculative_config=None))
|
||||
assert not VllmConfig._is_dflash2_draft(
|
||||
SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(method="dflash", draft_model_config=None)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_v2_model_runner", "expected_capture_sizes"),
|
||||
[
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [5, 8])
|
||||
def test_grouped_conv_matches_reference(block_size: int):
|
||||
torch.manual_seed(0)
|
||||
batch, taps, num_groups, group_size = 3, 3, 4, 2
|
||||
hidden = torch.randn(batch * block_size, num_groups * group_size)
|
||||
delta = torch.randn(batch * block_size, taps, num_groups)
|
||||
base = torch.randn(taps, num_groups * group_size)
|
||||
|
||||
actual = _grouped_conv(
|
||||
hidden, delta, base, block_size, num_groups, group_size, taps
|
||||
)
|
||||
hidden_blocks = hidden.view(batch, block_size, num_groups, group_size)
|
||||
expected = torch.zeros_like(hidden_blocks)
|
||||
base = base.view(taps, num_groups, group_size)
|
||||
delta = delta.view(batch, block_size, taps, num_groups)
|
||||
for position in range(block_size):
|
||||
for tap in range(min(taps, position + 1)):
|
||||
expected[:, position] += (
|
||||
base[tap] + delta[:, position, tap, :, None]
|
||||
) * hidden_blocks[:, position - tap]
|
||||
|
||||
torch.testing.assert_close(actual, expected.flatten(0, 1).flatten(-2))
|
||||
|
||||
|
||||
def test_selector_edges_match_sequential_reference():
|
||||
torch.manual_seed(1)
|
||||
batch, steps, top_k, rank = 2, 4, 3, 5
|
||||
vocab = 17
|
||||
predecessors = torch.randn(vocab, rank)
|
||||
successors = torch.randn(vocab, rank)
|
||||
candidate_ids = torch.randint(vocab, (batch, steps, top_k))
|
||||
unary = torch.randn(batch, steps, top_k)
|
||||
hidden = torch.randn(batch, steps, rank)
|
||||
anchors = torch.randint(vocab, (batch,))
|
||||
|
||||
actual = _score_edges(
|
||||
predecessors,
|
||||
successors,
|
||||
candidate_ids,
|
||||
unary,
|
||||
hidden,
|
||||
anchors,
|
||||
top_k,
|
||||
)
|
||||
expected = torch.empty_like(actual)
|
||||
for step in range(steps):
|
||||
pred = (
|
||||
anchors[:, None].expand(-1, top_k)
|
||||
if step == 0
|
||||
else candidate_ids[:, step - 1]
|
||||
)
|
||||
expected[:, step] = unary[:, step, None] + torch.einsum(
|
||||
"bpr,bcr->bpc",
|
||||
predecessors[pred] * hidden[:, step, None],
|
||||
successors[candidate_ids[:, step]],
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def _stub_base(monkeypatch, draft_logits):
|
||||
"""A DFlashSpeculator.__init__ that allocates only what the base class would.
|
||||
|
||||
The real base class fills draft_logits from draft_logits_spec, so callers
|
||||
pass a tensor already in that state.
|
||||
"""
|
||||
|
||||
def init_base(self, _vllm_config, device):
|
||||
self.draft_model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(dflash_config={"selector_top_k": 3})
|
||||
)
|
||||
self.max_num_reqs = 2
|
||||
self.num_query_per_req = 5
|
||||
self.num_speculative_steps = 4
|
||||
self.vocab_size = 17
|
||||
self.draft_tokens = torch.empty((2, 4), dtype=torch.int64, device=device)
|
||||
self.draft_logits = draft_logits
|
||||
|
||||
monkeypatch.setattr(DFlashSpeculator, "__init__", init_base)
|
||||
|
||||
|
||||
def test_selector_leaves_greedy_drafting_without_proposal_logits(monkeypatch):
|
||||
"""Greedy is the default, and it caches no proposal distribution.
|
||||
|
||||
The base class allocates draft_logits only for "probabilistic"; verification
|
||||
reads `draft_logits is None` to decide whether a distribution is on offer, so
|
||||
allocating one here would claim a proposal the walk never sampled from.
|
||||
"""
|
||||
_stub_base(monkeypatch, None)
|
||||
speculator = DFlash2Speculator(None, torch.device("cpu"))
|
||||
|
||||
assert speculator.draft_logits is None
|
||||
|
||||
|
||||
def test_selector_asks_for_fp32_proposal_logits():
|
||||
"""The spec the base class allocates from: fp32, filled -inf.
|
||||
|
||||
Not the head dtype -- rounding selector scores to bf16 moves the argmax of a
|
||||
candidate row often enough that the walk and the rejection sampler checking it
|
||||
would no longer read the same distribution.
|
||||
"""
|
||||
dtype, fill = DFlash2Speculator.draft_logits_spec(None, None)
|
||||
|
||||
assert dtype is torch.float32
|
||||
assert fill == float("-inf")
|
||||
@@ -21,12 +21,13 @@ from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
)
|
||||
|
||||
|
||||
def _config(num_hidden_layers, layer_types=None, causal_override=None):
|
||||
def _config(num_hidden_layers, layer_types=None, causal_override=None, is_causal=None):
|
||||
dflash_config = None if causal_override is None else {"causal": causal_override}
|
||||
return SimpleNamespace(
|
||||
num_hidden_layers=num_hidden_layers,
|
||||
layer_types=layer_types,
|
||||
dflash_config=dflash_config,
|
||||
is_causal=is_causal,
|
||||
)
|
||||
|
||||
|
||||
@@ -40,6 +41,19 @@ def _config(num_hidden_layers, layer_types=None, causal_override=None):
|
||||
_config(2, layer_types=["sliding_attention"] * 2, causal_override=False),
|
||||
True,
|
||||
),
|
||||
# DFlash2 stores the explicit attention semantics at the top level.
|
||||
(
|
||||
_config(
|
||||
2,
|
||||
layer_types=["sliding_attention"] * 2,
|
||||
is_causal=False,
|
||||
),
|
||||
True,
|
||||
),
|
||||
(
|
||||
_config(2, layer_types=["full_attention"] * 2, is_causal=True),
|
||||
False,
|
||||
),
|
||||
# SWA-derived: full-attention layers are non-causal.
|
||||
(_config(2, layer_types=["sliding_attention", "full_attention"]), True),
|
||||
# SWA-derived: all-sliding is fully causal.
|
||||
@@ -59,6 +73,16 @@ def test_dflash_layer_causal_is_per_layer():
|
||||
assert _dflash_layer_causal(config, 1) is False
|
||||
|
||||
|
||||
def test_dflash_layer_causal_honors_top_level_override():
|
||||
config = _config(
|
||||
2,
|
||||
layer_types=["sliding_attention", "full_attention"],
|
||||
is_causal=False,
|
||||
)
|
||||
assert _dflash_layer_causal(config, 0) is False
|
||||
assert _dflash_layer_causal(config, 1) is False
|
||||
|
||||
|
||||
def _vllm_config(**draft_config):
|
||||
config = SimpleNamespace(**draft_config)
|
||||
return SimpleNamespace(
|
||||
|
||||
@@ -670,6 +670,12 @@ class VllmConfig:
|
||||
if self._dflash_needs_multi_kv_group():
|
||||
return True
|
||||
|
||||
# The DFlash2 candidate selector exists only in the V2 speculator. On V1
|
||||
# the same checkpoint drafts through DFlashProposer, which never calls
|
||||
# it, so the draft degrades to DFlash1 silently. Force V2 as for dspark.
|
||||
if self._is_dflash2_draft():
|
||||
return True
|
||||
|
||||
if self.model_config is not None and self.model_config.is_diffusion:
|
||||
return True
|
||||
|
||||
@@ -693,6 +699,17 @@ class VllmConfig:
|
||||
|
||||
return True
|
||||
|
||||
def _is_dflash2_draft(self) -> bool:
|
||||
"""Whether the DFlash draft is a DFlash2 one, by the architecture the
|
||||
speculator selects on (v1/worker/gpu/spec_decode/__init__.py)."""
|
||||
spec = self.speculative_config
|
||||
if spec is None or spec.method != "dflash":
|
||||
return False
|
||||
draft_config = getattr(spec, "draft_model_config", None)
|
||||
if draft_config is None:
|
||||
return False
|
||||
return "DFlash2DraftModel" in (draft_config.architectures or [])
|
||||
|
||||
def _dflash_needs_multi_kv_group(self) -> bool:
|
||||
"""Whether a DFlash draft mixes sliding-window and full attention."""
|
||||
spec = self.speculative_config
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""A layer that compute logits from hidden_stats."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
@@ -10,12 +13,43 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_gather,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@cache
|
||||
def _flashinfer_topk() -> Callable[..., tuple[torch.Tensor, torch.Tensor]] | None:
|
||||
"""FlashInfer's radix top-k, or None for torch.topk.
|
||||
|
||||
The top-k spans the vocabulary, where the radix kernel is about twice
|
||||
torch.topk.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
return None
|
||||
if not has_flashinfer():
|
||||
logger.info_once(
|
||||
"flashinfer is unavailable; vocab-parallel top-k uses torch.topk, "
|
||||
"at roughly half the speed."
|
||||
)
|
||||
return None
|
||||
from flashinfer import top_k
|
||||
|
||||
return top_k
|
||||
|
||||
|
||||
def _topk(scores: torch.Tensor, k: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
impl = _flashinfer_topk()
|
||||
if impl is None or not scores.is_cuda:
|
||||
return torch.topk(scores, k, dim=-1)
|
||||
return impl(scores, k, sorted=True, deterministic=True)
|
||||
|
||||
|
||||
# --8<-- [start:logits_processor]
|
||||
@@ -204,6 +238,53 @@ class LogitsProcessor(PluggableLayer):
|
||||
top_tokens = gathered[:, :, 1].gather(dim=-1, index=max_rank_idx)
|
||||
return top_tokens.squeeze(-1).to(torch.int64)
|
||||
|
||||
def get_top_k_tokens(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
k: int,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Vocab-parallel top-k without all-gathering full logits.
|
||||
|
||||
The `get_top_tokens` reduction widened from one token to k, returning
|
||||
the values as well as the global ids. Communication is
|
||||
O(batch * 2k * tp_size) rather than O(batch * vocab_size).
|
||||
|
||||
Scale and soft cap are applied to the k selected values rather than
|
||||
the whole vocabulary; both are monotonic, so the selection is the same
|
||||
and only k entries are touched.
|
||||
"""
|
||||
if self.scale <= 0.0 and self.scale != 1.0:
|
||||
raise ValueError(
|
||||
"The local top-k reduction optimization is not supported for "
|
||||
"non-positive logit scaling factors."
|
||||
)
|
||||
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
|
||||
# Mask out padding entries beyond org_vocab_size on this shard.
|
||||
num_pad = lm_head.shard_indices.num_org_vocab_padding
|
||||
if num_pad > 0:
|
||||
logits[..., -num_pad:] = -float("inf")
|
||||
|
||||
values, ids = _topk(logits, k)
|
||||
# Convert shard-local indices to global vocab indices.
|
||||
ids = ids.to(torch.int64) + lm_head.shard_indices.org_vocab_start_index
|
||||
|
||||
if lm_head.tp_size > 1:
|
||||
values = tensor_model_parallel_all_gather(values, dim=-1)
|
||||
ids = tensor_model_parallel_all_gather(ids, dim=-1)
|
||||
values, selected = _topk(values, k)
|
||||
ids = ids.gather(-1, selected)
|
||||
|
||||
values = values.float()
|
||||
if self.scale != 1.0:
|
||||
values = values * self.scale
|
||||
if self.soft_cap is not None:
|
||||
values = torch.tanh(values / self.soft_cap) * self.soft_cap
|
||||
return ids, values
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"vocab_size={self.vocab_size}"
|
||||
s += f", org_vocab_size={self.org_vocab_size}"
|
||||
|
||||
@@ -56,10 +56,13 @@ _SLIDING_ATTENTION = "sliding_attention"
|
||||
|
||||
|
||||
def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
|
||||
"""``dflash_config.causal`` overrides all layers; else only SWA layers causal."""
|
||||
"""Resolve explicit causality before falling back to legacy layer defaults."""
|
||||
is_causal = getattr(config, "is_causal", None)
|
||||
if is_causal is not None:
|
||||
return bool(is_causal)
|
||||
override = (getattr(config, "dflash_config", None) or {}).get("causal")
|
||||
if override is not None:
|
||||
return override
|
||||
return bool(override)
|
||||
layer_types = getattr(config, "layer_types", None)
|
||||
return bool(layer_types) and layer_types[layer_idx] == _SLIDING_ATTENTION
|
||||
|
||||
@@ -374,6 +377,8 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class DFlashQwen3Model(nn.Module):
|
||||
decoder_layer_cls = DFlashQwen3DecoderLayer
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"midlayer.": "layers.0.",
|
||||
@@ -434,7 +439,7 @@ class DFlashQwen3Model(nn.Module):
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DFlashQwen3DecoderLayer(
|
||||
self.decoder_layer_cls(
|
||||
current_vllm_config,
|
||||
config=self.config,
|
||||
layer_idx=layer_idx,
|
||||
@@ -699,6 +704,8 @@ class DFlashQwen3Model(nn.Module):
|
||||
|
||||
|
||||
class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
model_cls = DFlashQwen3Model
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
@@ -708,7 +715,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
self.model = DFlashQwen3Model(
|
||||
self.model = self.model_cls(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
start_layer_id=target_layer_num,
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
from .qwen3_dflash import (
|
||||
DFlashQwen3DecoderLayer,
|
||||
DFlashQwen3ForCausalLM,
|
||||
DFlashQwen3Model,
|
||||
)
|
||||
from .utils import maybe_prefix
|
||||
|
||||
|
||||
def _grouped_conv(
|
||||
hidden_states: torch.Tensor,
|
||||
delta: torch.Tensor,
|
||||
base: torch.Tensor,
|
||||
block_size: int,
|
||||
num_groups: int,
|
||||
group_size: int,
|
||||
taps: int,
|
||||
) -> torch.Tensor:
|
||||
blocks = hidden_states.unflatten(-1, (num_groups, group_size))
|
||||
coefficients = base.view(1, taps, num_groups, group_size) + delta.unsqueeze(-1)
|
||||
output = coefficients[:, 0] * blocks
|
||||
position = torch.arange(hidden_states.shape[0], device=hidden_states.device)
|
||||
if block_size & (block_size - 1) == 0:
|
||||
position = position & (block_size - 1)
|
||||
else:
|
||||
position = position % block_size
|
||||
for tap in range(1, taps):
|
||||
shifted = F.pad(blocks[:-tap], (0, 0, 0, 0, tap, 0))
|
||||
output += coefficients[:, tap] * shifted * (position >= tap).view(-1, 1, 1)
|
||||
return output.flatten(-2)
|
||||
|
||||
|
||||
class DFlashGroupedConv(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
taps: int,
|
||||
group_size: int,
|
||||
block_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if hidden_size % group_size:
|
||||
raise ValueError(
|
||||
f"conv_group_size={group_size} must divide hidden_size={hidden_size}."
|
||||
)
|
||||
self.block_size = block_size
|
||||
self.taps = taps
|
||||
self.group_size = group_size
|
||||
self.num_groups = hidden_size // group_size
|
||||
self.base_kernel = nn.Parameter(
|
||||
torch.empty(2, taps, hidden_size, dtype=params_dtype),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.kernel_projection = ReplicatedLinear(
|
||||
hidden_size,
|
||||
2 * taps * self.num_groups,
|
||||
bias=False,
|
||||
params_dtype=params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "kernel_projection"),
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def _convolve(
|
||||
self, hidden_states: torch.Tensor, delta: torch.Tensor, side: int
|
||||
) -> torch.Tensor:
|
||||
return _grouped_conv(
|
||||
hidden_states,
|
||||
delta,
|
||||
self.base_kernel[side],
|
||||
self.block_size,
|
||||
self.num_groups,
|
||||
self.group_size,
|
||||
self.taps,
|
||||
)
|
||||
|
||||
def prepare(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
coefficients = self.kernel_projection(hidden_states).reshape(
|
||||
hidden_states.shape[0], 2, self.taps, self.num_groups
|
||||
)
|
||||
return self._convolve(hidden_states, coefficients[:, 0], 0), coefficients[:, 1]
|
||||
|
||||
def finish(
|
||||
self, hidden_states: torch.Tensor, coefficients: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return self._convolve(hidden_states, coefficients, 1)
|
||||
|
||||
|
||||
class DFlash2Qwen3DecoderLayer(DFlashQwen3DecoderLayer):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
config,
|
||||
layer_idx: int,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vllm_config,
|
||||
config=config,
|
||||
layer_idx=layer_idx,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
draft_config = config.dflash_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
conv_args = dict(
|
||||
hidden_size=config.hidden_size,
|
||||
taps=int(draft_config["conv_kernel_size"]),
|
||||
group_size=int(draft_config["conv_group_size"]),
|
||||
# Query tokens per request: the bonus token plus the mask tokens.
|
||||
block_size=1 + speculative_config.num_speculative_tokens,
|
||||
params_dtype=vllm_config.model_config.dtype,
|
||||
)
|
||||
self.attention_conv = DFlashGroupedConv(
|
||||
**conv_args, prefix=maybe_prefix(prefix, "attention_conv")
|
||||
)
|
||||
self.mlp_conv = DFlashGroupedConv(
|
||||
**conv_args, prefix=maybe_prefix(prefix, "mlp_conv")
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
hidden_states, coefficients = self.attention_conv.prepare(hidden_states)
|
||||
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
|
||||
hidden_states = self.attention_conv.finish(hidden_states, coefficients)
|
||||
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states, coefficients = self.mlp_conv.prepare(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.mlp_conv.finish(hidden_states, coefficients)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _score_edges(
|
||||
predecessor_table: torch.Tensor,
|
||||
successor_table: torch.Tensor,
|
||||
candidate_ids: torch.Tensor,
|
||||
unary_logits: torch.Tensor,
|
||||
hidden: torch.Tensor,
|
||||
anchor_token_ids: torch.Tensor,
|
||||
top_k: int,
|
||||
) -> torch.Tensor:
|
||||
successors = successor_table[candidate_ids]
|
||||
predecessor_ids = torch.cat(
|
||||
(
|
||||
anchor_token_ids[:, None, None].expand(-1, 1, top_k),
|
||||
candidate_ids[:, :-1],
|
||||
),
|
||||
dim=1,
|
||||
)
|
||||
predecessors = predecessor_table[predecessor_ids]
|
||||
return unary_logits[:, :, None] + torch.einsum(
|
||||
"blpr,blcr->blpc", predecessors * hidden[:, :, None], successors
|
||||
)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class CandidateSelector(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
vocab_size: int,
|
||||
rank: int,
|
||||
top_k: int,
|
||||
params_dtype: torch.dtype,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.top_k = top_k
|
||||
self.predecessor_codebook = nn.Parameter(
|
||||
torch.empty(vocab_size, rank, dtype=params_dtype), requires_grad=False
|
||||
)
|
||||
self.successor_codebook = nn.Parameter(
|
||||
torch.empty(vocab_size, rank, dtype=params_dtype), requires_grad=False
|
||||
)
|
||||
self.hidden_projection = ReplicatedLinear(
|
||||
hidden_size,
|
||||
rank,
|
||||
bias=False,
|
||||
params_dtype=params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "hidden_projection"),
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
unary_logits: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
anchor_token_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden = self.hidden_projection(hidden_states)
|
||||
return _score_edges(
|
||||
self.predecessor_codebook,
|
||||
self.successor_codebook,
|
||||
candidate_ids,
|
||||
unary_logits,
|
||||
hidden,
|
||||
anchor_token_ids,
|
||||
self.top_k,
|
||||
)
|
||||
|
||||
|
||||
class DFlash2Qwen3Model(DFlashQwen3Model):
|
||||
decoder_layer_cls = DFlash2Qwen3DecoderLayer
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
start_layer_id: int = 0,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
start_layer_id=start_layer_id,
|
||||
prefix=prefix,
|
||||
)
|
||||
draft_config = self.config.dflash_config
|
||||
self.input_embedding_scale = float(
|
||||
draft_config.get("input_embedding_scale", 1.0)
|
||||
)
|
||||
# Without its own tag the selector shares the draft head's compile cache.
|
||||
with set_model_tag("dflash2_candidate_selector"):
|
||||
self.candidate_selector = CandidateSelector(
|
||||
hidden_size=self.config.hidden_size,
|
||||
vocab_size=self.config.vocab_size,
|
||||
rank=int(draft_config["selector_rank"]),
|
||||
top_k=int(draft_config["selector_top_k"]),
|
||||
params_dtype=vllm_config.model_config.dtype,
|
||||
prefix=maybe_prefix(prefix, "candidate_selector"),
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return super().embed_input_ids(input_ids) * self.input_embedding_scale
|
||||
|
||||
|
||||
class DFlash2Qwen3ForCausalLM(DFlashQwen3ForCausalLM):
|
||||
model_cls = DFlash2Qwen3Model
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
draft_config = self.config.dflash_config
|
||||
softcap = float(draft_config.get("final_logit_softcapping") or 0.0)
|
||||
self.candidate_logits_processor = LogitsProcessor(
|
||||
vllm_config.model_config.get_vocab_size(),
|
||||
scale=float(draft_config.get("output_multiplier", 1.0)),
|
||||
soft_cap=softcap if softcap > 0 else None,
|
||||
)
|
||||
|
||||
def compute_candidates(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.candidate_logits_processor.get_top_k_tokens(
|
||||
self.lm_head, hidden_states, self.model.candidate_selector.top_k
|
||||
)
|
||||
|
||||
|
||||
EntryClass = DFlash2Qwen3ForCausalLM
|
||||
@@ -632,6 +632,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
|
||||
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
|
||||
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DFlash2DraftModel": ("qwen3_dflash2", "DFlash2Qwen3ForCausalLM"),
|
||||
# Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash
|
||||
# implementation. EAGLEConfig rewrites a dflash draft's architecture to
|
||||
# DFlash{arch} unless it already starts or ends with "DFlash" (see
|
||||
|
||||
@@ -81,6 +81,46 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gumbel_noised_argmax(
|
||||
logits,
|
||||
keys,
|
||||
mask,
|
||||
seed,
|
||||
pos,
|
||||
temp,
|
||||
USE_FP64: tl.constexpr,
|
||||
APPLY_TEMPERATURE: tl.constexpr = True,
|
||||
):
|
||||
"""Argmax of logits under Gumbel-max sampling, or plain argmax at temp 0.
|
||||
|
||||
`keys` indexes the noise, so the same token draws the same noise wherever it
|
||||
appears; `pos` and `seed` place the draw in the request's stream, which is
|
||||
what lets a draft and its verification agree.
|
||||
"""
|
||||
if temp != 0.0 and APPLY_TEMPERATURE:
|
||||
# Match the behavior of _temperature_kernel: if that kernel uses
|
||||
# tl.div_rn, this must too.
|
||||
logits = logits / temp
|
||||
|
||||
# fp32 is the default reduction dtype; fp64 is ~1/32-1/64x the throughput
|
||||
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
|
||||
if USE_FP64:
|
||||
logits = logits.to(tl.float64)
|
||||
if temp != 0.0:
|
||||
gumbel_seed = tl.randint(seed, pos)
|
||||
if USE_FP64:
|
||||
u = tl_rand64(gumbel_seed, keys, includes_zero=False)
|
||||
gumbel_noise = -tl.log(-tl.log(u))
|
||||
else:
|
||||
u = tl_rand32(gumbel_seed, keys, includes_zero=False)
|
||||
# log1p keeps the winning tail at u -> 0, where fp32 resolves it.
|
||||
gumbel_noise = -tl.log(-tldevice.log1p(-u))
|
||||
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
|
||||
|
||||
return tl.max(logits, axis=0, return_indices=True)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gumbel_block_argmax(
|
||||
logits,
|
||||
@@ -124,40 +164,18 @@ def gumbel_block_argmax(
|
||||
mask=mask & is_valid_req,
|
||||
)
|
||||
|
||||
if temp != 0.0 and APPLY_TEMPERATURE:
|
||||
# Apply temperature.
|
||||
# NOTE(woosuk): Match the behavior of _temperature_kernel.
|
||||
# E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too.
|
||||
logits = logits / temp
|
||||
|
||||
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
|
||||
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
|
||||
if USE_FP64:
|
||||
logits = logits.to(tl.float64)
|
||||
if temp != 0.0:
|
||||
# Calculate the seed for gumbel noise.
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + token_idx)
|
||||
gumbel_seed = tl.randint(seed, pos)
|
||||
|
||||
if USE_FP64:
|
||||
u = tl_rand64(gumbel_seed, block, includes_zero=False)
|
||||
gumbel_noise = -tl.log(-tl.log(u))
|
||||
else:
|
||||
u = tl_rand32(gumbel_seed, block, includes_zero=False)
|
||||
# Draw the large-noise tail (which decides the argmax winner) from u -> 0,
|
||||
# where fp32 has fine resolution, instead of u -> 1, where fp32 spacing is
|
||||
# ~2**-24. The naive `-log(-log(u))` puts the winning tail at u -> 1,
|
||||
# hard-capping the noise at ~16.6 and coarsely quantizing it; using
|
||||
# `log1p(-u)` == `log(1 - u)` keeps the tail in the well-resolved region.
|
||||
# Note `1 - u` would lose precision for small u, so `log1p` is required.
|
||||
gumbel_noise = -tl.log(-tldevice.log1p(-u))
|
||||
|
||||
# Apply gumbel noise.
|
||||
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
|
||||
|
||||
value, idx = tl.max(logits, axis=0, return_indices=True)
|
||||
return value, idx
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + token_idx)
|
||||
return gumbel_noised_argmax(
|
||||
logits,
|
||||
block,
|
||||
mask,
|
||||
seed,
|
||||
pos,
|
||||
temp,
|
||||
USE_FP64=USE_FP64,
|
||||
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -15,6 +15,12 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
|
||||
return ExtractHiddenStatesSpeculator(vllm_config, device)
|
||||
elif speculative_config.method == "dflash":
|
||||
if "DFlash2DraftModel" in speculative_config.draft_model_config.architectures:
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import (
|
||||
DFlash2Speculator,
|
||||
)
|
||||
|
||||
return DFlash2Speculator(vllm_config, device)
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
|
||||
DFlashSpeculator,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,215 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_noised_argmax
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _selector_walk_kernel(
|
||||
scores_ptr,
|
||||
candidate_ptr,
|
||||
sample_pos_ptr,
|
||||
req_state_ptr,
|
||||
temperature_ptr,
|
||||
seeds_ptr,
|
||||
tokens_ptr,
|
||||
realized_scores_ptr,
|
||||
num_steps: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
SAMPLE_PROBABILISTIC: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
offsets = tl.arange(0, BLOCK_K)
|
||||
mask = offsets < top_k
|
||||
req_state = tl.load(req_state_ptr + row * num_steps)
|
||||
valid = req_state >= 0
|
||||
temperature = tl.load(temperature_ptr + req_state, mask=valid, other=0.0)
|
||||
seed = tl.load(seeds_ptr + req_state, mask=valid, other=0)
|
||||
previous = 0
|
||||
for step in range(num_steps):
|
||||
flat = row * num_steps + step
|
||||
score_base = (flat * top_k + previous) * top_k
|
||||
scores = tl.load(
|
||||
scores_ptr + score_base + offsets,
|
||||
mask=mask & valid,
|
||||
other=float("-inf"),
|
||||
).to(tl.float64 if USE_FP64 else tl.float32)
|
||||
candidate_base = flat * top_k
|
||||
candidates = tl.load(
|
||||
candidate_ptr + candidate_base + offsets,
|
||||
mask=mask & valid,
|
||||
other=0,
|
||||
)
|
||||
|
||||
# Candidate ids key the noise, matching the target's own sampling.
|
||||
position = tl.load(sample_pos_ptr + flat) - 1
|
||||
_, index = gumbel_noised_argmax(
|
||||
scores,
|
||||
candidates,
|
||||
mask & valid,
|
||||
seed,
|
||||
position,
|
||||
temperature if SAMPLE_PROBABILISTIC else 0.0,
|
||||
USE_FP64=USE_FP64,
|
||||
)
|
||||
|
||||
tl.store(
|
||||
realized_scores_ptr + candidate_base + offsets,
|
||||
scores,
|
||||
mask=mask & valid,
|
||||
)
|
||||
token = tl.load(candidate_ptr + candidate_base + index, mask=valid, other=0)
|
||||
tl.store(tokens_ptr + flat, token, mask=valid)
|
||||
previous = index
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _cache_draft_logits_kernel(
|
||||
draft_logits_ptr,
|
||||
cached_candidate_ptr,
|
||||
candidate_ptr,
|
||||
scores_ptr,
|
||||
req_state_ptr,
|
||||
draft_logits_stride_0,
|
||||
draft_logits_stride_1,
|
||||
num_steps: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
flat = tl.program_id(0)
|
||||
req_state = tl.load(req_state_ptr + flat)
|
||||
step = flat % num_steps
|
||||
offsets = tl.arange(0, BLOCK_K)
|
||||
mask = (req_state >= 0) & (offsets < top_k)
|
||||
candidate_base = flat * top_k
|
||||
cache_base = (req_state * num_steps + step) * top_k
|
||||
old_token_ids = tl.load(cached_candidate_ptr + cache_base + offsets, mask=mask)
|
||||
logits_base = (
|
||||
draft_logits_ptr
|
||||
+ req_state * draft_logits_stride_0
|
||||
+ step * draft_logits_stride_1
|
||||
)
|
||||
tl.store(logits_base + old_token_ids, -float("inf"), mask=mask)
|
||||
token_ids = tl.load(candidate_ptr + candidate_base + offsets, mask=mask)
|
||||
scores = tl.load(scores_ptr + candidate_base + offsets, mask=mask)
|
||||
tl.store(logits_base + token_ids, scores, mask=mask)
|
||||
tl.store(cached_candidate_ptr + cache_base + offsets, token_ids, mask=mask)
|
||||
|
||||
|
||||
class DFlash2Speculator(DFlashSpeculator):
|
||||
_speculator_name = "DFlash2"
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
draft_config = self.draft_model_config.hf_config.dflash_config
|
||||
self.selector_top_k = int(draft_config["selector_top_k"])
|
||||
self._anchor_indices = (
|
||||
torch.arange(self.max_num_reqs, dtype=torch.int64, device=device)
|
||||
* self.num_query_per_req
|
||||
)
|
||||
self._selector_scores = torch.empty(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.selector_top_k,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self._cached_candidate_ids = torch.zeros(
|
||||
self._selector_scores.shape, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
def draft_logits_spec(self, vllm_config: VllmConfig) -> tuple[torch.dtype, float]:
|
||||
# fp32 so the walk and the rejection that checks it read the same
|
||||
# distribution; -inf because the cache kernel writes only the K
|
||||
# candidates.
|
||||
return torch.float32, -float("inf")
|
||||
|
||||
def _sample_path(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
scores: torch.Tensor,
|
||||
num_reqs: int,
|
||||
) -> None:
|
||||
block_k = triton.next_power_of_2(self.selector_top_k)
|
||||
_selector_walk_kernel[(num_reqs,)](
|
||||
scores.contiguous(),
|
||||
candidate_ids.contiguous(),
|
||||
self.sample_pos,
|
||||
self.sample_idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
self.draft_tokens,
|
||||
self._selector_scores,
|
||||
num_steps=self.num_speculative_steps,
|
||||
top_k=self.selector_top_k,
|
||||
BLOCK_K=block_k,
|
||||
SAMPLE_PROBABILISTIC=self.draft_logits is not None,
|
||||
USE_FP64=self.use_fp64_gumbel,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
def _cache_draft_logits(self, candidate_ids: torch.Tensor, num_sample: int) -> None:
|
||||
draft_logits = self.draft_logits
|
||||
assert draft_logits is not None
|
||||
block_k = triton.next_power_of_2(self.selector_top_k)
|
||||
_cache_draft_logits_kernel[(num_sample,)](
|
||||
draft_logits,
|
||||
self._cached_candidate_ids,
|
||||
candidate_ids,
|
||||
self._selector_scores,
|
||||
self.sample_idx_mapping,
|
||||
draft_logits.stride(0),
|
||||
draft_logits.stride(1),
|
||||
num_steps=self.num_speculative_steps,
|
||||
top_k=self.selector_top_k,
|
||||
BLOCK_K=block_k,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens_padded: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> None:
|
||||
last_hidden_states = self._run_model(
|
||||
num_tokens_padded,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cudagraph_runtime_mode,
|
||||
)
|
||||
num_sample = num_reqs * self.num_speculative_steps
|
||||
hidden_states = last_hidden_states[self.sample_indices[:num_sample]].view(
|
||||
num_reqs, self.num_speculative_steps, -1
|
||||
)
|
||||
candidate_ids, unary_logits = self.model.compute_candidates(
|
||||
hidden_states.flatten(0, 1)
|
||||
)
|
||||
candidate_ids = candidate_ids.view(
|
||||
num_reqs, self.num_speculative_steps, self.selector_top_k
|
||||
)
|
||||
unary_logits = unary_logits.view_as(candidate_ids)
|
||||
anchor_token_ids = self.input_buffers.input_ids[self._anchor_indices[:num_reqs]]
|
||||
scores = self.model.model.candidate_selector(
|
||||
candidate_ids,
|
||||
unary_logits,
|
||||
hidden_states,
|
||||
anchor_token_ids,
|
||||
)
|
||||
self._sample_path(candidate_ids, scores, num_reqs)
|
||||
if self.draft_logits is not None:
|
||||
self._cache_draft_logits(candidate_ids, num_sample)
|
||||
@@ -132,11 +132,15 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.draft_logits: torch.Tensor | None = None
|
||||
if self.speculative_config.draft_sample_method == "probabilistic":
|
||||
# Pre-temperature logits, cached from the previous decode step.
|
||||
self.draft_logits = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.vocab_size,
|
||||
dtype=vllm_config.model_config.head_dtype,
|
||||
dtype, fill = self.draft_logits_spec(vllm_config)
|
||||
self.draft_logits = torch.full(
|
||||
(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.vocab_size,
|
||||
),
|
||||
fill,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@@ -293,6 +297,13 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
def draft_logits_spec(self, vllm_config: VllmConfig) -> tuple[torch.dtype, float]:
|
||||
"""Dtype and fill for the cached proposal distribution.
|
||||
|
||||
Speculators that write only a subset of columns each step override this.
|
||||
"""
|
||||
return vllm_config.model_config.head_dtype, 0.0
|
||||
|
||||
def _validate_local_argmax_reduction(self) -> None:
|
||||
if not self.use_local_argmax_reduction:
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user