[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)

Signed-off-by: Albert Cheng <albecheng@nvidia.com>
Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Albert Cheng
2026-09-22 20:43:32 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Benjamin Chislett
parent 355d789f0d
commit 88aa0d287d
13 changed files with 203 additions and 22 deletions
+2 -1
View File
@@ -353,9 +353,10 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder):
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
max_query_len: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> Dots3NoteDecodeMetadata:
del max_seq_len, query_start_loc_device, num_decode_tokens
del max_seq_len, query_start_loc_device, num_decode_tokens, max_query_len
query_len = int(query_start_loc_cpu[1] - query_start_loc_cpu[0])
return Dots3NoteDecodeMetadata(
block_table=block_table_tensor,
+29 -1
View File
@@ -15,7 +15,10 @@ from vllm.model_executor.layers.linear import (
ReplicatedLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.model_executor.models.qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -188,6 +191,18 @@ class K3DSparkModel(nn.Module):
self.config.markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(self.config, "enable_confidence_head", False):
with_markov = getattr(self.config, "confidence_head_with_markov", False)
input_dim = self.config.hidden_size + (
self.config.markov_rank if with_markov else 0
)
self.confidence_head = DSparkConfidenceHead(
input_dim,
prefix=maybe_prefix(prefix, "confidence_head"),
bias=True,
with_markov=with_markov,
)
self._max_num_context_tokens = (
vllm_config.scheduler_config.max_num_batched_tokens
)
@@ -436,6 +451,12 @@ class K3DSparkForCausalLM(nn.Module):
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
if getattr(self.config, "enable_confidence_head", False):
# The class mapper drops the head as training-only; keep it for drafts
# that actually built one, since adaptive verification reads it.
self.hf_to_vllm_mapper = self.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_substr={"confidence_head": "confidence_head"}
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = K3DSparkModel(
vllm_config=vllm_config,
@@ -494,6 +515,13 @@ class K3DSparkForCausalLM(nn.Module):
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
return self.model.markov_head.bias(markov_embed, self.logits_processor)
def compute_confidence(
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
) -> torch.Tensor:
"""Per-position acceptance probability for each drafted token."""
assert self.model.confidence_head is not None
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
# read: 1. all weights. 2. context kv weights
+17 -1
View File
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.mamba.checkpoint import (
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
@@ -305,6 +305,10 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
# Overrides GDN's UNIFORM_BATCH: adaptive verification requires ALWAYS from every
# builder, and KDA reads per-request offsets off device within a fixed k+1 window,
# so one k+1 graph replays any 1..k+1 mix.
_cudagraph_support = AttentionCGSupport.ALWAYS
mamba_aligned_state_indices: torch.Tensor | None = None
def __init__(
@@ -407,6 +411,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
query_start_loc_cpu.diff() > 0
)
spec_sequence_masks_cpu |= active_decode_mask_cpu
# Adaptive's cost-table profiler probes eager dummy rows longer than
# RecoverSSM's num_spec+1 activation capacity; route them out instead
# of tripping the capacity check below.
spec_sequence_masks_cpu &= (
query_start_loc_cpu.diff() <= self.num_spec + 1
)
# Native KDA can use its regular decode path when no draft token
# was scheduled. RecoverSSM must preserve its extended conv window.
if (
@@ -750,3 +760,9 @@ class KimiK3KDAAttentionBackend(GDNAttentionBackend):
@staticmethod
def get_builder_cls() -> type[KimiK3KDAMetadataBuilder]:
return KimiK3KDAMetadataBuilder
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
# Pure spec-decode reads its plan from DEVICE offsets, so CPU/device
# mismatch is fine; the mixed branch still assumes CPU totals match.
return True