[Spec decode] Support variable-length decode for Kimi-K3 adaptive ver (#52988)
Signed-off-by: Albert Cheng <albecheng@nvidia.com> Signed-off-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-bia01.bia.clusters.nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Benjamin Chislett
parent
355d789f0d
commit
88aa0d287d
@@ -353,9 +353,10 @@ class Dots3NoteMLAMetadataBuilder(TritonMLAMetadataBuilder):
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
max_query_len: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> Dots3NoteDecodeMetadata:
|
||||
del max_seq_len, query_start_loc_device, num_decode_tokens
|
||||
del max_seq_len, query_start_loc_device, num_decode_tokens, max_query_len
|
||||
query_len = int(query_start_loc_cpu[1] - query_start_loc_cpu[0])
|
||||
return Dots3NoteDecodeMetadata(
|
||||
block_table=block_table_tensor,
|
||||
|
||||
@@ -15,7 +15,10 @@ from vllm.model_executor.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkConfidenceHead,
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -188,6 +191,18 @@ class K3DSparkModel(nn.Module):
|
||||
self.config.markov_rank,
|
||||
prefix=maybe_prefix(prefix, "markov_head"),
|
||||
)
|
||||
self.confidence_head: DSparkConfidenceHead | None = None
|
||||
if getattr(self.config, "enable_confidence_head", False):
|
||||
with_markov = getattr(self.config, "confidence_head_with_markov", False)
|
||||
input_dim = self.config.hidden_size + (
|
||||
self.config.markov_rank if with_markov else 0
|
||||
)
|
||||
self.confidence_head = DSparkConfidenceHead(
|
||||
input_dim,
|
||||
prefix=maybe_prefix(prefix, "confidence_head"),
|
||||
bias=True,
|
||||
with_markov=with_markov,
|
||||
)
|
||||
self._max_num_context_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
@@ -436,6 +451,12 @@ class K3DSparkForCausalLM(nn.Module):
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
if getattr(self.config, "enable_confidence_head", False):
|
||||
# The class mapper drops the head as training-only; keep it for drafts
|
||||
# that actually built one, since adaptive verification reads it.
|
||||
self.hf_to_vllm_mapper = self.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_substr={"confidence_head": "confidence_head"}
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
self.model = K3DSparkModel(
|
||||
vllm_config=vllm_config,
|
||||
@@ -494,6 +515,13 @@ class K3DSparkForCausalLM(nn.Module):
|
||||
def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.markov_head.bias(markov_embed, self.logits_processor)
|
||||
|
||||
def compute_confidence(
|
||||
self, head_hidden: torch.Tensor, markov_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Per-position acceptance probability for each drafted token."""
|
||||
assert self.model.confidence_head is not None
|
||||
return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed))
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
# read: 1. all weights. 2. context kv weights
|
||||
|
||||
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionBackend,
|
||||
GDNAttentionMetadata,
|
||||
@@ -305,6 +305,10 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||
|
||||
|
||||
class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
# Overrides GDN's UNIFORM_BATCH: adaptive verification requires ALWAYS from every
|
||||
# builder, and KDA reads per-request offsets off device within a fixed k+1 window,
|
||||
# so one k+1 graph replays any 1..k+1 mix.
|
||||
_cudagraph_support = AttentionCGSupport.ALWAYS
|
||||
mamba_aligned_state_indices: torch.Tensor | None = None
|
||||
|
||||
def __init__(
|
||||
@@ -407,6 +411,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
query_start_loc_cpu.diff() > 0
|
||||
)
|
||||
spec_sequence_masks_cpu |= active_decode_mask_cpu
|
||||
# Adaptive's cost-table profiler probes eager dummy rows longer than
|
||||
# RecoverSSM's num_spec+1 activation capacity; route them out instead
|
||||
# of tripping the capacity check below.
|
||||
spec_sequence_masks_cpu &= (
|
||||
query_start_loc_cpu.diff() <= self.num_spec + 1
|
||||
)
|
||||
# Native KDA can use its regular decode path when no draft token
|
||||
# was scheduled. RecoverSSM must preserve its extended conv window.
|
||||
if (
|
||||
@@ -750,3 +760,9 @@ class KimiK3KDAAttentionBackend(GDNAttentionBackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[KimiK3KDAMetadataBuilder]:
|
||||
return KimiK3KDAMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||
# Pure spec-decode reads its plan from DEVICE offsets, so CPU/device
|
||||
# mismatch is fine; the mixed branch still assumes CPU totals match.
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user