[Perf] Kimi K3 Specialized Decode: CuTeDSL Optimized GB300 (1.5-2x) (#61867)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Benjamin Chislett
2026-10-15 22:50:41 -04:00
committed by GitHub
co-authored by Claude Opus 5.5 vllm-agent
parent 3ddc6702a8
commit 642c18d64c
50 changed files with 12805 additions and 3 deletions
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kimi K3 specialized decode: a CuTe decode executor over the stock backbone, enabled
by ``additional_config={"decode_backend": "latency_specialized"}``."""
from dataclasses import dataclass
@dataclass(frozen=True)
class DecodeConfig:
max_context: int
mla_core: str
dcp: int
# Tokens per request in a verify step (1 + draft tokens; 1 without speculation).
q: int
capture_sizes: tuple[int, ...]
def specialized_decode_config(vllm_config) -> DecodeConfig | None:
"""The deployment's specialized decode settings, None for the stock decode.
Raises when the specialized decode is requested for an unsupported deployment."""
extra = vllm_config.additional_config
backend = extra.get("decode_backend", "auto") if isinstance(extra, dict) else "auto"
if backend == "auto":
return None
if backend != "latency_specialized":
raise ValueError("Supported decode_backend values are auto/latency_specialized")
reason = unsupported(vllm_config)
if reason is not None:
raise ValueError(f"Kimi K3 specialized decode does not support {reason}")
spec = vllm_config.speculative_config
return DecodeConfig(
max_context=vllm_config.model_config.max_model_len,
mla_core=extra.get("mla_core", "auto"),
dcp=vllm_config.parallel_config.decode_context_parallel_size,
q=1 + spec.num_speculative_tokens if spec is not None else 1,
capture_sizes=tuple(
vllm_config.compilation_config.cudagraph_capture_sizes or ()
),
)
def kda_dtypes(vllm_config):
"""(conv, recurrent) KDA state dtypes the deployment's cache config selects."""
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateDtypeCalculator
cache = vllm_config.cache_config
return MambaStateDtypeCalculator.kda_state_dtype(
vllm_config.model_config.dtype,
cache.mamba_cache_dtype,
cache.mamba_ssm_cache_dtype,
)
def unsupported(vllm_config) -> str | None:
import torch
from vllm import envs
from vllm.config import CompilationMode, CUDAGraphMode
from vllm.platforms import current_platform
parallel, spec = vllm_config.parallel_config, vllm_config.speculative_config
text = vllm_config.model_config.hf_text_config
moe = tuple(
getattr(text, name, None)
for name in (
"hidden_act",
"activation_situ_beta",
"activation_situ_linear_beta",
"num_experts",
"num_experts_per_token",
"num_expert_group",
"topk_group",
"moe_router_activation_func",
)
)
compilation = vllm_config.compilation_config
graphs = compilation.cudagraph_mode
checks = (
(not current_platform.is_device_capability_family(100), "this GPU (SM100/103)"),
(
moe != ("situ", 4.0, 25.0, 896, 16, 1, 1, "sigmoid"),
"MoE configs but the one-group sigmoid top-16 SiTU MoE of Kimi K3",
),
(parallel.tensor_parallel_size not in (8, 16), "TP other than 8/16"),
(parallel.pipeline_parallel_size != 1, "pipeline parallelism"),
(parallel.enable_expert_parallel, "expert parallelism"),
(parallel.use_ubatching, "DBO"),
(
parallel.decode_context_parallel_size
not in (1, parallel.tensor_parallel_size),
"decode context parallel other than 1 or TP",
),
(vllm_config.model_config.enable_sleep_mode, "sleep mode"),
(vllm_config.lora_config is not None, "LoRA"),
# The runner's ModelState hooks build the executor and prepare the KV layouts.
(not vllm_config.use_v2_model_runner, "the V1 model runner"),
# The decode loop runs in forward; a compiled forward would trace it.
(compilation.mode == CompilationMode.STOCK_TORCH_COMPILE, "torch.compile"),
# The loop runs only in FULL decode graphs; eager and PIECEWISE steps run stock.
(
graphs is None or graphs.decode_mode() != CUDAGraphMode.FULL,
"deployments without FULL decode CUDA graphs (e.g. enforce-eager)",
),
(
spec is not None and spec.method != "dspark",
"speculative methods but DSpark",
),
(
spec is not None and spec.enable_adaptive_verification,
"adaptive verification",
),
# KdaSplit keeps a BF16 conv state and an FP32 recurrent state in place; it
# maintains neither other cache dtypes nor ReplaySSM / RecoverSSM records.
(
kda_dtypes(vllm_config) != (torch.bfloat16, torch.float32),
"KDA cache dtypes other than a BF16 conv state and an FP32 recurrent "
"state (mamba cache dtype)",
),
(
vllm_config.cache_config.use_replayssm
or vllm_config.cache_config.use_kda_recoverssm,
"ReplaySSM / RecoverSSM (--use-replayssm)",
),
# The verify kernels take at most 8 tokens per request (KdaSplit's barriers).
(
spec is not None and spec.num_speculative_tokens > 7,
"more than 7 speculative tokens",
),
(
spec is not None and envs.VLLM_KIMI_K3_AUX_ATTN_RES_STREAM,
"the AttnRes aux stream",
),
)
return next((reason for failed, reason in checks if failed), None)
@@ -0,0 +1,280 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The specialized decode executor over the stock Kimi K3 backbone: one routing
decision per step, then every layer's prepared providers in one loop."""
from functools import partial
from typing import Any, NamedTuple
import torch
from vllm.config import CUDAGraphMode
from vllm.distributed import get_tp_group
from vllm.distributed import tensor_model_parallel_all_reduce as all_reduce
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from ..cutedsl.attn_res.prepared import PreparedAttnRes
from ..cutedsl.common.embedding import PreparedEmbedding
from ..cutedsl.common.twoshot import TwoShot
from ..cutedsl.moe.dense import PreparedDenseMLP
from ..cutedsl.moe.experts_runtime import PersistentExperts
from ..cutedsl.moe.front import SharedFront
from ..cutedsl.moe.prepared import MoEScratch, PreparedMoE
from ..cutedsl.moe.small_batch import chain as small_batch_chain
from ..cutedsl.moe.tail_runtime import MoeTailBox, PreparedTail
from ..cutedsl.oproj.prepared import OprojMailbox
from .prefetch import PrefetchController
from .providers import BATCHES, PREPARED_MAX, Attention, pack_front
class Step(NamedTuple):
q: int # tokens per request
metadata: list # per layer
slot_mapping: Any
def layer_name(module):
return getattr(module, "prefix", getattr(module, "layer_name", None))
def decode_metadata(attention, context, q, b):
"""One layer's metadata if this step is a pure decode (q = 1) or a uniform
verify step of b tokens, else None."""
md = context.attn_metadata
md = md.get(layer_name(attention)) if isinstance(md, dict) else None
if md is None:
return None
if q > 1:
return md if is_uniform_verify(md, q, b) else None
if md.num_prefills or getattr(md, "num_spec_decodes", 0):
return None
return md
def is_uniform_verify(md, q, b):
"""Whether every request of the b // q graph rows verifies q tokens (FULL graphs
pad whole requests: KDA stages NULL state rows, MLA zero-length decodes)."""
if md.num_prefills:
return False
if not hasattr(md, "num_spec_decodes"):
return md.num_decode_tokens == md.num_decodes * q == b
rows, tokens = md.spec_state_indices_tensor, md.num_spec_decode_tokens
full = rows is not None and rows.shape[0] * q == b
return full and not md.num_decodes and tokens == md.num_spec_decodes * q
def caches_of(attention):
cache = getattr(attention, "kv_cache", ())
return [cache] if isinstance(cache, torch.Tensor) else list(cache)
class SpecializedDecode:
"""Built once the weights are loaded; owns every prepared provider, the
mailboxes and the decode buffers. Reads the backbone, never modifies it."""
def __init__(self, backbone, config):
self.backbone, self.buffers = backbone, {}
q = config.q
# Verify steps decode q tokens per request at M = requests * q tokens; FULL
# verify graphs round each capture size up to whole requests.
rounded = {round_up(s, q) for s in config.capture_sizes}
verify = {m for m in rounded if m <= PREPARED_MAX}
self.batches, self.q = tuple(sorted({*BATCHES, *verify})), q
self.queries = (1, q) if q > 1 else (1,)
# The longest sequence the deployment serves bounds the MLA split count.
self.max_context, self.mla_core = config.max_context, config.mla_core
# Decode context parallel over the TP ranks: MLA KV is sequence-sharded.
self.dcp, self.dcp_box, self.mla_layers = config.dcp, None, 0
group = get_tp_group()
self.tp, self.rank = group.world_size, group.rank_in_group
self.group = group.device_group
self.device = torch.device(current_platform.current_device())
layers = backbone.layers[backbone.start_layer : backbone.end_layer]
self.layers = layers = tuple(layers)
self.build_providers()
self.embedding = PreparedEmbedding(backbone.embed_tokens, self.tp, self.rank)
self.before = {
d.layer_idx: PreparedAttnRes.for_target(d, has_delta=d is not layers[0])
for d in layers
}
# The final AttnRes also applies the backbone's final norm.
self.final = PreparedAttnRes.for_target(backbone, final=True)
# The next (or final) AttnRes consumes each MoE output from its MoE tail output
# row.
self.fused_before = {}
followers = [self.before[f.layer_idx] for f in layers[1:]] + [self.final]
for layer, consumer in zip(layers, followers):
moe = self.feed_forward[layer.layer_idx]
if isinstance(moe, PreparedMoE):
consumer.bind_ring(moe.tail.tail_box.out.buffer)
self.fused_before[layer.layer_idx] = consumer
# The AttnRes entering each aux layer (or the final one) also stores the
# updated prefix: the stock aux stream (prefix sum plus the last MLP output).
self.aux_layers = tuple(backbone.aux_hidden_state_layers)
end = layers[-1].layer_idx + 1
for index in self.aux_layers:
(self.final if index == end else self.before[index]).bind_aux()
empty = partial(torch.empty, device=self.device, dtype=torch.bfloat16)
self.hidden = (empty((64, 7168)), empty((64, 7168)))
self.prefix = empty((64, 7168))
self.blocks = empty((64, backbone.num_attn_res_blocks, 7168))
def build_providers(self):
"""Every layer's attention and feed-forward providers, the model's mailboxes
and the prefetch side stream. All collective allocations, weight packing and
kernel compiles finish here, before KV binding and CUDA graph capture."""
config, layers = self.backbone.config, self.layers
beta = config.activation_situ_beta
linear_beta = config.activation_situ_linear_beta
tp, rank, device, batches = self.tp, self.rank, self.device, self.batches
owner = self.group, device, tp, rank
packed = {
d.layer_idx: pack_front(d.mlp, rank, tp) for d in layers if d.is_moe_layer
}
prefetch = self.prefetch = PrefetchController(self, layers, packed)
state = MoEScratch(tp, device, batches)
mailbox, tail_box, moe_layers = (
OprojMailbox(*owner),
MoeTailBox(*owner),
list(packed),
)
twoshot = TwoShot(*owner)
front_box = OprojMailbox(*owner, (1, 64, 3584 + 2 * 896))
small_batch = small_batch_chain.SmallBatchBox(*owner)
self.attention: dict[int, Attention] = {}
self.feed_forward: dict[int, Any] = {}
for layer in layers:
index, module = layer.layer_idx, layer.mlp
boxes = (mailbox, small_batch, twoshot)
self.attention[index] = Attention(layer, self, prefetch, *boxes)
if not layer.is_moe_layer:
reduce = all_reduce if module.down_proj.reduce_results else None
self.feed_forward[index] = PreparedDenseMLP(module, batches, reduce)
continue
args = (module, packed[index], front_box, state.front, batches, beta)
shared = dict(plan=state.plan, counters=state.shared, q=self.q)
front = SharedFront(*args, linear_beta, **shared)
routed = module.experts.routed_experts
w13, w2 = (
w.view(torch.uint8) for w in (routed.w13_weight, routed.w2_weight)
)
weights = w13, routed.w13_weight_scale, w2, routed.w2_weight_scale
experts = PersistentExperts(*weights, beta=beta, linear_beta=linear_beta)
norm = module.routed_expert_norm.weight
up = module.routed_expert_up_proj.weight
op = module.experts._k3_latent_moe_tail_op
placement = tail_box, moe_layers.index(index), len(moe_layers)
tail = PreparedTail(op, norm, up, batches, placement)
moe = PreparedMoE(
front, experts, tail, state, prefetch=prefetch, layer_idx=index
)
moe.small_batch = small_batch_chain.SmallBatchChain(
small_batch, module, experts, tail, index, beta, linear_beta, prefetch
)
self.feed_forward[index] = moe
def prepare_kv(self):
"""Compile each layer's cores for its bound KV cache layout (the profiling
cache, then the real one)."""
for layer in self.layers:
caches = caches_of(layer.self_attn)
self.attention[layer.layer_idx].prepare_layout(caches)
def embed_input_ids(self, ids):
rows = ids.shape[0] if ids.ndim == 1 and ids.is_contiguous() else 0
if not 1 <= rows <= PREPARED_MAX:
return self.backbone.embed_input_ids(ids)
table = self.backbone.embed_tokens.weight
return self.embedding.run(ids, table.new_empty((ids.shape[0], table.shape[1])))
def plan(self, positions, intermediate_tensors):
"""This step's decode plan, or None to run the stock forward instead.
Steps that run the stock forward:
- pipeline-parallel stage inputs (``intermediate_tensors``);
- token counts without prepared providers (above 64 or not in ``batches``);
- steps outside a CUDA graph capture (profiling, warmup and the warmup run
before each capture), so every stock kernel runs before its capture;
- PIECEWISE steps, keyed by token count alone and so captured as stock;
- steps with prompt rows (prefills, chunked prefills, fresh prompts);
- speculative steps whose requests do not all verify q tokens (partial
drafts, verify rows mixed with plain decode rows).
Every predicate is the same on all TP ranks: the mailboxes of the specialized
decode need every rank to take the same path.
"""
b = positions.shape[0]
if intermediate_tensors is not None or b not in self.batches:
return None
if not torch.cuda.is_current_stream_capturing():
return None
if not is_forward_context_available():
return None
context = get_forward_context()
if context.cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE:
return None
by_layer = context.attn_metadata
rows: list[Any] = list(by_layer.values()) if isinstance(by_layer, dict) else []
q = self.q if any(getattr(md, "num_spec_decodes", 0) for md in rows) else 1
if b % q:
return None
metadata = [decode_metadata(d.self_attn, context, q, b) for d in self.layers]
if any(md is None for md in metadata):
return None
return Step(q, metadata, context.slot_mapping)
def run(self, step, input_ids, inputs_embeds):
"""Normalized hidden states (and the aux streams) of a planned step."""
x = inputs_embeds
if x is None:
x = self.embed_input_ids(input_ids)
out = torch.empty_like(x)
# The stock aux streams: the AttnRes prefix entering each aux layer.
aux_out = [torch.empty_like(x) for _ in self.aux_layers]
aux = dict(zip(self.aux_layers, aux_out))
b, q, slots = x.shape[0], step.q, step.slot_mapping
prefix, blocks = self.prefix[:b], self.blocks[:b]
first, second = self.hidden[0][:b], self.hidden[1][:b]
hidden, pre_ready = x, False
for layer, md in zip(self.layers, step.metadata):
index, caches = layer.layer_idx, caches_of(layer.self_attn)
if not pre_ready:
self.before[index].run(hidden, prefix, blocks, first, aux.get(index))
self.attention[index].run(
first, second, prefix, blocks, caches, md, slots, q
)
consumer = self.fused_before.get(index)
pre_ready = consumer is not None
if consumer is not None:
target = out if consumer is self.final else first
stream = aux.get(index + 1)
consume = partial(consumer.run_ring, prefix, blocks, aux=stream)
hidden = self.feed_forward[index].run(second, target, consume=consume)
else:
hidden = self.feed_forward[index].run(second, first)
first, second = second, first
if not pre_ready:
self.final.run(hidden, prefix, blocks, out, aux.get(index + 1))
self.prefetch.join()
return (out, aux_out) if aux_out else out
def normalize(self, output):
"""A stock step's output with the backbone's final norm applied, as the
specialized decode's last AttnRes applies it; the aux streams stay pre-norm.
Runs in chunks of at most 255 rows: the stock RMSNorm kernel reduces with
1024-thread blocks below 256 rows and 256-thread blocks above, so chunking keeps
the reduction order ``compute_logits`` uses for the few sampled rows, and
prefill logits match stock.
"""
hidden, aux = output if isinstance(output, tuple) else (output, None)
norm = self.backbone.norm
if hidden.ndim == 2 and hidden.shape[0] >= 256:
chunks = range(0, hidden.shape[0], 255)
hidden = torch.cat([norm(hidden[i : i + 255], None) for i in chunks])
else:
hidden = norm(hidden, None)
return hidden if aux is None else (hidden, aux)
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""L2 weight prefetch for the specialized decode.
Small-batch decode is bound by HBM bandwidth, and each kernel that starts on weights
not yet in L2 stalls on DRAM. Some kernels leave bandwidth idle, such as the attention
core and the all-reduces. At fixed points of each layer, this module launches a small
kernel (``cutedsl/common/prefetch.py``) on a side stream. It pulls the byte ranges of
the weights the next kernels read into L2, so that they start warm.
Prefetch points (``fork(point, layer, batch)``), each with its batches per TP:
- ``oproj_down``: after the o-projection (B3-8), the MoE front's packed
router + latent-down rows.
- ``attention_moe``: at TP8 only, after the KDA input projection (B5-8) or the MLA
core (B8), the same front rows plus the shared expert's weights.
- ``next_attention``: once the routed experts are launched, the next layer's
attention weights, capped in MiB per batch (``NEXT_ATTENTION``). Not above B8,
where the HBM-bound experts lose more bandwidth than the attention gains.
The front and shared-expert ranges are on the critical path. They launch with the
highest stream priority and the maximum shared-memory carveout, which needs a driver
launch because CuTe drops both attributes.
Each fork records an event on the main stream that the side stream waits on, so a
prefetch starts only after the work it overlaps. ``join()`` at the end of the step
makes the main stream wait for the side stream, so a captured graph always rejoins
it. This lives in ``runtime/`` rather than ``cutedsl/`` because it is host-side
orchestration across layers: a stream, events, and per-(point, layer, batch) tables
built from the model's layers.
"""
import ctypes
from functools import partial
import cuda.bindings.driver as cuda
import torch
from ...ops.cute_dsl.latent_moe_tail.primitives import to_cute
from ..cutedsl.common.host import _checked, resolve_kernel
from ..cutedsl.common.policy import batch_table
from ..cutedsl.common.prefetch import compile_kernel
# Per-TP batch -> MiB cap (or cap.carveout) of the next layer's attention weights,
# prefetched once the routed experts finish.
# Not at B>8: the HBM-bound experts lose more bandwidth than the attention gains.
NEXT_ATTENTION = {8: "1:96.100,2-3:96,4-7:64,8:32", 16: "1-8:1024"}
def attention_weights(attention):
"""Next-attention weights in their consumption order."""
names = (
*("in_proj_qkvgfab", "fused_qkv_a_g_proj", "fused_qkv_a_proj", "q_proj"),
*("kv_a_proj_with_mqa", "f_b_proj", "q_b_proj", "g_proj"),
)
weights = [getattr(getattr(attention, n, None), "weight", None) for n in names]
weights += [getattr(attention, "W_UK_T", None), getattr(attention, "W_UV", None)]
weights = [w for w in weights if isinstance(w, torch.Tensor)]
return (*weights, attention.o_proj.weight)
class PreparedWeightRanges:
"""One owner retains weights, immutable range table, executable and launch ABI."""
def __init__(self, weights, *, byte_cap=0, grid=32, carveout=None, priority=False):
self.weights = weights = tuple(weights)
device = weights[0].device
ranges, remaining = [], byte_cap
for weight in weights:
if weight.numel() == 0:
continue
if weight.data_ptr() % 16: # the kernel walks 16-byte aligned chunks
raise ValueError("L2 prefetch ranges must start 16-byte aligned")
span = 1 + sum((n - 1) * s for n, s in zip(weight.shape, weight.stride()))
size = span * weight.element_size()
if byte_cap:
size = min(size, remaining) // 16 * 16
remaining -= size
if size:
ranges.append((weight.data_ptr(), size))
if byte_cap and remaining < 16:
break
self.table = torch.tensor(ranges, dtype=torch.int64, device=device)
self.compiled = compile_kernel(len(ranges), grid, carveout, device.index)
self.config = None
if not priority and carveout is None:
self.views = to_cute(self.table, 16)
return
# Launch priority / carveout need a driver launch (CuTe drops both).
self.executor, self.function = resolve_kernel(self.compiled, device.index)
ids, self.attributes = cuda.CUlaunchAttributeID, []
if priority: # the greatest stream priority
top = cuda.CUlaunchAttribute()
top.id = ids.CU_LAUNCH_ATTRIBUTE_PRIORITY
top.value.priority = _checked(cuda.cuCtxGetStreamPriorityRange())[1]
self.attributes.append(top)
if carveout is not None:
smem = cuda.CUlaunchAttribute()
smem.id = ids.CU_LAUNCH_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT
smem.value.sharedMemCarveout = carveout
self.attributes.append(smem)
config = self.config = cuda.CUlaunchConfig()
config.gridDimX, config.gridDimY, config.gridDimZ = grid, 1, 1
config.blockDimX, config.blockDimY, config.blockDimZ = 32, 1, 1
config.sharedMemBytes, config.attrs = 0, self.attributes
config.numAttrs = len(self.attributes)
self.pointer = ctypes.c_void_p(self.table.data_ptr())
self.arguments = (ctypes.c_void_p * 1)(ctypes.addressof(self.pointer))
self.address = ctypes.addressof(self.arguments)
def launch(self, stream):
if self.config is None:
self.compiled(self.views, stream)
else:
self.config.hStream = stream
_checked(cuda.cuLaunchKernelEx(self.config, self.function, self.address, 0))
class PrefetchController:
"""Prefetch launches per (point, layer, batch) on one side stream."""
def __init__(self, runtime, layers, packed):
self.device, tp = runtime.device, runtime.tp
self.stream = torch.cuda.Stream(device=self.device)
self.events, self.pending, self.tables = {}, False, {}
urgent = partial(PreparedWeightRanges, carveout=100, priority=True)
following = batch_table(NEXT_ATTENTION[tp])
for position, layer in enumerate(layers):
if not layer.is_moe_layer:
continue
index, moe, attention = layer.layer_idx, layer.mlp, layer.self_attn
# The front GEMM reads [router shard; down shard] from the packed copy.
ranges = urgent((packed[index],), grid=8)
self._add("oproj_down", index, range(3, 9), ranges)
if tp == 8:
shared = moe.shared_experts
shared = shared.gate_up_proj.weight, shared.down_proj.weight
ranges = urgent((packed[index], *shared), grid=32)
batches = range(5, 9) if hasattr(attention, "in_proj_qkvgfab") else (8,)
self._add("attention_moe", index, batches, ranges)
if position + 1 < len(layers):
weights = attention_weights(layers[position + 1].self_attn)
self._add_capped("next_attention", index, weights, following)
def _add_capped(self, point, layer, weights, table):
groups: dict[tuple, list[int]] = {}
for batch, spec in table.items():
cap, carveout = spec if isinstance(spec, tuple) else (spec, None)
if cap:
groups.setdefault((cap, carveout), []).append(batch)
for (cap, carveout), batches in groups.items():
ranges = PreparedWeightRanges(
weights, byte_cap=cap * 1024**2, grid=32, carveout=carveout
)
self._add(point, layer, batches, ranges)
def _add(self, point, layer, batches, ranges):
key = point, layer
if key not in self.events:
event = torch.cuda.Event()
event.record(torch.cuda.current_stream(self.device))
self.events[key] = event
for batch in batches:
self.tables[point, layer, batch] = ranges
def fork(self, point, layer, batch):
ranges = self.tables.get((point, layer, batch))
if ranges is None:
return False
event = self.events[point, layer]
event.record(torch.cuda.current_stream(self.device))
self.stream.wait_event(event)
self.pending = True
ranges.launch(cuda.CUstream(self.stream.cuda_stream))
return True
def join(self):
if self.pending:
torch.cuda.current_stream(self.device).wait_stream(self.stream)
self.pending = False
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-layer attention providers (input projection, KDA/MLA core, o-projection) and
the helpers ``SpecializedDecode.build_providers`` uses."""
from typing import Any
import torch
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
SkinnyGemmConfig,
shape_dynamic_skinny_gemm,
)
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from ..cutedsl.gemm.projection import PreparedProjection
from ..cutedsl.kda.prepared import PreparedKda
from ..cutedsl.mla.dcp import DcpBox
from ..cutedsl.mla.prepared import PreparedMla
from ..cutedsl.oproj.prepared import PreparedOproj
# Decode batches (tokens) served by the prepared providers: every batch up to 8, the
# padded CUDA-graph batches and each further captured size; others run stock.
BATCHES = (*range(1, 9), 16, 32, 64)
PREPARED_MAX = BATCHES[-1]
def projection_policies(tp, rows, batches):
"""Split-K (M tile, N tile, split, stages) per execution rows' next tested bin."""
policies = {}
for batch in batches:
n = next(b for b in (8, 16, 32, 64) if batch <= b)
if rows == 6288 and (tp == 8 or n > 8):
tactic = (64, 64, 1, 6) if n == 64 else (128, n, 2, 5 if n == 32 else 6)
else: # TP16 KDA B64: a sixth stage, -0.7 us per layer in the KDA chain
stages = {8: 8, 16: 10, 32: 8, 64: 6 if rows == 3216 else 5}[n]
tactic = (64, n, 4 if n == 8 else 2, stages)
policies[batch] = tactic
return policies
class SkinnyProjection:
"""B1/B2 skinny GEMM; static K loads two weight tiles before the PDL wait."""
def __init__(self, weight, fallback):
self.weight, self.fallback, k = weight, fallback, weight.shape[1]
tactics = {1: (128, 4, 1), 2: (64, 4, 2)}
self.configs = {m: SkinnyGemmConfig(m, *t, 8, k) for m, t in tactics.items()}
for m, cfg in self.configs.items(): # compile before capture
shape_dynamic_skinny_gemm(weight.new_zeros(m, k), weight, cfg, None)
def run(self, x, out):
cfg = self.configs.get(x.shape[0])
if cfg is None:
return self.fallback.run(x, out)
return shape_dynamic_skinny_gemm(x, self.weight, cfg, None)
def scratch(runtime, name, shape, dtype=torch.bfloat16):
if name not in runtime.buffers:
runtime.buffers[name] = torch.empty(shape, device=runtime.device, dtype=dtype)
return runtime.buffers[name]
class Attention:
"""Input projection -> KDA/MLA core -> o-projection with its fused AttnRes."""
def __init__(self, decoder, runtime, prefetch, mailbox, small_batch, twoshot=None):
attention, tp = decoder.self_attn, runtime.tp
self.kind = kind = "kda" if hasattr(attention, "in_proj_qkvgfab") else "mla"
name = "in_proj_qkvgfab" if kind == "kda" else "fused_qkv_a_g_proj"
weight = getattr(attention, name).weight
pitch = weight.shape[0]
policies = projection_policies(tp, pitch, runtime.batches)
split_k = PreparedProjection(weight, policies).prepare()
self.projection = SkinnyProjection(weight, split_k)
self.core: Any # PreparedKda or PreparedMla
if kind == "kda":
mode = lambda b: (1 if b == 1 else 2 if b <= 8 else 0) if tp == 8 else 0
policies = {b: (pitch, mode(b)) for b in runtime.batches}
q = runtime.q # verify steps: B requests x q tokens
verify = [b // q for b in runtime.batches if q > 1 and b % q == 0]
policies.update({(b, q): (pitch, int(tp == 8)) for b in verify})
self.core = PreparedKda(attention, policies, is_conv_state_dim_first())
else:
heads = attention.num_local_heads
query = scratch(runtime, "mla.query", (64, heads, 576), torch.uint8)
policy = dict(max_context=runtime.max_context, core=runtime.mla_core)
policy.update(queries=runtime.queries)
if runtime.dcp > 1:
policy.update(
dcp=dcp_box(runtime, heads), parity=runtime.mla_layers % 2
)
runtime.mla_layers += 1
pitches = {b: pitch for b in runtime.batches}
self.core = PreparedMla(attention, pitches, query, **policy)
parity = decoder.layer_idx % 2
front: tuple | None = None
if decoder.is_moe_layer:
front = small_batch.front_views[parity], small_batch.hint_views[parity]
front += (small_batch.batches,)
self.output = PreparedOproj(
decoder, mailbox, runtime.batches, kind == "mla", front, twoshot
)
projected = scratch(runtime, kind + ".projected", (64, pitch))
local = scratch(runtime, "attention.local", (64, 12288 // tp))
self.prefetch, self.layer = prefetch, decoder
self.storage = {b: (projected[:b], local[:b]) for b in range(1, 65)}
def prepare_layout(self, caches):
if self.kind == "kda":
self.core.prepare_caches(caches)
else:
self.core.prepare_cache(caches[0])
def run(self, x, out, prefix, blocks, caches, metadata, slot_mapping, q=1):
b = x.shape[0]
projected, local = self.storage[b]
projected = self.projection.run(x, projected)
if self.kind == "kda":
self.prefetch.fork("attention_moe", self.layer.layer_idx, b)
if q > 1: # uniform q-token verify rows
slots = metadata.spec_state_indices_tensor
accepted = metadata.num_accepted_tokens
self.core.run(
projected, local, caches=caches, slots=slots, accepted=accepted
)
else:
slots = metadata.non_spec_state_indices_tensor
self.core.run(projected, local, caches=caches, slots=slots)
else:
decode = metadata.decode
tables, lengths = decode.block_table, decode.seq_lens
# Request metadata shorter than b: the core runs the prepared batch b
# with the missing rows as empty requests.
rows = min(b, q * min(lengths.numel(), tables.shape[0]))
if rows < b:
local[rows:].zero_()
slots = slot_mapping[self.layer.self_attn.layer_name]
mla = dict(cache=caches[0], block_table=tables, lengths=lengths)
# DCP: lengths are the local shard's, totals the full sequences'.
mla.update(totals=decode.dcp_tot_seq_lens)
self.core.run(projected, local, slots=slots, q=q, **mla)
self.prefetch.fork("attention_moe", self.layer.layer_idx, b)
self.output.run(local, prefix, blocks, out)
self.prefetch.fork("oproj_down", self.layer.layer_idx, b)
return out
def dcp_box(runtime, heads):
"""The model's DCP mailboxes (one set for every MLA layer)."""
if runtime.dcp_box is None:
from vllm.distributed import get_dcp_group
group = get_dcp_group()
assert group.world_size == runtime.dcp == runtime.tp
from vllm.config import get_current_vllm_config
args = group.device_group, runtime.device, group.world_size, group.rank_in_group
parallel = get_current_vllm_config().parallel_config
interleave = parallel.cp_kv_cache_interleave_size
runtime.dcp_box = DcpBox(*args, heads, interleave=interleave)
return runtime.dcp_box
def pack_front(moe, rank, tp):
"""[this rank's router rows (padded to 16); this rank's latent down shard]."""
router, down = moe.gate.weight, moe.routed_expert_down_proj.weight
rows, shard = 896 // tp, 3584 // tp
router = router[rank * rows : (rank + 1) * rows]
down = down[rank * shard : (rank + 1) * shard]
return torch.cat((router, router.new_zeros((-rows % 16, 7168)), down)).detach()