[Perf] Kimi K3 Specialized Decode: CuTeDSL Optimized GB300 (1.5-2x) (#61867)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 5.5
vllm-agent
parent
3ddc6702a8
commit
642c18d64c
@@ -0,0 +1,135 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi K3 specialized decode: a CuTe decode executor over the stock backbone, enabled
|
||||
by ``additional_config={"decode_backend": "latency_specialized"}``."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DecodeConfig:
|
||||
max_context: int
|
||||
mla_core: str
|
||||
dcp: int
|
||||
# Tokens per request in a verify step (1 + draft tokens; 1 without speculation).
|
||||
q: int
|
||||
capture_sizes: tuple[int, ...]
|
||||
|
||||
|
||||
def specialized_decode_config(vllm_config) -> DecodeConfig | None:
|
||||
"""The deployment's specialized decode settings, None for the stock decode.
|
||||
Raises when the specialized decode is requested for an unsupported deployment."""
|
||||
extra = vllm_config.additional_config
|
||||
backend = extra.get("decode_backend", "auto") if isinstance(extra, dict) else "auto"
|
||||
if backend == "auto":
|
||||
return None
|
||||
if backend != "latency_specialized":
|
||||
raise ValueError("Supported decode_backend values are auto/latency_specialized")
|
||||
reason = unsupported(vllm_config)
|
||||
if reason is not None:
|
||||
raise ValueError(f"Kimi K3 specialized decode does not support {reason}")
|
||||
spec = vllm_config.speculative_config
|
||||
return DecodeConfig(
|
||||
max_context=vllm_config.model_config.max_model_len,
|
||||
mla_core=extra.get("mla_core", "auto"),
|
||||
dcp=vllm_config.parallel_config.decode_context_parallel_size,
|
||||
q=1 + spec.num_speculative_tokens if spec is not None else 1,
|
||||
capture_sizes=tuple(
|
||||
vllm_config.compilation_config.cudagraph_capture_sizes or ()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def kda_dtypes(vllm_config):
|
||||
"""(conv, recurrent) KDA state dtypes the deployment's cache config selects."""
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateDtypeCalculator
|
||||
|
||||
cache = vllm_config.cache_config
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
vllm_config.model_config.dtype,
|
||||
cache.mamba_cache_dtype,
|
||||
cache.mamba_ssm_cache_dtype,
|
||||
)
|
||||
|
||||
|
||||
def unsupported(vllm_config) -> str | None:
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import CompilationMode, CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
parallel, spec = vllm_config.parallel_config, vllm_config.speculative_config
|
||||
text = vllm_config.model_config.hf_text_config
|
||||
moe = tuple(
|
||||
getattr(text, name, None)
|
||||
for name in (
|
||||
"hidden_act",
|
||||
"activation_situ_beta",
|
||||
"activation_situ_linear_beta",
|
||||
"num_experts",
|
||||
"num_experts_per_token",
|
||||
"num_expert_group",
|
||||
"topk_group",
|
||||
"moe_router_activation_func",
|
||||
)
|
||||
)
|
||||
compilation = vllm_config.compilation_config
|
||||
graphs = compilation.cudagraph_mode
|
||||
checks = (
|
||||
(not current_platform.is_device_capability_family(100), "this GPU (SM100/103)"),
|
||||
(
|
||||
moe != ("situ", 4.0, 25.0, 896, 16, 1, 1, "sigmoid"),
|
||||
"MoE configs but the one-group sigmoid top-16 SiTU MoE of Kimi K3",
|
||||
),
|
||||
(parallel.tensor_parallel_size not in (8, 16), "TP other than 8/16"),
|
||||
(parallel.pipeline_parallel_size != 1, "pipeline parallelism"),
|
||||
(parallel.enable_expert_parallel, "expert parallelism"),
|
||||
(parallel.use_ubatching, "DBO"),
|
||||
(
|
||||
parallel.decode_context_parallel_size
|
||||
not in (1, parallel.tensor_parallel_size),
|
||||
"decode context parallel other than 1 or TP",
|
||||
),
|
||||
(vllm_config.model_config.enable_sleep_mode, "sleep mode"),
|
||||
(vllm_config.lora_config is not None, "LoRA"),
|
||||
# The runner's ModelState hooks build the executor and prepare the KV layouts.
|
||||
(not vllm_config.use_v2_model_runner, "the V1 model runner"),
|
||||
# The decode loop runs in forward; a compiled forward would trace it.
|
||||
(compilation.mode == CompilationMode.STOCK_TORCH_COMPILE, "torch.compile"),
|
||||
# The loop runs only in FULL decode graphs; eager and PIECEWISE steps run stock.
|
||||
(
|
||||
graphs is None or graphs.decode_mode() != CUDAGraphMode.FULL,
|
||||
"deployments without FULL decode CUDA graphs (e.g. enforce-eager)",
|
||||
),
|
||||
(
|
||||
spec is not None and spec.method != "dspark",
|
||||
"speculative methods but DSpark",
|
||||
),
|
||||
(
|
||||
spec is not None and spec.enable_adaptive_verification,
|
||||
"adaptive verification",
|
||||
),
|
||||
# KdaSplit keeps a BF16 conv state and an FP32 recurrent state in place; it
|
||||
# maintains neither other cache dtypes nor ReplaySSM / RecoverSSM records.
|
||||
(
|
||||
kda_dtypes(vllm_config) != (torch.bfloat16, torch.float32),
|
||||
"KDA cache dtypes other than a BF16 conv state and an FP32 recurrent "
|
||||
"state (mamba cache dtype)",
|
||||
),
|
||||
(
|
||||
vllm_config.cache_config.use_replayssm
|
||||
or vllm_config.cache_config.use_kda_recoverssm,
|
||||
"ReplaySSM / RecoverSSM (--use-replayssm)",
|
||||
),
|
||||
# The verify kernels take at most 8 tokens per request (KdaSplit's barriers).
|
||||
(
|
||||
spec is not None and spec.num_speculative_tokens > 7,
|
||||
"more than 7 speculative tokens",
|
||||
),
|
||||
(
|
||||
spec is not None and envs.VLLM_KIMI_K3_AUX_ATTN_RES_STREAM,
|
||||
"the AttnRes aux stream",
|
||||
),
|
||||
)
|
||||
return next((reason for failed, reason in checks if failed), None)
|
||||
@@ -0,0 +1,280 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The specialized decode executor over the stock Kimi K3 backbone: one routing
|
||||
decision per step, then every layer's prepared providers in one loop."""
|
||||
|
||||
from functools import partial
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce as all_reduce
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
from ..cutedsl.attn_res.prepared import PreparedAttnRes
|
||||
from ..cutedsl.common.embedding import PreparedEmbedding
|
||||
from ..cutedsl.common.twoshot import TwoShot
|
||||
from ..cutedsl.moe.dense import PreparedDenseMLP
|
||||
from ..cutedsl.moe.experts_runtime import PersistentExperts
|
||||
from ..cutedsl.moe.front import SharedFront
|
||||
from ..cutedsl.moe.prepared import MoEScratch, PreparedMoE
|
||||
from ..cutedsl.moe.small_batch import chain as small_batch_chain
|
||||
from ..cutedsl.moe.tail_runtime import MoeTailBox, PreparedTail
|
||||
from ..cutedsl.oproj.prepared import OprojMailbox
|
||||
from .prefetch import PrefetchController
|
||||
from .providers import BATCHES, PREPARED_MAX, Attention, pack_front
|
||||
|
||||
|
||||
class Step(NamedTuple):
|
||||
q: int # tokens per request
|
||||
metadata: list # per layer
|
||||
slot_mapping: Any
|
||||
|
||||
|
||||
def layer_name(module):
|
||||
return getattr(module, "prefix", getattr(module, "layer_name", None))
|
||||
|
||||
|
||||
def decode_metadata(attention, context, q, b):
|
||||
"""One layer's metadata if this step is a pure decode (q = 1) or a uniform
|
||||
verify step of b tokens, else None."""
|
||||
md = context.attn_metadata
|
||||
md = md.get(layer_name(attention)) if isinstance(md, dict) else None
|
||||
if md is None:
|
||||
return None
|
||||
if q > 1:
|
||||
return md if is_uniform_verify(md, q, b) else None
|
||||
if md.num_prefills or getattr(md, "num_spec_decodes", 0):
|
||||
return None
|
||||
return md
|
||||
|
||||
|
||||
def is_uniform_verify(md, q, b):
|
||||
"""Whether every request of the b // q graph rows verifies q tokens (FULL graphs
|
||||
pad whole requests: KDA stages NULL state rows, MLA zero-length decodes)."""
|
||||
if md.num_prefills:
|
||||
return False
|
||||
if not hasattr(md, "num_spec_decodes"):
|
||||
return md.num_decode_tokens == md.num_decodes * q == b
|
||||
rows, tokens = md.spec_state_indices_tensor, md.num_spec_decode_tokens
|
||||
full = rows is not None and rows.shape[0] * q == b
|
||||
return full and not md.num_decodes and tokens == md.num_spec_decodes * q
|
||||
|
||||
|
||||
def caches_of(attention):
|
||||
cache = getattr(attention, "kv_cache", ())
|
||||
return [cache] if isinstance(cache, torch.Tensor) else list(cache)
|
||||
|
||||
|
||||
class SpecializedDecode:
|
||||
"""Built once the weights are loaded; owns every prepared provider, the
|
||||
mailboxes and the decode buffers. Reads the backbone, never modifies it."""
|
||||
|
||||
def __init__(self, backbone, config):
|
||||
self.backbone, self.buffers = backbone, {}
|
||||
q = config.q
|
||||
# Verify steps decode q tokens per request at M = requests * q tokens; FULL
|
||||
# verify graphs round each capture size up to whole requests.
|
||||
rounded = {round_up(s, q) for s in config.capture_sizes}
|
||||
verify = {m for m in rounded if m <= PREPARED_MAX}
|
||||
self.batches, self.q = tuple(sorted({*BATCHES, *verify})), q
|
||||
self.queries = (1, q) if q > 1 else (1,)
|
||||
# The longest sequence the deployment serves bounds the MLA split count.
|
||||
self.max_context, self.mla_core = config.max_context, config.mla_core
|
||||
# Decode context parallel over the TP ranks: MLA KV is sequence-sharded.
|
||||
self.dcp, self.dcp_box, self.mla_layers = config.dcp, None, 0
|
||||
group = get_tp_group()
|
||||
self.tp, self.rank = group.world_size, group.rank_in_group
|
||||
self.group = group.device_group
|
||||
self.device = torch.device(current_platform.current_device())
|
||||
|
||||
layers = backbone.layers[backbone.start_layer : backbone.end_layer]
|
||||
self.layers = layers = tuple(layers)
|
||||
self.build_providers()
|
||||
self.embedding = PreparedEmbedding(backbone.embed_tokens, self.tp, self.rank)
|
||||
self.before = {
|
||||
d.layer_idx: PreparedAttnRes.for_target(d, has_delta=d is not layers[0])
|
||||
for d in layers
|
||||
}
|
||||
# The final AttnRes also applies the backbone's final norm.
|
||||
self.final = PreparedAttnRes.for_target(backbone, final=True)
|
||||
# The next (or final) AttnRes consumes each MoE output from its MoE tail output
|
||||
# row.
|
||||
self.fused_before = {}
|
||||
followers = [self.before[f.layer_idx] for f in layers[1:]] + [self.final]
|
||||
for layer, consumer in zip(layers, followers):
|
||||
moe = self.feed_forward[layer.layer_idx]
|
||||
if isinstance(moe, PreparedMoE):
|
||||
consumer.bind_ring(moe.tail.tail_box.out.buffer)
|
||||
self.fused_before[layer.layer_idx] = consumer
|
||||
# The AttnRes entering each aux layer (or the final one) also stores the
|
||||
# updated prefix: the stock aux stream (prefix sum plus the last MLP output).
|
||||
self.aux_layers = tuple(backbone.aux_hidden_state_layers)
|
||||
end = layers[-1].layer_idx + 1
|
||||
for index in self.aux_layers:
|
||||
(self.final if index == end else self.before[index]).bind_aux()
|
||||
empty = partial(torch.empty, device=self.device, dtype=torch.bfloat16)
|
||||
self.hidden = (empty((64, 7168)), empty((64, 7168)))
|
||||
self.prefix = empty((64, 7168))
|
||||
self.blocks = empty((64, backbone.num_attn_res_blocks, 7168))
|
||||
|
||||
def build_providers(self):
|
||||
"""Every layer's attention and feed-forward providers, the model's mailboxes
|
||||
and the prefetch side stream. All collective allocations, weight packing and
|
||||
kernel compiles finish here, before KV binding and CUDA graph capture."""
|
||||
config, layers = self.backbone.config, self.layers
|
||||
beta = config.activation_situ_beta
|
||||
linear_beta = config.activation_situ_linear_beta
|
||||
tp, rank, device, batches = self.tp, self.rank, self.device, self.batches
|
||||
owner = self.group, device, tp, rank
|
||||
packed = {
|
||||
d.layer_idx: pack_front(d.mlp, rank, tp) for d in layers if d.is_moe_layer
|
||||
}
|
||||
prefetch = self.prefetch = PrefetchController(self, layers, packed)
|
||||
state = MoEScratch(tp, device, batches)
|
||||
mailbox, tail_box, moe_layers = (
|
||||
OprojMailbox(*owner),
|
||||
MoeTailBox(*owner),
|
||||
list(packed),
|
||||
)
|
||||
twoshot = TwoShot(*owner)
|
||||
front_box = OprojMailbox(*owner, (1, 64, 3584 + 2 * 896))
|
||||
small_batch = small_batch_chain.SmallBatchBox(*owner)
|
||||
self.attention: dict[int, Attention] = {}
|
||||
self.feed_forward: dict[int, Any] = {}
|
||||
for layer in layers:
|
||||
index, module = layer.layer_idx, layer.mlp
|
||||
boxes = (mailbox, small_batch, twoshot)
|
||||
self.attention[index] = Attention(layer, self, prefetch, *boxes)
|
||||
if not layer.is_moe_layer:
|
||||
reduce = all_reduce if module.down_proj.reduce_results else None
|
||||
self.feed_forward[index] = PreparedDenseMLP(module, batches, reduce)
|
||||
continue
|
||||
args = (module, packed[index], front_box, state.front, batches, beta)
|
||||
shared = dict(plan=state.plan, counters=state.shared, q=self.q)
|
||||
front = SharedFront(*args, linear_beta, **shared)
|
||||
routed = module.experts.routed_experts
|
||||
w13, w2 = (
|
||||
w.view(torch.uint8) for w in (routed.w13_weight, routed.w2_weight)
|
||||
)
|
||||
weights = w13, routed.w13_weight_scale, w2, routed.w2_weight_scale
|
||||
experts = PersistentExperts(*weights, beta=beta, linear_beta=linear_beta)
|
||||
norm = module.routed_expert_norm.weight
|
||||
up = module.routed_expert_up_proj.weight
|
||||
op = module.experts._k3_latent_moe_tail_op
|
||||
placement = tail_box, moe_layers.index(index), len(moe_layers)
|
||||
tail = PreparedTail(op, norm, up, batches, placement)
|
||||
moe = PreparedMoE(
|
||||
front, experts, tail, state, prefetch=prefetch, layer_idx=index
|
||||
)
|
||||
moe.small_batch = small_batch_chain.SmallBatchChain(
|
||||
small_batch, module, experts, tail, index, beta, linear_beta, prefetch
|
||||
)
|
||||
self.feed_forward[index] = moe
|
||||
|
||||
def prepare_kv(self):
|
||||
"""Compile each layer's cores for its bound KV cache layout (the profiling
|
||||
cache, then the real one)."""
|
||||
for layer in self.layers:
|
||||
caches = caches_of(layer.self_attn)
|
||||
self.attention[layer.layer_idx].prepare_layout(caches)
|
||||
|
||||
def embed_input_ids(self, ids):
|
||||
rows = ids.shape[0] if ids.ndim == 1 and ids.is_contiguous() else 0
|
||||
if not 1 <= rows <= PREPARED_MAX:
|
||||
return self.backbone.embed_input_ids(ids)
|
||||
table = self.backbone.embed_tokens.weight
|
||||
return self.embedding.run(ids, table.new_empty((ids.shape[0], table.shape[1])))
|
||||
|
||||
def plan(self, positions, intermediate_tensors):
|
||||
"""This step's decode plan, or None to run the stock forward instead.
|
||||
|
||||
Steps that run the stock forward:
|
||||
- pipeline-parallel stage inputs (``intermediate_tensors``);
|
||||
- token counts without prepared providers (above 64 or not in ``batches``);
|
||||
- steps outside a CUDA graph capture (profiling, warmup and the warmup run
|
||||
before each capture), so every stock kernel runs before its capture;
|
||||
- PIECEWISE steps, keyed by token count alone and so captured as stock;
|
||||
- steps with prompt rows (prefills, chunked prefills, fresh prompts);
|
||||
- speculative steps whose requests do not all verify q tokens (partial
|
||||
drafts, verify rows mixed with plain decode rows).
|
||||
|
||||
Every predicate is the same on all TP ranks: the mailboxes of the specialized
|
||||
decode need every rank to take the same path.
|
||||
"""
|
||||
b = positions.shape[0]
|
||||
if intermediate_tensors is not None or b not in self.batches:
|
||||
return None
|
||||
if not torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
if not is_forward_context_available():
|
||||
return None
|
||||
context = get_forward_context()
|
||||
if context.cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE:
|
||||
return None
|
||||
by_layer = context.attn_metadata
|
||||
rows: list[Any] = list(by_layer.values()) if isinstance(by_layer, dict) else []
|
||||
q = self.q if any(getattr(md, "num_spec_decodes", 0) for md in rows) else 1
|
||||
if b % q:
|
||||
return None
|
||||
metadata = [decode_metadata(d.self_attn, context, q, b) for d in self.layers]
|
||||
if any(md is None for md in metadata):
|
||||
return None
|
||||
return Step(q, metadata, context.slot_mapping)
|
||||
|
||||
def run(self, step, input_ids, inputs_embeds):
|
||||
"""Normalized hidden states (and the aux streams) of a planned step."""
|
||||
x = inputs_embeds
|
||||
if x is None:
|
||||
x = self.embed_input_ids(input_ids)
|
||||
out = torch.empty_like(x)
|
||||
# The stock aux streams: the AttnRes prefix entering each aux layer.
|
||||
aux_out = [torch.empty_like(x) for _ in self.aux_layers]
|
||||
aux = dict(zip(self.aux_layers, aux_out))
|
||||
b, q, slots = x.shape[0], step.q, step.slot_mapping
|
||||
prefix, blocks = self.prefix[:b], self.blocks[:b]
|
||||
first, second = self.hidden[0][:b], self.hidden[1][:b]
|
||||
hidden, pre_ready = x, False
|
||||
for layer, md in zip(self.layers, step.metadata):
|
||||
index, caches = layer.layer_idx, caches_of(layer.self_attn)
|
||||
if not pre_ready:
|
||||
self.before[index].run(hidden, prefix, blocks, first, aux.get(index))
|
||||
self.attention[index].run(
|
||||
first, second, prefix, blocks, caches, md, slots, q
|
||||
)
|
||||
consumer = self.fused_before.get(index)
|
||||
pre_ready = consumer is not None
|
||||
if consumer is not None:
|
||||
target = out if consumer is self.final else first
|
||||
stream = aux.get(index + 1)
|
||||
consume = partial(consumer.run_ring, prefix, blocks, aux=stream)
|
||||
hidden = self.feed_forward[index].run(second, target, consume=consume)
|
||||
else:
|
||||
hidden = self.feed_forward[index].run(second, first)
|
||||
first, second = second, first
|
||||
if not pre_ready:
|
||||
self.final.run(hidden, prefix, blocks, out, aux.get(index + 1))
|
||||
self.prefetch.join()
|
||||
return (out, aux_out) if aux_out else out
|
||||
|
||||
def normalize(self, output):
|
||||
"""A stock step's output with the backbone's final norm applied, as the
|
||||
specialized decode's last AttnRes applies it; the aux streams stay pre-norm.
|
||||
|
||||
Runs in chunks of at most 255 rows: the stock RMSNorm kernel reduces with
|
||||
1024-thread blocks below 256 rows and 256-thread blocks above, so chunking keeps
|
||||
the reduction order ``compute_logits`` uses for the few sampled rows, and
|
||||
prefill logits match stock.
|
||||
"""
|
||||
hidden, aux = output if isinstance(output, tuple) else (output, None)
|
||||
norm = self.backbone.norm
|
||||
if hidden.ndim == 2 and hidden.shape[0] >= 256:
|
||||
chunks = range(0, hidden.shape[0], 255)
|
||||
hidden = torch.cat([norm(hidden[i : i + 255], None) for i in chunks])
|
||||
else:
|
||||
hidden = norm(hidden, None)
|
||||
return hidden if aux is None else (hidden, aux)
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""L2 weight prefetch for the specialized decode.
|
||||
|
||||
Small-batch decode is bound by HBM bandwidth, and each kernel that starts on weights
|
||||
not yet in L2 stalls on DRAM. Some kernels leave bandwidth idle, such as the attention
|
||||
core and the all-reduces. At fixed points of each layer, this module launches a small
|
||||
kernel (``cutedsl/common/prefetch.py``) on a side stream. It pulls the byte ranges of
|
||||
the weights the next kernels read into L2, so that they start warm.
|
||||
|
||||
Prefetch points (``fork(point, layer, batch)``), each with its batches per TP:
|
||||
|
||||
- ``oproj_down``: after the o-projection (B3-8), the MoE front's packed
|
||||
router + latent-down rows.
|
||||
- ``attention_moe``: at TP8 only, after the KDA input projection (B5-8) or the MLA
|
||||
core (B8), the same front rows plus the shared expert's weights.
|
||||
- ``next_attention``: once the routed experts are launched, the next layer's
|
||||
attention weights, capped in MiB per batch (``NEXT_ATTENTION``). Not above B8,
|
||||
where the HBM-bound experts lose more bandwidth than the attention gains.
|
||||
|
||||
The front and shared-expert ranges are on the critical path. They launch with the
|
||||
highest stream priority and the maximum shared-memory carveout, which needs a driver
|
||||
launch because CuTe drops both attributes.
|
||||
|
||||
Each fork records an event on the main stream that the side stream waits on, so a
|
||||
prefetch starts only after the work it overlaps. ``join()`` at the end of the step
|
||||
makes the main stream wait for the side stream, so a captured graph always rejoins
|
||||
it. This lives in ``runtime/`` rather than ``cutedsl/`` because it is host-side
|
||||
orchestration across layers: a stream, events, and per-(point, layer, batch) tables
|
||||
built from the model's layers.
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
from functools import partial
|
||||
|
||||
import cuda.bindings.driver as cuda
|
||||
import torch
|
||||
|
||||
from ...ops.cute_dsl.latent_moe_tail.primitives import to_cute
|
||||
from ..cutedsl.common.host import _checked, resolve_kernel
|
||||
from ..cutedsl.common.policy import batch_table
|
||||
from ..cutedsl.common.prefetch import compile_kernel
|
||||
|
||||
# Per-TP batch -> MiB cap (or cap.carveout) of the next layer's attention weights,
|
||||
# prefetched once the routed experts finish.
|
||||
# Not at B>8: the HBM-bound experts lose more bandwidth than the attention gains.
|
||||
NEXT_ATTENTION = {8: "1:96.100,2-3:96,4-7:64,8:32", 16: "1-8:1024"}
|
||||
|
||||
|
||||
def attention_weights(attention):
|
||||
"""Next-attention weights in their consumption order."""
|
||||
names = (
|
||||
*("in_proj_qkvgfab", "fused_qkv_a_g_proj", "fused_qkv_a_proj", "q_proj"),
|
||||
*("kv_a_proj_with_mqa", "f_b_proj", "q_b_proj", "g_proj"),
|
||||
)
|
||||
weights = [getattr(getattr(attention, n, None), "weight", None) for n in names]
|
||||
weights += [getattr(attention, "W_UK_T", None), getattr(attention, "W_UV", None)]
|
||||
weights = [w for w in weights if isinstance(w, torch.Tensor)]
|
||||
return (*weights, attention.o_proj.weight)
|
||||
|
||||
|
||||
class PreparedWeightRanges:
|
||||
"""One owner retains weights, immutable range table, executable and launch ABI."""
|
||||
|
||||
def __init__(self, weights, *, byte_cap=0, grid=32, carveout=None, priority=False):
|
||||
self.weights = weights = tuple(weights)
|
||||
device = weights[0].device
|
||||
ranges, remaining = [], byte_cap
|
||||
for weight in weights:
|
||||
if weight.numel() == 0:
|
||||
continue
|
||||
if weight.data_ptr() % 16: # the kernel walks 16-byte aligned chunks
|
||||
raise ValueError("L2 prefetch ranges must start 16-byte aligned")
|
||||
span = 1 + sum((n - 1) * s for n, s in zip(weight.shape, weight.stride()))
|
||||
size = span * weight.element_size()
|
||||
if byte_cap:
|
||||
size = min(size, remaining) // 16 * 16
|
||||
remaining -= size
|
||||
if size:
|
||||
ranges.append((weight.data_ptr(), size))
|
||||
if byte_cap and remaining < 16:
|
||||
break
|
||||
self.table = torch.tensor(ranges, dtype=torch.int64, device=device)
|
||||
self.compiled = compile_kernel(len(ranges), grid, carveout, device.index)
|
||||
self.config = None
|
||||
if not priority and carveout is None:
|
||||
self.views = to_cute(self.table, 16)
|
||||
return
|
||||
# Launch priority / carveout need a driver launch (CuTe drops both).
|
||||
self.executor, self.function = resolve_kernel(self.compiled, device.index)
|
||||
ids, self.attributes = cuda.CUlaunchAttributeID, []
|
||||
if priority: # the greatest stream priority
|
||||
top = cuda.CUlaunchAttribute()
|
||||
top.id = ids.CU_LAUNCH_ATTRIBUTE_PRIORITY
|
||||
top.value.priority = _checked(cuda.cuCtxGetStreamPriorityRange())[1]
|
||||
self.attributes.append(top)
|
||||
if carveout is not None:
|
||||
smem = cuda.CUlaunchAttribute()
|
||||
smem.id = ids.CU_LAUNCH_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT
|
||||
smem.value.sharedMemCarveout = carveout
|
||||
self.attributes.append(smem)
|
||||
config = self.config = cuda.CUlaunchConfig()
|
||||
config.gridDimX, config.gridDimY, config.gridDimZ = grid, 1, 1
|
||||
config.blockDimX, config.blockDimY, config.blockDimZ = 32, 1, 1
|
||||
config.sharedMemBytes, config.attrs = 0, self.attributes
|
||||
config.numAttrs = len(self.attributes)
|
||||
self.pointer = ctypes.c_void_p(self.table.data_ptr())
|
||||
self.arguments = (ctypes.c_void_p * 1)(ctypes.addressof(self.pointer))
|
||||
self.address = ctypes.addressof(self.arguments)
|
||||
|
||||
def launch(self, stream):
|
||||
if self.config is None:
|
||||
self.compiled(self.views, stream)
|
||||
else:
|
||||
self.config.hStream = stream
|
||||
_checked(cuda.cuLaunchKernelEx(self.config, self.function, self.address, 0))
|
||||
|
||||
|
||||
class PrefetchController:
|
||||
"""Prefetch launches per (point, layer, batch) on one side stream."""
|
||||
|
||||
def __init__(self, runtime, layers, packed):
|
||||
self.device, tp = runtime.device, runtime.tp
|
||||
self.stream = torch.cuda.Stream(device=self.device)
|
||||
self.events, self.pending, self.tables = {}, False, {}
|
||||
urgent = partial(PreparedWeightRanges, carveout=100, priority=True)
|
||||
following = batch_table(NEXT_ATTENTION[tp])
|
||||
for position, layer in enumerate(layers):
|
||||
if not layer.is_moe_layer:
|
||||
continue
|
||||
index, moe, attention = layer.layer_idx, layer.mlp, layer.self_attn
|
||||
# The front GEMM reads [router shard; down shard] from the packed copy.
|
||||
ranges = urgent((packed[index],), grid=8)
|
||||
self._add("oproj_down", index, range(3, 9), ranges)
|
||||
if tp == 8:
|
||||
shared = moe.shared_experts
|
||||
shared = shared.gate_up_proj.weight, shared.down_proj.weight
|
||||
ranges = urgent((packed[index], *shared), grid=32)
|
||||
batches = range(5, 9) if hasattr(attention, "in_proj_qkvgfab") else (8,)
|
||||
self._add("attention_moe", index, batches, ranges)
|
||||
if position + 1 < len(layers):
|
||||
weights = attention_weights(layers[position + 1].self_attn)
|
||||
self._add_capped("next_attention", index, weights, following)
|
||||
|
||||
def _add_capped(self, point, layer, weights, table):
|
||||
groups: dict[tuple, list[int]] = {}
|
||||
for batch, spec in table.items():
|
||||
cap, carveout = spec if isinstance(spec, tuple) else (spec, None)
|
||||
if cap:
|
||||
groups.setdefault((cap, carveout), []).append(batch)
|
||||
for (cap, carveout), batches in groups.items():
|
||||
ranges = PreparedWeightRanges(
|
||||
weights, byte_cap=cap * 1024**2, grid=32, carveout=carveout
|
||||
)
|
||||
self._add(point, layer, batches, ranges)
|
||||
|
||||
def _add(self, point, layer, batches, ranges):
|
||||
key = point, layer
|
||||
if key not in self.events:
|
||||
event = torch.cuda.Event()
|
||||
event.record(torch.cuda.current_stream(self.device))
|
||||
self.events[key] = event
|
||||
for batch in batches:
|
||||
self.tables[point, layer, batch] = ranges
|
||||
|
||||
def fork(self, point, layer, batch):
|
||||
ranges = self.tables.get((point, layer, batch))
|
||||
if ranges is None:
|
||||
return False
|
||||
event = self.events[point, layer]
|
||||
event.record(torch.cuda.current_stream(self.device))
|
||||
self.stream.wait_event(event)
|
||||
self.pending = True
|
||||
ranges.launch(cuda.CUstream(self.stream.cuda_stream))
|
||||
return True
|
||||
|
||||
def join(self):
|
||||
if self.pending:
|
||||
torch.cuda.current_stream(self.device).wait_stream(self.stream)
|
||||
self.pending = False
|
||||
@@ -0,0 +1,172 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-layer attention providers (input projection, KDA/MLA core, o-projection) and
|
||||
the helpers ``SpecializedDecode.build_providers`` uses."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
SkinnyGemmConfig,
|
||||
shape_dynamic_skinny_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
|
||||
from ..cutedsl.gemm.projection import PreparedProjection
|
||||
from ..cutedsl.kda.prepared import PreparedKda
|
||||
from ..cutedsl.mla.dcp import DcpBox
|
||||
from ..cutedsl.mla.prepared import PreparedMla
|
||||
from ..cutedsl.oproj.prepared import PreparedOproj
|
||||
|
||||
# Decode batches (tokens) served by the prepared providers: every batch up to 8, the
|
||||
# padded CUDA-graph batches and each further captured size; others run stock.
|
||||
BATCHES = (*range(1, 9), 16, 32, 64)
|
||||
PREPARED_MAX = BATCHES[-1]
|
||||
|
||||
|
||||
def projection_policies(tp, rows, batches):
|
||||
"""Split-K (M tile, N tile, split, stages) per execution rows' next tested bin."""
|
||||
policies = {}
|
||||
for batch in batches:
|
||||
n = next(b for b in (8, 16, 32, 64) if batch <= b)
|
||||
if rows == 6288 and (tp == 8 or n > 8):
|
||||
tactic = (64, 64, 1, 6) if n == 64 else (128, n, 2, 5 if n == 32 else 6)
|
||||
else: # TP16 KDA B64: a sixth stage, -0.7 us per layer in the KDA chain
|
||||
stages = {8: 8, 16: 10, 32: 8, 64: 6 if rows == 3216 else 5}[n]
|
||||
tactic = (64, n, 4 if n == 8 else 2, stages)
|
||||
policies[batch] = tactic
|
||||
return policies
|
||||
|
||||
|
||||
class SkinnyProjection:
|
||||
"""B1/B2 skinny GEMM; static K loads two weight tiles before the PDL wait."""
|
||||
|
||||
def __init__(self, weight, fallback):
|
||||
self.weight, self.fallback, k = weight, fallback, weight.shape[1]
|
||||
tactics = {1: (128, 4, 1), 2: (64, 4, 2)}
|
||||
self.configs = {m: SkinnyGemmConfig(m, *t, 8, k) for m, t in tactics.items()}
|
||||
for m, cfg in self.configs.items(): # compile before capture
|
||||
shape_dynamic_skinny_gemm(weight.new_zeros(m, k), weight, cfg, None)
|
||||
|
||||
def run(self, x, out):
|
||||
cfg = self.configs.get(x.shape[0])
|
||||
if cfg is None:
|
||||
return self.fallback.run(x, out)
|
||||
return shape_dynamic_skinny_gemm(x, self.weight, cfg, None)
|
||||
|
||||
|
||||
def scratch(runtime, name, shape, dtype=torch.bfloat16):
|
||||
if name not in runtime.buffers:
|
||||
runtime.buffers[name] = torch.empty(shape, device=runtime.device, dtype=dtype)
|
||||
return runtime.buffers[name]
|
||||
|
||||
|
||||
class Attention:
|
||||
"""Input projection -> KDA/MLA core -> o-projection with its fused AttnRes."""
|
||||
|
||||
def __init__(self, decoder, runtime, prefetch, mailbox, small_batch, twoshot=None):
|
||||
attention, tp = decoder.self_attn, runtime.tp
|
||||
self.kind = kind = "kda" if hasattr(attention, "in_proj_qkvgfab") else "mla"
|
||||
name = "in_proj_qkvgfab" if kind == "kda" else "fused_qkv_a_g_proj"
|
||||
weight = getattr(attention, name).weight
|
||||
pitch = weight.shape[0]
|
||||
policies = projection_policies(tp, pitch, runtime.batches)
|
||||
split_k = PreparedProjection(weight, policies).prepare()
|
||||
self.projection = SkinnyProjection(weight, split_k)
|
||||
self.core: Any # PreparedKda or PreparedMla
|
||||
if kind == "kda":
|
||||
mode = lambda b: (1 if b == 1 else 2 if b <= 8 else 0) if tp == 8 else 0
|
||||
policies = {b: (pitch, mode(b)) for b in runtime.batches}
|
||||
q = runtime.q # verify steps: B requests x q tokens
|
||||
verify = [b // q for b in runtime.batches if q > 1 and b % q == 0]
|
||||
policies.update({(b, q): (pitch, int(tp == 8)) for b in verify})
|
||||
self.core = PreparedKda(attention, policies, is_conv_state_dim_first())
|
||||
else:
|
||||
heads = attention.num_local_heads
|
||||
query = scratch(runtime, "mla.query", (64, heads, 576), torch.uint8)
|
||||
policy = dict(max_context=runtime.max_context, core=runtime.mla_core)
|
||||
policy.update(queries=runtime.queries)
|
||||
if runtime.dcp > 1:
|
||||
policy.update(
|
||||
dcp=dcp_box(runtime, heads), parity=runtime.mla_layers % 2
|
||||
)
|
||||
runtime.mla_layers += 1
|
||||
pitches = {b: pitch for b in runtime.batches}
|
||||
self.core = PreparedMla(attention, pitches, query, **policy)
|
||||
parity = decoder.layer_idx % 2
|
||||
front: tuple | None = None
|
||||
if decoder.is_moe_layer:
|
||||
front = small_batch.front_views[parity], small_batch.hint_views[parity]
|
||||
front += (small_batch.batches,)
|
||||
self.output = PreparedOproj(
|
||||
decoder, mailbox, runtime.batches, kind == "mla", front, twoshot
|
||||
)
|
||||
projected = scratch(runtime, kind + ".projected", (64, pitch))
|
||||
local = scratch(runtime, "attention.local", (64, 12288 // tp))
|
||||
self.prefetch, self.layer = prefetch, decoder
|
||||
self.storage = {b: (projected[:b], local[:b]) for b in range(1, 65)}
|
||||
|
||||
def prepare_layout(self, caches):
|
||||
if self.kind == "kda":
|
||||
self.core.prepare_caches(caches)
|
||||
else:
|
||||
self.core.prepare_cache(caches[0])
|
||||
|
||||
def run(self, x, out, prefix, blocks, caches, metadata, slot_mapping, q=1):
|
||||
b = x.shape[0]
|
||||
projected, local = self.storage[b]
|
||||
projected = self.projection.run(x, projected)
|
||||
if self.kind == "kda":
|
||||
self.prefetch.fork("attention_moe", self.layer.layer_idx, b)
|
||||
if q > 1: # uniform q-token verify rows
|
||||
slots = metadata.spec_state_indices_tensor
|
||||
accepted = metadata.num_accepted_tokens
|
||||
self.core.run(
|
||||
projected, local, caches=caches, slots=slots, accepted=accepted
|
||||
)
|
||||
else:
|
||||
slots = metadata.non_spec_state_indices_tensor
|
||||
self.core.run(projected, local, caches=caches, slots=slots)
|
||||
else:
|
||||
decode = metadata.decode
|
||||
tables, lengths = decode.block_table, decode.seq_lens
|
||||
# Request metadata shorter than b: the core runs the prepared batch b
|
||||
# with the missing rows as empty requests.
|
||||
rows = min(b, q * min(lengths.numel(), tables.shape[0]))
|
||||
if rows < b:
|
||||
local[rows:].zero_()
|
||||
slots = slot_mapping[self.layer.self_attn.layer_name]
|
||||
mla = dict(cache=caches[0], block_table=tables, lengths=lengths)
|
||||
# DCP: lengths are the local shard's, totals the full sequences'.
|
||||
mla.update(totals=decode.dcp_tot_seq_lens)
|
||||
self.core.run(projected, local, slots=slots, q=q, **mla)
|
||||
self.prefetch.fork("attention_moe", self.layer.layer_idx, b)
|
||||
self.output.run(local, prefix, blocks, out)
|
||||
self.prefetch.fork("oproj_down", self.layer.layer_idx, b)
|
||||
return out
|
||||
|
||||
|
||||
def dcp_box(runtime, heads):
|
||||
"""The model's DCP mailboxes (one set for every MLA layer)."""
|
||||
if runtime.dcp_box is None:
|
||||
from vllm.distributed import get_dcp_group
|
||||
|
||||
group = get_dcp_group()
|
||||
assert group.world_size == runtime.dcp == runtime.tp
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
args = group.device_group, runtime.device, group.world_size, group.rank_in_group
|
||||
parallel = get_current_vllm_config().parallel_config
|
||||
interleave = parallel.cp_kv_cache_interleave_size
|
||||
runtime.dcp_box = DcpBox(*args, heads, interleave=interleave)
|
||||
return runtime.dcp_box
|
||||
|
||||
|
||||
def pack_front(moe, rank, tp):
|
||||
"""[this rank's router rows (padded to 16); this rank's latent down shard]."""
|
||||
router, down = moe.gate.weight, moe.routed_expert_down_proj.weight
|
||||
rows, shard = 896 // tp, 3584 // tp
|
||||
router = router[rank * rows : (rank + 1) * rows]
|
||||
down = down[rank * shard : (rank + 1) * shard]
|
||||
return torch.cat((router, router.new_zeros((-rows % 16, 7168)), down)).detach()
|
||||
Reference in New Issue
Block a user