Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
+13 -2
View File
@@ -8,7 +8,7 @@ from itertools import islice
import torch
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.distributed import get_pp_group, get_tensor_model_parallel_rank
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
@@ -71,6 +71,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
topk_indices_buffer=topk_indices_buffer,
)
self.zero_mlp_output = False
if (
config.n_routed_experts is not None
and layer_idx >= config.first_k_dense_replace
@@ -82,7 +83,15 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.mlp.experts.moe_config.skip_final_all_reduce = True
moe_cfg = self.mlp.experts.moe_config
pc = moe_cfg.moe_parallel_config
# Validated on MoRI only. DeepEP, pplx and the other combines
# keep the previous always-skip behavior.
if pc.use_mori_kernels:
# Combine is already all-reduced; the next fused norm sums TP.
self.zero_mlp_output = get_tensor_model_parallel_rank() > 0
else:
moe_cfg.skip_final_all_reduce = True
else:
self.mlp = DeepseekV2MLP(
hidden_size=config.hidden_size,
@@ -116,6 +125,8 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
hidden_states, residual, self.post_attention_layernorm
)
hidden_states = self.mlp(hidden_states)
if self.zero_mlp_output:
hidden_states = torch.zeros_like(hidden_states)
return hidden_states, residual
+4
View File
@@ -578,6 +578,9 @@ def _validate_aiter_mega_moe_config(vllm_config: VllmConfig) -> None:
class DeepseekV4MoE(nn.Module):
# None: use the padding vLLM added.
experts_intermediate_pad: int | None = None
def __init__(
self,
vllm_config: VllmConfig,
@@ -726,6 +729,7 @@ class DeepseekV4MoE(nn.Module):
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
intermediate_pad=self.experts_intermediate_pad,
renormalize=config.norm_topk_prob,
quant_config=quant_config,
prefix=f"{prefix}.experts",
+4
View File
@@ -77,6 +77,10 @@ logger = init_logger(__name__)
class DeepseekV4MoE(DeepseekV4MoEBase):
# TODO: revert once https://github.com/ROCm/aiter/pull/6598 is in the
# pinned AITER; with the padding set, its FlyDSL MoE can output NaN.
experts_intermediate_pad = 0
def __init__(
self,
vllm_config: VllmConfig,
+29 -9
View File
@@ -4,7 +4,7 @@
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
same short convolutions as the backbone; its attention is full or sliding-window
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
``chain_norm`` is applied after every depth; its output is both the logits input
@@ -28,18 +28,23 @@ from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.sequence import IntermediateTensors
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
from .model import (
InklingDecoderLayer,
InklingReplicatedEmbedding,
_TmlForCausalLMBase,
)
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
class InklingMTPDepthLayer(nn.Module):
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
def __init__(
self,
config: InklingModelConfig,
prefix: str,
is_local: bool,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.input_proj",
)
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
# full attention (the swa_* head config and sliding_window window) to
# match this depth's checkpoint transformer_block weights.
self.transformer_block = InklingDecoderLayer(
config,
layer_id=0,
is_local=is_local,
quant_config=None,
quant_config=quant_config,
prefix=f"{prefix}.transformer_block",
force_dense_mlp=True,
)
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
self.layers = nn.ModuleDict(
{
str(idx): InklingMTPDepthLayer(
config, f"{prefix}.layers.{idx}", idx in local_ids
config,
f"{prefix}.layers.{idx}",
idx in local_ids,
quant_config=vllm_config.quant_config,
)
for idx in range(self.num_mtp_layers)
}
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={"model.mtp.": "model."},
)
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
continue
# Dense MLP fused gate/up + down.
if ".mlp.w13_dn.weight" in name:
if ".mlp.w13_dn." in name:
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
elif ".mlp.w2_md.weight" in name:
elif ".mlp.w2_md." in name:
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
else:
if name.endswith(".bias") and name not in params:
+61 -59
View File
@@ -4,12 +4,15 @@
from __future__ import annotations
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Any, cast
import numpy as np
import regex as re
import torch
from PIL import Image
from transformers import InklingProcessor
from transformers.feature_extraction_utils import BatchFeature
from vllm.config.multimodal import (
@@ -29,18 +32,19 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.transformers_utils.processors.inkling import (
AUDIO_MARKER_ID,
AUDIO_TOKEN_ID,
IMAGE_MARKER_ID,
IMAGE_TOKEN_ID,
InklingAudioFeatureExtractor,
InklingImageProcessor,
InklingProcessor,
)
from ..configs import InklingMMConfig
# Block-start markers (<|content_image|>, <|content_audio_input|>), kept verbatim
IMAGE_MARKER_ID = 200005
AUDIO_MARKER_ID = 200020
# Per-patch / per-frame placeholders (<|unused_200054|>, <|unused_200053|>)
IMAGE_TOKEN_ID = 200054
AUDIO_TOKEN_ID = 200053
# Long-edge upscale applied before patchifying (upstream defaults to no rescale)
DEFAULT_RESCALE_IMAGE_FRAC = 2.0
# Maximum audio tokens accepted per clip. At the dMel rate of 20 tokens/s
# (50 ms hop) this is ~10 minutes of audio. It bounds the persistent per-request
# buffers and the encoder/memory budget; longer clips are rejected up front.
@@ -57,6 +61,24 @@ class InklingMultiModalDataParser(MultiModalDataParser):
return super()._parse_audio_data(data)
def _rescale_image(
image: Image.Image, frac: float | None, max_upscaled_long_edge: int | None
) -> Image.Image:
"""Scale the long edge by `frac` with PIL LANCZOS, as the reference does."""
image = image.convert("RGB")
if frac is None:
return image
long_edge = max(image.size)
target_long_edge = long_edge * frac
if max_upscaled_long_edge is not None:
target_long_edge = min(target_long_edge, max(max_upscaled_long_edge, long_edge))
ratio = target_long_edge / long_edge
if ratio == 1.0:
return image
size = tuple(max(1, math.floor(d * ratio + 0.5)) for d in image.size)
return image.resize(size, resample=Image.Resampling.LANCZOS)
def inkling_vision_enabled(config: InklingMMConfig) -> bool:
return getattr(config.vision_config, "decoder_dmodel", None) is not None
@@ -70,38 +92,13 @@ class InklingProcessingInfo(BaseProcessingInfo):
return self.ctx.get_hf_config(InklingMMConfig)
def get_hf_processor(self, **kwargs: object) -> InklingProcessor:
config = self.get_hf_config()
vision_config = config.vision_config
audio_config = config.audio_config
image_processor = InklingImageProcessor(
patch_size=getattr(vision_config, "patch_size", None) or 40,
)
if inkling_audio_enabled(config):
audio_params = {
"n_mels": audio_config.n_mel_bins,
"num_dmel_bins": audio_config.mel_vocab_size,
"dmel_min_value": audio_config.dmel_min_value,
"dmel_max_value": audio_config.dmel_max_value,
}
else:
audio_params = {}
audio_extractor = InklingAudioFeatureExtractor(params=audio_params)
return InklingProcessor(
image_processor=image_processor,
audio_feature_extractor=audio_extractor,
tokenizer=self.get_tokenizer(),
)
return self.ctx.get_hf_processor(InklingProcessor, **kwargs)
def get_data_parser(self) -> MultiModalDataParser:
# Audio inputs must be resampled to the dMel feature extractor's rate
# before `process_audios` (see InklingAudioFeatureExtractor._decode_one).
# Without a target_sr the default parser raises on any audio input.
extractor = self.get_hf_processor().audio_feature_extractor
# The feature extractor requires audio at its sampling rate
feature_extractor = self.get_hf_processor().feature_extractor
return InklingMultiModalDataParser(
target_sr=extractor.params.sample_rate,
target_sr=feature_extractor.sampling_rate,
target_channels=1,
expected_hidden_size=self._get_expected_hidden_size(),
)
@@ -163,9 +160,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
if num_audios:
# Size the dummy at the maximum allowed audio so memory/encoder
# budgeting reflects the largest clip we accept (MAX_AUDIO_TOKENS).
params = self.info.get_hf_processor().audio_feature_extractor.params
hop = int(round(params.audio_token_duration_s * params.sample_rate))
audio_len = MAX_AUDIO_TOKENS * hop
feature_extractor = self.info.get_hf_processor().feature_extractor
audio_len = MAX_AUDIO_TOKENS * feature_extractor.hop_length
mm_data["audio"] = self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
@@ -203,31 +199,37 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
data: dict[str, Any] = {"input_ids": [prompt_ids]}
if images:
img_feat = processor.process_images(images)
data["pixel_values"] = img_feat["vision_patches_bthwc"]
data["num_patches"] = torch.tensor(
img_feat["num_patches"], dtype=torch.int64
image_processor = processor.image_processor
frac = image_processor.rescale_image_frac or DEFAULT_RESCALE_IMAGE_FRAC
max_long_edge = image_processor.rescale_image_max_upscaled_long_edge
# Resize with PIL: torchvision LANCZOS does not match the reference
images = [_rescale_image(img, frac, max_long_edge) for img in images]
img_feat = image_processor(
images, rescale_image_frac=None, return_tensors="pt"
)
data["pixel_values"] = img_feat["pixel_values"].to(torch.bfloat16)
data["num_patches"] = img_feat["num_patches"].to(torch.int64)
if audios:
aud_feat = processor.process_audios(audios)
per_clip = aud_feat["dmel_bins"]
num_audio_tokens = aud_feat["num_audio_tokens"]
for i, n in enumerate(num_audio_tokens):
if int(n) > MAX_AUDIO_TOKENS:
feature_extractor = processor.feature_extractor
aud_feat = feature_extractor(
audios,
sampling_rate=feature_extractor.sampling_rate,
return_tensors="pt",
)
audio_input_ids = processor._extract_dmel_bins(aud_feat["input_features"])
num_audio_tokens = aud_feat["input_features_mask"].sum(-1)
for i, n in enumerate(num_audio_tokens.tolist()):
if n > MAX_AUDIO_TOKENS:
raise ValueError(
f"Audio clip {i} produces {int(n)} tokens, exceeding the "
f"Audio clip {i} produces {n} tokens, exceeding the "
f"maximum of {MAX_AUDIO_TOKENS} (~10 min at 20 tokens/s). "
"Provide a shorter clip."
)
if per_clip:
input_audio_features = torch.cat(
[torch.as_tensor(c) for c in per_clip], dim=0
)
else:
input_audio_features = torch.empty(0)
data["input_audio_features"] = input_audio_features
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
data["input_audio_features"] = torch.cat(
[ids[:n] for ids, n in zip(audio_input_ids, num_audio_tokens)]
)
data["num_audio_tokens"] = num_audio_tokens.to(torch.int64)
processed_data = BatchFeature(data=data, tensor_type=None)
return self._finalize_hf_mm_data(
+29 -9
View File
@@ -4,7 +4,7 @@
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
same short convolutions as the backbone; its attention is full or sliding-window
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
``chain_norm`` is applied after every depth; its output is both the logits input
@@ -28,18 +28,23 @@ from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.sequence import IntermediateTensors
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
from .model import (
InklingDecoderLayer,
InklingReplicatedEmbedding,
_TmlForCausalLMBase,
)
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
class InklingMTPDepthLayer(nn.Module):
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
def __init__(
self,
config: InklingModelConfig,
prefix: str,
is_local: bool,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.input_proj",
)
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
# full attention (the swa_* head config and sliding_window window) to
# match this depth's checkpoint transformer_block weights.
self.transformer_block = InklingDecoderLayer(
config,
layer_id=0,
is_local=is_local,
quant_config=None,
quant_config=quant_config,
prefix=f"{prefix}.transformer_block",
force_dense_mlp=True,
)
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
self.layers = nn.ModuleDict(
{
str(idx): InklingMTPDepthLayer(
config, f"{prefix}.layers.{idx}", idx in local_ids
config,
f"{prefix}.layers.{idx}",
idx in local_ids,
quant_config=vllm_config.quant_config,
)
for idx in range(self.num_mtp_layers)
}
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={"model.mtp.": "model."},
)
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
continue
# Dense MLP fused gate/up + down.
if ".mlp.w13_dn.weight" in name:
if ".mlp.w13_dn." in name:
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
elif ".mlp.w2_md.weight" in name:
elif ".mlp.w2_md." in name:
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
else:
if name.endswith(".bias") and name not in params:
+60 -41
View File
@@ -95,7 +95,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
assert isinstance(spec, MambaSpec)
# Only the fused path exports the checkpoint snapshots
can_checkpoint = (
self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate
self.use_prefill_checkpoint
and self.kda_prefill_backend == "fused"
and self.use_safe_gate
)
if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align":
logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.")
@@ -132,6 +134,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.conv_size = kda_config["short_conv_kernel_size"]
self.use_full_rank_gate = True
self._fused_qkv_conv_fn = None
if rocm_aiter_ops.is_enabled():
from aiter.ops.triton.gated_delta_net import (
causal_conv1d_split_qkv_triton_tile_fn,
)
self._fused_qkv_conv_fn = causal_conv1d_split_qkv_triton_tile_fn
# Keep f_a before the narrow beta shard, then pad each TP-local row to
# select the aligned BF16 GEMM path. The padding also avoids an Inductor
# correctness issue seen with the row-strided G view.
@@ -242,13 +252,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
"The fused KDA chunk kernel requires gfx950 and a build that "
"includes it."
)
self.use_fused_chunk = backend == "fused" or (
backend == "auto" and is_fused_kda_chunk_supported()
)
logger.info_once(
"Kimi-K3 KDA prefill backend: %s",
"fused" if self.use_fused_chunk else "triton",
)
self.kda_prefill_backend = backend
if backend == "auto":
if self.use_safe_gate and rocm_aiter_ops.is_kda_gluon_prefill_available():
self.kda_prefill_backend = "gluon"
elif is_fused_kda_chunk_supported():
self.kda_prefill_backend = "fused"
else:
self.kda_prefill_backend = "triton"
logger.info_once("Kimi-K3 KDA prefill backend: %s", self.kda_prefill_backend)
# TODO: Remove the extra config arg once tested
self.use_prefill_checkpoint = (
additional_config.get("kda_prefill_checkpoint", True)
@@ -576,35 +588,48 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if mixed_qkv_ns is not None:
assert g1_ns is not None and beta_ns is not None
if m.num_prefills > 0:
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
self.local_projection_size, dim=-1
)
# Packed prefill conv would require copying V solely to make
# it dense for KDA. Separate calls accept the strided inputs
# and produce dense Q/K/V without that extra traffic.
# TODO: Use packed conv once every KDA prefill backend accepts
# row-strided Q/K/V directly.
def _prefill_conv(
x: torch.Tensor,
state: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
return causal_conv1d_fn(
x.transpose(0, 1),
weight,
if self._fused_qkv_conv_fn is not None:
# One launch (fused qkv) instead of 3
q_ns, k_ns, v_ns = self._fused_qkv_conv_fn(
mixed_qkv_ns.transpose(0, 1),
conv_weights,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial_state,
conv_state,
non_spec_query_start_loc,
self.local_projection_size,
self.local_projection_size,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
has_initial_state=has_initial_state,
activation="silu",
block_m=8,
metadata=m,
).transpose(0, 1)
preserve_input_dtype=True,
)
else:
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
self.local_projection_size, dim=-1
)
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
def _prefill_conv(
x: torch.Tensor,
state: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
return causal_conv1d_fn(
x.transpose(0, 1),
weight,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial_state,
cache_indices=non_spec_state_indices_tensor,
query_start_loc=non_spec_query_start_loc,
metadata=m,
).transpose(0, 1)
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
q_ns, k_ns, v_ns = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in (q_ns, k_ns, v_ns)
@@ -694,7 +719,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
cu_seqlens=prefill_query_start_loc,
chunk_indices=m.chunk_indices,
chunk_offsets=m.chunk_offsets,
use_fused_chunk=self.use_fused_chunk,
backend=self.kda_prefill_backend,
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
checkpoint_state=checkpoint_state,
checkpoint_offsets=checkpoint_offsets,
@@ -748,15 +773,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
# ---------- merge spec and non-spec outputs ----------
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
# Mixed batches require indexed placement in the original order.
merged = torch.empty(
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
dtype=core_attn_out_spec.dtype,
device=core_attn_out_spec.device,
)
merged = core_attn_out[:, :num_actual_tokens]
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
elif core_attn_out_non_spec is not None:
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
+29
View File
@@ -11,6 +11,7 @@ from dataclasses import replace
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
@@ -18,12 +19,14 @@ from vllm.model_executor.layers.mamba.checkpoint import (
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.v1.attention.backend import max_decode_query_len
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import KVCacheSpec
logger = init_logger(__name__)
@@ -92,7 +95,24 @@ def prepare_chunk_metadata_device(
return chunk_indices, chunk_offsets
def _adaptive_verification_enabled(vllm_config: VllmConfig) -> bool:
speculative_config = vllm_config.speculative_config
return speculative_config is not None and bool(
getattr(speculative_config, "enable_adaptive_verification", False)
)
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
@classmethod
def get_varlen_cudagraph_max_query_len(
cls: type["KimiK3ROCmKDAMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
if not _adaptive_verification_enabled(vllm_config):
return None
return max_decode_query_len(vllm_config)
def build( # type: ignore[override]
self,
common_prefix_len: int,
@@ -108,6 +128,11 @@ class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
num_decode_draft_tokens_cpu,
fast_build,
)
if (
attn_metadata.uniform_spec_sequence_length is not None
and _adaptive_verification_enabled(self.vllm_config)
):
attn_metadata = replace(attn_metadata, uniform_spec_sequence_length=None)
if attn_metadata.checkpoint is None:
return attn_metadata
# The shared builder keys checkpoint rows on every request; the ROCm
@@ -163,3 +188,7 @@ class KimiK3ROCmKDABackend(GDNAttentionBackend):
@staticmethod
def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]:
return KimiK3ROCmKDAMetadataBuilder
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
return True
+9 -3
View File
@@ -23,8 +23,8 @@ _OPT_MIN_SIZE = 2048
class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm fusion and a fused
decode Q-prep path."""
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm and output gate
fusions and a fused decode Q-prep path."""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
@@ -303,6 +303,12 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
)
if self.g_proj is not None:
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
gate = self.g_proj(hidden_states)[0]
if rocm_aiter_ops.is_enabled():
from aiter.ops.triton.fusions.fused_sigmoid_mul import fused_sigmoid_mul
attn_out = fused_sigmoid_mul(attn_out, gate)
else:
attn_out = attn_out * gate.sigmoid()
return self.o_proj(attn_out)[0]
+39 -5
View File
@@ -9,6 +9,7 @@ path.
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
@@ -24,6 +25,11 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
logger = init_logger(__name__)
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
def get_gluon_chunk_kda_warmup_profiles() -> tuple[tuple[int, int], ...]:
return ((1, 64), (2, 64), (4, 64), (8, 64), (16, 64), (1, 4096), (2, 2048))
def chunk_kda_prefill(
q: torch.Tensor,
k: torch.Tensor,
@@ -40,7 +46,7 @@ def chunk_kda_prefill(
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_fused_chunk: bool = False,
backend: str = "triton",
out: torch.Tensor | None = None,
checkpoint_state: torch.Tensor | None = None,
checkpoint_offsets: torch.Tensor | None = None,
@@ -68,9 +74,10 @@ def chunk_kda_prefill(
cu_seqlens: int32 cumulative sequence lengths.
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
chunk_offsets: int32 per-sequence first chunk index.
use_fused_chunk: request the two-kernel ROCm path. It is used only when
every one of its preconditions holds; otherwise the Triton path
runs unchanged.
backend: ``"triton"`` (the vendored chunk path), ``"fused"`` (the
two-kernel ROCm path; used only when every one of its
preconditions holds, otherwise the Triton path runs unchanged) or
``"gluon"`` (AITER's ``chunk_kda``).
out: buffer the result must land in. Honoured by both backends, so the
caller can hand in a slice of its own output and skip a copy.
checkpoint_state: destination for mid-prefill recurrent state
@@ -89,6 +96,11 @@ def chunk_kda_prefill(
The output and, when requested, the final recurrent state.
"""
if backend not in ("triton", "fused", "gluon"):
raise ValueError(
"chunk_kda_prefill backend must be 'triton', 'fused' or 'gluon', "
f"got {backend!r}."
)
if scale is None:
scale = k.shape[-1] ** -0.5
@@ -96,7 +108,7 @@ def chunk_kda_prefill(
# needs the raw projections and a bounded gate rather than the pre-normalized
# tensors the Triton path takes.
fused = (
use_fused_chunk
backend == "fused"
and use_qk_l2norm_in_kernel
and cu_seqlens is not None
and lower_bound is not None
@@ -116,6 +128,28 @@ def chunk_kda_prefill(
if state_indices is None or has_initial_state is None:
raise ValueError("state_cache needs state_indices and has_initial_state")
if backend == "gluon":
return rocm_aiter_ops.kda_chunk_prefill(
q=q,
k=k,
v=v,
g=raw_g,
beta=raw_beta,
A_log=A_log,
dt_bias=g_bias,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
scale=scale,
out=out,
initial_state=initial_state,
output_final_state=output_final_state,
state_cache=state_cache,
state_indices=state_indices,
has_initial_state=has_initial_state,
)
# Only the fused walk addresses the paged rows directly. The Triton path
# gathers the rows it needs and scatters the results back here, so callers
# pass the cache the same way for either backend.