Merge branch 'main' into patches
This commit is contained in:
@@ -8,7 +8,7 @@ from itertools import islice
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_rank
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -71,6 +71,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
|
||||
self.zero_mlp_output = False
|
||||
if (
|
||||
config.n_routed_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
@@ -82,7 +83,15 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.mlp.experts.moe_config.skip_final_all_reduce = True
|
||||
moe_cfg = self.mlp.experts.moe_config
|
||||
pc = moe_cfg.moe_parallel_config
|
||||
# Validated on MoRI only. DeepEP, pplx and the other combines
|
||||
# keep the previous always-skip behavior.
|
||||
if pc.use_mori_kernels:
|
||||
# Combine is already all-reduced; the next fused norm sums TP.
|
||||
self.zero_mlp_output = get_tensor_model_parallel_rank() > 0
|
||||
else:
|
||||
moe_cfg.skip_final_all_reduce = True
|
||||
else:
|
||||
self.mlp = DeepseekV2MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -116,6 +125,8 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
hidden_states, residual, self.post_attention_layernorm
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
if self.zero_mlp_output:
|
||||
hidden_states = torch.zeros_like(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
|
||||
@@ -578,6 +578,9 @@ def _validate_aiter_mega_moe_config(vllm_config: VllmConfig) -> None:
|
||||
|
||||
|
||||
class DeepseekV4MoE(nn.Module):
|
||||
# None: use the padding vLLM added.
|
||||
experts_intermediate_pad: int | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -726,6 +729,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
intermediate_pad=self.experts_intermediate_pad,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
|
||||
@@ -77,6 +77,10 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
# TODO: revert once https://github.com/ROCm/aiter/pull/6598 is in the
|
||||
# pinned AITER; with the padding set, its FlyDSL MoE can output NaN.
|
||||
experts_intermediate_pad = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
|
||||
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
|
||||
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
|
||||
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
|
||||
same short convolutions as the backbone; its attention is full or sliding-window
|
||||
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
|
||||
``chain_norm`` is applied after every depth; its output is both the logits input
|
||||
@@ -28,18 +28,23 @@ from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
|
||||
from .model import (
|
||||
InklingDecoderLayer,
|
||||
InklingReplicatedEmbedding,
|
||||
_TmlForCausalLMBase,
|
||||
)
|
||||
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
|
||||
|
||||
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
|
||||
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
|
||||
class InklingMTPDepthLayer(nn.Module):
|
||||
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
|
||||
|
||||
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
prefix: str,
|
||||
is_local: bool,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.input_proj",
|
||||
)
|
||||
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
|
||||
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
|
||||
# full attention (the swa_* head config and sliding_window window) to
|
||||
# match this depth's checkpoint transformer_block weights.
|
||||
self.transformer_block = InklingDecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
is_local=is_local,
|
||||
quant_config=None,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.transformer_block",
|
||||
force_dense_mlp=True,
|
||||
)
|
||||
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleDict(
|
||||
{
|
||||
str(idx): InklingMTPDepthLayer(
|
||||
config, f"{prefix}.layers.{idx}", idx in local_ids
|
||||
config,
|
||||
f"{prefix}.layers.{idx}",
|
||||
idx in local_ids,
|
||||
quant_config=vllm_config.quant_config,
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
}
|
||||
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
|
||||
|
||||
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_prefix={"model.mtp.": "model."},
|
||||
)
|
||||
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
|
||||
continue
|
||||
|
||||
# Dense MLP fused gate/up + down.
|
||||
if ".mlp.w13_dn.weight" in name:
|
||||
if ".mlp.w13_dn." in name:
|
||||
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
|
||||
elif ".mlp.w2_md.weight" in name:
|
||||
elif ".mlp.w2_md." in name:
|
||||
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params:
|
||||
|
||||
@@ -4,12 +4,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import regex as re
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import InklingProcessor
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm.config.multimodal import (
|
||||
@@ -29,18 +32,19 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.transformers_utils.processors.inkling import (
|
||||
AUDIO_MARKER_ID,
|
||||
AUDIO_TOKEN_ID,
|
||||
IMAGE_MARKER_ID,
|
||||
IMAGE_TOKEN_ID,
|
||||
InklingAudioFeatureExtractor,
|
||||
InklingImageProcessor,
|
||||
InklingProcessor,
|
||||
)
|
||||
|
||||
from ..configs import InklingMMConfig
|
||||
|
||||
# Block-start markers (<|content_image|>, <|content_audio_input|>), kept verbatim
|
||||
IMAGE_MARKER_ID = 200005
|
||||
AUDIO_MARKER_ID = 200020
|
||||
# Per-patch / per-frame placeholders (<|unused_200054|>, <|unused_200053|>)
|
||||
IMAGE_TOKEN_ID = 200054
|
||||
AUDIO_TOKEN_ID = 200053
|
||||
|
||||
# Long-edge upscale applied before patchifying (upstream defaults to no rescale)
|
||||
DEFAULT_RESCALE_IMAGE_FRAC = 2.0
|
||||
|
||||
# Maximum audio tokens accepted per clip. At the dMel rate of 20 tokens/s
|
||||
# (50 ms hop) this is ~10 minutes of audio. It bounds the persistent per-request
|
||||
# buffers and the encoder/memory budget; longer clips are rejected up front.
|
||||
@@ -57,6 +61,24 @@ class InklingMultiModalDataParser(MultiModalDataParser):
|
||||
return super()._parse_audio_data(data)
|
||||
|
||||
|
||||
def _rescale_image(
|
||||
image: Image.Image, frac: float | None, max_upscaled_long_edge: int | None
|
||||
) -> Image.Image:
|
||||
"""Scale the long edge by `frac` with PIL LANCZOS, as the reference does."""
|
||||
image = image.convert("RGB")
|
||||
if frac is None:
|
||||
return image
|
||||
long_edge = max(image.size)
|
||||
target_long_edge = long_edge * frac
|
||||
if max_upscaled_long_edge is not None:
|
||||
target_long_edge = min(target_long_edge, max(max_upscaled_long_edge, long_edge))
|
||||
ratio = target_long_edge / long_edge
|
||||
if ratio == 1.0:
|
||||
return image
|
||||
size = tuple(max(1, math.floor(d * ratio + 0.5)) for d in image.size)
|
||||
return image.resize(size, resample=Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def inkling_vision_enabled(config: InklingMMConfig) -> bool:
|
||||
return getattr(config.vision_config, "decoder_dmodel", None) is not None
|
||||
|
||||
@@ -70,38 +92,13 @@ class InklingProcessingInfo(BaseProcessingInfo):
|
||||
return self.ctx.get_hf_config(InklingMMConfig)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> InklingProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
audio_config = config.audio_config
|
||||
|
||||
image_processor = InklingImageProcessor(
|
||||
patch_size=getattr(vision_config, "patch_size", None) or 40,
|
||||
)
|
||||
|
||||
if inkling_audio_enabled(config):
|
||||
audio_params = {
|
||||
"n_mels": audio_config.n_mel_bins,
|
||||
"num_dmel_bins": audio_config.mel_vocab_size,
|
||||
"dmel_min_value": audio_config.dmel_min_value,
|
||||
"dmel_max_value": audio_config.dmel_max_value,
|
||||
}
|
||||
else:
|
||||
audio_params = {}
|
||||
audio_extractor = InklingAudioFeatureExtractor(params=audio_params)
|
||||
|
||||
return InklingProcessor(
|
||||
image_processor=image_processor,
|
||||
audio_feature_extractor=audio_extractor,
|
||||
tokenizer=self.get_tokenizer(),
|
||||
)
|
||||
return self.ctx.get_hf_processor(InklingProcessor, **kwargs)
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
# Audio inputs must be resampled to the dMel feature extractor's rate
|
||||
# before `process_audios` (see InklingAudioFeatureExtractor._decode_one).
|
||||
# Without a target_sr the default parser raises on any audio input.
|
||||
extractor = self.get_hf_processor().audio_feature_extractor
|
||||
# The feature extractor requires audio at its sampling rate
|
||||
feature_extractor = self.get_hf_processor().feature_extractor
|
||||
return InklingMultiModalDataParser(
|
||||
target_sr=extractor.params.sample_rate,
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=1,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
)
|
||||
@@ -163,9 +160,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
||||
if num_audios:
|
||||
# Size the dummy at the maximum allowed audio so memory/encoder
|
||||
# budgeting reflects the largest clip we accept (MAX_AUDIO_TOKENS).
|
||||
params = self.info.get_hf_processor().audio_feature_extractor.params
|
||||
hop = int(round(params.audio_token_duration_s * params.sample_rate))
|
||||
audio_len = MAX_AUDIO_TOKENS * hop
|
||||
feature_extractor = self.info.get_hf_processor().feature_extractor
|
||||
audio_len = MAX_AUDIO_TOKENS * feature_extractor.hop_length
|
||||
mm_data["audio"] = self._get_dummy_audios(
|
||||
length=audio_len,
|
||||
num_audios=num_audios,
|
||||
@@ -203,31 +199,37 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
|
||||
data: dict[str, Any] = {"input_ids": [prompt_ids]}
|
||||
|
||||
if images:
|
||||
img_feat = processor.process_images(images)
|
||||
data["pixel_values"] = img_feat["vision_patches_bthwc"]
|
||||
data["num_patches"] = torch.tensor(
|
||||
img_feat["num_patches"], dtype=torch.int64
|
||||
image_processor = processor.image_processor
|
||||
frac = image_processor.rescale_image_frac or DEFAULT_RESCALE_IMAGE_FRAC
|
||||
max_long_edge = image_processor.rescale_image_max_upscaled_long_edge
|
||||
# Resize with PIL: torchvision LANCZOS does not match the reference
|
||||
images = [_rescale_image(img, frac, max_long_edge) for img in images]
|
||||
img_feat = image_processor(
|
||||
images, rescale_image_frac=None, return_tensors="pt"
|
||||
)
|
||||
data["pixel_values"] = img_feat["pixel_values"].to(torch.bfloat16)
|
||||
data["num_patches"] = img_feat["num_patches"].to(torch.int64)
|
||||
|
||||
if audios:
|
||||
aud_feat = processor.process_audios(audios)
|
||||
per_clip = aud_feat["dmel_bins"]
|
||||
num_audio_tokens = aud_feat["num_audio_tokens"]
|
||||
for i, n in enumerate(num_audio_tokens):
|
||||
if int(n) > MAX_AUDIO_TOKENS:
|
||||
feature_extractor = processor.feature_extractor
|
||||
aud_feat = feature_extractor(
|
||||
audios,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
return_tensors="pt",
|
||||
)
|
||||
audio_input_ids = processor._extract_dmel_bins(aud_feat["input_features"])
|
||||
num_audio_tokens = aud_feat["input_features_mask"].sum(-1)
|
||||
for i, n in enumerate(num_audio_tokens.tolist()):
|
||||
if n > MAX_AUDIO_TOKENS:
|
||||
raise ValueError(
|
||||
f"Audio clip {i} produces {int(n)} tokens, exceeding the "
|
||||
f"Audio clip {i} produces {n} tokens, exceeding the "
|
||||
f"maximum of {MAX_AUDIO_TOKENS} (~10 min at 20 tokens/s). "
|
||||
"Provide a shorter clip."
|
||||
)
|
||||
if per_clip:
|
||||
input_audio_features = torch.cat(
|
||||
[torch.as_tensor(c) for c in per_clip], dim=0
|
||||
)
|
||||
else:
|
||||
input_audio_features = torch.empty(0)
|
||||
data["input_audio_features"] = input_audio_features
|
||||
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
|
||||
data["input_audio_features"] = torch.cat(
|
||||
[ids[:n] for ids, n in zip(audio_input_ids, num_audio_tokens)]
|
||||
)
|
||||
data["num_audio_tokens"] = num_audio_tokens.to(torch.int64)
|
||||
|
||||
processed_data = BatchFeature(data=data, tensor_type=None)
|
||||
return self._finalize_hf_mm_data(
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
|
||||
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
|
||||
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
|
||||
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
|
||||
same short convolutions as the backbone; its attention is full or sliding-window
|
||||
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
|
||||
``chain_norm`` is applied after every depth; its output is both the logits input
|
||||
@@ -28,18 +28,23 @@ from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
|
||||
from .model import (
|
||||
InklingDecoderLayer,
|
||||
InklingReplicatedEmbedding,
|
||||
_TmlForCausalLMBase,
|
||||
)
|
||||
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
|
||||
|
||||
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
|
||||
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
|
||||
class InklingMTPDepthLayer(nn.Module):
|
||||
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
|
||||
|
||||
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
prefix: str,
|
||||
is_local: bool,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.input_proj",
|
||||
)
|
||||
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
|
||||
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
|
||||
# full attention (the swa_* head config and sliding_window window) to
|
||||
# match this depth's checkpoint transformer_block weights.
|
||||
self.transformer_block = InklingDecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
is_local=is_local,
|
||||
quant_config=None,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.transformer_block",
|
||||
force_dense_mlp=True,
|
||||
)
|
||||
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleDict(
|
||||
{
|
||||
str(idx): InklingMTPDepthLayer(
|
||||
config, f"{prefix}.layers.{idx}", idx in local_ids
|
||||
config,
|
||||
f"{prefix}.layers.{idx}",
|
||||
idx in local_ids,
|
||||
quant_config=vllm_config.quant_config,
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
}
|
||||
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
|
||||
|
||||
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_prefix={"model.mtp.": "model."},
|
||||
)
|
||||
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
|
||||
continue
|
||||
|
||||
# Dense MLP fused gate/up + down.
|
||||
if ".mlp.w13_dn.weight" in name:
|
||||
if ".mlp.w13_dn." in name:
|
||||
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
|
||||
elif ".mlp.w2_md.weight" in name:
|
||||
elif ".mlp.w2_md." in name:
|
||||
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params:
|
||||
|
||||
@@ -95,7 +95,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
assert isinstance(spec, MambaSpec)
|
||||
# Only the fused path exports the checkpoint snapshots
|
||||
can_checkpoint = (
|
||||
self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate
|
||||
self.use_prefill_checkpoint
|
||||
and self.kda_prefill_backend == "fused"
|
||||
and self.use_safe_gate
|
||||
)
|
||||
if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.")
|
||||
@@ -132,6 +134,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.use_full_rank_gate = True
|
||||
|
||||
self._fused_qkv_conv_fn = None
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
from aiter.ops.triton.gated_delta_net import (
|
||||
causal_conv1d_split_qkv_triton_tile_fn,
|
||||
)
|
||||
|
||||
self._fused_qkv_conv_fn = causal_conv1d_split_qkv_triton_tile_fn
|
||||
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row to
|
||||
# select the aligned BF16 GEMM path. The padding also avoids an Inductor
|
||||
# correctness issue seen with the row-strided G view.
|
||||
@@ -242,13 +252,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
"The fused KDA chunk kernel requires gfx950 and a build that "
|
||||
"includes it."
|
||||
)
|
||||
self.use_fused_chunk = backend == "fused" or (
|
||||
backend == "auto" and is_fused_kda_chunk_supported()
|
||||
)
|
||||
logger.info_once(
|
||||
"Kimi-K3 KDA prefill backend: %s",
|
||||
"fused" if self.use_fused_chunk else "triton",
|
||||
)
|
||||
self.kda_prefill_backend = backend
|
||||
if backend == "auto":
|
||||
if self.use_safe_gate and rocm_aiter_ops.is_kda_gluon_prefill_available():
|
||||
self.kda_prefill_backend = "gluon"
|
||||
elif is_fused_kda_chunk_supported():
|
||||
self.kda_prefill_backend = "fused"
|
||||
else:
|
||||
self.kda_prefill_backend = "triton"
|
||||
logger.info_once("Kimi-K3 KDA prefill backend: %s", self.kda_prefill_backend)
|
||||
# TODO: Remove the extra config arg once tested
|
||||
self.use_prefill_checkpoint = (
|
||||
additional_config.get("kda_prefill_checkpoint", True)
|
||||
@@ -576,35 +588,48 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Packed prefill conv would require copying V solely to make
|
||||
# it dense for KDA. Separate calls accept the strided inputs
|
||||
# and produce dense Q/K/V without that extra traffic.
|
||||
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||
# row-strided Q/K/V directly.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
if self._fused_qkv_conv_fn is not None:
|
||||
# One launch (fused qkv) instead of 3
|
||||
q_ns, k_ns, v_ns = self._fused_qkv_conv_fn(
|
||||
mixed_qkv_ns.transpose(0, 1),
|
||||
conv_weights,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
conv_state,
|
||||
non_spec_query_start_loc,
|
||||
self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
has_initial_state=has_initial_state,
|
||||
activation="silu",
|
||||
block_m=8,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
preserve_input_dtype=True,
|
||||
)
|
||||
else:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
@@ -694,7 +719,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
chunk_indices=m.chunk_indices,
|
||||
chunk_offsets=m.chunk_offsets,
|
||||
use_fused_chunk=self.use_fused_chunk,
|
||||
backend=self.kda_prefill_backend,
|
||||
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
|
||||
checkpoint_state=checkpoint_state,
|
||||
checkpoint_offsets=checkpoint_offsets,
|
||||
@@ -748,15 +773,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
|
||||
# ---------- merge spec and non-spec outputs ----------
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
# Mixed batches require indexed placement in the original order.
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_spec.dtype,
|
||||
device=core_attn_out_spec.device,
|
||||
)
|
||||
merged = core_attn_out[:, :num_actual_tokens]
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
|
||||
@@ -11,6 +11,7 @@ from dataclasses import replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
@@ -18,12 +19,14 @@ from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.v1.attention.backend import max_decode_query_len
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionBackend,
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -92,7 +95,24 @@ def prepare_chunk_metadata_device(
|
||||
return chunk_indices, chunk_offsets
|
||||
|
||||
|
||||
def _adaptive_verification_enabled(vllm_config: VllmConfig) -> bool:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
return speculative_config is not None and bool(
|
||||
getattr(speculative_config, "enable_adaptive_verification", False)
|
||||
)
|
||||
|
||||
|
||||
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(
|
||||
cls: type["KimiK3ROCmKDAMetadataBuilder"],
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> int | None:
|
||||
if not _adaptive_verification_enabled(vllm_config):
|
||||
return None
|
||||
return max_decode_query_len(vllm_config)
|
||||
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -108,6 +128,11 @@ class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
num_decode_draft_tokens_cpu,
|
||||
fast_build,
|
||||
)
|
||||
if (
|
||||
attn_metadata.uniform_spec_sequence_length is not None
|
||||
and _adaptive_verification_enabled(self.vllm_config)
|
||||
):
|
||||
attn_metadata = replace(attn_metadata, uniform_spec_sequence_length=None)
|
||||
if attn_metadata.checkpoint is None:
|
||||
return attn_metadata
|
||||
# The shared builder keys checkpoint rows on every request; the ROCm
|
||||
@@ -163,3 +188,7 @@ class KimiK3ROCmKDABackend(GDNAttentionBackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[KimiK3ROCmKDAMetadataBuilder]:
|
||||
return KimiK3ROCmKDAMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||
return True
|
||||
|
||||
@@ -23,8 +23,8 @@ _OPT_MIN_SIZE = 2048
|
||||
|
||||
|
||||
class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm fusion and a fused
|
||||
decode Q-prep path."""
|
||||
"""Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm and output gate
|
||||
fusions and a fused decode Q-prep path."""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -303,6 +303,12 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper):
|
||||
)
|
||||
|
||||
if self.g_proj is not None:
|
||||
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
|
||||
gate = self.g_proj(hidden_states)[0]
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
from aiter.ops.triton.fusions.fused_sigmoid_mul import fused_sigmoid_mul
|
||||
|
||||
attn_out = fused_sigmoid_mul(attn_out, gate)
|
||||
else:
|
||||
attn_out = attn_out * gate.sigmoid()
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
|
||||
@@ -9,6 +9,7 @@ path.
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
@@ -24,6 +25,11 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
|
||||
def get_gluon_chunk_kda_warmup_profiles() -> tuple[tuple[int, int], ...]:
|
||||
return ((1, 64), (2, 64), (4, 64), (8, 64), (16, 64), (1, 4096), (2, 2048))
|
||||
|
||||
|
||||
def chunk_kda_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -40,7 +46,7 @@ def chunk_kda_prefill(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_fused_chunk: bool = False,
|
||||
backend: str = "triton",
|
||||
out: torch.Tensor | None = None,
|
||||
checkpoint_state: torch.Tensor | None = None,
|
||||
checkpoint_offsets: torch.Tensor | None = None,
|
||||
@@ -68,9 +74,10 @@ def chunk_kda_prefill(
|
||||
cu_seqlens: int32 cumulative sequence lengths.
|
||||
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
|
||||
chunk_offsets: int32 per-sequence first chunk index.
|
||||
use_fused_chunk: request the two-kernel ROCm path. It is used only when
|
||||
every one of its preconditions holds; otherwise the Triton path
|
||||
runs unchanged.
|
||||
backend: ``"triton"`` (the vendored chunk path), ``"fused"`` (the
|
||||
two-kernel ROCm path; used only when every one of its
|
||||
preconditions holds, otherwise the Triton path runs unchanged) or
|
||||
``"gluon"`` (AITER's ``chunk_kda``).
|
||||
out: buffer the result must land in. Honoured by both backends, so the
|
||||
caller can hand in a slice of its own output and skip a copy.
|
||||
checkpoint_state: destination for mid-prefill recurrent state
|
||||
@@ -89,6 +96,11 @@ def chunk_kda_prefill(
|
||||
The output and, when requested, the final recurrent state.
|
||||
|
||||
"""
|
||||
if backend not in ("triton", "fused", "gluon"):
|
||||
raise ValueError(
|
||||
"chunk_kda_prefill backend must be 'triton', 'fused' or 'gluon', "
|
||||
f"got {backend!r}."
|
||||
)
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
@@ -96,7 +108,7 @@ def chunk_kda_prefill(
|
||||
# needs the raw projections and a bounded gate rather than the pre-normalized
|
||||
# tensors the Triton path takes.
|
||||
fused = (
|
||||
use_fused_chunk
|
||||
backend == "fused"
|
||||
and use_qk_l2norm_in_kernel
|
||||
and cu_seqlens is not None
|
||||
and lower_bound is not None
|
||||
@@ -116,6 +128,28 @@ def chunk_kda_prefill(
|
||||
if state_indices is None or has_initial_state is None:
|
||||
raise ValueError("state_cache needs state_indices and has_initial_state")
|
||||
|
||||
if backend == "gluon":
|
||||
return rocm_aiter_ops.kda_chunk_prefill(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=raw_g,
|
||||
beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=g_bias,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
scale=scale,
|
||||
out=out,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
state_cache=state_cache,
|
||||
state_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
)
|
||||
|
||||
# Only the fused walk addresses the paged rows directly. The Triton path
|
||||
# gathers the rows it needs and scatters the results back here, so callers
|
||||
# pass the cache the same way for either backend.
|
||||
|
||||
Reference in New Issue
Block a user