[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces

Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 21:06:19 +07:00
co-authored by Hinata
parent 5f3c7bd8ab
commit 5aa2ac64f8
269 changed files with 417 additions and 29932 deletions
@@ -800,6 +800,3 @@ class B12xExperts(mk.FusedMoEExpertsModular):
output=output,
unit_scale_contract=self._quant_mode == "w4a16",
)
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
raise NotImplementedError("LoRA is not supported for B12xExperts")
@@ -1300,7 +1300,6 @@ class FusedMoEConfig:
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
elastic_ep_max_dp_size: int | None = None
has_bias: bool = False
is_lora_enabled: bool = False
has_hash_routing: bool = False
# Routes are adjusted between routing and expert execution.
require_decomposed_backend: bool = False
@@ -307,7 +307,6 @@ class SubstitutedRoutedExperts(RoutedExperts):
not isinstance(self.quant_method, UnquantizedFusedMoEMethod),
),
("monolithic MoE backends", self.quant_method.is_monolithic),
("MoE LoRA", moe_config.is_lora_enabled),
(
"EPLB",
parallel_config.enable_eplb
@@ -352,7 +351,6 @@ class SubstitutedRoutedExperts(RoutedExperts):
ckpt_up_proj_name or self.ckpt_up_proj_name,
num_experts=self.expert_substitution.num_logical_experts,
routed_experts_prefix="",
lora_base_layer_prefix=self.lora_base_layer_prefix,
)
def forward_modular(
@@ -387,8 +387,3 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
use_w4_group_scaling=use_w4_group_scaling,
)
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
# No support for LoRA in flashinfer_cutlass_fused_moe.
# See TODOs in flashinfer functions runMoe and runMoeMinLatency.
raise NotImplementedError("LoRA is not supported for flashinfer_cutlass_moe")
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
import torch
from vllm.lora.punica_wrapper.punica_base import PunicaWrapperBase
@dataclass
class MoELoRAContext:
"""Carries all LoRA state for one MoE forward pass.
Built by FusedMoEWithLoRA.forward() and propagated explicitly through the
modular kernel path (FusedMoEKernel -> FusedMoEExpertsModular.apply) so
that TritonExperts.apply() can compute the LoRA contribution inline,
replacing the decorator-based monkey-patch approach.
"""
# LoRA weight tensors (same shapes as FusedMoEWithLoRA attributes)
w13_lora_a_stacked: tuple[torch.Tensor, ...]
w13_lora_b_stacked: tuple[torch.Tensor, ...]
w2_lora_a_stacked: tuple[torch.Tensor, ...]
w2_lora_b_stacked: tuple[torch.Tensor, ...]
# (max_loras + 1,) int32; slot 0 is the "no-adapter" sentinel
adapter_enabled: torch.Tensor
# Metadata
max_loras: int
top_k: int
w13_num_slices: int # 2 = gated (gate + up), 1 = non-gated or 3D-fused
fully_sharded: bool
tp_rank: int
tp_size: int
local_num_experts: int
punica_wrapper: PunicaWrapperBase
# Whether VLLM_TUNED_CONFIG_FOLDER is set; selects get_lora_op_configs vs
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
use_tuned_config: bool
# Shared MoE LoRA: w13 lora_A and w2 lora_B are shared across all experts
# (stored collapsed with expert-dim 1). When True, LoRAExpertsMixin expands
# them to local_num_experts via a stride-0 view before the kernel.
enable_moe_shared_loras: bool = False
# Optional dual-stream support for overlapping each (base GEMM, LoRA)
# pair. When aux_stream is None, the experts.apply() path runs the
# original sequential schedule. When set, base GEMM runs on the default
# stream and the LoRA fast-path writes the delta into a fresh buffer on
# aux_stream, which the default stream sums in afterwards.
# Events are paired one-per-overlap-pair: events[0,1] for w13,
# events[2,3] for w2, so the two pairs do not race on the same event.
aux_stream: torch.cuda.Stream | None = None
events: tuple[torch.cuda.Event, ...] | None = None
# Per-rank token→LoRA mapping after EP dispatch. Set by
# FusedMoEPrepareAndFinalizeModular.prepare() when EP+LoRA is active, read
# by LoRAExpertsMixin helpers in place of punica_wrapper's global mapping.
# None means no dispatch happened (non-EP path), in which case callers
# fall back to punica_wrapper.token_mapping_meta.
local_token_lora_mapping: torch.Tensor | None = None
# Original unquantized hidden states, stashed by the modular kernel
# before the prepare step potentially quantizes them. Used by
# apply_w13_lora so the LoRA kernel sees correct-magnitude activations
# instead of raw quantized values that are missing the activation scale.
# Set per forward pass; None until the modular kernel writes it.
original_hidden_states: torch.Tensor | None = None
@@ -1,159 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
class LoRAExpertsMixin:
"""Mixin for FusedMoEExpertsModular subclasses that natively handle
MoELoRAContext inside their apply() implementation.
Mixing this class in:
- Flips supports_lora() to True so _can_fused_experts_support lets
LoRA through the gate check.
- Stashes a MoELoRAContext on the experts instance via
set_lora_context(), which apply() consumes from self._lora_context.
- Provides apply_w13_lora / apply_w2_lora helpers that dispatch to
the PunicaWrapper kernels.
The helper methods are pure functions of their inputs; all required
state is on lora_context or passed as arguments.
"""
moe_config: FusedMoEConfig
_lora_context: MoELoRAContext | None = None
def set_lora_context(self, ctx: MoELoRAContext) -> None:
self._lora_context = ctx
@staticmethod
def supports_lora() -> bool:
return True
def apply_w13_lora(
self,
lora_context: MoELoRAContext,
*,
y: torch.Tensor,
x: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
expert_map: torch.Tensor | None,
w1: torch.Tensor,
w2: torch.Tensor,
num_tokens: int,
top_k_num: int,
add_inputs: bool = True,
swap_w13_slices: bool = False,
) -> tuple[
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor | None,
]:
w13_lora_a_stacked = lora_context.w13_lora_a_stacked
w13_lora_b_stacked = lora_context.w13_lora_b_stacked
if lora_context.enable_moe_shared_loras:
# w13 lora_A is shared across experts (collapsed expert-dim 1);
# broadcast to local_num_experts via a stride-0 view. The kernel
# derives num_experts from shape[1] and indexes via stride(1)==0.
w13_lora_a_stacked = tuple(
a.expand(-1, lora_context.local_num_experts, -1, -1)
for a in w13_lora_a_stacked
)
if swap_w13_slices:
# The expand kernel writes slice j into the j-th half of y's last
# dim. Reversing the (gate, up) slice tuples makes it emit
# [up, gate] order directly -- used by the FlashInfer trtllm path,
# whose SwiGLU expects the up half first, to avoid an out-of-place
# concat swap afterwards.
w13_lora_a_stacked = w13_lora_a_stacked[::-1]
w13_lora_b_stacked = w13_lora_b_stacked[::-1]
interleave_w13 = (
lora_context.w13_num_slices == 2
and self.moe_config.activation == MoEActivation.SWIGLUOAI
)
lora_output = torch.zeros_like(y) if interleave_w13 else y
result = lora_context.punica_wrapper.add_lora_w13(
lora_output,
x,
w13_lora_a_stacked,
w13_lora_b_stacked,
topk_ids,
topk_weights,
expert_map,
w1,
w2,
num_tokens,
top_k_num,
lora_context.max_loras,
lora_context.adapter_enabled,
lora_context.local_num_experts,
lora_context.top_k,
lora_context.w13_num_slices,
lora_context.fully_sharded,
lora_context.use_tuned_config,
add_inputs=add_inputs or interleave_w13,
token_lora_mapping=lora_context.local_token_lora_mapping,
)
if interleave_w13:
# Map gate/up slices onto even/odd output channels without a copy.
gate_up_delta = lora_output.unflatten(-1, (2, -1))
gate_up_output = y.unflatten(-1, (-1, 2)).transpose(-1, -2)
if add_inputs:
gate_up_output.add_(gate_up_delta)
else:
gate_up_output.copy_(gate_up_delta)
return result
def apply_w2_lora(
self,
lora_context: MoELoRAContext,
*,
y: torch.Tensor,
x: torch.Tensor,
topk_weights: torch.Tensor,
sorted_token_ids_lora: torch.Tensor | None,
expert_ids_lora: torch.Tensor | None,
num_tokens_post_padded_lora: torch.Tensor | None,
token_lora_mapping: torch.Tensor | None,
num_tokens: int,
w1: torch.Tensor,
w2: torch.Tensor,
top_k_num: int,
add_inputs: bool = True,
) -> None:
w2_lora_b_stacked = lora_context.w2_lora_b_stacked
if lora_context.enable_moe_shared_loras:
# w2 lora_B is shared across experts (collapsed expert-dim 1);
# broadcast to local_num_experts via a stride-0 view.
w2_lora_b_stacked = tuple(
b.expand(-1, lora_context.local_num_experts, -1, -1)
for b in w2_lora_b_stacked
)
lora_context.punica_wrapper.add_lora_w2(
y,
x,
lora_context.w2_lora_a_stacked,
w2_lora_b_stacked,
topk_weights,
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
token_lora_mapping,
num_tokens,
w1,
w2,
top_k_num,
lora_context.max_loras,
lora_context.adapter_enabled,
lora_context.top_k,
lora_context.fully_sharded,
lora_context.tp_rank,
lora_context.use_tuned_config,
add_inputs=add_inputs,
)
@@ -21,9 +21,6 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
LoRAExpertsMixin,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
batched_moe_align_block_size,
moe_align_block_size,
@@ -673,7 +670,7 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
return E, M, N, K, topk
class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
class MarlinExperts(MarlinExpertsBase):
"""Marlin-based fused MoE expert implementation."""
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
@@ -736,121 +733,7 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
assert self.w1_scale is not None
assert self.w2_scale is not None
ctx = self._lora_context
if ctx is None:
fused_marlin_moe(
hidden_states=hidden_states,
w1=w1,
w2=w2,
bias1=self.w1_bias,
bias2=self.w2_bias,
w1_scale=self.w1_scale,
w2_scale=self.w2_scale,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_scale1=self.g1_alphas,
global_scale2=self.g2_alphas,
input_global_scale1=self.a1_gscale,
input_global_scale2=self.a2_gscale,
w1_zeros=self.w1_zp,
w2_zeros=self.w2_zp,
quant_type_id=self.quant_type_id,
apply_router_weight_on_input=apply_router_weight_on_input,
global_num_experts=global_num_experts,
activation=activation,
activation_func=self.activation,
activation_config=self.activation_config,
moe_sum=self.moe_sum,
expert_map=expert_map,
output=output,
# Workspaces are swapped in workspace_shapes() to account for proper
# output buffer allocation. Please refer to workspace_shapes().
intermediate_cache13=workspace2,
intermediate_cache2=workspace13,
input_dtype=self.input_dtype,
)
return
# LoRA path: wrap activation_func and moe_sum to inject LoRA at the
# two natural injection points.
#
# Marlin uses moe_align_block_size (same as TritonExperts) so
# intermediate_cache1 is indexed by flat (token, expert) pair index,
# which is compatible with add_lora_fused_moe's scatter mechanism.
M = hidden_states.size(0)
top_k_num = topk_ids.size(1)
lora_state: dict = {}
def activation_with_lora(
act_enum: MoEActivation,
act_output: torch.Tensor,
act_input: torch.Tensor,
*,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
) -> None:
# act_input = intermediate_cache1 (M*topk, 2N for gated)
# act_output = intermediate_cache2 (M*topk, N)
(
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
token_lora_mapping,
) = self.apply_w13_lora(
ctx,
y=act_input,
x=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
w1=w1,
w2=w2,
num_tokens=M,
top_k_num=top_k_num,
)
lora_state.update(
{
"sorted": sorted_token_ids_lora,
"eids": expert_ids_lora,
"npad": num_tokens_post_padded_lora,
"tlm": token_lora_mapping,
}
)
self.activation(
act_enum,
act_output,
act_input,
topk_ids=topk_ids,
expert_map=expert_map,
)
lora_state["cache2"] = act_output
def moe_sum_with_lora(
moe_out: torch.Tensor,
out: torch.Tensor,
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
) -> None:
# moe_out shape: (M, topk, K)
self.apply_w2_lora(
ctx,
y=moe_out,
x=lora_state["cache2"],
topk_weights=topk_weights,
sorted_token_ids_lora=lora_state["sorted"],
expert_ids_lora=lora_state["eids"],
num_tokens_post_padded_lora=lora_state["npad"],
token_lora_mapping=lora_state["tlm"],
num_tokens=M,
w1=w1,
w2=w2,
top_k_num=top_k_num,
)
self.moe_sum(moe_out, out, topk_ids, expert_map)
return fused_marlin_moe(
fused_marlin_moe(
hidden_states=hidden_states,
w1=w1,
w2=w2,
@@ -870,11 +753,13 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
apply_router_weight_on_input=apply_router_weight_on_input,
global_num_experts=global_num_experts,
activation=activation,
activation_func=activation_with_lora,
activation_func=self.activation,
activation_config=self.activation_config,
moe_sum=moe_sum_with_lora,
moe_sum=self.moe_sum,
expert_map=expert_map,
output=output,
# Workspaces are swapped in workspace_shapes() to account for proper
# output buffer allocation. Please refer to workspace_shapes().
intermediate_cache13=workspace2,
intermediate_cache2=workspace13,
input_dtype=self.input_dtype,
@@ -374,10 +374,6 @@ class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
# Used in experts/triton_moe.py and passed to moe_kernel_quantize_input.
return self.a1_gscale
@staticmethod
def supports_lora() -> bool:
return False
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
@@ -386,8 +382,6 @@ class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
if moe_config.is_lora_enabled:
return False, "kernel does not support LoRA"
if moe_config.has_bias:
return False, "kernel does not support bias"
@@ -15,9 +15,6 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
LoRAExpertsMixin,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
_prepare_expert_assignment,
invoke_fused_moe_triton_kernel,
@@ -59,10 +56,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
class TritonExperts(mk.FusedMoEExpertsModular):
"""Triton-based fused MoE expert implementation."""
@staticmethod
@@ -104,16 +100,6 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@property
def expects_unquantized_inputs(self) -> bool:
# Defer activation quantization to apply() only when LoRA is active AND
# tokens are dispatched across ranks (DP+EP all2all).
return (
self._lora_context is not None
and self.quant_dtype is not None
and self.moe_config.moe_parallel_config.use_all2all_kernels
)
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cuda_alike() or current_platform.is_xpu()
@@ -279,16 +265,13 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
torch.int8,
]
# We declared expects_unquantized_inputs (LoRA + DP/EP all2all), so the
# prepare step deferred activation quantization to this kernel:
# `hidden_states` arrives unquantized. Keep the unquantized tensor for
# the LoRA shrink input and quantize a copy here for the base GEMM
# (mirrors what the prepare step would have done, but after the
# all-gather so the layout matches the gathered topk_ids / token map).
lora_unquantized_hidden_states: torch.Tensor | None = None
# The prepare step deferred activation quantization to this kernel:
# `hidden_states` arrives unquantized. Quantize a copy here for the
# base GEMM (mirrors what the prepare step would have done, but after
# the all-gather so the layout matches the gathered topk_ids / token
# map).
if self.expects_unquantized_inputs:
assert a1q_scale is None
lora_unquantized_hidden_states = hidden_states
hidden_states, a1q_scale = moe_kernel_quantize_input(
hidden_states,
self.a1_scale,
@@ -343,7 +326,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
# Their rows in the workspace caches are then never written, so every
# consumer of those caches must be row-local: a dynamic per-tensor
# scale for the second GEMM's activation would take its amax over the
# untouched rows, and LoRA aligns the full `topk_ids` itself.
# untouched rows.
a2_scale_is_global = (
self.quant_dtype is not None
and a2_scale is None
@@ -352,7 +335,6 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
)
skip_invalid = (
expert_map is not None
and self._lora_context is None
and not a2_scale_is_global
)
@@ -373,34 +355,6 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
)
)
# LoRA w13: applied to intermediate_cache1 before activation. When
# the LoRA layer requested a dual-stream schedule, we run base w13
# GEMM on the default stream and the LoRA fast-path on aux_stream;
# the LoRA writes its delta into a fresh zero buffer (add_inputs=
# False) and we sum it into intermediate_cache1 after both finish.
#
# The LoRA shrink kernel needs unquantized, gathered-layout
# activations. When activation quant was deferred to this kernel
# (expects_unquantized_inputs), the input we quantized above is exactly
# that, so use it directly. Otherwise fall back to the context stash
# (e.g. weight-only quant), guarding on a row-count match so a
# DP-gathered layout never indexes a local stash out of bounds.
sorted_token_ids_lora = None
expert_ids_lora = None
num_tokens_post_padded_lora = None
token_lora_mapping = None
lora_context = self._lora_context
if lora_unquantized_hidden_states is not None:
lora_x = lora_unquantized_hidden_states
elif (
lora_context is not None
and lora_context.original_hidden_states is not None
and lora_context.original_hidden_states.shape[0] == hidden_states.shape[0]
):
lora_x = lora_context.original_hidden_states
else:
lora_x = hidden_states
# TODO: The fallback to self.a1_scale was added for deferred static
# activation quantization in https://github.com/vllm-project/vllm/pull/40857.
# Activation emulation relies solely on `a1q_scale` output of
@@ -412,93 +366,34 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
else (a1q_scale if a1q_scale is not None else self.a1_scale)
)
def _base_w13_fn():
invoke_fused_moe_triton_kernel(
hidden_states,
w1,
intermediate_cache1,
input_scale,
self.w1_scale,
None, # topk_weights
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
False, # mul_routed_weights
top_k_num,
config,
compute_type=compute_type,
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
use_int8_w8a8=self.quant_config.use_int8_w8a8,
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
per_channel_quant=self.per_act_token_quant,
block_shape=self.block_shape,
B_bias=self.w1_bias,
)
if lora_context is not None and lora_context.aux_stream is not None:
# add_inputs=False: kernel overwrites lora_delta_w13. zeros (not
# empty) so untouched rows -- e.g. blocks where every program
# early-exits because lora_id<0 -- stay at zero and the trailing
# add_() is a no-op there.
lora_delta_w13 = torch.zeros_like(intermediate_cache1)
def _lora_w13_fn():
return self.apply_w13_lora(
lora_context,
y=lora_delta_w13,
x=lora_x,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
w1=w1,
w2=w2,
num_tokens=num_tokens,
top_k_num=top_k_num,
add_inputs=False,
)
assert lora_context.events is not None
_, lora_meta = maybe_execute_in_parallel(
_base_w13_fn,
_lora_w13_fn,
lora_context.events[0],
lora_context.events[1],
lora_context.aux_stream,
)
(
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
token_lora_mapping,
) = lora_meta
intermediate_cache1.add_(lora_delta_w13)
else:
_base_w13_fn()
if lora_context is not None:
(
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
token_lora_mapping,
) = self.apply_w13_lora(
lora_context,
y=intermediate_cache1,
x=lora_x,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
w1=w1,
w2=w2,
num_tokens=num_tokens,
top_k_num=top_k_num,
)
invoke_fused_moe_triton_kernel(
hidden_states,
w1,
intermediate_cache1,
input_scale,
self.w1_scale,
None, # topk_weights
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
False, # mul_routed_weights
top_k_num,
config,
compute_type=compute_type,
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
use_int8_w8a8=self.quant_config.use_int8_w8a8,
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
per_channel_quant=self.per_act_token_quant,
block_shape=self.block_shape,
B_bias=self.w1_bias,
)
a2q_scale: torch.Tensor | None = None
# Fuse SiLU+Mul + FP8 block quantize into a single kernel
# when conditions permit (gated SiLU, fp8 block quant with
# group_size=128, no LoRA requiring the BF16 intermediate).
# group_size=128).
# The fused kernel has no clamp parameter, so a configured
# SwiGLU clamp limit falls through to the unfused path.
if (
@@ -506,7 +401,6 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
and self.activation_config.clamp_limit is None
and self.quant_config.use_fp8_w8a8
and self.block_shape == [128, 128]
and lora_context is None
and not is_deep_gemm_e8m0_used()
):
qintermediate_cache2, a2q_scale = ops.silu_and_mul_per_block_quant(
@@ -528,83 +422,29 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
quantization_emulation=self.quantization_emulation,
)
# LoRA w2: applied to intermediate_cache3 before moe_sum, using the
# unquantized intermediate_cache2 as the lora_a input. Reuses the
# sorted_token_ids_lora computed above. Same dual-stream pattern as
# the w13 pair: base GEMM on default stream, LoRA delta on aux,
# join via .add_() into intermediate_cache3.
def _base_w2_fn():
invoke_fused_moe_triton_kernel(
qintermediate_cache2,
w2,
intermediate_cache3,
a2q_scale,
self.w2_scale,
topk_weights,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
not apply_router_weight_on_input,
1,
config,
compute_type=compute_type,
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
use_int8_w8a8=self.quant_config.use_int8_w8a8,
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
per_channel_quant=self.per_act_token_quant,
block_shape=self.block_shape,
B_bias=self.w2_bias,
)
invoke_fused_moe_triton_kernel(
qintermediate_cache2,
w2,
intermediate_cache3,
a2q_scale,
self.w2_scale,
topk_weights,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
not apply_router_weight_on_input,
1,
config,
compute_type=compute_type,
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
use_int8_w8a8=self.quant_config.use_int8_w8a8,
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
per_channel_quant=self.per_act_token_quant,
block_shape=self.block_shape,
B_bias=self.w2_bias,
)
if lora_context is not None and lora_context.aux_stream is not None:
lora_delta_w2 = torch.zeros_like(intermediate_cache3)
def _lora_w2_fn():
self.apply_w2_lora(
lora_context,
y=lora_delta_w2,
x=intermediate_cache2,
topk_weights=topk_weights,
sorted_token_ids_lora=sorted_token_ids_lora,
expert_ids_lora=expert_ids_lora,
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
token_lora_mapping=token_lora_mapping,
num_tokens=num_tokens,
w1=w1,
w2=w2,
top_k_num=top_k_num,
add_inputs=False,
)
assert lora_context.events is not None
maybe_execute_in_parallel(
_base_w2_fn,
_lora_w2_fn,
lora_context.events[2],
lora_context.events[3],
lora_context.aux_stream,
)
intermediate_cache3.add_(lora_delta_w2)
else:
_base_w2_fn()
if lora_context is not None:
self.apply_w2_lora(
lora_context,
y=intermediate_cache3,
x=intermediate_cache2,
topk_weights=topk_weights,
sorted_token_ids_lora=sorted_token_ids_lora,
expert_ids_lora=expert_ids_lora,
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
token_lora_mapping=token_lora_mapping,
num_tokens=num_tokens,
w1=w1,
w2=w2,
top_k_num=top_k_num,
)
# separate function is required for MoE + LoRA
self.moe_sum(intermediate_cache3, output, topk_ids, expert_map)
def moe_sum(
@@ -804,5 +644,4 @@ class TritonWNA16Experts(TritonExperts):
block_shape=self.block_shape,
)
# separate function is required for MoE + LoRA
self.moe_sum(intermediate_cache3, output)
@@ -1,731 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""LoRA-aware FlashInfer TRT-LLM MoE experts (BF16, MXFP4 x BF16).
Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153
(BF16) and PR #3987 (FP4 block-scale):
* The W13 (gate_up) LoRA delta is passed directly as ``gemm1_lora_delta`` to
the routed kernel, which fuses it into FC1 before SwiGLU (BiasType::Mn).
* The W2 (down) LoRA cannot be fused -- we take the FC1 activation output
returned by the kernel (``gemm1_activation_output``, permuted) together with
``expanded_idx_to_permuted_idx``, unpermute it, compute the W2 delta out of
kernel via punica, and add it to the already-finalized output.
Constraints (matching the PR support matrix; final gating lives in the oracle):
* SM100+ (Blackwell), gated SwiGLU, shuffled weights only;
* BF16 weights, or MXFP4 weights with BF16 activations;
* routing must be computed outside the MoE (the Modular path satisfies this).
"""
import functools
from abc import abstractmethod
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
LoRAExpertsMixin,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
view_as_block_major_k,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsBase,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
@triton.jit
def _unpermute_activation_kernel(
act_ptr, # act_permuted: (num_permuted, num_cols)
idx_ptr, # idx_map: (num_rows,), values in [0, num_permuted) or -1
out_ptr, # out: (num_rows, num_cols)
num_cols,
stride_ar,
stride_or,
BLOCK_I: tl.constexpr,
):
row = tl.program_id(0)
col_offs = tl.program_id(1) * BLOCK_I + tl.arange(0, BLOCK_I)
col_mask = col_offs < num_cols
idx = tl.load(idx_ptr + row)
out_ptrs = out_ptr + row * stride_or + col_offs
if idx >= 0:
vals = tl.load(act_ptr + idx * stride_ar + col_offs, mask=col_mask, other=0.0)
tl.store(out_ptrs, vals, mask=col_mask)
else:
zeros = tl.zeros((BLOCK_I,), dtype=out_ptr.dtype.element_ty)
tl.store(out_ptrs, zeros, mask=col_mask)
@triton.jit
def _finalize_lora_kernel(
gemm2_ptr, # (num_permuted, K) base FC2 output, permuted, unweighted
weight_ptr, # (num_tokens * top_k,) routing weights (expanded order)
idx_ptr, # (num_tokens * top_k,) expanded_idx -> permuted_idx or -1
delta_ptr, # (num_tokens, top_k, K) W2 LoRA delta, already routing-weighted
out_ptr, # (num_tokens, K)
K,
stride_g0,
stride_d0,
stride_d1,
stride_o0,
scale,
TOP_K: tl.constexpr,
BLOCK_K: tl.constexpr,
):
token = tl.program_id(0)
col = tl.program_id(1) * BLOCK_K + tl.arange(0, BLOCK_K)
mask = col < K
acc_base = tl.zeros((BLOCK_K,), dtype=tl.float32)
acc_delta = tl.zeros((BLOCK_K,), dtype=tl.float32)
for k in tl.static_range(TOP_K):
eid = token * TOP_K + k
pidx = tl.load(idx_ptr + eid)
if pidx >= 0:
w = tl.load(weight_ptr + eid).to(tl.float32)
base = tl.load(gemm2_ptr + pidx * stride_g0 + col, mask=mask, other=0.0).to(
tl.float32
)
acc_base += w * base
acc_delta += tl.load(
delta_ptr + token * stride_d0 + k * stride_d1 + col, mask=mask, other=0.0
).to(tl.float32)
out = acc_base * scale + acc_delta
tl.store(
out_ptr + token * stride_o0 + col, out.to(out_ptr.dtype.element_ty), mask=mask
)
class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
"""LoRA-aware trtllm MoE experts."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.routing_method_type = moe_config.routing_method
self.topk = moe_config.experts_per_token
self.intermediate_size_per_partition = (
moe_config.intermediate_size_per_partition
)
self.hidden_dim = moe_config.hidden_dim
self.local_num_experts = moe_config.num_local_experts
self.ep_rank = moe_config.moe_parallel_config.ep_rank
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
p = current_platform
return (
p.is_cuda()
and p.is_device_capability_family(100)
and has_flashinfer_trtllm_fused_moe()
)
def workspace_shapes(
self,
M,
N,
K,
topk,
global_num_experts,
local_num_experts,
expert_tokens_meta,
activation,
):
# flashinfer manages its own workspace; only declare output (M, K)
return (0,), (0,), (M, K)
def moe_problem_size(self, a1, w1, w2, topk_ids):
"""Override the base 3D-weight assumption.
FusedMoEKernel._fused_experts calls moe_problem_size before apply(),
but the base impl asserts ``len(w1.shape) == 3``. The flashinfer
trtllm path stores shuffled weights in 4D BlockMajorK layout, so we
derive the (E, M, N, K, topk) tuple from config + inputs instead.
The N/K here only feed workspace sizing, which we zero out in
workspace_shapes(); the real shapes are handled inside flashinfer.
"""
E = self.local_num_experts
N = 2 * self.intermediate_size_per_partition
K = self.hidden_dim
M = a1.size(0) if a1.dim() == 2 else a1.size(1)
topk = topk_ids.size(1)
return E, M, N, K, topk
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
# apply() writes the fully finalized result into `output` (fused base
# finalize + W2 LoRA reduction), so this is a No-Op.
return TopKWeightAndReduceNoOP()
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return (
not moe_parallel_config.use_all2all_kernels
or moe_parallel_config.use_ag_rs_all2all_kernels
) and not moe_parallel_config.enable_eplb
@staticmethod
def _supports_router_logits_dtype(router_logits_dtype, routing_method) -> bool:
return True
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False # gated only
@property
def expects_unquantized_inputs(self) -> bool:
return True
@abstractmethod
def invoke_routed_moe(
self,
*,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids_and_weights: tuple[torch.Tensor, torch.Tensor],
gemm1_lora_delta: torch.Tensor | None,
global_num_experts: int,
a1q_scale: torch.Tensor | None,
output: torch.Tensor,
) -> list[torch.Tensor]:
"""Call the dtype-specific trtllm_*_routed_moe and return list[Tensor].
The LoRA path always sets gemm1_lora_delta and runs with
do_finalize=False so the base finalize can be fused with the W2 LoRA
reduction (see _finalize_with_w2_lora). Return contract:
gemm1_lora_delta is None -> [output] (do_finalize=True)
otherwise -> [gemm2_output(permuted, unweighted),
expert_weights,
expanded_idx_to_permuted_idx,
gemm1_activation_output(permuted)]
"""
raise NotImplementedError
def prepare_gemm1_lora_delta(
self,
gemm1_lora_delta: torch.Tensor,
lora_context: MoELoRAContext,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Hook to rewrite the W13 delta before it is handed to the kernel."""
return gemm1_lora_delta
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
lora_context = self._lora_context
assert lora_context is not None, "LoRA context must be set"
num_tokens = hidden_states.size(0)
top_k = self.topk
intermediate_size = self.intermediate_size_per_partition
# LoRA buffers use the padded hidden size; `output` may be narrower.
K = self.hidden_dim
# ---- Base-model fast path ----
# When no token in the batch selects a LoRA adapter, skip the LoRA machinery
# and run the plain base MoE with do_finalize=True, which writes the finalized
# result straight into `output`.
if self._batch_has_no_lora(lora_context):
self.invoke_routed_moe(
hidden_states=hidden_states,
w1=w1,
w2=w2,
topk_ids_and_weights=(topk_ids, topk_weights),
gemm1_lora_delta=None, # without LoRA, no delta
global_num_experts=global_num_experts,
a1q_scale=a1q_scale,
output=output,
)
return
# The LoRA tile-config heuristic (try_get_optimal_moe_config) unpacks
# w1/w2 as standard 3D MoE weights, but flashinfer stores shuffled
# 4D BlockMajorK weights. add_lora_w13/add_lora_w2 only read .shape
# from w1/w2 (the actual GEMM uses lora_a/b_stacked), so pass
# zero-storage meta tensors carrying the logical 3D shapes:
# w1: (E, 2I, H) w2: (E, H, I)
w1_cfg = torch.empty(
(self.local_num_experts, 2 * intermediate_size, K),
device="meta",
dtype=torch.bfloat16,
)
w2_cfg = torch.empty(
(self.local_num_experts, K, intermediate_size),
device="meta",
dtype=torch.bfloat16,
)
# ---- 1) W13 LoRA delta -> gemm1_lora_delta (bf16, [T, top_k, 2I]) ----
gemm1_lora_delta = None
w13_meta = (None, None, None, None)
# zeros (not empty): under EP the punica expand kernel only writes
# slots whose expert is local to this rank; the kernel skips the rest.
gemm1_lora_delta = torch.zeros(
num_tokens,
top_k,
2 * intermediate_size,
dtype=torch.bfloat16,
device=hidden_states.device,
)
lora_x = hidden_states
if not self.expects_unquantized_inputs:
orig = lora_context.original_hidden_states
assert orig is not None and orig.shape[0] == hidden_states.shape[0], (
"quantized trtllm LoRA path requires original_hidden_states"
)
lora_x = orig
# add_inputs=False: write the pure delta only (the base is fused in
# by the kernel) and do NOT multiply by the routing weight (it is a
# pre-SwiGLU bias).
# swap_w13_slices=True: apply_w13_lora writes in vLLM's w13 order
# (gate=w1 first, up=w3 second), but FlashInfer's gemm1_lora_delta
# expects [up, gate]; reversing the slices emits that order directly,
# avoiding an out-of-place concat swap.
w13_meta = self.apply_w13_lora(
lora_context,
y=gemm1_lora_delta,
x=lora_x,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
w1=w1_cfg,
w2=w2_cfg,
num_tokens=num_tokens,
top_k_num=top_k,
add_inputs=False,
swap_w13_slices=True,
)
gemm1_lora_delta = self.prepare_gemm1_lora_delta(
gemm1_lora_delta, lora_context, topk_ids
)
# ---- 2) Call the routed flashinfer kernel ----
ret = self.invoke_routed_moe(
hidden_states=hidden_states,
w1=w1,
w2=w2,
topk_ids_and_weights=(topk_ids, topk_weights),
gemm1_lora_delta=gemm1_lora_delta,
global_num_experts=global_num_experts,
a1q_scale=a1q_scale,
output=output,
)
# ---- 3) W2 LoRA (computed out of kernel) ----
# do_finalize=False: flashinfer returns the *unfinalized* base output.
# ret = [gemm2_output(permuted, unweighted),
# expert_weights, expanded_idx_to_permuted_idx,
# gemm1_activation_output(permuted)]
gemm2_permuted = ret[0]
expert_weights = ret[1]
expanded_idx_to_permuted_idx = ret[2]
gemm1_act_permuted = ret[3] # [max_padded, I], post-act
act = self._unpermute_activation(
gemm1_act_permuted,
expanded_idx_to_permuted_idx,
num_tokens,
top_k,
intermediate_size,
) # (T*top_k, I) -- same layout as the triton path's intermediate_cache2
(
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
token_lora_mapping,
) = w13_meta
w2_delta = torch.zeros(
num_tokens,
top_k,
K,
dtype=output.dtype,
device=output.device,
)
self.apply_w2_lora(
lora_context,
y=w2_delta,
x=act,
topk_weights=topk_weights,
sorted_token_ids_lora=sorted_token_ids_lora,
expert_ids_lora=expert_ids_lora,
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
token_lora_mapping=token_lora_mapping,
num_tokens=num_tokens,
w1=w1_cfg,
w2=w2_cfg,
top_k_num=top_k,
add_inputs=False,
)
# ---- 4) Fused finalize: reduce the base path over top_k (with routing
# weights) and add the already-weighted W2 delta, in a single kernel.
# This replaces flashinfer's internal finalize launch plus a separate
# w2_delta.sum(dim=1) + add_.
self._finalize_with_w2_lora(
output,
gemm2_permuted,
expert_weights,
expanded_idx_to_permuted_idx,
w2_delta,
num_tokens,
top_k,
scale=1.0,
)
@staticmethod
def _batch_has_no_lora(lora_context: MoELoRAContext) -> bool:
"""True when no token in the batch selects a LoRA adapter.
Mirrors the no-lora fast path in
``PunicaWrapperGPU.add_lora_fused_moe``: the punica kernel metadata
carries a CPU ``no_lora_flag`` computed once per forward from the
token->LoRA mapping. Reading it is a host-only check (no device sync),
and under CUDA graphs the branch is frozen at capture time against the
graph's ``has_lora`` dispatch key, so it stays correct on replay.
"""
meta = getattr(lora_context.punica_wrapper, "token_mapping_meta", None)
if meta is None:
return False
flag = meta.no_lora_flag_cpu
return bool(flag.numel() == 1 and flag.item())
@staticmethod
def _unpermute_activation(
act_permuted: torch.Tensor,
idx_map: torch.Tensor,
num_tokens: int,
top_k: int,
intermediate_size: int,
) -> torch.Tensor:
"""Permuted FC1 activation -> (num_tokens*top_k, I).
expanded_idx = token*top_k + k; idx_map[expanded_idx] = permuted_idx or -1.
Fused gather + drop-masking: each output row copies the matching
permuted row, or is zeroed when idx_map < 0.
"""
num_rows = num_tokens * top_k
out = torch.empty(
(num_rows, intermediate_size),
dtype=act_permuted.dtype,
device=act_permuted.device,
)
BLOCK_I = 1024
grid = (num_rows, triton.cdiv(intermediate_size, BLOCK_I))
_unpermute_activation_kernel[grid](
act_permuted,
idx_map,
out,
intermediate_size,
act_permuted.stride(0),
out.stride(0),
BLOCK_I=BLOCK_I,
)
return out
@staticmethod
def _finalize_with_w2_lora(
output: torch.Tensor,
gemm2_permuted: torch.Tensor,
expert_weights: torch.Tensor,
idx_map: torch.Tensor,
w2_delta: torch.Tensor,
num_tokens: int,
top_k: int,
scale: float = 1.0,
) -> None:
"""Fused base finalize + W2 LoRA reduction, written into ``output``.
For each token: sum the routing-weighted permuted base rows over top_k
(``expert_weights`` in expanded order, ``idx_map < 0`` dropped), scale by
``scale``, and add the already-weighted ``w2_delta`` reduced over top_k.
"""
# Inputs may carry hidden padding that `output` does not.
K = output.size(1)
assert gemm2_permuted.size(1) >= K and w2_delta.size(2) >= K
BLOCK_K = 512
grid = (num_tokens, triton.cdiv(K, BLOCK_K))
_finalize_lora_kernel[grid](
gemm2_permuted,
expert_weights.reshape(-1),
idx_map,
w2_delta,
output,
K,
gemm2_permuted.stride(0),
w2_delta.stride(0),
w2_delta.stride(1),
output.stride(0),
scale,
TOP_K=top_k,
BLOCK_K=BLOCK_K,
)
# BF16 unquantized trtllm MoE + LoRA
class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase):
"""BF16 unquantized trtllm MoE + LoRA."""
@staticmethod
def _supports_quant_scheme(weight_key, activation_key) -> bool:
return weight_key is None and activation_key is None
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [MoEActivation.SILU]
@staticmethod
def _supports_routing_method(routing_method, weight_key, activation_key) -> bool:
return routing_method in [
RoutingMethodType.DeepSeekV3,
RoutingMethodType.Llama4,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
def invoke_routed_moe(
self,
*,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids_and_weights: tuple[torch.Tensor, torch.Tensor],
gemm1_lora_delta: torch.Tensor | None,
global_num_experts: int,
a1q_scale: torch.Tensor | None,
output: torch.Tensor,
) -> list[torch.Tensor]:
import flashinfer
# With gemm1_lora_delta set (the LoRA path) run do_finalize=False and
# return the unfinalized permuted base output so apply() can fuse the
# finalize with the W2 LoRA reduction (see _finalize_with_w2_lora).
# Without a delta (base path), run do_finalize=True and hand flashinfer
# the caller's buffer via output= so it finalizes in place -- no copy.
do_finalize = gemm1_lora_delta is None
ret = flashinfer.fused_moe.trtllm_bf16_routed_moe(
topk_ids=topk_ids_and_weights,
hidden_states=hidden_states,
gemm1_weights=view_as_block_major_k(w1),
gemm2_weights=view_as_block_major_k(w2),
gemm1_lora_delta=gemm1_lora_delta,
num_experts=global_num_experts,
top_k=self.topk,
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=self.ep_rank * self.local_num_experts,
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
routing_method_type=self.routing_method_type,
do_finalize=do_finalize,
output=output if do_finalize else None,
)
if not do_finalize:
# [gemm2_output, expert_weights, expanded_idx, gemm1_activation]
return list(ret)
# do_finalize=True finalized directly into `output`.
return [output]
# MXFP4 weights x BF16 activations trtllm MoE + LoRA
class TrtLlmMxfp4LoRAExperts(_TrtLlmLoRAExpertsBase, TrtLlmMxfp4ExpertsBase):
"""MXFP4-weight, BF16-activation trtllm MoE + LoRA. Not MXFP8 activations:
the kernel then returns the FC1 activation quantized, without its scales."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
_TrtLlmLoRAExpertsBase.__init__(self, moe_config, quant_config)
# gemm1_alpha / gemm1_beta / gemm1_clamp_limit, hidden_dim_unpadded.
TrtLlmMxfp4ExpertsBase.__init__(self, moe_config, quant_config)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return weight_key == kMxfp4Static and activation_key is None
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
@staticmethod
def _supports_routing_method(routing_method, weight_key, activation_key) -> bool:
return True # routing is done outside the kernel
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
# DP gathers tokens, but the no-LoRA fast path reads the local batch.
p = moe_parallel_config
return p.dp_size == 1 and not p.use_all2all_kernels and not p.enable_eplb
def workspace_shapes(
self,
M,
N,
K,
topk,
global_num_experts,
local_num_experts,
expert_tokens_meta,
activation,
):
# flashinfer manages its own workspace and drops the hidden padding.
return (0,), (0,), (M, self.hidden_dim_unpadded)
def output_hidden_dim(self, hidden_dim: int) -> int:
# FusedMoEExpertsModular precedes TrtLlmMxfp4ExpertsBase in the MRO.
return self.hidden_dim_unpadded
@functools.cached_property
def _w13_bias_up_gate_index(self) -> torch.Tensor:
"""Columns of the kernel-format FC1 bias in [up | gate] order."""
from flashinfer.fused_moe.core import get_w2_permute_indices_with_cache
# Kernel format is the (up_i, gate_i) bias permuted for the tile.
assert self.w1_bias is not None
bias = self.w1_bias.reshape(self.local_num_experts, -1)[0]
permute = get_w2_permute_indices_with_cache({}, bias.reshape(-1, 1), 128)
interleaved = torch.argsort(permute.to(bias.device))
return torch.cat([interleaved[0::2], interleaved[1::2]])
def prepare_gemm1_lora_delta(
self,
gemm1_lora_delta: torch.Tensor,
lora_context: MoELoRAContext,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Bring the W13 delta to [up | gate] halves and fold in the FC1 bias."""
intermediate_size = gemm1_lora_delta.size(-1) // 2
# View as (I, 2) pairs of the first and second w13 half. SWIGLUOAI
# weights, and the deltas that follow them, are interleaved.
if self.moe_config.activation == MoEActivation.SWIGLUOAI:
pairs = gemm1_lora_delta.unflatten(-1, (intermediate_size, 2))
else:
pairs = gemm1_lora_delta.unflatten(-1, (2, intermediate_size)).mT
# Per-expert deltas arrive as (up, gate); fused 3D ones follow the
# checkpoint (gate, up) order.
if lora_context.w13_num_slices != 2:
pairs = pairs.flip(-1)
delta = pairs.mT.flatten(-2) # [up | gate]
# The kernel has a single FC1 bias slot, shared by the static bias and
# the per-token delta, so the static bias rides along in the delta.
if self.w1_bias is not None:
bias = self.w1_bias.reshape(self.local_num_experts, -1)
bias = bias[:, self._w13_bias_up_gate_index].float()
local_ids = topk_ids.long() - self.ep_rank * self.local_num_experts
local_ids = local_ids.clamp_(0, self.local_num_experts - 1)
delta = (delta.float() + bias[local_ids]).to(torch.bfloat16)
return delta.contiguous()
def invoke_routed_moe(
self,
*,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids_and_weights: tuple[torch.Tensor, torch.Tensor],
gemm1_lora_delta: torch.Tensor | None,
global_num_experts: int,
a1q_scale: torch.Tensor | None,
output: torch.Tensor,
) -> list[torch.Tensor]:
from flashinfer import trtllm_fp4_block_scale_routed_moe
assert a1q_scale is None and hidden_states.dtype == torch.bfloat16
topk_ids, topk_weights = topk_ids_and_weights
# See TrtLlmBf16LoRAExperts.invoke_routed_moe for the return contract.
do_finalize = gemm1_lora_delta is None
ret = trtllm_fp4_block_scale_routed_moe(
topk_ids=(topk_ids.to(torch.int32), topk_weights),
routing_bias=None,
hidden_states=hidden_states,
hidden_states_scale=None,
gemm1_weights=w1,
gemm1_weights_scale=self.w1_scale,
# With a delta the bias is folded into it (prepare_gemm1_lora_delta).
gemm1_bias=self.w1_bias if do_finalize else None,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.w2_scale,
gemm2_bias=self.w2_bias,
output1_scale_scalar=None,
output1_scale_gate_scalar=None,
output2_scale_scalar=None,
num_experts=global_num_experts,
top_k=self.topk,
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=self.ep_rank * self.local_num_experts,
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
# Routing is already done; Renormalize is a safe default.
routing_method_type=RoutingMethodType.Renormalize,
do_finalize=do_finalize,
enable_pdl=True,
activation_type=self._flashinfer_activation_type(
self.moe_config.activation
),
output=output if do_finalize else None,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
gemm1_lora_delta=gemm1_lora_delta,
)
return [output] if do_finalize else list(ret)
@@ -371,7 +371,6 @@ def FusedMoEFactory(
max_num_tokens=max_num_batched_tokens,
elastic_ep_max_dp_size=vllm_config.parallel_config.elastic_ep_max_dp_size,
has_bias=has_bias,
is_lora_enabled=vllm_config.lora_config is not None,
shared_expert_prefix=shared_expert_prefix,
activation=moe_activation,
device=vllm_config.device_config.device,
@@ -606,8 +606,6 @@ class FusedMoEExperts(ABC):
return False, _make_reason(f"{activation_format.value} activation format")
elif envs.VLLM_BATCH_INVARIANT and not cls._supports_batch_invariance():
return False, _make_reason("batch invariance")
elif moe_config.is_lora_enabled and not cls.supports_lora():
return False, _make_reason("LoRA")
return True, None
@staticmethod
@@ -772,15 +770,6 @@ class FusedMoEExperts(ABC):
def g2_alphas(self) -> torch.Tensor | None:
return self.quant_config.g2_alphas
@staticmethod
def supports_lora() -> bool:
"""Return True if this expert impl natively handles LoRA.
LoRA-aware experts should mix in LoRAExpertsMixin, which flips this
to True and provides the per-forward LoRA state plumbing.
"""
return False
def supports_packed_ue8m0_act_scales(self) -> bool:
"""A flag indicating whether or not this class can process packed ue8m0
activation scales.
@@ -1531,13 +1520,6 @@ class FusedMoEKernelModularImpl:
apply_router_weight_on_input,
)
# Stash the original unquantized hidden states on the LoRA context
# so apply_w13_lora sees correct-magnitude activations instead of
# the potentially quantized values produced by _prepare().
lora_ctx = getattr(self.fused_experts, "_lora_context", None)
if lora_ctx is not None:
lora_ctx.original_hidden_states = hidden_states
fused_out = self._fused_experts(
in_dtype=hidden_states.dtype,
a1q=a1q,
@@ -1555,9 +1537,6 @@ class FusedMoEKernelModularImpl:
output_alias=output,
)
if lora_ctx is not None:
lora_ctx.original_hidden_states = None
if isinstance(fused_out, UnfinalizedMoEOutput):
# Nothing below can run on an unfinalized output: finalize is the
# local top-k reduction (and, for DP/EP, the combine) that the
@@ -1733,9 +1712,6 @@ class FusedMoEKernel:
def moe_config(self) -> FusedMoEConfig:
return self.fused_experts.moe_config
def supports_lora(self) -> bool:
return self.fused_experts.supports_lora()
def _post_init_setup(self):
"""Resolve any leftover setup dependencies between self.prepare_finalize
and self.fused_experts here.
@@ -337,13 +337,11 @@ def pad_tp_shard_to_weight_blocks(
config.intermediate_size % block_n != 0
or config.hidden_dim % block_n != 0
or config.ep_size != 1
or config.is_lora_enabled
or config.has_bias
):
raise ValueError(
f"Block-aligned FP8 TP sharding requires {block_n}-aligned "
"global expert dimensions, pure TP, and no LoRA or "
"expert bias."
"global expert dimensions, pure TP, and no expert bias."
)
config.tp_shard_with_padding = True
config.intermediate_size_per_partition = round_up(
@@ -522,30 +522,6 @@ def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
return f"{base} since {reason}." if reason else f"{base}."
def _requested_kernel_cls(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
) -> list[type[mk.FusedMoEExperts]]:
"""Kernel classes for an explicitly requested backend. The LoRA-aware
TRT-LLM experts are opt-in: "auto" keeps its current LoRA selection."""
candidates = backend_to_kernel_cls(backend)
if not config.is_lora_enabled or backend not in TRTLLM_BACKENDS:
return candidates
if backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8:
raise ValueError(
_make_log_unsupported(
backend,
"expert LoRA is not supported with MXFP8 activations; use "
"moe_backend='flashinfer_trtllm'",
)
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
TrtLlmMxfp4LoRAExperts,
)
return [TrtLlmMxfp4LoRAExperts, *candidates]
def _return_or_raise(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
@@ -555,7 +531,7 @@ def _return_or_raise(
scope: Literal["process", "global", "local"] = "local",
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
reason: str | None = None
for k_cls in _requested_kernel_cls(backend, config):
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, config, weight_key, activation_key, activation_format
)
@@ -194,31 +194,6 @@ def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend
)
def _trtllm_bf16_lora_supported(moe_config: FusedMoEConfig) -> bool:
"""Gate for routing LoRA-enabled BF16 MoE to the FlashInfer TRT-LLM
gemm1_lora_delta path (PR #3153).
"""
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
TrtLlmBf16LoRAExperts,
)
# LoRA path returns before the oracle loop; reuse is_supported_config here.
supported, _ = TrtLlmBf16LoRAExperts.is_supported_config(
TrtLlmBf16LoRAExperts,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
if not supported:
return False
# The flashinfer trtllm fused-MoE kernel requires the per-partition
# intermediate size to be a multiple of 128. Plain TP shards the MoE
# intermediate dim (e.g. 768 -> 192 at tp=4), which would crash the kernel
# at runtime; fall back to Triton in that case.
return moe_config.intermediate_size_per_partition % 128 == 0
def select_unquantized_moe_backend(
moe_config: FusedMoEConfig,
) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]:
@@ -231,22 +206,6 @@ def select_unquantized_moe_backend(
if current_platform.is_out_of_tree():
return UnquantizedMoeBackend.OOT, None
if moe_config.is_lora_enabled:
if _trtllm_bf16_lora_supported(moe_config):
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
TrtLlmBf16LoRAExperts,
)
logger.info_once(
"Using TrtLlmBf16LoRAExperts Unquantized MoE LoRA backend "
"(TrtLlmBf16LoRAExperts)."
)
return UnquantizedMoeBackend.FLASHINFER_TRTLLM, TrtLlmBf16LoRAExperts
logger.info_once("Using TRITON Unquantized MoE LoRA backend")
return UnquantizedMoeBackend.TRITON, backend_to_kernel_cls(
UnquantizedMoeBackend.TRITON
)[0]
# NOTE: the kernels are selected in the following order.
AVAILABLE_BACKENDS = _get_priority_backends(moe_config)
@@ -93,14 +93,6 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
super().__init__()
self.is_sequence_parallel = is_sequence_parallel
self._num_dispatchers = num_dispatchers
# Set by FusedMoEWithLoRA.set_mapping() when LoRA is active. When
# present, prepare() dispatches the per-token LoRA mapping alongside
# hidden_states and writes the gathered result back to the context so
# experts can use the per-rank-local mapping.
self._lora_context = None
def set_lora_context(self, ctx) -> None:
self._lora_context = ctx
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
@@ -193,27 +185,9 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
a1, quant_config, defer_input_quant
)
# When LoRA is active, dispatch the per-token LoRA id along with
# hidden_states so every rank receives the correct mapping for the
# tokens it ends up processing. The punica_wrapper stores indices as
# int64 but the moe_lora_align_block_size kernel expects int32, so
# pull the pre-cast view from token_mapping_meta.
lora_ctx = self._lora_context
local_token_lora_mapping = None
if lora_ctx is not None:
local_token_lora_mapping = (
lora_ctx.punica_wrapper.token_mapping_meta.token_lora_mapping[
: a1.shape[0]
]
)
extra_tensors: list[torch.Tensor] | None = None
if scales is not None:
extra_tensors = list(scales)
if local_token_lora_mapping is not None:
if extra_tensors is None:
extra_tensors = []
extra_tensors.append(local_token_lora_mapping)
with self._comm_region():
res = get_ep_group().dispatch(
@@ -232,10 +206,6 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
assert len(res) == 4
a1q, topk_weights, topk_ids, gathered_extras = res
gathered_extras = list(gathered_extras)
if local_token_lora_mapping is not None:
dispatched_lora_mapping = gathered_extras.pop()
assert lora_ctx is not None
lora_ctx.local_token_lora_mapping = dispatched_lora_mapping
if scales is not None:
a1q_scale = _unwrap_scale_and_prepare_for_moe(
gathered_extras, quant_config
@@ -212,8 +212,6 @@ class RoutedExperts(PluggableLayer):
self.quant_method.create_weights(layer=self, **moe_quant_params)
self.lora_base_layer_prefix = ""
# TODO(bnell): Temporary hack. Get rid of this.
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
self.quant_method = quant_method
@@ -1121,7 +1119,6 @@ class RoutedExperts(PluggableLayer):
num_experts=moe_config.num_logical_experts + num_fused_shared_experts,
num_redundant_experts=num_redundant_experts,
routed_experts_prefix="",
lora_base_layer_prefix=self.lora_base_layer_prefix,
include_fused=include_fused,
is_gated=moe_config.is_act_and_mul,
)
@@ -1136,17 +1133,11 @@ class RoutedExperts(PluggableLayer):
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
) -> list[tuple[str, str, int, str]]:
"""Build the expert mapping, detecting the LoRA `base_layer.` prefix by
scanning `model`'s parameters.
Legacy entry point for models that still hand-roll `load_weights`; the
`RoutedExperts` weight loader uses `get_expert_mapping` /
`build_expert_params_mapping` instead (which take the prefix directly).
See `build_expert_params_mapping` for the returned tuple format.
"""Legacy entry point for models that still hand-roll `load_weights`;
the `RoutedExperts` weight loader uses `get_expert_mapping` /
`build_expert_params_mapping` instead. See
`build_expert_params_mapping` for the returned tuple format.
"""
has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters())
prefix = "base_layer." if has_base_layer else ""
# These loaders index ``params_dict[full_name]``, so both sides get it.
return RoutedExperts.build_expert_params_mapping(
ckpt_gate_proj_name,
ckpt_down_proj_name,
@@ -1154,8 +1145,6 @@ class RoutedExperts(PluggableLayer):
num_experts,
num_redundant_experts,
routed_experts_prefix,
lora_base_layer_prefix=prefix,
lora_base_layer_prefix_on_param_name=prefix,
)
@staticmethod
@@ -1166,8 +1155,6 @@ class RoutedExperts(PluggableLayer):
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
lora_base_layer_prefix: str = "",
lora_base_layer_prefix_on_param_name: str = "",
include_fused: bool = False,
is_gated: bool = True,
) -> list[tuple[str, str, int, str]]:
@@ -1182,14 +1169,6 @@ class RoutedExperts(PluggableLayer):
ckpt_up_proj_name: Name of up projection in checkpoint
num_experts: Number of logical (non-redundant) experts
num_redundant_experts: Number of redundant experts
lora_base_layer_prefix: LoRA ``base_layer.`` prefix for the
``weight_name`` (checkpoint) side
lora_base_layer_prefix_on_param_name: same, for the ``param_name``
side. Independent because ``get_expert_mapping`` resolves
``param_name`` via ``getattr`` against this layer's bare
``w13_weight``/``w2_weight`` (no prefix), while
``make_expert_params_mapping`` indexes the model-wide
``params_dict`` (prefix included).
include_fused: Prepend the fused pre-fused-checkpoint entries
is_gated: Whether w13 holds gate and up (gated activation) or a
single up projection. Selects the fused checkpoint layout.
@@ -1219,10 +1198,8 @@ class RoutedExperts(PluggableLayer):
if routed_experts_prefix != "":
routed_experts_prefix = f"{routed_experts_prefix}."
w13 = (
f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w13_"
)
w2 = f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w2_"
w13 = f"experts.{routed_experts_prefix}w13_"
w2 = f"experts.{routed_experts_prefix}w2_"
fused_mapping = []
if include_fused:
@@ -1260,10 +1237,7 @@ class RoutedExperts(PluggableLayer):
fused_mapping.extend(
(
w13,
(
f"experts.{physical_to_logical_map[expert_id]}."
f"{gate_up}.{lora_base_layer_prefix}"
),
f"experts.{physical_to_logical_map[expert_id]}.{gate_up}",
expert_id,
shard_id,
)
@@ -1275,7 +1249,7 @@ class RoutedExperts(PluggableLayer):
# (param_name, weight_name, expert_id, shard_id)
(
w13 if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] else w2,
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{lora_base_layer_prefix}",
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}",
expert_id,
shard_id,
)
@@ -195,8 +195,6 @@ def _moe_forward_shared_fake(
return shared_out, fused_out
# NOTE: `moe_forward` and `moe_forward_shared` being opaque custom ops is a
# load-bearing assumption for the MoE-LoRA dual-stream path.
direct_register_custom_op(
op_name="moe_forward",
op_func=_moe_forward,
@@ -83,7 +83,7 @@ class LogitsProcessor(PluggableLayer):
self.vocab_size = vocab_size
# Whether the input is logits (default is hidden states).
self.logits_as_input = logits_as_input
# original vocabulary size (without LoRA).
# original vocabulary size (without added embeddings).
self.org_vocab_size = org_vocab_size or vocab_size
# Soft cap the logits. Used in Gemma 2.
self.soft_cap = soft_cap
@@ -607,8 +607,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
# projection of the input hidden states
# Qwen3-Next and Qwen3.5 has a different qkv_proj layout,
# we need to create qkvz_proj adaptively here.
# When create_in_proj_qkvz is False (e.g. LoRA enabled in Qwen3.5),
# in_proj_qkv and in_proj_z are created separately instead.
# When create_in_proj_qkvz is False, in_proj_qkv and in_proj_z
# are created separately instead.
self.in_proj_qkvz = self.create_qkvz_proj(
hidden_size=self.hidden_size,
key_dim=self.key_dim,
@@ -1281,7 +1281,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
assert not hasattr(self, "in_proj_qkv"), "lora isn't supported on CPU."
assert not hasattr(self, "in_proj_qkv"), (
"the split in_proj_qkv/in_proj_z layout is not supported on CPU"
)
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
ba, _ = self.in_proj_ba(hidden_states)
@@ -106,7 +106,6 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
from vllm.platforms.rocm import on_gfx950
vllm_config = get_current_vllm_config()
is_lora_disabled = vllm_config.lora_config is None
moe_backend = vllm_config.kernel_config.moe_backend
group_size = weight_quant.group_size or -1
if (
@@ -114,7 +113,6 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
and weight_quant.type == QuantizationType.INT
and group_size == 32
and weight_quant.num_bits == 4
and is_lora_disabled
and on_gfx950()
and moe_backend == "flydsl"
):
@@ -518,12 +518,6 @@ def get_and_maybe_dequant_weights(
Fp8PerTensorOnlineLinearMethod,
)
# LoRA linear wrappers store quantization metadata on `base_layer`.
# Unwrap here so callers can pass either a raw linear layer or its LoRA
# wrapper without special-casing.
while hasattr(layer, "base_layer") and hasattr(layer.base_layer, "quant_method"):
layer = layer.base_layer
weight = get_attribute_fallback(layer, ["weight", "qweight", "weight_packed"])
# Unquantized layer: just return base weights
@@ -209,9 +209,9 @@ class VocabParallelEmbedding(PluggableLayer):
Adapted from torch.nn.Embedding, note that we pad the vocabulary size to
make sure it is divisible by the number of model parallel GPUs.
In order to support various loading methods, we ensure that LoRA-added
In order to support various loading methods, we ensure that added
embeddings are always at the end of TP-sharded tensors. In other words,
we shard base embeddings and LoRA embeddings separately (both padded),
we shard base embeddings and added embeddings separately (both padded),
and place them in the same tensor.
In this example, we will have the original vocab size = 1010,
added vocab size = 16 and padding to 64. Therefore, the total
@@ -219,16 +219,16 @@ class VocabParallelEmbedding(PluggableLayer):
1024, add 16, and then pad to 1088).
Therefore, the tensor format looks like the following:
TP1, rank 0 (no sharding):
|< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >|
|< --------BASE-------- >|< -BASE PADDING-- >|< -----ADDED----- >|< -ADDED PADDING- >|
corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1025 | -1 | ... | -1 |
index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 |
TP2, rank 0:
|< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >|
|< --------------------BASE--------------------- >|< -----ADDED----- >|< -ADDED PADDING- >|
corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1010 | ... | 1025 | -1 | ... | -1 |
index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 |
TP2, rank 1:
|< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >|
|< -----------BASE----------- >|< -BASE PADDING- >|< ----------ADDED PADDING---------- >|
corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 |
index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 |
@@ -236,7 +236,7 @@ class VocabParallelEmbedding(PluggableLayer):
num_embeddings: vocabulary size.
embedding_dim: size of hidden state.
params_dtype: type of the parameters.
org_num_embeddings: original vocabulary size (without LoRA).
org_num_embeddings: original vocabulary size (without added embeddings).
padding_size: padding size for the vocabulary.
quant_config: quant config for the layer
prefix: full name of the layer in the state dict
@@ -587,7 +587,7 @@ class ParallelLMHead(VocabParallelEmbedding):
embedding_dim: size of hidden state.
bias: whether to use bias.
params_dtype: type of the parameters.
org_num_embeddings: original vocabulary size (without LoRA).
org_num_embeddings: original vocabulary size (without added embeddings).
padding_size: padding size for the vocabulary.
disable_tp: If true, tensor parallelism will be disabled for this layer.
@@ -20,12 +20,8 @@ __all__ = ["get_parameter_for_reload", "reload_weights"]
def get_parameter_for_reload(model: nn.Module, name: str) -> nn.Parameter:
"""Resolve checkpoint names without changing the model's module tree."""
from vllm.lora.layers import BaseLayerWithLoRA
module_name, _, parameter_name = name.rpartition(".")
module = model.get_submodule(module_name)
if isinstance(module, BaseLayerWithLoRA):
module = module.base_layer
return module.get_parameter(parameter_name)
@@ -39,7 +35,7 @@ def reload_weights(
"""Reload a model's weights in place from a weights iterator or from disk.
Callers are responsible for invalidating any state derived from the old
weights (e.g. LoRA, encoder and multimodal caches).
weights (e.g. encoder and multimodal caches).
Args:
vllm_config: config of the model being reloaded. If `weights_path` is
@@ -63,10 +59,7 @@ def reload_weights(
model_config = vllm_config.model_config
load_config = vllm_config.load_config
weights_to_load = {
name.replace(".base_layer.", ".") if vllm_config.lora_config else name
for name, _ in model.named_parameters()
}
weights_to_load = {name for name, _ in model.named_parameters()}
counter_before_reloading = time.perf_counter()
# load weights from disk if none are provided
@@ -217,8 +217,8 @@ class ShardedStateLoader(BaseModelLoader):
state_dict = self._filter_subtensors(model.state_dict())
counter_before_loading_weights = time.perf_counter()
for key, tensor in self.iterate_over_files(filepaths):
# If loading with LoRA enabled, additional padding may
# be added to certain parameters. We only load into a
# Parameters padded beyond their checkpoint tensors
# may carry extra trailing rows. We only load into a
# narrowed view of the parameter data.
param = state_dict[key]
if param.dtype != tensor.dtype:
@@ -119,8 +119,6 @@ def initialize_model(
kwargs["cache_config"] = vllm_config.cache_config
if "quant_config" in all_params:
kwargs["quant_config"] = vllm_config.quant_config
if "lora_config" in all_params:
kwargs["lora_config"] = vllm_config.lora_config
if "scheduler_config" in all_params:
kwargs["scheduler_config"] = vllm_config.scheduler_config
with set_current_vllm_config(vllm_config, check_compile=True, prefix=prefix):
-8
View File
@@ -4,20 +4,16 @@
from .interfaces import (
HasInnerState,
SupportsHCSP,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsMultiModalEmbeddings,
SupportsPP,
SupportsTranscription,
has_inner_state,
supports_hc_sp,
supports_lora,
supports_mrope,
supports_multimodal,
supports_multimodal_embeddings,
supports_pp,
supports_transcription,
)
from .interfaces_base import (
VllmModelForPooling,
@@ -37,8 +33,6 @@ __all__ = [
"has_inner_state",
"SupportsHCSP",
"supports_hc_sp",
"SupportsLoRA",
"supports_lora",
"SupportsMultiModal",
"supports_multimodal",
"SupportsMultiModalEmbeddings",
@@ -47,6 +41,4 @@ __all__ = [
"supports_mrope",
"SupportsPP",
"supports_pp",
"SupportsTranscription",
"supports_transcription",
]
+5 -374
View File
@@ -1,19 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import weakref
from bisect import bisect_right
from collections.abc import (
AsyncGenerator,
Callable,
Hashable,
Mapping,
MutableSequence,
Sequence,
)
from contextlib import ExitStack, contextmanager, nullcontext
from dataclasses import dataclass
from operator import attrgetter
from typing import (
TYPE_CHECKING,
@@ -27,11 +23,9 @@ from typing import (
runtime_checkable,
)
import numpy as np
import torch
import torch.nn as nn
from torch import Tensor
from transformers.models.whisper.tokenization_whisper import LANGUAGES
from typing_extensions import TypeIs
from vllm.logger import init_logger
@@ -42,14 +36,10 @@ from vllm.utils.func_utils import supports_kw
if TYPE_CHECKING:
from vllm.config import (
ModelConfig,
SpeechToTextConfig,
SpeechToTextParams,
VllmConfig,
)
from vllm.config.multimodal import VideoPruningMethod
from vllm.inputs import PromptType, TokensPrompt
from vllm.lora.model_manager import LoRAModelManager
from vllm.inputs import TokensPrompt
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.mamba_utils import (
@@ -58,11 +48,10 @@ if TYPE_CHECKING:
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
from vllm.tokenizers import TokenizerLike
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
from vllm.v1.worker.encoder_cudagraph_defs import (
@@ -105,23 +94,6 @@ MambaStateShapes: TypeAlias = (
)
@dataclass(frozen=True, slots=True)
class DiarizedTranscriptionSegment:
"""A timestamped, speaker-attributed segment produced by an ASR model."""
start: float
end: float
speaker: str
text: str
class StreamingTranscriptionPostProcessor:
"""Stateful streaming post-processor for transcription deltas."""
def process_delta(self, text_delta: str, finished: bool) -> str:
return text_delta
def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
"""A helper function to be used in the context of
[vllm.model_executor.models.interfaces.SupportsMultiModal.embed_input_ids][]
@@ -206,12 +178,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
`multimodal_config.mm_device_do_normalize`.
"""
supports_tower_connector_lora: ClassVar[bool] = False
"""
A flag that indicates whether this model supports
`lora_config.enable_tower_connector_lora`.
"""
requires_raw_input_tokens: ClassVar[bool] = False
"""
A flag that indicates this model processes input id tokens
@@ -432,24 +398,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
yield
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: "MultiModalKwargsItem | None",
num_mm_embeds: int,
) -> tuple[int, int | None]:
"""Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
MM LoRA uses these counts to build adapter mappings for the tower and
connector forwards. Models with multiple modalities can override this
when each modality has different encoder padding or pooling behavior.
"""
raise NotImplementedError(
f"{type(self).__name__} must implement get_mm_lora_token_counts "
"to support tower/connector LoRA."
)
def _embed_text_input_ids(
self,
input_ids: Tensor,
@@ -459,9 +407,9 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
) -> Tensor:
if is_multimodal is not None and self._has_oov_mm_tokens:
# Force all input IDs to be in vocab; we do this instead of squeezing
# to ensure that any external configuration requiring offset tracking,
# e.g., LoRA, are applied correctly regardless of whether or not
# we have multimodal tokens.
# to ensure that any external configuration requiring offset tracking
# is applied correctly regardless of whether or not we have
# multimodal tokens.
in_vocab_ids = input_ids.masked_fill(
is_multimodal.to(device=input_ids.device, non_blocking=True), 0
)
@@ -712,85 +660,6 @@ def supports_kv_cache_grouping(
return isinstance(model, SupportsKVCacheGrouping)
@runtime_checkable
class SupportsLoRA(Protocol):
"""The interface required for all models that support LoRA."""
supports_lora: ClassVar[Literal[True]] = True
"""
A flag that indicates this model supports LoRA.
Note:
There is no need to redefine this flag if this class is in the
MRO of your model class.
"""
is_3d_moe_weight: ClassVar[bool] = False
is_non_gated_moe: ClassVar[bool] = False
# The `embedding_module` and `embedding_padding_modules`
# are empty by default.
embedding_modules: ClassVar[dict[str, str]] = {}
packed_modules_mapping: dict[str, list[str]] = {}
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
lora_skip_prefixes: ClassVar[list[str]] = []
lora_manager: "LoRAModelManager | None"
# We can't use runtime_checkable with ClassVar for issubclass checks
# so we need to treat the class as an instance and use isinstance instead
@runtime_checkable
class _SupportsLoRAType(Protocol):
supports_lora: Literal[True]
packed_modules_mapping: dict[str, list[str]]
embedding_modules: dict[str, str]
@overload
def supports_lora(model: type[object]) -> TypeIs[type[SupportsLoRA]]: ...
@overload
def supports_lora(model: object) -> TypeIs[SupportsLoRA]: ...
def supports_lora(
model: type[object] | object,
) -> TypeIs[type[SupportsLoRA]] | TypeIs[SupportsLoRA]:
result = _supports_lora(model)
if not result:
lora_attrs = (
"packed_modules_mapping",
"embedding_modules",
)
missing_attrs = tuple(attr for attr in lora_attrs if not hasattr(model, attr))
if getattr(model, "supports_lora", False):
if missing_attrs:
logger.warning(
"The model (%s) sets `supports_lora=True`, "
"but is missing LoRA-specific attributes: %s",
model,
missing_attrs,
)
else:
if not missing_attrs:
logger.warning(
"The model (%s) contains all LoRA-specific attributes, "
"but does not set `supports_lora=True`.",
model,
)
return result
def _supports_lora(model: type[object] | object) -> bool:
if isinstance(model, type):
return isinstance(model, _SupportsLoRAType)
return isinstance(model, SupportsLoRA)
class _MakeEmptyIntermediateTensors(Protocol):
def __call__(
self,
@@ -1289,244 +1158,6 @@ class SupportsQuant:
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
@runtime_checkable
class SupportsRealtime(Protocol):
"""The interface required for all models that support transcription."""
supports_realtime: ClassVar[Literal[True]] = True
realtime_max_tokens: ClassVar[int] = 1
"""Maximum tokens to generate per streaming audio segment.
Override in subclasses based on the model's expected output length."""
@classmethod
def buffer_realtime_audio(
cls,
audio_stream: AsyncGenerator[np.ndarray, None],
input_stream: asyncio.Queue[list[int]],
model_config: "ModelConfig",
) -> AsyncGenerator["PromptType", None]: ...
@overload
def supports_realtime(
model: type[object],
) -> TypeIs[type[SupportsRealtime]]: ...
@overload
def supports_realtime(model: object) -> TypeIs[SupportsRealtime]: ...
def supports_realtime(
model: type[object] | object,
) -> TypeIs[type[SupportsRealtime]] | TypeIs[SupportsRealtime]:
return getattr(model, "supports_realtime", False)
@runtime_checkable
class SupportsTranscription(Protocol):
"""The interface required for all models that support transcription."""
# Mapping from ISO639_1 language codes: language names
supported_languages: ClassVar[Mapping[str, str]]
supports_transcription: ClassVar[Literal[True]] = True
supports_transcription_only: ClassVar[bool] = False
"""
Transcription models can opt out of text generation by setting this to
`True`.
"""
supports_segment_timestamp: ClassVar[bool] = False
"""
Enables the segment timestamp option for supported models by setting this to `True`.
"""
supports_diarized_transcription: ClassVar[bool] = False
"""Enables the ``diarized_json`` response format for the model."""
supports_explicit_language_detection: ClassVar[bool] = False
"""
Transcription models that require an explicit language detection step
(e.g. Whisper needs a separate forward pass to predict the language
token) should set this to ``True`` and implement
:meth:`get_language_detection_prompt` and
:meth:`parse_language_detection_output` and
:meth:`get_language_token_ids`.
"""
no_space_languages: ClassVar[set[str]] = {"ja", "zh"}
"""
Languages that don't need a space between words.
For example, Japanese (ja) and Chinese (zh) don't need a space between words.
"""
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
# language codes in supported_languages
# that don't exist in the full language map
invalid = set(cls.supported_languages) - set(LANGUAGES.keys())
if invalid:
raise ValueError(
f"{cls.__name__}.supported_languages contains invalid "
f"language codes: {sorted(invalid)}\n. "
f"Valid choices are: {sorted(LANGUAGES.keys())}"
)
@classmethod
def get_generation_prompt(
cls,
stt_params: "SpeechToTextParams",
) -> "PromptType":
"""Get the prompt for the ASR model.
The model has control over the construction, as long as it
returns a valid PromptType."""
...
@classmethod
def get_other_languages(cls) -> Mapping[str, str]:
# other possible language codes from the whisper map
return {k: v for k, v in LANGUAGES.items() if k not in cls.supported_languages}
@classmethod
def validate_language(cls, language: str | None) -> str | None:
"""Ensure the language specified in the transcription request
is a valid ISO 639-1 language code. If the request language is
valid, but not natively supported by the model, trigger a
warning (but not an exception).
"""
if language is None or language in cls.supported_languages:
return language
elif language in cls.get_other_languages():
logger.warning(
"Language %r is not natively supported by %s; "
"results may be less accurate. Supported languages: %r",
language,
cls.__name__,
list(cls.supported_languages.keys()),
)
return language
else:
raise ValueError(
f"Unsupported language: {language!r}. Must be one of "
f"{list(cls.supported_languages.keys())}."
)
@classmethod
def get_speech_to_text_config(
cls, model_config: "ModelConfig", task_type: Literal["transcribe", "translate"]
) -> "SpeechToTextConfig":
"""Get the speech to text config for the ASR model."""
...
@classmethod
def get_num_audio_tokens(
cls,
audio_duration_s: float,
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
) -> int | None:
"""Map from audio duration to number of audio tokens produced by the ASR
model, without running a forward pass.
This is used for estimating the amount of processing for this audio.
"""
return None
@classmethod
def post_process_output(cls, text: str) -> str:
"""Post-process the raw model output text.
Some ASR models output structured formats (e.g., language tags,
special tokens) that need to be stripped before returning to the user.
Args:
text: Raw decoded text from the model.
Returns:
Cleaned transcription text.
"""
return text
@classmethod
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
"""Parse the model-specific diarized transcript format.
Only models that set ``supports_diarized_transcription`` must override
this method.
"""
raise NotImplementedError
@classmethod
def get_streaming_post_processor_cls(
cls,
) -> type[StreamingTranscriptionPostProcessor]:
"""Return a stateful post-processor class for streaming output deltas.
Each instance receives the next decoded text delta and whether the
request output is final. It returns the cleaned delta that should be
sent to the client.
"""
return StreamingTranscriptionPostProcessor
@classmethod
def get_language_detection_prompt(
cls,
audio: np.ndarray,
stt_config: "SpeechToTextConfig",
) -> "PromptType":
"""Return a prompt that triggers language detection.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@classmethod
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: "TokenizerLike",
) -> str:
"""Parse the detected language from model output token IDs.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@classmethod
def get_language_token_ids(
cls,
tokenizer: "TokenizerLike",
) -> list[int] | None:
"""Return token IDs that represent valid language tokens.
Used to constrain language detection to only produce valid language tokens.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@overload
def supports_transcription(
model: type[object],
) -> TypeIs[type[SupportsTranscription]]: ...
@overload
def supports_transcription(model: object) -> TypeIs[SupportsTranscription]: ...
def supports_transcription(
model: type[object] | object,
) -> TypeIs[type[SupportsTranscription]] | TypeIs[SupportsTranscription]:
return getattr(model, "supports_transcription", False)
@runtime_checkable
class SupportsEagleBase(Protocol):
"""Base interface for models that support EAGLE-based speculative decoding."""
+1 -2
View File
@@ -62,7 +62,6 @@ from .interfaces import (
EagleModelMixin,
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
SupportsQuant,
)
@@ -456,7 +455,7 @@ class Qwen2Model(nn.Module, EagleModelMixin):
class Qwen2ForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3, SupportsQuant
nn.Module, SupportsPP, SupportsEagle, SupportsEagle3, SupportsQuant
):
hf_to_vllm_mapper = Qwen2Model.hf_to_vllm_mapper
packed_modules_mapping = {
-17
View File
@@ -76,7 +76,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -103,7 +102,6 @@ from .interfaces import (
SupportsEagle,
SupportsEagle3,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsMultiModalPruning,
@@ -1295,7 +1293,6 @@ class Qwen2_5_VLForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsPP,
SupportsQuant,
SupportsEagle,
@@ -1323,7 +1320,6 @@ class Qwen2_5_VLForConditionalGeneration(
supports_encoder_tp_data = True
supports_mm_device_do_normalize = True
supports_tower_connector_lora = True
def iter_mm_grid_thw(
self, mm_features: list[MultiModalFeatureSpec]
@@ -2129,16 +2125,3 @@ class Qwen2_5_VLForConditionalGeneration(
connector="visual.merger.",
tower_model="visual.",
)
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_mm_embeds * merge_size**2, num_mm_embeds
+2 -2
View File
@@ -60,7 +60,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
)
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
from .interfaces import SupportsPP
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -420,7 +420,7 @@ class Qwen2MoeModel(nn.Module):
return hidden_states
class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
fall_back_to_pt_during_load = False
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
-17
View File
@@ -70,7 +70,6 @@ from vllm.multimodal.inputs import (
ImageItem,
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -97,7 +96,6 @@ from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
from .interfaces import (
MultiModalEmbeddings,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsPP,
@@ -1286,7 +1284,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo])
class Qwen2VLForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsLoRA,
SupportsPP,
SupportsMRoPE,
SupportsEncoderCudaGraph,
@@ -1305,7 +1302,6 @@ class Qwen2VLForConditionalGeneration(
supports_encoder_tp_data = True
supports_mm_device_do_normalize = True
supports_tower_connector_lora = True
def iter_mm_grid_thw(
self, mm_features: list[MultiModalFeatureSpec]
@@ -1860,16 +1856,3 @@ class Qwen2VLForConditionalGeneration(
connector="visual.merger.",
tower_model="visual.",
)
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_mm_embeds * merge_size**2, num_mm_embeds
+1 -6
View File
@@ -52,7 +52,6 @@ from .interfaces import (
LocalArgmaxMixin,
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
)
from .qwen2 import Qwen2MLP as Qwen3MLP
@@ -278,17 +277,13 @@ class Qwen3Model(Qwen2Model):
class Qwen3ForCausalLM(
LocalArgmaxMixin, nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
LocalArgmaxMixin, nn.Module, SupportsPP, SupportsEagle, SupportsEagle3
):
hf_to_vllm_mapper = Qwen3Model.hf_to_vllm_mapper
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
embedding_modules = {
"embed_tokens": "input_embeddings",
"lm_head": "output_embeddings",
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
-11
View File
@@ -25,7 +25,6 @@
"""Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
from collections.abc import Iterable
from typing import ClassVar
import torch
from torch import nn
@@ -76,7 +75,6 @@ from .interfaces import (
MixtureOfExperts,
MultiModalEmbeddings,
SupportsEagle3,
SupportsLoRA,
SupportsMRoPE,
SupportsPP,
_require_is_multimodal,
@@ -328,7 +326,6 @@ class Qwen3_5ForCausalLMBase(
HasInnerState,
IsHybrid,
SupportsEagle3,
SupportsLoRA,
SupportsMRoPE,
SupportsPP,
):
@@ -343,11 +340,6 @@ class Qwen3_5ForCausalLMBase(
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
"in_proj_ba": ["in_proj_b", "in_proj_a"],
}
# Maps PEFT embed/lm_head LoRA targets onto vLLM embedding wrappers.
embedding_modules = {
"embed_tokens": "input_embeddings",
"lm_head": "output_embeddings",
}
# Some community text-only checkpoints keep the extraneous
# `model.language_model.` prefix inherited from the VL training stack.
@@ -731,9 +723,6 @@ class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts):
class Qwen3_5MoeForConditionalGeneration(
Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts
):
# For MoE LoRA weights loading
is_3d_moe_weight: ClassVar[bool] = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
# protocols have not __init__ method, so we need to use nn.Module.__init__
nn.Module.__init__(self)
-2
View File
@@ -68,7 +68,6 @@ from .interfaces import (
IsHybrid,
MixtureOfExperts,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
)
from .utils import (
@@ -820,7 +819,6 @@ class QwenNextMixtureOfExperts(MixtureOfExperts):
class Qwen3NextForCausalLM(
nn.Module,
HasInnerState,
SupportsLoRA,
SupportsPP,
QwenNextMixtureOfExperts,
IsHybrid,
-16
View File
@@ -129,7 +129,6 @@ from .interfaces import (
SupportsEagle,
SupportsEagle3,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsMultiModalPruning,
@@ -1961,7 +1960,6 @@ class Qwen3VLForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsPP,
SupportsMRoPE,
SupportsEagle,
@@ -1982,7 +1980,6 @@ class Qwen3VLForConditionalGeneration(
}
supports_encoder_tp_data = True
supports_tower_connector_lora = True
supports_mm_device_do_normalize = True
supported_video_pruning_methods = ("evs", "vidcom2")
@@ -3258,19 +3255,6 @@ class Qwen3VLForConditionalGeneration(
tower_model="visual.",
)
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_mm_embeds * merge_size**2, num_mm_embeds
@lru_cache
def _cached_tensor(x, device) -> torch.Tensor:
-28
View File
@@ -55,7 +55,6 @@ from .interfaces import (
supports_multimodal_raw_input_only,
supports_pp,
supports_replayssm,
supports_transcription,
)
from .interfaces_base import (
get_attn_type,
@@ -232,8 +231,6 @@ class _ModelInfo:
is_hybrid: bool
has_noops: bool
supports_replayssm: bool
supports_transcription: bool
supports_transcription_only: bool
supports_kv_cache_grouping: bool
supported_video_pruning_methods: tuple[str, ...]
supports_mm_device_do_normalize: bool
@@ -262,10 +259,6 @@ class _ModelInfo:
is_attention_free=is_attention_free(model),
is_hybrid=is_hybrid(model),
supports_replayssm=supports_replayssm(model),
supports_transcription=supports_transcription(model),
supports_transcription_only=(
supports_transcription(model) and model.supports_transcription_only
),
supports_kv_cache_grouping=supports_kv_cache_grouping(model),
has_noops=has_noops(model),
supported_video_pruning_methods=getattr(
@@ -859,22 +852,6 @@ class _ModelRegistry:
model_info, _ = self.inspect_model_cls(architectures, model_config)
return model_info.has_noops
def is_transcription_model(
self,
architectures: str | list[str],
model_config: ModelConfig,
) -> bool:
model_info, _ = self.inspect_model_cls(architectures, model_config)
return model_info.supports_transcription
def is_transcription_only_model(
self,
architectures: str | list[str],
model_config: ModelConfig,
) -> bool:
model_info, _ = self.inspect_model_cls(architectures, model_config)
return model_info.supports_transcription_only
def _resolve_module_name(mod_relname: str) -> str:
# Allow registry entries to point at fully-qualified module paths (e.g.
@@ -929,11 +906,6 @@ def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
def _run() -> None:
# Setup plugins
from vllm.plugins import load_general_plugins
load_general_plugins()
fn, output_file = pickle.loads(sys.stdin.buffer.read())
result = fn()
+3 -3
View File
@@ -181,13 +181,13 @@ class WeightsMapper:
"""Mapper variant keeping only the renames.
This is what consumers that *name* modules rather than load them need:
LoRA name parsing and the quantization config's layer lists.
the quantization config's layer lists.
Stacked maps are dropped so that constituent names (e.g. `q_proj`) survive
rather than being rewritten to the stacked vLLM name (`qkv_proj`). Mappings to
`None` are dropped because "do not load this weight" is meaningless to such a
consumer, and applying it would silently shrink a quantization config's ignore
list or make LoRA name parsing fail."""
consumer, and applying it would silently shrink a quantization config's
ignore list."""
remove_none = lambda d: {k: v for k, v in d.items() if v is not None}
return replace(
self,