[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -800,6 +800,3 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
output=output,
|
||||
unit_scale_contract=self._quant_mode == "w4a16",
|
||||
)
|
||||
|
||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
||||
raise NotImplementedError("LoRA is not supported for B12xExperts")
|
||||
|
||||
@@ -1300,7 +1300,6 @@ class FusedMoEConfig:
|
||||
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
|
||||
elastic_ep_max_dp_size: int | None = None
|
||||
has_bias: bool = False
|
||||
is_lora_enabled: bool = False
|
||||
has_hash_routing: bool = False
|
||||
# Routes are adjusted between routing and expert execution.
|
||||
require_decomposed_backend: bool = False
|
||||
|
||||
@@ -307,7 +307,6 @@ class SubstitutedRoutedExperts(RoutedExperts):
|
||||
not isinstance(self.quant_method, UnquantizedFusedMoEMethod),
|
||||
),
|
||||
("monolithic MoE backends", self.quant_method.is_monolithic),
|
||||
("MoE LoRA", moe_config.is_lora_enabled),
|
||||
(
|
||||
"EPLB",
|
||||
parallel_config.enable_eplb
|
||||
@@ -352,7 +351,6 @@ class SubstitutedRoutedExperts(RoutedExperts):
|
||||
ckpt_up_proj_name or self.ckpt_up_proj_name,
|
||||
num_experts=self.expert_substitution.num_logical_experts,
|
||||
routed_experts_prefix="",
|
||||
lora_base_layer_prefix=self.lora_base_layer_prefix,
|
||||
)
|
||||
|
||||
def forward_modular(
|
||||
|
||||
@@ -387,8 +387,3 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
|
||||
use_w4_group_scaling=use_w4_group_scaling,
|
||||
)
|
||||
|
||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
||||
# No support for LoRA in flashinfer_cutlass_fused_moe.
|
||||
# See TODOs in flashinfer functions runMoe and runMoeMinLatency.
|
||||
raise NotImplementedError("LoRA is not supported for flashinfer_cutlass_moe")
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.lora.punica_wrapper.punica_base import PunicaWrapperBase
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoELoRAContext:
|
||||
"""Carries all LoRA state for one MoE forward pass.
|
||||
|
||||
Built by FusedMoEWithLoRA.forward() and propagated explicitly through the
|
||||
modular kernel path (FusedMoEKernel -> FusedMoEExpertsModular.apply) so
|
||||
that TritonExperts.apply() can compute the LoRA contribution inline,
|
||||
replacing the decorator-based monkey-patch approach.
|
||||
"""
|
||||
|
||||
# LoRA weight tensors (same shapes as FusedMoEWithLoRA attributes)
|
||||
w13_lora_a_stacked: tuple[torch.Tensor, ...]
|
||||
w13_lora_b_stacked: tuple[torch.Tensor, ...]
|
||||
w2_lora_a_stacked: tuple[torch.Tensor, ...]
|
||||
w2_lora_b_stacked: tuple[torch.Tensor, ...]
|
||||
|
||||
# (max_loras + 1,) int32; slot 0 is the "no-adapter" sentinel
|
||||
adapter_enabled: torch.Tensor
|
||||
|
||||
# Metadata
|
||||
max_loras: int
|
||||
top_k: int
|
||||
w13_num_slices: int # 2 = gated (gate + up), 1 = non-gated or 3D-fused
|
||||
fully_sharded: bool
|
||||
tp_rank: int
|
||||
tp_size: int
|
||||
local_num_experts: int
|
||||
|
||||
punica_wrapper: PunicaWrapperBase
|
||||
|
||||
# Whether VLLM_TUNED_CONFIG_FOLDER is set; selects get_lora_op_configs vs
|
||||
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
|
||||
use_tuned_config: bool
|
||||
|
||||
# Shared MoE LoRA: w13 lora_A and w2 lora_B are shared across all experts
|
||||
# (stored collapsed with expert-dim 1). When True, LoRAExpertsMixin expands
|
||||
# them to local_num_experts via a stride-0 view before the kernel.
|
||||
enable_moe_shared_loras: bool = False
|
||||
|
||||
# Optional dual-stream support for overlapping each (base GEMM, LoRA)
|
||||
# pair. When aux_stream is None, the experts.apply() path runs the
|
||||
# original sequential schedule. When set, base GEMM runs on the default
|
||||
# stream and the LoRA fast-path writes the delta into a fresh buffer on
|
||||
# aux_stream, which the default stream sums in afterwards.
|
||||
# Events are paired one-per-overlap-pair: events[0,1] for w13,
|
||||
# events[2,3] for w2, so the two pairs do not race on the same event.
|
||||
aux_stream: torch.cuda.Stream | None = None
|
||||
events: tuple[torch.cuda.Event, ...] | None = None
|
||||
|
||||
# Per-rank token→LoRA mapping after EP dispatch. Set by
|
||||
# FusedMoEPrepareAndFinalizeModular.prepare() when EP+LoRA is active, read
|
||||
# by LoRAExpertsMixin helpers in place of punica_wrapper's global mapping.
|
||||
# None means no dispatch happened (non-EP path), in which case callers
|
||||
# fall back to punica_wrapper.token_mapping_meta.
|
||||
local_token_lora_mapping: torch.Tensor | None = None
|
||||
|
||||
# Original unquantized hidden states, stashed by the modular kernel
|
||||
# before the prepare step potentially quantizes them. Used by
|
||||
# apply_w13_lora so the LoRA kernel sees correct-magnitude activations
|
||||
# instead of raw quantized values that are missing the activation scale.
|
||||
# Set per forward pass; None until the modular kernel writes it.
|
||||
original_hidden_states: torch.Tensor | None = None
|
||||
@@ -1,159 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
|
||||
|
||||
|
||||
class LoRAExpertsMixin:
|
||||
"""Mixin for FusedMoEExpertsModular subclasses that natively handle
|
||||
MoELoRAContext inside their apply() implementation.
|
||||
|
||||
Mixing this class in:
|
||||
- Flips supports_lora() to True so _can_fused_experts_support lets
|
||||
LoRA through the gate check.
|
||||
- Stashes a MoELoRAContext on the experts instance via
|
||||
set_lora_context(), which apply() consumes from self._lora_context.
|
||||
- Provides apply_w13_lora / apply_w2_lora helpers that dispatch to
|
||||
the PunicaWrapper kernels.
|
||||
|
||||
The helper methods are pure functions of their inputs; all required
|
||||
state is on lora_context or passed as arguments.
|
||||
"""
|
||||
|
||||
moe_config: FusedMoEConfig
|
||||
_lora_context: MoELoRAContext | None = None
|
||||
|
||||
def set_lora_context(self, ctx: MoELoRAContext) -> None:
|
||||
self._lora_context = ctx
|
||||
|
||||
@staticmethod
|
||||
def supports_lora() -> bool:
|
||||
return True
|
||||
|
||||
def apply_w13_lora(
|
||||
self,
|
||||
lora_context: MoELoRAContext,
|
||||
*,
|
||||
y: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
expert_map: torch.Tensor | None,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_tokens: int,
|
||||
top_k_num: int,
|
||||
add_inputs: bool = True,
|
||||
swap_w13_slices: bool = False,
|
||||
) -> tuple[
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
w13_lora_a_stacked = lora_context.w13_lora_a_stacked
|
||||
w13_lora_b_stacked = lora_context.w13_lora_b_stacked
|
||||
if lora_context.enable_moe_shared_loras:
|
||||
# w13 lora_A is shared across experts (collapsed expert-dim 1);
|
||||
# broadcast to local_num_experts via a stride-0 view. The kernel
|
||||
# derives num_experts from shape[1] and indexes via stride(1)==0.
|
||||
w13_lora_a_stacked = tuple(
|
||||
a.expand(-1, lora_context.local_num_experts, -1, -1)
|
||||
for a in w13_lora_a_stacked
|
||||
)
|
||||
if swap_w13_slices:
|
||||
# The expand kernel writes slice j into the j-th half of y's last
|
||||
# dim. Reversing the (gate, up) slice tuples makes it emit
|
||||
# [up, gate] order directly -- used by the FlashInfer trtllm path,
|
||||
# whose SwiGLU expects the up half first, to avoid an out-of-place
|
||||
# concat swap afterwards.
|
||||
w13_lora_a_stacked = w13_lora_a_stacked[::-1]
|
||||
w13_lora_b_stacked = w13_lora_b_stacked[::-1]
|
||||
interleave_w13 = (
|
||||
lora_context.w13_num_slices == 2
|
||||
and self.moe_config.activation == MoEActivation.SWIGLUOAI
|
||||
)
|
||||
lora_output = torch.zeros_like(y) if interleave_w13 else y
|
||||
result = lora_context.punica_wrapper.add_lora_w13(
|
||||
lora_output,
|
||||
x,
|
||||
w13_lora_a_stacked,
|
||||
w13_lora_b_stacked,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
expert_map,
|
||||
w1,
|
||||
w2,
|
||||
num_tokens,
|
||||
top_k_num,
|
||||
lora_context.max_loras,
|
||||
lora_context.adapter_enabled,
|
||||
lora_context.local_num_experts,
|
||||
lora_context.top_k,
|
||||
lora_context.w13_num_slices,
|
||||
lora_context.fully_sharded,
|
||||
lora_context.use_tuned_config,
|
||||
add_inputs=add_inputs or interleave_w13,
|
||||
token_lora_mapping=lora_context.local_token_lora_mapping,
|
||||
)
|
||||
if interleave_w13:
|
||||
# Map gate/up slices onto even/odd output channels without a copy.
|
||||
gate_up_delta = lora_output.unflatten(-1, (2, -1))
|
||||
gate_up_output = y.unflatten(-1, (-1, 2)).transpose(-1, -2)
|
||||
if add_inputs:
|
||||
gate_up_output.add_(gate_up_delta)
|
||||
else:
|
||||
gate_up_output.copy_(gate_up_delta)
|
||||
return result
|
||||
|
||||
def apply_w2_lora(
|
||||
self,
|
||||
lora_context: MoELoRAContext,
|
||||
*,
|
||||
y: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
sorted_token_ids_lora: torch.Tensor | None,
|
||||
expert_ids_lora: torch.Tensor | None,
|
||||
num_tokens_post_padded_lora: torch.Tensor | None,
|
||||
token_lora_mapping: torch.Tensor | None,
|
||||
num_tokens: int,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
top_k_num: int,
|
||||
add_inputs: bool = True,
|
||||
) -> None:
|
||||
w2_lora_b_stacked = lora_context.w2_lora_b_stacked
|
||||
if lora_context.enable_moe_shared_loras:
|
||||
# w2 lora_B is shared across experts (collapsed expert-dim 1);
|
||||
# broadcast to local_num_experts via a stride-0 view.
|
||||
w2_lora_b_stacked = tuple(
|
||||
b.expand(-1, lora_context.local_num_experts, -1, -1)
|
||||
for b in w2_lora_b_stacked
|
||||
)
|
||||
lora_context.punica_wrapper.add_lora_w2(
|
||||
y,
|
||||
x,
|
||||
lora_context.w2_lora_a_stacked,
|
||||
w2_lora_b_stacked,
|
||||
topk_weights,
|
||||
sorted_token_ids_lora,
|
||||
expert_ids_lora,
|
||||
num_tokens_post_padded_lora,
|
||||
token_lora_mapping,
|
||||
num_tokens,
|
||||
w1,
|
||||
w2,
|
||||
top_k_num,
|
||||
lora_context.max_loras,
|
||||
lora_context.adapter_enabled,
|
||||
lora_context.top_k,
|
||||
lora_context.fully_sharded,
|
||||
lora_context.tp_rank,
|
||||
lora_context.use_tuned_config,
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
@@ -21,9 +21,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
|
||||
LoRAExpertsMixin,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
batched_moe_align_block_size,
|
||||
moe_align_block_size,
|
||||
@@ -673,7 +670,7 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
return E, M, N, K, topk
|
||||
|
||||
|
||||
class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
class MarlinExperts(MarlinExpertsBase):
|
||||
"""Marlin-based fused MoE expert implementation."""
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
@@ -736,121 +733,7 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
|
||||
ctx = self._lora_context
|
||||
if ctx is None:
|
||||
fused_marlin_moe(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
bias1=self.w1_bias,
|
||||
bias2=self.w2_bias,
|
||||
w1_scale=self.w1_scale,
|
||||
w2_scale=self.w2_scale,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_scale1=self.g1_alphas,
|
||||
global_scale2=self.g2_alphas,
|
||||
input_global_scale1=self.a1_gscale,
|
||||
input_global_scale2=self.a2_gscale,
|
||||
w1_zeros=self.w1_zp,
|
||||
w2_zeros=self.w2_zp,
|
||||
quant_type_id=self.quant_type_id,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
global_num_experts=global_num_experts,
|
||||
activation=activation,
|
||||
activation_func=self.activation,
|
||||
activation_config=self.activation_config,
|
||||
moe_sum=self.moe_sum,
|
||||
expert_map=expert_map,
|
||||
output=output,
|
||||
# Workspaces are swapped in workspace_shapes() to account for proper
|
||||
# output buffer allocation. Please refer to workspace_shapes().
|
||||
intermediate_cache13=workspace2,
|
||||
intermediate_cache2=workspace13,
|
||||
input_dtype=self.input_dtype,
|
||||
)
|
||||
return
|
||||
|
||||
# LoRA path: wrap activation_func and moe_sum to inject LoRA at the
|
||||
# two natural injection points.
|
||||
#
|
||||
# Marlin uses moe_align_block_size (same as TritonExperts) so
|
||||
# intermediate_cache1 is indexed by flat (token, expert) pair index,
|
||||
# which is compatible with add_lora_fused_moe's scatter mechanism.
|
||||
|
||||
M = hidden_states.size(0)
|
||||
top_k_num = topk_ids.size(1)
|
||||
lora_state: dict = {}
|
||||
|
||||
def activation_with_lora(
|
||||
act_enum: MoEActivation,
|
||||
act_output: torch.Tensor,
|
||||
act_input: torch.Tensor,
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# act_input = intermediate_cache1 (M*topk, 2N for gated)
|
||||
# act_output = intermediate_cache2 (M*topk, N)
|
||||
|
||||
(
|
||||
sorted_token_ids_lora,
|
||||
expert_ids_lora,
|
||||
num_tokens_post_padded_lora,
|
||||
token_lora_mapping,
|
||||
) = self.apply_w13_lora(
|
||||
ctx,
|
||||
y=act_input,
|
||||
x=hidden_states,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
expert_map=expert_map,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
num_tokens=M,
|
||||
top_k_num=top_k_num,
|
||||
)
|
||||
lora_state.update(
|
||||
{
|
||||
"sorted": sorted_token_ids_lora,
|
||||
"eids": expert_ids_lora,
|
||||
"npad": num_tokens_post_padded_lora,
|
||||
"tlm": token_lora_mapping,
|
||||
}
|
||||
)
|
||||
self.activation(
|
||||
act_enum,
|
||||
act_output,
|
||||
act_input,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
lora_state["cache2"] = act_output
|
||||
|
||||
def moe_sum_with_lora(
|
||||
moe_out: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
expert_map: torch.Tensor | None,
|
||||
) -> None:
|
||||
# moe_out shape: (M, topk, K)
|
||||
self.apply_w2_lora(
|
||||
ctx,
|
||||
y=moe_out,
|
||||
x=lora_state["cache2"],
|
||||
topk_weights=topk_weights,
|
||||
sorted_token_ids_lora=lora_state["sorted"],
|
||||
expert_ids_lora=lora_state["eids"],
|
||||
num_tokens_post_padded_lora=lora_state["npad"],
|
||||
token_lora_mapping=lora_state["tlm"],
|
||||
num_tokens=M,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
top_k_num=top_k_num,
|
||||
)
|
||||
self.moe_sum(moe_out, out, topk_ids, expert_map)
|
||||
|
||||
return fused_marlin_moe(
|
||||
fused_marlin_moe(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
@@ -870,11 +753,13 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
global_num_experts=global_num_experts,
|
||||
activation=activation,
|
||||
activation_func=activation_with_lora,
|
||||
activation_func=self.activation,
|
||||
activation_config=self.activation_config,
|
||||
moe_sum=moe_sum_with_lora,
|
||||
moe_sum=self.moe_sum,
|
||||
expert_map=expert_map,
|
||||
output=output,
|
||||
# Workspaces are swapped in workspace_shapes() to account for proper
|
||||
# output buffer allocation. Please refer to workspace_shapes().
|
||||
intermediate_cache13=workspace2,
|
||||
intermediate_cache2=workspace13,
|
||||
input_dtype=self.input_dtype,
|
||||
|
||||
@@ -374,10 +374,6 @@ class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
|
||||
# Used in experts/triton_moe.py and passed to moe_kernel_quantize_input.
|
||||
return self.a1_gscale
|
||||
|
||||
@staticmethod
|
||||
def supports_lora() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
@@ -386,8 +382,6 @@ class Nvfp4QuantizationEmulationTritonExperts(TritonExperts):
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
if moe_config.is_lora_enabled:
|
||||
return False, "kernel does not support LoRA"
|
||||
if moe_config.has_bias:
|
||||
return False, "kernel does not support bias"
|
||||
|
||||
|
||||
@@ -15,9 +15,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
|
||||
LoRAExpertsMixin,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
_prepare_expert_assignment,
|
||||
invoke_fused_moe_triton_kernel,
|
||||
@@ -59,10 +56,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl
|
||||
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
||||
|
||||
|
||||
class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
class TritonExperts(mk.FusedMoEExpertsModular):
|
||||
"""Triton-based fused MoE expert implementation."""
|
||||
|
||||
@staticmethod
|
||||
@@ -104,16 +100,6 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# Defer activation quantization to apply() only when LoRA is active AND
|
||||
# tokens are dispatched across ranks (DP+EP all2all).
|
||||
return (
|
||||
self._lora_context is not None
|
||||
and self.quant_dtype is not None
|
||||
and self.moe_config.moe_parallel_config.use_all2all_kernels
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
@@ -279,16 +265,13 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
torch.int8,
|
||||
]
|
||||
|
||||
# We declared expects_unquantized_inputs (LoRA + DP/EP all2all), so the
|
||||
# prepare step deferred activation quantization to this kernel:
|
||||
# `hidden_states` arrives unquantized. Keep the unquantized tensor for
|
||||
# the LoRA shrink input and quantize a copy here for the base GEMM
|
||||
# (mirrors what the prepare step would have done, but after the
|
||||
# all-gather so the layout matches the gathered topk_ids / token map).
|
||||
lora_unquantized_hidden_states: torch.Tensor | None = None
|
||||
# The prepare step deferred activation quantization to this kernel:
|
||||
# `hidden_states` arrives unquantized. Quantize a copy here for the
|
||||
# base GEMM (mirrors what the prepare step would have done, but after
|
||||
# the all-gather so the layout matches the gathered topk_ids / token
|
||||
# map).
|
||||
if self.expects_unquantized_inputs:
|
||||
assert a1q_scale is None
|
||||
lora_unquantized_hidden_states = hidden_states
|
||||
hidden_states, a1q_scale = moe_kernel_quantize_input(
|
||||
hidden_states,
|
||||
self.a1_scale,
|
||||
@@ -343,7 +326,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
# Their rows in the workspace caches are then never written, so every
|
||||
# consumer of those caches must be row-local: a dynamic per-tensor
|
||||
# scale for the second GEMM's activation would take its amax over the
|
||||
# untouched rows, and LoRA aligns the full `topk_ids` itself.
|
||||
# untouched rows.
|
||||
a2_scale_is_global = (
|
||||
self.quant_dtype is not None
|
||||
and a2_scale is None
|
||||
@@ -352,7 +335,6 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
)
|
||||
skip_invalid = (
|
||||
expert_map is not None
|
||||
and self._lora_context is None
|
||||
and not a2_scale_is_global
|
||||
)
|
||||
|
||||
@@ -373,34 +355,6 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
)
|
||||
)
|
||||
|
||||
# LoRA w13: applied to intermediate_cache1 before activation. When
|
||||
# the LoRA layer requested a dual-stream schedule, we run base w13
|
||||
# GEMM on the default stream and the LoRA fast-path on aux_stream;
|
||||
# the LoRA writes its delta into a fresh zero buffer (add_inputs=
|
||||
# False) and we sum it into intermediate_cache1 after both finish.
|
||||
#
|
||||
# The LoRA shrink kernel needs unquantized, gathered-layout
|
||||
# activations. When activation quant was deferred to this kernel
|
||||
# (expects_unquantized_inputs), the input we quantized above is exactly
|
||||
# that, so use it directly. Otherwise fall back to the context stash
|
||||
# (e.g. weight-only quant), guarding on a row-count match so a
|
||||
# DP-gathered layout never indexes a local stash out of bounds.
|
||||
sorted_token_ids_lora = None
|
||||
expert_ids_lora = None
|
||||
num_tokens_post_padded_lora = None
|
||||
token_lora_mapping = None
|
||||
lora_context = self._lora_context
|
||||
if lora_unquantized_hidden_states is not None:
|
||||
lora_x = lora_unquantized_hidden_states
|
||||
elif (
|
||||
lora_context is not None
|
||||
and lora_context.original_hidden_states is not None
|
||||
and lora_context.original_hidden_states.shape[0] == hidden_states.shape[0]
|
||||
):
|
||||
lora_x = lora_context.original_hidden_states
|
||||
else:
|
||||
lora_x = hidden_states
|
||||
|
||||
# TODO: The fallback to self.a1_scale was added for deferred static
|
||||
# activation quantization in https://github.com/vllm-project/vllm/pull/40857.
|
||||
# Activation emulation relies solely on `a1q_scale` output of
|
||||
@@ -412,93 +366,34 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
else (a1q_scale if a1q_scale is not None else self.a1_scale)
|
||||
)
|
||||
|
||||
def _base_w13_fn():
|
||||
invoke_fused_moe_triton_kernel(
|
||||
hidden_states,
|
||||
w1,
|
||||
intermediate_cache1,
|
||||
input_scale,
|
||||
self.w1_scale,
|
||||
None, # topk_weights
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
False, # mul_routed_weights
|
||||
top_k_num,
|
||||
config,
|
||||
compute_type=compute_type,
|
||||
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
|
||||
use_int8_w8a8=self.quant_config.use_int8_w8a8,
|
||||
use_int8_w8a16=self.quant_config.use_int8_w8a16,
|
||||
use_int4_w4a16=self.quant_config.use_int4_w4a16,
|
||||
per_channel_quant=self.per_act_token_quant,
|
||||
block_shape=self.block_shape,
|
||||
B_bias=self.w1_bias,
|
||||
)
|
||||
|
||||
if lora_context is not None and lora_context.aux_stream is not None:
|
||||
# add_inputs=False: kernel overwrites lora_delta_w13. zeros (not
|
||||
# empty) so untouched rows -- e.g. blocks where every program
|
||||
# early-exits because lora_id<0 -- stay at zero and the trailing
|
||||
# add_() is a no-op there.
|
||||
lora_delta_w13 = torch.zeros_like(intermediate_cache1)
|
||||
|
||||
def _lora_w13_fn():
|
||||
return self.apply_w13_lora(
|
||||
lora_context,
|
||||
y=lora_delta_w13,
|
||||
x=lora_x,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
expert_map=expert_map,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
num_tokens=num_tokens,
|
||||
top_k_num=top_k_num,
|
||||
add_inputs=False,
|
||||
)
|
||||
|
||||
assert lora_context.events is not None
|
||||
_, lora_meta = maybe_execute_in_parallel(
|
||||
_base_w13_fn,
|
||||
_lora_w13_fn,
|
||||
lora_context.events[0],
|
||||
lora_context.events[1],
|
||||
lora_context.aux_stream,
|
||||
)
|
||||
(
|
||||
sorted_token_ids_lora,
|
||||
expert_ids_lora,
|
||||
num_tokens_post_padded_lora,
|
||||
token_lora_mapping,
|
||||
) = lora_meta
|
||||
intermediate_cache1.add_(lora_delta_w13)
|
||||
else:
|
||||
_base_w13_fn()
|
||||
if lora_context is not None:
|
||||
(
|
||||
sorted_token_ids_lora,
|
||||
expert_ids_lora,
|
||||
num_tokens_post_padded_lora,
|
||||
token_lora_mapping,
|
||||
) = self.apply_w13_lora(
|
||||
lora_context,
|
||||
y=intermediate_cache1,
|
||||
x=lora_x,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
expert_map=expert_map,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
num_tokens=num_tokens,
|
||||
top_k_num=top_k_num,
|
||||
)
|
||||
invoke_fused_moe_triton_kernel(
|
||||
hidden_states,
|
||||
w1,
|
||||
intermediate_cache1,
|
||||
input_scale,
|
||||
self.w1_scale,
|
||||
None, # topk_weights
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
False, # mul_routed_weights
|
||||
top_k_num,
|
||||
config,
|
||||
compute_type=compute_type,
|
||||
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
|
||||
use_int8_w8a8=self.quant_config.use_int8_w8a8,
|
||||
use_int8_w8a16=self.quant_config.use_int8_w8a16,
|
||||
use_int4_w4a16=self.quant_config.use_int4_w4a16,
|
||||
per_channel_quant=self.per_act_token_quant,
|
||||
block_shape=self.block_shape,
|
||||
B_bias=self.w1_bias,
|
||||
)
|
||||
|
||||
a2q_scale: torch.Tensor | None = None
|
||||
|
||||
# Fuse SiLU+Mul + FP8 block quantize into a single kernel
|
||||
# when conditions permit (gated SiLU, fp8 block quant with
|
||||
# group_size=128, no LoRA requiring the BF16 intermediate).
|
||||
# group_size=128).
|
||||
# The fused kernel has no clamp parameter, so a configured
|
||||
# SwiGLU clamp limit falls through to the unfused path.
|
||||
if (
|
||||
@@ -506,7 +401,6 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
and self.activation_config.clamp_limit is None
|
||||
and self.quant_config.use_fp8_w8a8
|
||||
and self.block_shape == [128, 128]
|
||||
and lora_context is None
|
||||
and not is_deep_gemm_e8m0_used()
|
||||
):
|
||||
qintermediate_cache2, a2q_scale = ops.silu_and_mul_per_block_quant(
|
||||
@@ -528,83 +422,29 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
quantization_emulation=self.quantization_emulation,
|
||||
)
|
||||
|
||||
# LoRA w2: applied to intermediate_cache3 before moe_sum, using the
|
||||
# unquantized intermediate_cache2 as the lora_a input. Reuses the
|
||||
# sorted_token_ids_lora computed above. Same dual-stream pattern as
|
||||
# the w13 pair: base GEMM on default stream, LoRA delta on aux,
|
||||
# join via .add_() into intermediate_cache3.
|
||||
def _base_w2_fn():
|
||||
invoke_fused_moe_triton_kernel(
|
||||
qintermediate_cache2,
|
||||
w2,
|
||||
intermediate_cache3,
|
||||
a2q_scale,
|
||||
self.w2_scale,
|
||||
topk_weights,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
not apply_router_weight_on_input,
|
||||
1,
|
||||
config,
|
||||
compute_type=compute_type,
|
||||
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
|
||||
use_int8_w8a8=self.quant_config.use_int8_w8a8,
|
||||
use_int8_w8a16=self.quant_config.use_int8_w8a16,
|
||||
use_int4_w4a16=self.quant_config.use_int4_w4a16,
|
||||
per_channel_quant=self.per_act_token_quant,
|
||||
block_shape=self.block_shape,
|
||||
B_bias=self.w2_bias,
|
||||
)
|
||||
invoke_fused_moe_triton_kernel(
|
||||
qintermediate_cache2,
|
||||
w2,
|
||||
intermediate_cache3,
|
||||
a2q_scale,
|
||||
self.w2_scale,
|
||||
topk_weights,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
not apply_router_weight_on_input,
|
||||
1,
|
||||
config,
|
||||
compute_type=compute_type,
|
||||
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
|
||||
use_int8_w8a8=self.quant_config.use_int8_w8a8,
|
||||
use_int8_w8a16=self.quant_config.use_int8_w8a16,
|
||||
use_int4_w4a16=self.quant_config.use_int4_w4a16,
|
||||
per_channel_quant=self.per_act_token_quant,
|
||||
block_shape=self.block_shape,
|
||||
B_bias=self.w2_bias,
|
||||
)
|
||||
|
||||
if lora_context is not None and lora_context.aux_stream is not None:
|
||||
lora_delta_w2 = torch.zeros_like(intermediate_cache3)
|
||||
|
||||
def _lora_w2_fn():
|
||||
self.apply_w2_lora(
|
||||
lora_context,
|
||||
y=lora_delta_w2,
|
||||
x=intermediate_cache2,
|
||||
topk_weights=topk_weights,
|
||||
sorted_token_ids_lora=sorted_token_ids_lora,
|
||||
expert_ids_lora=expert_ids_lora,
|
||||
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
|
||||
token_lora_mapping=token_lora_mapping,
|
||||
num_tokens=num_tokens,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
top_k_num=top_k_num,
|
||||
add_inputs=False,
|
||||
)
|
||||
|
||||
assert lora_context.events is not None
|
||||
maybe_execute_in_parallel(
|
||||
_base_w2_fn,
|
||||
_lora_w2_fn,
|
||||
lora_context.events[2],
|
||||
lora_context.events[3],
|
||||
lora_context.aux_stream,
|
||||
)
|
||||
intermediate_cache3.add_(lora_delta_w2)
|
||||
else:
|
||||
_base_w2_fn()
|
||||
if lora_context is not None:
|
||||
self.apply_w2_lora(
|
||||
lora_context,
|
||||
y=intermediate_cache3,
|
||||
x=intermediate_cache2,
|
||||
topk_weights=topk_weights,
|
||||
sorted_token_ids_lora=sorted_token_ids_lora,
|
||||
expert_ids_lora=expert_ids_lora,
|
||||
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
|
||||
token_lora_mapping=token_lora_mapping,
|
||||
num_tokens=num_tokens,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
top_k_num=top_k_num,
|
||||
)
|
||||
|
||||
# separate function is required for MoE + LoRA
|
||||
self.moe_sum(intermediate_cache3, output, topk_ids, expert_map)
|
||||
|
||||
def moe_sum(
|
||||
@@ -804,5 +644,4 @@ class TritonWNA16Experts(TritonExperts):
|
||||
block_shape=self.block_shape,
|
||||
)
|
||||
|
||||
# separate function is required for MoE + LoRA
|
||||
self.moe_sum(intermediate_cache3, output)
|
||||
|
||||
@@ -1,731 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""LoRA-aware FlashInfer TRT-LLM MoE experts (BF16, MXFP4 x BF16).
|
||||
|
||||
Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153
|
||||
(BF16) and PR #3987 (FP4 block-scale):
|
||||
|
||||
* The W13 (gate_up) LoRA delta is passed directly as ``gemm1_lora_delta`` to
|
||||
the routed kernel, which fuses it into FC1 before SwiGLU (BiasType::Mn).
|
||||
* The W2 (down) LoRA cannot be fused -- we take the FC1 activation output
|
||||
returned by the kernel (``gemm1_activation_output``, permuted) together with
|
||||
``expanded_idx_to_permuted_idx``, unpermute it, compute the W2 delta out of
|
||||
kernel via punica, and add it to the already-finalized output.
|
||||
|
||||
Constraints (matching the PR support matrix; final gating lives in the oracle):
|
||||
* SM100+ (Blackwell), gated SwiGLU, shuffled weights only;
|
||||
* BF16 weights, or MXFP4 weights with BF16 activations;
|
||||
* routing must be computed outside the MoE (the Modular path satisfies this).
|
||||
|
||||
"""
|
||||
|
||||
import functools
|
||||
from abc import abstractmethod
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
|
||||
LoRAExpertsMixin,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
view_as_block_major_k,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
|
||||
TrtLlmMxfp4ExpertsBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _unpermute_activation_kernel(
|
||||
act_ptr, # act_permuted: (num_permuted, num_cols)
|
||||
idx_ptr, # idx_map: (num_rows,), values in [0, num_permuted) or -1
|
||||
out_ptr, # out: (num_rows, num_cols)
|
||||
num_cols,
|
||||
stride_ar,
|
||||
stride_or,
|
||||
BLOCK_I: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col_offs = tl.program_id(1) * BLOCK_I + tl.arange(0, BLOCK_I)
|
||||
col_mask = col_offs < num_cols
|
||||
|
||||
idx = tl.load(idx_ptr + row)
|
||||
out_ptrs = out_ptr + row * stride_or + col_offs
|
||||
if idx >= 0:
|
||||
vals = tl.load(act_ptr + idx * stride_ar + col_offs, mask=col_mask, other=0.0)
|
||||
tl.store(out_ptrs, vals, mask=col_mask)
|
||||
else:
|
||||
zeros = tl.zeros((BLOCK_I,), dtype=out_ptr.dtype.element_ty)
|
||||
tl.store(out_ptrs, zeros, mask=col_mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _finalize_lora_kernel(
|
||||
gemm2_ptr, # (num_permuted, K) base FC2 output, permuted, unweighted
|
||||
weight_ptr, # (num_tokens * top_k,) routing weights (expanded order)
|
||||
idx_ptr, # (num_tokens * top_k,) expanded_idx -> permuted_idx or -1
|
||||
delta_ptr, # (num_tokens, top_k, K) W2 LoRA delta, already routing-weighted
|
||||
out_ptr, # (num_tokens, K)
|
||||
K,
|
||||
stride_g0,
|
||||
stride_d0,
|
||||
stride_d1,
|
||||
stride_o0,
|
||||
scale,
|
||||
TOP_K: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
col = tl.program_id(1) * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
mask = col < K
|
||||
|
||||
acc_base = tl.zeros((BLOCK_K,), dtype=tl.float32)
|
||||
acc_delta = tl.zeros((BLOCK_K,), dtype=tl.float32)
|
||||
for k in tl.static_range(TOP_K):
|
||||
eid = token * TOP_K + k
|
||||
pidx = tl.load(idx_ptr + eid)
|
||||
if pidx >= 0:
|
||||
w = tl.load(weight_ptr + eid).to(tl.float32)
|
||||
base = tl.load(gemm2_ptr + pidx * stride_g0 + col, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
acc_base += w * base
|
||||
acc_delta += tl.load(
|
||||
delta_ptr + token * stride_d0 + k * stride_d1 + col, mask=mask, other=0.0
|
||||
).to(tl.float32)
|
||||
|
||||
out = acc_base * scale + acc_delta
|
||||
tl.store(
|
||||
out_ptr + token * stride_o0 + col, out.to(out_ptr.dtype.element_ty), mask=mask
|
||||
)
|
||||
|
||||
|
||||
class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
"""LoRA-aware trtllm MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
self.routing_method_type = moe_config.routing_method
|
||||
self.topk = moe_config.experts_per_token
|
||||
self.intermediate_size_per_partition = (
|
||||
moe_config.intermediate_size_per_partition
|
||||
)
|
||||
self.hidden_dim = moe_config.hidden_dim
|
||||
self.local_num_experts = moe_config.num_local_experts
|
||||
self.ep_rank = moe_config.moe_parallel_config.ep_rank
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
p = current_platform
|
||||
return (
|
||||
p.is_cuda()
|
||||
and p.is_device_capability_family(100)
|
||||
and has_flashinfer_trtllm_fused_moe()
|
||||
)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
topk,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
expert_tokens_meta,
|
||||
activation,
|
||||
):
|
||||
# flashinfer manages its own workspace; only declare output (M, K)
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def moe_problem_size(self, a1, w1, w2, topk_ids):
|
||||
"""Override the base 3D-weight assumption.
|
||||
|
||||
FusedMoEKernel._fused_experts calls moe_problem_size before apply(),
|
||||
but the base impl asserts ``len(w1.shape) == 3``. The flashinfer
|
||||
trtllm path stores shuffled weights in 4D BlockMajorK layout, so we
|
||||
derive the (E, M, N, K, topk) tuple from config + inputs instead.
|
||||
The N/K here only feed workspace sizing, which we zero out in
|
||||
workspace_shapes(); the real shapes are handled inside flashinfer.
|
||||
"""
|
||||
E = self.local_num_experts
|
||||
N = 2 * self.intermediate_size_per_partition
|
||||
K = self.hidden_dim
|
||||
M = a1.size(0) if a1.dim() == 2 else a1.size(1)
|
||||
topk = topk_ids.size(1)
|
||||
return E, M, N, K, topk
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
# apply() writes the fully finalized result into `output` (fused base
|
||||
# finalize + W2 LoRA reduction), so this is a No-Op.
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return (
|
||||
not moe_parallel_config.use_all2all_kernels
|
||||
or moe_parallel_config.use_ag_rs_all2all_kernels
|
||||
) and not moe_parallel_config.enable_eplb
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(router_logits_dtype, routing_method) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False # gated only
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@abstractmethod
|
||||
def invoke_routed_moe(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids_and_weights: tuple[torch.Tensor, torch.Tensor],
|
||||
gemm1_lora_delta: torch.Tensor | None,
|
||||
global_num_experts: int,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Call the dtype-specific trtllm_*_routed_moe and return list[Tensor].
|
||||
|
||||
The LoRA path always sets gemm1_lora_delta and runs with
|
||||
do_finalize=False so the base finalize can be fused with the W2 LoRA
|
||||
reduction (see _finalize_with_w2_lora). Return contract:
|
||||
gemm1_lora_delta is None -> [output] (do_finalize=True)
|
||||
otherwise -> [gemm2_output(permuted, unweighted),
|
||||
expert_weights,
|
||||
expanded_idx_to_permuted_idx,
|
||||
gemm1_activation_output(permuted)]
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_gemm1_lora_delta(
|
||||
self,
|
||||
gemm1_lora_delta: torch.Tensor,
|
||||
lora_context: MoELoRAContext,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Hook to rewrite the W13 delta before it is handed to the kernel."""
|
||||
return gemm1_lora_delta
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
lora_context = self._lora_context
|
||||
assert lora_context is not None, "LoRA context must be set"
|
||||
num_tokens = hidden_states.size(0)
|
||||
top_k = self.topk
|
||||
intermediate_size = self.intermediate_size_per_partition
|
||||
# LoRA buffers use the padded hidden size; `output` may be narrower.
|
||||
K = self.hidden_dim
|
||||
|
||||
# ---- Base-model fast path ----
|
||||
# When no token in the batch selects a LoRA adapter, skip the LoRA machinery
|
||||
# and run the plain base MoE with do_finalize=True, which writes the finalized
|
||||
# result straight into `output`.
|
||||
if self._batch_has_no_lora(lora_context):
|
||||
self.invoke_routed_moe(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_ids_and_weights=(topk_ids, topk_weights),
|
||||
gemm1_lora_delta=None, # without LoRA, no delta
|
||||
global_num_experts=global_num_experts,
|
||||
a1q_scale=a1q_scale,
|
||||
output=output,
|
||||
)
|
||||
return
|
||||
|
||||
# The LoRA tile-config heuristic (try_get_optimal_moe_config) unpacks
|
||||
# w1/w2 as standard 3D MoE weights, but flashinfer stores shuffled
|
||||
# 4D BlockMajorK weights. add_lora_w13/add_lora_w2 only read .shape
|
||||
# from w1/w2 (the actual GEMM uses lora_a/b_stacked), so pass
|
||||
# zero-storage meta tensors carrying the logical 3D shapes:
|
||||
# w1: (E, 2I, H) w2: (E, H, I)
|
||||
w1_cfg = torch.empty(
|
||||
(self.local_num_experts, 2 * intermediate_size, K),
|
||||
device="meta",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
w2_cfg = torch.empty(
|
||||
(self.local_num_experts, K, intermediate_size),
|
||||
device="meta",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
# ---- 1) W13 LoRA delta -> gemm1_lora_delta (bf16, [T, top_k, 2I]) ----
|
||||
gemm1_lora_delta = None
|
||||
w13_meta = (None, None, None, None)
|
||||
|
||||
# zeros (not empty): under EP the punica expand kernel only writes
|
||||
# slots whose expert is local to this rank; the kernel skips the rest.
|
||||
gemm1_lora_delta = torch.zeros(
|
||||
num_tokens,
|
||||
top_k,
|
||||
2 * intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
lora_x = hidden_states
|
||||
if not self.expects_unquantized_inputs:
|
||||
orig = lora_context.original_hidden_states
|
||||
assert orig is not None and orig.shape[0] == hidden_states.shape[0], (
|
||||
"quantized trtllm LoRA path requires original_hidden_states"
|
||||
)
|
||||
lora_x = orig
|
||||
# add_inputs=False: write the pure delta only (the base is fused in
|
||||
# by the kernel) and do NOT multiply by the routing weight (it is a
|
||||
# pre-SwiGLU bias).
|
||||
# swap_w13_slices=True: apply_w13_lora writes in vLLM's w13 order
|
||||
# (gate=w1 first, up=w3 second), but FlashInfer's gemm1_lora_delta
|
||||
# expects [up, gate]; reversing the slices emits that order directly,
|
||||
# avoiding an out-of-place concat swap.
|
||||
w13_meta = self.apply_w13_lora(
|
||||
lora_context,
|
||||
y=gemm1_lora_delta,
|
||||
x=lora_x,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
expert_map=expert_map,
|
||||
w1=w1_cfg,
|
||||
w2=w2_cfg,
|
||||
num_tokens=num_tokens,
|
||||
top_k_num=top_k,
|
||||
add_inputs=False,
|
||||
swap_w13_slices=True,
|
||||
)
|
||||
gemm1_lora_delta = self.prepare_gemm1_lora_delta(
|
||||
gemm1_lora_delta, lora_context, topk_ids
|
||||
)
|
||||
|
||||
# ---- 2) Call the routed flashinfer kernel ----
|
||||
ret = self.invoke_routed_moe(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_ids_and_weights=(topk_ids, topk_weights),
|
||||
gemm1_lora_delta=gemm1_lora_delta,
|
||||
global_num_experts=global_num_experts,
|
||||
a1q_scale=a1q_scale,
|
||||
output=output,
|
||||
)
|
||||
# ---- 3) W2 LoRA (computed out of kernel) ----
|
||||
# do_finalize=False: flashinfer returns the *unfinalized* base output.
|
||||
# ret = [gemm2_output(permuted, unweighted),
|
||||
# expert_weights, expanded_idx_to_permuted_idx,
|
||||
# gemm1_activation_output(permuted)]
|
||||
gemm2_permuted = ret[0]
|
||||
expert_weights = ret[1]
|
||||
expanded_idx_to_permuted_idx = ret[2]
|
||||
gemm1_act_permuted = ret[3] # [max_padded, I], post-act
|
||||
act = self._unpermute_activation(
|
||||
gemm1_act_permuted,
|
||||
expanded_idx_to_permuted_idx,
|
||||
num_tokens,
|
||||
top_k,
|
||||
intermediate_size,
|
||||
) # (T*top_k, I) -- same layout as the triton path's intermediate_cache2
|
||||
|
||||
(
|
||||
sorted_token_ids_lora,
|
||||
expert_ids_lora,
|
||||
num_tokens_post_padded_lora,
|
||||
token_lora_mapping,
|
||||
) = w13_meta
|
||||
|
||||
w2_delta = torch.zeros(
|
||||
num_tokens,
|
||||
top_k,
|
||||
K,
|
||||
dtype=output.dtype,
|
||||
device=output.device,
|
||||
)
|
||||
self.apply_w2_lora(
|
||||
lora_context,
|
||||
y=w2_delta,
|
||||
x=act,
|
||||
topk_weights=topk_weights,
|
||||
sorted_token_ids_lora=sorted_token_ids_lora,
|
||||
expert_ids_lora=expert_ids_lora,
|
||||
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
|
||||
token_lora_mapping=token_lora_mapping,
|
||||
num_tokens=num_tokens,
|
||||
w1=w1_cfg,
|
||||
w2=w2_cfg,
|
||||
top_k_num=top_k,
|
||||
add_inputs=False,
|
||||
)
|
||||
# ---- 4) Fused finalize: reduce the base path over top_k (with routing
|
||||
# weights) and add the already-weighted W2 delta, in a single kernel.
|
||||
# This replaces flashinfer's internal finalize launch plus a separate
|
||||
# w2_delta.sum(dim=1) + add_.
|
||||
self._finalize_with_w2_lora(
|
||||
output,
|
||||
gemm2_permuted,
|
||||
expert_weights,
|
||||
expanded_idx_to_permuted_idx,
|
||||
w2_delta,
|
||||
num_tokens,
|
||||
top_k,
|
||||
scale=1.0,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _batch_has_no_lora(lora_context: MoELoRAContext) -> bool:
|
||||
"""True when no token in the batch selects a LoRA adapter.
|
||||
|
||||
Mirrors the no-lora fast path in
|
||||
``PunicaWrapperGPU.add_lora_fused_moe``: the punica kernel metadata
|
||||
carries a CPU ``no_lora_flag`` computed once per forward from the
|
||||
token->LoRA mapping. Reading it is a host-only check (no device sync),
|
||||
and under CUDA graphs the branch is frozen at capture time against the
|
||||
graph's ``has_lora`` dispatch key, so it stays correct on replay.
|
||||
"""
|
||||
meta = getattr(lora_context.punica_wrapper, "token_mapping_meta", None)
|
||||
if meta is None:
|
||||
return False
|
||||
flag = meta.no_lora_flag_cpu
|
||||
return bool(flag.numel() == 1 and flag.item())
|
||||
|
||||
@staticmethod
|
||||
def _unpermute_activation(
|
||||
act_permuted: torch.Tensor,
|
||||
idx_map: torch.Tensor,
|
||||
num_tokens: int,
|
||||
top_k: int,
|
||||
intermediate_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Permuted FC1 activation -> (num_tokens*top_k, I).
|
||||
|
||||
expanded_idx = token*top_k + k; idx_map[expanded_idx] = permuted_idx or -1.
|
||||
Fused gather + drop-masking: each output row copies the matching
|
||||
permuted row, or is zeroed when idx_map < 0.
|
||||
"""
|
||||
num_rows = num_tokens * top_k
|
||||
out = torch.empty(
|
||||
(num_rows, intermediate_size),
|
||||
dtype=act_permuted.dtype,
|
||||
device=act_permuted.device,
|
||||
)
|
||||
BLOCK_I = 1024
|
||||
grid = (num_rows, triton.cdiv(intermediate_size, BLOCK_I))
|
||||
_unpermute_activation_kernel[grid](
|
||||
act_permuted,
|
||||
idx_map,
|
||||
out,
|
||||
intermediate_size,
|
||||
act_permuted.stride(0),
|
||||
out.stride(0),
|
||||
BLOCK_I=BLOCK_I,
|
||||
)
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _finalize_with_w2_lora(
|
||||
output: torch.Tensor,
|
||||
gemm2_permuted: torch.Tensor,
|
||||
expert_weights: torch.Tensor,
|
||||
idx_map: torch.Tensor,
|
||||
w2_delta: torch.Tensor,
|
||||
num_tokens: int,
|
||||
top_k: int,
|
||||
scale: float = 1.0,
|
||||
) -> None:
|
||||
"""Fused base finalize + W2 LoRA reduction, written into ``output``.
|
||||
|
||||
For each token: sum the routing-weighted permuted base rows over top_k
|
||||
(``expert_weights`` in expanded order, ``idx_map < 0`` dropped), scale by
|
||||
``scale``, and add the already-weighted ``w2_delta`` reduced over top_k.
|
||||
"""
|
||||
# Inputs may carry hidden padding that `output` does not.
|
||||
K = output.size(1)
|
||||
assert gemm2_permuted.size(1) >= K and w2_delta.size(2) >= K
|
||||
BLOCK_K = 512
|
||||
grid = (num_tokens, triton.cdiv(K, BLOCK_K))
|
||||
_finalize_lora_kernel[grid](
|
||||
gemm2_permuted,
|
||||
expert_weights.reshape(-1),
|
||||
idx_map,
|
||||
w2_delta,
|
||||
output,
|
||||
K,
|
||||
gemm2_permuted.stride(0),
|
||||
w2_delta.stride(0),
|
||||
w2_delta.stride(1),
|
||||
output.stride(0),
|
||||
scale,
|
||||
TOP_K=top_k,
|
||||
BLOCK_K=BLOCK_K,
|
||||
)
|
||||
|
||||
|
||||
# BF16 unquantized trtllm MoE + LoRA
|
||||
class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase):
|
||||
"""BF16 unquantized trtllm MoE + LoRA."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(weight_key, activation_key) -> bool:
|
||||
return weight_key is None and activation_key is None
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [MoEActivation.SILU]
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(routing_method, weight_key, activation_key) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
RoutingMethodType.Llama4,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
|
||||
def invoke_routed_moe(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids_and_weights: tuple[torch.Tensor, torch.Tensor],
|
||||
gemm1_lora_delta: torch.Tensor | None,
|
||||
global_num_experts: int,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
) -> list[torch.Tensor]:
|
||||
import flashinfer
|
||||
|
||||
# With gemm1_lora_delta set (the LoRA path) run do_finalize=False and
|
||||
# return the unfinalized permuted base output so apply() can fuse the
|
||||
# finalize with the W2 LoRA reduction (see _finalize_with_w2_lora).
|
||||
# Without a delta (base path), run do_finalize=True and hand flashinfer
|
||||
# the caller's buffer via output= so it finalizes in place -- no copy.
|
||||
do_finalize = gemm1_lora_delta is None
|
||||
ret = flashinfer.fused_moe.trtllm_bf16_routed_moe(
|
||||
topk_ids=topk_ids_and_weights,
|
||||
hidden_states=hidden_states,
|
||||
gemm1_weights=view_as_block_major_k(w1),
|
||||
gemm2_weights=view_as_block_major_k(w2),
|
||||
gemm1_lora_delta=gemm1_lora_delta,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=self.routing_method_type,
|
||||
do_finalize=do_finalize,
|
||||
output=output if do_finalize else None,
|
||||
)
|
||||
if not do_finalize:
|
||||
# [gemm2_output, expert_weights, expanded_idx, gemm1_activation]
|
||||
return list(ret)
|
||||
# do_finalize=True finalized directly into `output`.
|
||||
return [output]
|
||||
|
||||
|
||||
# MXFP4 weights x BF16 activations trtllm MoE + LoRA
|
||||
class TrtLlmMxfp4LoRAExperts(_TrtLlmLoRAExpertsBase, TrtLlmMxfp4ExpertsBase):
|
||||
"""MXFP4-weight, BF16-activation trtllm MoE + LoRA. Not MXFP8 activations:
|
||||
the kernel then returns the FC1 activation quantized, without its scales."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
_TrtLlmLoRAExpertsBase.__init__(self, moe_config, quant_config)
|
||||
# gemm1_alpha / gemm1_beta / gemm1_clamp_limit, hidden_dim_unpadded.
|
||||
TrtLlmMxfp4ExpertsBase.__init__(self, moe_config, quant_config)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return weight_key == kMxfp4Static and activation_key is None
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(routing_method, weight_key, activation_key) -> bool:
|
||||
return True # routing is done outside the kernel
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
# DP gathers tokens, but the no-LoRA fast path reads the local batch.
|
||||
p = moe_parallel_config
|
||||
return p.dp_size == 1 and not p.use_all2all_kernels and not p.enable_eplb
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
topk,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
expert_tokens_meta,
|
||||
activation,
|
||||
):
|
||||
# flashinfer manages its own workspace and drops the hidden padding.
|
||||
return (0,), (0,), (M, self.hidden_dim_unpadded)
|
||||
|
||||
def output_hidden_dim(self, hidden_dim: int) -> int:
|
||||
# FusedMoEExpertsModular precedes TrtLlmMxfp4ExpertsBase in the MRO.
|
||||
return self.hidden_dim_unpadded
|
||||
|
||||
@functools.cached_property
|
||||
def _w13_bias_up_gate_index(self) -> torch.Tensor:
|
||||
"""Columns of the kernel-format FC1 bias in [up | gate] order."""
|
||||
from flashinfer.fused_moe.core import get_w2_permute_indices_with_cache
|
||||
|
||||
# Kernel format is the (up_i, gate_i) bias permuted for the tile.
|
||||
assert self.w1_bias is not None
|
||||
bias = self.w1_bias.reshape(self.local_num_experts, -1)[0]
|
||||
permute = get_w2_permute_indices_with_cache({}, bias.reshape(-1, 1), 128)
|
||||
interleaved = torch.argsort(permute.to(bias.device))
|
||||
return torch.cat([interleaved[0::2], interleaved[1::2]])
|
||||
|
||||
def prepare_gemm1_lora_delta(
|
||||
self,
|
||||
gemm1_lora_delta: torch.Tensor,
|
||||
lora_context: MoELoRAContext,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Bring the W13 delta to [up | gate] halves and fold in the FC1 bias."""
|
||||
intermediate_size = gemm1_lora_delta.size(-1) // 2
|
||||
# View as (I, 2) pairs of the first and second w13 half. SWIGLUOAI
|
||||
# weights, and the deltas that follow them, are interleaved.
|
||||
if self.moe_config.activation == MoEActivation.SWIGLUOAI:
|
||||
pairs = gemm1_lora_delta.unflatten(-1, (intermediate_size, 2))
|
||||
else:
|
||||
pairs = gemm1_lora_delta.unflatten(-1, (2, intermediate_size)).mT
|
||||
# Per-expert deltas arrive as (up, gate); fused 3D ones follow the
|
||||
# checkpoint (gate, up) order.
|
||||
if lora_context.w13_num_slices != 2:
|
||||
pairs = pairs.flip(-1)
|
||||
delta = pairs.mT.flatten(-2) # [up | gate]
|
||||
|
||||
# The kernel has a single FC1 bias slot, shared by the static bias and
|
||||
# the per-token delta, so the static bias rides along in the delta.
|
||||
if self.w1_bias is not None:
|
||||
bias = self.w1_bias.reshape(self.local_num_experts, -1)
|
||||
bias = bias[:, self._w13_bias_up_gate_index].float()
|
||||
local_ids = topk_ids.long() - self.ep_rank * self.local_num_experts
|
||||
local_ids = local_ids.clamp_(0, self.local_num_experts - 1)
|
||||
delta = (delta.float() + bias[local_ids]).to(torch.bfloat16)
|
||||
return delta.contiguous()
|
||||
|
||||
def invoke_routed_moe(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids_and_weights: tuple[torch.Tensor, torch.Tensor],
|
||||
gemm1_lora_delta: torch.Tensor | None,
|
||||
global_num_experts: int,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
) -> list[torch.Tensor]:
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
assert a1q_scale is None and hidden_states.dtype == torch.bfloat16
|
||||
topk_ids, topk_weights = topk_ids_and_weights
|
||||
# See TrtLlmBf16LoRAExperts.invoke_routed_moe for the return contract.
|
||||
do_finalize = gemm1_lora_delta is None
|
||||
ret = trtllm_fp4_block_scale_routed_moe(
|
||||
topk_ids=(topk_ids.to(torch.int32), topk_weights),
|
||||
routing_bias=None,
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_scale=None,
|
||||
gemm1_weights=w1,
|
||||
gemm1_weights_scale=self.w1_scale,
|
||||
# With a delta the bias is folded into it (prepare_gemm1_lora_delta).
|
||||
gemm1_bias=self.w1_bias if do_finalize else None,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.w2_scale,
|
||||
gemm2_bias=self.w2_bias,
|
||||
output1_scale_scalar=None,
|
||||
output1_scale_gate_scalar=None,
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
# Routing is already done; Renormalize is a safe default.
|
||||
routing_method_type=RoutingMethodType.Renormalize,
|
||||
do_finalize=do_finalize,
|
||||
enable_pdl=True,
|
||||
activation_type=self._flashinfer_activation_type(
|
||||
self.moe_config.activation
|
||||
),
|
||||
output=output if do_finalize else None,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
gemm1_lora_delta=gemm1_lora_delta,
|
||||
)
|
||||
return [output] if do_finalize else list(ret)
|
||||
@@ -371,7 +371,6 @@ def FusedMoEFactory(
|
||||
max_num_tokens=max_num_batched_tokens,
|
||||
elastic_ep_max_dp_size=vllm_config.parallel_config.elastic_ep_max_dp_size,
|
||||
has_bias=has_bias,
|
||||
is_lora_enabled=vllm_config.lora_config is not None,
|
||||
shared_expert_prefix=shared_expert_prefix,
|
||||
activation=moe_activation,
|
||||
device=vllm_config.device_config.device,
|
||||
|
||||
@@ -606,8 +606,6 @@ class FusedMoEExperts(ABC):
|
||||
return False, _make_reason(f"{activation_format.value} activation format")
|
||||
elif envs.VLLM_BATCH_INVARIANT and not cls._supports_batch_invariance():
|
||||
return False, _make_reason("batch invariance")
|
||||
elif moe_config.is_lora_enabled and not cls.supports_lora():
|
||||
return False, _make_reason("LoRA")
|
||||
return True, None
|
||||
|
||||
@staticmethod
|
||||
@@ -772,15 +770,6 @@ class FusedMoEExperts(ABC):
|
||||
def g2_alphas(self) -> torch.Tensor | None:
|
||||
return self.quant_config.g2_alphas
|
||||
|
||||
@staticmethod
|
||||
def supports_lora() -> bool:
|
||||
"""Return True if this expert impl natively handles LoRA.
|
||||
|
||||
LoRA-aware experts should mix in LoRAExpertsMixin, which flips this
|
||||
to True and provides the per-forward LoRA state plumbing.
|
||||
"""
|
||||
return False
|
||||
|
||||
def supports_packed_ue8m0_act_scales(self) -> bool:
|
||||
"""A flag indicating whether or not this class can process packed ue8m0
|
||||
activation scales.
|
||||
@@ -1531,13 +1520,6 @@ class FusedMoEKernelModularImpl:
|
||||
apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
# Stash the original unquantized hidden states on the LoRA context
|
||||
# so apply_w13_lora sees correct-magnitude activations instead of
|
||||
# the potentially quantized values produced by _prepare().
|
||||
lora_ctx = getattr(self.fused_experts, "_lora_context", None)
|
||||
if lora_ctx is not None:
|
||||
lora_ctx.original_hidden_states = hidden_states
|
||||
|
||||
fused_out = self._fused_experts(
|
||||
in_dtype=hidden_states.dtype,
|
||||
a1q=a1q,
|
||||
@@ -1555,9 +1537,6 @@ class FusedMoEKernelModularImpl:
|
||||
output_alias=output,
|
||||
)
|
||||
|
||||
if lora_ctx is not None:
|
||||
lora_ctx.original_hidden_states = None
|
||||
|
||||
if isinstance(fused_out, UnfinalizedMoEOutput):
|
||||
# Nothing below can run on an unfinalized output: finalize is the
|
||||
# local top-k reduction (and, for DP/EP, the combine) that the
|
||||
@@ -1733,9 +1712,6 @@ class FusedMoEKernel:
|
||||
def moe_config(self) -> FusedMoEConfig:
|
||||
return self.fused_experts.moe_config
|
||||
|
||||
def supports_lora(self) -> bool:
|
||||
return self.fused_experts.supports_lora()
|
||||
|
||||
def _post_init_setup(self):
|
||||
"""Resolve any leftover setup dependencies between self.prepare_finalize
|
||||
and self.fused_experts here.
|
||||
|
||||
@@ -337,13 +337,11 @@ def pad_tp_shard_to_weight_blocks(
|
||||
config.intermediate_size % block_n != 0
|
||||
or config.hidden_dim % block_n != 0
|
||||
or config.ep_size != 1
|
||||
or config.is_lora_enabled
|
||||
or config.has_bias
|
||||
):
|
||||
raise ValueError(
|
||||
f"Block-aligned FP8 TP sharding requires {block_n}-aligned "
|
||||
"global expert dimensions, pure TP, and no LoRA or "
|
||||
"expert bias."
|
||||
"global expert dimensions, pure TP, and no expert bias."
|
||||
)
|
||||
config.tp_shard_with_padding = True
|
||||
config.intermediate_size_per_partition = round_up(
|
||||
|
||||
@@ -522,30 +522,6 @@ def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
return f"{base} since {reason}." if reason else f"{base}."
|
||||
|
||||
|
||||
def _requested_kernel_cls(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
"""Kernel classes for an explicitly requested backend. The LoRA-aware
|
||||
TRT-LLM experts are opt-in: "auto" keeps its current LoRA selection."""
|
||||
candidates = backend_to_kernel_cls(backend)
|
||||
if not config.is_lora_enabled or backend not in TRTLLM_BACKENDS:
|
||||
return candidates
|
||||
if backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8:
|
||||
raise ValueError(
|
||||
_make_log_unsupported(
|
||||
backend,
|
||||
"expert LoRA is not supported with MXFP8 activations; use "
|
||||
"moe_backend='flashinfer_trtllm'",
|
||||
)
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
|
||||
TrtLlmMxfp4LoRAExperts,
|
||||
)
|
||||
|
||||
return [TrtLlmMxfp4LoRAExperts, *candidates]
|
||||
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
@@ -555,7 +531,7 @@ def _return_or_raise(
|
||||
scope: Literal["process", "global", "local"] = "local",
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in _requested_kernel_cls(backend, config):
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
|
||||
@@ -194,31 +194,6 @@ def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend
|
||||
)
|
||||
|
||||
|
||||
def _trtllm_bf16_lora_supported(moe_config: FusedMoEConfig) -> bool:
|
||||
"""Gate for routing LoRA-enabled BF16 MoE to the FlashInfer TRT-LLM
|
||||
gemm1_lora_delta path (PR #3153).
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
|
||||
TrtLlmBf16LoRAExperts,
|
||||
)
|
||||
|
||||
# LoRA path returns before the oracle loop; reuse is_supported_config here.
|
||||
supported, _ = TrtLlmBf16LoRAExperts.is_supported_config(
|
||||
TrtLlmBf16LoRAExperts,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
if not supported:
|
||||
return False
|
||||
# The flashinfer trtllm fused-MoE kernel requires the per-partition
|
||||
# intermediate size to be a multiple of 128. Plain TP shards the MoE
|
||||
# intermediate dim (e.g. 768 -> 192 at tp=4), which would crash the kernel
|
||||
# at runtime; fall back to Triton in that case.
|
||||
return moe_config.intermediate_size_per_partition % 128 == 0
|
||||
|
||||
|
||||
def select_unquantized_moe_backend(
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]:
|
||||
@@ -231,22 +206,6 @@ def select_unquantized_moe_backend(
|
||||
if current_platform.is_out_of_tree():
|
||||
return UnquantizedMoeBackend.OOT, None
|
||||
|
||||
if moe_config.is_lora_enabled:
|
||||
if _trtllm_bf16_lora_supported(moe_config):
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
|
||||
TrtLlmBf16LoRAExperts,
|
||||
)
|
||||
|
||||
logger.info_once(
|
||||
"Using TrtLlmBf16LoRAExperts Unquantized MoE LoRA backend "
|
||||
"(TrtLlmBf16LoRAExperts)."
|
||||
)
|
||||
return UnquantizedMoeBackend.FLASHINFER_TRTLLM, TrtLlmBf16LoRAExperts
|
||||
logger.info_once("Using TRITON Unquantized MoE LoRA backend")
|
||||
return UnquantizedMoeBackend.TRITON, backend_to_kernel_cls(
|
||||
UnquantizedMoeBackend.TRITON
|
||||
)[0]
|
||||
|
||||
# NOTE: the kernels are selected in the following order.
|
||||
AVAILABLE_BACKENDS = _get_priority_backends(moe_config)
|
||||
|
||||
|
||||
@@ -93,14 +93,6 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
super().__init__()
|
||||
self.is_sequence_parallel = is_sequence_parallel
|
||||
self._num_dispatchers = num_dispatchers
|
||||
# Set by FusedMoEWithLoRA.set_mapping() when LoRA is active. When
|
||||
# present, prepare() dispatches the per-token LoRA mapping alongside
|
||||
# hidden_states and writes the gathered result back to the context so
|
||||
# experts can use the per-rank-local mapping.
|
||||
self._lora_context = None
|
||||
|
||||
def set_lora_context(self, ctx) -> None:
|
||||
self._lora_context = ctx
|
||||
|
||||
@property
|
||||
def activation_format(self) -> mk.FusedMoEActivationFormat:
|
||||
@@ -193,27 +185,9 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
a1, quant_config, defer_input_quant
|
||||
)
|
||||
|
||||
# When LoRA is active, dispatch the per-token LoRA id along with
|
||||
# hidden_states so every rank receives the correct mapping for the
|
||||
# tokens it ends up processing. The punica_wrapper stores indices as
|
||||
# int64 but the moe_lora_align_block_size kernel expects int32, so
|
||||
# pull the pre-cast view from token_mapping_meta.
|
||||
lora_ctx = self._lora_context
|
||||
local_token_lora_mapping = None
|
||||
if lora_ctx is not None:
|
||||
local_token_lora_mapping = (
|
||||
lora_ctx.punica_wrapper.token_mapping_meta.token_lora_mapping[
|
||||
: a1.shape[0]
|
||||
]
|
||||
)
|
||||
|
||||
extra_tensors: list[torch.Tensor] | None = None
|
||||
if scales is not None:
|
||||
extra_tensors = list(scales)
|
||||
if local_token_lora_mapping is not None:
|
||||
if extra_tensors is None:
|
||||
extra_tensors = []
|
||||
extra_tensors.append(local_token_lora_mapping)
|
||||
|
||||
with self._comm_region():
|
||||
res = get_ep_group().dispatch(
|
||||
@@ -232,10 +206,6 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
assert len(res) == 4
|
||||
a1q, topk_weights, topk_ids, gathered_extras = res
|
||||
gathered_extras = list(gathered_extras)
|
||||
if local_token_lora_mapping is not None:
|
||||
dispatched_lora_mapping = gathered_extras.pop()
|
||||
assert lora_ctx is not None
|
||||
lora_ctx.local_token_lora_mapping = dispatched_lora_mapping
|
||||
if scales is not None:
|
||||
a1q_scale = _unwrap_scale_and_prepare_for_moe(
|
||||
gathered_extras, quant_config
|
||||
|
||||
@@ -212,8 +212,6 @@ class RoutedExperts(PluggableLayer):
|
||||
|
||||
self.quant_method.create_weights(layer=self, **moe_quant_params)
|
||||
|
||||
self.lora_base_layer_prefix = ""
|
||||
|
||||
# TODO(bnell): Temporary hack. Get rid of this.
|
||||
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
||||
self.quant_method = quant_method
|
||||
@@ -1121,7 +1119,6 @@ class RoutedExperts(PluggableLayer):
|
||||
num_experts=moe_config.num_logical_experts + num_fused_shared_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
routed_experts_prefix="",
|
||||
lora_base_layer_prefix=self.lora_base_layer_prefix,
|
||||
include_fused=include_fused,
|
||||
is_gated=moe_config.is_act_and_mul,
|
||||
)
|
||||
@@ -1136,17 +1133,11 @@ class RoutedExperts(PluggableLayer):
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""Build the expert mapping, detecting the LoRA `base_layer.` prefix by
|
||||
scanning `model`'s parameters.
|
||||
|
||||
Legacy entry point for models that still hand-roll `load_weights`; the
|
||||
`RoutedExperts` weight loader uses `get_expert_mapping` /
|
||||
`build_expert_params_mapping` instead (which take the prefix directly).
|
||||
See `build_expert_params_mapping` for the returned tuple format.
|
||||
"""Legacy entry point for models that still hand-roll `load_weights`;
|
||||
the `RoutedExperts` weight loader uses `get_expert_mapping` /
|
||||
`build_expert_params_mapping` instead. See
|
||||
`build_expert_params_mapping` for the returned tuple format.
|
||||
"""
|
||||
has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters())
|
||||
prefix = "base_layer." if has_base_layer else ""
|
||||
# These loaders index ``params_dict[full_name]``, so both sides get it.
|
||||
return RoutedExperts.build_expert_params_mapping(
|
||||
ckpt_gate_proj_name,
|
||||
ckpt_down_proj_name,
|
||||
@@ -1154,8 +1145,6 @@ class RoutedExperts(PluggableLayer):
|
||||
num_experts,
|
||||
num_redundant_experts,
|
||||
routed_experts_prefix,
|
||||
lora_base_layer_prefix=prefix,
|
||||
lora_base_layer_prefix_on_param_name=prefix,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1166,8 +1155,6 @@ class RoutedExperts(PluggableLayer):
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
lora_base_layer_prefix: str = "",
|
||||
lora_base_layer_prefix_on_param_name: str = "",
|
||||
include_fused: bool = False,
|
||||
is_gated: bool = True,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
@@ -1182,14 +1169,6 @@ class RoutedExperts(PluggableLayer):
|
||||
ckpt_up_proj_name: Name of up projection in checkpoint
|
||||
num_experts: Number of logical (non-redundant) experts
|
||||
num_redundant_experts: Number of redundant experts
|
||||
lora_base_layer_prefix: LoRA ``base_layer.`` prefix for the
|
||||
``weight_name`` (checkpoint) side
|
||||
lora_base_layer_prefix_on_param_name: same, for the ``param_name``
|
||||
side. Independent because ``get_expert_mapping`` resolves
|
||||
``param_name`` via ``getattr`` against this layer's bare
|
||||
``w13_weight``/``w2_weight`` (no prefix), while
|
||||
``make_expert_params_mapping`` indexes the model-wide
|
||||
``params_dict`` (prefix included).
|
||||
include_fused: Prepend the fused pre-fused-checkpoint entries
|
||||
is_gated: Whether w13 holds gate and up (gated activation) or a
|
||||
single up projection. Selects the fused checkpoint layout.
|
||||
@@ -1219,10 +1198,8 @@ class RoutedExperts(PluggableLayer):
|
||||
if routed_experts_prefix != "":
|
||||
routed_experts_prefix = f"{routed_experts_prefix}."
|
||||
|
||||
w13 = (
|
||||
f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w13_"
|
||||
)
|
||||
w2 = f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w2_"
|
||||
w13 = f"experts.{routed_experts_prefix}w13_"
|
||||
w2 = f"experts.{routed_experts_prefix}w2_"
|
||||
|
||||
fused_mapping = []
|
||||
if include_fused:
|
||||
@@ -1260,10 +1237,7 @@ class RoutedExperts(PluggableLayer):
|
||||
fused_mapping.extend(
|
||||
(
|
||||
w13,
|
||||
(
|
||||
f"experts.{physical_to_logical_map[expert_id]}."
|
||||
f"{gate_up}.{lora_base_layer_prefix}"
|
||||
),
|
||||
f"experts.{physical_to_logical_map[expert_id]}.{gate_up}",
|
||||
expert_id,
|
||||
shard_id,
|
||||
)
|
||||
@@ -1275,7 +1249,7 @@ class RoutedExperts(PluggableLayer):
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
(
|
||||
w13 if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] else w2,
|
||||
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{lora_base_layer_prefix}",
|
||||
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}",
|
||||
expert_id,
|
||||
shard_id,
|
||||
)
|
||||
|
||||
@@ -195,8 +195,6 @@ def _moe_forward_shared_fake(
|
||||
return shared_out, fused_out
|
||||
|
||||
|
||||
# NOTE: `moe_forward` and `moe_forward_shared` being opaque custom ops is a
|
||||
# load-bearing assumption for the MoE-LoRA dual-stream path.
|
||||
direct_register_custom_op(
|
||||
op_name="moe_forward",
|
||||
op_func=_moe_forward,
|
||||
|
||||
@@ -83,7 +83,7 @@ class LogitsProcessor(PluggableLayer):
|
||||
self.vocab_size = vocab_size
|
||||
# Whether the input is logits (default is hidden states).
|
||||
self.logits_as_input = logits_as_input
|
||||
# original vocabulary size (without LoRA).
|
||||
# original vocabulary size (without added embeddings).
|
||||
self.org_vocab_size = org_vocab_size or vocab_size
|
||||
# Soft cap the logits. Used in Gemma 2.
|
||||
self.soft_cap = soft_cap
|
||||
|
||||
@@ -607,8 +607,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
# projection of the input hidden states
|
||||
# Qwen3-Next and Qwen3.5 has a different qkv_proj layout,
|
||||
# we need to create qkvz_proj adaptively here.
|
||||
# When create_in_proj_qkvz is False (e.g. LoRA enabled in Qwen3.5),
|
||||
# in_proj_qkv and in_proj_z are created separately instead.
|
||||
# When create_in_proj_qkvz is False, in_proj_qkv and in_proj_z
|
||||
# are created separately instead.
|
||||
self.in_proj_qkvz = self.create_qkvz_proj(
|
||||
hidden_size=self.hidden_size,
|
||||
key_dim=self.key_dim,
|
||||
@@ -1281,7 +1281,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
assert not hasattr(self, "in_proj_qkv"), "lora isn't supported on CPU."
|
||||
assert not hasattr(self, "in_proj_qkv"), (
|
||||
"the split in_proj_qkv/in_proj_z layout is not supported on CPU"
|
||||
)
|
||||
|
||||
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
ba, _ = self.in_proj_ba(hidden_states)
|
||||
|
||||
-2
@@ -106,7 +106,6 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
is_lora_disabled = vllm_config.lora_config is None
|
||||
moe_backend = vllm_config.kernel_config.moe_backend
|
||||
group_size = weight_quant.group_size or -1
|
||||
if (
|
||||
@@ -114,7 +113,6 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
and weight_quant.type == QuantizationType.INT
|
||||
and group_size == 32
|
||||
and weight_quant.num_bits == 4
|
||||
and is_lora_disabled
|
||||
and on_gfx950()
|
||||
and moe_backend == "flydsl"
|
||||
):
|
||||
|
||||
@@ -518,12 +518,6 @@ def get_and_maybe_dequant_weights(
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
|
||||
# LoRA linear wrappers store quantization metadata on `base_layer`.
|
||||
# Unwrap here so callers can pass either a raw linear layer or its LoRA
|
||||
# wrapper without special-casing.
|
||||
while hasattr(layer, "base_layer") and hasattr(layer.base_layer, "quant_method"):
|
||||
layer = layer.base_layer
|
||||
|
||||
weight = get_attribute_fallback(layer, ["weight", "qweight", "weight_packed"])
|
||||
|
||||
# Unquantized layer: just return base weights
|
||||
|
||||
@@ -209,9 +209,9 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
Adapted from torch.nn.Embedding, note that we pad the vocabulary size to
|
||||
make sure it is divisible by the number of model parallel GPUs.
|
||||
|
||||
In order to support various loading methods, we ensure that LoRA-added
|
||||
In order to support various loading methods, we ensure that added
|
||||
embeddings are always at the end of TP-sharded tensors. In other words,
|
||||
we shard base embeddings and LoRA embeddings separately (both padded),
|
||||
we shard base embeddings and added embeddings separately (both padded),
|
||||
and place them in the same tensor.
|
||||
In this example, we will have the original vocab size = 1010,
|
||||
added vocab size = 16 and padding to 64. Therefore, the total
|
||||
@@ -219,16 +219,16 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
1024, add 16, and then pad to 1088).
|
||||
Therefore, the tensor format looks like the following:
|
||||
TP1, rank 0 (no sharding):
|
||||
|< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >|
|
||||
|< --------BASE-------- >|< -BASE PADDING-- >|< -----ADDED----- >|< -ADDED PADDING- >|
|
||||
corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1025 | -1 | ... | -1 |
|
||||
index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 |
|
||||
|
||||
TP2, rank 0:
|
||||
|< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >|
|
||||
|< --------------------BASE--------------------- >|< -----ADDED----- >|< -ADDED PADDING- >|
|
||||
corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1010 | ... | 1025 | -1 | ... | -1 |
|
||||
index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 |
|
||||
TP2, rank 1:
|
||||
|< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >|
|
||||
|< -----------BASE----------- >|< -BASE PADDING- >|< ----------ADDED PADDING---------- >|
|
||||
corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 |
|
||||
index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 |
|
||||
|
||||
@@ -236,7 +236,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
num_embeddings: vocabulary size.
|
||||
embedding_dim: size of hidden state.
|
||||
params_dtype: type of the parameters.
|
||||
org_num_embeddings: original vocabulary size (without LoRA).
|
||||
org_num_embeddings: original vocabulary size (without added embeddings).
|
||||
padding_size: padding size for the vocabulary.
|
||||
quant_config: quant config for the layer
|
||||
prefix: full name of the layer in the state dict
|
||||
@@ -587,7 +587,7 @@ class ParallelLMHead(VocabParallelEmbedding):
|
||||
embedding_dim: size of hidden state.
|
||||
bias: whether to use bias.
|
||||
params_dtype: type of the parameters.
|
||||
org_num_embeddings: original vocabulary size (without LoRA).
|
||||
org_num_embeddings: original vocabulary size (without added embeddings).
|
||||
padding_size: padding size for the vocabulary.
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
|
||||
|
||||
@@ -20,12 +20,8 @@ __all__ = ["get_parameter_for_reload", "reload_weights"]
|
||||
|
||||
def get_parameter_for_reload(model: nn.Module, name: str) -> nn.Parameter:
|
||||
"""Resolve checkpoint names without changing the model's module tree."""
|
||||
from vllm.lora.layers import BaseLayerWithLoRA
|
||||
|
||||
module_name, _, parameter_name = name.rpartition(".")
|
||||
module = model.get_submodule(module_name)
|
||||
if isinstance(module, BaseLayerWithLoRA):
|
||||
module = module.base_layer
|
||||
return module.get_parameter(parameter_name)
|
||||
|
||||
|
||||
@@ -39,7 +35,7 @@ def reload_weights(
|
||||
"""Reload a model's weights in place from a weights iterator or from disk.
|
||||
|
||||
Callers are responsible for invalidating any state derived from the old
|
||||
weights (e.g. LoRA, encoder and multimodal caches).
|
||||
weights (e.g. encoder and multimodal caches).
|
||||
|
||||
Args:
|
||||
vllm_config: config of the model being reloaded. If `weights_path` is
|
||||
@@ -63,10 +59,7 @@ def reload_weights(
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
load_config = vllm_config.load_config
|
||||
weights_to_load = {
|
||||
name.replace(".base_layer.", ".") if vllm_config.lora_config else name
|
||||
for name, _ in model.named_parameters()
|
||||
}
|
||||
weights_to_load = {name for name, _ in model.named_parameters()}
|
||||
counter_before_reloading = time.perf_counter()
|
||||
|
||||
# load weights from disk if none are provided
|
||||
|
||||
@@ -217,8 +217,8 @@ class ShardedStateLoader(BaseModelLoader):
|
||||
state_dict = self._filter_subtensors(model.state_dict())
|
||||
counter_before_loading_weights = time.perf_counter()
|
||||
for key, tensor in self.iterate_over_files(filepaths):
|
||||
# If loading with LoRA enabled, additional padding may
|
||||
# be added to certain parameters. We only load into a
|
||||
# Parameters padded beyond their checkpoint tensors
|
||||
# may carry extra trailing rows. We only load into a
|
||||
# narrowed view of the parameter data.
|
||||
param = state_dict[key]
|
||||
if param.dtype != tensor.dtype:
|
||||
|
||||
@@ -119,8 +119,6 @@ def initialize_model(
|
||||
kwargs["cache_config"] = vllm_config.cache_config
|
||||
if "quant_config" in all_params:
|
||||
kwargs["quant_config"] = vllm_config.quant_config
|
||||
if "lora_config" in all_params:
|
||||
kwargs["lora_config"] = vllm_config.lora_config
|
||||
if "scheduler_config" in all_params:
|
||||
kwargs["scheduler_config"] = vllm_config.scheduler_config
|
||||
with set_current_vllm_config(vllm_config, check_compile=True, prefix=prefix):
|
||||
|
||||
@@ -4,20 +4,16 @@
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalEmbeddings,
|
||||
SupportsPP,
|
||||
SupportsTranscription,
|
||||
has_inner_state,
|
||||
supports_hc_sp,
|
||||
supports_lora,
|
||||
supports_mrope,
|
||||
supports_multimodal,
|
||||
supports_multimodal_embeddings,
|
||||
supports_pp,
|
||||
supports_transcription,
|
||||
)
|
||||
from .interfaces_base import (
|
||||
VllmModelForPooling,
|
||||
@@ -37,8 +33,6 @@ __all__ = [
|
||||
"has_inner_state",
|
||||
"SupportsHCSP",
|
||||
"supports_hc_sp",
|
||||
"SupportsLoRA",
|
||||
"supports_lora",
|
||||
"SupportsMultiModal",
|
||||
"supports_multimodal",
|
||||
"SupportsMultiModalEmbeddings",
|
||||
@@ -47,6 +41,4 @@ __all__ = [
|
||||
"supports_mrope",
|
||||
"SupportsPP",
|
||||
"supports_pp",
|
||||
"SupportsTranscription",
|
||||
"supports_transcription",
|
||||
]
|
||||
|
||||
@@ -1,19 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import weakref
|
||||
from bisect import bisect_right
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
Hashable,
|
||||
Mapping,
|
||||
MutableSequence,
|
||||
Sequence,
|
||||
)
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from operator import attrgetter
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
@@ -27,11 +23,9 @@ from typing import (
|
||||
runtime_checkable,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -42,14 +36,10 @@ from vllm.utils.func_utils import supports_kw
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import (
|
||||
ModelConfig,
|
||||
SpeechToTextConfig,
|
||||
SpeechToTextParams,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.multimodal import VideoPruningMethod
|
||||
from vllm.inputs import PromptType, TokensPrompt
|
||||
from vllm.lora.model_manager import LoRAModelManager
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
@@ -58,11 +48,10 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.multimodal.registry import _ProcessorFactories
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import ScoreType
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
@@ -105,23 +94,6 @@ MambaStateShapes: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DiarizedTranscriptionSegment:
|
||||
"""A timestamped, speaker-attributed segment produced by an ASR model."""
|
||||
|
||||
start: float
|
||||
end: float
|
||||
speaker: str
|
||||
text: str
|
||||
|
||||
|
||||
class StreamingTranscriptionPostProcessor:
|
||||
"""Stateful streaming post-processor for transcription deltas."""
|
||||
|
||||
def process_delta(self, text_delta: str, finished: bool) -> str:
|
||||
return text_delta
|
||||
|
||||
|
||||
def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
|
||||
"""A helper function to be used in the context of
|
||||
[vllm.model_executor.models.interfaces.SupportsMultiModal.embed_input_ids][]
|
||||
@@ -206,12 +178,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
`multimodal_config.mm_device_do_normalize`.
|
||||
"""
|
||||
|
||||
supports_tower_connector_lora: ClassVar[bool] = False
|
||||
"""
|
||||
A flag that indicates whether this model supports
|
||||
`lora_config.enable_tower_connector_lora`.
|
||||
"""
|
||||
|
||||
requires_raw_input_tokens: ClassVar[bool] = False
|
||||
"""
|
||||
A flag that indicates this model processes input id tokens
|
||||
@@ -432,24 +398,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
|
||||
yield
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: "MultiModalKwargsItem | None",
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
"""Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
|
||||
|
||||
MM LoRA uses these counts to build adapter mappings for the tower and
|
||||
connector forwards. Models with multiple modalities can override this
|
||||
when each modality has different encoder padding or pooling behavior.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} must implement get_mm_lora_token_counts "
|
||||
"to support tower/connector LoRA."
|
||||
)
|
||||
|
||||
def _embed_text_input_ids(
|
||||
self,
|
||||
input_ids: Tensor,
|
||||
@@ -459,9 +407,9 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
) -> Tensor:
|
||||
if is_multimodal is not None and self._has_oov_mm_tokens:
|
||||
# Force all input IDs to be in vocab; we do this instead of squeezing
|
||||
# to ensure that any external configuration requiring offset tracking,
|
||||
# e.g., LoRA, are applied correctly regardless of whether or not
|
||||
# we have multimodal tokens.
|
||||
# to ensure that any external configuration requiring offset tracking
|
||||
# is applied correctly regardless of whether or not we have
|
||||
# multimodal tokens.
|
||||
in_vocab_ids = input_ids.masked_fill(
|
||||
is_multimodal.to(device=input_ids.device, non_blocking=True), 0
|
||||
)
|
||||
@@ -712,85 +660,6 @@ def supports_kv_cache_grouping(
|
||||
return isinstance(model, SupportsKVCacheGrouping)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsLoRA(Protocol):
|
||||
"""The interface required for all models that support LoRA."""
|
||||
|
||||
supports_lora: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
A flag that indicates this model supports LoRA.
|
||||
|
||||
Note:
|
||||
There is no need to redefine this flag if this class is in the
|
||||
MRO of your model class.
|
||||
"""
|
||||
is_3d_moe_weight: ClassVar[bool] = False
|
||||
is_non_gated_moe: ClassVar[bool] = False
|
||||
# The `embedding_module` and `embedding_padding_modules`
|
||||
# are empty by default.
|
||||
embedding_modules: ClassVar[dict[str, str]] = {}
|
||||
packed_modules_mapping: dict[str, list[str]] = {}
|
||||
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
|
||||
lora_skip_prefixes: ClassVar[list[str]] = []
|
||||
lora_manager: "LoRAModelManager | None"
|
||||
|
||||
|
||||
# We can't use runtime_checkable with ClassVar for issubclass checks
|
||||
# so we need to treat the class as an instance and use isinstance instead
|
||||
@runtime_checkable
|
||||
class _SupportsLoRAType(Protocol):
|
||||
supports_lora: Literal[True]
|
||||
|
||||
packed_modules_mapping: dict[str, list[str]]
|
||||
embedding_modules: dict[str, str]
|
||||
|
||||
|
||||
@overload
|
||||
def supports_lora(model: type[object]) -> TypeIs[type[SupportsLoRA]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_lora(model: object) -> TypeIs[SupportsLoRA]: ...
|
||||
|
||||
|
||||
def supports_lora(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsLoRA]] | TypeIs[SupportsLoRA]:
|
||||
result = _supports_lora(model)
|
||||
|
||||
if not result:
|
||||
lora_attrs = (
|
||||
"packed_modules_mapping",
|
||||
"embedding_modules",
|
||||
)
|
||||
missing_attrs = tuple(attr for attr in lora_attrs if not hasattr(model, attr))
|
||||
|
||||
if getattr(model, "supports_lora", False):
|
||||
if missing_attrs:
|
||||
logger.warning(
|
||||
"The model (%s) sets `supports_lora=True`, "
|
||||
"but is missing LoRA-specific attributes: %s",
|
||||
model,
|
||||
missing_attrs,
|
||||
)
|
||||
else:
|
||||
if not missing_attrs:
|
||||
logger.warning(
|
||||
"The model (%s) contains all LoRA-specific attributes, "
|
||||
"but does not set `supports_lora=True`.",
|
||||
model,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _supports_lora(model: type[object] | object) -> bool:
|
||||
if isinstance(model, type):
|
||||
return isinstance(model, _SupportsLoRAType)
|
||||
|
||||
return isinstance(model, SupportsLoRA)
|
||||
|
||||
|
||||
class _MakeEmptyIntermediateTensors(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
@@ -1289,244 +1158,6 @@ class SupportsQuant:
|
||||
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsRealtime(Protocol):
|
||||
"""The interface required for all models that support transcription."""
|
||||
|
||||
supports_realtime: ClassVar[Literal[True]] = True
|
||||
|
||||
realtime_max_tokens: ClassVar[int] = 1
|
||||
"""Maximum tokens to generate per streaming audio segment.
|
||||
Override in subclasses based on the model's expected output length."""
|
||||
|
||||
@classmethod
|
||||
def buffer_realtime_audio(
|
||||
cls,
|
||||
audio_stream: AsyncGenerator[np.ndarray, None],
|
||||
input_stream: asyncio.Queue[list[int]],
|
||||
model_config: "ModelConfig",
|
||||
) -> AsyncGenerator["PromptType", None]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_realtime(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsRealtime]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_realtime(model: object) -> TypeIs[SupportsRealtime]: ...
|
||||
|
||||
|
||||
def supports_realtime(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsRealtime]] | TypeIs[SupportsRealtime]:
|
||||
return getattr(model, "supports_realtime", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsTranscription(Protocol):
|
||||
"""The interface required for all models that support transcription."""
|
||||
|
||||
# Mapping from ISO639_1 language codes: language names
|
||||
supported_languages: ClassVar[Mapping[str, str]]
|
||||
|
||||
supports_transcription: ClassVar[Literal[True]] = True
|
||||
|
||||
supports_transcription_only: ClassVar[bool] = False
|
||||
"""
|
||||
Transcription models can opt out of text generation by setting this to
|
||||
`True`.
|
||||
"""
|
||||
supports_segment_timestamp: ClassVar[bool] = False
|
||||
"""
|
||||
Enables the segment timestamp option for supported models by setting this to `True`.
|
||||
"""
|
||||
|
||||
supports_diarized_transcription: ClassVar[bool] = False
|
||||
"""Enables the ``diarized_json`` response format for the model."""
|
||||
|
||||
supports_explicit_language_detection: ClassVar[bool] = False
|
||||
"""
|
||||
Transcription models that require an explicit language detection step
|
||||
(e.g. Whisper needs a separate forward pass to predict the language
|
||||
token) should set this to ``True`` and implement
|
||||
:meth:`get_language_detection_prompt` and
|
||||
:meth:`parse_language_detection_output` and
|
||||
:meth:`get_language_token_ids`.
|
||||
"""
|
||||
|
||||
no_space_languages: ClassVar[set[str]] = {"ja", "zh"}
|
||||
"""
|
||||
Languages that don't need a space between words.
|
||||
For example, Japanese (ja) and Chinese (zh) don't need a space between words.
|
||||
"""
|
||||
|
||||
def __init_subclass__(cls, **kwargs):
|
||||
super().__init_subclass__(**kwargs)
|
||||
# language codes in supported_languages
|
||||
# that don't exist in the full language map
|
||||
invalid = set(cls.supported_languages) - set(LANGUAGES.keys())
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
f"{cls.__name__}.supported_languages contains invalid "
|
||||
f"language codes: {sorted(invalid)}\n. "
|
||||
f"Valid choices are: {sorted(LANGUAGES.keys())}"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_generation_prompt(
|
||||
cls,
|
||||
stt_params: "SpeechToTextParams",
|
||||
) -> "PromptType":
|
||||
"""Get the prompt for the ASR model.
|
||||
The model has control over the construction, as long as it
|
||||
returns a valid PromptType."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_other_languages(cls) -> Mapping[str, str]:
|
||||
# other possible language codes from the whisper map
|
||||
return {k: v for k, v in LANGUAGES.items() if k not in cls.supported_languages}
|
||||
|
||||
@classmethod
|
||||
def validate_language(cls, language: str | None) -> str | None:
|
||||
"""Ensure the language specified in the transcription request
|
||||
is a valid ISO 639-1 language code. If the request language is
|
||||
valid, but not natively supported by the model, trigger a
|
||||
warning (but not an exception).
|
||||
"""
|
||||
if language is None or language in cls.supported_languages:
|
||||
return language
|
||||
elif language in cls.get_other_languages():
|
||||
logger.warning(
|
||||
"Language %r is not natively supported by %s; "
|
||||
"results may be less accurate. Supported languages: %r",
|
||||
language,
|
||||
cls.__name__,
|
||||
list(cls.supported_languages.keys()),
|
||||
)
|
||||
return language
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported language: {language!r}. Must be one of "
|
||||
f"{list(cls.supported_languages.keys())}."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls, model_config: "ModelConfig", task_type: Literal["transcribe", "translate"]
|
||||
) -> "SpeechToTextConfig":
|
||||
"""Get the speech to text config for the ASR model."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_num_audio_tokens(
|
||||
cls,
|
||||
audio_duration_s: float,
|
||||
stt_config: "SpeechToTextConfig",
|
||||
model_config: "ModelConfig",
|
||||
) -> int | None:
|
||||
"""Map from audio duration to number of audio tokens produced by the ASR
|
||||
model, without running a forward pass.
|
||||
This is used for estimating the amount of processing for this audio.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def post_process_output(cls, text: str) -> str:
|
||||
"""Post-process the raw model output text.
|
||||
|
||||
Some ASR models output structured formats (e.g., language tags,
|
||||
special tokens) that need to be stripped before returning to the user.
|
||||
|
||||
Args:
|
||||
text: Raw decoded text from the model.
|
||||
|
||||
Returns:
|
||||
Cleaned transcription text.
|
||||
|
||||
"""
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
|
||||
"""Parse the model-specific diarized transcript format.
|
||||
|
||||
Only models that set ``supports_diarized_transcription`` must override
|
||||
this method.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def get_streaming_post_processor_cls(
|
||||
cls,
|
||||
) -> type[StreamingTranscriptionPostProcessor]:
|
||||
"""Return a stateful post-processor class for streaming output deltas.
|
||||
|
||||
Each instance receives the next decoded text delta and whether the
|
||||
request output is final. It returns the cleaned delta that should be
|
||||
sent to the client.
|
||||
"""
|
||||
return StreamingTranscriptionPostProcessor
|
||||
|
||||
@classmethod
|
||||
def get_language_detection_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
stt_config: "SpeechToTextConfig",
|
||||
) -> "PromptType":
|
||||
"""Return a prompt that triggers language detection.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def parse_language_detection_output(
|
||||
cls,
|
||||
token_ids: list[int],
|
||||
tokenizer: "TokenizerLike",
|
||||
) -> str:
|
||||
"""Parse the detected language from model output token IDs.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def get_language_token_ids(
|
||||
cls,
|
||||
tokenizer: "TokenizerLike",
|
||||
) -> list[int] | None:
|
||||
"""Return token IDs that represent valid language tokens.
|
||||
|
||||
Used to constrain language detection to only produce valid language tokens.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@overload
|
||||
def supports_transcription(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsTranscription]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_transcription(model: object) -> TypeIs[SupportsTranscription]: ...
|
||||
|
||||
|
||||
def supports_transcription(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsTranscription]] | TypeIs[SupportsTranscription]:
|
||||
return getattr(model, "supports_transcription", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsEagleBase(Protocol):
|
||||
"""Base interface for models that support EAGLE-based speculative decoding."""
|
||||
|
||||
@@ -62,7 +62,6 @@ from .interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
@@ -456,7 +455,7 @@ class Qwen2Model(nn.Module, EagleModelMixin):
|
||||
|
||||
|
||||
class Qwen2ForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3, SupportsQuant
|
||||
nn.Module, SupportsPP, SupportsEagle, SupportsEagle3, SupportsQuant
|
||||
):
|
||||
hf_to_vllm_mapper = Qwen2Model.hf_to_vllm_mapper
|
||||
packed_modules_mapping = {
|
||||
|
||||
@@ -76,7 +76,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -103,7 +102,6 @@ from .interfaces import (
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalPruning,
|
||||
@@ -1295,7 +1293,6 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
SupportsEagle,
|
||||
@@ -1323,7 +1320,6 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_mm_device_do_normalize = True
|
||||
supports_tower_connector_lora = True
|
||||
|
||||
def iter_mm_grid_thw(
|
||||
self, mm_features: list[MultiModalFeatureSpec]
|
||||
@@ -2129,16 +2125,3 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
connector="visual.merger.",
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
@@ -60,7 +60,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .interfaces import SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -420,7 +420,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
|
||||
fall_back_to_pt_during_load = False
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
|
||||
@@ -70,7 +70,6 @@ from vllm.multimodal.inputs import (
|
||||
ImageItem,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -97,7 +96,6 @@ from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
@@ -1286,7 +1284,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo])
|
||||
class Qwen2VLForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
SupportsEncoderCudaGraph,
|
||||
@@ -1305,7 +1302,6 @@ class Qwen2VLForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_mm_device_do_normalize = True
|
||||
supports_tower_connector_lora = True
|
||||
|
||||
def iter_mm_grid_thw(
|
||||
self, mm_features: list[MultiModalFeatureSpec]
|
||||
@@ -1860,16 +1856,3 @@ class Qwen2VLForConditionalGeneration(
|
||||
connector="visual.merger.",
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
@@ -52,7 +52,6 @@ from .interfaces import (
|
||||
LocalArgmaxMixin,
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
)
|
||||
from .qwen2 import Qwen2MLP as Qwen3MLP
|
||||
@@ -278,17 +277,13 @@ class Qwen3Model(Qwen2Model):
|
||||
|
||||
|
||||
class Qwen3ForCausalLM(
|
||||
LocalArgmaxMixin, nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
LocalArgmaxMixin, nn.Module, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
):
|
||||
hf_to_vllm_mapper = Qwen3Model.hf_to_vllm_mapper
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
embedding_modules = {
|
||||
"embed_tokens": "input_embeddings",
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
"""Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -76,7 +75,6 @@ from .interfaces import (
|
||||
MixtureOfExperts,
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
_require_is_multimodal,
|
||||
@@ -328,7 +326,6 @@ class Qwen3_5ForCausalLMBase(
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
):
|
||||
@@ -343,11 +340,6 @@ class Qwen3_5ForCausalLMBase(
|
||||
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
|
||||
"in_proj_ba": ["in_proj_b", "in_proj_a"],
|
||||
}
|
||||
# Maps PEFT embed/lm_head LoRA targets onto vLLM embedding wrappers.
|
||||
embedding_modules = {
|
||||
"embed_tokens": "input_embeddings",
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
# Some community text-only checkpoints keep the extraneous
|
||||
# `model.language_model.` prefix inherited from the VL training stack.
|
||||
@@ -731,9 +723,6 @@ class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts):
|
||||
class Qwen3_5MoeForConditionalGeneration(
|
||||
Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts
|
||||
):
|
||||
# For MoE LoRA weights loading
|
||||
is_3d_moe_weight: ClassVar[bool] = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
||||
# protocols have not __init__ method, so we need to use nn.Module.__init__
|
||||
nn.Module.__init__(self)
|
||||
|
||||
@@ -68,7 +68,6 @@ from .interfaces import (
|
||||
IsHybrid,
|
||||
MixtureOfExperts,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
)
|
||||
from .utils import (
|
||||
@@ -820,7 +819,6 @@ class QwenNextMixtureOfExperts(MixtureOfExperts):
|
||||
class Qwen3NextForCausalLM(
|
||||
nn.Module,
|
||||
HasInnerState,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
QwenNextMixtureOfExperts,
|
||||
IsHybrid,
|
||||
|
||||
@@ -129,7 +129,6 @@ from .interfaces import (
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalPruning,
|
||||
@@ -1961,7 +1960,6 @@ class Qwen3VLForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
SupportsEagle,
|
||||
@@ -1982,7 +1980,6 @@ class Qwen3VLForConditionalGeneration(
|
||||
}
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_tower_connector_lora = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
supported_video_pruning_methods = ("evs", "vidcom2")
|
||||
@@ -3258,19 +3255,6 @@ class Qwen3VLForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _cached_tensor(x, device) -> torch.Tensor:
|
||||
|
||||
@@ -55,7 +55,6 @@ from .interfaces import (
|
||||
supports_multimodal_raw_input_only,
|
||||
supports_pp,
|
||||
supports_replayssm,
|
||||
supports_transcription,
|
||||
)
|
||||
from .interfaces_base import (
|
||||
get_attn_type,
|
||||
@@ -232,8 +231,6 @@ class _ModelInfo:
|
||||
is_hybrid: bool
|
||||
has_noops: bool
|
||||
supports_replayssm: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
supports_kv_cache_grouping: bool
|
||||
supported_video_pruning_methods: tuple[str, ...]
|
||||
supports_mm_device_do_normalize: bool
|
||||
@@ -262,10 +259,6 @@ class _ModelInfo:
|
||||
is_attention_free=is_attention_free(model),
|
||||
is_hybrid=is_hybrid(model),
|
||||
supports_replayssm=supports_replayssm(model),
|
||||
supports_transcription=supports_transcription(model),
|
||||
supports_transcription_only=(
|
||||
supports_transcription(model) and model.supports_transcription_only
|
||||
),
|
||||
supports_kv_cache_grouping=supports_kv_cache_grouping(model),
|
||||
has_noops=has_noops(model),
|
||||
supported_video_pruning_methods=getattr(
|
||||
@@ -859,22 +852,6 @@ class _ModelRegistry:
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.has_noops
|
||||
|
||||
def is_transcription_model(
|
||||
self,
|
||||
architectures: str | list[str],
|
||||
model_config: ModelConfig,
|
||||
) -> bool:
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.supports_transcription
|
||||
|
||||
def is_transcription_only_model(
|
||||
self,
|
||||
architectures: str | list[str],
|
||||
model_config: ModelConfig,
|
||||
) -> bool:
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.supports_transcription_only
|
||||
|
||||
|
||||
def _resolve_module_name(mod_relname: str) -> str:
|
||||
# Allow registry entries to point at fully-qualified module paths (e.g.
|
||||
@@ -929,11 +906,6 @@ def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
|
||||
|
||||
|
||||
def _run() -> None:
|
||||
# Setup plugins
|
||||
from vllm.plugins import load_general_plugins
|
||||
|
||||
load_general_plugins()
|
||||
|
||||
fn, output_file = pickle.loads(sys.stdin.buffer.read())
|
||||
|
||||
result = fn()
|
||||
|
||||
@@ -181,13 +181,13 @@ class WeightsMapper:
|
||||
"""Mapper variant keeping only the renames.
|
||||
|
||||
This is what consumers that *name* modules rather than load them need:
|
||||
LoRA name parsing and the quantization config's layer lists.
|
||||
the quantization config's layer lists.
|
||||
|
||||
Stacked maps are dropped so that constituent names (e.g. `q_proj`) survive
|
||||
rather than being rewritten to the stacked vLLM name (`qkv_proj`). Mappings to
|
||||
`None` are dropped because "do not load this weight" is meaningless to such a
|
||||
consumer, and applying it would silently shrink a quantization config's ignore
|
||||
list or make LoRA name parsing fail."""
|
||||
consumer, and applying it would silently shrink a quantization config's
|
||||
ignore list."""
|
||||
remove_none = lambda d: {k: v for k, v in d.items() if v is not None}
|
||||
return replace(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user