[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -135,8 +135,7 @@ class CustomOp(nn.Module):
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
"""PyTorch-native implementation; OOT plugins override via
|
||||
:meth:`forward_oot`.
|
||||
"""
|
||||
:meth:`forward_oot`."""
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_oot(self, *args, **kwargs):
|
||||
|
||||
@@ -10,8 +10,7 @@ from vllm.model_executor.hw_agnostic.custom_op import CustomOp
|
||||
@CustomOp.register("rms_norm")
|
||||
class RMSNorm(CustomOp):
|
||||
"""``x -> w * x / sqrt(E[x^2] + eps)``. With ``residual``, fuses
|
||||
``residual += x`` then RMSNorm and returns ``(normalized, residual)``.
|
||||
"""
|
||||
``residual += x`` then RMSNorm and returns ``(normalized, residual)``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -881,8 +881,7 @@ def choose_mp_linear_kernel(
|
||||
|
||||
def init_mxfp8_linear_kernel(*, bmm_batch_size: int | None = None) -> Mxfp8LinearKernel:
|
||||
"""Select and instantiate the best MXFP8 linear kernel for the
|
||||
current platform.
|
||||
"""
|
||||
current platform."""
|
||||
config = Mxfp8LinearLayerConfig(bmm_batch_size=bmm_batch_size)
|
||||
|
||||
platform = current_platform._enum
|
||||
@@ -934,8 +933,7 @@ def init_mxfp4_linear_kernel(
|
||||
activation_quant_key: QuantKey | None = None,
|
||||
) -> MxFp4LinearKernel:
|
||||
"""Select and instantiate the best MXFP4 linear kernel for the
|
||||
current platform.
|
||||
"""
|
||||
current platform."""
|
||||
config = MxFp4LinearLayerConfig(
|
||||
activation_quant_key=activation_quant_key,
|
||||
)
|
||||
@@ -982,8 +980,7 @@ def init_mxfp6_linear_kernel(
|
||||
activation_quant_key: QuantKey | None = None,
|
||||
) -> MxFp6LinearKernel:
|
||||
"""Select and instantiate the best MXFP6 linear kernel for the
|
||||
current platform.
|
||||
"""
|
||||
current platform."""
|
||||
config = MxFp6LinearLayerConfig(
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
@@ -1071,8 +1068,7 @@ def init_wfp8_a16_linear_kernel(
|
||||
|
||||
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
"""Select and instantiate the best NVFP4 linear kernel for the
|
||||
current platform.
|
||||
"""
|
||||
current platform."""
|
||||
config = NvFp4LinearLayerConfig()
|
||||
a16_kernels = (
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
|
||||
@@ -177,8 +177,7 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""This kernel supports dynamic quantization of the input. By
|
||||
convention, pre-quantized blockscales must use the swizzled layout.
|
||||
"""
|
||||
convention, pre-quantized blockscales must use the swizzled layout."""
|
||||
return kNvfp4Dynamic
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -41,8 +41,7 @@ class FP8BlockParams(FP8Params):
|
||||
@property
|
||||
def block_scale_attr(self) -> str:
|
||||
"""Fp8LinearMethod registers the block scale as ``weight_scale_inv``,
|
||||
compressed-tensors as ``weight_scale``.
|
||||
"""
|
||||
compressed-tensors as ``weight_scale``."""
|
||||
return (
|
||||
self.WEIGHT_SCALE
|
||||
if self.weight_scale_inv is None
|
||||
|
||||
@@ -174,8 +174,7 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""Only static per-tensor activation quantization is supported for external
|
||||
quantization.
|
||||
"""
|
||||
quantization."""
|
||||
if self.config.activation_quant_key == kFp8StaticTensorSym:
|
||||
return kFp8StaticTensorSym
|
||||
return None
|
||||
@@ -185,8 +184,7 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
x: torch.Tensor, dim: int, alignment: int, value: float = 0.0
|
||||
) -> torch.Tensor:
|
||||
"""Pad tensor ``x`` along ``dim`` to the next multiple of
|
||||
``alignment``.
|
||||
"""
|
||||
``alignment``."""
|
||||
remainder = x.shape[dim] % alignment
|
||||
if remainder == 0:
|
||||
return x
|
||||
|
||||
@@ -804,8 +804,7 @@ _ACTIVATION_REGISTRY = LazyDict(
|
||||
|
||||
def _get_gelu_pytorch_tanh() -> nn.Module:
|
||||
"""Get PyTorch GELU with tanh approximation, with ROCm fallback
|
||||
and fast GELU for ARM.
|
||||
"""
|
||||
and fast GELU for ARM."""
|
||||
if current_platform.is_rocm():
|
||||
# TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile
|
||||
logger.warning_once(
|
||||
|
||||
@@ -1568,8 +1568,7 @@ class MLACommonBackend(AttentionBackend):
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Per-token-head modes pack an inline fp32 scale pair after the
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA).
|
||||
"""
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
|
||||
spec = super().customize_spec(spec)
|
||||
mode = spec.kv_quant_mode
|
||||
if spec.state_content_bytes is not None or not mode.is_per_token_head:
|
||||
|
||||
@@ -55,8 +55,7 @@ _FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
|
||||
"""Workspace token budget for flashinfer fused all-reduce, or None if the
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``.
|
||||
"""
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
|
||||
|
||||
@@ -170,8 +170,7 @@ def _fused_embed_eh_norm_kernel(
|
||||
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
|
||||
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
|
||||
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
|
||||
cat. Requires the full table on-rank (replicated embedding).
|
||||
"""
|
||||
cat. Requires the full table on-rank (replicated embedding)."""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
@@ -56,8 +56,7 @@ class MoEActivation(Enum):
|
||||
@property
|
||||
def custom_op_name(self) -> str:
|
||||
"""Maps to the CustomOp name of activations
|
||||
in vllm/model_executor/layers/activation.py.
|
||||
"""
|
||||
in vllm/model_executor/layers/activation.py."""
|
||||
return _CUSTOM_OP_NAMES[self]
|
||||
|
||||
def without_mul(self) -> "MoEActivation":
|
||||
|
||||
@@ -36,8 +36,7 @@ def is_aiter_mxfp8_moe_available() -> bool:
|
||||
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
|
||||
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
|
||||
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
|
||||
closed (returns False -> triton dot_scaled), which is always safe.
|
||||
"""
|
||||
closed (returns False -> triton dot_scaled), which is always safe."""
|
||||
if not (current_platform.is_rocm() and current_platform.supports_mx()):
|
||||
return False
|
||||
try:
|
||||
|
||||
@@ -192,8 +192,7 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
def _pad_moe_intermediate(self, layer: torch.nn.Module) -> None:
|
||||
"""Zero-pad the per-partition MoE intermediate dim of both weights and
|
||||
the expert bias, see `_padded_intermediate_size`.
|
||||
"""
|
||||
the expert bias, see `_padded_intermediate_size`."""
|
||||
intermediate_size = self.moe_config.intermediate_size_per_partition
|
||||
padded_size = self._padded_intermediate_size(self.moe_config)
|
||||
if padded_size == intermediate_size:
|
||||
@@ -953,8 +952,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular):
|
||||
|
||||
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
|
||||
modular MoE experts.
|
||||
"""
|
||||
modular MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1287,8 +1285,7 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
|
||||
class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
|
||||
quantization, dispatched through zentorch.
|
||||
"""
|
||||
quantization, dispatched through zentorch."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -34,8 +34,7 @@ from vllm.triton_utils import tl, triton
|
||||
|
||||
def _select_cfg(M, N, K, block_m):
|
||||
"""Pick the launch config from host constants only (M=num_valid_tokens, N, K,
|
||||
block_m) — graph-capture safe (no GPU-scalar branch).
|
||||
"""
|
||||
block_m) — graph-capture safe (no GPU-scalar branch)."""
|
||||
# Per-regime winners (measured, isolated cuda-event A/B on gfx950, GPU 3):
|
||||
# BLOCK_K=256 (fewer K-iters + bigger MX scale-load coalesced with the dot),
|
||||
# num_stages=2 (software-pipeline overlaps the E8M0 scale-load with the scaled
|
||||
|
||||
@@ -55,28 +55,24 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
|
||||
@abstractmethod
|
||||
def backend_enum_cls(self) -> type[BackendT]:
|
||||
"""Return the concrete `Enum` class enumerating this oracle's
|
||||
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`).
|
||||
"""
|
||||
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`)."""
|
||||
|
||||
@abstractmethod
|
||||
def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]:
|
||||
"""Return platform-appropriate backends in priority order for
|
||||
this `moe_config`.
|
||||
"""
|
||||
this `moe_config`."""
|
||||
|
||||
@abstractmethod
|
||||
def backend_to_kernel_cls(
|
||||
self, backend: BackendT
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
"""Map a backend enum value to its concrete `FusedMoEExperts`
|
||||
subclasses, in selection priority order.
|
||||
"""
|
||||
subclasses, in selection priority order."""
|
||||
|
||||
@abstractmethod
|
||||
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
|
||||
"""Map a user-facing `MoEBackend` (from the runner config) to
|
||||
this oracle's enum.
|
||||
"""
|
||||
this oracle's enum."""
|
||||
|
||||
@abstractmethod
|
||||
def select_backend(
|
||||
@@ -108,8 +104,7 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Construct the `FusedMoEKernel` (Prepare/Finalize + Experts
|
||||
combinator) for the chosen backend.
|
||||
"""
|
||||
combinator) for the chosen backend."""
|
||||
|
||||
def convert_to_kernel_format(
|
||||
self,
|
||||
|
||||
@@ -451,8 +451,7 @@ def _humming_fp8_weight_schema(
|
||||
layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
|
||||
) -> dict[str, Any]:
|
||||
"""Build the humming weight schema from the canonical on-device fp8/mxfp8
|
||||
tensors (scale dtype/shape, block size), not the producing quant method.
|
||||
"""
|
||||
tensors (scale dtype/shape, block size), not the producing quant method."""
|
||||
# mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
|
||||
# no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
|
||||
if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
|
||||
|
||||
@@ -224,8 +224,7 @@ def _humming_int8_weight_schema(
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor
|
||||
) -> dict[str, Any]:
|
||||
"""Build the humming compressed-tensors int8 schema from the canonical
|
||||
on-device tensors; humming does the signed-int8 -> native conversion.
|
||||
"""
|
||||
on-device tensors; humming does the signed-int8 -> native conversion."""
|
||||
config: dict[str, Any] = {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "int-quantized",
|
||||
|
||||
@@ -512,8 +512,7 @@ def _process_weights_flashinfer(
|
||||
|
||||
def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor:
|
||||
"""Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)``
|
||||
tensor along its last dim, from ``unit`` to ``padded_unit`` columns.
|
||||
"""
|
||||
tensor along its last dim, from ``unit`` to ``padded_unit`` columns."""
|
||||
if padded_unit == unit:
|
||||
return x
|
||||
e, rows, _ = x.shape
|
||||
@@ -1081,8 +1080,7 @@ def _humming_wna16_weight_schema(
|
||||
quant_config: QuantizationConfig | QuantizationArgs | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Humming weight schema for a WNA16 checkpoint, derived from the quant
|
||||
config rather than the running kernel.
|
||||
"""
|
||||
config rather than the running kernel."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
|
||||
|
||||
@@ -331,8 +331,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
|
||||
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
"""Available backends in priority order, BF16-act variant before
|
||||
activation-quantized variant within each vendor family.
|
||||
"""
|
||||
activation-quantized variant within each vendor family."""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
@@ -411,8 +410,7 @@ def _resolve_activation_key(
|
||||
model_activation_key: QuantKey | None,
|
||||
) -> QuantKey | None:
|
||||
"""Combine the model-supplied activation key with the user override.
|
||||
Raises on conflict (both set and disagreeing).
|
||||
"""
|
||||
Raises on conflict (both set and disagreeing)."""
|
||||
user_override = _user_moe_activation_override()
|
||||
if user_override is None:
|
||||
return model_activation_key
|
||||
@@ -462,8 +460,7 @@ def _filter_by_activation(
|
||||
) -> list[Mxfp4MoeBackend]:
|
||||
"""Pick variants matching ``requested_activation_key``; without one,
|
||||
prefer BF16 if the list has any, else keep the list as-is so explicit
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land.
|
||||
"""
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
|
||||
if requested_activation_key is not None:
|
||||
return [
|
||||
b
|
||||
|
||||
@@ -71,8 +71,7 @@ class ZeroExpertRouter(BaseRouter):
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute routing with full bias, compute zero expert output,
|
||||
mask zero expert IDs.
|
||||
"""
|
||||
mask zero expert IDs."""
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
|
||||
@@ -163,8 +163,7 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
) -> str:
|
||||
"""Resolve the decode top-k implementation from the configured
|
||||
backend ("auto" = the pre-existing chain, or a validated explicit
|
||||
value).
|
||||
"""
|
||||
value)."""
|
||||
if self._backend == "auto":
|
||||
return self._resolve_auto(logits, topk_tokens, num_rows)
|
||||
|
||||
@@ -211,8 +210,7 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""The pre-existing priority chain: cooperative -> persistent ->
|
||||
per_row. deep_select/flashinfer/torch are opt-in only.
|
||||
"""
|
||||
per_row. deep_select/flashinfer/torch are opt-in only."""
|
||||
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
|
||||
return "cooperative"
|
||||
if self._is_cuda and topk_tokens in (512, 1024, 2048):
|
||||
@@ -274,8 +272,7 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
"""Run the resolved decode top-k implementation, writing into
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
|
||||
"""
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
|
||||
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
|
||||
if backend == "deep_select":
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
|
||||
@@ -158,8 +158,7 @@ class LinearMethodBase(QuantizeMethodBase):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the weights in layer to the input tensor.
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
|
||||
@@ -861,8 +861,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""ROCm forward using AITER Triton fused projection+attention when
|
||||
available, otherwise falling back to the generic CUDA path.
|
||||
"""
|
||||
available, otherwise falling back to the generic CUDA path."""
|
||||
if GDN_AITER_TRITON_AVAILABLE:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
|
||||
@@ -261,8 +261,7 @@ class MambaStateShapeCalculator:
|
||||
@classmethod
|
||||
def extra_groups_for_head_shards(cls, ngroups: int, tp_size: int):
|
||||
"""Compute the increase in group numbers to account for
|
||||
replication in order to accompany the head shards.
|
||||
"""
|
||||
replication in order to accompany the head shards."""
|
||||
# in the case ngoups % tp_size == 0, this will be zero
|
||||
if ngroups % tp_size == 0:
|
||||
return 0
|
||||
|
||||
@@ -59,8 +59,7 @@ def get_ssm_device_name() -> str:
|
||||
|
||||
def _canonical_cache_dtype(cache_dtype: str) -> str:
|
||||
"""Canonical key for config lookup. bf16 and fp16 share the same tuned
|
||||
configs because the kernel only sees bit width when accessing state.
|
||||
"""
|
||||
configs because the kernel only sees bit width when accessing state."""
|
||||
return "float16" if cache_dtype == "bfloat16" else cache_dtype
|
||||
|
||||
|
||||
@@ -161,8 +160,7 @@ _ssm_config_override: tuple[int, int] | None = None
|
||||
@contextmanager
|
||||
def override_ssm_config(config: tuple[int, int]):
|
||||
"""Pin ``try_get_optimal_ssm_config`` to ``config`` for the duration of
|
||||
the context. Used by the tuning benchmark to time specific configs.
|
||||
"""
|
||||
the context. Used by the tuning benchmark to time specific configs."""
|
||||
global _ssm_config_override
|
||||
prev = _ssm_config_override
|
||||
_ssm_config_override = config
|
||||
|
||||
@@ -210,8 +210,7 @@ class LamportWorkspace:
|
||||
@property
|
||||
def workspace(self) -> torch.Tensor:
|
||||
"""Device tensor (int64) that can be passed to the kernel
|
||||
as ``void** workspace``.
|
||||
"""
|
||||
as ``void** workspace``."""
|
||||
return self._workspace
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -46,24 +46,21 @@ class QuantizeMethodBase(ABC):
|
||||
):
|
||||
"""Create weights for a layer.
|
||||
|
||||
The weights will be set as attributes of the layer.
|
||||
"""
|
||||
The weights will be set as attributes of the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
|
||||
"""Apply the weights in layer to the input tensor.
|
||||
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
# Not required functions
|
||||
def embedding(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
|
||||
"""Gather embeddings in the layer based on indices in the input tensor.
|
||||
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
# Not required functions
|
||||
@@ -76,8 +73,7 @@ class QuantizeMethodBase(ABC):
|
||||
Quantization methods that need special weight handling (e.g. repacked
|
||||
weights) override this.
|
||||
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
layer.weight = embed_tokens.weight
|
||||
return layer
|
||||
|
||||
|
||||
@@ -665,8 +665,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
) -> bool:
|
||||
"""Weight N-bit INT (pack-quantized for sub-byte, int-quantized for 8-bit)
|
||||
with static per-tensor INT8 input/output activation quant, applied as a float
|
||||
fake-quant around a weight-only matmul.
|
||||
"""
|
||||
fake-quant around a weight-only matmul."""
|
||||
is_int_pack_format = format in (
|
||||
CompressionFormat.pack_quantized.value,
|
||||
CompressionFormat.int_quantized.value,
|
||||
@@ -702,8 +701,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
format: str | None,
|
||||
) -> bool:
|
||||
"""Weight N-bit INT with symmetric dynamic INT activation quant
|
||||
via Humming kernel.
|
||||
"""
|
||||
via Humming kernel."""
|
||||
if input_quant is None:
|
||||
return False
|
||||
is_pack_format = format == CompressionFormat.pack_quantized.value
|
||||
|
||||
+1
-2
@@ -37,8 +37,7 @@ def _dequant_gather_kernel(
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Gather embedding rows by token id, unpack int32-packed INT weights, and
|
||||
dequantize to ``out`` dtype in one pass (no int8 intermediate).
|
||||
"""
|
||||
dequantize to ``out`` dtype in one pass (no int8 intermediate)."""
|
||||
row = tl.program_id(0)
|
||||
col = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
col_mask = col < hidden
|
||||
|
||||
+1
-2
@@ -211,8 +211,7 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
|
||||
|
||||
def _pack_int_quantized_weight(self, layer: torch.nn.Module) -> None:
|
||||
"""Normalize an int-quantized (plain int8) weight to the canonical
|
||||
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect.
|
||||
"""
|
||||
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect."""
|
||||
weight = layer.weight
|
||||
out_features, in_features = weight.shape
|
||||
packed = pack_to_int32(weight.data.contiguous(), self.num_bits)
|
||||
|
||||
@@ -86,8 +86,7 @@ class INCConfigParser:
|
||||
|
||||
def is_explicitly_configured(name: str) -> bool:
|
||||
"""Return True if *name* has an explicit entry in extra_config,
|
||||
either via exact key match or via a regex pattern key.
|
||||
"""
|
||||
either via exact key match or via a regex pattern key."""
|
||||
if not self._config.extra_config:
|
||||
return False
|
||||
if name in self._config.extra_config:
|
||||
|
||||
@@ -48,8 +48,7 @@ class INCScheme(ABC):
|
||||
layer_config: "INCLayerConfig",
|
||||
) -> "FusedMoEMethodBase | None":
|
||||
"""Optional. Override if this scheme supports MoE.
|
||||
Default raises NotImplementedError.
|
||||
"""
|
||||
Default raises NotImplementedError."""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support MoE layers. "
|
||||
f"Layer config: {layer_config}"
|
||||
|
||||
@@ -1988,8 +1988,7 @@ class KNvfp4Static(QuantKeyScheme):
|
||||
|
||||
class KNvfp4Dynamic(QuantKeyScheme):
|
||||
"""NVFP4 activation scheme (W4A4). Has a static global input scale on disk;
|
||||
the per-group scale is computed at runtime inside the kernel.
|
||||
"""
|
||||
the per-group scale is computed at runtime inside the kernel."""
|
||||
|
||||
key = kNvfp4Dynamic
|
||||
|
||||
@@ -2026,8 +2025,7 @@ class KNvfp4Dynamic(QuantKeyScheme):
|
||||
|
||||
class KFp8StaticTensor(QuantKeyScheme):
|
||||
"""Plain per-tensor static FP8 — bivalent: serves BOTH the weight slot and
|
||||
the activation slot (W8A8). One key in both QuantSpec slots.
|
||||
"""
|
||||
the activation slot (W8A8). One key in both QuantSpec slots."""
|
||||
|
||||
key = kFp8StaticTensorSym
|
||||
|
||||
@@ -2097,8 +2095,7 @@ class KFp8StaticTensor(QuantKeyScheme):
|
||||
|
||||
class KFp8StaticChannel(QuantKeyScheme):
|
||||
"""Per-channel static FP8 weight (the 'PcPt' weight). Weight-role only —
|
||||
there is no static per-channel *activation* today.
|
||||
"""
|
||||
there is no static per-channel *activation* today."""
|
||||
|
||||
key = kFp8StaticTokenSym
|
||||
|
||||
@@ -2139,8 +2136,7 @@ class KFp8StaticChannel(QuantKeyScheme):
|
||||
class KFp8Block128(QuantKeyScheme):
|
||||
"""128x128 block-static FP8 weight ('PbWo'). Weight-role only. ModelOpt
|
||||
exports the scale 4-D [out_blk,1,in_blk,1]; process squeezes to 2-D.
|
||||
No transpose (block kernel keeps [out,in]).
|
||||
"""
|
||||
No transpose (block kernel keeps [out,in])."""
|
||||
|
||||
key = kFp8Static128BlockSym
|
||||
|
||||
@@ -2196,8 +2192,7 @@ class KFp8Block128(QuantKeyScheme):
|
||||
|
||||
class KMxfp8Static(QuantKeyScheme):
|
||||
"""MXFP8 weight: fp8-e4m3 values + per-32-block e8m0 (uint8) scale.
|
||||
Weight-role only. process is validate-only plus an idempotency guard.
|
||||
"""
|
||||
Weight-role only. process is validate-only plus an idempotency guard."""
|
||||
|
||||
key = kMxfp8Static
|
||||
|
||||
@@ -2272,8 +2267,7 @@ class KDynamicNoParam(QuantKeyScheme):
|
||||
"""Dynamic activation with no stored scale (W8A8): quantized at runtime in
|
||||
the kernel. NOT the same as activation=None (weight-only) — init_fp8 needs a
|
||||
non-None activation key. Activation-role only. Serves the fp8 per-token, fp8
|
||||
per-block, and mxfp8 dynamic activation keys.
|
||||
"""
|
||||
per-block, and mxfp8 dynamic activation keys."""
|
||||
|
||||
def create_weights(self, layer, role, ctx, shapes, wl) -> None:
|
||||
if role is not ACT:
|
||||
|
||||
@@ -79,8 +79,7 @@ class Mxfp4Config(QuantizationConfig):
|
||||
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
"""MoE method for RoutedExperts. Subclasses override to pick a
|
||||
checkpoint-specific kernel family.
|
||||
"""
|
||||
checkpoint-specific kernel family."""
|
||||
return Mxfp4MoEMethod(moe)
|
||||
|
||||
def get_quant_method(
|
||||
|
||||
@@ -139,8 +139,7 @@ def _find_matching_targets(
|
||||
|
||||
class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""Model-level config for online quantization (quantize fp16/bf16 weights
|
||||
during model loading, without requiring a pre-quantized checkpoint).
|
||||
"""
|
||||
during model loading, without requiring a pre-quantized checkpoint)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -118,8 +118,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
|
||||
|
||||
class OnlineLinearBase(LinearMethodBase):
|
||||
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time.
|
||||
"""
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
@@ -163,8 +162,7 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
|
||||
"""
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -265,8 +263,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online blockwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading.
|
||||
"""
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -440,8 +437,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
"""Shared base for online FP8 MoE methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time.
|
||||
"""
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
# Declared here for mypy; actual values are set in __init__.
|
||||
fp8_backend: "Fp8MoeBackend"
|
||||
@@ -562,8 +558,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
|
||||
class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
"""Online tensorwise FP8 MoE quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
|
||||
"""
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -620,8 +615,7 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
|
||||
class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
"""Online blockwise FP8 MoE quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading.
|
||||
"""
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -69,8 +69,7 @@ def input_to_float8(
|
||||
x: torch.Tensor, dtype: torch.dtype | None = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This function quantizes input values to float8 values "
|
||||
"with tensor-wise quantization.
|
||||
"""
|
||||
"with tensor-wise quantization."""
|
||||
dtype = current_platform.fp8_dtype() if dtype is None else dtype
|
||||
finfo = torch.finfo(dtype)
|
||||
min_val, max_val = x.aminmax()
|
||||
@@ -1195,8 +1194,7 @@ def prepare_fp8_moe_layer_for_deepgemm(
|
||||
|
||||
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Pad the weight tensor. This is an optimization on ROCm platform, which
|
||||
can benefit from tensors located far enough from one another in memory
|
||||
"""
|
||||
can benefit from tensors located far enough from one another in memory"""
|
||||
if (
|
||||
envs.VLLM_ROCM_FP8_PADDING
|
||||
and current_platform.is_rocm()
|
||||
|
||||
@@ -297,8 +297,7 @@ def _humming_input_schema_to_quant_key(
|
||||
schema: "HummingInputSchema",
|
||||
) -> QuantKey | None:
|
||||
"""Convert a HummingInputSchema to a QuantKey. Returns None if
|
||||
the schema represents unquantized (bf16/fp16) inputs.
|
||||
"""
|
||||
the schema represents unquantized (bf16/fp16) inputs."""
|
||||
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
|
||||
return None
|
||||
|
||||
@@ -427,8 +426,7 @@ def convert_linear_layer_to_humming_standard(
|
||||
):
|
||||
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
|
||||
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``).
|
||||
"""
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
|
||||
for name, checkpoint_name in name_map.items():
|
||||
tensor = getattr(layer, checkpoint_name)
|
||||
delattr(layer, checkpoint_name)
|
||||
|
||||
@@ -245,8 +245,7 @@ def marlin_padded_nk(size_n: int, size_k: int, group_size: int = -1) -> tuple[in
|
||||
|
||||
def marlin_repacked_nk(qweight: torch.Tensor, num_bits: int) -> tuple[int, int]:
|
||||
"""Recover the (size_n, size_k) a Marlin weight was repacked with
|
||||
(including any tile padding) from its packed shape.
|
||||
"""
|
||||
(including any tile padding) from its packed shape."""
|
||||
pack_factor = 32 // num_bits
|
||||
size_k = qweight.size(0) * GPTQ_MARLIN_TILE
|
||||
size_n = qweight.size(1) * pack_factor // GPTQ_MARLIN_TILE
|
||||
@@ -257,8 +256,7 @@ def marlin_pad_qweight(
|
||||
qweight: torch.Tensor, size_n: int, size_k: int, padded_n: int, padded_k: int
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad a GPTQ-layout packed weight (size_k / pack, size_n) for
|
||||
gptq_marlin_repack.
|
||||
"""
|
||||
gptq_marlin_repack."""
|
||||
if (padded_n, padded_k) == (size_n, size_k):
|
||||
return qweight
|
||||
pack_factor = size_k // qweight.size(0)
|
||||
@@ -276,8 +274,7 @@ def marlin_pad_scales(
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad weight scales (num_groups, size_n); call before
|
||||
marlin_permute_scales and pass the padded extents to it.
|
||||
"""
|
||||
marlin_permute_scales and pass the padded extents to it."""
|
||||
if (padded_n, padded_k) == (size_n, size_k):
|
||||
return scales
|
||||
pad_rows = padded_k // group_size - scales.size(0) if group_size > 0 else 0
|
||||
|
||||
@@ -41,8 +41,7 @@ def _nvfp4_compute_scale_factor(
|
||||
) -> float:
|
||||
"""Compute the power-of-2 scale_factor needed so that all non-zero
|
||||
values in marlin_scales * 2^7 are >= 2 after rescaling.
|
||||
Returns a Python float (power of 2, >= 1.0).
|
||||
"""
|
||||
Returns a Python float (power of 2, >= 1.0)."""
|
||||
# Since half has a smaller dynamic range compared to bfloat16,
|
||||
# no rescaling is applied here if active dtype is half.
|
||||
if a_dtype is not None and a_dtype == torch.half:
|
||||
@@ -376,8 +375,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
def pad_w13(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a (E, num_shards * N, cols)
|
||||
tensor to padded_N rows.
|
||||
"""
|
||||
tensor to padded_N rows."""
|
||||
if padded_N == N:
|
||||
return x
|
||||
x = x.view(E, num_shards, N, x.size(-1))
|
||||
@@ -386,8 +384,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
def pad_w2(x: torch.Tensor, packing: int) -> torch.Tensor:
|
||||
"""Zero-pad the packed N (last) dim of a (E, K, N / packing)
|
||||
tensor.
|
||||
"""
|
||||
tensor."""
|
||||
if padded_N == N:
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
|
||||
|
||||
@@ -217,8 +217,7 @@ def prepare_fp8_layer_for_marlin(
|
||||
|
||||
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.
|
||||
"""
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
|
||||
if padded_n == n:
|
||||
return x
|
||||
e = x.size(0)
|
||||
|
||||
@@ -9,8 +9,7 @@ from .base import RotaryEmbedding
|
||||
|
||||
class NTKScalingRotaryEmbedding(RotaryEmbedding):
|
||||
"""RotaryEmbedding extended with fixed and mixed NTK scaling.
|
||||
https://kexue.fm/archives/9706
|
||||
"""
|
||||
https://kexue.fm/archives/9706"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -285,8 +285,7 @@ def _gather_workspace_shapes(
|
||||
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
|
||||
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
|
||||
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales.
|
||||
"""
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
|
||||
if use_fp4_cache:
|
||||
return (
|
||||
((total_seq_lens, head_dim // 2), torch.uint8),
|
||||
@@ -304,8 +303,7 @@ def kv_cache_as_quant_view(
|
||||
use_fp4_cache: bool,
|
||||
) -> torch.Tensor:
|
||||
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation.
|
||||
"""
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation."""
|
||||
if use_fp4_cache:
|
||||
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
|
||||
num_blocks, block_size, _ = kv_cache.shape
|
||||
|
||||
@@ -41,8 +41,7 @@ def _prefill_k_workspaces(
|
||||
total_seq_lens: int, head_dim: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The packed MXFP4 K-gather workspace ``(values, scales)``, shared with
|
||||
the dense indexer layers of the same model.
|
||||
"""
|
||||
the dense indexer layers of the same model."""
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, current_platform.fp8_dtype(), use_fp4_cache=True
|
||||
)
|
||||
@@ -120,8 +119,7 @@ class SparseMQAIndexer(nn.Module):
|
||||
|
||||
def _reserve_workspaces(self, device: torch.device) -> None:
|
||||
"""Profiling run: claim the K-gather workspace and the peak sparse
|
||||
logits allocation so the memory estimate covers them.
|
||||
"""
|
||||
logits allocation so the memory estimate covers them."""
|
||||
_prefill_k_workspaces(self.max_total_seq_len, self.head_dim)
|
||||
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
|
||||
torch.empty(max_logits_bytes, dtype=torch.uint8, device=device)
|
||||
|
||||
@@ -376,8 +376,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
) -> VocabParallelEmbeddingShardIndices:
|
||||
"""Get start and end indices for vocab parallel embedding, following the
|
||||
layout outlined in the class docstring, based on the given tp_rank and
|
||||
tp_size.
|
||||
"""
|
||||
tp_size."""
|
||||
num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded
|
||||
padded_org_vocab_start_index, padded_org_vocab_end_index = (
|
||||
vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size)
|
||||
|
||||
@@ -36,8 +36,7 @@ class BaseModelLoader(ABC):
|
||||
@abstractmethod
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
"""Load weights into a model. This standalone API allows
|
||||
inplace weights loading for an already-initialized model
|
||||
"""
|
||||
inplace weights loading for an already-initialized model"""
|
||||
raise NotImplementedError
|
||||
|
||||
def create_model(
|
||||
|
||||
@@ -135,8 +135,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
) -> tuple[str, list[str], bool]:
|
||||
"""Prepare weights for the model.
|
||||
|
||||
If the model is not local, it will be downloaded.
|
||||
"""
|
||||
If the model is not local, it will be downloaded."""
|
||||
model_name_or_path = (
|
||||
maybe_download_from_modelscope(model_name_or_path, revision)
|
||||
or model_name_or_path
|
||||
|
||||
@@ -23,8 +23,7 @@ def parse_expert_id(weight_name: str) -> int | None:
|
||||
|
||||
Returns ``None`` for dense weights (attention, layernorm, embedding),
|
||||
shared experts, and 3D fused-expert tensors where all experts are stored
|
||||
in a single tensor without a numeric expert id in the name.
|
||||
"""
|
||||
in a single tensor without a numeric expert id in the name."""
|
||||
m = _EXPERT_ID_RE.search(weight_name)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
@@ -71,8 +70,7 @@ def should_skip_weight(
|
||||
local_expert_ids: set[int] | None,
|
||||
) -> bool:
|
||||
"""Return ``True`` if *weight_name* is an expert weight that does not
|
||||
belong to the local rank and should be skipped during loading.
|
||||
"""
|
||||
belong to the local rank and should be skipped during loading."""
|
||||
if local_expert_ids is None:
|
||||
return False
|
||||
eid = parse_expert_id(weight_name)
|
||||
|
||||
@@ -308,8 +308,7 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
||||
|
||||
Assumes dtype/shapes of attention tensors do not change during
|
||||
processing, since we use .data.copy_() to preserve kernel tensor
|
||||
references.
|
||||
"""
|
||||
references."""
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if quant_method is not None:
|
||||
# Re-create scale Parameters with sentinel values so unloaded scales
|
||||
@@ -388,8 +387,7 @@ def _get_weight_loader(tensor: torch.Tensor):
|
||||
|
||||
def _copy_and_restore_kernel_tensors(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
"""Copy processed values into original kernel tensor storage and restore
|
||||
kernel tensor references on the layer. Preserves cudagraph references.
|
||||
"""
|
||||
kernel tensor references on the layer. Preserves cudagraph references."""
|
||||
assert info.kernel_tensors is not None
|
||||
parameters, buffers = info.kernel_tensors
|
||||
non_persistent = info.kernel_non_persistent_buffers
|
||||
|
||||
@@ -81,8 +81,7 @@ class RunaiModelStreamerLoader(BaseModelLoader):
|
||||
) -> list[str]:
|
||||
"""Prepare weights for the model.
|
||||
|
||||
If the model is not local, it will be downloaded.
|
||||
"""
|
||||
If the model is not local, it will be downloaded."""
|
||||
is_object_storage_path = is_runai_obj_uri(model_name_or_path)
|
||||
is_local = os.path.isdir(model_name_or_path)
|
||||
safetensors_pattern = "*.safetensors"
|
||||
|
||||
@@ -497,8 +497,7 @@ def _check_tensors_on_meta_device(model: nn.Module) -> None:
|
||||
|
||||
def _resize_lora_embeddings(model: nn.Module):
|
||||
"""Modify LoRA embedding layers to use bigger tensors
|
||||
to allow for adapter added tokens.
|
||||
"""
|
||||
to allow for adapter added tokens."""
|
||||
for child in model.modules():
|
||||
if (
|
||||
isinstance(child, VocabParallelEmbedding)
|
||||
|
||||
@@ -105,8 +105,7 @@ class TensorizerLoader(BaseModelLoader):
|
||||
|
||||
Expects a vLLM-tensorized model. See the
|
||||
examples/features/tensorize_vllm_model.py example script
|
||||
for serializing vLLM models.
|
||||
"""
|
||||
for serializing vLLM models."""
|
||||
if is_vllm_tensorized(self.tensorizer_config):
|
||||
tensorizer_config = self._patch_tensorizer_config(model_config)
|
||||
deserialize_tensorizer_model(model, tensorizer_config)
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""IPC model loader: maps post-quantized weights from a local weight cache
|
||||
daemon via CUDA IPC instead of loading from disk.
|
||||
"""
|
||||
daemon via CUDA IPC instead of loading from disk."""
|
||||
|
||||
import dataclasses
|
||||
import socket
|
||||
|
||||
@@ -78,8 +78,7 @@ def maybe_retie_word_embeddings(model: nn.Module, model_config: ModelConfig) ->
|
||||
|
||||
Checkpoints produced by quantization or fine-tuning tooling often keep a redundant
|
||||
copy of the tied `lm_head`. Sharing the storage again reclaims the memory it would
|
||||
otherwise cost.
|
||||
"""
|
||||
otherwise cost."""
|
||||
if not model_config.word_embeddings_untied_by_checkpoint:
|
||||
return
|
||||
if (untied := _get_untied_lm_head(model)) is None:
|
||||
|
||||
@@ -144,8 +144,7 @@ def atomic_writer(
|
||||
|
||||
def _natural_sort_key(filepath: str) -> list:
|
||||
"""Natural sort key for filenames with numeric components, such as
|
||||
model-00001-of-00005.safetensors -> ['model-', 1, '-of-', 5, '.safetensors']
|
||||
"""
|
||||
model-00001-of-00005.safetensors -> ['model-', 1, '-of-', 5, '.safetensors']"""
|
||||
return [
|
||||
int(s) if s.isdigit() else s
|
||||
for s in re.split(r"(\d+)", os.path.basename(filepath))
|
||||
@@ -162,8 +161,7 @@ def maybe_download_from_modelscope(
|
||||
"""Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
|
||||
|
||||
Returns the path to the downloaded model, or None if the model is not
|
||||
downloaded from ModelScope.
|
||||
"""
|
||||
downloaded from ModelScope."""
|
||||
if envs.VLLM_USE_MODELSCOPE:
|
||||
# download model from ModelScope hub,
|
||||
# lazy import so that modelscope is not required for normal use.
|
||||
@@ -1148,8 +1146,7 @@ def instanttensor_weights_iterator(
|
||||
use_tqdm_on_load: bool,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files
|
||||
using instanttensor library.
|
||||
"""
|
||||
using instanttensor library."""
|
||||
try:
|
||||
import instanttensor
|
||||
except ImportError as e:
|
||||
|
||||
@@ -47,8 +47,7 @@ from .utils import (
|
||||
|
||||
class ArceeMLP(nn.Module):
|
||||
"""Feed-forward layer for Arcee using ReLU^2 activation
|
||||
(no gating as in LLaMA).
|
||||
"""
|
||||
(no gating as in LLaMA)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -96,8 +95,7 @@ class ArceeMLP(nn.Module):
|
||||
|
||||
class ArceeDecoderLayer(nn.Module):
|
||||
"""Transformer decoder block for Arcee, with self-attention and
|
||||
ReLU^2 MLP.
|
||||
"""
|
||||
ReLU^2 MLP."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -177,8 +175,7 @@ class ArceeDecoderLayer(nn.Module):
|
||||
@support_torch_compile
|
||||
class ArceeModel(nn.Module, EagleModelMixin):
|
||||
"""The transformer model backbone for Arcee (embedding layer + stacked
|
||||
decoder blocks + final norm).
|
||||
"""
|
||||
decoder blocks + final norm)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -280,8 +277,7 @@ class ArceeForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
):
|
||||
"""Arcee Model for causal language modeling, integrated with vLLM
|
||||
runtime.
|
||||
"""
|
||||
runtime."""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
@@ -362,8 +358,7 @@ class ArceeForCausalLM(
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights into the model (delegates to inner model and handles
|
||||
tied embeddings).
|
||||
"""
|
||||
tied embeddings)."""
|
||||
loader = AutoWeightsLoader(self)
|
||||
# AutoWeightLoader handles weight name remapping, including fusing
|
||||
# separate q_proj, k_proj, v_proj into qkv_proj
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Minimal implementation of BlipVisionModel intended to be only used
|
||||
within a vision language model.
|
||||
"""
|
||||
within a vision language model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
@@ -96,8 +96,7 @@ class RMSNorm(nn.Module):
|
||||
|
||||
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
|
||||
"""Returns (norm_class, eps). Uses RMSNorm when config.rms_norm_eps is set,
|
||||
otherwise falls back to LayerNorm with config.layer_norm_eps.
|
||||
"""
|
||||
otherwise falls back to LayerNorm with config.layer_norm_eps."""
|
||||
rms_eps = getattr(config, "rms_norm_eps", None)
|
||||
if rms_eps is not None:
|
||||
return RMSNorm, rms_eps
|
||||
|
||||
@@ -667,8 +667,7 @@ class ConvSubsampling(nn.Module):
|
||||
repeat_num: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Calculates the output length of a Tensor passed
|
||||
through a convolution or max pooling layer
|
||||
"""
|
||||
through a convolution or max pooling layer"""
|
||||
add_pad: float = all_paddings - kernel_size
|
||||
one: float = 1.0
|
||||
for i in range(repeat_num):
|
||||
|
||||
@@ -363,8 +363,7 @@ class ImageEncoderViT(nn.Module):
|
||||
|
||||
class Block(nn.Module):
|
||||
"""Transformer blocks with support of window attention and residual propagation
|
||||
blocks
|
||||
"""
|
||||
blocks"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -139,8 +139,7 @@ class NoRepeatNGramLogitsProcessor:
|
||||
|
||||
class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
"""Example of overriding the wrapper class `__init__()` in order to utilize
|
||||
info about the device type
|
||||
"""
|
||||
info about the device type"""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
|
||||
@@ -511,8 +511,7 @@ def _compiled_sample_step(
|
||||
accept/renoise → convergence, all as vectorized PyTorch ops.
|
||||
|
||||
Returns the temperature-scaled logits ``[num_decode, CL, vocab]`` so the
|
||||
caller can compute logprobs outside the compiled region.
|
||||
"""
|
||||
caller can compute logprobs outside the compiled region."""
|
||||
num_decode = decode_slots.shape[0]
|
||||
device = decode_slots.device
|
||||
|
||||
|
||||
@@ -814,8 +814,7 @@ class Gemma3nCrossDecoder(nn.Module):
|
||||
def _kv_sharing_weights_mapper(config: Gemma3nTextConfig) -> WeightsMapper:
|
||||
"""KV-shared layers only have q_proj, so qkv_proj packing applies to the
|
||||
other layers. Original checkpoints still ship K/V tensors for the shared
|
||||
layers (fine-tuned ones omit them); those are dropped.
|
||||
"""
|
||||
layers (fine-tuned ones omit them); those are dropped."""
|
||||
first_kv_shared_layer_idx = config.num_hidden_layers - config.num_kv_shared_layers
|
||||
return WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
|
||||
@@ -499,8 +499,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
|
||||
class Gemma3nMultimodalEmbedder(nn.Module):
|
||||
"""Embeds token ids or soft tokens for multimodal content into language
|
||||
model space.
|
||||
"""
|
||||
model space."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -25,8 +25,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.1V & GLM-4.6V-Flash, AutoGLM-Phone-9B model
|
||||
compatible with HuggingFace weights.
|
||||
"""
|
||||
compatible with HuggingFace weights."""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
|
||||
|
||||
@@ -22,8 +22,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 model
|
||||
compatible with HuggingFace weights.
|
||||
"""
|
||||
compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
@@ -22,8 +22,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 MTP
|
||||
model compatible with HuggingFace weights.
|
||||
"""
|
||||
model compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
@@ -462,8 +462,7 @@ class GraniteSpeechConformerConvModule(nn.Module):
|
||||
|
||||
class GraniteSpeechConformerBlock(nn.Module):
|
||||
"""Conformer block, consisting largely of linear layers,
|
||||
attention, and convolutional layers.
|
||||
"""
|
||||
attention, and convolutional layers."""
|
||||
|
||||
def __init__(self, config: PretrainedConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -892,8 +892,7 @@ def has_inner_state(
|
||||
@runtime_checkable
|
||||
class IsAttentionFree(Protocol):
|
||||
"""The interface required for all models like Mamba that lack attention,
|
||||
but do have state whose size is constant wrt the number of tokens.
|
||||
"""
|
||||
but do have state whose size is constant wrt the number of tokens."""
|
||||
|
||||
is_attention_free: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -921,8 +920,7 @@ def is_attention_free(
|
||||
class IsHybrid(Protocol):
|
||||
"""The interface required for all models like Jamba that have both
|
||||
attention and mamba blocks, indicates that
|
||||
hf_config has 'layers_block_type'
|
||||
"""
|
||||
hf_config has 'layers_block_type'"""
|
||||
|
||||
is_hybrid: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -1338,8 +1336,7 @@ class SupportsTranscription(Protocol):
|
||||
) -> "PromptType":
|
||||
"""Get the prompt for the ASR model.
|
||||
The model has control over the construction, as long as it
|
||||
returns a valid PromptType.
|
||||
"""
|
||||
returns a valid PromptType."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
@@ -1631,8 +1628,7 @@ class EagleModelMixin:
|
||||
@runtime_checkable
|
||||
class SupportsEagle(SupportsEagleBase, Protocol):
|
||||
"""The interface required for models that support
|
||||
EAGLE-1 and EAGLE-2 speculative decoding.
|
||||
"""
|
||||
EAGLE-1 and EAGLE-2 speculative decoding."""
|
||||
|
||||
supports_eagle: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -1662,8 +1658,7 @@ def supports_eagle(
|
||||
@runtime_checkable
|
||||
class SupportsEagle3(SupportsEagleBase, Protocol):
|
||||
"""The interface required for models that support
|
||||
EAGLE-3 speculative decoding.
|
||||
"""
|
||||
EAGLE-3 speculative decoding."""
|
||||
|
||||
supports_eagle3: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
|
||||
@@ -345,8 +345,7 @@ def apply_rotary_pos_emb_flashatt(
|
||||
|
||||
class KeyeSiglipAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You
|
||||
Need' paper.
|
||||
"""
|
||||
Need' paper."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Implementation of Siglip2VisionModel intended to be only used
|
||||
within a vision language model.
|
||||
"""
|
||||
within a vision language model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
@@ -51,8 +51,7 @@ class Medusa(nn.Module):
|
||||
truncated_vocab_size < vocab_size. To use this technique, one has to find
|
||||
the top-k most frequent tokens in target dataset and add that as a tensor
|
||||
in the draft checkpoint (using key token_map). Also, the draft config
|
||||
needs to have truncated_vocab_size (=k) as an attribute.
|
||||
"""
|
||||
needs to have truncated_vocab_size (=k) as an attribute."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
|
||||
@@ -858,8 +858,7 @@ class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
|
||||
|
||||
class MiniCPMV4_6DownsampleMLP(nn.Module):
|
||||
"""Match HF (transformers v5.7+) parameter naming: pre_norm/linear_1/
|
||||
act/linear_2 (instead of pre_norm + Sequential(mlp.0/mlp.2)).
|
||||
"""
|
||||
act/linear_2 (instead of pre_norm + Sequential(mlp.0/mlp.2))."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1090,8 +1090,7 @@ class Llama4ForConditionalGeneration(
|
||||
|
||||
def _rename_weight_for_modelopt_checkpoint(self, name: str) -> str:
|
||||
"""Rename weights from ModelOpt llama4 fp8 checkpoints to vLLM
|
||||
format.
|
||||
"""
|
||||
format."""
|
||||
if name.startswith("model.") or name.startswith("language_model.model."):
|
||||
renamed = (
|
||||
name.replace("model.", "language_model.model.", 1)
|
||||
|
||||
@@ -114,8 +114,7 @@ logger = init_logger(__name__)
|
||||
def _text_config(config):
|
||||
"""MuseGlimmer checkpoints may nest the text config under ``text_config``
|
||||
(multimodal ``MuseGlimmerConfig``) or expose it directly
|
||||
(``MuseGlimmerTextConfig``).
|
||||
"""
|
||||
(``MuseGlimmerTextConfig``)."""
|
||||
return getattr(config, "text_config", config)
|
||||
|
||||
|
||||
@@ -489,8 +488,7 @@ class MuseGlimmerMultiModalProcessor(
|
||||
def _muse_glimmer_use_qk_norm(config) -> bool:
|
||||
"""Whether QK-norm is applied. MuseGlimmer ALWAYS applies QK-norm; the modular HF
|
||||
``text_config`` schema simply omits ``use_qk_norm`` (reads as ``None``).
|
||||
Treat a missing/None flag as True — only an explicit ``False`` disables it.
|
||||
"""
|
||||
Treat a missing/None flag as True — only an explicit ``False`` disables it."""
|
||||
val = getattr(config, "use_qk_norm", None)
|
||||
return True if val is None else bool(val)
|
||||
|
||||
@@ -498,8 +496,7 @@ def _muse_glimmer_use_qk_norm(config) -> bool:
|
||||
def _muse_glimmer_use_attn_output_gate(config) -> bool:
|
||||
"""Whether the per-head sigmoid attention output gate is applied. MuseGlimmer ALWAYS
|
||||
applies it; the modular HF ``text_config`` omits ``use_attn_output_gate``
|
||||
(reads as ``None``). Missing/None -> True; only explicit ``False`` disables.
|
||||
"""
|
||||
(reads as ``None``). Missing/None -> True; only explicit ``False`` disables."""
|
||||
val = getattr(config, "use_attn_output_gate", None)
|
||||
return True if val is None else bool(val)
|
||||
|
||||
|
||||
@@ -1188,8 +1188,7 @@ class NemotronH_Nano_VL_V2(
|
||||
self, video_input: NanoNemotronVLVideoPixelInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Process video input and create final embeddings with video content
|
||||
and indicator tokens.
|
||||
"""
|
||||
and indicator tokens."""
|
||||
T = self.video_temporal_patch_size
|
||||
|
||||
if T > 1:
|
||||
|
||||
@@ -1462,8 +1462,7 @@ class NemoConvSubsampling(torch.nn.Module):
|
||||
|
||||
def conv_split_by_channel(self, x: Tensor) -> Tensor:
|
||||
"""For dw convs, tries to split input by time, run conv and concat
|
||||
results
|
||||
"""
|
||||
results"""
|
||||
x = self.conv[0](x) # full conv2D
|
||||
x = self.conv[1](x) # activation
|
||||
|
||||
|
||||
@@ -69,8 +69,7 @@ def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
|
||||
|
||||
def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
|
||||
"""Whether the draft needs a non-causal-capable backend, resolved from config
|
||||
(config mirror of the model's ``get_draft_attn_causal``, usable pre-build).
|
||||
"""
|
||||
(config mirror of the model's ``get_draft_attn_causal``, usable pre-build)."""
|
||||
return not all(
|
||||
_dflash_layer_causal(config, i) for i in range(config.num_hidden_layers)
|
||||
)
|
||||
@@ -155,8 +154,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
|
||||
Context KVs are pre-inserted into the KV cache before the forward pass.
|
||||
This layer handles only query tokens via standard attention.
|
||||
Adapted from Qwen3Attention.
|
||||
"""
|
||||
Adapted from Qwen3Attention."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -250,8 +248,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
"""DFlash attention assumes that the KV cache is already populated
|
||||
with the context K/V from the target model's hidden states. This forward op
|
||||
computes attention for the query tokens only.
|
||||
See also: precompute_and_store_context_kv
|
||||
"""
|
||||
See also: precompute_and_store_context_kv"""
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
@@ -743,8 +740,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
|
||||
def get_draft_attn_causal(self) -> list[bool]:
|
||||
"""Per-layer attention causality, aligned with
|
||||
get_draft_kv_cache_layer_names.
|
||||
"""
|
||||
get_draft_kv_cache_layer_names."""
|
||||
return [layer.self_attn.causal for layer in self.model.layers]
|
||||
|
||||
def compute_logits(
|
||||
|
||||
@@ -1321,8 +1321,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
|
||||
"""Transformers>=5.10 processors override `replace_video_token`
|
||||
to expand only the bare video token, keeping the prompt's outer
|
||||
`<|vision_start|>`/`<|vision_end|>` markers.
|
||||
"""
|
||||
`<|vision_start|>`/`<|vision_end|>` markers."""
|
||||
mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
|
||||
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
||||
return proc_impl is not None and proc_impl is not mixin_impl
|
||||
|
||||
@@ -599,8 +599,7 @@ class RadioInternVisionModel(nn.Module):
|
||||
self, imgs_sizes: list[tuple[int, int]], device: torch.device
|
||||
) -> MaskMetadata:
|
||||
"""Build mask metadata from image pixel sizes. Adds num_skip to each
|
||||
sequence length (cls/register tokens) to match patch generator output.
|
||||
"""
|
||||
sequence length (cls/register tokens) to match patch generator output."""
|
||||
patch_size = self.patch_generator.patch_size
|
||||
num_skip = self.patch_generator.num_skip
|
||||
|
||||
@@ -613,8 +612,7 @@ class RadioInternVisionModel(nn.Module):
|
||||
) -> MaskMetadata:
|
||||
"""Build mask metadata from actual sequence lengths (already including
|
||||
cls/register tokens, i.e. patch_count + num_skip per item).
|
||||
Use inter_image_mask_metadata() when you only have imgs_sizes.
|
||||
"""
|
||||
Use inter_image_mask_metadata() when you only have imgs_sizes."""
|
||||
assert len(seq_lens) > 0
|
||||
cu_seqlens = torch.tensor(
|
||||
list(accumulate(seq_lens, initial=0)), dtype=torch.int32, device=device
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Implementation of SiglipVisionModel intended to be only used
|
||||
within a vision language model.
|
||||
"""
|
||||
within a vision language model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
@@ -20,8 +20,7 @@
|
||||
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
|
||||
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
|
||||
"""Inference-only StableLM (https://github.com/Stability-AI/StableLM)
|
||||
model compatible with HuggingFace weights.
|
||||
"""
|
||||
model compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
@@ -207,8 +207,7 @@ class Base(
|
||||
def _mark_model_components(self, vllm_config: "VllmConfig"):
|
||||
"""Mark language model and tower submodules as `self.model` is created.
|
||||
|
||||
Nothing to do in `Base`, `MultiModalMixin` will override.
|
||||
"""
|
||||
Nothing to do in `Base`, `MultiModalMixin` will override."""
|
||||
yield
|
||||
|
||||
@cached_property
|
||||
@@ -281,8 +280,7 @@ class Base(
|
||||
|
||||
def _decorate_for_torch_compile(self):
|
||||
"""Decorate the model's decoder class to indicate to vLLM that it
|
||||
supports torch compile if `can_enable_torch_compile` is True.
|
||||
"""
|
||||
supports torch compile if `can_enable_torch_compile` is True."""
|
||||
self._decorate_cls_for_torch_compile(
|
||||
cls=self._pre_trained_model_classes.decoder,
|
||||
# Applied to a PreTrainedModel so the batch dimension will exist
|
||||
|
||||
@@ -55,20 +55,17 @@ class BaseFuser(ABC):
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Apply the fusion to an already-validated `module`, returning the
|
||||
module to install in its place (mutated in place, or freshly built).
|
||||
"""
|
||||
module to install in its place (mutated in place, or freshly built)."""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
|
||||
(none unless it stacks weights).
|
||||
"""
|
||||
(none unless it stacks weights)."""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`packed_modules_mapping` entries this fuser contributes (none unless
|
||||
it stacks weights).
|
||||
"""
|
||||
it stacks weights)."""
|
||||
return {}
|
||||
|
||||
|
||||
@@ -122,8 +119,7 @@ class RewriteFuser(BaseFuser):
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward.
|
||||
"""
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
@@ -132,8 +128,7 @@ class RewriteFuser(BaseFuser):
|
||||
@dataclass
|
||||
class StackedFuser(RewriteFuser):
|
||||
"""A fuser that merges sibling projections into one stacked linear and
|
||||
rewrites the forward to call it.
|
||||
"""
|
||||
rewrites the forward to call it."""
|
||||
|
||||
merged_name: ClassVar[str]
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
@@ -152,16 +147,14 @@ class StackedFuser(RewriteFuser):
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
"""Each projection's original name and its shard id in the merged module.
|
||||
|
||||
Source for both `orig_to_new_stacked` and `packed_modules_mapping`.
|
||||
"""
|
||||
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
|
||||
|
||||
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
|
||||
so only this exact layer is remapped, never a same-named projection
|
||||
elsewhere (e.g. an unfused MoE expert's `gate_proj`).
|
||||
"""
|
||||
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
|
||||
merged = maybe_prefix(prefix, self.merged_name)
|
||||
return {
|
||||
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
|
||||
@@ -170,8 +163,7 @@ class StackedFuser(RewriteFuser):
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`{merged_name: [projection names]}` so quantization can unpack the
|
||||
fused layer into its per-shard configs.
|
||||
"""
|
||||
fused layer into its per-shard configs."""
|
||||
return {self.merged_name: [name for name, _ in self.shards]}
|
||||
|
||||
def _unguarded_calls(
|
||||
@@ -182,8 +174,7 @@ class StackedFuser(RewriteFuser):
|
||||
`update_attrs` deletes the projections it merges, so any reference to one beyond
|
||||
its call site must be a guard on its existence; folding those to their constant
|
||||
value keeps the rewritten forward off a name that no longer exists. A reference
|
||||
that is not such a guard raises, so fusion is skipped.
|
||||
"""
|
||||
that is not such a guard raises, so fusion is skipped."""
|
||||
calls = []
|
||||
for name in names:
|
||||
call, refs = self_call_and_refs(funcdef, name)
|
||||
|
||||
@@ -232,8 +232,7 @@ class MLAFuser(StackedFuser):
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
|
||||
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
|
||||
attention interface unexpanded.
|
||||
"""
|
||||
attention interface unexpanded."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
if self.has_q_lora:
|
||||
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
|
||||
|
||||
@@ -80,8 +80,7 @@ def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
|
||||
|
||||
Computations that must run in higher precision say so in their forward (e.g.
|
||||
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
|
||||
even when the config does not name it.
|
||||
"""
|
||||
even when the config does not name it."""
|
||||
for node in nodes:
|
||||
if node.op not in ("call_method", "call_function"):
|
||||
continue
|
||||
@@ -126,8 +125,7 @@ class SharedExpertMLP(nn.Module):
|
||||
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Standard MoE block forward.
|
||||
|
||||
Routing and any shared experts are handled inside `self.experts: MoERunner`.
|
||||
"""
|
||||
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
|
||||
num_tokens = hidden_states.shape[0]
|
||||
@@ -154,8 +152,7 @@ class MoEBlockFuser:
|
||||
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function and the dtype the router computes in.
|
||||
"""
|
||||
Returns the scoring function and the dtype the router computes in."""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
@@ -283,8 +280,7 @@ class MoEBlockFuser:
|
||||
self, moe_block: nn.Module, prefix: str
|
||||
) -> SharedExpertMLP | None:
|
||||
"""Build the HF shared expert (and its optional gate)
|
||||
as a `SharedExpertMLP` for vLLM's fused MoE.
|
||||
"""
|
||||
as a `SharedExpertMLP` for vLLM's fused MoE."""
|
||||
if self.shared_name is None:
|
||||
return None
|
||||
shared_experts = getattr(moe_block, self.shared_name)
|
||||
|
||||
@@ -43,8 +43,7 @@ def _reference_weight(module: nn.Module) -> torch.Tensor | None:
|
||||
"""A weight whose trailing dim is the module's hidden size.
|
||||
|
||||
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype.
|
||||
"""
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
|
||||
for child in module.modules():
|
||||
if isinstance(child, nn.Linear):
|
||||
return child.weight
|
||||
@@ -59,8 +58,7 @@ class _MetaProxy(fx.Proxy):
|
||||
|
||||
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
|
||||
executing each op on the meta values, so PyTorch's meta kernels are the
|
||||
single source of shape inference — no per-op rules.
|
||||
"""
|
||||
single source of shape inference — no per-op rules."""
|
||||
|
||||
meta: object = _UNKNOWN
|
||||
|
||||
@@ -84,8 +82,7 @@ class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
|
||||
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
|
||||
|
||||
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too.
|
||||
"""
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too."""
|
||||
|
||||
def __init__(self, root: fx.Proxy, attr: str):
|
||||
super().__init__(root, attr)
|
||||
@@ -221,8 +218,7 @@ def _leaf_attention_interfaces():
|
||||
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
|
||||
|
||||
`vllm_attention_function` needs runtime context so it is untraceable.
|
||||
Every interface returns `(attn_output, attn_weights)`.
|
||||
"""
|
||||
Every interface returns `(attn_output, attn_weights)`."""
|
||||
from transformers.modeling_utils import AttentionInterface
|
||||
|
||||
original = AttentionInterface.get_interface
|
||||
@@ -300,8 +296,7 @@ def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
|
||||
def forward_input_count(cls: type[nn.Module]) -> int:
|
||||
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
|
||||
any `*args`/`**kwargs`. Read from the signature, so it is independent of
|
||||
whether the trace completes (unlike counting placeholders).
|
||||
"""
|
||||
whether the trace completes (unlike counting placeholders)."""
|
||||
params = list(forward_parameters(cls).values())
|
||||
if not params:
|
||||
return 1 # uninspectable: assume a single input and let matching decide
|
||||
@@ -378,8 +373,7 @@ def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
|
||||
|
||||
In these positions the object's identity never escapes, so a reference that
|
||||
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
|
||||
yield an operand, so the module could escape (`x and self.<name>`).
|
||||
"""
|
||||
yield an operand, so the module could escape (`x and self.<name>`)."""
|
||||
for node in ast.walk(funcdef):
|
||||
if (
|
||||
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
|
||||
@@ -412,8 +406,7 @@ def bypass_existence_guard(
|
||||
`not self.<name>`): the reference itself to `True`.
|
||||
|
||||
Any other surviving reference escapes the projection's value, which no longer
|
||||
exists after fusion, so refuse rather than change semantics.
|
||||
"""
|
||||
exists after fusion, so refuse rather than change semantics."""
|
||||
for node in ast.walk(funcdef):
|
||||
# `self.<name> is (not) None`, either operand order.
|
||||
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
|
||||
@@ -474,8 +467,7 @@ def _base_name(node: ast.expr) -> str | None:
|
||||
def _rebound_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names rebound outright within `region` (`x = ...`, `del x`).
|
||||
|
||||
These are `Name` nodes in a `Store`/`Del` context.
|
||||
"""
|
||||
These are `Name` nodes in a `Store`/`Del` context."""
|
||||
return {
|
||||
node.id
|
||||
for stmt in region
|
||||
@@ -489,8 +481,7 @@ def _inplace_target(node: ast.AST) -> str | None:
|
||||
|
||||
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
|
||||
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
|
||||
plain `Name` store (see `_rebound_names`).
|
||||
"""
|
||||
plain `Name` store (see `_rebound_names`)."""
|
||||
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
|
||||
node.ctx, (ast.Store, ast.Del)
|
||||
):
|
||||
@@ -711,8 +702,7 @@ def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
|
||||
Non-linear submodules are transparent too (e.g. the dropout GPT-style
|
||||
attentions apply after their output projection). Never walks through a leaf
|
||||
call (e.g. an attention interface): its inputs are what attention consumes,
|
||||
not what produced the value.
|
||||
"""
|
||||
not what produced the value."""
|
||||
stack = [node]
|
||||
seen: set[fx.Node] = set()
|
||||
while stack:
|
||||
@@ -735,8 +725,7 @@ def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
|
||||
|
||||
Never walks through a leaf call (e.g. an attention interface): what crosses
|
||||
it is consumed by the attention computation, not projected.
|
||||
"""
|
||||
it is consumed by the attention computation, not projected."""
|
||||
queue = list(node.users)
|
||||
seen: set[fx.Node] = set()
|
||||
while queue:
|
||||
|
||||
@@ -22,8 +22,7 @@ _VLLM_PKG = "vllm.model_executor.layers"
|
||||
|
||||
def _resolve(module: str, name: str):
|
||||
"""Return `name` from the hw-agnostic `module` when enabled and available,
|
||||
else from vLLM. Logs which source was used.
|
||||
"""
|
||||
else from vLLM. Logs which source was used."""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
|
||||
|
||||
@@ -75,8 +75,7 @@ class TransformersMoERunner(MoERunner):
|
||||
) -> torch.Tensor:
|
||||
"""In Transformers `experts.forward` will have this signature.
|
||||
|
||||
We discard any extra kwargs because we cannot use them here.
|
||||
"""
|
||||
We discard any extra kwargs because we cannot use them here."""
|
||||
# Note: we need to forward through a custom op so the topk_ids
|
||||
# can be transferred without interfering with cudagraphs.
|
||||
return torch.ops.vllm.transformers_moe_forward(
|
||||
@@ -368,8 +367,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier.
|
||||
"""
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
|
||||
@@ -424,8 +424,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Empty, because `apply` writes the placeholder ranges itself rather than
|
||||
deriving them from updates.
|
||||
"""
|
||||
deriving them from updates."""
|
||||
return []
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
@@ -459,8 +458,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
num_audios: int,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Take each contiguous run of the audio token as one item's placeholder,
|
||||
and record how many tokens the run holds.
|
||||
"""
|
||||
and record how many tokens the run holds."""
|
||||
audio_token_ids = self._get_mm_token_ids("audio")
|
||||
prompt_tensor = torch.tensor(prompt_ids)
|
||||
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
|
||||
@@ -518,8 +516,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
mm_token_type_ids: torch.Tensor | None,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Split the positions the processor marks as image into one placeholder per
|
||||
item, sized by the token count it reports for each image.
|
||||
"""
|
||||
item, sized by the token count it reports for each image."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if mm_token_type_ids is None:
|
||||
raise ValueError(
|
||||
@@ -572,8 +569,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
"""Process the prompt and every multi-modal item in one HF processor call,
|
||||
then read the placeholder ranges out of the token ids it returns.
|
||||
"""
|
||||
then read the placeholder ranges out of the token ids it returns."""
|
||||
prompt = inputs.prompt
|
||||
mm_items = inputs.mm_data_items
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
@@ -711,8 +707,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Replace each modality's placeholder token with the token ids that item's
|
||||
replacement text encodes to, marking which of them hold embeddings.
|
||||
"""
|
||||
replacement text encodes to, marking which of them hold embeddings."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
@@ -907,8 +902,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
self, model: "PreTrainedModel"
|
||||
) -> dict[str, type["PreTrainedModel"]]:
|
||||
"""Modalities whose encoder cannot be told apart from the model itself are
|
||||
omitted, as are those `get_encoder` rejects.
|
||||
"""
|
||||
omitted, as are those `get_encoder` rejects."""
|
||||
encoder_classes: dict[str, type[PreTrainedModel]] = {}
|
||||
for modality in _MODALITY_TO_TOKEN_TYPE_ID:
|
||||
try:
|
||||
@@ -1027,8 +1021,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
"""Transformers modeling backend multimodal classes do not contain a separate
|
||||
vLLM language model class. Therefore, in order to return a language model vLLM
|
||||
class, we use a wrapper to give `self` the same interface as a text model.
|
||||
"""
|
||||
class, we use a wrapper to give `self` the same interface as a text model."""
|
||||
# Exclude self and object
|
||||
bases = self.__class__.mro()[1:-1]
|
||||
# Keep only classes defined in `vllm.model_executor.models.transformers`
|
||||
@@ -1182,8 +1175,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
) -> dict[str, Any]:
|
||||
"""Narrow the entries of `kwargs` that hold one row per item down to the item
|
||||
at `index`. Length is all there is to match on, so an unrelated entry of the
|
||||
same length is narrowed too.
|
||||
"""
|
||||
same length is narrowed too."""
|
||||
return {
|
||||
key: value[index : index + 1]
|
||||
if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
|
||||
|
||||
@@ -201,8 +201,7 @@ def attrsetter(attr: str) -> Callable[[object, object], None]:
|
||||
|
||||
class _UninitializedEmbedding(nn.Embedding):
|
||||
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`.
|
||||
"""
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
@@ -210,8 +209,7 @@ class _UninitializedEmbedding(nn.Embedding):
|
||||
|
||||
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
|
||||
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding.
|
||||
"""
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@@ -47,8 +47,7 @@ ShardId: TypeAlias = str | int | tuple[int, ...]
|
||||
class WeightsMapper:
|
||||
"""Maps the name of each weight if they match the following patterns.
|
||||
|
||||
If a key maps to a value of `None`, the corresponding weight is ignored.
|
||||
"""
|
||||
If a key maps to a value of `None`, the corresponding weight is ignored."""
|
||||
|
||||
orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list)
|
||||
orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict)
|
||||
@@ -172,8 +171,7 @@ class WeightsMapper:
|
||||
rather than being rewritten to the stacked vLLM name (`qkv_proj`). Mappings to
|
||||
`None` are dropped because "do not load this weight" is meaningless to such a
|
||||
consumer, and applying it would silently shrink a quantization config's ignore
|
||||
list or make LoRA name parsing fail.
|
||||
"""
|
||||
list or make LoRA name parsing fail."""
|
||||
remove_none = lambda d: {k: v for k, v in d.items() if v is not None}
|
||||
return replace(
|
||||
self,
|
||||
|
||||
@@ -116,8 +116,7 @@ def get_multimodal_config() -> MultiModalConfig | None:
|
||||
"""Return the current ``MultiModalConfig``, or ``None`` when no engine
|
||||
config context is active (e.g., during unit tests) or when the current
|
||||
``model_config`` does not carry a ``multimodal_config`` (e.g., minimal
|
||||
stubs used in tests).
|
||||
"""
|
||||
stubs used in tests)."""
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None or vllm_config.model_config is None:
|
||||
return None
|
||||
|
||||
@@ -174,8 +174,7 @@ def get_moe_expert_mapping(
|
||||
|
||||
It will be retrieved from the first module that has a `get_expert_mapping` method.
|
||||
If the model manually implements `get_expert_mapping`, it will be used.
|
||||
Otherwise, it will use the first RoutedExperts layer.
|
||||
"""
|
||||
Otherwise, it will use the first RoutedExperts layer."""
|
||||
for _, module in model.named_modules():
|
||||
get_mapping = getattr(module, "get_expert_mapping", None)
|
||||
if get_mapping is not None:
|
||||
|
||||
@@ -18,8 +18,7 @@ logger = init_logger(__name__)
|
||||
|
||||
def qwen4_exp_qsa_triton_warmup(worker: "Worker") -> None:
|
||||
"""Warm every reachable QSA specialization: indexer decode-query-length
|
||||
profiles plus the sparse attention split-K/merge configs.
|
||||
"""
|
||||
profiles plus the sparse attention split-K/merge configs."""
|
||||
qsa_module = sys.modules.get("vllm.models.qwen4_exp.nvidia.indexer_qsa")
|
||||
attn_module = sys.modules.get("vllm.models.qwen4_exp.nvidia.qsa")
|
||||
if qsa_module is None or attn_module is None:
|
||||
|
||||
Reference in New Issue
Block a user