[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -804,8 +804,7 @@ _ACTIVATION_REGISTRY = LazyDict(
|
||||
|
||||
def _get_gelu_pytorch_tanh() -> nn.Module:
|
||||
"""Get PyTorch GELU with tanh approximation, with ROCm fallback
|
||||
and fast GELU for ARM.
|
||||
"""
|
||||
and fast GELU for ARM."""
|
||||
if current_platform.is_rocm():
|
||||
# TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile
|
||||
logger.warning_once(
|
||||
|
||||
@@ -1568,8 +1568,7 @@ class MLACommonBackend(AttentionBackend):
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Per-token-head modes pack an inline fp32 scale pair after the
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA).
|
||||
"""
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
|
||||
spec = super().customize_spec(spec)
|
||||
mode = spec.kv_quant_mode
|
||||
if spec.state_content_bytes is not None or not mode.is_per_token_head:
|
||||
|
||||
@@ -55,8 +55,7 @@ _FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
|
||||
"""Workspace token budget for flashinfer fused all-reduce, or None if the
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``.
|
||||
"""
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
|
||||
|
||||
@@ -170,8 +170,7 @@ def _fused_embed_eh_norm_kernel(
|
||||
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
|
||||
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
|
||||
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
|
||||
cat. Requires the full table on-rank (replicated embedding).
|
||||
"""
|
||||
cat. Requires the full table on-rank (replicated embedding)."""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
@@ -56,8 +56,7 @@ class MoEActivation(Enum):
|
||||
@property
|
||||
def custom_op_name(self) -> str:
|
||||
"""Maps to the CustomOp name of activations
|
||||
in vllm/model_executor/layers/activation.py.
|
||||
"""
|
||||
in vllm/model_executor/layers/activation.py."""
|
||||
return _CUSTOM_OP_NAMES[self]
|
||||
|
||||
def without_mul(self) -> "MoEActivation":
|
||||
|
||||
@@ -36,8 +36,7 @@ def is_aiter_mxfp8_moe_available() -> bool:
|
||||
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
|
||||
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
|
||||
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
|
||||
closed (returns False -> triton dot_scaled), which is always safe.
|
||||
"""
|
||||
closed (returns False -> triton dot_scaled), which is always safe."""
|
||||
if not (current_platform.is_rocm() and current_platform.supports_mx()):
|
||||
return False
|
||||
try:
|
||||
|
||||
@@ -192,8 +192,7 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
def _pad_moe_intermediate(self, layer: torch.nn.Module) -> None:
|
||||
"""Zero-pad the per-partition MoE intermediate dim of both weights and
|
||||
the expert bias, see `_padded_intermediate_size`.
|
||||
"""
|
||||
the expert bias, see `_padded_intermediate_size`."""
|
||||
intermediate_size = self.moe_config.intermediate_size_per_partition
|
||||
padded_size = self._padded_intermediate_size(self.moe_config)
|
||||
if padded_size == intermediate_size:
|
||||
@@ -953,8 +952,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular):
|
||||
|
||||
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
|
||||
modular MoE experts.
|
||||
"""
|
||||
modular MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1287,8 +1285,7 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
|
||||
class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
|
||||
quantization, dispatched through zentorch.
|
||||
"""
|
||||
quantization, dispatched through zentorch."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -34,8 +34,7 @@ from vllm.triton_utils import tl, triton
|
||||
|
||||
def _select_cfg(M, N, K, block_m):
|
||||
"""Pick the launch config from host constants only (M=num_valid_tokens, N, K,
|
||||
block_m) — graph-capture safe (no GPU-scalar branch).
|
||||
"""
|
||||
block_m) — graph-capture safe (no GPU-scalar branch)."""
|
||||
# Per-regime winners (measured, isolated cuda-event A/B on gfx950, GPU 3):
|
||||
# BLOCK_K=256 (fewer K-iters + bigger MX scale-load coalesced with the dot),
|
||||
# num_stages=2 (software-pipeline overlaps the E8M0 scale-load with the scaled
|
||||
|
||||
@@ -55,28 +55,24 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
|
||||
@abstractmethod
|
||||
def backend_enum_cls(self) -> type[BackendT]:
|
||||
"""Return the concrete `Enum` class enumerating this oracle's
|
||||
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`).
|
||||
"""
|
||||
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`)."""
|
||||
|
||||
@abstractmethod
|
||||
def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]:
|
||||
"""Return platform-appropriate backends in priority order for
|
||||
this `moe_config`.
|
||||
"""
|
||||
this `moe_config`."""
|
||||
|
||||
@abstractmethod
|
||||
def backend_to_kernel_cls(
|
||||
self, backend: BackendT
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
"""Map a backend enum value to its concrete `FusedMoEExperts`
|
||||
subclasses, in selection priority order.
|
||||
"""
|
||||
subclasses, in selection priority order."""
|
||||
|
||||
@abstractmethod
|
||||
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
|
||||
"""Map a user-facing `MoEBackend` (from the runner config) to
|
||||
this oracle's enum.
|
||||
"""
|
||||
this oracle's enum."""
|
||||
|
||||
@abstractmethod
|
||||
def select_backend(
|
||||
@@ -108,8 +104,7 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Construct the `FusedMoEKernel` (Prepare/Finalize + Experts
|
||||
combinator) for the chosen backend.
|
||||
"""
|
||||
combinator) for the chosen backend."""
|
||||
|
||||
def convert_to_kernel_format(
|
||||
self,
|
||||
|
||||
@@ -451,8 +451,7 @@ def _humming_fp8_weight_schema(
|
||||
layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
|
||||
) -> dict[str, Any]:
|
||||
"""Build the humming weight schema from the canonical on-device fp8/mxfp8
|
||||
tensors (scale dtype/shape, block size), not the producing quant method.
|
||||
"""
|
||||
tensors (scale dtype/shape, block size), not the producing quant method."""
|
||||
# mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
|
||||
# no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
|
||||
if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
|
||||
|
||||
@@ -224,8 +224,7 @@ def _humming_int8_weight_schema(
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor
|
||||
) -> dict[str, Any]:
|
||||
"""Build the humming compressed-tensors int8 schema from the canonical
|
||||
on-device tensors; humming does the signed-int8 -> native conversion.
|
||||
"""
|
||||
on-device tensors; humming does the signed-int8 -> native conversion."""
|
||||
config: dict[str, Any] = {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "int-quantized",
|
||||
|
||||
@@ -512,8 +512,7 @@ def _process_weights_flashinfer(
|
||||
|
||||
def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor:
|
||||
"""Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)``
|
||||
tensor along its last dim, from ``unit`` to ``padded_unit`` columns.
|
||||
"""
|
||||
tensor along its last dim, from ``unit`` to ``padded_unit`` columns."""
|
||||
if padded_unit == unit:
|
||||
return x
|
||||
e, rows, _ = x.shape
|
||||
@@ -1081,8 +1080,7 @@ def _humming_wna16_weight_schema(
|
||||
quant_config: QuantizationConfig | QuantizationArgs | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Humming weight schema for a WNA16 checkpoint, derived from the quant
|
||||
config rather than the running kernel.
|
||||
"""
|
||||
config rather than the running kernel."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
|
||||
|
||||
@@ -331,8 +331,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
|
||||
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
"""Available backends in priority order, BF16-act variant before
|
||||
activation-quantized variant within each vendor family.
|
||||
"""
|
||||
activation-quantized variant within each vendor family."""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
@@ -411,8 +410,7 @@ def _resolve_activation_key(
|
||||
model_activation_key: QuantKey | None,
|
||||
) -> QuantKey | None:
|
||||
"""Combine the model-supplied activation key with the user override.
|
||||
Raises on conflict (both set and disagreeing).
|
||||
"""
|
||||
Raises on conflict (both set and disagreeing)."""
|
||||
user_override = _user_moe_activation_override()
|
||||
if user_override is None:
|
||||
return model_activation_key
|
||||
@@ -462,8 +460,7 @@ def _filter_by_activation(
|
||||
) -> list[Mxfp4MoeBackend]:
|
||||
"""Pick variants matching ``requested_activation_key``; without one,
|
||||
prefer BF16 if the list has any, else keep the list as-is so explicit
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land.
|
||||
"""
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
|
||||
if requested_activation_key is not None:
|
||||
return [
|
||||
b
|
||||
|
||||
@@ -71,8 +71,7 @@ class ZeroExpertRouter(BaseRouter):
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute routing with full bias, compute zero expert output,
|
||||
mask zero expert IDs.
|
||||
"""
|
||||
mask zero expert IDs."""
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
|
||||
@@ -163,8 +163,7 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
) -> str:
|
||||
"""Resolve the decode top-k implementation from the configured
|
||||
backend ("auto" = the pre-existing chain, or a validated explicit
|
||||
value).
|
||||
"""
|
||||
value)."""
|
||||
if self._backend == "auto":
|
||||
return self._resolve_auto(logits, topk_tokens, num_rows)
|
||||
|
||||
@@ -211,8 +210,7 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""The pre-existing priority chain: cooperative -> persistent ->
|
||||
per_row. deep_select/flashinfer/torch are opt-in only.
|
||||
"""
|
||||
per_row. deep_select/flashinfer/torch are opt-in only."""
|
||||
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
|
||||
return "cooperative"
|
||||
if self._is_cuda and topk_tokens in (512, 1024, 2048):
|
||||
@@ -274,8 +272,7 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
"""Run the resolved decode top-k implementation, writing into
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
|
||||
"""
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
|
||||
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
|
||||
if backend == "deep_select":
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
|
||||
@@ -158,8 +158,7 @@ class LinearMethodBase(QuantizeMethodBase):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the weights in layer to the input tensor.
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
|
||||
@@ -861,8 +861,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""ROCm forward using AITER Triton fused projection+attention when
|
||||
available, otherwise falling back to the generic CUDA path.
|
||||
"""
|
||||
available, otherwise falling back to the generic CUDA path."""
|
||||
if GDN_AITER_TRITON_AVAILABLE:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
|
||||
@@ -261,8 +261,7 @@ class MambaStateShapeCalculator:
|
||||
@classmethod
|
||||
def extra_groups_for_head_shards(cls, ngroups: int, tp_size: int):
|
||||
"""Compute the increase in group numbers to account for
|
||||
replication in order to accompany the head shards.
|
||||
"""
|
||||
replication in order to accompany the head shards."""
|
||||
# in the case ngoups % tp_size == 0, this will be zero
|
||||
if ngroups % tp_size == 0:
|
||||
return 0
|
||||
|
||||
@@ -59,8 +59,7 @@ def get_ssm_device_name() -> str:
|
||||
|
||||
def _canonical_cache_dtype(cache_dtype: str) -> str:
|
||||
"""Canonical key for config lookup. bf16 and fp16 share the same tuned
|
||||
configs because the kernel only sees bit width when accessing state.
|
||||
"""
|
||||
configs because the kernel only sees bit width when accessing state."""
|
||||
return "float16" if cache_dtype == "bfloat16" else cache_dtype
|
||||
|
||||
|
||||
@@ -161,8 +160,7 @@ _ssm_config_override: tuple[int, int] | None = None
|
||||
@contextmanager
|
||||
def override_ssm_config(config: tuple[int, int]):
|
||||
"""Pin ``try_get_optimal_ssm_config`` to ``config`` for the duration of
|
||||
the context. Used by the tuning benchmark to time specific configs.
|
||||
"""
|
||||
the context. Used by the tuning benchmark to time specific configs."""
|
||||
global _ssm_config_override
|
||||
prev = _ssm_config_override
|
||||
_ssm_config_override = config
|
||||
|
||||
@@ -210,8 +210,7 @@ class LamportWorkspace:
|
||||
@property
|
||||
def workspace(self) -> torch.Tensor:
|
||||
"""Device tensor (int64) that can be passed to the kernel
|
||||
as ``void** workspace``.
|
||||
"""
|
||||
as ``void** workspace``."""
|
||||
return self._workspace
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -46,24 +46,21 @@ class QuantizeMethodBase(ABC):
|
||||
):
|
||||
"""Create weights for a layer.
|
||||
|
||||
The weights will be set as attributes of the layer.
|
||||
"""
|
||||
The weights will be set as attributes of the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
|
||||
"""Apply the weights in layer to the input tensor.
|
||||
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
# Not required functions
|
||||
def embedding(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
|
||||
"""Gather embeddings in the layer based on indices in the input tensor.
|
||||
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
# Not required functions
|
||||
@@ -76,8 +73,7 @@ class QuantizeMethodBase(ABC):
|
||||
Quantization methods that need special weight handling (e.g. repacked
|
||||
weights) override this.
|
||||
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
layer.weight = embed_tokens.weight
|
||||
return layer
|
||||
|
||||
|
||||
@@ -665,8 +665,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
) -> bool:
|
||||
"""Weight N-bit INT (pack-quantized for sub-byte, int-quantized for 8-bit)
|
||||
with static per-tensor INT8 input/output activation quant, applied as a float
|
||||
fake-quant around a weight-only matmul.
|
||||
"""
|
||||
fake-quant around a weight-only matmul."""
|
||||
is_int_pack_format = format in (
|
||||
CompressionFormat.pack_quantized.value,
|
||||
CompressionFormat.int_quantized.value,
|
||||
@@ -702,8 +701,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
format: str | None,
|
||||
) -> bool:
|
||||
"""Weight N-bit INT with symmetric dynamic INT activation quant
|
||||
via Humming kernel.
|
||||
"""
|
||||
via Humming kernel."""
|
||||
if input_quant is None:
|
||||
return False
|
||||
is_pack_format = format == CompressionFormat.pack_quantized.value
|
||||
|
||||
+1
-2
@@ -37,8 +37,7 @@ def _dequant_gather_kernel(
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Gather embedding rows by token id, unpack int32-packed INT weights, and
|
||||
dequantize to ``out`` dtype in one pass (no int8 intermediate).
|
||||
"""
|
||||
dequantize to ``out`` dtype in one pass (no int8 intermediate)."""
|
||||
row = tl.program_id(0)
|
||||
col = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
col_mask = col < hidden
|
||||
|
||||
+1
-2
@@ -211,8 +211,7 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
|
||||
|
||||
def _pack_int_quantized_weight(self, layer: torch.nn.Module) -> None:
|
||||
"""Normalize an int-quantized (plain int8) weight to the canonical
|
||||
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect.
|
||||
"""
|
||||
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect."""
|
||||
weight = layer.weight
|
||||
out_features, in_features = weight.shape
|
||||
packed = pack_to_int32(weight.data.contiguous(), self.num_bits)
|
||||
|
||||
@@ -86,8 +86,7 @@ class INCConfigParser:
|
||||
|
||||
def is_explicitly_configured(name: str) -> bool:
|
||||
"""Return True if *name* has an explicit entry in extra_config,
|
||||
either via exact key match or via a regex pattern key.
|
||||
"""
|
||||
either via exact key match or via a regex pattern key."""
|
||||
if not self._config.extra_config:
|
||||
return False
|
||||
if name in self._config.extra_config:
|
||||
|
||||
@@ -48,8 +48,7 @@ class INCScheme(ABC):
|
||||
layer_config: "INCLayerConfig",
|
||||
) -> "FusedMoEMethodBase | None":
|
||||
"""Optional. Override if this scheme supports MoE.
|
||||
Default raises NotImplementedError.
|
||||
"""
|
||||
Default raises NotImplementedError."""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support MoE layers. "
|
||||
f"Layer config: {layer_config}"
|
||||
|
||||
@@ -1988,8 +1988,7 @@ class KNvfp4Static(QuantKeyScheme):
|
||||
|
||||
class KNvfp4Dynamic(QuantKeyScheme):
|
||||
"""NVFP4 activation scheme (W4A4). Has a static global input scale on disk;
|
||||
the per-group scale is computed at runtime inside the kernel.
|
||||
"""
|
||||
the per-group scale is computed at runtime inside the kernel."""
|
||||
|
||||
key = kNvfp4Dynamic
|
||||
|
||||
@@ -2026,8 +2025,7 @@ class KNvfp4Dynamic(QuantKeyScheme):
|
||||
|
||||
class KFp8StaticTensor(QuantKeyScheme):
|
||||
"""Plain per-tensor static FP8 — bivalent: serves BOTH the weight slot and
|
||||
the activation slot (W8A8). One key in both QuantSpec slots.
|
||||
"""
|
||||
the activation slot (W8A8). One key in both QuantSpec slots."""
|
||||
|
||||
key = kFp8StaticTensorSym
|
||||
|
||||
@@ -2097,8 +2095,7 @@ class KFp8StaticTensor(QuantKeyScheme):
|
||||
|
||||
class KFp8StaticChannel(QuantKeyScheme):
|
||||
"""Per-channel static FP8 weight (the 'PcPt' weight). Weight-role only —
|
||||
there is no static per-channel *activation* today.
|
||||
"""
|
||||
there is no static per-channel *activation* today."""
|
||||
|
||||
key = kFp8StaticTokenSym
|
||||
|
||||
@@ -2139,8 +2136,7 @@ class KFp8StaticChannel(QuantKeyScheme):
|
||||
class KFp8Block128(QuantKeyScheme):
|
||||
"""128x128 block-static FP8 weight ('PbWo'). Weight-role only. ModelOpt
|
||||
exports the scale 4-D [out_blk,1,in_blk,1]; process squeezes to 2-D.
|
||||
No transpose (block kernel keeps [out,in]).
|
||||
"""
|
||||
No transpose (block kernel keeps [out,in])."""
|
||||
|
||||
key = kFp8Static128BlockSym
|
||||
|
||||
@@ -2196,8 +2192,7 @@ class KFp8Block128(QuantKeyScheme):
|
||||
|
||||
class KMxfp8Static(QuantKeyScheme):
|
||||
"""MXFP8 weight: fp8-e4m3 values + per-32-block e8m0 (uint8) scale.
|
||||
Weight-role only. process is validate-only plus an idempotency guard.
|
||||
"""
|
||||
Weight-role only. process is validate-only plus an idempotency guard."""
|
||||
|
||||
key = kMxfp8Static
|
||||
|
||||
@@ -2272,8 +2267,7 @@ class KDynamicNoParam(QuantKeyScheme):
|
||||
"""Dynamic activation with no stored scale (W8A8): quantized at runtime in
|
||||
the kernel. NOT the same as activation=None (weight-only) — init_fp8 needs a
|
||||
non-None activation key. Activation-role only. Serves the fp8 per-token, fp8
|
||||
per-block, and mxfp8 dynamic activation keys.
|
||||
"""
|
||||
per-block, and mxfp8 dynamic activation keys."""
|
||||
|
||||
def create_weights(self, layer, role, ctx, shapes, wl) -> None:
|
||||
if role is not ACT:
|
||||
|
||||
@@ -79,8 +79,7 @@ class Mxfp4Config(QuantizationConfig):
|
||||
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
"""MoE method for RoutedExperts. Subclasses override to pick a
|
||||
checkpoint-specific kernel family.
|
||||
"""
|
||||
checkpoint-specific kernel family."""
|
||||
return Mxfp4MoEMethod(moe)
|
||||
|
||||
def get_quant_method(
|
||||
|
||||
@@ -139,8 +139,7 @@ def _find_matching_targets(
|
||||
|
||||
class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""Model-level config for online quantization (quantize fp16/bf16 weights
|
||||
during model loading, without requiring a pre-quantized checkpoint).
|
||||
"""
|
||||
during model loading, without requiring a pre-quantized checkpoint)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -118,8 +118,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
|
||||
|
||||
class OnlineLinearBase(LinearMethodBase):
|
||||
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time.
|
||||
"""
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
@@ -163,8 +162,7 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
|
||||
"""
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -265,8 +263,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online blockwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading.
|
||||
"""
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -440,8 +437,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
"""Shared base for online FP8 MoE methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time.
|
||||
"""
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
# Declared here for mypy; actual values are set in __init__.
|
||||
fp8_backend: "Fp8MoeBackend"
|
||||
@@ -562,8 +558,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
|
||||
class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
"""Online tensorwise FP8 MoE quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
|
||||
"""
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -620,8 +615,7 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
|
||||
class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
"""Online blockwise FP8 MoE quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading.
|
||||
"""
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -69,8 +69,7 @@ def input_to_float8(
|
||||
x: torch.Tensor, dtype: torch.dtype | None = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This function quantizes input values to float8 values "
|
||||
"with tensor-wise quantization.
|
||||
"""
|
||||
"with tensor-wise quantization."""
|
||||
dtype = current_platform.fp8_dtype() if dtype is None else dtype
|
||||
finfo = torch.finfo(dtype)
|
||||
min_val, max_val = x.aminmax()
|
||||
@@ -1195,8 +1194,7 @@ def prepare_fp8_moe_layer_for_deepgemm(
|
||||
|
||||
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Pad the weight tensor. This is an optimization on ROCm platform, which
|
||||
can benefit from tensors located far enough from one another in memory
|
||||
"""
|
||||
can benefit from tensors located far enough from one another in memory"""
|
||||
if (
|
||||
envs.VLLM_ROCM_FP8_PADDING
|
||||
and current_platform.is_rocm()
|
||||
|
||||
@@ -297,8 +297,7 @@ def _humming_input_schema_to_quant_key(
|
||||
schema: "HummingInputSchema",
|
||||
) -> QuantKey | None:
|
||||
"""Convert a HummingInputSchema to a QuantKey. Returns None if
|
||||
the schema represents unquantized (bf16/fp16) inputs.
|
||||
"""
|
||||
the schema represents unquantized (bf16/fp16) inputs."""
|
||||
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
|
||||
return None
|
||||
|
||||
@@ -427,8 +426,7 @@ def convert_linear_layer_to_humming_standard(
|
||||
):
|
||||
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
|
||||
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``).
|
||||
"""
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
|
||||
for name, checkpoint_name in name_map.items():
|
||||
tensor = getattr(layer, checkpoint_name)
|
||||
delattr(layer, checkpoint_name)
|
||||
|
||||
@@ -245,8 +245,7 @@ def marlin_padded_nk(size_n: int, size_k: int, group_size: int = -1) -> tuple[in
|
||||
|
||||
def marlin_repacked_nk(qweight: torch.Tensor, num_bits: int) -> tuple[int, int]:
|
||||
"""Recover the (size_n, size_k) a Marlin weight was repacked with
|
||||
(including any tile padding) from its packed shape.
|
||||
"""
|
||||
(including any tile padding) from its packed shape."""
|
||||
pack_factor = 32 // num_bits
|
||||
size_k = qweight.size(0) * GPTQ_MARLIN_TILE
|
||||
size_n = qweight.size(1) * pack_factor // GPTQ_MARLIN_TILE
|
||||
@@ -257,8 +256,7 @@ def marlin_pad_qweight(
|
||||
qweight: torch.Tensor, size_n: int, size_k: int, padded_n: int, padded_k: int
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad a GPTQ-layout packed weight (size_k / pack, size_n) for
|
||||
gptq_marlin_repack.
|
||||
"""
|
||||
gptq_marlin_repack."""
|
||||
if (padded_n, padded_k) == (size_n, size_k):
|
||||
return qweight
|
||||
pack_factor = size_k // qweight.size(0)
|
||||
@@ -276,8 +274,7 @@ def marlin_pad_scales(
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad weight scales (num_groups, size_n); call before
|
||||
marlin_permute_scales and pass the padded extents to it.
|
||||
"""
|
||||
marlin_permute_scales and pass the padded extents to it."""
|
||||
if (padded_n, padded_k) == (size_n, size_k):
|
||||
return scales
|
||||
pad_rows = padded_k // group_size - scales.size(0) if group_size > 0 else 0
|
||||
|
||||
@@ -41,8 +41,7 @@ def _nvfp4_compute_scale_factor(
|
||||
) -> float:
|
||||
"""Compute the power-of-2 scale_factor needed so that all non-zero
|
||||
values in marlin_scales * 2^7 are >= 2 after rescaling.
|
||||
Returns a Python float (power of 2, >= 1.0).
|
||||
"""
|
||||
Returns a Python float (power of 2, >= 1.0)."""
|
||||
# Since half has a smaller dynamic range compared to bfloat16,
|
||||
# no rescaling is applied here if active dtype is half.
|
||||
if a_dtype is not None and a_dtype == torch.half:
|
||||
@@ -376,8 +375,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
def pad_w13(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a (E, num_shards * N, cols)
|
||||
tensor to padded_N rows.
|
||||
"""
|
||||
tensor to padded_N rows."""
|
||||
if padded_N == N:
|
||||
return x
|
||||
x = x.view(E, num_shards, N, x.size(-1))
|
||||
@@ -386,8 +384,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
def pad_w2(x: torch.Tensor, packing: int) -> torch.Tensor:
|
||||
"""Zero-pad the packed N (last) dim of a (E, K, N / packing)
|
||||
tensor.
|
||||
"""
|
||||
tensor."""
|
||||
if padded_N == N:
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
|
||||
|
||||
@@ -217,8 +217,7 @@ def prepare_fp8_layer_for_marlin(
|
||||
|
||||
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.
|
||||
"""
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
|
||||
if padded_n == n:
|
||||
return x
|
||||
e = x.size(0)
|
||||
|
||||
@@ -9,8 +9,7 @@ from .base import RotaryEmbedding
|
||||
|
||||
class NTKScalingRotaryEmbedding(RotaryEmbedding):
|
||||
"""RotaryEmbedding extended with fixed and mixed NTK scaling.
|
||||
https://kexue.fm/archives/9706
|
||||
"""
|
||||
https://kexue.fm/archives/9706"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -285,8 +285,7 @@ def _gather_workspace_shapes(
|
||||
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
|
||||
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
|
||||
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales.
|
||||
"""
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
|
||||
if use_fp4_cache:
|
||||
return (
|
||||
((total_seq_lens, head_dim // 2), torch.uint8),
|
||||
@@ -304,8 +303,7 @@ def kv_cache_as_quant_view(
|
||||
use_fp4_cache: bool,
|
||||
) -> torch.Tensor:
|
||||
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation.
|
||||
"""
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation."""
|
||||
if use_fp4_cache:
|
||||
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
|
||||
num_blocks, block_size, _ = kv_cache.shape
|
||||
|
||||
@@ -41,8 +41,7 @@ def _prefill_k_workspaces(
|
||||
total_seq_lens: int, head_dim: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The packed MXFP4 K-gather workspace ``(values, scales)``, shared with
|
||||
the dense indexer layers of the same model.
|
||||
"""
|
||||
the dense indexer layers of the same model."""
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, current_platform.fp8_dtype(), use_fp4_cache=True
|
||||
)
|
||||
@@ -120,8 +119,7 @@ class SparseMQAIndexer(nn.Module):
|
||||
|
||||
def _reserve_workspaces(self, device: torch.device) -> None:
|
||||
"""Profiling run: claim the K-gather workspace and the peak sparse
|
||||
logits allocation so the memory estimate covers them.
|
||||
"""
|
||||
logits allocation so the memory estimate covers them."""
|
||||
_prefill_k_workspaces(self.max_total_seq_len, self.head_dim)
|
||||
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
|
||||
torch.empty(max_logits_bytes, dtype=torch.uint8, device=device)
|
||||
|
||||
@@ -376,8 +376,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
) -> VocabParallelEmbeddingShardIndices:
|
||||
"""Get start and end indices for vocab parallel embedding, following the
|
||||
layout outlined in the class docstring, based on the given tp_rank and
|
||||
tp_size.
|
||||
"""
|
||||
tp_size."""
|
||||
num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded
|
||||
padded_org_vocab_start_index, padded_org_vocab_end_index = (
|
||||
vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size)
|
||||
|
||||
Reference in New Issue
Block a user