Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -209,6 +209,7 @@ def candidate_blocks_to_sparse_indices(
|
||||
``i * sparse_block_kv + ks % sparse_block_kv``.
|
||||
end: [rows] int32 count of valid sparse-token columns; valid columns
|
||||
always form a prefix because the block ids are sorted.
|
||||
|
||||
"""
|
||||
rows, num_candidates = candidate_blocks.shape
|
||||
ratio = candidate_block_size // sparse_block_kv
|
||||
@@ -273,8 +274,17 @@ def sparse_topk_remap(
|
||||
count come back as -1.
|
||||
|
||||
Args:
|
||||
logits: [rows, width] bf16 sparse logits, as produced by the sparse
|
||||
kernels.
|
||||
sparse_indices: [rows, S] int32 sparse block ids backing ``logits``.
|
||||
end: [rows] int32 per-row count of valid logit columns.
|
||||
row_ks: [rows] int32 per-row K range start.
|
||||
sparse_block_kv: Positions per sparse block (8 or 16).
|
||||
topk_tokens: Number of tokens to keep per row.
|
||||
topk_indices: [rows, topk_tokens] int32 output buffer.
|
||||
col_indices: [rows, topk_tokens] int32 scratch for the sparse-column
|
||||
top-k result (row stride 32B-aligned for DeepSelect).
|
||||
|
||||
"""
|
||||
rows, width = logits.shape
|
||||
assert logits.dtype == torch.bfloat16 and width >= topk_tokens
|
||||
@@ -329,6 +339,9 @@ def sparse_mqa_logits_prefill_chunk(
|
||||
cu_seqlen_ke: [rows] int32 per-token K end bounds in the packed
|
||||
workspace.
|
||||
candidate_blocks: [rows, K] int32 request-local candidate block ids.
|
||||
candidate_block_size: Positions per candidate block.
|
||||
sparse_block_kv: Positions per sparse block (8 or 16).
|
||||
topk_tokens: Number of tokens to keep per row.
|
||||
topk_indices: [rows, topk_tokens] output buffer.
|
||||
sparse_indices: Caller-owned scratch, see
|
||||
`candidate_blocks_to_sparse_indices`.
|
||||
@@ -341,6 +354,7 @@ def sparse_mqa_logits_prefill_chunk(
|
||||
|
||||
Returns:
|
||||
The DeepGEMM schedule metadata used, for reuse by later layers.
|
||||
|
||||
"""
|
||||
assert weights.dtype == torch.bfloat16, weights.dtype
|
||||
if kernel_metadata is None:
|
||||
@@ -417,6 +431,9 @@ def sparse_mqa_logits_paged_decode(
|
||||
row_indices: [rows] int32 row -> request map (pairing only affects
|
||||
scheduling).
|
||||
candidate_blocks: [rows, K] int32 candidate block ids.
|
||||
candidate_block_size: Positions per candidate block.
|
||||
sparse_block_kv: Positions per sparse block (8 or 16).
|
||||
topk_tokens: Number of tokens to keep per row.
|
||||
topk_indices: [rows, topk_tokens] output buffer.
|
||||
row_ks: [rows] int32 zeros (paged blocks are context-relative).
|
||||
sparse_indices: Caller-owned scratch.
|
||||
@@ -426,6 +443,7 @@ def sparse_mqa_logits_paged_decode(
|
||||
|
||||
Returns:
|
||||
The DeepGEMM schedule metadata used, for reuse by later layers.
|
||||
|
||||
"""
|
||||
rows = q.shape[0]
|
||||
page_kv = kv_cache.shape[1]
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
This module re-exports linear kernel implementations to provide a
|
||||
"""This module re-exports linear kernel implementations to provide a
|
||||
stable import interface during an ongoing reorganization. Upcoming
|
||||
PRs will remove the scaled_mm and mixed_precision subdirectories
|
||||
and reorganize kernels by provider (aiter, cutlass, flashinfer, etc.)
|
||||
@@ -637,8 +636,7 @@ def choose_scaled_mm_linear_kernel(
|
||||
*,
|
||||
quantization: str,
|
||||
) -> type[_KernelT]:
|
||||
"""
|
||||
Choose a _KernelT that can implement the given config for the
|
||||
"""Choose a _KernelT that can implement the given config for the
|
||||
given compute capability. Attempts to choose the best kernel in terms of
|
||||
performance.
|
||||
|
||||
@@ -660,8 +658,8 @@ def choose_scaled_mm_linear_kernel(
|
||||
|
||||
Returns:
|
||||
_KernelT: Chosen kernel.
|
||||
"""
|
||||
|
||||
"""
|
||||
failure_reason_list = []
|
||||
|
||||
if force_kernel is not None:
|
||||
@@ -815,8 +813,7 @@ def init_int8_linear_kernel(
|
||||
def choose_mp_linear_kernel(
|
||||
config: MPLinearLayerConfig, compute_capability: int | None = None
|
||||
) -> type[MPLinearKernel]:
|
||||
"""
|
||||
Choose an MPLinearKernel that can implement the given config for the given
|
||||
"""Choose an MPLinearKernel that can implement the given config for the given
|
||||
compute capability. Attempts to choose the best kernel in terms of
|
||||
performance.
|
||||
|
||||
@@ -832,6 +829,7 @@ def choose_mp_linear_kernel(
|
||||
|
||||
Returns:
|
||||
type[MPLinearKernel]: Chosen kernel.
|
||||
|
||||
"""
|
||||
if compute_capability is None:
|
||||
if current_platform is None:
|
||||
@@ -883,7 +881,8 @@ def choose_mp_linear_kernel(
|
||||
|
||||
def init_mxfp8_linear_kernel(*, bmm_batch_size: int | None = None) -> Mxfp8LinearKernel:
|
||||
"""Select and instantiate the best MXFP8 linear kernel for the
|
||||
current platform."""
|
||||
current platform.
|
||||
"""
|
||||
config = Mxfp8LinearLayerConfig(bmm_batch_size=bmm_batch_size)
|
||||
|
||||
platform = current_platform._enum
|
||||
@@ -935,7 +934,8 @@ def init_mxfp4_linear_kernel(
|
||||
activation_quant_key: QuantKey | None = None,
|
||||
) -> MxFp4LinearKernel:
|
||||
"""Select and instantiate the best MXFP4 linear kernel for the
|
||||
current platform."""
|
||||
current platform.
|
||||
"""
|
||||
config = MxFp4LinearLayerConfig(
|
||||
activation_quant_key=activation_quant_key,
|
||||
)
|
||||
@@ -982,7 +982,8 @@ def init_mxfp6_linear_kernel(
|
||||
activation_quant_key: QuantKey | None = None,
|
||||
) -> MxFp6LinearKernel:
|
||||
"""Select and instantiate the best MXFP6 linear kernel for the
|
||||
current platform."""
|
||||
current platform.
|
||||
"""
|
||||
config = MxFp6LinearLayerConfig(
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
@@ -1070,7 +1071,8 @@ def init_wfp8_a16_linear_kernel(
|
||||
|
||||
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
"""Select and instantiate the best NVFP4 linear kernel for the
|
||||
current platform."""
|
||||
current platform.
|
||||
"""
|
||||
config = NvFp4LinearLayerConfig()
|
||||
a16_kernels = (
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
@@ -1193,8 +1195,7 @@ def register_linear_kernel(
|
||||
platform: PlatformEnum,
|
||||
kernel_type: str = "mp",
|
||||
) -> None:
|
||||
"""
|
||||
Register a new linear kernel class to be considered in kernel selection.
|
||||
"""Register a new linear kernel class to be considered in kernel selection.
|
||||
|
||||
Args:
|
||||
kernel_class (type): The kernel class to register.
|
||||
@@ -1204,6 +1205,7 @@ def register_linear_kernel(
|
||||
|
||||
Raises:
|
||||
ValueError: If the kernel_type is not recognized.
|
||||
|
||||
"""
|
||||
if kernel_type == "mp":
|
||||
if platform not in _POSSIBLE_KERNELS:
|
||||
|
||||
@@ -50,6 +50,7 @@ class Params:
|
||||
# Access typed parameters
|
||||
output = func(input, params.weight, params.weight_scale)
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
weight: torch.Tensor
|
||||
@@ -72,7 +73,7 @@ class Params:
|
||||
|
||||
@dataclass
|
||||
class FP8Params(Params):
|
||||
"""FP8 layer parameters with typed fields"""
|
||||
"""FP8 layer parameters with typed fields."""
|
||||
|
||||
input_scale_ub: torch.Tensor | None
|
||||
|
||||
@@ -80,7 +81,7 @@ class FP8Params(Params):
|
||||
|
||||
@classmethod
|
||||
def from_layer(cls, layer: torch.nn.Module) -> "FP8Params":
|
||||
"""Extract parameters from layer"""
|
||||
"""Extract parameters from layer."""
|
||||
return cls(
|
||||
weight=getattr(layer, cls.WEIGHT),
|
||||
weight_scale=getattr(layer, cls.WEIGHT_SCALE),
|
||||
@@ -163,6 +164,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
2. Initialization: __init__() creates kernel instance with config
|
||||
3. Weight loading: process_weights_after_loading() preprocesses weights
|
||||
4. Inference: apply_weights() executes the quantized matmul
|
||||
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
@@ -184,6 +186,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
A tuple of (is_supported, reason):
|
||||
- is_supported: True if the kernel can run on this hardware
|
||||
- reason: If not supported, a string explaining why; otherwise None
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -204,7 +207,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
A tuple of (can_implement, reason):
|
||||
- can_implement: True if this kernel supports the config
|
||||
- reason: If not supported, a string explaining why; otherwise None
|
||||
```
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -214,6 +217,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
Args:
|
||||
config: Kernel-specific configuration containing settings like
|
||||
quantization keys, output dtypes, etc.
|
||||
|
||||
"""
|
||||
self.config = config
|
||||
|
||||
@@ -245,6 +249,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
weight_reordered = reorder_weights(params.weight)
|
||||
replace_parameter(layer, params.WEIGHT, weight_reordered)
|
||||
```
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -270,6 +275,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
def _get_layer_params(self, layer, **kwargs):
|
||||
return MyKernelParams.from_layer(layer)
|
||||
```
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -281,6 +287,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
|
||||
Returns:
|
||||
Number of tokens to pad, or None for no padding (default)
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -306,5 +313,6 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
|
||||
|
||||
Returns:
|
||||
Output tensor of shape [..., out_features]
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Hybrid W4A16 kernel: Triton for prefill, HIP skinny for decode.
|
||||
"""Hybrid W4A16 kernel: Triton for prefill, HIP skinny for decode.
|
||||
|
||||
Routes based on batch size M:
|
||||
M <= MAX_SKINNY_BATCH_SIZE: HIP skinny GEMM (wvSplitK_int4_g)
|
||||
@@ -94,8 +93,7 @@ def _triton_w4a16_skinny_fmt_kernel(
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fused W4A16 GEMM reading weights from skinny format [N, K//8].
|
||||
"""Fused W4A16 GEMM reading weights from skinny format [N, K//8].
|
||||
|
||||
B is stored as [N, K//8] int32 using ExLlama shuffle packing:
|
||||
each int32 packs 8 K-values with interleave [0,2,4,6,1,3,5,7]:
|
||||
@@ -196,8 +194,7 @@ def triton_w4a16_skinny_fmt_gemm(
|
||||
zp_bias: int = 8,
|
||||
zp: torch.Tensor | None = None, # [N//8, K//G] int32 zero-points
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Fused W4A16 GEMM reading from skinny weight format [N, K//8].
|
||||
"""Fused W4A16 GEMM reading from skinny weight format [N, K//8].
|
||||
|
||||
Args:
|
||||
a: Activation matrix [M, K], float16 or bfloat16.
|
||||
@@ -211,6 +208,7 @@ def triton_w4a16_skinny_fmt_gemm(
|
||||
|
||||
Returns:
|
||||
Output matrix [M, N], same dtype as a.
|
||||
|
||||
"""
|
||||
assert a.is_contiguous(), "Activation matrix must be contiguous"
|
||||
assert b_q.is_contiguous(), "Weight matrix must be contiguous"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Triton-based W4A16 GEMM kernel for ROCm MI300.
|
||||
"""Triton-based W4A16 GEMM kernel for ROCm MI300.
|
||||
|
||||
Implements fused int4-weight dequantization + fp16 GEMM in a single kernel,
|
||||
using GPTQ sequential packing (8 int4 values per int32, shifts [0,4,...,28]).
|
||||
@@ -67,8 +66,7 @@ def triton_w4a16_gemm_kernel(
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Fused W4A16 GEMM: C[M,N] = A[M,K] @ dequant(B)[K,N]
|
||||
"""Fused W4A16 GEMM: C[M,N] = A[M,K] @ dequant(B)[K,N].
|
||||
|
||||
B is stored as [K, N//8] int32 using GPTQ sequential packing:
|
||||
each int32 packs 8 consecutive N-values at bit offsets [0,4,8,12,16,20,24,28].
|
||||
@@ -170,8 +168,7 @@ def _triton_w4a16_gemm_impl(
|
||||
group_size: int,
|
||||
zp_bias: int = 8, # bias for uint4b8 when qzeros is None
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Fused W4A16 GEMM using GPTQ-packed int4 weights.
|
||||
"""Fused W4A16 GEMM using GPTQ-packed int4 weights.
|
||||
|
||||
Args:
|
||||
a: Activation matrix [M, K], float16 or bfloat16.
|
||||
@@ -184,6 +181,7 @@ def _triton_w4a16_gemm_impl(
|
||||
|
||||
Returns:
|
||||
Output matrix [M, N], same dtype as a.
|
||||
|
||||
"""
|
||||
assert a.is_contiguous(), "Activation matrix must be contiguous"
|
||||
assert b_q.is_contiguous(), "Weight matrix must be contiguous"
|
||||
@@ -302,8 +300,7 @@ def triton_w4a16_gemm(
|
||||
|
||||
|
||||
class TritonW4A16LinearKernel(MPLinearKernel):
|
||||
"""
|
||||
Triton-based W4A16 GEMM kernel for ROCm (MI300 and newer).
|
||||
"""Triton-based W4A16 GEMM kernel for ROCm (MI300 and newer).
|
||||
|
||||
Supports GPTQ-format int4 weights (uint4b8 symmetric, uint4 asymmetric)
|
||||
with grouped quantization. Weight tensors are transposed from the
|
||||
@@ -360,8 +357,7 @@ class TritonW4A16LinearKernel(MPLinearKernel):
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Convert compressed-tensors checkpoint layout to kernel layout.
|
||||
"""Convert compressed-tensors checkpoint layout to kernel layout.
|
||||
|
||||
Checkpoint (from compressed_tensors_wNa16.create_weights):
|
||||
weight_packed: [N, K//8] int32 input_dim=1, output_dim=0, packed_dim=1
|
||||
|
||||
@@ -19,6 +19,7 @@ class MxFp4LinearLayerConfig:
|
||||
Attributes:
|
||||
activation_quant_key: Identifies the activation quantization format,
|
||||
or `None` when activations must not be quantized.
|
||||
|
||||
"""
|
||||
|
||||
activation_quant_key: QuantKey | None = None
|
||||
|
||||
@@ -21,6 +21,7 @@ class MxFp6LinearLayerConfig:
|
||||
kMxfp6E2M3Static or kMxfp6E3M2Static.
|
||||
activation_quant_key: Identifies the activation quantization format,
|
||||
or `None` when activations must not be quantized.
|
||||
|
||||
"""
|
||||
|
||||
weight_quant_key: QuantKey
|
||||
|
||||
@@ -177,7 +177,8 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""This kernel supports dynamic quantization of the input. By
|
||||
convention, pre-quantized blockscales must use the swizzled layout."""
|
||||
convention, pre-quantized blockscales must use the swizzled layout.
|
||||
"""
|
||||
return kNvfp4Dynamic
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -41,7 +41,8 @@ class FP8BlockParams(FP8Params):
|
||||
@property
|
||||
def block_scale_attr(self) -> str:
|
||||
"""Fp8LinearMethod registers the block scale as ``weight_scale_inv``,
|
||||
compressed-tensors as ``weight_scale``."""
|
||||
compressed-tensors as ``weight_scale``.
|
||||
"""
|
||||
return (
|
||||
self.WEIGHT_SCALE
|
||||
if self.weight_scale_inv is None
|
||||
|
||||
@@ -71,8 +71,7 @@ class AiterInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
`AiterInt8ScaledMMLinearKernel` implements a fused version of
|
||||
"""`AiterInt8ScaledMMLinearKernel` implements a fused version of
|
||||
`output = torch.mm((scale_a * a), (scale_b * b)).to(out_dtype)`
|
||||
where scale_a * a and scale_b * b are implemented using numpy-style
|
||||
broadcasting.
|
||||
|
||||
@@ -174,7 +174,8 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
"""Only static per-tensor activation quantization is supported for external
|
||||
quantization."""
|
||||
quantization.
|
||||
"""
|
||||
if self.config.activation_quant_key == kFp8StaticTensorSym:
|
||||
return kFp8StaticTensorSym
|
||||
return None
|
||||
@@ -184,7 +185,8 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
x: torch.Tensor, dim: int, alignment: int, value: float = 0.0
|
||||
) -> torch.Tensor:
|
||||
"""Pad tensor ``x`` along ``dim`` to the next multiple of
|
||||
``alignment``."""
|
||||
``alignment``.
|
||||
"""
|
||||
remainder = x.shape[dim] % alignment
|
||||
if remainder == 0:
|
||||
return x
|
||||
|
||||
@@ -149,8 +149,7 @@ class FlashInferFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
class FlashInferFp8DeepGEMMDynamicBlockScaledKernel(
|
||||
Fp8BlockScaledDynamicMMLinearKernel
|
||||
):
|
||||
"""
|
||||
Conditional FlashInfer / DeepGEMM FP8 block-scaled GEMM.
|
||||
"""Conditional FlashInfer / DeepGEMM FP8 block-scaled GEMM.
|
||||
|
||||
Dispatches between two kernels based on input batch size:
|
||||
- Small batches (M < 32): FlashInfer's swapAB trick for better utilisation.
|
||||
@@ -214,9 +213,7 @@ def _flashinfer_fp8_blockscale_gemm_fake(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Required fake/meta implementation for torch.compile graph tracing.
|
||||
"""
|
||||
"""Required fake/meta implementation for torch.compile graph tracing."""
|
||||
return torch.empty(
|
||||
input.shape[0], weight.shape[0], dtype=torch.bfloat16, device=input.device
|
||||
)
|
||||
@@ -236,8 +233,7 @@ def _dynamic_flashinfer_deepgemm_blockscale_gemm_impl(
|
||||
group_size: int,
|
||||
use_deep_gemm_e8m0: bool,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Conditional FlashInfer FP8 blockscale GEMM with batch-size-dependent selection.
|
||||
"""Conditional FlashInfer FP8 blockscale GEMM with batch-size-dependent selection.
|
||||
|
||||
This function switches between two optimized kernels based on the input batch size:
|
||||
- For small batches (M < 32): Uses FlashInfer's DeepGEMM swapAB optimization.
|
||||
@@ -260,6 +256,7 @@ def _dynamic_flashinfer_deepgemm_blockscale_gemm_impl(
|
||||
|
||||
Returns:
|
||||
Output tensor of shape (batch_size, output_dim) in bfloat16 format
|
||||
|
||||
"""
|
||||
|
||||
def run_flashinfer_deepgemm_swapAB(
|
||||
@@ -323,9 +320,7 @@ def _dynamic_flashinfer_deepgemm_blockscale_gemm_fake(
|
||||
group_size: int,
|
||||
use_deep_gemm_e8m0: bool,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Required fake/meta implementation for torch.compile graph tracing.
|
||||
"""
|
||||
"""Required fake/meta implementation for torch.compile graph tracing."""
|
||||
return torch.empty(
|
||||
input.shape[0], weight.shape[0], dtype=torch.bfloat16, device=input.device
|
||||
)
|
||||
|
||||
@@ -27,8 +27,7 @@ from .ScaledMMLinearKernel import (
|
||||
|
||||
|
||||
class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
"""
|
||||
FP8 Marlin kernel for GPUs that lack FP8 hardware support.
|
||||
"""FP8 Marlin kernel for GPUs that lack FP8 hardware support.
|
||||
Leverages the Marlin kernel for fast weight-only FP8 quantization.
|
||||
"""
|
||||
|
||||
|
||||
@@ -50,8 +50,7 @@ def _supports_torch_fp8_scaled_mm() -> bool:
|
||||
|
||||
|
||||
class TorchFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
"""
|
||||
Base class for FP8 linear kernels using Torch.
|
||||
"""Base class for FP8 linear kernels using Torch.
|
||||
Each subclass represents a kernel variant for
|
||||
specific device capabilities and torch versions.
|
||||
"""
|
||||
|
||||
@@ -19,8 +19,7 @@ def mhc_pre_aiter(
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
"""Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
@@ -40,8 +39,8 @@ def mhc_pre_aiter(
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
@@ -120,7 +119,7 @@ def mhc_pre_delayed_aiter(
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
residual_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""mHC pre with the pre-mix carried in from the previous sublayer.
|
||||
"""MHC pre with the pre-mix carried in from the previous sublayer.
|
||||
|
||||
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
|
||||
rather than the gate computed here, and that gate is returned as the
|
||||
@@ -151,6 +150,7 @@ def mhc_pre_delayed_aiter(
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
next_pre_mix: shape (..., hc_mult), dtype torch.float32
|
||||
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
|
||||
@@ -109,6 +109,7 @@ def mhc_pre_delayed_tilelang(
|
||||
Post and residual coefficients, optionally normalized BF16 layer input,
|
||||
and the next FP32 pre-mix, with shapes (tokens, hc_mult, 1),
|
||||
(tokens, hc_mult, hc_mult), (tokens, hidden_size), and (tokens, hc_mult).
|
||||
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
@@ -320,6 +321,7 @@ def mhc_fused_post_pre_delayed_tilelang(
|
||||
(tokens, hc_mult, hidden_size), (tokens, hc_mult, 1),
|
||||
(tokens, hc_mult, hc_mult), (tokens, hidden_size), (tokens, hc_mult),
|
||||
and (tokens, hidden_size).
|
||||
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
@@ -551,8 +553,7 @@ def mhc_pre_tilelang(
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
"""Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
@@ -576,6 +577,7 @@ def mhc_pre_tilelang(
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
@@ -827,8 +829,7 @@ def mhc_fused_post_pre_tilelang(
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Run one MHC post block followed by the next MHC pre block.
|
||||
"""Run one MHC post block followed by the next MHC pre block.
|
||||
|
||||
When ``norm_weight`` is provided, the layer_input_cur output is the
|
||||
RMSNorm'd activation (fused into the kernel); otherwise it is the
|
||||
@@ -842,8 +843,8 @@ def mhc_fused_post_pre_tilelang(
|
||||
post_mix_cur: shape (..., hc_mult, 1)
|
||||
comb_mix_cur: shape (..., hc_mult, hc_mult)
|
||||
layer_input_cur: shape (..., hidden_size)
|
||||
"""
|
||||
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_MHC_FUSED_TILELANG_KERNEL,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
|
||||
@@ -661,7 +661,7 @@ def mhc_fused_tilelang(
|
||||
tile_n: int = 1,
|
||||
split_k: int = 1,
|
||||
) -> tilelang.JITKernel:
|
||||
"""Fused mhc post-mapping + pre-norm GEMM FMA"""
|
||||
"""Fused mhc post-mapping + pre-norm GEMM FMA."""
|
||||
m = T.dynamic("num_tokens")
|
||||
split_k = T.dynamic("split_k")
|
||||
h = hidden
|
||||
|
||||
@@ -57,8 +57,7 @@ def mhc_pre_torch(
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
"""Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
@@ -76,8 +75,8 @@ def mhc_pre_torch(
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
|
||||
"""
|
||||
# Validate shapes
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
|
||||
Reference in New Issue
Block a user