Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -209,6 +209,7 @@ def candidate_blocks_to_sparse_indices(
``i * sparse_block_kv + ks % sparse_block_kv``.
end: [rows] int32 count of valid sparse-token columns; valid columns
always form a prefix because the block ids are sorted.
"""
rows, num_candidates = candidate_blocks.shape
ratio = candidate_block_size // sparse_block_kv
@@ -273,8 +274,17 @@ def sparse_topk_remap(
count come back as -1.
Args:
logits: [rows, width] bf16 sparse logits, as produced by the sparse
kernels.
sparse_indices: [rows, S] int32 sparse block ids backing ``logits``.
end: [rows] int32 per-row count of valid logit columns.
row_ks: [rows] int32 per-row K range start.
sparse_block_kv: Positions per sparse block (8 or 16).
topk_tokens: Number of tokens to keep per row.
topk_indices: [rows, topk_tokens] int32 output buffer.
col_indices: [rows, topk_tokens] int32 scratch for the sparse-column
top-k result (row stride 32B-aligned for DeepSelect).
"""
rows, width = logits.shape
assert logits.dtype == torch.bfloat16 and width >= topk_tokens
@@ -329,6 +339,9 @@ def sparse_mqa_logits_prefill_chunk(
cu_seqlen_ke: [rows] int32 per-token K end bounds in the packed
workspace.
candidate_blocks: [rows, K] int32 request-local candidate block ids.
candidate_block_size: Positions per candidate block.
sparse_block_kv: Positions per sparse block (8 or 16).
topk_tokens: Number of tokens to keep per row.
topk_indices: [rows, topk_tokens] output buffer.
sparse_indices: Caller-owned scratch, see
`candidate_blocks_to_sparse_indices`.
@@ -341,6 +354,7 @@ def sparse_mqa_logits_prefill_chunk(
Returns:
The DeepGEMM schedule metadata used, for reuse by later layers.
"""
assert weights.dtype == torch.bfloat16, weights.dtype
if kernel_metadata is None:
@@ -417,6 +431,9 @@ def sparse_mqa_logits_paged_decode(
row_indices: [rows] int32 row -> request map (pairing only affects
scheduling).
candidate_blocks: [rows, K] int32 candidate block ids.
candidate_block_size: Positions per candidate block.
sparse_block_kv: Positions per sparse block (8 or 16).
topk_tokens: Number of tokens to keep per row.
topk_indices: [rows, topk_tokens] output buffer.
row_ks: [rows] int32 zeros (paged blocks are context-relative).
sparse_indices: Caller-owned scratch.
@@ -426,6 +443,7 @@ def sparse_mqa_logits_paged_decode(
Returns:
The DeepGEMM schedule metadata used, for reuse by later layers.
"""
rows = q.shape[0]
page_kv = kv_cache.shape[1]
+15 -13
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This module re-exports linear kernel implementations to provide a
"""This module re-exports linear kernel implementations to provide a
stable import interface during an ongoing reorganization. Upcoming
PRs will remove the scaled_mm and mixed_precision subdirectories
and reorganize kernels by provider (aiter, cutlass, flashinfer, etc.)
@@ -637,8 +636,7 @@ def choose_scaled_mm_linear_kernel(
*,
quantization: str,
) -> type[_KernelT]:
"""
Choose a _KernelT that can implement the given config for the
"""Choose a _KernelT that can implement the given config for the
given compute capability. Attempts to choose the best kernel in terms of
performance.
@@ -660,8 +658,8 @@ def choose_scaled_mm_linear_kernel(
Returns:
_KernelT: Chosen kernel.
"""
"""
failure_reason_list = []
if force_kernel is not None:
@@ -815,8 +813,7 @@ def init_int8_linear_kernel(
def choose_mp_linear_kernel(
config: MPLinearLayerConfig, compute_capability: int | None = None
) -> type[MPLinearKernel]:
"""
Choose an MPLinearKernel that can implement the given config for the given
"""Choose an MPLinearKernel that can implement the given config for the given
compute capability. Attempts to choose the best kernel in terms of
performance.
@@ -832,6 +829,7 @@ def choose_mp_linear_kernel(
Returns:
type[MPLinearKernel]: Chosen kernel.
"""
if compute_capability is None:
if current_platform is None:
@@ -883,7 +881,8 @@ def choose_mp_linear_kernel(
def init_mxfp8_linear_kernel(*, bmm_batch_size: int | None = None) -> Mxfp8LinearKernel:
"""Select and instantiate the best MXFP8 linear kernel for the
current platform."""
current platform.
"""
config = Mxfp8LinearLayerConfig(bmm_batch_size=bmm_batch_size)
platform = current_platform._enum
@@ -935,7 +934,8 @@ def init_mxfp4_linear_kernel(
activation_quant_key: QuantKey | None = None,
) -> MxFp4LinearKernel:
"""Select and instantiate the best MXFP4 linear kernel for the
current platform."""
current platform.
"""
config = MxFp4LinearLayerConfig(
activation_quant_key=activation_quant_key,
)
@@ -982,7 +982,8 @@ def init_mxfp6_linear_kernel(
activation_quant_key: QuantKey | None = None,
) -> MxFp6LinearKernel:
"""Select and instantiate the best MXFP6 linear kernel for the
current platform."""
current platform.
"""
config = MxFp6LinearLayerConfig(
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
@@ -1070,7 +1071,8 @@ def init_wfp8_a16_linear_kernel(
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
"""Select and instantiate the best NVFP4 linear kernel for the
current platform."""
current platform.
"""
config = NvFp4LinearLayerConfig()
a16_kernels = (
FlashInferCuteDslNvFp4W4A16LinearKernel,
@@ -1193,8 +1195,7 @@ def register_linear_kernel(
platform: PlatformEnum,
kernel_type: str = "mp",
) -> None:
"""
Register a new linear kernel class to be considered in kernel selection.
"""Register a new linear kernel class to be considered in kernel selection.
Args:
kernel_class (type): The kernel class to register.
@@ -1204,6 +1205,7 @@ def register_linear_kernel(
Raises:
ValueError: If the kernel_type is not recognized.
"""
if kernel_type == "mp":
if platform not in _POSSIBLE_KERNELS:
+11 -3
View File
@@ -50,6 +50,7 @@ class Params:
# Access typed parameters
output = func(input, params.weight, params.weight_scale)
```
"""
weight: torch.Tensor
@@ -72,7 +73,7 @@ class Params:
@dataclass
class FP8Params(Params):
"""FP8 layer parameters with typed fields"""
"""FP8 layer parameters with typed fields."""
input_scale_ub: torch.Tensor | None
@@ -80,7 +81,7 @@ class FP8Params(Params):
@classmethod
def from_layer(cls, layer: torch.nn.Module) -> "FP8Params":
"""Extract parameters from layer"""
"""Extract parameters from layer."""
return cls(
weight=getattr(layer, cls.WEIGHT),
weight_scale=getattr(layer, cls.WEIGHT_SCALE),
@@ -163,6 +164,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
2. Initialization: __init__() creates kernel instance with config
3. Weight loading: process_weights_after_loading() preprocesses weights
4. Inference: apply_weights() executes the quantized matmul
"""
@classmethod
@@ -184,6 +186,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
A tuple of (is_supported, reason):
- is_supported: True if the kernel can run on this hardware
- reason: If not supported, a string explaining why; otherwise None
"""
raise NotImplementedError
@@ -204,7 +207,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
A tuple of (can_implement, reason):
- can_implement: True if this kernel supports the config
- reason: If not supported, a string explaining why; otherwise None
```
"""
raise NotImplementedError
@@ -214,6 +217,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
Args:
config: Kernel-specific configuration containing settings like
quantization keys, output dtypes, etc.
"""
self.config = config
@@ -245,6 +249,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
weight_reordered = reorder_weights(params.weight)
replace_parameter(layer, params.WEIGHT, weight_reordered)
```
"""
raise NotImplementedError
@@ -270,6 +275,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
def _get_layer_params(self, layer, **kwargs):
return MyKernelParams.from_layer(layer)
```
"""
raise NotImplementedError
@@ -281,6 +287,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
Returns:
Number of tokens to pad, or None for no padding (default)
"""
return None
@@ -306,5 +313,6 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]):
Returns:
Output tensor of shape [..., out_features]
"""
raise NotImplementedError
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Hybrid W4A16 kernel: Triton for prefill, HIP skinny for decode.
"""Hybrid W4A16 kernel: Triton for prefill, HIP skinny for decode.
Routes based on batch size M:
M <= MAX_SKINNY_BATCH_SIZE: HIP skinny GEMM (wvSplitK_int4_g)
@@ -94,8 +93,7 @@ def _triton_w4a16_skinny_fmt_kernel(
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
):
"""
Fused W4A16 GEMM reading weights from skinny format [N, K//8].
"""Fused W4A16 GEMM reading weights from skinny format [N, K//8].
B is stored as [N, K//8] int32 using ExLlama shuffle packing:
each int32 packs 8 K-values with interleave [0,2,4,6,1,3,5,7]:
@@ -196,8 +194,7 @@ def triton_w4a16_skinny_fmt_gemm(
zp_bias: int = 8,
zp: torch.Tensor | None = None, # [N//8, K//G] int32 zero-points
) -> torch.Tensor:
"""
Fused W4A16 GEMM reading from skinny weight format [N, K//8].
"""Fused W4A16 GEMM reading from skinny weight format [N, K//8].
Args:
a: Activation matrix [M, K], float16 or bfloat16.
@@ -211,6 +208,7 @@ def triton_w4a16_skinny_fmt_gemm(
Returns:
Output matrix [M, N], same dtype as a.
"""
assert a.is_contiguous(), "Activation matrix must be contiguous"
assert b_q.is_contiguous(), "Weight matrix must be contiguous"
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Triton-based W4A16 GEMM kernel for ROCm MI300.
"""Triton-based W4A16 GEMM kernel for ROCm MI300.
Implements fused int4-weight dequantization + fp16 GEMM in a single kernel,
using GPTQ sequential packing (8 int4 values per int32, shifts [0,4,...,28]).
@@ -67,8 +66,7 @@ def triton_w4a16_gemm_kernel(
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
):
"""
Fused W4A16 GEMM: C[M,N] = A[M,K] @ dequant(B)[K,N]
"""Fused W4A16 GEMM: C[M,N] = A[M,K] @ dequant(B)[K,N].
B is stored as [K, N//8] int32 using GPTQ sequential packing:
each int32 packs 8 consecutive N-values at bit offsets [0,4,8,12,16,20,24,28].
@@ -170,8 +168,7 @@ def _triton_w4a16_gemm_impl(
group_size: int,
zp_bias: int = 8, # bias for uint4b8 when qzeros is None
) -> torch.Tensor:
"""
Fused W4A16 GEMM using GPTQ-packed int4 weights.
"""Fused W4A16 GEMM using GPTQ-packed int4 weights.
Args:
a: Activation matrix [M, K], float16 or bfloat16.
@@ -184,6 +181,7 @@ def _triton_w4a16_gemm_impl(
Returns:
Output matrix [M, N], same dtype as a.
"""
assert a.is_contiguous(), "Activation matrix must be contiguous"
assert b_q.is_contiguous(), "Weight matrix must be contiguous"
@@ -302,8 +300,7 @@ def triton_w4a16_gemm(
class TritonW4A16LinearKernel(MPLinearKernel):
"""
Triton-based W4A16 GEMM kernel for ROCm (MI300 and newer).
"""Triton-based W4A16 GEMM kernel for ROCm (MI300 and newer).
Supports GPTQ-format int4 weights (uint4b8 symmetric, uint4 asymmetric)
with grouped quantization. Weight tensors are transposed from the
@@ -360,8 +357,7 @@ class TritonW4A16LinearKernel(MPLinearKernel):
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""
Convert compressed-tensors checkpoint layout to kernel layout.
"""Convert compressed-tensors checkpoint layout to kernel layout.
Checkpoint (from compressed_tensors_wNa16.create_weights):
weight_packed: [N, K//8] int32 input_dim=1, output_dim=0, packed_dim=1
@@ -19,6 +19,7 @@ class MxFp4LinearLayerConfig:
Attributes:
activation_quant_key: Identifies the activation quantization format,
or `None` when activations must not be quantized.
"""
activation_quant_key: QuantKey | None = None
@@ -21,6 +21,7 @@ class MxFp6LinearLayerConfig:
kMxfp6E2M3Static or kMxfp6E3M2Static.
activation_quant_key: Identifies the activation quantization format,
or `None` when activations must not be quantized.
"""
weight_quant_key: QuantKey
@@ -177,7 +177,8 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
def input_quant_key(self) -> QuantKey | None:
"""This kernel supports dynamic quantization of the input. By
convention, pre-quantized blockscales must use the swizzled layout."""
convention, pre-quantized blockscales must use the swizzled layout.
"""
return kNvfp4Dynamic
@classmethod
@@ -41,7 +41,8 @@ class FP8BlockParams(FP8Params):
@property
def block_scale_attr(self) -> str:
"""Fp8LinearMethod registers the block scale as ``weight_scale_inv``,
compressed-tensors as ``weight_scale``."""
compressed-tensors as ``weight_scale``.
"""
return (
self.WEIGHT_SCALE
if self.weight_scale_inv is None
@@ -71,8 +71,7 @@ class AiterInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
`AiterInt8ScaledMMLinearKernel` implements a fused version of
"""`AiterInt8ScaledMMLinearKernel` implements a fused version of
`output = torch.mm((scale_a * a), (scale_b * b)).to(out_dtype)`
where scale_a * a and scale_b * b are implemented using numpy-style
broadcasting.
@@ -174,7 +174,8 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
def input_quant_key(self) -> QuantKey | None:
"""Only static per-tensor activation quantization is supported for external
quantization."""
quantization.
"""
if self.config.activation_quant_key == kFp8StaticTensorSym:
return kFp8StaticTensorSym
return None
@@ -184,7 +185,8 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
x: torch.Tensor, dim: int, alignment: int, value: float = 0.0
) -> torch.Tensor:
"""Pad tensor ``x`` along ``dim`` to the next multiple of
``alignment``."""
``alignment``.
"""
remainder = x.shape[dim] % alignment
if remainder == 0:
return x
@@ -149,8 +149,7 @@ class FlashInferFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
class FlashInferFp8DeepGEMMDynamicBlockScaledKernel(
Fp8BlockScaledDynamicMMLinearKernel
):
"""
Conditional FlashInfer / DeepGEMM FP8 block-scaled GEMM.
"""Conditional FlashInfer / DeepGEMM FP8 block-scaled GEMM.
Dispatches between two kernels based on input batch size:
- Small batches (M < 32): FlashInfer's swapAB trick for better utilisation.
@@ -214,9 +213,7 @@ def _flashinfer_fp8_blockscale_gemm_fake(
weight: torch.Tensor,
weight_scale: torch.Tensor,
) -> torch.Tensor:
"""
Required fake/meta implementation for torch.compile graph tracing.
"""
"""Required fake/meta implementation for torch.compile graph tracing."""
return torch.empty(
input.shape[0], weight.shape[0], dtype=torch.bfloat16, device=input.device
)
@@ -236,8 +233,7 @@ def _dynamic_flashinfer_deepgemm_blockscale_gemm_impl(
group_size: int,
use_deep_gemm_e8m0: bool,
) -> torch.Tensor:
"""
Conditional FlashInfer FP8 blockscale GEMM with batch-size-dependent selection.
"""Conditional FlashInfer FP8 blockscale GEMM with batch-size-dependent selection.
This function switches between two optimized kernels based on the input batch size:
- For small batches (M < 32): Uses FlashInfer's DeepGEMM swapAB optimization.
@@ -260,6 +256,7 @@ def _dynamic_flashinfer_deepgemm_blockscale_gemm_impl(
Returns:
Output tensor of shape (batch_size, output_dim) in bfloat16 format
"""
def run_flashinfer_deepgemm_swapAB(
@@ -323,9 +320,7 @@ def _dynamic_flashinfer_deepgemm_blockscale_gemm_fake(
group_size: int,
use_deep_gemm_e8m0: bool,
) -> torch.Tensor:
"""
Required fake/meta implementation for torch.compile graph tracing.
"""
"""Required fake/meta implementation for torch.compile graph tracing."""
return torch.empty(
input.shape[0], weight.shape[0], dtype=torch.bfloat16, device=input.device
)
@@ -27,8 +27,7 @@ from .ScaledMMLinearKernel import (
class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
"""
FP8 Marlin kernel for GPUs that lack FP8 hardware support.
"""FP8 Marlin kernel for GPUs that lack FP8 hardware support.
Leverages the Marlin kernel for fast weight-only FP8 quantization.
"""
@@ -50,8 +50,7 @@ def _supports_torch_fp8_scaled_mm() -> bool:
class TorchFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
"""
Base class for FP8 linear kernels using Torch.
"""Base class for FP8 linear kernels using Torch.
Each subclass represents a kernel variant for
specific device capabilities and torch versions.
"""
+4 -4
View File
@@ -19,8 +19,7 @@ def mhc_pre_aiter(
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
@@ -40,8 +39,8 @@ def mhc_pre_aiter(
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
@@ -120,7 +119,7 @@ def mhc_pre_delayed_aiter(
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""mHC pre with the pre-mix carried in from the previous sublayer.
"""MHC pre with the pre-mix carried in from the previous sublayer.
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
rather than the gate computed here, and that gate is returned as the
@@ -151,6 +150,7 @@ def mhc_pre_delayed_aiter(
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
next_pre_mix: shape (..., hc_mult), dtype torch.float32
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
+6 -5
View File
@@ -109,6 +109,7 @@ def mhc_pre_delayed_tilelang(
Post and residual coefficients, optionally normalized BF16 layer input,
and the next FP32 pre-mix, with shapes (tokens, hc_mult, 1),
(tokens, hc_mult, hc_mult), (tokens, hidden_size), and (tokens, hc_mult).
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
@@ -320,6 +321,7 @@ def mhc_fused_post_pre_delayed_tilelang(
(tokens, hc_mult, hidden_size), (tokens, hc_mult, 1),
(tokens, hc_mult, hc_mult), (tokens, hidden_size), (tokens, hc_mult),
and (tokens, hidden_size).
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
@@ -551,8 +553,7 @@ def mhc_pre_tilelang(
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
@@ -576,6 +577,7 @@ def mhc_pre_tilelang(
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
@@ -827,8 +829,7 @@ def mhc_fused_post_pre_tilelang(
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Run one MHC post block followed by the next MHC pre block.
"""Run one MHC post block followed by the next MHC pre block.
When ``norm_weight`` is provided, the layer_input_cur output is the
RMSNorm'd activation (fused into the kernel); otherwise it is the
@@ -842,8 +843,8 @@ def mhc_fused_post_pre_tilelang(
post_mix_cur: shape (..., hc_mult, 1)
comb_mix_cur: shape (..., hc_mult, hc_mult)
layer_input_cur: shape (..., hidden_size)
"""
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_MHC_FUSED_TILELANG_KERNEL,
_MHC_POST_TILELANG_KERNEL,
@@ -661,7 +661,7 @@ def mhc_fused_tilelang(
tile_n: int = 1,
split_k: int = 1,
) -> tilelang.JITKernel:
"""Fused mhc post-mapping + pre-norm GEMM FMA"""
"""Fused mhc post-mapping + pre-norm GEMM FMA."""
m = T.dynamic("num_tokens")
split_k = T.dynamic("split_k")
h = hidden
+2 -3
View File
@@ -57,8 +57,7 @@ def mhc_pre_torch(
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Forward pass for mHC pre block.
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
@@ -76,8 +75,8 @@ def mhc_pre_torch(
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
"""
# Validate shapes
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32