Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -7,13 +7,16 @@ import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8)
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_group_quant_int8, per_token_quant_int8)
|
||||
per_token_group_quant_int8,
|
||||
per_token_quant_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_quantize)
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import mxfp8_quantize
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils import cdiv
|
||||
@@ -21,26 +24,28 @@ from vllm.utils.flashinfer import fp4_quantize
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _count_expert_num_tokens(topk_ids_ptr, expert_num_tokens_ptr, num_experts,
|
||||
topk_numel, expert_map,
|
||||
HAS_EXPERT_MAP: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr):
|
||||
|
||||
def _count_expert_num_tokens(
|
||||
topk_ids_ptr,
|
||||
expert_num_tokens_ptr,
|
||||
num_experts,
|
||||
topk_numel,
|
||||
expert_map,
|
||||
HAS_EXPERT_MAP: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
curr_expert = tl.program_id(0)
|
||||
|
||||
offsets = tl.arange(0, BLOCK_SIZE)
|
||||
topk_ids_ptrs = topk_ids_ptr + offsets
|
||||
|
||||
acc = tl.zeros((BLOCK_SIZE, ), dtype=tl.int32)
|
||||
acc = tl.zeros((BLOCK_SIZE,), dtype=tl.int32)
|
||||
for x in range(tl.cdiv(topk_numel, BLOCK_SIZE)):
|
||||
mask = offsets < (topk_numel - x * BLOCK_SIZE)
|
||||
expert_ids = tl.load(topk_ids_ptrs, mask=mask, other=-1)
|
||||
if HAS_EXPERT_MAP:
|
||||
expert_map_ptrs = expert_map + expert_ids
|
||||
expert_map_mask = expert_ids >= 0
|
||||
expert_ids = tl.load(expert_map_ptrs,
|
||||
mask=expert_map_mask,
|
||||
other=-1)
|
||||
expert_ids = tl.load(expert_map_ptrs, mask=expert_map_mask, other=-1)
|
||||
|
||||
has_curr_expert = tl.where(expert_ids == curr_expert, 1, 0)
|
||||
acc = acc + has_curr_expert
|
||||
@@ -51,8 +56,8 @@ def _count_expert_num_tokens(topk_ids_ptr, expert_num_tokens_ptr, num_experts,
|
||||
|
||||
|
||||
def count_expert_num_tokens(
|
||||
topk_ids: torch.Tensor, num_local_experts: int,
|
||||
expert_map: Optional[torch.Tensor]) -> torch.Tensor:
|
||||
topk_ids: torch.Tensor, num_local_experts: int, expert_map: Optional[torch.Tensor]
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Count the number to tokens assigned to each expert.
|
||||
|
||||
@@ -68,17 +73,16 @@ def count_expert_num_tokens(
|
||||
A tensor of size num_local_experts, where tensor[i] holds the number
|
||||
of tokens assigned to the ith expert.
|
||||
"""
|
||||
assert topk_ids.dtype.is_signed, (
|
||||
"The kernel uses -1 to represent invalid topk_ids")
|
||||
expert_num_tokens = torch.empty((num_local_experts),
|
||||
device=topk_ids.device,
|
||||
dtype=torch.int32)
|
||||
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
|
||||
expert_num_tokens = torch.empty(
|
||||
(num_local_experts), device=topk_ids.device, dtype=torch.int32
|
||||
)
|
||||
|
||||
grid = num_local_experts
|
||||
BLOCK_SIZE = min(topk_ids.numel(), 1024)
|
||||
BLOCK_SIZE = triton.next_power_of_2(BLOCK_SIZE)
|
||||
|
||||
_count_expert_num_tokens[(grid, )](
|
||||
_count_expert_num_tokens[(grid,)](
|
||||
topk_ids,
|
||||
expert_num_tokens,
|
||||
num_local_experts,
|
||||
@@ -96,9 +100,10 @@ def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
|
||||
Shrink the given tensor and apply the given view to it. This is
|
||||
used to resize the intermediate fused_moe caches.
|
||||
"""
|
||||
assert prod(v) <= x.numel(
|
||||
), f"{v} ({prod(v)}) <= {x.shape} ({x.numel()})" # CUDAGRAPH unfriendly?
|
||||
return x.flatten()[:prod(v)].view(*v)
|
||||
assert prod(v) <= x.numel(), (
|
||||
f"{v} ({prod(v)}) <= {x.shape} ({x.numel()})"
|
||||
) # CUDAGRAPH unfriendly?
|
||||
return x.flatten()[: prod(v)].view(*v)
|
||||
|
||||
|
||||
def _fp4_quantize(
|
||||
@@ -106,9 +111,7 @@ def _fp4_quantize(
|
||||
A_scale: Optional[torch.Tensor],
|
||||
is_sf_swizzled_layout: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return fp4_quantize(A,
|
||||
A_scale,
|
||||
is_sf_swizzled_layout=is_sf_swizzled_layout)
|
||||
return fp4_quantize(A, A_scale, is_sf_swizzled_layout=is_sf_swizzled_layout)
|
||||
|
||||
|
||||
def _fp8_quantize(
|
||||
@@ -125,7 +128,8 @@ def _fp8_quantize(
|
||||
# TODO(luka): use QuantFP8 custom op
|
||||
# https://github.com/vllm-project/vllm/issues/20711
|
||||
A, A_scale = ops.scaled_fp8_quant(
|
||||
A, A_scale, use_per_token_if_dynamic=per_act_token)
|
||||
A, A_scale, use_per_token_if_dynamic=per_act_token
|
||||
)
|
||||
else:
|
||||
assert not per_act_token
|
||||
assert len(block_shape) == 2
|
||||
@@ -151,8 +155,7 @@ def _int8_quantize(
|
||||
# activations apply per-token quantization. Otherwise, assume
|
||||
# activation tensor-wise fp8/int8 quantization, dynamic or static
|
||||
if block_shape is None:
|
||||
assert per_act_token, \
|
||||
"int8 quantization only supports block or channel-wise"
|
||||
assert per_act_token, "int8 quantization only supports block or channel-wise"
|
||||
A, A_scale = per_token_quant_int8(A)
|
||||
else:
|
||||
assert not per_act_token
|
||||
@@ -204,9 +207,7 @@ def moe_kernel_quantize_input(
|
||||
elif quant_dtype == torch.int8:
|
||||
return _int8_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
elif quant_dtype == "nvfp4":
|
||||
return _fp4_quantize(A,
|
||||
A_scale,
|
||||
is_sf_swizzled_layout=is_fp4_scale_swizzled)
|
||||
return _fp4_quantize(A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled)
|
||||
elif quant_dtype == "mxfp4":
|
||||
return _mxfp4_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
elif quant_dtype == "mxfp8":
|
||||
@@ -225,8 +226,7 @@ def _fp8_perm(m: torch.Tensor, idx: torch.Tensor) -> torch.Tensor:
|
||||
return m[idx, ...]
|
||||
|
||||
|
||||
def normalize_scales_shape(
|
||||
scales: Optional[torch.Tensor]) -> Optional[torch.Tensor]:
|
||||
def normalize_scales_shape(scales: Optional[torch.Tensor]) -> Optional[torch.Tensor]:
|
||||
if scales is not None:
|
||||
if scales.numel() == 1:
|
||||
scales = scales.view(1, 1)
|
||||
@@ -242,8 +242,9 @@ def normalize_batched_scales_shape(
|
||||
if scales is not None and scales.ndim < 3:
|
||||
if scales.numel() == 1:
|
||||
scales = scales.view(1)
|
||||
scales = torch.repeat_interleave(scales, num_experts,
|
||||
dim=0).view(num_experts, 1, 1)
|
||||
scales = torch.repeat_interleave(scales, num_experts, dim=0).view(
|
||||
num_experts, 1, 1
|
||||
)
|
||||
else:
|
||||
scales = scales.view(num_experts, -1, scales.size(-1))
|
||||
|
||||
@@ -263,7 +264,8 @@ def _validate_scale_shape(
|
||||
assert a_scale.numel() == 1, f"{a_scale.shape}"
|
||||
elif per_act_token_quant:
|
||||
assert a_scale.shape[0] == a.shape[0] and a_scale.shape[1] == 1, (
|
||||
f"{a_scale.shape[0]} == {a.shape[0]} and {a_scale.shape[1]} == 1")
|
||||
f"{a_scale.shape[0]} == {a.shape[0]} and {a_scale.shape[1]} == 1"
|
||||
)
|
||||
else:
|
||||
assert block_shape is not None
|
||||
expected = (a.shape[0], cdiv(a.shape[1], block_shape[1]))
|
||||
|
||||
Reference in New Issue
Block a user