Merge branch 'main' into patches
This commit is contained in:
@@ -2024,6 +2024,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_local_block_size: int,
|
||||
dcp_virtual_block_size: int,
|
||||
dcp_manager: MLADCPManager | None = None,
|
||||
align_indptr: bool = False,
|
||||
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||
"""Build chunked-context metadata for an MLA prefill.
|
||||
|
||||
@@ -2043,6 +2044,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_local_block_size: Per-rank interleave block size for DCP.
|
||||
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
|
||||
dcp_manager: Shared MLA DCP collective manager.
|
||||
align_indptr: Align packed int32 indptr slices to 16 bytes for CuTe DSL.
|
||||
|
||||
Returns:
|
||||
The chunked-context metadata, or None when no prefill has any context.
|
||||
@@ -2123,6 +2125,15 @@ def build_mla_chunked_context_metadata(
|
||||
num_tokens = sum(plan.seq_lens)
|
||||
num_local_tokens = num_tokens
|
||||
|
||||
if align_indptr:
|
||||
# CuTe DSL requires 16-byte-aligned int32 indptrs.
|
||||
padding = -boundary_offset % 4
|
||||
cu_seq_lens_flat.extend([0] * padding)
|
||||
cu_seqlens_q_flat.extend([0] * padding)
|
||||
if use_dcp:
|
||||
padded_local_cu_seq_lens_flat.extend([0] * padding)
|
||||
boundary_offset += padding
|
||||
|
||||
seq_lens_flat.extend(plan.seq_lens)
|
||||
cu_seq_lens_flat.extend(itertools.accumulate(plan.seq_lens, initial=0))
|
||||
query_base = query_start_loc[plan.request_start]
|
||||
@@ -2616,6 +2627,14 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
|
||||
)
|
||||
|
||||
align_indptr = False
|
||||
if (
|
||||
self._prefill_backend.get_name() == "TRTLLM_RAGGED"
|
||||
and self.vllm_config.attention_config.trtllm_ragged_backend
|
||||
== "cute-dsl"
|
||||
):
|
||||
align_indptr = True
|
||||
|
||||
chunked_context_metadata = build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=context_lens_cpu,
|
||||
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||
@@ -2628,6 +2647,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
dcp_manager=self.dcp_manager,
|
||||
align_indptr=align_indptr,
|
||||
)
|
||||
|
||||
prefill_metadata = MLACommonPrefillMetadata(
|
||||
|
||||
@@ -359,13 +359,6 @@ class ExpertMapManager:
|
||||
return True
|
||||
return self._expert_map[global_id] != -1
|
||||
|
||||
def get_local_expert_ids(self) -> list[int]:
|
||||
"""Get list of global IDs for experts on this rank."""
|
||||
if self._expert_map is None:
|
||||
return list(range(self.global_num_experts))
|
||||
|
||||
return torch.where(self._expert_map != -1)[0].tolist()
|
||||
|
||||
def update(
|
||||
self,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
|
||||
@@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _act_mul_quant(
|
||||
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
expert_ends: torch.Tensor | None = None,
|
||||
expert_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
block_k = self._ACT_BLOCK_K
|
||||
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
|
||||
@@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
expert_ends=expert_ends,
|
||||
expert_alignment=expert_alignment,
|
||||
)
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
@@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
||||
use_psum_layout = current_platform.is_device_capability_family(100)
|
||||
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
@@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
expert_map=expert_map,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
aq_out=a1q_perm,
|
||||
use_psum_layout=use_psum_layout,
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
@@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
(a1q, a1q_scale),
|
||||
(w1.view(torch.int8), self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
use_psum_layout=use_psum_layout,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
# SwiGLU activation + FP8 requant
|
||||
# SiLU+mul and FP8 requant use the same live expert ranges as GEMM.
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
input=mm1_out.view(-1, N),
|
||||
output=quant_out,
|
||||
activation=activation,
|
||||
expert_ends=grouped_layout if use_psum_layout else None,
|
||||
expert_alignment=align_used if use_psum_layout else 0,
|
||||
)
|
||||
|
||||
# FC2: FP8 activations x FP4 weights
|
||||
@@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
(a2q, a2q_scale),
|
||||
(w2.view(torch.int8), self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
use_psum_layout=use_psum_layout,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
@@ -517,6 +517,7 @@ _TILE_CONFIG_OVERRIDES = {
|
||||
256: (128, 32),
|
||||
512: (128, 19),
|
||||
},
|
||||
(5120, 256): {64: (64, 64)}, # GB200
|
||||
(6144, 128): { # MiniMax-M3
|
||||
64: (32, 64),
|
||||
128: (64, 64),
|
||||
@@ -533,7 +534,7 @@ def _pick_tile_config(N: int, K: int, M: int, num_sms: int) -> tuple[int, int]:
|
||||
if 32 < N <= 512:
|
||||
token_bucket = triton.next_power_of_2(N)
|
||||
overrides = _TILE_CONFIG_OVERRIDES.get((K, M))
|
||||
if overrides is not None:
|
||||
if overrides is not None and token_bucket in overrides:
|
||||
BN, split_k = overrides[token_bucket]
|
||||
return BN, min(split_k, k_tiles)
|
||||
|
||||
@@ -609,6 +610,8 @@ def _bf16x3_router_gemm(X: torch.Tensor, W: torch.Tensor) -> torch.Tensor:
|
||||
"""Return ``X @ W.T`` using the SM100 BF16x3 router GEMM kernel."""
|
||||
N, K = X.shape
|
||||
M, _ = W.shape
|
||||
if N == 0:
|
||||
return X.new_empty((N, M), dtype=torch.float32)
|
||||
num_sms = torch.cuda.get_device_properties(X.device).multi_processor_count
|
||||
|
||||
BN, split_k = _pick_tile_config(N, K, M, num_sms)
|
||||
|
||||
@@ -24,8 +24,8 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
|
||||
any shape w/ K divisible by 8)
|
||||
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
|
||||
model-specific shapes, gfx950 any shape)
|
||||
2. fp32 specialized kernel (bf16/fp32 in, fp32 out; SM90+
|
||||
model-specific shapes and batch limits, gfx950 any shape with M<=32)
|
||||
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
|
||||
any shape)
|
||||
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
@@ -46,9 +46,14 @@ class GateLinear(ReplicatedLinear):
|
||||
split-K from the token count and skip the batch-invariant linear path.
|
||||
"""
|
||||
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
|
||||
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
|
||||
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel.
|
||||
FP32_SUPPORTED_SHAPES = {
|
||||
(3072, 256),
|
||||
(6144, 128),
|
||||
(5120, 256),
|
||||
(4096, 352),
|
||||
(8192, 256),
|
||||
}
|
||||
FP32_MAX_TOKENS = 32
|
||||
# Largest batch for which tier 1 beats the cuBLAS epilogue. Subclasses
|
||||
# whose router shape stays ahead further up the range may raise it.
|
||||
@@ -454,9 +459,9 @@ def fp32_router_gemm_dispatch_impl(
|
||||
weight: torch.Tensor,
|
||||
allow_bf16x3_router_gemm: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Dynamically run fp32 specialized gemm if num_tokens <= FP32_MAX_TOKENS,
|
||||
otherwise optionally run the experimental BF16x3 kernel for medium/large
|
||||
SM100 router batches, then fall back to F.linear.
|
||||
"""Run the FP32 kernel for small batches, with shape-specific limits.
|
||||
|
||||
Larger SM100 BF16 batches use BF16x3; other inputs fall back to F.linear.
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
does not support runtime dispatching on num_tokens.
|
||||
"""
|
||||
@@ -473,7 +478,15 @@ def fp32_router_gemm_dispatch_impl(
|
||||
if weight.dtype == torch.bfloat16:
|
||||
return torch.mm(x, weight.T, out_dtype=torch.float32)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
|
||||
if (
|
||||
allow_bf16x3_router_gemm
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.shape == (256, 5120)
|
||||
):
|
||||
# GB200: four token groups win through 48 for multiples of eight.
|
||||
max_tokens = 48 if x.shape[0] % 8 == 0 else 16
|
||||
if x.shape[0] <= max_tokens:
|
||||
return ops.fp32_router_gemm(x, weight)
|
||||
|
||||
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
|
||||
@@ -9,139 +9,162 @@
|
||||
from packaging import version
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.triton_helpers import fast_exp
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
TRITON_22 = version.parse(triton.__version__) >= version.parse("2.2.0")
|
||||
|
||||
|
||||
def _drop_racy_wgmma_configs(configs: list[triton.Config]) -> list[triton.Config]:
|
||||
"""Drop the configs that Triton miscompiles on sm90.
|
||||
|
||||
With whole warpgroups (num_warps % 4 == 0) and BLOCK_SIZE_M % 64 == 0,
|
||||
`tl.dot(cb, x)` lowers to wgmma with `cb` in registers. With
|
||||
num_stages > 1, Triton leaves that wgmma running across the loop
|
||||
back-edge while the next iteration overwrites the registers of `cb`,
|
||||
which writes NaN/Inf into whole output rows.
|
||||
"""
|
||||
if not (current_platform.is_cuda() and current_platform.is_device_capability(90)):
|
||||
return configs
|
||||
return [
|
||||
c
|
||||
for c in configs
|
||||
if c.num_stages == 1
|
||||
or c.num_warps % 4 != 0
|
||||
or c.kwargs["BLOCK_SIZE_M"] % 64 != 0
|
||||
]
|
||||
|
||||
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
# =================================================================
|
||||
# Higher warp count configs for better latency hiding
|
||||
# More warps = more instructions in flight = better memory latency hiding
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
# Smaller tiles with more stages for software pipelining
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=3,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# =================================================================
|
||||
# Low register pressure configs (num_stages=1) for large dstate
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
# num_stages=2 configs - moderate register pressure
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# Original configs for larger dstate values
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
|
||||
num_stages=3,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=2,
|
||||
),
|
||||
],
|
||||
configs=_drop_racy_wgmma_configs(
|
||||
[
|
||||
# =================================================================
|
||||
# Higher warp count configs for better latency hiding
|
||||
# More warps = more instructions in flight = better memory latency hiding
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=8,
|
||||
),
|
||||
# Smaller tiles with more stages for software pipelining
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=3,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# =================================================================
|
||||
# Low register pressure configs (num_stages=1) for large dstate
|
||||
# =================================================================
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=1,
|
||||
num_warps=4,
|
||||
),
|
||||
# num_stages=2 configs - moderate register pressure
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=2,
|
||||
num_warps=4,
|
||||
),
|
||||
# Original configs for larger dstate values
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
|
||||
num_stages=3,
|
||||
num_warps=8,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=4,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=5,
|
||||
num_warps=2,
|
||||
),
|
||||
triton.Config(
|
||||
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
|
||||
num_stages=4,
|
||||
num_warps=2,
|
||||
),
|
||||
]
|
||||
),
|
||||
key=["chunk_size", "hdim", "dstate", "IS_CAUSAL"],
|
||||
)
|
||||
@triton.jit
|
||||
|
||||
@@ -12,35 +12,6 @@ from vllm.triton_utils import tl, triton
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def block_dequant(
|
||||
x_q_block: torch.Tensor,
|
||||
x_s: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""This function conducts block-wise dequantization.
|
||||
The inputs are block-wise quantization tensor `x_q_block`,
|
||||
block-wise quantization scale and the block size.
|
||||
The outputs are dequantized tensor.
|
||||
"""
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
n, k = x_q_block.shape
|
||||
n_tiles = (n + block_n - 1) // block_n
|
||||
k_tiles = (k + block_k - 1) // block_k
|
||||
assert n_tiles == x_s.shape[0]
|
||||
assert k_tiles == x_s.shape[1]
|
||||
|
||||
x_dq_block = x_q_block.to(torch.float32)
|
||||
|
||||
for i in range(k_tiles):
|
||||
for j in range(n_tiles):
|
||||
x_dq_block[
|
||||
j * block_n : min((j + 1) * block_n, n),
|
||||
i * block_k : min((i + 1) * block_k, k),
|
||||
] *= x_s[j][i]
|
||||
|
||||
return x_dq_block
|
||||
|
||||
|
||||
if current_platform.is_rocm():
|
||||
|
||||
@triton.jit
|
||||
|
||||
Reference in New Issue
Block a user