[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -339,17 +339,6 @@ def fused_add_rms_norm(
|
||||
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
|
||||
|
||||
|
||||
def muse_spark_mul_tanh_soft_cap(
|
||||
x: torch.Tensor,
|
||||
multiplier: float,
|
||||
soft_cap_temp: float,
|
||||
) -> torch.Tensor:
|
||||
"""Cast to FP32 and apply ``temp * tanh(x * multiplier / temp)`` in one pass."""
|
||||
out = torch.empty_like(x, dtype=torch.float32)
|
||||
torch.ops._C.muse_spark_mul_tanh_soft_cap(out, x, multiplier, soft_cap_temp)
|
||||
return out
|
||||
|
||||
|
||||
def fused_qk_norm_rope(
|
||||
qkv: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
@@ -2609,197 +2598,6 @@ def reshape_and_cache_flash(
|
||||
)
|
||||
|
||||
|
||||
def fused_qknorm_rope_cache(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rotary_dim: int,
|
||||
eps: float,
|
||||
norm_weight_bias: float = 0.0,
|
||||
kv_cache_dtype: str = "auto",
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fuse per-head RMSNorm, partial NeoX RoPE and KV insertion.
|
||||
|
||||
Norm weights may be FP32 independently of QKV. A zero rotary dimension
|
||||
skips RoPE; norm outputs are rounded to QKV dtype before rotation.
|
||||
"""
|
||||
if rotary_dim < 0 or rotary_dim > head_dim or rotary_dim % 8:
|
||||
raise ValueError("rotary_dim must be a multiple of 8 in [0, head_dim]")
|
||||
q = qkv.new_empty((qkv.shape[0], num_q_heads * head_dim))
|
||||
k = qkv.new_empty((qkv.shape[0], num_kv_heads * head_dim))
|
||||
v = torch.empty_like(k)
|
||||
cos, sin = cos_sin_cache.chunk(2, dim=-1)
|
||||
torch.ops._C.fused_qknorm_rope_cache(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos,
|
||||
sin,
|
||||
positions,
|
||||
slot_mapping,
|
||||
key_cache,
|
||||
value_cache,
|
||||
head_dim,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim // 2,
|
||||
eps,
|
||||
norm_weight_bias,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
return q, k, v
|
||||
|
||||
|
||||
def fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
cos_sin_cache: torch.Tensor | None = None,
|
||||
rotary_dim: int = 0,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
block_size: int = 0,
|
||||
kv_cache_dtype: str = "auto",
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
q_fp8_out: torch.Tensor | None = None,
|
||||
q_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""In-place weightless QK RMSNorm + GPT-J RoPE, optionally writing k/v to cache.
|
||||
|
||||
When ``kv_cache`` is given, the caller must skip the attention layer's own KV write.
|
||||
With ``q_fp8_out``, write scaled E4M3 Q there instead of rewriting Q in ``qkv``.
|
||||
"""
|
||||
torch.ops._C.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
cos_sin_cache,
|
||||
rotary_dim,
|
||||
slot_mapping,
|
||||
kv_cache,
|
||||
block_size,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
q_fp8_out,
|
||||
q_scale,
|
||||
)
|
||||
|
||||
|
||||
def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
rotary_dim: int,
|
||||
eps: float,
|
||||
index_q_norm_weight: torch.Tensor | None = None,
|
||||
index_k_norm_weight: torch.Tensor | None = None,
|
||||
num_index_heads: int = 0,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
index_slot_mapping: torch.Tensor | None = None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
index_cache: torch.Tensor | None = None,
|
||||
block_size: int = 0,
|
||||
q_out: torch.Tensor | None = None,
|
||||
index_q_out: torch.Tensor | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
skip_index_branch: bool = False,
|
||||
q_fp8_out: torch.Tensor | None = None,
|
||||
q_fp8_scale: float = 1.0,
|
||||
kv_k_scale: torch.Tensor | None = None,
|
||||
kv_v_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Fused MiniMax-M3 attention pre-processing (in-place).
|
||||
|
||||
Applies Gemma RMSNorm + partial NeoX RoPE to ``qkv`` in place. ``qkv`` is a
|
||||
single fused tensor:
|
||||
|
||||
- dense layer (``num_index_heads == 0``): ``[q | k | v]``;
|
||||
- sparse layer (``num_index_heads > 0``): ``[q | k | v | index_q |
|
||||
index_k]`` — the index branch is read straight out of ``qkv``.
|
||||
|
||||
When ``kv_cache`` is given (sparse serving), also scatter-inserts the
|
||||
normed/roped k & v into the paged KV cache by ``slot_mapping`` and the
|
||||
index key into ``index_cache`` by ``index_slot_mapping``. ``kv_cache_dtype``
|
||||
selects the cache storage/conversion path. If
|
||||
``index_slot_mapping`` is omitted, ``slot_mapping`` is used for both caches.
|
||||
|
||||
If ``q_out`` / ``index_q_out`` (contiguous ``[N, nq*128]`` / ``[N,
|
||||
niq*128]``) are given, the normed/roped q / index_q are written there
|
||||
instead of in place — folding the de-interleave into this kernel's store so
|
||||
callers skip a separate ``.contiguous()`` copy before the SM100 sparse
|
||||
attention's flat TMA descriptor.
|
||||
|
||||
If ``q_fp8_out`` is given, the same normalized q is also written in FP8
|
||||
E4M3 using ``q_fp8_scale`` as its dequantization scale. Without ``q_out``,
|
||||
q is then written only in FP8 and the q slice of ``qkv`` is left as is.
|
||||
|
||||
``kv_cache_dtype="nvfp4"`` quantizes k/v into the packed HND
|
||||
``[num_blocks, 2 * num_kv_heads, block_size, 72]`` uint8 cache using the
|
||||
device dequantization scales ``kv_k_scale``/``kv_v_scale``. Slot
|
||||
``2 * head + side`` (K = 0, V = 1) holds that head's E2M1 data followed by
|
||||
its E4M3 block scales, so every head is one contiguous run of the page.
|
||||
|
||||
When ``skip_index_branch`` is true, sparse rows still keep their packed
|
||||
``[index_q | index_k]`` tail, but the kernel only processes the main q/k/v
|
||||
branches and main KV cache. This is used by MiniMax-M3 index-topk reuse
|
||||
layers that consume top-k block ids selected by an earlier sparse layer.
|
||||
"""
|
||||
torch.ops._C.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
index_q_norm_weight,
|
||||
index_k_norm_weight,
|
||||
num_index_heads,
|
||||
slot_mapping,
|
||||
index_slot_mapping,
|
||||
kv_cache,
|
||||
index_cache,
|
||||
block_size,
|
||||
q_out,
|
||||
index_q_out,
|
||||
kv_cache_dtype,
|
||||
skip_index_branch,
|
||||
q_fp8_out,
|
||||
q_fp8_scale,
|
||||
kv_k_scale,
|
||||
kv_v_scale,
|
||||
)
|
||||
|
||||
|
||||
def fused_kda_decode(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -2922,35 +2720,6 @@ def concat_and_cache_mla_grouped(
|
||||
)
|
||||
|
||||
|
||||
def kimi_k3_attn_res(
|
||||
prefix: torch.Tensor,
|
||||
delta: torch.Tensor | None,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
output_norm_weight: torch.Tensor | None,
|
||||
num_blocks: int,
|
||||
block_write_idx: int,
|
||||
eps: float,
|
||||
output_norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty_like(prefix)
|
||||
torch.ops._C.kimi_k3_attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
output,
|
||||
num_blocks,
|
||||
block_write_idx,
|
||||
eps,
|
||||
output_norm_eps,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def concat_and_cache_mla_rope_fused(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
@@ -3435,24 +3204,6 @@ def sm100_cutlass_mla_get_workspace_size(
|
||||
)
|
||||
|
||||
|
||||
def dsv3_fused_a_gemm(
|
||||
output: torch.Tensor,
|
||||
mat_a: torch.Tensor,
|
||||
mat_b: torch.Tensor,
|
||||
enable_pdl: bool = False,
|
||||
) -> None:
|
||||
"""Low-latency fused-A-style GEMM (SM 9.0+, BF16, 1-16 tokens).
|
||||
|
||||
Computes ``output = mat_a @ mat_b`` for the compiled Kimi K3 and
|
||||
DeepSeek V3 projection shapes. ``mat_a`` and ``output`` are row-major;
|
||||
``mat_b`` is the column-major transposed weight. ``enable_pdl`` permits
|
||||
programmatic dependent launch for callers that have validated it.
|
||||
|
||||
Requires SM 9.0+.
|
||||
"""
|
||||
torch.ops._C.dsv3_fused_a_gemm(output, mat_a, mat_b, enable_pdl)
|
||||
|
||||
|
||||
def weight_packed_linear_cpu(
|
||||
x: torch.Tensor,
|
||||
packed_weight: torch.Tensor,
|
||||
@@ -4841,24 +4592,3 @@ if hasattr(torch.ops._C, "hadacore_transform"):
|
||||
@register_fake("_C::hadacore_transform")
|
||||
def _hadacore_transform_fake(x: torch.Tensor, inplace: bool) -> torch.Tensor:
|
||||
return torch.empty_like(x) if not inplace else x
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "minimax_allreduce_rms_qk"):
|
||||
|
||||
@register_fake("_C::minimax_allreduce_rms_qk")
|
||||
def _minimax_allreduce_rms_qk_fake(
|
||||
qkv: torch.Tensor,
|
||||
norm_weight_q: torch.Tensor,
|
||||
norm_weight_k: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
rank: int,
|
||||
nranks: int,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
token_num = qkv.shape[0]
|
||||
return (
|
||||
torch.empty([token_num, q_size], dtype=qkv.dtype, device=qkv.device),
|
||||
torch.empty([token_num, kv_size], dtype=qkv.dtype, device=qkv.device),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user