[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from .mrope import MRotaryEmbedding
|
||||
|
||||
|
||||
class Ernie4_5_VLRotaryEmbedding(MRotaryEmbedding):
|
||||
"""3D rotary positional embedding. 3D is t:time h:height w:width."""
|
||||
|
||||
def forward_native( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert positions.ndim == 1 or positions.ndim == 2
|
||||
assert key is not None
|
||||
|
||||
num_tokens = positions.shape[-1]
|
||||
cos_sin = self.cos_sin_cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if positions.ndim == 2:
|
||||
assert self.mrope_section
|
||||
|
||||
section_h = self.mrope_section[0] # 22
|
||||
section_w = self.mrope_section[1] # 22
|
||||
section_t = self.mrope_section[2] # 20
|
||||
assert section_h == section_w
|
||||
# Split according to [h w h w h w h w... t t t...]
|
||||
section_cos_t = cos[..., -section_t:]
|
||||
section_cos_h = cos[..., : section_h + section_w : 2]
|
||||
section_cos_w = cos[..., 1 : section_h + section_w : 2]
|
||||
|
||||
cos_t, cos_h, cos_w = section_cos_t[0], section_cos_h[1], section_cos_w[2]
|
||||
cos_hw = torch.stack([cos_h, cos_w], dim=-1).reshape(
|
||||
cos_h.shape[:-1] + (cos_h.shape[-1] * 2,)
|
||||
)
|
||||
cos = torch.cat([cos_hw, cos_t], dim=-1)
|
||||
|
||||
section_sin_t = sin[..., -section_t:]
|
||||
section_sin_h = sin[..., : section_h + section_w : 2]
|
||||
section_sin_w = sin[..., 1 : section_h + section_w : 2]
|
||||
|
||||
sin_t, sin_h, sin_w = section_sin_t[0], section_sin_h[1], section_sin_w[2]
|
||||
sin_hw = torch.stack([sin_h, sin_w], dim=-1).reshape(
|
||||
sin_h.shape[:-1] + (sin_h.shape[-1] * 2,)
|
||||
)
|
||||
sin = torch.cat([sin_hw, sin_t], dim=-1)
|
||||
|
||||
query_shape = query.shape
|
||||
query = query.view(num_tokens, -1, self.head_size)
|
||||
query_rot = query[..., : self.rotary_dim]
|
||||
query_pass = query[..., self.rotary_dim :]
|
||||
query_rot = self.apply_rotary_emb.forward_native(
|
||||
query_rot,
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
|
||||
|
||||
key_shape = key.shape
|
||||
key = key.view(num_tokens, -1, self.head_size)
|
||||
key_rot = key[..., : self.rotary_dim]
|
||||
key_pass = key[..., self.rotary_dim :]
|
||||
key_rot = self.apply_rotary_emb.forward_native(
|
||||
key_rot,
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
|
||||
return query, key
|
||||
|
||||
def forward_cuda( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
return self.forward_native(positions, query, key)
|
||||
|
||||
def forward_xpu( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
# No fused XPU kernel for this 3D t/h/w rope; base
|
||||
# MRotaryEmbedding.forward_xpu forwards an extra `offsets` arg that
|
||||
# this class's forward_cuda override doesn't accept. Use native path.
|
||||
return self.forward_native(positions, query, key)
|
||||
@@ -1,153 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr
|
||||
# is an offset slice of the packed H2D buffer, so its 16B alignment varies
|
||||
# with grid count; without these the kernel recompiles mid-serving.
|
||||
@triton.jit(
|
||||
do_not_specialize=["num_tokens", "num_grids"],
|
||||
do_not_specialize_on_alignment=["grids_ptr"],
|
||||
)
|
||||
def _vit_mrope_setup_kernel(
|
||||
inv_freq_t_ptr,
|
||||
inv_freq_h_ptr,
|
||||
inv_freq_w_ptr,
|
||||
cos_out_ptr,
|
||||
sin_out_ptr,
|
||||
cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid
|
||||
grids_ptr, # (G, 3) int32: t, h, w per grid
|
||||
num_tokens,
|
||||
num_grids,
|
||||
half_t: tl.constexpr,
|
||||
half_h: tl.constexpr,
|
||||
half_w: tl.constexpr,
|
||||
spatial_merge: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one
|
||||
launch: compute each token's (t, h, w) position arithmetically and the
|
||||
rotation in fp32 from the per-axis inv_freq. Only each axis's mrope
|
||||
section columns are written; the rest is left uninitialized (the
|
||||
consumer's per-axis masked loads read only the section columns)."""
|
||||
pid = tl.program_id(0)
|
||||
tok = pid * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
tok_mask = tok < num_tokens
|
||||
|
||||
# Grid (segment) index per token; cu_seqlens is nondecreasing.
|
||||
seg = tl.zeros([BLOCK_N], dtype=tl.int32)
|
||||
for g in range(num_grids):
|
||||
end = tl.load(cu_seqlens_ptr + g + 1)
|
||||
seg += (tok >= end).to(tl.int32)
|
||||
|
||||
seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0)
|
||||
grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1)
|
||||
grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1)
|
||||
|
||||
local = tok - seg_start
|
||||
rem = local % (grid_h * grid_w)
|
||||
tpos = local // (grid_h * grid_w)
|
||||
# Within a frame tokens are ordered by spatial-merge block (bh, bw, mh,
|
||||
# mw), matching the reference ViT position-id permute.
|
||||
bh = rem // (spatial_merge * grid_w)
|
||||
r = rem % (spatial_merge * grid_w)
|
||||
bw = r // (spatial_merge * spatial_merge)
|
||||
r2 = r % (spatial_merge * spatial_merge)
|
||||
hpos = bh * spatial_merge + r2 // spatial_merge
|
||||
wpos = bw * spatial_merge + r2 % spatial_merge
|
||||
|
||||
d = tl.arange(0, BLOCK_D)
|
||||
half_rot = half_t + half_h + half_w
|
||||
row = (tok * half_rot)[:, None] + d[None, :]
|
||||
plane = num_tokens * half_rot
|
||||
|
||||
# Per-axis compute+store to cap live registers; the fp32 trig sequence
|
||||
# wants few elements per thread (hence small BLOCK_N, many warps).
|
||||
in_t = d < half_t
|
||||
inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0)
|
||||
freq = tpos.to(tl.float32)[:, None] * inv_t[None, :]
|
||||
mask = tok_mask[:, None] & in_t[None, :]
|
||||
tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask)
|
||||
|
||||
in_h = (d >= half_t) & (d < half_t + half_h)
|
||||
inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0)
|
||||
freq = hpos.to(tl.float32)[:, None] * inv_h[None, :]
|
||||
mask = tok_mask[:, None] & in_h[None, :]
|
||||
tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask)
|
||||
|
||||
in_w = (d >= half_t + half_h) & (d < half_rot)
|
||||
inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0)
|
||||
freq = wpos.to(tl.float32)[:, None] * inv_w[None, :]
|
||||
mask = tok_mask[:, None] & in_w[None, :]
|
||||
tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask)
|
||||
|
||||
|
||||
def vit_mrope_setup(
|
||||
inv_freq_t: torch.Tensor,
|
||||
inv_freq_h: torch.Tensor,
|
||||
inv_freq_w: torch.Tensor,
|
||||
grid_thw: list[list[int]],
|
||||
spatial_merge_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope.
|
||||
|
||||
One fused kernel: positions are computed arithmetically from the grids
|
||||
and the rotation comes from the per-axis inv_freq, so there is no
|
||||
cos/sin cache to size or grow.
|
||||
|
||||
Args:
|
||||
inv_freq_t: t-axis inverse frequencies, (t_dim // 2,).
|
||||
inv_freq_h: h-axis inverse frequencies, (h_dim // 2,).
|
||||
inv_freq_w: w-axis inverse frequencies, (w_dim // 2,).
|
||||
grid_thw: per-grid [t, h, w] patch counts (host metadata).
|
||||
spatial_merge_size: spatial merge factor of the ViT.
|
||||
|
||||
"""
|
||||
device = inv_freq_t.device
|
||||
cu = [0]
|
||||
for t, h, w in grid_thw:
|
||||
cu.append(cu[-1] + t * h * w)
|
||||
n = cu[-1]
|
||||
g = len(grid_thw)
|
||||
# grid_thw is per-request host metadata, so one H2D is unavoidable;
|
||||
# pack cu_seqlens and grids into a single pinned async copy.
|
||||
buf = async_tensor_h2d(
|
||||
cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32
|
||||
)
|
||||
cu_seqlens = buf[: g + 1]
|
||||
grids = buf[g + 1 :].view(g, 3)
|
||||
|
||||
half_t = inv_freq_t.numel()
|
||||
half_h = inv_freq_h.numel()
|
||||
half_w = inv_freq_w.numel()
|
||||
half_rot = half_t + half_h + half_w
|
||||
cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
|
||||
sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
|
||||
_vit_mrope_setup_kernel[(triton.cdiv(n, 32),)](
|
||||
inv_freq_t,
|
||||
inv_freq_h,
|
||||
inv_freq_w,
|
||||
cos,
|
||||
sin,
|
||||
cu_seqlens,
|
||||
grids,
|
||||
n,
|
||||
g,
|
||||
half_t=half_t,
|
||||
half_h=half_h,
|
||||
half_w=half_w,
|
||||
spatial_merge=spatial_merge_size,
|
||||
BLOCK_N=32,
|
||||
BLOCK_D=triton.next_power_of_2(half_rot),
|
||||
num_warps=8,
|
||||
)
|
||||
return cos, sin
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""In-place interleaved RoPE on the Q/K slices of a packed QKV buffer."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
# seqlen is not specialized: image sizes vary per request.
|
||||
@triton.jit(do_not_specialize=["seqlen"])
|
||||
def _packed_qk_rope_kernel(
|
||||
qk_ptr, # (seqlen, 2 * nheads, headdim) strided view of the packed QKV
|
||||
freqs_ptr, # (seqlen, rotary_dim) fp32 re/im view of the complex freqs
|
||||
seqlen,
|
||||
qk_row_stride,
|
||||
qk_head_stride,
|
||||
rotary_dim: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
BLOCK_K: tl.constexpr = triton.next_power_of_2(rotary_dim)
|
||||
pid_m = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
rk = tl.arange(0, BLOCK_K)
|
||||
mask_m = rm < seqlen
|
||||
|
||||
base = qk_ptr + pid_h * qk_head_stride + rm[:, None] * qk_row_stride
|
||||
mask_x = mask_m[:, None] & (rk < rotary_dim)[None, :]
|
||||
x = tl.load(base + rk[None, :], mask=mask_x, other=0.0).to(tl.float32)
|
||||
f = tl.load(freqs_ptr + rm[:, None] * rotary_dim + rk[None, :], mask=mask_x)
|
||||
cos, sin = tl.split(tl.reshape(f, (BLOCK_M, BLOCK_K // 2, 2)))
|
||||
|
||||
x0, x1 = tl.split(tl.reshape(x, (BLOCK_M, BLOCK_K // 2, 2)))
|
||||
o0 = x0 * cos - x1 * sin
|
||||
o1 = x1 * cos + x0 * sin
|
||||
tl.store(base + rk[None, :], tl.interleave(o0, o1), mask=mask_x)
|
||||
|
||||
|
||||
def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None:
|
||||
"""Rotate Q and K in place inside a packed QKV buffer.
|
||||
|
||||
Args:
|
||||
xqkv: contiguous (seqlen, 3, nheads, headdim); only the Q and K
|
||||
slices are rotated.
|
||||
freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs,
|
||||
read through its interleaved re/im fp32 view.
|
||||
|
||||
Matches ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied to Q and K
|
||||
separately to within one ULP, but in one kernel with ~5x less memory
|
||||
traffic. Both compute in fp32; the gap is which product the compiler
|
||||
keeps exact inside the fused multiply-add, which differs per backend.
|
||||
Requires triton: callers must check HAS_TRITON and use the unfused path
|
||||
otherwise. Contract violations raise.
|
||||
|
||||
"""
|
||||
assert xqkv.ndim == 4 and xqkv.size(1) == 3, xqkv.shape
|
||||
assert xqkv.is_contiguous(), xqkv.shape
|
||||
assert freqs_cis.ndim == 2 and freqs_cis.is_contiguous(), freqs_cis.shape
|
||||
assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype
|
||||
|
||||
seq_length, _, num_heads, head_dim = xqkv.shape
|
||||
rotary_dim = freqs_cis.size(-1) * 2
|
||||
assert freqs_cis.size(0) == seq_length, (freqs_cis.shape, xqkv.shape)
|
||||
assert rotary_dim == head_dim, (freqs_cis.shape, xqkv.shape) # full rotation
|
||||
qk = xqkv.as_strided(
|
||||
(seq_length, 2 * num_heads, head_dim),
|
||||
(3 * num_heads * head_dim, head_dim, 1),
|
||||
)
|
||||
|
||||
BLOCK_M = 4
|
||||
# seqlen on grid.x: HIP caps gridDim.y at 65535.
|
||||
grid = (triton.cdiv(seq_length, BLOCK_M), 2 * num_heads)
|
||||
_packed_qk_rope_kernel[grid](
|
||||
qk,
|
||||
freqs_cis.view(torch.float32),
|
||||
seq_length,
|
||||
qk.stride(0),
|
||||
qk.stride(1),
|
||||
rotary_dim=rotary_dim,
|
||||
BLOCK_M=BLOCK_M,
|
||||
num_warps=2 if rotary_dim <= 64 else 4,
|
||||
)
|
||||
Reference in New Issue
Block a user