[DSV4.1] Add encoder cuda graph support for deepseek-v4.1-flash (#56625)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-20 21:51:13 +08:00
committed by GitHub
parent 9b2f34cad4
commit 27757dde02
6 changed files with 599 additions and 63 deletions
+156 -7
View File
@@ -40,8 +40,7 @@ from vllm.model_executor.models.vision import (
)
@lru_cache(8)
def get_vision_cos_sin(
def _compute_vision_cos_sin(
n_h: int, n_w: int, dim: int, theta: float
) -> tuple[torch.Tensor, torch.Tensor]:
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
@@ -52,6 +51,13 @@ def get_vision_cos_sin(
return freqs.cos().unsqueeze(1), freqs.sin().unsqueeze(1)
@lru_cache(8)
def get_vision_cos_sin(
n_h: int, n_w: int, dim: int, theta: float
) -> tuple[torch.Tensor, torch.Tensor]:
return _compute_vision_cos_sin(n_h, n_w, dim, theta)
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
dtype = x.dtype
x1, x2 = x.float().chunk(2, dim=-1)
@@ -123,15 +129,23 @@ class DeepseekV4VisionAttention(nn.Module):
)
def forward(
self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
self,
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
n = x.size(0)
qkv, _ = self.wqkv(x)
q, k, v = (t.view(n, self.n_heads, self.head_dim) for t in qkv.chunk(3, -1))
q = apply_rotary(q, cos, sin).unsqueeze(0) # (b=1, n, h, d)
k = apply_rotary(k, cos, sin).unsqueeze(0)
# One image per call: a dense batch, no varlen packing metadata.
o = self.attn(q, k, v.unsqueeze(0))
# Dense (cu_seqlens=None): one image per call. Varlen (cu_seqlens
# set): packed multi-image batch for encoder CUDA graph replay.
o = self.attn(
q, k, v.unsqueeze(0), cu_seqlens=cu_seqlens, max_seqlen=max_seqlen
)
out, _ = self.wo(o.reshape(n, -1))
return out
@@ -173,9 +187,14 @@ class DeepseekV4VisionBlock(nn.Module):
self.mlp = DeepseekV4VisionMLP(config, prefix=f"{prefix}.mlp")
def forward(
self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
self,
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
x = x + self.attn(self.norm1(x), cos, sin)
x = x + self.attn(self.norm1(x), cos, sin, cu_seqlens, max_seqlen)
return x + self.mlp(self.norm2(x))
@@ -206,6 +225,115 @@ class DeepseekV4ViT(nn.Module):
x = block(x, cos, sin)
return self.norm(x)
def forward_packed(
self,
patches: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
) -> torch.Tensor:
"""Varlen path for encoder CUDA graphs: multiple images packed along
rows, per-image attention via ``cu_seqlens``, RoPE tables precomputed
on device. Pure-tensor (no host reads, no H2D), safe to capture."""
x = self.patch_embed(patches)
for block in self.blocks:
x = block(x, cos, sin, cu_seqlens, max_seqlen)
return self.norm(x)
def build_packed_vit_metadata(
grids: list[list[int]] | list[tuple[int, int]],
*,
rope_dim: int,
rope_theta: float,
device: torch.device,
max_seqlen_override: int | None = None,
cached: bool = True,
) -> dict[str, torch.Tensor]:
"""Precompute packed-batch ViT metadata for ``DeepseekV4ViT.forward_packed``.
Args:
grids: ``[n_vit_h, n_vit_w]`` per image, in packing order.
rope_dim: RoPE embedding dimension of the ViT.
rope_theta: RoPE base frequency of the ViT.
device: Device for the returned tensors (except ``max_seqlen``,
which stays on the host).
max_seqlen_override: Worst-case value baked in at CUDA graph capture
(the attention wrapper reads ``max_seqlen`` on the host, so the
capture-time value becomes a graph constant).
cached: Use the shared ``lru_cache`` for RoPE tables. Capture-time
dummy grids pass ``False`` to avoid evicting real entries.
"""
cos_sin = get_vision_cos_sin if cached else _compute_vision_cos_sin
cos_list: list[torch.Tensor] = []
sin_list: list[torch.Tensor] = []
cu_seqlens = [0]
max_seqlen = 0
for n_h, n_w in grids:
cos, sin = cos_sin(n_h, n_w, rope_dim, rope_theta)
cos_list.append(cos)
sin_list.append(sin)
n = n_h * n_w
cu_seqlens.append(cu_seqlens[-1] + n)
max_seqlen = max(max_seqlen, n)
if cos_list:
cos = torch.cat(cos_list).to(device)
sin = torch.cat(sin_list).to(device)
else:
cos = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
sin = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
if max_seqlen_override is not None:
max_seqlen = max_seqlen_override
return {
"vit_cos": cos,
"vit_sin": sin,
"cu_seqlens": torch.tensor(cu_seqlens, dtype=torch.int32, device=device),
# Read on the host by the attention wrapper; keep on CPU.
"max_seqlen": torch.tensor(max_seqlen, dtype=torch.int32),
}
def build_packed_merge_metadata(
grids: list[list[int]] | list[tuple[int, int]],
downsample_ratio: int,
*,
device: torch.device,
dtype: torch.dtype,
) -> dict[str, torch.Tensor]:
"""Gather indices/mask for ``DeepseekV4Aligner.forward_packed``.
For each aligner output row, the ``r x r`` source patch rows in the
packed ViT output; positions past the image edge (the eager path pads
them with zeros) get mask 0 and a clamped in-range index.
"""
r = downsample_ratio
idx_list: list[torch.Tensor] = []
mask_list: list[torch.Tensor] = []
offset = 0
for n_h, n_w in grids:
rows = -(-n_h // r)
cols = -(-n_w // r)
bi = torch.arange(rows).repeat_interleave(cols)
bj = torch.arange(cols).repeat(rows)
di = torch.arange(r).repeat_interleave(r)
dj = torch.arange(r).repeat(r)
hi = bi[:, None] * r + di[None, :]
wj = bj[:, None] * r + dj[None, :]
valid = (hi < n_h) & (wj < n_w)
idx = offset + hi.clamp(max=n_h - 1) * n_w + wj.clamp(max=n_w - 1)
idx_list.append(idx)
mask_list.append(valid)
offset += n_h * n_w
if idx_list:
merge_idx = torch.cat(idx_list).to(device)
merge_mask = torch.cat(mask_list).unsqueeze(-1).to(device=device, dtype=dtype)
else:
merge_idx = torch.zeros((0, r * r), dtype=torch.int64, device=device)
merge_mask = torch.zeros((0, r * r, 1), dtype=dtype, device=device)
return {"merge_idx": merge_idx, "merge_mask": merge_mask}
class DeepseekV4Aligner(nn.Module):
"""Spatial merge (downsample_ratio x downsample_ratio) + MLP projector."""
@@ -240,6 +368,27 @@ class DeepseekV4Aligner(nn.Module):
out, _ = self.w2(F.gelu(hidden))
return out
def forward_packed(
self,
x: torch.Tensor,
merge_idx: torch.Tensor,
merge_mask: torch.Tensor,
) -> torch.Tensor:
"""Packed multi-image path for encoder CUDA graphs.
``merge_idx``/``merge_mask`` come from ``build_packed_merge_metadata``;
gather+mask reproduces the eager ``F.pad`` + ``F.unfold`` merge
exactly (unfold rows are channel-major, so the gathered patch rows
are transposed before flattening).
"""
r2 = self.downsample_ratio**2
m = merge_idx.shape[0]
gathered = (x[merge_idx] * merge_mask).view(m, r2, -1)
gathered = gathered.permute(0, 2, 1).reshape(m, -1)
hidden, _ = self.w1(gathered)
out, _ = self.w2(F.gelu(hidden))
return out
def run_dp_sharded_vision_tower(
vision_model: DeepseekV4ViT,