[DSV4.1] Add encoder cuda graph support for deepseek-v4.1-flash (#56625)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -40,8 +40,7 @@ from vllm.model_executor.models.vision import (
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(8)
|
||||
def get_vision_cos_sin(
|
||||
def _compute_vision_cos_sin(
|
||||
n_h: int, n_w: int, dim: int, theta: float
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
|
||||
@@ -52,6 +51,13 @@ def get_vision_cos_sin(
|
||||
return freqs.cos().unsqueeze(1), freqs.sin().unsqueeze(1)
|
||||
|
||||
|
||||
@lru_cache(8)
|
||||
def get_vision_cos_sin(
|
||||
n_h: int, n_w: int, dim: int, theta: float
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return _compute_vision_cos_sin(n_h, n_w, dim, theta)
|
||||
|
||||
|
||||
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
|
||||
dtype = x.dtype
|
||||
x1, x2 = x.float().chunk(2, dim=-1)
|
||||
@@ -123,15 +129,23 @@ class DeepseekV4VisionAttention(nn.Module):
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
n = x.size(0)
|
||||
qkv, _ = self.wqkv(x)
|
||||
q, k, v = (t.view(n, self.n_heads, self.head_dim) for t in qkv.chunk(3, -1))
|
||||
q = apply_rotary(q, cos, sin).unsqueeze(0) # (b=1, n, h, d)
|
||||
k = apply_rotary(k, cos, sin).unsqueeze(0)
|
||||
# One image per call: a dense batch, no varlen packing metadata.
|
||||
o = self.attn(q, k, v.unsqueeze(0))
|
||||
# Dense (cu_seqlens=None): one image per call. Varlen (cu_seqlens
|
||||
# set): packed multi-image batch for encoder CUDA graph replay.
|
||||
o = self.attn(
|
||||
q, k, v.unsqueeze(0), cu_seqlens=cu_seqlens, max_seqlen=max_seqlen
|
||||
)
|
||||
out, _ = self.wo(o.reshape(n, -1))
|
||||
return out
|
||||
|
||||
@@ -173,9 +187,14 @@ class DeepseekV4VisionBlock(nn.Module):
|
||||
self.mlp = DeepseekV4VisionMLP(config, prefix=f"{prefix}.mlp")
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = x + self.attn(self.norm1(x), cos, sin)
|
||||
x = x + self.attn(self.norm1(x), cos, sin, cu_seqlens, max_seqlen)
|
||||
return x + self.mlp(self.norm2(x))
|
||||
|
||||
|
||||
@@ -206,6 +225,115 @@ class DeepseekV4ViT(nn.Module):
|
||||
x = block(x, cos, sin)
|
||||
return self.norm(x)
|
||||
|
||||
def forward_packed(
|
||||
self,
|
||||
patches: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Varlen path for encoder CUDA graphs: multiple images packed along
|
||||
rows, per-image attention via ``cu_seqlens``, RoPE tables precomputed
|
||||
on device. Pure-tensor (no host reads, no H2D), safe to capture."""
|
||||
x = self.patch_embed(patches)
|
||||
for block in self.blocks:
|
||||
x = block(x, cos, sin, cu_seqlens, max_seqlen)
|
||||
return self.norm(x)
|
||||
|
||||
|
||||
def build_packed_vit_metadata(
|
||||
grids: list[list[int]] | list[tuple[int, int]],
|
||||
*,
|
||||
rope_dim: int,
|
||||
rope_theta: float,
|
||||
device: torch.device,
|
||||
max_seqlen_override: int | None = None,
|
||||
cached: bool = True,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Precompute packed-batch ViT metadata for ``DeepseekV4ViT.forward_packed``.
|
||||
|
||||
Args:
|
||||
grids: ``[n_vit_h, n_vit_w]`` per image, in packing order.
|
||||
rope_dim: RoPE embedding dimension of the ViT.
|
||||
rope_theta: RoPE base frequency of the ViT.
|
||||
device: Device for the returned tensors (except ``max_seqlen``,
|
||||
which stays on the host).
|
||||
max_seqlen_override: Worst-case value baked in at CUDA graph capture
|
||||
(the attention wrapper reads ``max_seqlen`` on the host, so the
|
||||
capture-time value becomes a graph constant).
|
||||
cached: Use the shared ``lru_cache`` for RoPE tables. Capture-time
|
||||
dummy grids pass ``False`` to avoid evicting real entries.
|
||||
|
||||
"""
|
||||
cos_sin = get_vision_cos_sin if cached else _compute_vision_cos_sin
|
||||
cos_list: list[torch.Tensor] = []
|
||||
sin_list: list[torch.Tensor] = []
|
||||
cu_seqlens = [0]
|
||||
max_seqlen = 0
|
||||
for n_h, n_w in grids:
|
||||
cos, sin = cos_sin(n_h, n_w, rope_dim, rope_theta)
|
||||
cos_list.append(cos)
|
||||
sin_list.append(sin)
|
||||
n = n_h * n_w
|
||||
cu_seqlens.append(cu_seqlens[-1] + n)
|
||||
max_seqlen = max(max_seqlen, n)
|
||||
if cos_list:
|
||||
cos = torch.cat(cos_list).to(device)
|
||||
sin = torch.cat(sin_list).to(device)
|
||||
else:
|
||||
cos = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
|
||||
sin = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
|
||||
if max_seqlen_override is not None:
|
||||
max_seqlen = max_seqlen_override
|
||||
return {
|
||||
"vit_cos": cos,
|
||||
"vit_sin": sin,
|
||||
"cu_seqlens": torch.tensor(cu_seqlens, dtype=torch.int32, device=device),
|
||||
# Read on the host by the attention wrapper; keep on CPU.
|
||||
"max_seqlen": torch.tensor(max_seqlen, dtype=torch.int32),
|
||||
}
|
||||
|
||||
|
||||
def build_packed_merge_metadata(
|
||||
grids: list[list[int]] | list[tuple[int, int]],
|
||||
downsample_ratio: int,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Gather indices/mask for ``DeepseekV4Aligner.forward_packed``.
|
||||
|
||||
For each aligner output row, the ``r x r`` source patch rows in the
|
||||
packed ViT output; positions past the image edge (the eager path pads
|
||||
them with zeros) get mask 0 and a clamped in-range index.
|
||||
"""
|
||||
r = downsample_ratio
|
||||
idx_list: list[torch.Tensor] = []
|
||||
mask_list: list[torch.Tensor] = []
|
||||
offset = 0
|
||||
for n_h, n_w in grids:
|
||||
rows = -(-n_h // r)
|
||||
cols = -(-n_w // r)
|
||||
bi = torch.arange(rows).repeat_interleave(cols)
|
||||
bj = torch.arange(cols).repeat(rows)
|
||||
di = torch.arange(r).repeat_interleave(r)
|
||||
dj = torch.arange(r).repeat(r)
|
||||
hi = bi[:, None] * r + di[None, :]
|
||||
wj = bj[:, None] * r + dj[None, :]
|
||||
valid = (hi < n_h) & (wj < n_w)
|
||||
idx = offset + hi.clamp(max=n_h - 1) * n_w + wj.clamp(max=n_w - 1)
|
||||
idx_list.append(idx)
|
||||
mask_list.append(valid)
|
||||
offset += n_h * n_w
|
||||
if idx_list:
|
||||
merge_idx = torch.cat(idx_list).to(device)
|
||||
merge_mask = torch.cat(mask_list).unsqueeze(-1).to(device=device, dtype=dtype)
|
||||
else:
|
||||
merge_idx = torch.zeros((0, r * r), dtype=torch.int64, device=device)
|
||||
merge_mask = torch.zeros((0, r * r, 1), dtype=dtype, device=device)
|
||||
return {"merge_idx": merge_idx, "merge_mask": merge_mask}
|
||||
|
||||
|
||||
class DeepseekV4Aligner(nn.Module):
|
||||
"""Spatial merge (downsample_ratio x downsample_ratio) + MLP projector."""
|
||||
@@ -240,6 +368,27 @@ class DeepseekV4Aligner(nn.Module):
|
||||
out, _ = self.w2(F.gelu(hidden))
|
||||
return out
|
||||
|
||||
def forward_packed(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
merge_idx: torch.Tensor,
|
||||
merge_mask: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Packed multi-image path for encoder CUDA graphs.
|
||||
|
||||
``merge_idx``/``merge_mask`` come from ``build_packed_merge_metadata``;
|
||||
gather+mask reproduces the eager ``F.pad`` + ``F.unfold`` merge
|
||||
exactly (unfold rows are channel-major, so the gathered patch rows
|
||||
are transposed before flattening).
|
||||
"""
|
||||
r2 = self.downsample_ratio**2
|
||||
m = merge_idx.shape[0]
|
||||
gathered = (x[merge_idx] * merge_mask).view(m, r2, -1)
|
||||
gathered = gathered.permute(0, 2, 1).reshape(m, -1)
|
||||
hidden, _ = self.w1(gathered)
|
||||
out, _ = self.w2(F.gelu(hidden))
|
||||
return out
|
||||
|
||||
|
||||
def run_dp_sharded_vision_tower(
|
||||
vision_model: DeepseekV4ViT,
|
||||
|
||||
Reference in New Issue
Block a user