[Model] Support DeepSeek-V4.1-Flash (#56214)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jee Jee Li
andyluo7
OpenAI Codex
zjy0516
Claude Opus 5
parent
e6821ceac9
commit
e77daef89e
@@ -11,7 +11,7 @@ import torch.nn.functional as F
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.config import CUDAGraphMode, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
@@ -819,6 +819,32 @@ def rocm_fp8_mqa_logits(
|
||||
return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)
|
||||
|
||||
|
||||
def _max_decode_logits_rows(num_batched_tokens: int) -> int:
|
||||
"""Upper bound on decode rows the paged-MQA logits buffer can ever hold.
|
||||
|
||||
``rocm_fp8_paged_mqa_logits`` sizes its workspace as
|
||||
``(batch_size * next_n, max_model_len)``. ``batch_size`` is bounded by
|
||||
``max_num_seqs`` and ``next_n`` by ``1 + num_speculative_tokens``, which is
|
||||
far tighter than ``max_num_batched_tokens`` -- 192 vs 16384 for a typical
|
||||
32-seq DSpark-5 deployment. The loose bound is harmless at short contexts
|
||||
but scales with ``max_model_len``, so at the model's full context it asks
|
||||
for tens of TiB and the engine cannot start. Take whichever valid bound is
|
||||
smaller; the workspace is locked after profiling, so it must not be under-
|
||||
estimated.
|
||||
"""
|
||||
try:
|
||||
vllm_config = get_current_vllm_config()
|
||||
except Exception:
|
||||
return num_batched_tokens
|
||||
scheduler_config = getattr(vllm_config, "scheduler_config", None)
|
||||
max_num_seqs = getattr(scheduler_config, "max_num_seqs", None)
|
||||
if not max_num_seqs:
|
||||
return num_batched_tokens
|
||||
speculative_config = getattr(vllm_config, "speculative_config", None)
|
||||
num_spec = getattr(speculative_config, "num_speculative_tokens", 0) or 0
|
||||
return min(num_batched_tokens, max_num_seqs * (1 + num_spec))
|
||||
|
||||
|
||||
def rocm_aiter_sparse_attn_indexer_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
k_cache_prefix: LayerNameType,
|
||||
@@ -835,6 +861,9 @@ def rocm_aiter_sparse_attn_indexer_fake(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool = False,
|
||||
compress_ratio: int = 1,
|
||||
candidate_blocks: torch.Tensor | None = None,
|
||||
candidate_block_size: int = 0,
|
||||
candidate_write: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return topk_indices_buffer
|
||||
|
||||
@@ -856,6 +885,9 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool = False,
|
||||
compress_ratio: int = 1,
|
||||
candidate_blocks: torch.Tensor | None = None,
|
||||
candidate_block_size: int = 0,
|
||||
candidate_write: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
forward_context = get_forward_context()
|
||||
@@ -881,15 +913,15 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
)
|
||||
|
||||
# Decode logits buffer, used by rocm_fp8_paged_mqa_logits.
|
||||
# batch_size * next_n <= hidden_states.shape[0] == max_num_batched_tokens
|
||||
decode_rows = _max_decode_logits_rows(hidden_states.shape[0])
|
||||
if _ON_GFX942 or _ON_GFX950:
|
||||
workspace_manager.get_simultaneous(
|
||||
((hidden_states.shape[0], max_model_len), torch.float32),
|
||||
((decode_rows, max_model_len), torch.float32),
|
||||
)
|
||||
else:
|
||||
workspace_manager.get_simultaneous(
|
||||
(
|
||||
(q_fp8.shape[1], hidden_states.shape[0], max_model_len),
|
||||
(q_fp8.shape[1], decode_rows, max_model_len),
|
||||
torch.float32,
|
||||
),
|
||||
)
|
||||
@@ -919,6 +951,9 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
compress_ratio,
|
||||
candidate_blocks,
|
||||
candidate_block_size,
|
||||
candidate_write,
|
||||
)
|
||||
layer_attn_metadata = attn_metadata[k_cache_prefix]
|
||||
assert isinstance(layer_attn_metadata, DeepseekV32IndexerMetadata)
|
||||
@@ -974,6 +1009,30 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
)
|
||||
if candidate_blocks is not None:
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
_apply_candidate_mask,
|
||||
_select_candidate_blocks,
|
||||
)
|
||||
|
||||
chunk_candidates = candidate_blocks[chunk.token_start : chunk.token_end]
|
||||
if candidate_write:
|
||||
_select_candidate_blocks(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
chunk_candidates.shape[1],
|
||||
candidate_block_size,
|
||||
chunk_candidates,
|
||||
)
|
||||
else:
|
||||
_apply_candidate_mask(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
chunk_candidates,
|
||||
candidate_block_size,
|
||||
)
|
||||
topk_indices = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
@@ -1041,6 +1100,37 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
|
||||
if candidate_blocks is not None:
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
_apply_candidate_mask,
|
||||
_select_candidate_blocks,
|
||||
)
|
||||
|
||||
num_rows = logits.shape[0]
|
||||
visible = decode_metadata.seq_lens.reshape(-1)
|
||||
if visible.numel() != num_rows:
|
||||
visible = visible.repeat_interleave(next_n)
|
||||
visible = visible[:num_rows].to(torch.int64)
|
||||
row_starts = torch.zeros_like(visible)
|
||||
decode_candidates = candidate_blocks[:num_rows]
|
||||
if candidate_write:
|
||||
_select_candidate_blocks(
|
||||
logits,
|
||||
row_starts,
|
||||
visible,
|
||||
decode_candidates.shape[1],
|
||||
candidate_block_size,
|
||||
decode_candidates,
|
||||
)
|
||||
else:
|
||||
_apply_candidate_mask(
|
||||
logits,
|
||||
row_starts,
|
||||
visible,
|
||||
decode_candidates,
|
||||
candidate_block_size,
|
||||
)
|
||||
|
||||
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
|
||||
num_rows = logits.shape[0]
|
||||
|
||||
@@ -1097,6 +1187,10 @@ def _decode_e8m0_scales(scale: torch.Tensor) -> torch.Tensor:
|
||||
)
|
||||
|
||||
return _upcast_e8m0_to_fp32(scale).contiguous()
|
||||
if scale.dtype == torch.uint8:
|
||||
# MXFP8 parameters preserve E8M0 scales as their raw exponent bytes.
|
||||
# They are biased exponents, not numeric uint8 scale values.
|
||||
return torch.exp2(scale.to(torch.int16).to(torch.float32) - 127.0)
|
||||
return scale.to(torch.float32)
|
||||
|
||||
|
||||
@@ -1226,7 +1320,15 @@ def _get_cached_wo_a_bf16(
|
||||
)
|
||||
|
||||
wo_a_scale_param = get_fp8_block_weight_scale(wo_a)
|
||||
if wo_a_scale_param is not None:
|
||||
if wo_a_scale_param is None:
|
||||
# ModelOpt MXFP8 stores the multiplicative E8M0 scale without the
|
||||
# historical ``_inv`` suffix.
|
||||
wo_a_scale_param = getattr(wo_a, "weight_scale", None)
|
||||
# Emulated MXFP8 kernels can replace the original one-byte weight with an
|
||||
# already-dequantized BF16 tensor while retaining the scale attribute for
|
||||
# metadata. Applying that retained scale again would double-dequantize the
|
||||
# weight. Block scaling is only valid while the one-byte FP8 storage remains.
|
||||
if wo_a_scale_param is not None and wo_a.weight.element_size() == 1:
|
||||
wo_a_weight = wo_a.weight.view(n_local_groups, o_lora_rank, hidden_dim).to(
|
||||
torch.float32
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user